Add bounded source context analysis with lexical follow-up searches

src/viano_bug_analysis/analysis/investigation_prompt.py

@@ -18,6 +18,9 @@ def output_schema(context=None, *, strict=False):
     if context and context['tool_schema'].get('x-project-bindings', {}).get('relation_profiles'):
         from viano_bug_analysis.tools.database_relations import extend_tool_schema
         schema['properties']['next'] = extend_tool_schema(schema['properties']['next'])
+    if context and context['tool_schema'].get('x-project-bindings', {}).get('source_analysis'):
+        from viano_bug_analysis.tools.source_context import extend_tool_schema
+        schema['properties']['next'] = extend_tool_schema(schema['properties']['next'])
     if context and context.get('prompt_version') in {'investigation_project_v12','investigation_project_v13','investigation_project_v14','investigation_project_v15','investigation_project_v16','investigation_project_v17','investigation_project_v18'}:
         schema['properties']['update']['properties']['causal_review'] = json.loads(
             files('viano_bug_analysis.schemas').joinpath('investigation-causal-review-v1.schema.json').read_text())

src/viano_bug_analysis/analysis/investigation_step.py

@@ -29,6 +29,9 @@ def _result_status(capture):
         status['completeness']='complete_for_configured_edges_only' if complete and result.get('coverage')=='complete_configured_relations' else 'partial_or_unknown'
         status['nodes']=[{k:node.get(k) for k in ('node','table','returned_rows','complete','upstream_complete','local_truncated')}
                          for node in items] if isinstance(items,list) else []
+    elif query.get('kind')=='source.context':
+        status['completeness']='complete_for_selected_lexical_context_only' if complete and result.get('coverage')=='complete_configured_source_context' else 'partial_or_unknown'
+        status['limits_reached']=result.get('limits_reached',[])
     # Unknown providers and selected source ranges never imply complete absence.
     return status
 

src/viano_bug_analysis/analysis/investigator.py

@@ -256,6 +256,11 @@ class Investigator:
                     'upstream truncation invalidates downstream absence claims.')
                 items = context['tool_schema']['properties']['requests']['items']
                 context['tool_schema']['properties']['requests']['items'] = extend_tool_schema(items)
+            if service.source_context:
+                from viano_bug_analysis.tools.source_context import extend_tool_schema
+                context['tool_schema']['x-project-bindings']['source_analysis'] = service.source_context.catalog()
+                items = context['tool_schema']['properties']['requests']['items']
+                context['tool_schema']['properties']['requests']['items'] = extend_tool_schema(items)
         else:
             variants=context["tool_schema"]["properties"]["requests"]["items"]["oneOf"]
             context["tool_schema"]["properties"]["requests"]["items"]["oneOf"]=[
@@ -583,6 +588,8 @@ class Investigator:
             raise ToolError("PROJECT_BINDING_CHANGED")
         if binding.get('relation_profiles') is not None and binding['relation_profiles'] != (service.relations.catalog() if service.relations else None):
             raise ToolError('PROJECT_BINDING_CHANGED')
+        if binding.get('source_analysis') is not None and binding['source_analysis'] != (service.source_context.catalog() if service.source_context else None):
+            raise ToolError('PROJECT_BINDING_CHANGED')
 
     def _invalid(self,state,budget,code,llm_ref,*,feedback=None):
         active=state["active"];h=state["hypotheses"][active["hypothesis_id"]]

src/viano_bug_analysis/schemas/investigation-request-v1.schema.json

@@ -709,6 +709,29 @@
                         "maxLength": 200
                       }
                     }
+                  },
+                  {
+                    "type": "object",
+                    "additionalProperties": false,
+                    "required": [
+                      "kind",
+                      "text",
+                      "path_prefix"
+                    ],
+                    "properties": {
+                      "kind": {
+                        "const": "source.context"
+                      },
+                      "text": {
+                        "type": "string",
+                        "minLength": 1,
+                        "maxLength": 200
+                      },
+                      "path_prefix": {
+                        "type": "string",
+                        "maxLength": 250
+                      }
+                    }
                   }
                 ]
               }

src/viano_bug_analysis/schemas/investigation-state-v1.schema.json

@@ -1710,6 +1710,29 @@
                                       "maxLength": 200
                                     }
                                   }
+                                },
+                                {
+                                  "type": "object",
+                                  "additionalProperties": false,
+                                  "required": [
+                                    "kind",
+                                    "text",
+                                    "path_prefix"
+                                  ],
+                                  "properties": {
+                                    "kind": {
+                                      "const": "source.context"
+                                    },
+                                    "text": {
+                                      "type": "string",
+                                      "minLength": 1,
+                                      "maxLength": 200
+                                    },
+                                    "path_prefix": {
+                                      "type": "string",
+                                      "maxLength": 250
+                                    }
+                                  }
                                 }
                               ]
                             }

src/viano_bug_analysis/schemas/project-capture-v1.schema.json

@@ -264,6 +264,29 @@
               "maxLength": 200
             }
           }
+        },
+        {
+          "type": "object",
+          "additionalProperties": false,
+          "required": [
+            "kind",
+            "text",
+            "path_prefix"
+          ],
+          "properties": {
+            "kind": {
+              "const": "source.context"
+            },
+            "text": {
+              "type": "string",
+              "minLength": 1,
+              "maxLength": 200
+            },
+            "path_prefix": {
+              "type": "string",
+              "maxLength": 250
+            }
+          }
         }
       ]
     },
@@ -298,7 +321,9 @@
             "partial_search",
             "complete_selected_text_search",
             "complete_configured_relations",
-            "partial_configured_relations"
+            "partial_configured_relations",
+            "complete_configured_source_context",
+            "partial_configured_source_context"
           ]
         },
         "truncated": {
@@ -348,6 +373,18 @@
             "null"
           ],
           "minimum": 1
+        },
+        "searches": {
+          "type": "array",
+          "items": {
+            "type": "object"
+          }
+        },
+        "limits_reached": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
         }
       }
     }

src/viano_bug_analysis/tools/project_query.py

@@ -35,7 +35,7 @@ class ProjectResearch:
     a URL, directory, credentials, database, branch, or arbitrary SQL.
     """
 
-    def __init__(self, project, *, source=None, database=None, limits=None, relations=None):
+    def __init__(self, project, *, source=None, database=None, limits=None, relations=None, source_analysis=None):
         self.project = copy.deepcopy(project)
         self.source, self.database = source, database
         self.limits = limits or ProjectLimits()
@@ -47,6 +47,10 @@ class ProjectResearch:
             self.relations = DatabaseRelations(database, relations) if relations else None
         except (ValueError, TypeError, KeyError):
             raise ToolError('PROJECT_RELATIONS_CONFIG') from None
+        from .source_context import SourceContextAnalysis
+        self.source_context = SourceContextAnalysis(source, source_analysis) if source_analysis is not None else None
+        if self.source_context and not self.source_context.config.enabled:
+            self.source_context = None
 
     @classmethod
     def load(cls, path):
@@ -55,7 +59,7 @@ class ProjectResearch:
         try:
             path = Path(path).resolve()
             config = tomllib.loads(path.read_text(encoding="utf-8"))
-            if set(config) - {"project", "source", "database", "limits", "enrichment", "entity_resolution", "relations", "relations_file"}:
+            if set(config) - {"project", "source", "database", "limits", "enrichment", "entity_resolution", "relations", "relations_file", "source_analysis"}:
                 raise ValueError()
             limits = ProjectLimits(**config.get("limits", {}))
             source = None
@@ -72,7 +76,8 @@ class ProjectResearch:
             relations = config.get('relations', {})
             if 'relations_file' in config:
                 relations = tomllib.loads((path.parent / config['relations_file']).read_text(encoding='utf-8'))['relations']
-            service = cls(config["project"], source=source, database=database, limits=limits, relations=relations)
+            service = cls(config["project"], source=source, database=database, limits=limits, relations=relations,
+                          source_analysis=config.get('source_analysis'))
             service.enrichment = copy.deepcopy(config.get("enrichment", {}))
             service.entity_resolution = copy.deepcopy(config.get("entity_resolution", {}))
             return service
@@ -89,7 +94,11 @@ class ProjectResearch:
 
     def query_schema(self):
         from .database_relations import extend_query_schema
-        return extend_query_schema(self.schema()) if self.relations else self.schema()
+        schema = extend_query_schema(self.schema()) if self.relations else self.schema()
+        if self.source_context:
+            from .source_context import extend_query_schema
+            schema = extend_query_schema(schema)
+        return schema
 
     def prepare(self, context, request):
         self.check_project(context.reader.manifest["project"])
@@ -111,6 +120,9 @@ class ProjectResearch:
         if arguments['kind'] == 'database.relations':
             self.relations.validate(arguments)
             return
+        if arguments['kind'] == 'source.context':
+            self.source_context.validate(arguments)
+            return
         provider.validate(arguments)
 
     def collect(self, arguments):
@@ -122,12 +134,20 @@ class ProjectResearch:
                        'consistency': 'single_repeatable_read_read_only_transaction',
                        'relation_plan': copy.deepcopy(self.relations.profiles[arguments['profile']]),
                        'relation_profile_sha256': self.relations.binding(arguments['profile'])}
+        if arguments['kind'] == 'source.context':
+            settings = self.source_context.catalog()
+            binding = {**binding, 'source_analysis': settings,
+                       'source_analysis_sha256': hashlib.sha256(json_bytes(settings)).hexdigest()}
         cache_key = hashlib.sha256(json_bytes({"arguments": arguments, "binding": binding})).hexdigest()
         # A request is frozen for the controller's validation/persistence cycle.
         # Durable resume uses the captured reply in the package, never this cache.
         if cache_key not in self._captures:
-            result = (self.relations.query(copy.deepcopy(arguments)) if arguments['kind'] == 'database.relations'
-                      else provider.query(copy.deepcopy(arguments)))
+            if arguments['kind'] == 'database.relations':
+                result = self.relations.query(copy.deepcopy(arguments))
+            elif arguments['kind'] == 'source.context':
+                result = self.source_context.query(copy.deepcopy(arguments))
+            else:
+                result = provider.query(copy.deepcopy(arguments))
             capture = {"format": "viano-project-capture", "format_version": 1,
                        "query": copy.deepcopy(arguments), "origin": binding, "result": result}
             from viano_bug_analysis.case.contracts import validate

src/viano_bug_analysis/tools/repository.py

@@ -112,7 +112,7 @@ class GitSourceProvider:
             if prefix != ".":
                 command.append(":(literal)" + prefix)
         else:
-            extensions = ["sql"] if args["kind"] == "source.schema" else ["cpp", "h", "hpp", "c", "sql", "txt", "md", "conf", "cmake"]
+            extensions = args.get('_extensions') or (["sql"] if args["kind"] == "source.schema" else ["cpp", "h", "hpp", "c", "sql", "txt", "md", "conf", "cmake"])
             command.extend(":(glob)**/*." + ext for ext in extensions)
         stream_queue = queue.Queue(maxsize=2)
         stop = threading.Event()
@@ -165,6 +165,8 @@ class GitSourceProvider:
                     continue
                 if not name.startswith(args["path_prefix"]) or (args["kind"] == "source.schema" and not name.endswith(".sql")):
                     continue
+                if '_extensions' in args and name.rsplit('.', 1)[-1] not in args['_extensions']:
+                    continue
                 if name not in sources:
                     try:
                         _, sources[name] = self._read(name)

src/viano_bug_analysis/tools/router.py

@@ -49,7 +49,7 @@ class InvestigationToolRouter:
             if branch['properties']['operation'].get('const') == 'project.query':
                 variants = branch['properties']['arguments']['oneOf']
                 branch['properties']['arguments']['oneOf'] = [v for v in variants
-                    if v['properties']['kind'].get('const') != 'database.relations']
+                    if v['properties']['kind'].get('const') not in {'database.relations', 'source.context'}]
         return schema
 
     def query(self, package, batch):

src/viano_bug_analysis/tools/source_context.py

@@ -0,0 +1,211 @@
+"""Bounded lexical context chains over immutable Git objects, never call graphs."""
+import copy
+import math
+import re
+import time
+from dataclasses import asdict, dataclass, replace
+
+from viano_bug_analysis.lifecycle.storage import json_bytes
+from .config import ToolError
+from .repository import safe_path
+
+
+def request_schema():
+    return {'type': 'object', 'additionalProperties': False,
+            'required': ['kind', 'text', 'path_prefix'], 'properties': {
+                'kind': {'const': 'source.context'},
+                'text': {'type': 'string', 'minLength': 1, 'maxLength': 200},
+                'path_prefix': {'type': 'string', 'maxLength': 250}}}
+
+
+def extend_query_schema(schema):
+    schema = copy.deepcopy(schema)
+    schema['oneOf'].append(request_schema())
+    return schema
+
+
+def extend_tool_schema(schema):
+    schema = copy.deepcopy(schema)
+    for branch in schema.get('oneOf', []):
+        if branch.get('properties', {}).get('operation', {}).get('const') == 'project.query':
+            branch['properties']['arguments'] = extend_query_schema(branch['properties']['arguments'])
+    return schema
+
+
+@dataclass(frozen=True)
+class SourceContextConfig:
+    enabled: bool = True
+    context_lines: int = 8
+    max_searches: int = 4
+    max_matches: int = 12
+    max_files: int = 8
+    max_blocks: int = 12
+    max_total_lines: int = 160
+    registration_prefixes: tuple = ()
+    extensions: tuple = ('cpp', 'h', 'hpp', 'c', 'sql', 'py', 'js', 'ts', 'tsx',
+                         'cs', 'go', 'rs', 'lua', 'java', 'cmake', 'txt')
+
+    def __post_init__(self):
+        bounds = {'context_lines': 100, 'max_searches': 12, 'max_matches': 100,
+                  'max_files': 30, 'max_blocks': 100, 'max_total_lines': 2000}
+        if type(self.enabled) is not bool:
+            raise ToolError('PROJECT_SOURCE_CONTEXT_CONFIG')
+        for key, maximum in bounds.items():
+            value = getattr(self, key)
+            if type(value) is not int or not 1 <= value <= maximum:
+                raise ToolError('PROJECT_SOURCE_CONTEXT_CONFIG')
+        for values, maximum, pattern in ((self.registration_prefixes, 8, r'[A-Za-z_][A-Za-z0-9_]{0,63}'),
+                                          (self.extensions, 30, r'[A-Za-z0-9]{1,12}')):
+            if (not isinstance(values, (list, tuple)) or len(values) > maximum
+                    or any(not isinstance(v, str) or not re.fullmatch(pattern, v) for v in values)):
+                raise ToolError('PROJECT_SOURCE_CONTEXT_CONFIG')
+        if not self.extensions:
+            raise ToolError('PROJECT_SOURCE_CONTEXT_CONFIG')
+
+
+class SourceContextAnalysis:
+    def __init__(self, provider, settings):
+        self.provider = provider
+        try:
+            self.config = SourceContextConfig(**settings)
+        except (TypeError, ValueError):
+            raise ToolError('PROJECT_SOURCE_CONTEXT_CONFIG') from None
+        if self.config.enabled and (not callable(getattr(provider, '_read', None)) or not callable(getattr(provider, 'query', None))):
+            raise ToolError('PROJECT_SOURCE_CONTEXT_CONFIG')
+
+    def catalog(self):
+        settings = {k: list(v) if isinstance(v, tuple) else v for k, v in asdict(self.config).items()}
+        return {'version': 1, 'settings': settings,
+                'preference': 'Prefer source.context for a symbol or entity identifier before separate '
+                              'search/read/registration calls. Returned links are lexical candidates, '
+                              'not verified calls, registration semantics or exclusions of dynamic paths.'}
+
+    def validate(self, args):
+        safe_path(args['path_prefix'], empty=True)
+        if not args['text'].strip() or any(ord(c) < 32 or ord(c) == 127 for c in args['text']):
+            raise ToolError('PROJECT_SOURCE_TEXT')
+
+    def query(self, args):
+        self.validate(args)
+        config, provider = self.config, self.provider
+        deadline = time.monotonic() + provider.limits.timeout_seconds
+        # Independent limits: do not mutate the shared provider during a query.
+        probe = copy.copy(provider)
+        files, anchors, searches, reached = {}, {}, [], set()
+        pending = [(args['text'], 'requested_term', None)]
+        seen, queued, registration_scanned = set(), {args['text']}, set()
+        read_bytes = 0
+
+        def remaining():
+            seconds = deadline - time.monotonic()
+            if seconds <= 0:
+                raise ToolError('PROJECT_SOURCE_TIMEOUT')
+            probe.limits = replace(provider.limits, timeout_seconds=max(1, math.ceil(seconds)),
+                                   max_search_bytes=max(1, provider.limits.max_search_bytes // config.max_searches))
+
+        def read_file(path):
+            nonlocal read_bytes
+            if path in files:
+                return files[path]
+            if len(files) >= config.max_files:
+                reached.add('file_limit'); return None
+            remaining()
+            text, origin = probe._read(path)
+            read_bytes += origin['size']
+            if read_bytes > provider.limits.max_search_bytes:
+                raise ToolError('PROJECT_SOURCE_INPUT_LIMIT')
+            files[path] = (text.splitlines(), origin)
+            return files[path]
+
+        def candidate(term, reason, parent):
+            if term in seen or term in queued: return
+            if len(queued) >= config.max_searches * config.max_matches:
+                reached.add('candidate_limit'); return
+            queued.add(term); pending.append((term, reason, parent))
+
+        while pending and len(searches) < config.max_searches:
+            term, reason, parent = pending.pop(0)
+            if term in seen:
+                continue
+            seen.add(term); remaining()
+            query = {'kind': 'source.symbol' if re.fullmatch(r'[A-Za-z_][A-Za-z0-9_]*', term) else 'source.search',
+                     'text': term, 'path_prefix': args['path_prefix'], 'case_sensitive': True,
+                     'offset': 0, 'limit': min(config.max_matches, provider.limits.max_rows),
+                     '_extensions': list(config.extensions)}
+            result = probe.query(query)
+            qid = 'S' + str(len(searches) + 1)
+            search = {'id': qid, 'reason': reason, 'parent_search': parent, 'query': query,
+                      'coverage': result['coverage'], 'truncated': result['truncated'],
+                      'skipped_files': result.get('skipped_files', []),
+                      'returned_matches': len(result['items'])}
+            searches.append(search)
+            if result['truncated']: reached.add('match_limit')
+            if result.get('skipped_files') or result['coverage'] != 'complete_selected_text_search':
+                reached.add('incomplete_search')
+            for hit in result['items']:
+                path, line = hit['path'], hit['line']
+                content = read_file(path)
+                if content is None: continue
+                lines, origin = content
+                if not 1 <= line <= len(lines):
+                    raise ToolError('PROJECT_SOURCE_RANGE')
+                anchors.setdefault(path, {}).setdefault(line, set()).add(qid)
+                # Equality syntax is a candidate alias, not a semantic identity proof.
+                match = re.search(r'\b([A-Za-z_][A-Za-z0-9_]*)\s*=\s*' + re.escape(term) + r'(?![A-Za-z0-9_])', lines[line-1])
+                if match and match[1] != term:
+                    candidate(match[1], 'lexical_assignment_candidate', qid)
+                # Trusted project-specific prefixes select nearby registration context.
+                if (reason == 'lexical_registration_candidate' or path in registration_scanned
+                        or any(term.startswith(prefix) for prefix in config.registration_prefixes)): continue
+                registration_scanned.add(path)
+                for number, text in enumerate(lines, 1):
+                    if number % 256 == 0 and time.monotonic() >= deadline:
+                        raise ToolError('PROJECT_SOURCE_TIMEOUT')
+                    if len(anchors.get(path, {})) >= config.max_searches * config.max_matches:
+                        reached.add('registration_anchor_limit'); break
+                    for prefix in config.registration_prefixes:
+                        for found in re.finditer(r'\b(' + re.escape(prefix) + r'[A-Za-z0-9_]*)\s*\(', text):
+                            symbol = found[1]
+                            anchors.setdefault(path, {}).setdefault(number, set()).update({qid, 'registration_candidate'})
+                            if symbol != term: candidate(symbol, 'lexical_registration_candidate', qid)
+        if any(term not in seen for term, _, _ in pending): reached.add('followup_search_limit')
+
+        items, used_lines, used_blocks = [], 0, 0
+        for path, (lines, origin) in files.items():
+            ranges = []
+            for line in sorted(anchors.get(path, {})):
+                start, end = max(1, line-config.context_lines), min(len(lines), line+config.context_lines)
+                if ranges and start <= ranges[-1][1] + 1:
+                    ranges[-1][1] = max(ranges[-1][1], end)
+                else: ranges.append([start, end])
+            blocks = []
+            for start, end in ranges:
+                if used_blocks >= config.max_blocks: reached.add('block_limit'); break
+                count = min(end-start+1, config.max_total_lines-used_lines)
+                if count <= 0: reached.add('line_limit'); break
+                if count < end-start+1: reached.add('line_limit')
+                selected = lines[start-1:start-1+count]
+                truncated = [start+i for i, text in enumerate(selected) if len(text) > 2048]
+                if truncated: reached.add('line_text_limit')
+                blocks.append({'start_line': start, 'lines': [text[:2048] for text in selected],
+                               'truncated_lines': truncated})
+                used_lines += count; used_blocks += 1
+            visible = {start+i for b in blocks for start in [b['start_line']] for i in range(len(b['lines']))}
+            selected_anchors = [{'line': line, 'search_ids': sorted(ids-{'registration_candidate'}),
+                                 'registration_candidate': 'registration_candidate' in ids, 'context_returned': line in visible}
+                                for line, ids in sorted(anchors.get(path, {}).items())]
+            omitted = sum(not a['context_returned'] for a in selected_anchors)
+            if omitted: reached.add('omitted_anchors')
+            if blocks:
+                items.append({**origin, 'total_lines': len(lines), 'blocks': blocks,
+                              'anchors': selected_anchors, 'omitted_anchors': omitted})
+        remaining()
+        result = {'items': items, 'searches': searches, 'limits_reached': sorted(reached),
+                  'coverage': 'partial_configured_source_context' if reached else 'complete_configured_source_context',
+                  'truncated': bool(reached),
+                  'interpretation': 'Selected lexical searches and bounded context windows at the configured commit only. '
+                                    'Assignments and registration names are candidates, not verified semantic relationships. '
+                                    'No complete call graph, runtime observation, root cause or global absence claim.'}
+        if len(json_bytes(result)) > provider.limits.max_result_bytes:
+            raise ToolError('PROJECT_RESULT_LIMIT')
+        return result