diff --git a/docs/investigation-contract-v2.md b/docs/investigation-contract-v2.md index 1e514e5..18d7c8f 100644 --- a/docs/investigation-contract-v2.md +++ b/docs/investigation-contract-v2.md @@ -60,3 +60,49 @@ Cache-Aufteilung und frühe Diagnose-Routingentscheidung. Validierung am 20.10.2026: 307 lokale Tests bestanden (171,87 Sekunden), `git diff --check` ohne Befund. Keine kostenpflichtigen LLM-Tests ausgeführt. + + +## Quellenraum und gezielte Korrekturen (20.10.2026) + +Neue v2-Kontexte speichern `research_guidance_version=1`. Ein kleiner, im stabilen +Fallkontext cachebarer Quellenkatalog unterscheidet konfigurierte Git-Recherche +(`project.query`) von den deklarierten Dateien im Paket (`source.*`). Höchstens +drei Paketpfade mit zusammen höchstens 700 Zeichen werden aufgeführt; weitere +Referenzen werden als ausgelassen gezählt. Historische Kontexte und ihre Requests +bleiben unverändert rekonstruierbar. Die vollständige Eingabe unterliegt weiterhin +der bestehenden 20.000-Zeichen-Grenze und dem 10.000-Zeichen-Richtwert. + +Fehlender Query-Kind, unvollständige `source.search`-Argumente und falscher +Quellenraum erhalten kurze, feldbezogene Hinweise. Es werden weder fehlende +Argumente erfunden noch Repository-Pfade automatisch in Paketquellen umgedeutet. +Die Hinweise passen in das bestehende Limit von 160 Zeichen. + +Die Antwortreparatur entfernt bei einem bekannten, einzelnen `recall` lediglich +die eindeutige `arguments`-Hülle. Beleg-ID, optionale Seitennummer, Beobachtung und +Bewertung bleiben erhalten. Mehrdeutige Hüllen, unbekannte IDs, mehrere Belege, +negative oder nicht ganzzahlige Seiten und unbekannte Aktionen werden nicht +repariert. Anschließend gelten unverändert Schema-, Provenienz-, Seiten- und +Budgetprüfungen; Originalantwort und Reparaturregel werden protokolliert. + +Die neue allgemeine Untersuchungsanweisung verlangt bei vermeintlich fehlenden +Hooks auch Aufrufer, Overrides, indirekte Übergaben und Registrierung zu prüfen. +Vorhandener Implementierungscode allein ist kein Beleg für einen Defekt. Das +ändert keine gespeicherten Confidence-Werte und erzwingt keine Bug17-spezifische +Schlussfolgerung. Ob diese Anleitung die fachliche Qualität verbessert, muss ein +späterer LLM-Lauf zeigen. + + +Der blockierte Bug17-Lauf wurde ohne API-Aufruf von Generation 204 über einen +protokollierten manuellen technischen Retry auf Generation 206 vorbereitet. +806 vorhandene Archiveinträge bleiben erhalten; Budget, Hypothesenstände und +Versuchsnummer bleiben unverändert. Der neue Kontext ergänzt ausschließlich die +versionierten Quellenhinweise und den fehlenden Korrekturhinweis. Die vollständige +logische Anfrage misst 13.849 Zeichen einschließlich Cache-Kontext. Der portable +Paketvalidator akzeptiert die Ausgabe. Details: +`test-results/token-usage/bug17-source-contract-repair-20261020.json`. + +Validierung der Quellenkorrekturen: 269 lokale Regressionstests bestanden in +192,85 Sekunden. Darunter 18 gezielte Tests für die kuratierten Suchfehler, +eindeutige und verweigerte Recall-Reparaturen, vollständige Controller-Prüfung, +Budgetneutralität, Cache-Aufteilung und die Zeichenobergrenze. +`git diff --check` ohne Befund; keine kostenpflichtigen Aufrufe. diff --git a/docs/test-results/token-usage/bug17-source-contract-repair-20261020.json b/docs/test-results/token-usage/bug17-source-contract-repair-20261020.json new file mode 100644 index 0000000..770671e --- /dev/null +++ b/docs/test-results/token-usage/bug17-source-contract-repair-20261020.json @@ -0,0 +1,20 @@ +{ + "date": "2026-10-20", + "case_id": "BUG-000017", + "package_id": "trial-bounded-full-BUG-000017-20261020", + "from_generation": 204, + "to_generation": 206, + "state": "INVESTIGATING_HYPOTHESES", + "api_calls": 0, + "full_logical_input_characters": 13849, + "preserved_original_entries": 806, + "budget_unchanged": true, + "hypothesis_states_unchanged": true, + "active_attempt_unchanged": true, + "new_context_ref": "analysis/investigation/0001/contexts/H1-0019-attempt-0003-guidance-v1.json", + "changes": [ + "manual technical retry", + "append versioned source guidance and precise rejection details" + ], + "package_valid": true +} diff --git a/src/viano_bug_analysis/analysis/investigation_contract.py b/src/viano_bug_analysis/analysis/investigation_contract.py index 2b4cfb4..c7a096b 100644 --- a/src/viano_bug_analysis/analysis/investigation_contract.py +++ b/src/viano_bug_analysis/analysis/investigation_contract.py @@ -127,6 +127,17 @@ def feedback(error, document): and action.get('operation') == 'project.query' and 'arguments' not in action): result.update(validation_path='/next/arguments', validation_fields=['arguments'], requirement='next.arguments must contain one complete query object, including its query kind; do not flatten query fields into next or infer missing values') + elif (path.startswith('/next') and isinstance(action,dict) and action.get('kind') == 'tool' + and action.get('operation') == 'project.query' and isinstance(action.get('arguments'),dict) + and 'kind' not in action['arguments']): + result.update(validation_path='/next/arguments/kind',validation_fields=['kind'], + requirement='next.arguments.kind is required. Choose a literal query kind; source.search needs text,path_prefix,case_sensitive,offset,limit. Do not guess kind.') + elif (path.startswith('/next') and isinstance(action,dict) and action.get('operation') == 'project.query' + and isinstance(action.get('arguments'),dict) and action['arguments'].get('kind') == 'source.search'): + result['requirement'] = 'source.search requires kind,text,path_prefix,case_sensitive,offset,limit. Supply missing fields; offset >= 0, limit > 0 within configured maximum. Keep intent.' + elif path.startswith('/next') and isinstance(action,dict) and action.get('kind') == 'recall' and 'arguments' in action: + result.update(validation_path='/next',validation_fields=['evidence_refs','page'], + requirement='recall has evidence_refs and optional page directly in next, without arguments; preserve the requested evidence ID and page') elif '/entity_refs' in path: result['requirement'] = 'entity_refs must use exact N references from input.entities; raw domain IDs are not references; never invent or guess a mapping' elif path == '/next/arguments' and isinstance(action,dict) and isinstance(action.get('arguments'),dict) and 'choices' in action['arguments']: diff --git a/src/viano_bug_analysis/analysis/investigation_window.py b/src/viano_bug_analysis/analysis/investigation_window.py index 6409461..7ef9ea7 100644 --- a/src/viano_bug_analysis/analysis/investigation_window.py +++ b/src/viano_bug_analysis/analysis/investigation_window.py @@ -232,6 +232,9 @@ def project(request, context): if v2(context): from .investigation_contract import entity_catalog view['entities'] = entity_catalog(context) + if context.get('research_guidance_version') == 1: + from .research_guidance import source_access + view['source_access'] = source_access(context) evidence = context['evidence'] ordered = sorted(evidence) previous = context.get('previous_result') @@ -251,6 +254,9 @@ def project(request, context): if key in old: body[key] = old[key] result['messages'] = [{'role': 'system', 'content': instructions_v2() if v2(context) else INSTRUCTIONS}, {'role': 'user', 'content': compact(body)}] + if context.get('research_guidance_version') == 1: + from .research_guidance import INSTRUCTIONS as research_instructions + result['messages'][0]['content'] += research_instructions if context.get('startup_explore'): result['messages'][0]['content'] += '\nInitial round: assessment=null; choose a tool only. No consolidation or confidence update.' def refresh(): diff --git a/src/viano_bug_analysis/analysis/investigator.py b/src/viano_bug_analysis/analysis/investigator.py index 0aa94a4..4e2f1ba 100644 --- a/src/viano_bug_analysis/analysis/investigator.py +++ b/src/viano_bug_analysis/analysis/investigator.py @@ -249,6 +249,7 @@ class Investigator: if state['policy'].get('prompt_window_version') == 2: # Frozen per context: historical requests remain reconstructable. context['assessment_scope_version'] = 1 + context['research_guidance_version'] = 1 if state.get("prompt_protocol") == 2: context["prompt_version"] = "investigation_project_v2" context["case_sources"] = interpretation_context["sources"] @@ -490,8 +491,12 @@ class Investigator: except (InvestigationError,CaseBuildError,BudgetError,ToolError) as error: if getattr(error,"code",None) in {"PROJECT_PROVIDER_UNAVAILABLE", "PROJECT_SOURCE_UNAVAILABLE", "PROJECT_DATABASE_UNAVAILABLE", "PROJECT_DATABASE_CREDENTIALS", "PROJECT_DATABASE_DRIVER", "PROJECT_BINDING_CHANGED"}: return self._research_blocked(state,error.code) + detail = getattr(error,'feedback',None) + if getattr(error,'code',None) == 'TOOL_SOURCE_NOT_DECLARED' and context['policy'].get('prompt_window_version') == 2: + from .research_guidance import namespace_feedback + detail = namespace_feedback(decision.get('next')) or detail return self._invalid(state,budget,getattr(error,"code","INVESTIGATION_OUTPUT"),outcome.result_ref, - feedback=getattr(error,'feedback',None)) + feedback=detail) active["decision"]=decision;active["llm_ref"]=outcome.result_ref;active["phase"]="executing" if call['outcome'].get('normalization'): state['pending_format_notice']=call['outcome']['normalization']['rules'][:16] diff --git a/src/viano_bug_analysis/analysis/research_guidance.py b/src/viano_bug_analysis/analysis/research_guidance.py new file mode 100644 index 0000000..385ea06 --- /dev/null +++ b/src/viano_bug_analysis/analysis/research_guidance.py @@ -0,0 +1,31 @@ +"""Small, versioned source namespaces and causal-method guidance; no bug rules.""" + + +def source_access(context): + refs = context.get('case',{}).get('source_context',{}).get('refs',[]) + paths = [v['path'] for v in refs if isinstance(v,dict) and isinstance(v.get('path'),str)] + shown = [] + for path in paths: + if len(shown) == 3 or sum(map(len,shown)) + len(path) > 700: + break + shown.append(path) + binding = context['tool_schema'].get('x-project-bindings',{}) + return {'repository_available':bool(binding.get('source')), + 'repository_operation':'project.query', + 'repository_search_fields':['kind','text','path_prefix','case_sensitive','offset','limit'], + 'package_operations':['source.find_text','source.find_symbol','source.read_context'], + 'package_source_refs':shown,'omitted_package_source_refs':len(paths)-len(shown)} + + +INSTRUCTIONS = ''' +Source namespaces: configured Git uses operation project.query and a complete query object. source.search requires kind,text,path_prefix,case_sensitive,offset,limit. Package source.* operations accept only exact package_source_refs, never repository paths. Do not guess missing kind or translate namespaces silently. Prefer source.context for call/binding analysis when available, then targeted primitives. +Test the proposed mechanism, not just a preferred method name. Before declaring a hook missing, check callers, overrides, indirect delegation and registration. An absent wrapper method alone does not exclude another valid entry point. Existing implementation is neutral unless it establishes a causal defect; retrieve relevant contrary evidence. Distinguish not-yet-read from absent.''' + + +def namespace_feedback(action): + if (not isinstance(action,dict) or action.get('kind') != 'tool' + or action.get('operation') not in {'source.find_text','source.find_symbol','source.read_context'}): + return None + field = 'source_ref' if action['operation'] == 'source.read_context' else 'source_refs' + return {'validation_path':'/next/arguments/'+field,'validation_fields':[field], + 'requirement':'package source.* requires declared package_source_refs, not repository paths. For Git use project.query and a complete source.search query; Do not guess.'} diff --git a/src/viano_bug_analysis/llm/response_recovery.py b/src/viano_bug_analysis/llm/response_recovery.py index 7b031ea..7ded9b2 100644 --- a/src/viano_bug_analysis/llm/response_recovery.py +++ b/src/viano_bug_analysis/llm/response_recovery.py @@ -103,6 +103,28 @@ def normalize(document, *, request=None): and isinstance(filters[1],str) and filters[1] in {'eq','ne','lt','le','gt','ge','contains','in'}): args['filters']=[dict(zip(('column','operator','value'),filters))] rules.append('flat_filter:/next/arguments/filters') + # A known recall has a single unambiguous container error. Never change its + # evidence ID/page, select a reference, invent a page or move sibling fields. + if (request and enabled(request) and request.get('output_schema',{}).get('x-viano-investigation-window') == 2 + and isinstance(action,dict) and set(action) == {'kind','arguments'} and action['kind'] == 'recall' + and isinstance(args,dict) and set(args) in ({'evidence_refs'},{'evidence_refs','page'}) + and isinstance(args.get('evidence_refs'),list) and len(args['evidence_refs']) == 1 + and isinstance(args['evidence_refs'][0],str) and re.fullmatch(r'E[0-9]{6,}',args['evidence_refs'][0]) + and ('page' not in args or type(args['page']) is int and args['page'] >= 0)): + try: + body = json.loads(request['messages'][1]['content']) + visible = set(body['input'].get('evidence',{})) | set(body['input'].get('archive',{}).get('recent_ids',[])) + except (ValueError,KeyError,IndexError,TypeError): + visible = set() + if args['evidence_refs'][0] in visible: + from jsonschema import Draft202012Validator + candidate = copy.deepcopy(result) + candidate['next'] = {'kind':'recall',**copy.deepcopy(args)} + if Draft202012Validator(request['output_schema']).is_valid(candidate): + result = candidate + action = result['next'] + args = {} + rules.append('unwrap_recall_arguments:/next') # A recall is not a newly charged research action. Discard only a well-formed # but non-applicable budget score, never causal findings, evidence references, # action intent or missing required values. The original score stays audited. diff --git a/src/viano_bug_analysis/llm/stable_context.py b/src/viano_bug_analysis/llm/stable_context.py index a29732c..ebe99f9 100644 --- a/src/viano_bug_analysis/llm/stable_context.py +++ b/src/viano_bug_analysis/llm/stable_context.py @@ -62,7 +62,7 @@ def split_document(document, prompt_version): # Cache its one tool/response contract plus immutable, targeted case data. current = copy.deepcopy(document) stable_input = {k:current['input'].pop(k) for k in - ('case','behavior','tool_profiles','entities') if k in current['input']} + ('case','behavior','tool_profiles','entities','source_access') if k in current['input']} descriptor = {'encoding':'bounded_investigation_v1','input':stable_input, 'response_contract':current.pop('response_contract')} descriptor['id'] = hashlib.sha256(json_bytes(descriptor)).hexdigest() diff --git a/src/viano_bug_analysis/schemas/investigation-context-v1.schema.json b/src/viano_bug_analysis/schemas/investigation-context-v1.schema.json index b2c8b83..16742c7 100644 --- a/src/viano_bug_analysis/schemas/investigation-context-v1.schema.json +++ b/src/viano_bug_analysis/schemas/investigation-context-v1.schema.json @@ -904,6 +904,9 @@ }, "assessment_scope_version": { "const": 1 + }, + "research_guidance_version": { + "const": 1 } } } diff --git a/tests/test_source_search_recovery.py b/tests/test_source_search_recovery.py new file mode 100644 index 0000000..a8c7beb --- /dev/null +++ b/tests/test_source_search_recovery.py @@ -0,0 +1,156 @@ +import copy +import json +from pathlib import Path + +import pytest +from jsonschema import Draft202012Validator + +from test_bounded_contract_v2 import context, loop_for +from test_case_builder import case_setup +from test_investigation_tools import tool_case +from test_investigation_startup import Adapter +from test_investigator import CASE, run, state +from viano_bug_analysis.analysis.investigation_prompt import build_prompt +from viano_bug_analysis.analysis.investigation_contract import feedback +from viano_bug_analysis.analysis.research_guidance import source_access, namespace_feedback +from viano_bug_analysis.llm.response_recovery import normalize +from viano_bug_analysis.llm.stable_context import split_document, restore_document +from viano_bug_analysis.llm.gemini import GeminiProvider +from viano_bug_analysis.llm.investigation_size import check_wire +from viano_bug_analysis.llm import ProviderResponse +from viano_bug_analysis.package import PackageReader, PackageValidator +from viano_bug_analysis.budget import BudgetManager +from viano_bug_analysis.tools.project_query import ProjectResearch + +CORPUS = Path(__file__).parent/'fixtures/llm_rejections/20261020/bug17-source-search-contract' + + +def captured(name): + folder = CORPUS/name + return tuple(json.loads((folder/f).read_text(encoding='utf-8')) for f in ['request.json','parsed-answer.json']) + + +def test_nested_recall_is_recovered_without_changing_reference_page_or_assessment(): + request,answer = captured('nested-recall') + before = copy.deepcopy(answer) + result,audit = normalize(answer,request=request) + assert result['next'] == {'kind':'recall',**answer['next']['arguments']} + assert result['assessment'] == answer['assessment'] + assert result['observation'] == answer['observation'] and answer == before + assert audit['original_document'] == before + assert audit['rules'] == ['unwrap_recall_arguments:/next'] + assert normalize(result,request=request) == (result,None) + + +@pytest.mark.parametrize('change',['extra','sibling','many','unknown','negative','boolean','string','operation','legacy','disabled']) +def test_ambiguous_recall_is_not_repaired(change): + request,answer = captured('nested-recall') + if change == 'extra': answer['next']['arguments']['other'] = 1 + if change == 'sibling': answer['next']['page'] = 4 + if change == 'many': answer['next']['arguments']['evidence_refs'].append('E000001') + if change == 'unknown': answer['next']['arguments']['evidence_refs'] = ['E999999'] + if change == 'negative': answer['next']['arguments']['page'] = -1 + if change == 'boolean': answer['next']['arguments']['page'] = True + if change == 'string': answer['next']['arguments']['page'] = '1' + if change == 'operation': answer['next']['operation'] = 'invented' + if change == 'legacy': request['output_schema']['x-viano-investigation-window'] = 1 + if change == 'disabled': request['output_schema'].pop('x-viano-response-recovery') + assert normalize(answer,request=request) == (answer,None) + + +@pytest.mark.parametrize('name,fields',[('eb01f48a',{'offset','limit'}),('a99adf87',{'kind'})]) +def test_actual_search_errors_get_precise_feedback_without_guessing(name,fields): + request,answer = captured(name) + query_schema = ProjectResearch.schema() + schema = {'properties':{'next':{'properties':{'arguments':query_schema}}}} + error = next(Draft202012Validator(schema).iter_errors(answer)) + detail = feedback(error,answer) + assert fields <= set(detail['validation_fields']) + assert 'source.search' in detail['requirement'] + assert len(detail['requirement']) <= 160 + assert normalize(answer,request=request) == (answer,None) + + +def test_package_reference_feedback_does_not_rewrite_or_echo_path(): + request,answer = captured('60d98274') + detail = namespace_feedback(answer['next']) + assert detail['validation_path'] == '/next/arguments/source_refs' + assert 'project.query' in detail['requirement'] and 'package' in detail['requirement'] + assert 'arathi' not in json.dumps(detail) + assert len(detail['requirement']) <= 160 + assert normalize(answer,request=request) == (answer,None) + + +def test_versioned_guidance_is_cached_bounded_and_generic(): + c = context();old = build_prompt(c) + c['case']['source_context'] = {'refs':[{'path':f'original/source-{i}.cpp'} for i in range(9)]} + before = copy.deepcopy(c) + c['research_guidance_version'] = 1 + request = build_prompt(c) + body = json.loads(request['messages'][1]['content']) + assert len(body['input']['source_access']['package_source_refs']) == 3 + assert body['input']['source_access']['omitted_package_source_refs'] == 6 + assert 'overrides' in request['messages'][0]['content'] + assert 'QuestAccept' not in request['messages'][0]['content'] + parts = split_document(body,request['prompt_version']) + assert 'source_access' in parts[0]['input'] + assert restore_document(*parts) == body + system,prefix,live = GeminiProvider._messages(request,stable_case_context=True) + assert check_wire(system,prefix+live) <= 20000 + c.pop('research_guidance_version') + assert c == before + assert 'source_access' not in json.loads(old['messages'][1]['content'])['input'] + + +def test_long_package_paths_do_not_expand_catalog(): + c = context();c['case']['source_context'] = {'refs':[{'path':'x'*701}]} + result = source_access(c) + assert result['package_source_refs'] == [] and result['omitted_package_source_refs'] == 1 + + +def test_bad_namespace_gets_controller_feedback_without_budget_side_effect(tool_case): + class Wrong(Adapter): + async def generate(self,config,request,**kwargs): + body = json.loads(request['messages'][1]['content']) + if body['task'] != 'investigate_one_step': + return await super().generate(config,request,**kwargs) + return ProviderResponse({'assessment':None,'observation':'Search the existing source.', + 'next':{'kind':'tool','operation':'source.find_text','arguments':{ + 'source_refs':['src/not-a-package-reference.cpp'],'text':'Start', + 'case_sensitive':True,'offset':0,'limit':10}}},'fixture',100,50) + loop = loop_for(tool_case,Wrong());run(loop,steps=4) + before = BudgetManager(tool_case[0]).snapshot(*CASE).state + result = run(loop,steps=1) + saved = state(result) + assert saved['hypotheses']['H1']['notes'][-1]['code'] == 'TOOL_SOURCE_NOT_DECLARED' + with PackageReader(result.package) as reader: + c = json.loads(reader.read_bytes(saved['active']['context_ref'])) + assert 'package_source_refs' in c['previous_rejection_details']['requirement'] + assert c['previous_rejection_details']['validation_path'] == '/next/arguments/source_refs' + assert BudgetManager(tool_case[0]).snapshot(*CASE).state == before + assert PackageValidator().validate(result.package).valid + + +def test_nested_recall_is_executed_without_new_research_charge(tool_case): + class Recall(Adapter): + async def generate(self,config,request,**kwargs): + body = json.loads(request['messages'][1]['content']) + if body['task'] != 'investigate_one_step': + return await super().generate(config,request,**kwargs) + self.calls.append(body) + return ProviderResponse({'assessment':None,'observation':'Read recorded evidence.', + 'next':{'kind':'recall','arguments':{'evidence_refs':['E000001'],'page':0}}}, + 'fixture',100,50) + adapter = Recall();loop = loop_for(tool_case,adapter) + run(loop,steps=4) + before = BudgetManager(tool_case[0]).snapshot(*CASE).state + result = run(loop,steps=1) + saved = state(result) + assert saved['hypotheses']['H1']['invalid_responses'] == 0 + assert BudgetManager(tool_case[0]).snapshot(*CASE).state == before + with PackageReader(result.package) as reader: + ref = saved['hypotheses']['H1']['notes'][-1]['llm_ref'] + call = json.loads(reader.read_bytes(ref)) + assert call['outcome']['document']['next'] == {'kind':'recall','evidence_refs':['E000001'],'page':0} + assert call['outcome']['normalization']['rules'] == ['unwrap_recall_arguments:/next'] + assert PackageValidator().validate(result.package).valid