From 29816dc45b546dc4935ee5b084ecab502d73f903 Mon Sep 17 00:00:00 2001 From: Nitsu45 Date: Sun, 18 Oct 2026 09:17:24 +0200 Subject: [PATCH] Resolve nested tool schema failures into actionable repair feedback --- docs/investigation-step-v17.md | 8 ++ .../analysis/investigation_compact.py | 13 ++- src/viano_bug_analysis/llm/diagnostics.py | 40 ++++++--- src/viano_bug_analysis/llm/gateway.py | 2 +- tests/test_tool_repair_feedback.py | 85 +++++++++++++++++++ 5 files changed, 131 insertions(+), 17 deletions(-) create mode 100644 tests/test_tool_repair_feedback.py diff --git a/docs/investigation-step-v17.md b/docs/investigation-step-v17.md index 2323190..8faca9d 100644 --- a/docs/investigation-step-v17.md +++ b/docs/investigation-step-v17.md @@ -28,3 +28,11 @@ Die enge Frühabschluss-Bewertung verwendet unverändert investigation_review_v1 ## Kostenbewertung Diese Maßnahme reduziert die zu lösende Denkaufgabe, nicht unmittelbar die Eingabezeichen. Die zusätzlichen mechanischen Hinweise kosten etwas Input. Offline-Größen derselben gespeicherten Bug20-Kontexte stehen in [investigation-v17-framing-20261018.json](measurements/investigation-v17-framing-20261018.json). Sie sind keine Messung tatsächlicher Tokens oder eingesparter Denkarbeit. Der nächste Live-Test muss Denktokens, Aufrufzahl, Laufzeit und Lösungsqualität gemeinsam prüfen. Keine zusätzlichen LLM-Zusammenfassungen, kein niedrigeres allgemeines Denkniveau, keine kostenpflichtigen Testaufrufe während der Implementierung. + +## Präzise Korrekturhinweise für Werkzeugargumente + +Verschachtelte oneOf-/anyOf-Fehler werden anhand der vorhandenen kind- und operation-Diskriminatoren bis zur passenden Variante verfolgt, einschließlich enum-basierter Queryarten. Dafür werden die bereits gegen das vollständige Schema aufgelösten Unterfehler verwendet; isoliertes erneutes Validieren einer Variante verliert sonst Root-Definitionen und Pfadinformationen. + +Beispiel: Eine source.search-Anfrage ohne path_prefix erhält validation_path=/next/arguments, validation_fields=[path_prefix] und den Hinweis, dass path_prefix="" eine Suche ab Repository-Wurzel bezeichnet. Andere fehlende Pflichtfelder, etwa value in einem DB-Filter, werden entsprechend an ihrer tatsächlichen Position genannt. Typfehler werden von fehlenden Feldern unterschieden. Unbekannte oder mehrdeutige Diskriminatoren bleiben ein allgemeiner Formfehler, statt Felder einer geratenen Variante zu verlangen. + +Es werden weiterhin nur erlaubte Feldnamen, Fehlerkategorien und begrenzte Schema-Pfade ausgegeben, keine Anfragewerte oder Rohantworten. Die Hinweise gelangen über den normalen Checkpoint in den nächsten Korrekturprompt. Keine automatische Ergänzung von Werkzeugargumenten, kein höheres Wiederholungslimit, keine zusätzliche LLM-Abfrage. Die beiden abgelehnten Antworten des ersten v17-Bug20-Laufs wurden lokal erneut geprüft; beide benennen nun das fehlende path_prefix korrekt. diff --git a/src/viano_bug_analysis/analysis/investigation_compact.py b/src/viano_bug_analysis/analysis/investigation_compact.py index 8fbd0e6..1e25089 100644 --- a/src/viano_bug_analysis/analysis/investigation_compact.py +++ b/src/viano_bug_analysis/analysis/investigation_compact.py @@ -239,12 +239,21 @@ def validate_tool_shape(context,decision): error=next(Draft202012Validator(output_schema(context,strict=True)).iter_errors(decision),None) if error: from viano_bug_analysis.llm.diagnostics import schema_feedback - detail=schema_feedback(error) + detail=schema_feedback(error,include_keyword=True) + keyword=detail.pop('validation_keyword',None) if (context.get('prompt_version') in {'investigation_project_v16','investigation_project_v17'} and 'evidence_refs' in error.absolute_path): detail['requirement'] = 'research_E_ids_only_not_case_F_ids_empty_findings_before_research' raise InvestigationError('INVESTIGATION_OUTPUT', feedback=detail) - raise InvestigationError('INVESTIGATION_OUTPUT',feedback={**detail,'requirement':'match_full_controller_contract_at_validation_path'}) + requirement={'required':'provide_missing_required_fields', + 'additionalProperties':'remove_unrecognized_fields', + 'type':'match_expected_type_at_validation_path', + 'enum':'use_allowed_value_at_validation_path', + 'const':'use_allowed_value_at_validation_path'}.get(keyword, + 'match_full_controller_contract_at_validation_path') + if keyword=='required' and 'path_prefix' in detail['validation_fields']: + requirement='provide_missing_required_fields; path_prefix="" searches_repository_root' + raise InvestigationError('INVESTIGATION_OUTPUT',feedback={**detail,'requirement':requirement}) view=review_view(context) if preparing_review(context) else model_view(context) readable=set(view['evidence_policy']['full_visible_ids']) | set(view.get('known_schemas', {})) for memory in [view['working_brief'],*view['other_working_briefs'].values()]: diff --git a/src/viano_bug_analysis/llm/diagnostics.py b/src/viano_bug_analysis/llm/diagnostics.py index b50edaf..f642779 100644 --- a/src/viano_bug_analysis/llm/diagnostics.py +++ b/src/viano_bug_analysis/llm/diagnostics.py @@ -13,22 +13,31 @@ VALIDATION_FIELDS = frozenset({'assessment','semantic_score','reason','update',' 'value','text','case_sensitive','path_prefix','source_ref','source_refs','note','causal_review'}) -def schema_feedback(error, _depth=0): +def schema_feedback(error, _depth=0, *, include_keyword=False): """Schema location/required field names only; never echo response values.""" - if _depth < 4 and error.validator in {'oneOf','anyOf'} and isinstance(error.instance,dict) and 'kind' in error.instance: + if _depth < 8 and error.validator in {'oneOf','anyOf'} and isinstance(error.instance,dict): branches=error.schema.get(error.validator,[]) - matching=[b for b in branches if isinstance(b,dict) and b.get('properties',{}).get('kind',{}).get('const')==error.instance['kind']] + matching=[] + for index,branch in enumerate(branches): + if not isinstance(branch,dict):continue + properties=branch.get('properties',{}) + compared=False;matches=True + for discriminator in ('kind','operation'): + rule=properties.get(discriminator,{}) + if not isinstance(rule,dict):continue + if discriminator not in error.instance:continue + if 'const' in rule: + compared=True;matches &= error.instance[discriminator]==rule['const'] + elif 'enum' in rule: + compared=True;matches &= error.instance[discriminator] in rule['enum'] + if compared and matches:matching.append(index) if len(matching)==1: - from jsonschema import Draft202012Validator - try: - nested=next(Draft202012Validator(matching[0]).iter_errors(error.instance),None) - except Exception: - nested=None # Optional diagnostics must never break controlled rejection. - if nested: - info=schema_feedback(nested, _depth+1) - prefix='/'.join(p if isinstance(p,str) and p in VALIDATION_FIELDS else '?' for p in error.absolute_path) - info['validation_path']=('/'+prefix if prefix else '')+info['validation_path'] - return info + # Reuse errors resolved against the full schema; revalidating an + # isolated branch loses its root $defs and original field path. + nested=next((child for child in error.context if child.schema_path + and child.schema_path[0]==matching[0]),None) + if nested is not None: + return schema_feedback(nested,_depth+1,include_keyword=include_keyword) path = '/'.join(str(p) if type(p) is int and 0 <= p <= 1000000 else p if isinstance(p,str) and p in VALIDATION_FIELDS else '?' for p in list(error.absolute_path)[:8]) @@ -37,7 +46,10 @@ def schema_feedback(error, _depth=0): fields = [f for f in error.validator_value if f not in error.instance and f in VALIDATION_FIELDS][:8] elif error.validator == 'additionalProperties' and isinstance(error.instance,dict): fields = sorted((set(error.instance)-set(error.schema.get('properties',{}))) & VALIDATION_FIELDS)[:8] - return {'validation_path':'/'+path, 'validation_fields':fields} + result={'validation_path':'/'+path, 'validation_fields':fields} + if include_keyword and error.validator in KEYWORDS: + result['validation_keyword']=error.validator + return result def count(value): diff --git a/src/viano_bug_analysis/llm/gateway.py b/src/viano_bug_analysis/llm/gateway.py index cc85680..c2aac42 100644 --- a/src/viano_bug_analysis/llm/gateway.py +++ b/src/viano_bug_analysis/llm/gateway.py @@ -157,7 +157,7 @@ class LLMGateway: info={'validation_keyword':invalid.validator} if request['prompt_version'] in {'investigation_project_v14','investigation_project_v15','investigation_project_v16','investigation_project_v17'}: from .diagnostics import schema_feedback - info.update(schema_feedback(invalid)) + info.update(schema_feedback(invalid,include_keyword=True)) error = LLMError("LLM_OUTPUT_SCHEMA",diagnostics=info) self._failure_metadata(error,provider,request,started,response,attempts) raise error from None diff --git a/tests/test_tool_repair_feedback.py b/tests/test_tool_repair_feedback.py new file mode 100644 index 0000000..7f65fb5 --- /dev/null +++ b/tests/test_tool_repair_feedback.py @@ -0,0 +1,85 @@ +import asyncio +import json + +import pytest +from jsonschema import Draft202012Validator + +from test_compact_investigation import context +from test_lean_investigation_decision import reply +from test_case_builder import case_setup +from test_investigation_tools import tool_case +from test_project_research import source +from viano_bug_analysis.analysis.investigation_prompt import output_schema +from viano_bug_analysis.analysis.investigation_compact import validate_tool_shape +from viano_bug_analysis.analysis.investigation_config import InvestigationError +from viano_bug_analysis.llm.diagnostics import schema_feedback,sanitize + + +@pytest.mark.parametrize('kind',['source.search','source.symbol','source.registration','source.schema']) +def test_nested_enum_tool_feedback_names_missing_path_prefix(kind): + c=context();c['prompt_version']='investigation_project_v17';r=reply() + r['next']={'kind':'tool','operation':'project.query','arguments':{ + 'kind':kind,'text':'PRIVATE_QUERY','case_sensitive':False,'offset':0,'limit':20}} + with pytest.raises(InvestigationError) as caught:validate_tool_shape(c,r) + info=caught.value.feedback + assert info['validation_path']=='/next/arguments' + assert info['validation_fields']==['path_prefix'] + assert 'path_prefix="" searches_repository_root' in info['requirement'] + assert 'PRIVATE_QUERY' not in json.dumps(info) + + +def test_feedback_descends_through_nested_filters_and_root_definitions(): + c=context();c['prompt_version']='investigation_project_v17';r=reply() + r['next']={'kind':'tool','operation':'project.query','arguments':{ + 'kind':'database.rows','table':'creature','columns':['guid'], + 'filters':[{'column':'id','operator':'eq'}],'offset':0,'limit':20}} + with pytest.raises(InvestigationError) as caught:validate_tool_shape(c,r) + assert caught.value.feedback=={'validation_path':'/next/arguments/filters/0', + 'validation_fields':['value'],'requirement':'provide_missing_required_fields'} + + +def test_feedback_reports_type_error_without_recommending_missing_fields(): + c=context();r=reply();r['next']={'kind':'tool','operation':'project.query','arguments':{ + 'kind':'source.search','text':'query','path_prefix':'','case_sensitive':False,'offset':0,'limit':'PRIVATE'}} + with pytest.raises(InvestigationError) as caught:validate_tool_shape(c,r) + assert caught.value.feedback['validation_path']=='/next/arguments/limit' + assert caught.value.feedback['requirement']=='match_expected_type_at_validation_path' + assert 'PRIVATE' not in json.dumps(caught.value.feedback) + + +def test_unknown_discriminator_does_not_choose_arbitrary_branch(): + c=context();r=reply();r['next']={'kind':'tool','operation':'PRIVATE_OPERATION','arguments':{}} + error=next(Draft202012Validator(output_schema(c,strict=True)).iter_errors(r)) + info=schema_feedback(error,include_keyword=True) + assert info['validation_path']=='/next' and info['validation_fields']==[] + assert sanitize(info)==info and 'PRIVATE' not in json.dumps(info) + + +def test_precise_feedback_reaches_checkpoint_then_one_corrected_source_query(tool_case,source): + from test_investigator import LoopAdapter,CASE,state + from test_case_interpreter import gateway + from viano_bug_analysis.analysis import Investigator + from viano_bug_analysis.tools import InvestigationToolRouter + from viano_bug_analysis.tools.project_query import ProjectResearch + from viano_bug_analysis.package import PackageReader,PackageValidator + from viano_bug_analysis.budget import BudgetManager + store=tool_case[0] + with PackageReader(tool_case[1]) as reader:project=reader.manifest['project'] + service=ProjectResearch(project,source=source);calls=[];original=source.query + def counted(args):calls.append(args);return original(args) + source.query=counted + def change(reply,c): + args={'kind':'source.search','text':'activate','case_sensitive':False,'offset':0,'limit':20} + if c.get('previous_rejection_details'): + assert c['previous_rejection_details']['validation_fields']==['path_prefix'] + args['path_prefix']='' + reply['next']={'kind':'tool','operation':'project.query','arguments':args} + adapter=LoopAdapter(change);loop=Investigator(store,gateway(adapter),router=InvestigationToolRouter(project_research=service)) + first=asyncio.run(loop.step(*CASE));assert first.status=='REJECTED' and not calls + assert not BudgetManager(store).snapshot(*CASE).state['accounts']['H1']['actions'] + with PackageReader(first.package) as reader: + c=json.loads(reader.read_bytes(state(first)['active']['context_ref'])) + assert c['previous_rejection_details']['validation_path']=='/next/arguments' + second=asyncio.run(loop.step(*CASE)) + assert second.status=='PROGRESSED' and len(calls)==1 + assert PackageValidator().validate(second.package).valid