Resolve nested tool schema failures into actionable repair feedback

This commit is contained in:
Nico committed 2026-10-18 09:17:24 +02:00
1 parent 6a9788114d
commit 29816dc45b
5 files changed
+131 -17

No files matched your search

+8
View File
@@ -28,3 +28,11 @@ Die enge Frühabschluss-Bewertung verwendet unverändert investigation_review_v1
## Kostenbewertung
Diese Maßnahme reduziert die zu lösende Denkaufgabe, nicht unmittelbar die Eingabezeichen. Die zusätzlichen mechanischen Hinweise kosten etwas Input. Offline-Größen derselben gespeicherten Bug20-Kontexte stehen in [investigation-v17-framing-20261018.json](measurements/investigation-v17-framing-20261018.json). Sie sind keine Messung tatsächlicher Tokens oder eingesparter Denkarbeit. Der nächste Live-Test muss Denktokens, Aufrufzahl, Laufzeit und Lösungsqualität gemeinsam prüfen. Keine zusätzlichen LLM-Zusammenfassungen, kein niedrigeres allgemeines Denkniveau, keine kostenpflichtigen Testaufrufe während der Implementierung.
## Präzise Korrekturhinweise für Werkzeugargumente
Verschachtelte oneOf-/anyOf-Fehler werden anhand der vorhandenen kind- und operation-Diskriminatoren bis zur passenden Variante verfolgt, einschließlich enum-basierter Queryarten. Dafür werden die bereits gegen das vollständige Schema aufgelösten Unterfehler verwendet; isoliertes erneutes Validieren einer Variante verliert sonst Root-Definitionen und Pfadinformationen.
Beispiel: Eine source.search-Anfrage ohne path_prefix erhält validation_path=/next/arguments, validation_fields=[path_prefix] und den Hinweis, dass path_prefix="" eine Suche ab Repository-Wurzel bezeichnet. Andere fehlende Pflichtfelder, etwa value in einem DB-Filter, werden entsprechend an ihrer tatsächlichen Position genannt. Typfehler werden von fehlenden Feldern unterschieden. Unbekannte oder mehrdeutige Diskriminatoren bleiben ein allgemeiner Formfehler, statt Felder einer geratenen Variante zu verlangen.
Es werden weiterhin nur erlaubte Feldnamen, Fehlerkategorien und begrenzte Schema-Pfade ausgegeben, keine Anfragewerte oder Rohantworten. Die Hinweise gelangen über den normalen Checkpoint in den nächsten Korrekturprompt. Keine automatische Ergänzung von Werkzeugargumenten, kein höheres Wiederholungslimit, keine zusätzliche LLM-Abfrage. Die beiden abgelehnten Antworten des ersten v17-Bug20-Laufs wurden lokal erneut geprüft; beide benennen nun das fehlende path_prefix korrekt.
@@ -239,12 +239,21 @@ def validate_tool_shape(context,decision):
error=next(Draft202012Validator(output_schema(context,strict=True)).iter_errors(decision),None)
if error:
from viano_bug_analysis.llm.diagnostics import schema_feedback
detail=schema_feedback(error)
detail=schema_feedback(error,include_keyword=True)
keyword=detail.pop('validation_keyword',None)
if (context.get('prompt_version') in {'investigation_project_v16','investigation_project_v17'}
and 'evidence_refs' in error.absolute_path):
detail['requirement'] = 'research_E_ids_only_not_case_F_ids_empty_findings_before_research'
raise InvestigationError('INVESTIGATION_OUTPUT', feedback=detail)
raise InvestigationError('INVESTIGATION_OUTPUT',feedback={**detail,'requirement':'match_full_controller_contract_at_validation_path'})
requirement={'required':'provide_missing_required_fields',
'additionalProperties':'remove_unrecognized_fields',
'type':'match_expected_type_at_validation_path',
'enum':'use_allowed_value_at_validation_path',
'const':'use_allowed_value_at_validation_path'}.get(keyword,
'match_full_controller_contract_at_validation_path')
if keyword=='required' and 'path_prefix' in detail['validation_fields']:
requirement='provide_missing_required_fields; path_prefix="" searches_repository_root'
raise InvestigationError('INVESTIGATION_OUTPUT',feedback={**detail,'requirement':requirement})
view=review_view(context) if preparing_review(context) else model_view(context)
readable=set(view['evidence_policy']['full_visible_ids']) | set(view.get('known_schemas', {}))
for memory in [view['working_brief'],*view['other_working_briefs'].values()]:
+26 -14
View File
@@ -13,22 +13,31 @@ VALIDATION_FIELDS = frozenset({'assessment','semantic_score','reason','update','
'value','text','case_sensitive','path_prefix','source_ref','source_refs','note','causal_review'})
def schema_feedback(error, _depth=0):
def schema_feedback(error, _depth=0, *, include_keyword=False):
"""Schema location/required field names only; never echo response values."""
if _depth < 4 and error.validator in {'oneOf','anyOf'} and isinstance(error.instance,dict) and 'kind' in error.instance:
if _depth < 8 and error.validator in {'oneOf','anyOf'} and isinstance(error.instance,dict):
branches=error.schema.get(error.validator,[])
matching=[b for b in branches if isinstance(b,dict) and b.get('properties',{}).get('kind',{}).get('const')==error.instance['kind']]
matching=[]
for index,branch in enumerate(branches):
if not isinstance(branch,dict):continue
properties=branch.get('properties',{})
compared=False;matches=True
for discriminator in ('kind','operation'):
rule=properties.get(discriminator,{})
if not isinstance(rule,dict):continue
if discriminator not in error.instance:continue
if 'const' in rule:
compared=True;matches &= error.instance[discriminator]==rule['const']
elif 'enum' in rule:
compared=True;matches &= error.instance[discriminator] in rule['enum']
if compared and matches:matching.append(index)
if len(matching)==1:
from jsonschema import Draft202012Validator
try:
nested=next(Draft202012Validator(matching[0]).iter_errors(error.instance),None)
except Exception:
nested=None # Optional diagnostics must never break controlled rejection.
if nested:
info=schema_feedback(nested, _depth+1)
prefix='/'.join(p if isinstance(p,str) and p in VALIDATION_FIELDS else '?' for p in error.absolute_path)
info['validation_path']=('/'+prefix if prefix else '')+info['validation_path']
return info
# Reuse errors resolved against the full schema; revalidating an
# isolated branch loses its root $defs and original field path.
nested=next((child for child in error.context if child.schema_path
and child.schema_path[0]==matching[0]),None)
if nested is not None:
return schema_feedback(nested,_depth+1,include_keyword=include_keyword)
path = '/'.join(str(p) if type(p) is int and 0 <= p <= 1000000 else
p if isinstance(p,str) and p in VALIDATION_FIELDS else '?'
for p in list(error.absolute_path)[:8])
@@ -37,7 +46,10 @@ def schema_feedback(error, _depth=0):
fields = [f for f in error.validator_value if f not in error.instance and f in VALIDATION_FIELDS][:8]
elif error.validator == 'additionalProperties' and isinstance(error.instance,dict):
fields = sorted((set(error.instance)-set(error.schema.get('properties',{}))) & VALIDATION_FIELDS)[:8]
return {'validation_path':'/'+path, 'validation_fields':fields}
result={'validation_path':'/'+path, 'validation_fields':fields}
if include_keyword and error.validator in KEYWORDS:
result['validation_keyword']=error.validator
return result
def count(value):
+1 -1
View File
@@ -157,7 +157,7 @@ class LLMGateway:
info={'validation_keyword':invalid.validator}
if request['prompt_version'] in {'investigation_project_v14','investigation_project_v15','investigation_project_v16','investigation_project_v17'}:
from .diagnostics import schema_feedback
info.update(schema_feedback(invalid))
info.update(schema_feedback(invalid,include_keyword=True))
error = LLMError("LLM_OUTPUT_SCHEMA",diagnostics=info)
self._failure_metadata(error,provider,request,started,response,attempts)
raise error from None
+85
View File
@@ -0,0 +1,85 @@
import asyncio
import json
import pytest
from jsonschema import Draft202012Validator
from test_compact_investigation import context
from test_lean_investigation_decision import reply
from test_case_builder import case_setup
from test_investigation_tools import tool_case
from test_project_research import source
from viano_bug_analysis.analysis.investigation_prompt import output_schema
from viano_bug_analysis.analysis.investigation_compact import validate_tool_shape
from viano_bug_analysis.analysis.investigation_config import InvestigationError
from viano_bug_analysis.llm.diagnostics import schema_feedback,sanitize
@pytest.mark.parametrize('kind',['source.search','source.symbol','source.registration','source.schema'])
def test_nested_enum_tool_feedback_names_missing_path_prefix(kind):
c=context();c['prompt_version']='investigation_project_v17';r=reply()
r['next']={'kind':'tool','operation':'project.query','arguments':{
'kind':kind,'text':'PRIVATE_QUERY','case_sensitive':False,'offset':0,'limit':20}}
with pytest.raises(InvestigationError) as caught:validate_tool_shape(c,r)
info=caught.value.feedback
assert info['validation_path']=='/next/arguments'
assert info['validation_fields']==['path_prefix']
assert 'path_prefix="" searches_repository_root' in info['requirement']
assert 'PRIVATE_QUERY' not in json.dumps(info)
def test_feedback_descends_through_nested_filters_and_root_definitions():
c=context();c['prompt_version']='investigation_project_v17';r=reply()
r['next']={'kind':'tool','operation':'project.query','arguments':{
'kind':'database.rows','table':'creature','columns':['guid'],
'filters':[{'column':'id','operator':'eq'}],'offset':0,'limit':20}}
with pytest.raises(InvestigationError) as caught:validate_tool_shape(c,r)
assert caught.value.feedback=={'validation_path':'/next/arguments/filters/0',
'validation_fields':['value'],'requirement':'provide_missing_required_fields'}
def test_feedback_reports_type_error_without_recommending_missing_fields():
c=context();r=reply();r['next']={'kind':'tool','operation':'project.query','arguments':{
'kind':'source.search','text':'query','path_prefix':'','case_sensitive':False,'offset':0,'limit':'PRIVATE'}}
with pytest.raises(InvestigationError) as caught:validate_tool_shape(c,r)
assert caught.value.feedback['validation_path']=='/next/arguments/limit'
assert caught.value.feedback['requirement']=='match_expected_type_at_validation_path'
assert 'PRIVATE' not in json.dumps(caught.value.feedback)
def test_unknown_discriminator_does_not_choose_arbitrary_branch():
c=context();r=reply();r['next']={'kind':'tool','operation':'PRIVATE_OPERATION','arguments':{}}
error=next(Draft202012Validator(output_schema(c,strict=True)).iter_errors(r))
info=schema_feedback(error,include_keyword=True)
assert info['validation_path']=='/next' and info['validation_fields']==[]
assert sanitize(info)==info and 'PRIVATE' not in json.dumps(info)
def test_precise_feedback_reaches_checkpoint_then_one_corrected_source_query(tool_case,source):
from test_investigator import LoopAdapter,CASE,state
from test_case_interpreter import gateway
from viano_bug_analysis.analysis import Investigator
from viano_bug_analysis.tools import InvestigationToolRouter
from viano_bug_analysis.tools.project_query import ProjectResearch
from viano_bug_analysis.package import PackageReader,PackageValidator
from viano_bug_analysis.budget import BudgetManager
store=tool_case[0]
with PackageReader(tool_case[1]) as reader:project=reader.manifest['project']
service=ProjectResearch(project,source=source);calls=[];original=source.query
def counted(args):calls.append(args);return original(args)
source.query=counted
def change(reply,c):
args={'kind':'source.search','text':'activate','case_sensitive':False,'offset':0,'limit':20}
if c.get('previous_rejection_details'):
assert c['previous_rejection_details']['validation_fields']==['path_prefix']
args['path_prefix']=''
reply['next']={'kind':'tool','operation':'project.query','arguments':args}
adapter=LoopAdapter(change);loop=Investigator(store,gateway(adapter),router=InvestigationToolRouter(project_research=service))
first=asyncio.run(loop.step(*CASE));assert first.status=='REJECTED' and not calls
assert not BudgetManager(store).snapshot(*CASE).state['accounts']['H1']['actions']
with PackageReader(first.package) as reader:
c=json.loads(reader.read_bytes(state(first)['active']['context_ref']))
assert c['previous_rejection_details']['validation_path']=='/next/arguments'
second=asyncio.run(loop.step(*CASE))
assert second.status=='PROGRESSED' and len(calls)==1
assert PackageValidator().validate(second.package).valid