Resolve nested tool schema failures into actionable repair feedback
This commit is contained in:
1 parent
6a9788114d
commit
29816dc45b
5 files changed
+131
-17
No files matched your search
@@ -28,3 +28,11 @@ Die enge Frühabschluss-Bewertung verwendet unverändert investigation_review_v1
|
||||
## Kostenbewertung
|
||||
|
||||
Diese Maßnahme reduziert die zu lösende Denkaufgabe, nicht unmittelbar die Eingabezeichen. Die zusätzlichen mechanischen Hinweise kosten etwas Input. Offline-Größen derselben gespeicherten Bug20-Kontexte stehen in [investigation-v17-framing-20261018.json](measurements/investigation-v17-framing-20261018.json). Sie sind keine Messung tatsächlicher Tokens oder eingesparter Denkarbeit. Der nächste Live-Test muss Denktokens, Aufrufzahl, Laufzeit und Lösungsqualität gemeinsam prüfen. Keine zusätzlichen LLM-Zusammenfassungen, kein niedrigeres allgemeines Denkniveau, keine kostenpflichtigen Testaufrufe während der Implementierung.
|
||||
|
||||
## Präzise Korrekturhinweise für Werkzeugargumente
|
||||
|
||||
Verschachtelte oneOf-/anyOf-Fehler werden anhand der vorhandenen kind- und operation-Diskriminatoren bis zur passenden Variante verfolgt, einschließlich enum-basierter Queryarten. Dafür werden die bereits gegen das vollständige Schema aufgelösten Unterfehler verwendet; isoliertes erneutes Validieren einer Variante verliert sonst Root-Definitionen und Pfadinformationen.
|
||||
|
||||
Beispiel: Eine source.search-Anfrage ohne path_prefix erhält validation_path=/next/arguments, validation_fields=[path_prefix] und den Hinweis, dass path_prefix="" eine Suche ab Repository-Wurzel bezeichnet. Andere fehlende Pflichtfelder, etwa value in einem DB-Filter, werden entsprechend an ihrer tatsächlichen Position genannt. Typfehler werden von fehlenden Feldern unterschieden. Unbekannte oder mehrdeutige Diskriminatoren bleiben ein allgemeiner Formfehler, statt Felder einer geratenen Variante zu verlangen.
|
||||
|
||||
Es werden weiterhin nur erlaubte Feldnamen, Fehlerkategorien und begrenzte Schema-Pfade ausgegeben, keine Anfragewerte oder Rohantworten. Die Hinweise gelangen über den normalen Checkpoint in den nächsten Korrekturprompt. Keine automatische Ergänzung von Werkzeugargumenten, kein höheres Wiederholungslimit, keine zusätzliche LLM-Abfrage. Die beiden abgelehnten Antworten des ersten v17-Bug20-Laufs wurden lokal erneut geprüft; beide benennen nun das fehlende path_prefix korrekt.
|
||||
@@ -239,12 +239,21 @@ def validate_tool_shape(context,decision):
|
||||
error=next(Draft202012Validator(output_schema(context,strict=True)).iter_errors(decision),None)
|
||||
if error:
|
||||
from viano_bug_analysis.llm.diagnostics import schema_feedback
|
||||
detail=schema_feedback(error)
|
||||
detail=schema_feedback(error,include_keyword=True)
|
||||
keyword=detail.pop('validation_keyword',None)
|
||||
if (context.get('prompt_version') in {'investigation_project_v16','investigation_project_v17'}
|
||||
and 'evidence_refs' in error.absolute_path):
|
||||
detail['requirement'] = 'research_E_ids_only_not_case_F_ids_empty_findings_before_research'
|
||||
raise InvestigationError('INVESTIGATION_OUTPUT', feedback=detail)
|
||||
raise InvestigationError('INVESTIGATION_OUTPUT',feedback={**detail,'requirement':'match_full_controller_contract_at_validation_path'})
|
||||
requirement={'required':'provide_missing_required_fields',
|
||||
'additionalProperties':'remove_unrecognized_fields',
|
||||
'type':'match_expected_type_at_validation_path',
|
||||
'enum':'use_allowed_value_at_validation_path',
|
||||
'const':'use_allowed_value_at_validation_path'}.get(keyword,
|
||||
'match_full_controller_contract_at_validation_path')
|
||||
if keyword=='required' and 'path_prefix' in detail['validation_fields']:
|
||||
requirement='provide_missing_required_fields; path_prefix="" searches_repository_root'
|
||||
raise InvestigationError('INVESTIGATION_OUTPUT',feedback={**detail,'requirement':requirement})
|
||||
view=review_view(context) if preparing_review(context) else model_view(context)
|
||||
readable=set(view['evidence_policy']['full_visible_ids']) | set(view.get('known_schemas', {}))
|
||||
for memory in [view['working_brief'],*view['other_working_briefs'].values()]:
|
||||
|
||||
@@ -13,22 +13,31 @@ VALIDATION_FIELDS = frozenset({'assessment','semantic_score','reason','update','
|
||||
'value','text','case_sensitive','path_prefix','source_ref','source_refs','note','causal_review'})
|
||||
|
||||
|
||||
def schema_feedback(error, _depth=0):
|
||||
def schema_feedback(error, _depth=0, *, include_keyword=False):
|
||||
"""Schema location/required field names only; never echo response values."""
|
||||
if _depth < 4 and error.validator in {'oneOf','anyOf'} and isinstance(error.instance,dict) and 'kind' in error.instance:
|
||||
if _depth < 8 and error.validator in {'oneOf','anyOf'} and isinstance(error.instance,dict):
|
||||
branches=error.schema.get(error.validator,[])
|
||||
matching=[b for b in branches if isinstance(b,dict) and b.get('properties',{}).get('kind',{}).get('const')==error.instance['kind']]
|
||||
matching=[]
|
||||
for index,branch in enumerate(branches):
|
||||
if not isinstance(branch,dict):continue
|
||||
properties=branch.get('properties',{})
|
||||
compared=False;matches=True
|
||||
for discriminator in ('kind','operation'):
|
||||
rule=properties.get(discriminator,{})
|
||||
if not isinstance(rule,dict):continue
|
||||
if discriminator not in error.instance:continue
|
||||
if 'const' in rule:
|
||||
compared=True;matches &= error.instance[discriminator]==rule['const']
|
||||
elif 'enum' in rule:
|
||||
compared=True;matches &= error.instance[discriminator] in rule['enum']
|
||||
if compared and matches:matching.append(index)
|
||||
if len(matching)==1:
|
||||
from jsonschema import Draft202012Validator
|
||||
try:
|
||||
nested=next(Draft202012Validator(matching[0]).iter_errors(error.instance),None)
|
||||
except Exception:
|
||||
nested=None # Optional diagnostics must never break controlled rejection.
|
||||
if nested:
|
||||
info=schema_feedback(nested, _depth+1)
|
||||
prefix='/'.join(p if isinstance(p,str) and p in VALIDATION_FIELDS else '?' for p in error.absolute_path)
|
||||
info['validation_path']=('/'+prefix if prefix else '')+info['validation_path']
|
||||
return info
|
||||
# Reuse errors resolved against the full schema; revalidating an
|
||||
# isolated branch loses its root $defs and original field path.
|
||||
nested=next((child for child in error.context if child.schema_path
|
||||
and child.schema_path[0]==matching[0]),None)
|
||||
if nested is not None:
|
||||
return schema_feedback(nested,_depth+1,include_keyword=include_keyword)
|
||||
path = '/'.join(str(p) if type(p) is int and 0 <= p <= 1000000 else
|
||||
p if isinstance(p,str) and p in VALIDATION_FIELDS else '?'
|
||||
for p in list(error.absolute_path)[:8])
|
||||
@@ -37,7 +46,10 @@ def schema_feedback(error, _depth=0):
|
||||
fields = [f for f in error.validator_value if f not in error.instance and f in VALIDATION_FIELDS][:8]
|
||||
elif error.validator == 'additionalProperties' and isinstance(error.instance,dict):
|
||||
fields = sorted((set(error.instance)-set(error.schema.get('properties',{}))) & VALIDATION_FIELDS)[:8]
|
||||
return {'validation_path':'/'+path, 'validation_fields':fields}
|
||||
result={'validation_path':'/'+path, 'validation_fields':fields}
|
||||
if include_keyword and error.validator in KEYWORDS:
|
||||
result['validation_keyword']=error.validator
|
||||
return result
|
||||
|
||||
|
||||
def count(value):
|
||||
|
||||
@@ -157,7 +157,7 @@ class LLMGateway:
|
||||
info={'validation_keyword':invalid.validator}
|
||||
if request['prompt_version'] in {'investigation_project_v14','investigation_project_v15','investigation_project_v16','investigation_project_v17'}:
|
||||
from .diagnostics import schema_feedback
|
||||
info.update(schema_feedback(invalid))
|
||||
info.update(schema_feedback(invalid,include_keyword=True))
|
||||
error = LLMError("LLM_OUTPUT_SCHEMA",diagnostics=info)
|
||||
self._failure_metadata(error,provider,request,started,response,attempts)
|
||||
raise error from None
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
import pytest
|
||||
from jsonschema import Draft202012Validator
|
||||
|
||||
from test_compact_investigation import context
|
||||
from test_lean_investigation_decision import reply
|
||||
from test_case_builder import case_setup
|
||||
from test_investigation_tools import tool_case
|
||||
from test_project_research import source
|
||||
from viano_bug_analysis.analysis.investigation_prompt import output_schema
|
||||
from viano_bug_analysis.analysis.investigation_compact import validate_tool_shape
|
||||
from viano_bug_analysis.analysis.investigation_config import InvestigationError
|
||||
from viano_bug_analysis.llm.diagnostics import schema_feedback,sanitize
|
||||
|
||||
|
||||
@pytest.mark.parametrize('kind',['source.search','source.symbol','source.registration','source.schema'])
|
||||
def test_nested_enum_tool_feedback_names_missing_path_prefix(kind):
|
||||
c=context();c['prompt_version']='investigation_project_v17';r=reply()
|
||||
r['next']={'kind':'tool','operation':'project.query','arguments':{
|
||||
'kind':kind,'text':'PRIVATE_QUERY','case_sensitive':False,'offset':0,'limit':20}}
|
||||
with pytest.raises(InvestigationError) as caught:validate_tool_shape(c,r)
|
||||
info=caught.value.feedback
|
||||
assert info['validation_path']=='/next/arguments'
|
||||
assert info['validation_fields']==['path_prefix']
|
||||
assert 'path_prefix="" searches_repository_root' in info['requirement']
|
||||
assert 'PRIVATE_QUERY' not in json.dumps(info)
|
||||
|
||||
|
||||
def test_feedback_descends_through_nested_filters_and_root_definitions():
|
||||
c=context();c['prompt_version']='investigation_project_v17';r=reply()
|
||||
r['next']={'kind':'tool','operation':'project.query','arguments':{
|
||||
'kind':'database.rows','table':'creature','columns':['guid'],
|
||||
'filters':[{'column':'id','operator':'eq'}],'offset':0,'limit':20}}
|
||||
with pytest.raises(InvestigationError) as caught:validate_tool_shape(c,r)
|
||||
assert caught.value.feedback=={'validation_path':'/next/arguments/filters/0',
|
||||
'validation_fields':['value'],'requirement':'provide_missing_required_fields'}
|
||||
|
||||
|
||||
def test_feedback_reports_type_error_without_recommending_missing_fields():
|
||||
c=context();r=reply();r['next']={'kind':'tool','operation':'project.query','arguments':{
|
||||
'kind':'source.search','text':'query','path_prefix':'','case_sensitive':False,'offset':0,'limit':'PRIVATE'}}
|
||||
with pytest.raises(InvestigationError) as caught:validate_tool_shape(c,r)
|
||||
assert caught.value.feedback['validation_path']=='/next/arguments/limit'
|
||||
assert caught.value.feedback['requirement']=='match_expected_type_at_validation_path'
|
||||
assert 'PRIVATE' not in json.dumps(caught.value.feedback)
|
||||
|
||||
|
||||
def test_unknown_discriminator_does_not_choose_arbitrary_branch():
|
||||
c=context();r=reply();r['next']={'kind':'tool','operation':'PRIVATE_OPERATION','arguments':{}}
|
||||
error=next(Draft202012Validator(output_schema(c,strict=True)).iter_errors(r))
|
||||
info=schema_feedback(error,include_keyword=True)
|
||||
assert info['validation_path']=='/next' and info['validation_fields']==[]
|
||||
assert sanitize(info)==info and 'PRIVATE' not in json.dumps(info)
|
||||
|
||||
|
||||
def test_precise_feedback_reaches_checkpoint_then_one_corrected_source_query(tool_case,source):
|
||||
from test_investigator import LoopAdapter,CASE,state
|
||||
from test_case_interpreter import gateway
|
||||
from viano_bug_analysis.analysis import Investigator
|
||||
from viano_bug_analysis.tools import InvestigationToolRouter
|
||||
from viano_bug_analysis.tools.project_query import ProjectResearch
|
||||
from viano_bug_analysis.package import PackageReader,PackageValidator
|
||||
from viano_bug_analysis.budget import BudgetManager
|
||||
store=tool_case[0]
|
||||
with PackageReader(tool_case[1]) as reader:project=reader.manifest['project']
|
||||
service=ProjectResearch(project,source=source);calls=[];original=source.query
|
||||
def counted(args):calls.append(args);return original(args)
|
||||
source.query=counted
|
||||
def change(reply,c):
|
||||
args={'kind':'source.search','text':'activate','case_sensitive':False,'offset':0,'limit':20}
|
||||
if c.get('previous_rejection_details'):
|
||||
assert c['previous_rejection_details']['validation_fields']==['path_prefix']
|
||||
args['path_prefix']=''
|
||||
reply['next']={'kind':'tool','operation':'project.query','arguments':args}
|
||||
adapter=LoopAdapter(change);loop=Investigator(store,gateway(adapter),router=InvestigationToolRouter(project_research=service))
|
||||
first=asyncio.run(loop.step(*CASE));assert first.status=='REJECTED' and not calls
|
||||
assert not BudgetManager(store).snapshot(*CASE).state['accounts']['H1']['actions']
|
||||
with PackageReader(first.package) as reader:
|
||||
c=json.loads(reader.read_bytes(state(first)['active']['context_ref']))
|
||||
assert c['previous_rejection_details']['validation_path']=='/next/arguments'
|
||||
second=asyncio.run(loop.step(*CASE))
|
||||
assert second.status=='PROGRESSED' and len(calls)==1
|
||||
assert PackageValidator().validate(second.package).valid
|
||||
Reference in new issue
Block a user