diff --git a/docs/tool-guidance-repair-v1.md b/docs/tool-guidance-repair-v1.md new file mode 100644 index 0000000..9d87e56 --- /dev/null +++ b/docs/tool-guidance-repair-v1.md @@ -0,0 +1,55 @@ +# Explizite Werkzeugverträge und Filterkorrektur + +Der Live-Test `bug20-analysis-tools-20261018` wurde in der ersten H2-Runde +blockiert. Nach einem leeren Filterarray lieferte das Modell zweimal eine flache +Liste statt eines Filterobjekts. Die Anfragen wurden nicht ausgeführt. Die neuen +größeren Werkzeuge waren im Katalog enthalten, ihre konkreten Aufrufformen aber +nicht im festen Instruktionstext der Anfangsrunden erklärt. + +## Korrekturhinweise + +Neue technische Fehler unter `/next/arguments/filters` erhalten einen konstanten, +wertfreien Hinweis mit der korrekten Form: + +```json +[{"column":"observed_column","operator":"eq","value":123}] +``` + +Spalte und Wert sind illustrative Platzhalter und müssen aus belegten Daten +gewählt werden. Der Hinweis verlangt ausdrücklich eine nichtleere Liste von +Objekten statt einer flachen Liste. Er enthält keine Daten aus der abgelehnten +Antwort und bleibt innerhalb der bestehenden 160-Zeichen-Grenze. Das gilt für +volle Anfangsrunden, die schlanke Schleife und deren bisherigen v19-Pfad. + +Es gibt keine automatische Reparatur oder Ausführung einer ungültigen Anfrage. +Die nächste Modellantwort wird erneut vollständig validiert. Bestehende +gespeicherte Korrekturhinweise werden beim Promptaufbau nicht rückwirkend ersetzt. + +## Werkzeuganweisung + +Neue Kontexte mit mindestens einem der größeren Werkzeuge erhalten den Marker +`tool_guidance_version = 1`. Nur diese Kontexte ergänzen die Systeminstruktionen +um die konkreten Formen von `database.relations` und/oder `source.context`, +entsprechend dem jeweils verfügbaren Katalog. Dazu kommen die nichtleere +Filterstruktur und der Hinweis, Ausgabe-JSON nicht wie komprimierte Eingabedaten +zu kodieren. Passende größere Werkzeuge werden empfohlen, Grundwerkzeuge bleiben +verfügbar. + +Die Anweisung steht in vollständigen Anfangsrunden und schlanken Rechercheschritten +einmal im Systemtext. In Konsolidierung und vorbereitender Abschlussbewertung +wird sie nicht angefügt. Sie kann mit den bestehenden stabilen Instruktionen +gecacht werden; zusätzliche LLM-Aufrufe entstehen nicht. + +Kontexte ohne Marker behalten ihre bisherigen Nachrichten und Schemas. Die vier +gespeicherten Untersuchungsaufrufe des blockierten Tests wurden offline exakt +rekonstruiert. Ein erneuter Aufruf desselben alten, blockierten Arbeitsstands +bekommt dadurch nicht automatisch einen neuen Promptvertrag. Für die Erprobung +der ergänzten Werkzeuganweisung ist ein frischer Kontext erforderlich. + +## Validierung + +Lokale Tests reproduzieren beide fehlerhaften Filterformen in vollen und schlanken +Antworten, prüfen die Annahme korrekter Filterobjekte, die persistierte Korrektur +und eine danach erfolgreiche Werkzeugaktion. Weitere Prüfungen betreffen die +Anweisung in beiden Promptarten, historische Verträge, beide größeren Werkzeuge, +Paket-Wiederaufnahme und Caching. Es wurden keine bezahlten API-Aufrufe ausgeführt. diff --git a/src/viano_bug_analysis/analysis/investigation_compact.py b/src/viano_bug_analysis/analysis/investigation_compact.py index 17fe1e6..9da84e9 100644 --- a/src/viano_bug_analysis/analysis/investigation_compact.py +++ b/src/viano_bug_analysis/analysis/investigation_compact.py @@ -243,7 +243,8 @@ def validate_tool_shape(context,decision): from viano_bug_analysis.llm.diagnostics import schema_feedback from .investigation_config import InvestigationError from .investigation_lean import repair_feedback - raise InvestigationError('INVESTIGATION_OUTPUT',feedback=repair_feedback(schema_feedback(error,include_keyword=True))) + from .tool_guidance import filter_feedback + raise InvestigationError('INVESTIGATION_OUTPUT',feedback=filter_feedback(repair_feedback(schema_feedback(error,include_keyword=True)))) from .investigation_lean import checkpoint,build if checkpoint(context): import json @@ -279,7 +280,8 @@ def validate_tool_shape(context,decision): 'match_full_controller_contract_at_validation_path') if keyword=='required' and 'path_prefix' in detail['validation_fields']: requirement='provide_missing_required_fields; path_prefix="" searches_repository_root' - raise InvestigationError('INVESTIGATION_OUTPUT',feedback={**detail,'requirement':requirement}) + from .tool_guidance import filter_feedback + raise InvestigationError('INVESTIGATION_OUTPUT',feedback=filter_feedback({**detail,'requirement':requirement})) view=review_view(context) if preparing_review(context) else model_view(context) readable=set(view['evidence_policy']['full_visible_ids']) | set(view.get('known_schemas', {})) for memory in [view['working_brief'],*view['other_working_briefs'].values()]: diff --git a/src/viano_bug_analysis/analysis/investigation_lean_v20.py b/src/viano_bug_analysis/analysis/investigation_lean_v20.py index 4294eeb..24992ff 100644 --- a/src/viano_bug_analysis/analysis/investigation_lean_v20.py +++ b/src/viano_bug_analysis/analysis/investigation_lean_v20.py @@ -125,7 +125,8 @@ def validate_reply(context,reply): from viano_bug_analysis.llm.diagnostics import schema_feedback error=next(Draft202012Validator(schema(context,strict=True)).iter_errors(reply),None) if error: - raise InvestigationError('INVESTIGATION_OUTPUT',feedback=previous.repair_feedback(schema_feedback(error,include_keyword=True))) + from .tool_guidance import filter_feedback + raise InvestigationError('INVESTIGATION_OUTPUT',feedback=filter_feedback(previous.repair_feedback(schema_feedback(error,include_keyword=True)))) if previous.checkpoint(context): from .investigation_compact import validate_tool_shape validate_tool_shape(legacy(context),wire(context,reply)) diff --git a/src/viano_bug_analysis/analysis/investigation_prompt.py b/src/viano_bug_analysis/analysis/investigation_prompt.py index 062d2c3..af5c560 100644 --- a/src/viano_bug_analysis/analysis/investigation_prompt.py +++ b/src/viano_bug_analysis/analysis/investigation_prompt.py @@ -82,6 +82,15 @@ def output_schema(context=None, *, strict=False): def build_prompt(context): + request = _build_prompt(context) + from .tool_guidance import instructions, HEADER + guide = instructions(context) + if guide and HEADER not in request['messages'][0]['content']: + request['messages'][0]['content'] += guide + return request + + +def _build_prompt(context): if context.get('lean_protocol')==2: from .investigation_lean_v20 import build return build(context) diff --git a/src/viano_bug_analysis/analysis/investigator.py b/src/viano_bug_analysis/analysis/investigator.py index babbb3f..82f6a86 100644 --- a/src/viano_bug_analysis/analysis/investigator.py +++ b/src/viano_bug_analysis/analysis/investigator.py @@ -261,6 +261,8 @@ class Investigator: context['tool_schema']['x-project-bindings']['source_analysis'] = service.source_context.catalog() items = context['tool_schema']['properties']['requests']['items'] context['tool_schema']['properties']['requests']['items'] = extend_tool_schema(items) + if service.relations or service.source_context: + context['tool_guidance_version'] = 1 else: variants=context["tool_schema"]["properties"]["requests"]["items"]["oneOf"] context["tool_schema"]["properties"]["requests"]["items"]["oneOf"]=[ diff --git a/src/viano_bug_analysis/analysis/tool_guidance.py b/src/viano_bug_analysis/analysis/tool_guidance.py new file mode 100644 index 0000000..cbb52e2 --- /dev/null +++ b/src/viano_bug_analysis/analysis/tool_guidance.py @@ -0,0 +1,42 @@ +"""Versioned tool shapes and value-free repair instructions, no query rewriting.""" +import copy +import re + +HEADER = '\nTool argument contracts v1 (plain JSON output):\n' +FILTER_REQUIREMENT = ('filters must be a nonempty array of objects: ' + '[{"column":"observed_column","operator":"eq","value":123}]; ' + 'use verified values, never a flat list.') + + +def filter_feedback(details): + result = copy.deepcopy(details) + if re.fullmatch(r'/next/arguments/filters(?:/[0-9]+)?', result.get('validation_path', '')): + result['requirement'] = FILTER_REQUIREMENT + return result + + +def instructions(context): + if context.get('tool_guidance_version') != 1: + return '' + from .investigation_compact import preparing_review + if preparing_review(context) or context.get('lean_session', {}).get('checkpoint'): + return '' + bindings = context['tool_schema'].get('x-project-bindings', {}) + text = (HEADER + 'Use operation="project.query" for these arguments. Prefer a matching larger tool ' + 'before separate schema/search/read calls; primitive tools remain available.\n') + if bindings.get('relation_profiles'): + text += ('database.relations arguments: {"kind":"database.relations",' + '"profile":"NAME_FROM_relation_profiles","value":123}. ' + 'Select the profile from the catalog and replace 123 with the observed start key ' + '(integer for numeric IDs, string for textual keys). No table or SQL argument.\n') + if bindings.get('source_analysis'): + text += ('source.context arguments: {"kind":"source.context",' + '"text":"SYMBOL_OR_IDENTIFIER_FROM_EVIDENCE","path_prefix":""}. ' + 'Replace text with the actual term; empty path_prefix means repository root. ' + 'No registration, offset, limit or command argument.\n') + text += ('For primitive database.rows/database.count, filters MUST be a nonempty array of objects, ' + 'for example [{"column":"observed_column","operator":"eq","value":123}]. ' + 'Replace the illustrative column/value with schema-verified, case-grounded inputs. ' + 'Never emit [] or flatten an object into ["column",...,"operator",...,"value",...]. ' + 'Input compression encodings do not apply to output: return ordinary JSON objects.\n') + return text diff --git a/src/viano_bug_analysis/schemas/investigation-context-v1.schema.json b/src/viano_bug_analysis/schemas/investigation-context-v1.schema.json index d5b46b5..f4d153a 100644 --- a/src/viano_bug_analysis/schemas/investigation-context-v1.schema.json +++ b/src/viano_bug_analysis/schemas/investigation-context-v1.schema.json @@ -773,6 +773,9 @@ }, "bootstrap_json_framing": { "const": true + }, + "tool_guidance_version": { + "const": 1 } } } diff --git a/tests/test_tool_guidance.py b/tests/test_tool_guidance.py new file mode 100644 index 0000000..b3322bc --- /dev/null +++ b/tests/test_tool_guidance.py @@ -0,0 +1,97 @@ +import copy +import json +import pytest + +from test_investigation_lean import lean_context +from test_lean_investigation_decision import reply +from test_case_builder import case_setup +from test_investigation_tools import tool_case +from viano_bug_analysis.analysis import InvestigationError +from viano_bug_analysis.analysis.investigation_prompt import build_prompt +from viano_bug_analysis.analysis.investigation_compact import validate_tool_shape +from viano_bug_analysis.analysis.tool_guidance import HEADER,FILTER_REQUIREMENT,filter_feedback + + +def context(lean=False): + c=lean_context() + if lean:c['lean_protocol']=2 + else:c.pop('lean_session') + return c + + +@pytest.mark.parametrize('lean',[False,True]) +@pytest.mark.parametrize('filters',[[],['column','id','operator','eq','value',23487]]) +def test_recorded_filter_failures_get_exact_value_free_repair_and_valid_shape_passes(lean,filters): + c=context(lean) + action={'kind':'tool','operation':'project.query','arguments':{ + 'kind':'database.rows','table':'objects','columns':['id'], + 'filters':filters,'offset':0,'limit':5}} + if lean:r={'assessment':None,'observation':'Scoped observation.','next':action} + else: + r=reply();r['next']=action;r['update']['next_step']=None + r['update']['working_brief']['findings']=[] + before=copy.deepcopy(r) + with pytest.raises(InvestigationError) as error:validate_tool_shape(c,r) + assert r==before + feedback=error.value.feedback + assert feedback['requirement']==FILTER_REQUIREMENT and len(FILTER_REQUIREMENT)<=160 + assert '23487' not in json.dumps(feedback) + fixed=copy.deepcopy(r);fixed['next']['arguments']['filters']=[{'column':'id','operator':'eq','value':23487}] + validate_tool_shape(c,fixed) + + +@pytest.mark.parametrize('lean',[False,True]) +def test_explicit_argument_shapes_once_in_full_and_slim_requests(lean): + c=context(lean);c['tool_guidance_version']=1 + b=c['tool_schema']['x-project-bindings'];b['relation_profiles']={'inspect':{}};b['source_analysis']={'version':1} + request=build_prompt(c);system=request['messages'][0]['content'] + assert system.count(HEADER)==1 + assert '"kind":"database.relations"' in system and '"profile"' in system + assert '"kind":"source.context"' in system and '"path_prefix":""' in system + assert 'nonempty array of objects' in system and 'No table or SQL argument' in system + # Already stored contexts without the new marker keep their old messages. + c.pop('tool_guidance_version') + assert HEADER not in build_prompt(c)['messages'][0]['content'] + + +def test_no_new_research_instructions_during_consolidation(): + c=context(True);c['tool_guidance_version']=1;c['lean_session']['checkpoint']=True + c['tool_schema']['x-project-bindings']['relation_profiles']={'inspect':{}} + assert HEADER not in build_prompt(c)['messages'][0]['content'] + + +def test_saved_generic_feedback_is_not_retroactively_rewritten(): + from viano_bug_analysis.analysis.investigation_lean import repair_feedback + old={'validation_path':'/next/arguments/filters/0','validation_fields':[], + 'requirement':'match_expected_type_at_validation_path'} + assert repair_feedback(old)==old + assert filter_feedback(old)['requirement']==FILTER_REQUIREMENT + nested={'validation_path':'/next/arguments/filters/0/operator','requirement':'use_allowed_value_at_validation_path'} + assert filter_feedback(nested)==nested + + +def test_feedback_persists_then_fixture_model_corrects_without_extra_tool_actions(tool_case): + from test_working_brief import service_for + from test_investigator import LoopAdapter,run,state + from test_case_interpreter import gateway + from test_project_research import rows + from viano_bug_analysis.analysis import Investigator + from viano_bug_analysis.tools import InvestigationToolRouter + from viano_bug_analysis.package import PackageReader + attempts=[] + def change(r,c): + if c.get('previous_rejection_details'): + assert c['previous_rejection_details']['requirement']==FILTER_REQUIREMENT + q=rows() + if not attempts:q['filters']=[] + attempts.append(q) + r['next']={'kind':'tool','operation':'project.query','arguments':q} + adapter=LoopAdapter(change) + loop=Investigator(tool_case[0],gateway(adapter),router=InvestigationToolRouter(project_research=service_for(tool_case))) + first=run(loop,steps=1) + assert state(first)['hypotheses']['H1']['invalid_responses']==1 + second=run(loop,steps=1) + assert len(attempts)==2 and state(second)['active'] is None + assert state(second)['hypotheses']['H1']['invalid_responses']==0 + with PackageReader(second.package) as reader: + assert len([p for p in reader.files if p.startswith('analysis/evidence/captures/')])==1