From 67843454e499668646e9e45bd93ef729c42c038f Mon Sep 17 00:00:00 2001 From: Nitsu45 Date: Sun, 18 Oct 2026 23:30:33 +0200 Subject: [PATCH] Report JSON types and guide malformed investigation finding repair --- docs/bootstrap-schema-diagnostics.md | 27 +++++++++++++ .../analysis/investigation_prompt.py | 2 + .../analysis/investigator.py | 1 + .../analysis/tool_guidance.py | 14 +++++++ src/viano_bug_analysis/llm/diagnostics.py | 19 ++++++++++ .../investigation-context-v1.schema.json | 26 +++++++++++++ .../schemas/llm-call-v1.schema.json | 26 +++++++++++++ .../llm-failure-diagnostics-v1.schema.json | 38 +++++++++++++++++++ tests/test_bootstrap_schema_diagnostics.py | 37 ++++++++++++++++-- 9 files changed, 187 insertions(+), 3 deletions(-) diff --git a/docs/bootstrap-schema-diagnostics.md b/docs/bootstrap-schema-diagnostics.md index ad96eca..69035fa 100644 --- a/docs/bootstrap-schema-diagnostics.md +++ b/docs/bootstrap-schema-diagnostics.md @@ -23,3 +23,30 @@ Datenschutz und den Weg vom Gateway über das Paket bis zur Wiederaufnahme und erfolgreichen Korrektur mit einem simulierten Anbieter. Es werden keine kostenpflichtigen Anfragen ausgeführt. Historisch nicht gespeicherte Fehlerpfade lassen sich damit nicht nachträglich rekonstruieren. + +## Präzisierung nach dem Source-/Schema-Test + +Der folgende echte Test zeigte dreimal denselben Typfehler an +`/update/working_brief/findings/0`. Die zuvor erfolgreichen Bootstrap-Antworten +enthielten leere Beleglisten. Ein Erfolg dieser Antworten prüfte daher nicht die +Struktur ausgefüllter Belegobjekte. Der Transport prüft weiterhin nur die äußere +Form nativ; das vollständige lokale Schema bleibt maßgeblich. + +Typfehler enthalten jetzt zusätzlich `expected_types` und `actual_type`, ausschließlich +aus den sieben JSON-Schematypen. Boolesche Werte werden nicht als Integer gemeldet. +Es werden weder Antwortwerte noch Python-Klassennamen übertragen. Diese Angaben +bleiben im Fehlerprotokoll und im wiederaufnehmbaren Korrekturkontext erhalten. + +Bei einem fehlerhaften Belegelement mit erwartetem Typ `object` erhält nur der +Korrekturaufruf ein kurzes englisches Objektbeispiel mit den benötigten Feldern. +Die Beispielwerte sind ausdrücklich illustrativ: IDs, Text und Geltungsbereich +müssen aus den tatsächlich sichtbaren Belegen stammen. Komprimierte Datensätze, +Tupel und Zeichenketten sind keine zulässigen Belegobjekte; bestehende Erkenntnisse +dürfen nicht zum Umgehen der Prüfung gelöscht werden. Der Controller konvertiert +keine Modellantwort automatisch und startet keine zusätzliche Recherche. + +Unveränderte Erstaufrufe und historische Korrekturkontexte ohne diese neuen +Typangaben erhalten keinen zusätzlichen Prompttext. Frühere Prompts bleiben so +reproduzierbar. Die lokal simulierte Paketwiederaufnahme prüft denselben Fehlerpfad +wie der echte Lauf und anschließend einen korrigierten Aufruf. Dies ersetzt keinen +Erfolgsnachweis mit dem echten Modell. diff --git a/src/viano_bug_analysis/analysis/investigation_prompt.py b/src/viano_bug_analysis/analysis/investigation_prompt.py index 11b457f..249bd12 100644 --- a/src/viano_bug_analysis/analysis/investigation_prompt.py +++ b/src/viano_bug_analysis/analysis/investigation_prompt.py @@ -88,6 +88,8 @@ def build_prompt(context): guide = instructions(context) if guide and HEADER not in request['messages'][0]['content']: request['messages'][0]['content'] += guide + from .tool_guidance import finding_repair_instructions + request['messages'][0]['content'] += finding_repair_instructions(context) return request diff --git a/src/viano_bug_analysis/analysis/investigator.py b/src/viano_bug_analysis/analysis/investigator.py index 93a6f6c..66f835a 100644 --- a/src/viano_bug_analysis/analysis/investigator.py +++ b/src/viano_bug_analysis/analysis/investigator.py @@ -619,6 +619,7 @@ class Investigator: 'anyOf':'match_an_allowed_shape', }.get(info.get('validation_keyword'),'match_schema_at_validation_path') if context.get('bootstrap_json_framing'): + context['previous_rejection_details'].update({k:info[k] for k in ('expected_types','actual_type') if k in info}) context['previous_rejection_details']['requirement']={ 'type':'match_expected_type_at_validation_path', 'enum':'use_allowed_value_at_validation_path', diff --git a/src/viano_bug_analysis/analysis/tool_guidance.py b/src/viano_bug_analysis/analysis/tool_guidance.py index b7277d6..68c4737 100644 --- a/src/viano_bug_analysis/analysis/tool_guidance.py +++ b/src/viano_bug_analysis/analysis/tool_guidance.py @@ -8,6 +8,20 @@ FILTER_REQUIREMENT = ('filters must be a nonempty array of objects: ' 'use verified values, never a flat list.') +def finding_repair_instructions(context): + detail=context.get('previous_rejection_details',{}) + if (not context.get('bootstrap_json_framing') or detail.get('expected_types')!=['object'] + or not re.fullmatch(r'/update/working_brief/findings/[0-9]+',detail.get('validation_path',''))): + return '' + return ('\nFinding type repair: each findings element must be a JSON OBJECT, not a string, ' + 'array, tuple or encoded record. Shape example: ' + '{"id":"B0001","direction":"neutral","text":"Scoped observation",' + '"scope":"Examined snapshot","evidence_refs":["E000001"]}. ' + 'These are illustrative values: preserve existing B IDs and use only actual visible E IDs ' + 'and evidence-grounded text/scope. Do not copy invented evidence or erase findings to bypass validation. ' + 'No $record/$table encoding in output. Repair the response without repeating research.\n') + + def filter_feedback(details): result = copy.deepcopy(details) if re.fullmatch(r'/next/arguments/filters(?:/[0-9]+)?', result.get('validation_path', '')): diff --git a/src/viano_bug_analysis/llm/diagnostics.py b/src/viano_bug_analysis/llm/diagnostics.py index 26e8a61..1e78740 100644 --- a/src/viano_bug_analysis/llm/diagnostics.py +++ b/src/viano_bug_analysis/llm/diagnostics.py @@ -4,6 +4,16 @@ COUNTS = {'input_tokens','output_tokens','reasoning_tokens','cached_input_tokens 'cache_created_tokens','cache_ttl_seconds','response_bytes','output_characters','max_output_tokens', 'parse_line','parse_column'} KEYWORDS = {'type','required','additionalProperties','enum','const','pattern','minItems','maxItems','minimum','maximum','anyOf','oneOf'} +JSON_TYPES = {'object','array','string','integer','number','boolean','null'} + + +def json_type(value): + if value is None:return 'null' + if isinstance(value,bool):return 'boolean' + if isinstance(value,dict):return 'object' + if isinstance(value,list):return 'array' + if isinstance(value,str):return 'string' + return 'integer' if isinstance(value,int) else 'number' VALIDATION_FIELDS = frozenset({'retire','finding_id','intent','hypothesis_id','assessment','semantic_score','reason','update','confidence', 'working_brief','format_version','findings','retired','revision_reason','direction', 'text','scope','evidence_refs','id','uncertainties','next_question','resolve_questions', @@ -50,6 +60,11 @@ def schema_feedback(error, _depth=0, *, include_keyword=False): result={'validation_path':'/'+path, 'validation_fields':fields} if include_keyword and error.validator in KEYWORDS: result['validation_keyword']=error.validator + if error.validator=='type': + expected=error.validator_value + expected=[expected] if isinstance(expected,str) else expected + result['expected_types']=[t for t in expected if t in JSON_TYPES][:7] + result['actual_type']=json_type(error.instance) return result @@ -71,6 +86,10 @@ def sanitize(data): result[key] = data[key] if isinstance(data.get('validation_keyword'),str) and data['validation_keyword'] in KEYWORDS: result['validation_keyword'] = data['validation_keyword'] + if isinstance(data.get('actual_type'),str) and data['actual_type'] in JSON_TYPES: + result['actual_type']=data['actual_type'] + if isinstance(data.get('expected_types'),list): + result['expected_types']=[t for t in data['expected_types'] if isinstance(t,str) and t in JSON_TYPES][:7] for key, allowed in { 'json_error_kind':{'missing_delimiter','missing_key','unterminated_string','invalid_escape','extra_data','unexpected_value','other'}, 'response_stage':{'http_envelope','candidate','output_json'}, diff --git a/src/viano_bug_analysis/schemas/investigation-context-v1.schema.json b/src/viano_bug_analysis/schemas/investigation-context-v1.schema.json index f4d153a..f775013 100644 --- a/src/viano_bug_analysis/schemas/investigation-context-v1.schema.json +++ b/src/viano_bug_analysis/schemas/investigation-context-v1.schema.json @@ -497,6 +497,32 @@ }, "json_error_at_end": { "type": "boolean" + }, + "expected_types": { + "type": "array", + "maxItems": 7, + "items": { + "enum": [ + "object", + "array", + "string", + "integer", + "number", + "boolean", + "null" + ] + } + }, + "actual_type": { + "enum": [ + "object", + "array", + "string", + "integer", + "number", + "boolean", + "null" + ] } } }, diff --git a/src/viano_bug_analysis/schemas/llm-call-v1.schema.json b/src/viano_bug_analysis/schemas/llm-call-v1.schema.json index 33a18f5..071a1d7 100644 --- a/src/viano_bug_analysis/schemas/llm-call-v1.schema.json +++ b/src/viano_bug_analysis/schemas/llm-call-v1.schema.json @@ -282,6 +282,32 @@ ], "minimum": 0, "maximum": 1000000000000 + }, + "expected_types": { + "type": "array", + "maxItems": 7, + "items": { + "enum": [ + "object", + "array", + "string", + "integer", + "number", + "boolean", + "null" + ] + } + }, + "actual_type": { + "enum": [ + "object", + "array", + "string", + "integer", + "number", + "boolean", + "null" + ] } }, "additionalProperties": false diff --git a/src/viano_bug_analysis/schemas/llm-failure-diagnostics-v1.schema.json b/src/viano_bug_analysis/schemas/llm-failure-diagnostics-v1.schema.json index 817c528..bd34ad6 100644 --- a/src/viano_bug_analysis/schemas/llm-failure-diagnostics-v1.schema.json +++ b/src/viano_bug_analysis/schemas/llm-failure-diagnostics-v1.schema.json @@ -117,6 +117,44 @@ ], "minimum": 0, "maximum": 1000000000000 + }, + "expected_types": { + "type": "array", + "maxItems": 7, + "items": { + "enum": [ + "object", + "array", + "string", + "integer", + "number", + "boolean", + "null" + ] + } + }, + "actual_type": { + "enum": [ + "object", + "array", + "string", + "integer", + "number", + "boolean", + "null" + ] + }, + "validation_path": { + "type": "string", + "maxLength": 240 + }, + "validation_fields": { + "type": "array", + "maxItems": 8, + "items": { + "type": "string", + "maxLength": 64 + } } }, "additionalProperties": false diff --git a/tests/test_bootstrap_schema_diagnostics.py b/tests/test_bootstrap_schema_diagnostics.py index ba3711e..4d148da 100644 --- a/tests/test_bootstrap_schema_diagnostics.py +++ b/tests/test_bootstrap_schema_diagnostics.py @@ -37,6 +37,34 @@ def test_bootstrap_gateway_reports_safe_nested_schema_location(fault,path,keywor assert caught.value.metadata['input_tokens']==100 +@pytest.mark.parametrize('value,actual', [('PRIVATE_VALUE','string'),(['PRIVATE_VALUE'],'array'),(None,'null'),(True,'boolean'),(42,'integer'),(1.5,'number')]) +def test_finding_item_types_are_preserved_without_response_values(value,actual): + from jsonschema import Draft202012Validator + from viano_bug_analysis.llm.diagnostics import schema_feedback,sanitize + from viano_bug_analysis.case.contracts import validate + request=bootstrap();document=reply();document['update']['next_step']=None + document['update']['working_brief']['findings']=[value] + error=next(Draft202012Validator(request['output_schema']).iter_errors(document)) + details=sanitize(schema_feedback(error,include_keyword=True)) + assert details['validation_path']=='/update/working_brief/findings/0' + assert details['expected_types']==['object'] and details['actual_type']==actual + assert 'PRIVATE_VALUE' not in json.dumps(details) + validate(details,'llm-failure-diagnostics') + + +def test_type_sanitizer_rejects_arbitrary_values_and_distinguishes_bool(): + from viano_bug_analysis.llm.diagnostics import sanitize + assert sanitize({'actual_type':'PRIVATE','expected_types':['object','PRIVATE',{},True]})=={'expected_types':['object']} + + +def test_old_contexts_and_other_error_paths_get_no_new_prompt_text(): + from viano_bug_analysis.analysis.tool_guidance import finding_repair_instructions + assert finding_repair_instructions({'bootstrap_json_framing':True,'previous_rejection_details':{ + 'validation_path':'/update/working_brief/findings/0'}})=='' + assert finding_repair_instructions({'bootstrap_json_framing':True,'previous_rejection_details':{ + 'validation_path':'/next/arguments','expected_types':['object']}})=='' + + def test_bootstrap_schema_feedback_survives_package_resume_and_repairs_once(tool_case,tmp_path): from test_investigation_hybrid import HybridAdapter from test_working_brief import service_for @@ -53,7 +81,7 @@ def test_bootstrap_schema_feedback_survives_package_resume_and_repairs_once(tool assert request['prompt_version']=='investigation_bootstrap_v21' response=await super().generate(config,dict(request,prompt_version='investigation_project_v18'),**kwargs) if len(self.sent)==1: - response.document['update']['working_brief']['findings']='PRIVATE_RESPONSE_VALUE' + response.document['update']['working_brief']['findings']=['PRIVATE_RESPONSE_VALUE'] return response adapter=Adapter();router=InvestigationToolRouter(project_research=service_for(tool_case)) loop=Investigator(tool_case[0],gateway(adapter),InvestigationConfig( @@ -65,7 +93,8 @@ def test_bootstrap_schema_feedback_survives_package_resume_and_repairs_once(tool context=json.loads(reader.read_bytes(state(first)['active']['context_ref'])) call=json.loads(reader.read_bytes(state(first)['hypotheses']['H1']['notes'][-1]['llm_ref'])) feedback=context['previous_rejection_details'] - assert feedback=={'validation_path':'/update/working_brief/findings','validation_fields':[], + assert feedback=={'validation_path':'/update/working_brief/findings/0','validation_fields':[], + 'expected_types':['object'],'actual_type':'string', 'requirement':'match_expected_type_at_validation_path'} assert call['outcome']['diagnostics']['validation_path']==feedback['validation_path'] assert 'PRIVATE_RESPONSE_VALUE' not in json.dumps(call) @@ -73,5 +102,7 @@ def test_bootstrap_schema_feedback_survives_package_resume_and_repairs_once(tool result=asyncio.run(Investigator(portable,gateway(adapter),router=router).step(*CASE)) assert result.status=='PROGRESSED' and len(adapter.sent)==2 assert 'match_expected_type_at_validation_path' in json.dumps(adapter.sent[1]) - assert '/update/working_brief/findings' in json.dumps(adapter.sent[1]) + assert '/update/working_brief/findings/0' in json.dumps(adapter.sent[1]) + assert 'Finding type repair' in adapter.sent[1]['messages'][0]['content'] + assert 'No $record/$table encoding in output' in adapter.sent[1]['messages'][0]['content'] assert PackageValidator().validate(result.package).valid