From bfb353eca289abfea8cfbf2f4bbaa9e28bb1bdd1 Mon Sep 17 00:00:00 2001 From: Nitsu45 Date: Mon, 19 Oct 2026 08:21:40 +0200 Subject: [PATCH] Normalize investigation formats and repair bounded response fragments --- config/investigation.lean.toml | 3 + docs/response-format-recovery.md | 79 ++++++ .../analysis/investigation_config.py | 5 +- .../analysis/investigation_prompt.py | 14 ++ .../analysis/investigator.py | 29 ++- src/viano_bug_analysis/llm/gateway.py | 27 ++- src/viano_bug_analysis/llm/gemini.py | 8 +- .../llm/openai_compatible.py | 8 +- src/viano_bug_analysis/llm/provider_base.py | 1 + .../llm/response_recovery.py | 199 +++++++++++++++ src/viano_bug_analysis/llm/runner.py | 2 + .../investigation-context-v1.schema.json | 64 +++++ .../investigation-state-v1.schema.json | 15 ++ .../schemas/llm-call-v1.schema.json | 48 ++++ tests/test_response_recovery.py | 229 ++++++++++++++++++ 15 files changed, 722 insertions(+), 9 deletions(-) create mode 100644 docs/response-format-recovery.md create mode 100644 src/viano_bug_analysis/llm/response_recovery.py create mode 100644 tests/test_response_recovery.py diff --git a/config/investigation.lean.toml b/config/investigation.lean.toml index 98a55b3..949dc20 100644 --- a/config/investigation.lean.toml +++ b/config/investigation.lean.toml @@ -7,3 +7,6 @@ bootstrap_json_framing = true # Versioned native object shapes for nonempty bootstrap findings. bootstrap_framing_version = 2 + +# Deterministic normalization and bounded format-only repair. +response_recovery_version = 1 diff --git a/docs/response-format-recovery.md b/docs/response-format-recovery.md new file mode 100644 index 0000000..9882c48 --- /dev/null +++ b/docs/response-format-recovery.md @@ -0,0 +1,79 @@ +# Begrenzte Formatrettung in der Hypothesenuntersuchung + +Die versionierte Untersuchungspolicy `response_recovery_version = 1` aktiviert +zwei Mechanismen. Das schlanke Testprofil aktiviert diese Version ausdrücklich; +der Standardwert 0 und alte gespeicherte Policies behalten das bisherige Verhalten. +Andere Analysephasen werden nicht umgestellt. `docs/architecture.md` bleibt +unverändert: Es wird weiterhin ausschließlich eine vollständig validierte Antwort +atomar verarbeitet, niemals ein herausgelöster gültiger Rechercheauftrag neben +ungültigen Zustandsänderungen. + +## Deterministische Normalisierung + +Die Anfrage trägt den Schema-Marker `x-viano-response-recovery: 1`. Nur markierte +Investigation-Aufrufe erhalten die folgenden Regeln: + +- Ein einzelner äußerer Markdown-Codeblock ohne Sprachangabe oder mit `json` + wird entfernt. Zusätzliche Prosa und beliebige JSON-Syntaxfehler werden nicht + repariert. Doppelte Schlüssel bleiben ungültig. +- Ein einmal JSON-kodiertes Objekt wird streng dekodiert: als gesamte Antwort + oder an bekannten Objektstellen wie `next.arguments` und einem Belegobjekt. + Freie Texte werden nicht in fachliche Objekte umgedeutet. +- Primitive DB-Filter `["id","eq",1]` oder + `["column","id","operator","eq","value",1]` werden bei eindeutig gültigem + Spaltennamen und bekanntem Operator zu + `[{"column":"id","operator":"eq","value":1}]` normalisiert. Die Werte + bleiben unverändert; Tabellen, Spalten, Operatoren und Grenzen werden danach + weiterhin durch den vollständigen Werkzeugvertrag geprüft. + +Originaldarstellung, normalisierte Antwort und angewandte Regeln werden im +append-only LLM-Aufruf gespeichert. Rohtexte befinden sich ausschließlich im +Paket-Audit, nicht in Fehlerdiagnosen oder Logs. Erst nach vollständiger Schema-, +Beleg-, Budget- und Toolprüfung wird eine Aktion ausgeführt. Ein kurzer Hinweis +auf die Normalisierung wird im nächsten regulären Aufruf mitgegeben, ohne einen +zusätzlichen Korrekturaufruf zu erzeugen. + +## Kleiner LLM-Reparaturaufruf + +Verbleibende, ausdrücklich zugelassene Container-Typfehler können einen +`investigation_format_repair_v1`-Aufruf erzeugen. Er enthält die bereits generierte +Antwort, den Fehlerpfad, den betroffenen Schemaausschnitt und wenige Bindungsdaten +(Hypothese, vorheriges Ergebnis und sichtbare E-IDs). Er enthält keine erneute +Fallinterpretation, Codehistorie, Rohbelege oder allgemeinen Rechercheanweisungen. + +Die LLM gibt ausschließlich `replacement` für genau diesen Strukturpfad zurück. +Das Programm setzt die Antwort selbst zusammen. Alle skalaren Werte müssen samt +Typ und Häufigkeit erhalten bleiben; vorhandene benannte skalare Zuordnungen +dürfen nicht verschwinden. Bei erkennbaren Filtertripeln bleiben außerdem die +Positionen von Spalte, Operator und Wert verbindlich; ein Tausch gleichartiger +Strings zwischen Spalte und Wert wird abgelehnt. Außerhalb des Pfads bleibt die Antwort exakt gleich. +Die vollständige rekonstruierte Antwort wird anschließend erneut gegen den +ursprünglichen fachlichen Vertrag geprüft. Quelle, Hash, Patch und rekonstruierte +Antwort bleiben über Checkpoints auch nach Paketwiederaufnahme nachvollziehbar. + +Erfundene Belege, unbekannte Aktionen, Confidence-Änderungen und nicht eindeutig +strukturbezogene Fehler werden damit nicht legitimiert. Ein bloßer Prosasatz +anstelle eines Belegobjekts lässt sich beispielsweise nicht ohne erfundene +ID-/Rollen-/Referenzangaben vervollständigen und erhält keine solche Reparatur. +Ein Änderungsvorschlag mit anderen Abfragewerten wird abgelehnt. Bestehende +Invalid-Response-Limits gelten auch für Reparaturversuche. Transportfehler behalten +die vorhandenen Failover-/Retry-Regeln. Nicht unterstützte Fehler bleiben im +bisherigen kontrollierten Ablehnungspfad. + +## Grenzen und Aktivierung + +Die Formatrettung ist auf 65.536 UTF-8-Bytes Antwort-/Reparaturdaten sowie 16 +Normalisierungen begrenzt. Das sind Sicherheitsgrenzen dieses versionierten +Mechanismus, keine Grenzen des Case-Paketformats. Größere Antworten werden nicht +für einen Reparaturaufruf dupliziert. Zu große oder nicht reparierbare Antworten +werden nicht abgeschnitten oder teilweise ausgeführt. + +Neue Untersuchungen müssen die Policy initialisieren. Eine laufende Untersuchung +wird nicht still auf eine andere Policy umgestellt. Historische Kontexte ohne +Marker, Audit oder Reparaturdescriptor ergeben unveränderte Prompts. + +Lokale Tests decken Originalerhaltung, einmalige Ausführung, Hinweise ohne +Extraaufruf, Wiederaufnahme und die Ablehnung geänderter Abfragewerte sowie +unbekannter Aktionen/Belege ab. Ein gezielter Reparaturprompt bleibt im geprüften +Fixture bei 2.199 statt 10.353 Zeichen des normalen Untersuchungsaufrufs. Das ist keine +allgemeine Kosten- oder Qualitätsgarantie; bezahlte Tests erfolgen separat. diff --git a/src/viano_bug_analysis/analysis/investigation_config.py b/src/viano_bug_analysis/analysis/investigation_config.py index 5fa8e5f..8897575 100644 --- a/src/viano_bug_analysis/analysis/investigation_config.py +++ b/src/viano_bug_analysis/analysis/investigation_config.py @@ -25,9 +25,10 @@ class InvestigationConfig: lean_protocol: int = 1 bootstrap_json_framing: bool = False bootstrap_framing_version: int = 2 + response_recovery_version: int = 0 def __post_init__(self): - if any(type(value) is not int or value <= 0 for name,value in asdict(self).items() if name not in {'lean_loop','lean_checkpoint_steps','bootstrap_json_framing'}): + if any(type(value) is not int or value <= 0 for name,value in asdict(self).items() if name not in {'lean_loop','lean_checkpoint_steps','bootstrap_json_framing','response_recovery_version'}): raise InvestigationError("INVESTIGATION_CONFIG") if type(self.lean_checkpoint_steps) is not int or self.lean_checkpoint_steps<0 or self.lean_protocol not in {1,2}: @@ -36,6 +37,8 @@ class InvestigationConfig: raise InvestigationError('INVESTIGATION_CONFIG') if self.bootstrap_framing_version not in {1,2}: raise InvestigationError('INVESTIGATION_CONFIG') + if type(self.response_recovery_version) is not int or self.response_recovery_version not in {0,1}: + raise InvestigationError('INVESTIGATION_CONFIG') @classmethod def load(cls, path): diff --git a/src/viano_bug_analysis/analysis/investigation_prompt.py b/src/viano_bug_analysis/analysis/investigation_prompt.py index fa16504..6f7d948 100644 --- a/src/viano_bug_analysis/analysis/investigation_prompt.py +++ b/src/viano_bug_analysis/analysis/investigation_prompt.py @@ -83,6 +83,9 @@ def output_schema(context=None, *, strict=False): def build_prompt(context): + if context.get('response_repair'): + from viano_bug_analysis.llm.response_recovery import repair_request + return repair_request(context) request = _build_prompt(context) from .tool_guidance import instructions, HEADER guide = instructions(context) @@ -90,6 +93,17 @@ def build_prompt(context): request['messages'][0]['content'] += guide from .tool_guidance import finding_repair_instructions request['messages'][0]['content'] += finding_repair_instructions(context) + if context.get('policy',{}).get('response_recovery_version')==1: + from viano_bug_analysis.llm.response_recovery import MARKER + request['output_schema'][MARKER]=1 + body=json.loads(request['messages'][1]['content']) + if 'output_schema' in body:body['output_schema']=request['output_schema'] + request['messages'][1]['content']=json_bytes(body).decode('utf-8') + if context.get('format_notice'): + body=json.loads(request['messages'][1]['content']) + body['format_notice']={'normalized_previous_reply':context['format_notice'], + 'instruction':'Use ordinary JSON objects next time; no extra research or reply needed.'} + request['messages'][1]['content']=json_bytes(body).decode('utf-8') return request diff --git a/src/viano_bug_analysis/analysis/investigator.py b/src/viano_bug_analysis/analysis/investigator.py index 44c4d38..7570ce5 100644 --- a/src/viano_bug_analysis/analysis/investigator.py +++ b/src/viano_bug_analysis/analysis/investigator.py @@ -295,6 +295,8 @@ class Investigator: context['bootstrap_json_framing']=True if state['policy'].get('bootstrap_framing_version',1)==2: context['bootstrap_framing_version']=2 + if state.get('pending_format_notice'): + context['format_notice']=state.pop('pending_format_notice') if len(json_bytes(context))>self.limits.max_context_bytes:raise InvestigationError("INVESTIGATION_CONTEXT_LIMIT") validate(context,"investigation-context") return context @@ -438,6 +440,10 @@ class Investigator: return self._invalid(state,budget,getattr(error,"code","INVESTIGATION_OUTPUT"),outcome.result_ref, feedback=getattr(error,'feedback',None)) active["decision"]=decision;active["llm_ref"]=outcome.result_ref;active["phase"]="executing" + if call['outcome'].get('normalization'): + state['pending_format_notice']=call['outcome']['normalization']['rules'][:16] + if call['outcome'].get('repair'): + state['pending_format_notice']=['format_repair:/'+ '/'.join(map(str,context['response_repair']['path']))] if decision["next"]["kind"]=="test": now=self.clock() if now.tzinfo is None:raise InvestigationError("INVESTIGATION_CLOCK") @@ -642,6 +648,13 @@ class Investigator: 'return_one_valid_JSON_object_no_fences_or_duplicate_keys' if info.get('response_stage') == 'output_json' else 'provider_response_unreadable_repeat_same_task_without_research_side_effects') + if context['policy'].get('response_recovery_version')==1 and not context.get('response_repair'): + with PackageReader(self._latest(state['case_id'],state['package_id']),self.store.limits) as reader: + call=self._read(reader,llm_ref,'llm-call') + from viano_bug_analysis.llm.response_recovery import make_repair + prepared=make_repair(context,call,llm_ref) + if prepared: + context['response_repair'],context['repair_binding']=prepared active["context_ref"]=f"analysis/investigation/{state['analysis_revision']:04d}/contexts/{active['hypothesis_id']}-{active['iteration']:04d}-attempt-{active['attempt']:04d}.json" state,result=self._save(state,context=context) self.runner.logger.error("Investigation response rejected case_id=%s code=%s",state["case_id"],code) @@ -671,7 +684,21 @@ class Investigator: # State permits several frozen wire versions. Recheck against this # exact request before any resumed budget or tool side effect. from jsonschema import Draft202012Validator - if not Draft202012Validator(saved_request['output_schema']).is_valid(decision): + from .investigation_prompt import output_schema + contract=output_schema(context) if context.get('response_repair') else saved_request['output_schema'] + if context.get('response_repair'): + from viano_bug_analysis.llm.response_recovery import apply_repair,sha + source=self._read(reader,context['response_repair']['source_ref'],'llm-call') + original=source['outcome'].get('rejected_document',source['outcome'].get('document')) + if (sha(source)!=context['response_repair']['source_sha256'] + or original!=context['response_repair']['document'] + or source['phase']!=State.INVESTIGATING_HYPOTHESES + or any(source[k]!=state[k] for k in ('case_id','package_id','analysis_revision'))): + raise InvestigationError('INVESTIGATION_REPAIR_BINDING') + patch=call['outcome'].get('repair',{}).get('patch') + if not patch or apply_repair(saved_request,patch)!=decision: + raise InvestigationError('INVESTIGATION_REPAIR_BINDING') + if not Draft202012Validator(contract).is_valid(decision): raise InvestigationError('INVESTIGATION_PLAN_BINDING') from .investigation_compact import validate_tool_shape validate_tool_shape(context,decision) diff --git a/src/viano_bug_analysis/llm/gateway.py b/src/viano_bug_analysis/llm/gateway.py index 8cb48db..f408ddf 100644 --- a/src/viano_bug_analysis/llm/gateway.py +++ b/src/viano_bug_analysis/llm/gateway.py @@ -116,9 +116,21 @@ class LLMGateway: if (response.cached_input_tokens is None or response.cached_input_tokens <= 0 or response.input_tokens is None or response.cached_input_tokens > response.input_tokens): raise LLMError('LLM_CACHE_USAGE_UNCONFIRMED') + from .response_recovery import enabled,normalize,REPAIR_PROMPT,apply_repair + if enabled(request): + document,audit=normalize(response.document) + if response.normalization: + original=copy.deepcopy(response.normalization) + if audit:original['rules'].extend(audit['rules']) + audit=original + if audit and len(audit['rules'])>16:raise LLMError('LLM_RESPONSE_LIMIT') + response=replace(response,document=document,normalization=audit) encoded = json_bytes(response.document) if len(encoded) > self.config.max_response_bytes: raise LLMError("LLM_RESPONSE_LIMIT") + if response.normalization and len(json_bytes({'document':response.document, + 'normalization':response.normalization}))>self.config.max_response_bytes: + raise LLMError('LLM_RESPONSE_LIMIT') if not isinstance(response.document, dict): raise LLMError("LLM_OUTPUT_SCHEMA") Draft202012Validator(request["output_schema"], format_checker=FORMAT_CHECKER).validate(response.document) @@ -133,8 +145,17 @@ class LLMGateway: if any(usage[k] is not None for k in ('estimated_input_cost', 'estimated_cached_input_cost', 'estimated_output_cost', 'estimated_cache_storage_cost')): metadata['cost_currency'] = provider.cost_currency + result={"document": parse_manifest(encoded), "metadata": metadata} + if response.normalization:result['normalization']=response.normalization + if request['prompt_version']==REPAIR_PROMPT: + result['document']=apply_repair(request,response.document) + if len(json_bytes(result['document']))>self.config.max_response_bytes: + raise LLMError('LLM_RESPONSE_LIMIT') + result['repair']={'version':1,'patch':copy.deepcopy(response.document)} + if len(json_bytes({'document':result['document'],'repair':result['repair']}))>self.config.max_response_bytes: + raise LLMError('LLM_RESPONSE_LIMIT') LOGGER.info("LLM request succeeded provider=%s", provider.id) - return {"document": parse_manifest(encoded), "metadata": metadata} + return result except (ProviderUnavailable, TimeoutError) as error: if provider.cache_required and (provider.cache_required_prompt_prefix is None or request['prompt_version'].startswith(provider.cache_required_prompt_prefix)): @@ -159,6 +180,10 @@ class LLMGateway: from .diagnostics import schema_feedback info.update(schema_feedback(invalid,include_keyword=True)) error = LLMError("LLM_OUTPUT_SCHEMA",diagnostics=info) + from .response_recovery import enabled,LIMIT + if enabled(request) and isinstance(response.document,dict) and len(json_bytes(response.document))<=LIMIT: + error.rejected_document=copy.deepcopy(response.document) + if response.normalization:error.normalization=copy.deepcopy(response.normalization) self._failure_metadata(error,provider,request,started,response,attempts) raise error from None except LLMError as error: diff --git a/src/viano_bug_analysis/llm/gemini.py b/src/viano_bug_analysis/llm/gemini.py index b119ff3..dd3cf9f 100644 --- a/src/viano_bug_analysis/llm/gemini.py +++ b/src/viano_bug_analysis/llm/gemini.py @@ -140,7 +140,8 @@ class GeminiProvider: if config.response_format != 'none': generation['responseMimeType'] = 'application/json' if config.response_format == 'json_schema': - generation['responseJsonSchema'] = request['output_schema'] + generation['responseJsonSchema'] = {k:v for k,v in request['output_schema'].items() + if k!='x-viano-response-recovery'} if request['prompt_version']=='investigation_bootstrap_v21' and config.response_format=='json_object': # Small, native syntax/outer-shape guard. The complete controller # schema stays in the prefix and remains the local authority. @@ -232,9 +233,10 @@ class GeminiProvider: raise LLMError('LLM_INCOMPLETE_OUTPUT') text = ''.join(part['text'] for part in parts if 'text' in part and not part.get('thought')) diagnostics['output_characters'] = len(text) - document = parse_response_object(text.encode('utf-8'), stage='output_json') + from .response_recovery import parse_text + document,normalization = parse_text(text,request) return ProviderResponse(document, envelope.get('modelVersion', config.model), prompt, output, - sanitize(diagnostics), cached, thoughts) + sanitize(diagnostics), cached, thoughts,normalization) except LLMError as error: error.diagnostics = sanitize(diagnostics) | sanitize(error.diagnostics) raise diff --git a/src/viano_bug_analysis/llm/openai_compatible.py b/src/viano_bug_analysis/llm/openai_compatible.py index fe5640d..512d6e2 100644 --- a/src/viano_bug_analysis/llm/openai_compatible.py +++ b/src/viano_bug_analysis/llm/openai_compatible.py @@ -29,7 +29,8 @@ class OpenAICompatibleProvider: payload["reasoning_effort"] = config.reasoning_effort if config.response_format == "json_schema": payload["response_format"] = {"type": "json_schema", "json_schema": { - "name": "case_output", "schema": request["output_schema"], "strict": True}} + "name": "case_output", "schema": {k:v for k,v in request["output_schema"].items() + if k!='x-viano-response-recovery'}, "strict": True}} elif config.response_format == "json_object": payload["response_format"] = {"type": "json_object"} headers = {"Content-Type": "application/json", "Accept": "application/json"} @@ -75,7 +76,8 @@ class OpenAICompatibleProvider: content = message["content"] if not isinstance(content, str): raise LLMError("LLM_RESPONSE",diagnostics=diagnostics) - document = parse_response_object(content.encode("utf-8"), stage='output_json') + from .response_recovery import parse_text + document,normalization = parse_text(content,request) model = envelope.get("model", config.model) if not isinstance(model, str) or not model.strip() or len(model) > 256 or any(ord(c) < 32 for c in model): raise LLMError("LLM_RESPONSE",diagnostics=diagnostics) @@ -85,7 +87,7 @@ class OpenAICompatibleProvider: if count is not None and (isinstance(count, bool) or not isinstance(count, int) or count < 0): raise LLMError("LLM_RESPONSE",diagnostics=diagnostics) return ProviderResponse(document, model, input_tokens, output_tokens, diagnostics, - diagnostics.get('cached_input_tokens'), diagnostics.get('reasoning_tokens')) + diagnostics.get('cached_input_tokens'), diagnostics.get('reasoning_tokens'),normalization) except LLMError as error: error.diagnostics = diagnostics | error.diagnostics raise diff --git a/src/viano_bug_analysis/llm/provider_base.py b/src/viano_bug_analysis/llm/provider_base.py index f5e0165..d872348 100644 --- a/src/viano_bug_analysis/llm/provider_base.py +++ b/src/viano_bug_analysis/llm/provider_base.py @@ -13,6 +13,7 @@ class ProviderResponse: diagnostics: dict | None = None cached_input_tokens: int | None = None reasoning_tokens: int | None = None + normalization: dict | None = None class LLMProvider(Protocol): diff --git a/src/viano_bug_analysis/llm/response_recovery.py b/src/viano_bug_analysis/llm/response_recovery.py new file mode 100644 index 0000000..4363d04 --- /dev/null +++ b/src/viano_bug_analysis/llm/response_recovery.py @@ -0,0 +1,199 @@ +"""Bounded, opt-in format recovery. Never infer evidence, actions or missing values.""" +import copy +import hashlib +import json +import re +from collections import Counter + +from viano_bug_analysis.lifecycle.storage import json_bytes +from viano_bug_analysis.package.manifest import parse_manifest +from .errors import LLMError + +MARKER='x-viano-response-recovery' +LIMIT=65536 +REPAIR_PROMPT='investigation_format_repair_v1' +OBJECT_PATH=re.compile(r'(?:/assessment|/update|/next|/next/arguments|/update/working_brief|/update/next_step|/update/working_brief/findings/[0-9]+|/next/arguments/filters/[0-9]+)') +REPAIR_PATH=re.compile(r'(?:/update/working_brief/findings(?:/[0-9]+)?|/next/arguments/filters(?:/[0-9]+)?)') + + +def enabled(request): + return (request.get('prompt_version','').startswith('investigation_') + and type(request.get('output_schema',{}).get(MARKER)) is int + and request['output_schema'][MARKER]==1) + + +def sha(document):return hashlib.sha256(json_bytes(document)).hexdigest() + + +def parse_text(text,request): + from .diagnostics import parse_response_object + if enabled(request) and len(text.encode('utf-8'))<=LIMIT: + match=re.fullmatch(r'\s*```(?:json)?\s*\n(.*?)\n```\s*',text,re.S) + candidate=match[1] if match else text + try:outer=parse_manifest(candidate.encode('utf-8')) + except (ValueError,UnicodeError,RecursionError):outer=None + if isinstance(outer,str): + document=parse_response_object(outer.encode('utf-8'),stage='output_json') + return document,{'version':1,'original_text':text,'original_document':copy.deepcopy(document), + 'rules':(['strip_json_fence:/'] if match else [])+['decode_root_object:/']} + if match: + document=parse_response_object(match[1].encode('utf-8'),stage='output_json') + return document,{'version':1,'original_text':text,'original_document':copy.deepcopy(document), + 'rules':['strip_json_fence:/']} + return parse_response_object(text.encode('utf-8'),stage='output_json'),None + + +def normalize(document): + original=copy.deepcopy(document);result=copy.deepcopy(document);rules=[] + if not isinstance(result,dict) or len(json_bytes(result))>LIMIT:return result,None + def visit(value,path=''): + if isinstance(value,str) and OBJECT_PATH.fullmatch(path): + try:decoded=parse_manifest(value.encode('utf-8')) + except (ValueError,UnicodeError,RecursionError):return value + if isinstance(decoded,dict): + rules.append('decode_object:'+path);value=decoded + if isinstance(value,dict):return {k:visit(v,path+'/'+k) for k,v in value.items()} + if isinstance(value,list):return [visit(v,path+'/'+str(i)) for i,v in enumerate(value)] + return value + result=visit(result) + action=result.get('next',{});args=action.get('arguments',{}) if isinstance(action,dict) else {} + if (isinstance(args,dict) and action.get('kind')=='tool' and action.get('operation')=='project.query' + and args.get('kind') in {'database.rows','database.count'}): + filters=args.get('filters') + if (isinstance(filters,list) and len(filters)==6 + and all(isinstance(k,str) for k in filters[::2]) + and set(filters[::2])=={'column','operator','value'}): + paired=dict(zip(filters[::2],filters[1::2])) + if (isinstance(paired['column'],str) and re.fullmatch(r'[A-Za-z_][A-Za-z0-9_]*',paired['column']) + and isinstance(paired['operator'],str) + and paired['operator'] in {'eq','ne','lt','le','gt','ge','contains','in'}): + args['filters']=[paired] + rules.append('flat_filter_pairs:/next/arguments/filters') + if (isinstance(filters,list) and len(filters)==3 and isinstance(filters[0],str) + and re.fullmatch(r'[A-Za-z_][A-Za-z0-9_]*',filters[0]) + and isinstance(filters[1],str) and filters[1] in {'eq','ne','lt','le','gt','ge','contains','in'}): + args['filters']=[dict(zip(('column','operator','value'),filters))] + rules.append('flat_filter:/next/arguments/filters') + if len(rules)>16:return original,None + return result,({'version':1,'original_document':original,'rules':rules} if rules else None) + + +def at(document,path): + for key in path:document=document[key] + return document + + +def target_schema(schema,path): + for key in path: + schema=schema['items'] if type(key) is int else schema['properties'][key] + return copy.deepcopy(schema) + + +def eligible(context,document,details): + path=details.get('validation_path','') + if (not isinstance(document,dict) or len(json_bytes(document))>LIMIT + or details.get('validation_keyword')!='type' or not REPAIR_PATH.fullmatch(path)): + return None + return [int(k) if k.isdigit() else k for k in path.strip('/').split('/')] + + +def atoms(value): + result=Counter() + def visit(item): + if isinstance(item,dict): + for v in item.values():visit(v) + elif isinstance(item,list): + for v in item:visit(v) + else:result[json_bytes(item)]+=1 + visit(value) + return result + + +def apply_repair(request,patch): + """One replacement, same scalar values and existing named scalar bindings.""" + data=json.loads(request['messages'][1]['content'])['repair'] + original=data['document'];path=data['path'] + if not REPAIR_PATH.fullmatch('/'+'/'.join(map(str,path))):raise LLMError('LLM_REPAIR_BINDING') + if sha(original)!=data['document_sha256']:raise LLMError('LLM_REPAIR_BINDING') + before=at(original,path);after=patch['replacement'] + if atoms(before)!=atoms(after):raise LLMError('LLM_REPAIR_CONTENT_CHANGED') + def named(value): + found=[] + if isinstance(value,dict): + for key,item in value.items(): + if not isinstance(item,(dict,list)) or key in {'value','evidence_refs'}: + found.append((key,json_bytes(item))) + found.extend(named(item)) + elif isinstance(value,list): + if (len(value)==3 and isinstance(value[0],str) and isinstance(value[1],str) + and value[1] in {'eq','ne','lt','le','gt','ge','contains','in'}): + found.extend((key,json_bytes(item)) for key,item in zip(('column','operator','value'),value)) + for item in value:found.extend(named(item)) + return found + old=Counter(named(before));new=Counter(named(after)) + if any(new[pair]LIMIT:return None + return descriptor,binding diff --git a/src/viano_bug_analysis/llm/runner.py b/src/viano_bug_analysis/llm/runner.py index 422431b..7e4cf10 100644 --- a/src/viano_bug_analysis/llm/runner.py +++ b/src/viano_bug_analysis/llm/runner.py @@ -158,6 +158,8 @@ class CaseLLMRunner: outcome['diagnostics'] = error.diagnostics if error.http_status is not None: outcome["http_status"] = error.http_status + for field in ('rejected_document','normalization'): + if hasattr(error,field):outcome[field]=getattr(error,field) result_ref = base + f"/round-{rounds:04d}.json" details["result_ref"] = result_ref record = {"format": "viano-llm-call", "format_version": 1, diff --git a/src/viano_bug_analysis/schemas/investigation-context-v1.schema.json b/src/viano_bug_analysis/schemas/investigation-context-v1.schema.json index 2e105a9..30b665c 100644 --- a/src/viano_bug_analysis/schemas/investigation-context-v1.schema.json +++ b/src/viano_bug_analysis/schemas/investigation-context-v1.schema.json @@ -117,6 +117,13 @@ 1, 2 ] + }, + "response_recovery_version": { + "type": "integer", + "enum": [ + 0, + 1 + ] } } }, @@ -812,6 +819,63 @@ }, "bootstrap_framing_version": { "const": 2 + }, + "format_notice": { + "type": "array", + "maxItems": 16, + "items": { + "type": "string", + "maxLength": 160 + } + }, + "response_repair": { + "type": "object", + "additionalProperties": false, + "required": [ + "source_ref", + "source_sha256", + "document", + "document_sha256", + "path", + "schema" + ], + "properties": { + "source_ref": { + "type": "string", + "maxLength": 1024 + }, + "source_sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "document": { + "type": "object" + }, + "document_sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "path": { + "type": "array", + "minItems": 1, + "maxItems": 8, + "items": { + "type": [ + "string", + "integer" + ] + } + }, + "schema": { + "type": "object" + }, + "definitions": { + "type": "object" + } + } + }, + "repair_binding": { + "type": "object" } } } diff --git a/src/viano_bug_analysis/schemas/investigation-state-v1.schema.json b/src/viano_bug_analysis/schemas/investigation-state-v1.schema.json index e953f06..fd4759b 100644 --- a/src/viano_bug_analysis/schemas/investigation-state-v1.schema.json +++ b/src/viano_bug_analysis/schemas/investigation-state-v1.schema.json @@ -113,6 +113,13 @@ 1, 2 ] + }, + "response_recovery_version": { + "type": "integer", + "enum": [ + 0, + 1 + ] } } }, @@ -2151,6 +2158,14 @@ "lean_preferred_hypothesis": { "type": "string", "pattern": "^H[1-9][0-9]*$" + }, + "pending_format_notice": { + "type": "array", + "maxItems": 16, + "items": { + "type": "string", + "maxLength": 160 + } } } } diff --git a/src/viano_bug_analysis/schemas/llm-call-v1.schema.json b/src/viano_bug_analysis/schemas/llm-call-v1.schema.json index 071a1d7..f256e55 100644 --- a/src/viano_bug_analysis/schemas/llm-call-v1.schema.json +++ b/src/viano_bug_analysis/schemas/llm-call-v1.schema.json @@ -311,6 +311,54 @@ } }, "additionalProperties": false + }, + "rejected_document": { + "type": "object" + }, + "normalization": { + "type": "object", + "additionalProperties": false, + "required": [ + "version", + "original_document", + "rules" + ], + "properties": { + "version": { + "const": 1 + }, + "original_document": { + "type": "object" + }, + "original_text": { + "type": "string", + "maxLength": 65536 + }, + "rules": { + "type": "array", + "maxItems": 16, + "items": { + "type": "string", + "maxLength": 160 + } + } + } + }, + "repair": { + "type": "object", + "additionalProperties": false, + "required": [ + "version", + "patch" + ], + "properties": { + "version": { + "const": 1 + }, + "patch": { + "type": "object" + } + } } } } diff --git a/tests/test_response_recovery.py b/tests/test_response_recovery.py new file mode 100644 index 0000000..38579c9 --- /dev/null +++ b/tests/test_response_recovery.py @@ -0,0 +1,229 @@ +import asyncio +import copy +import json + +import pytest + +from test_case_builder import case_setup +from test_investigation_tools import tool_case +from test_case_interpreter import gateway +from test_investigator import LoopAdapter,CASE,state +from test_working_brief import service_for +from test_project_research import rows +from viano_bug_analysis.analysis import Investigator,InvestigationConfig +from viano_bug_analysis.budget import BudgetManager +from viano_bug_analysis.lifecycle import Workspace,CheckpointStore +from viano_bug_analysis.llm import ProviderResponse,LLMError +from viano_bug_analysis.llm.response_recovery import normalize,parse_text,MARKER,REPAIR_PROMPT,apply_repair,sha +from viano_bug_analysis.package import PackageReader,PackageValidator +from viano_bug_analysis.tools import InvestigationToolRouter + + +def loop(tool_case,adapter): + return Investigator(tool_case[0],gateway(adapter),InvestigationConfig(response_recovery_version=1), + router=InvestigationToolRouter(project_research=service_for(tool_case))) + + +def test_local_normalization_is_bounded_explicit_and_preserves_values(): + raw={'update':json.dumps({'confidence':'plausible'}),'next':{'kind':'tool','operation':'project.query', + 'arguments':rows(filters=['id','eq',1])}} + before=copy.deepcopy(raw);fixed,audit=normalize(raw) + assert raw==before and audit['original_document']==raw + assert fixed['next']['arguments']['filters']==[{'column':'id','operator':'eq','value':1}] + assert fixed['update']=={'confidence':'plausible'} + assert len(audit['rules'])==2 + assert normalize(fixed)==(fixed,None) + + +@pytest.mark.parametrize('filters',[[],['id','eq'],['id','unknown',1],['id','eq',1,'extra']]) +def test_ambiguous_filters_are_not_guessed(filters): + raw={'next':{'kind':'tool','operation':'project.query','arguments':rows(filters=filters)}} + assert normalize(raw)==(raw,None) + + +def test_fence_is_opt_in_and_duplicate_keys_remain_invalid(): + req={'prompt_version':'investigation_bootstrap_v22','output_schema':{MARKER:1}} + original='```json\n{"next":{}}\n```' + doc,audit=parse_text(original,req) + assert doc=={'next':{}} and audit['original_text']==original + with pytest.raises(LLMError):parse_text(original,{'output_schema':{}}) + with pytest.raises(LLMError):parse_text('```json\n{"a":1,"a":2}\n```',req) + doc,audit=parse_text(json.dumps('{"next":{}}'),req) + assert doc=={'next':{}} and audit['rules']==['decode_root_object:/'] + + +def test_flat_key_value_filter_pairs_are_unambiguous(): + raw={'next':{'kind':'tool','operation':'project.query', + 'arguments':rows(filters=['column','id','operator','eq','value',1])}} + fixed,audit=normalize(raw) + assert fixed['next']['arguments']['filters']==[{'column':'id','operator':'eq','value':1}] + assert audit['rules']==['flat_filter_pairs:/next/arguments/filters'] + + +def test_flat_filter_executes_once_with_audit_and_deferred_notice(tool_case): + def change(reply,c):reply['next']={'kind':'tool','operation':'project.query','arguments':rows(filters=['id','eq',1])} + adapter=LoopAdapter(change);worker=loop(tool_case,adapter) + result=asyncio.run(worker.step(*CASE));assert result.status=='PROGRESSED' + assert len(adapter.contexts)==1 + with PackageReader(result.package) as r: + s=state(result);call=json.loads(r.read_bytes(s['hypotheses']['H1']['notes'][-1]['llm_ref'])) + assert call['outcome']['normalization']['original_document']['next']['arguments']['filters']==['id','eq',1] + assert call['outcome']['document']['next']['arguments']['filters']==[{'column':'id','operator':'eq','value':1}] + assert len(BudgetManager(tool_case[0]).snapshot(*CASE).state['accounts']['H1']['actions'])==1 + second=asyncio.run(worker.step(*CASE));assert second.status=='PROGRESSED' + with PackageReader(second.package) as r: + requests=[json.loads(r.read_bytes(n)) for n in r.files if n.startswith('llm/') and n.endswith('/request.json')] + assert any('format_notice' in req['messages'][1]['content'] for req in requests) + assert PackageValidator().validate(second.package).valid + + +class RepairAdapter(LoopAdapter): + def __init__(self,tamper=False):super().__init__();self.requests=[];self.tamper=tamper + async def generate(self,config,request,**kwargs): + self.requests.append(copy.deepcopy(request)) + if request['prompt_version']==REPAIR_PROMPT: + replacement={'column':'id','operator':'eq','value':999 if self.tamper else 1} + return ProviderResponse({'replacement':replacement},'fixture',100,20) + response=await super().generate(config,request,**kwargs) + response.document['next']={'kind':'tool','operation':'project.query','arguments':rows(filters=[['id','eq',1]])} + return response + + +def test_small_repair_resumes_portably_and_preserves_whole_response(tool_case,tmp_path): + adapter=RepairAdapter();worker=loop(tool_case,adapter) + first=asyncio.run(worker.step(*CASE));assert first.status=='REJECTED' + assert not BudgetManager(tool_case[0]).snapshot(*CASE).state['accounts']['H1']['actions'] + with PackageReader(first.package) as r: + c=json.loads(r.read_bytes(state(first)['active']['context_ref'])) + assert c['response_repair']['path']==['next','arguments','filters',0] + portable=CheckpointStore(Workspace(tmp_path/'portable'));portable.intake(first.package) + from viano_bug_analysis.tools import InvestigationToolRouter + resumed=Investigator(portable,gateway(adapter),router=InvestigationToolRouter(project_research=service_for(tool_case))) + second=asyncio.run(resumed.step(*CASE));assert second.status=='PROGRESSED' + assert adapter.requests[-1]['prompt_version']==REPAIR_PROMPT + sizes=[sum(len(m['content']) for m in req['messages']) for req in adapter.requests] + assert sizes[-1]