From 2fe1e478a545bf9bb4de69d22a6370dcc857c465 Mon Sep 17 00:00:00 2001 From: Nitsu45 Date: Mon, 19 Oct 2026 10:30:59 +0200 Subject: [PATCH] Reject impossible format repairs and retain rejected proposals --- docs/response-format-recovery.md | 38 +++++ .../analysis/investigator.py | 10 +- src/viano_bug_analysis/llm/diagnostics.py | 2 +- src/viano_bug_analysis/llm/gateway.py | 20 ++- .../llm/response_recovery.py | 120 ++++++++++++- src/viano_bug_analysis/llm/runner.py | 2 +- .../schemas/llm-call-v1.schema.json | 56 ++++++ .../llm-failure-diagnostics-v1.schema.json | 14 ++ tests/test_repair_admission.py | 160 ++++++++++++++++++ 9 files changed, 416 insertions(+), 6 deletions(-) create mode 100644 tests/test_repair_admission.py diff --git a/docs/response-format-recovery.md b/docs/response-format-recovery.md index 9882c48..ab6994f 100644 --- a/docs/response-format-recovery.md +++ b/docs/response-format-recovery.md @@ -77,3 +77,41 @@ Extraaufruf, Wiederaufnahme und die Ablehnung geänderter Abfragewerte sowie unbekannter Aktionen/Belege ab. Ein gezielter Reparaturprompt bleibt im geprüften Fixture bei 2.199 statt 10.353 Zeichen des normalen Untersuchungsaufrufs. Das ist keine allgemeine Kosten- oder Qualitätsgarantie; bezahlte Tests erfolgen separat. + +## Zulassung und Ablehnungs-Audit nach dem ersten echten Lauf + +Der erste echte Lauf normalisierte einen `name/contains/Wolpertinger`-Filter +erfolgreich. Ein weiterer Filter enthielt dagegen +`["LogTitle","like","%Wolpertinger%"]`. Das ist nicht rein strukturell: `like` +gehört nicht zum Werkzeugvertrag. Die damalige Reparatur zielte zusätzlich +fälschlich auf den einzelnen String `filters[0]`. Ein vollständiges Objekt mit +Spalte, Operator und Wert kann daraus nicht werterhaltend erzeugt werden. + +Die Zulassung berücksichtigt jetzt die vollständige Filtereinheit und löst den +zugehörigen Schemaausschnitt über die passende Aktions-/Query-Alternative auf. +Ein Fehlerpfad innerhalb einer flachen Liste wird, soweit überhaupt reparierbar, +auf deren vollständigen Filterbereich angehoben. Skalare Ziele, unvollständige +Filter, fehlende Werte und unbekannte Operatoren erhalten keinen Format-Reparaturaufruf. +Es wird weder SQL `LIKE` in `contains` übersetzt noch werden Prozentzeichen entfernt. +Stattdessen bekommt der normale Ablehnungspfad einen kurzen, ausdrücklichen +Operator-Vertragshinweis. Eine neue fachliche Entscheidung bleibt Sache der LLM. + +Vor der Zulassung wird mit den bereits vorhandenen Filterwerten eine rein +strukturelle Prüffassung gebildet. Sie muss die gesamte ursprüngliche Ausgabestruktur +erfüllen und die Erhaltungsregeln einhalten. Weitere unveränderte Strukturfehler +verhindern so ebenfalls einen aussichtslosen Reparaturaufruf. Diese Prüffassung +wird nicht ausgeführt; eine tatsächliche Modellreparatur muss später nochmals +alle Kontrollen einschließlich fachlicher Beleg- und Budgetprüfung bestehen. +Alte gespeicherte Reparaturpläne werden vor einem erneuten Anbieteraufruf ebenfalls +auf ungeeignete Zieltypen und Operatoren geprüft, ohne ihre historischen Prompts +umzuschreiben. + +Abgelehnte Reparaturantworten stehen separat in `outcome.rejected_repair`, mit +Version, Ablehnungsgrund und SHA-256 über die kanonische JSON-Darstellung des +gespeicherten Inhalts. Ein geparster Vorschlag wird als `patch`, kaputtes JSON +als `original_text` gespeichert. Die Grenze liegt bei höchstens 65.536 UTF-8-Bytes +und zusätzlich innerhalb des konfigurierten Response-Limits. Größere Vorschläge +werden nicht gekürzt oder teilweise übernommen. HTTP-Fehlerkörper und Header +werden dadurch nicht aufbewahrt. Logs und bereinigte Diagnosen enthalten weiterhin +keinen Antworttext. Das Aufrufprotokoll erlaubt die zusätzlichen sicheren +JSON-Parserdiagnosen, damit deren Speicherung selbst nicht mit `INPUT_SCHEMA` scheitert. diff --git a/src/viano_bug_analysis/analysis/investigator.py b/src/viano_bug_analysis/analysis/investigator.py index 7570ce5..c6398b1 100644 --- a/src/viano_bug_analysis/analysis/investigator.py +++ b/src/viano_bug_analysis/analysis/investigator.py @@ -410,6 +410,11 @@ class Investigator: if budget.snapshot(case_id,package_id).state!=context["budget"]: raise InvestigationError("INVESTIGATION_CONTEXT_CHANGED") operation=f"investigation:v1:{active['hypothesis_id']}:{active['iteration']}:{active['attempt']}:{active['retry_token']}" + if context.get('response_repair'): + from viano_bug_analysis.llm.response_recovery import admission_reason + plan=context['response_repair'] + if admission_reason(plan['document'],plan['path'],plan['schema']): + return self._research_blocked(state,'LLM_REPAIR_INELIGIBLE','INVALID_LLM_RESPONSES') try: request=build_prompt(context) if context.get('prompt_version') in {'investigation_project_v15','investigation_project_v16','investigation_project_v17','investigation_project_v18'}: @@ -651,7 +656,10 @@ class Investigator: if context['policy'].get('response_recovery_version')==1 and not context.get('response_repair'): with PackageReader(self._latest(state['case_id'],state['package_id']),self.store.limits) as reader: call=self._read(reader,llm_ref,'llm-call') - from viano_bug_analysis.llm.response_recovery import make_repair + from viano_bug_analysis.llm.response_recovery import make_repair,filter_contract_feedback + document=call['outcome'].get('rejected_document',call['outcome'].get('document')) + contract_feedback=filter_contract_feedback(document) + if contract_feedback:context['previous_rejection_details']=contract_feedback prepared=make_repair(context,call,llm_ref) if prepared: context['response_repair'],context['repair_binding']=prepared diff --git a/src/viano_bug_analysis/llm/diagnostics.py b/src/viano_bug_analysis/llm/diagnostics.py index 1e78740..d4c2fbe 100644 --- a/src/viano_bug_analysis/llm/diagnostics.py +++ b/src/viano_bug_analysis/llm/diagnostics.py @@ -21,7 +21,7 @@ VALIDATION_FIELDS = frozenset({'retire','finding_id','intent','hypothesis_id','a 'prerequisites','requested_observations','environment','countercheck_refs','path', 'start_line','end_line','limit','offset','table','columns','filters','column','operator', 'value','text','case_sensitive','path_prefix','source_ref','source_refs','note','causal_review', - 'next_step','goal','supports_if','weakens_if','investigate_if','profile','entity_refs'}) + 'next_step','goal','supports_if','weakens_if','investigate_if','profile','entity_refs','replacement'}) def schema_feedback(error, _depth=0, *, include_keyword=False): diff --git a/src/viano_bug_analysis/llm/gateway.py b/src/viano_bug_analysis/llm/gateway.py index f408ddf..1a7142c 100644 --- a/src/viano_bug_analysis/llm/gateway.py +++ b/src/viano_bug_analysis/llm/gateway.py @@ -17,6 +17,7 @@ from .provider_base import ProviderResponse from .diagnostics import sanitize, count from .gemini import GeminiProvider from .usage import usage_metadata +from .response_recovery import REPAIR_PROMPT LOGGER = logging.getLogger("viano_bug_analysis.llm") PRIVACY = {"local_only": {"local"}, "self_hosted": {"local", "self_hosted"}, "cloud_allowed": {"local", "self_hosted", "cloud"}} @@ -116,7 +117,7 @@ class LLMGateway: if (response.cached_input_tokens is None or response.cached_input_tokens <= 0 or response.input_tokens is None or response.cached_input_tokens > response.input_tokens): raise LLMError('LLM_CACHE_USAGE_UNCONFIRMED') - from .response_recovery import enabled,normalize,REPAIR_PROMPT,apply_repair + from .response_recovery import enabled,normalize,apply_repair if enabled(request): document,audit=normalize(response.document) if response.normalization: @@ -176,7 +177,7 @@ class LLMGateway: LOGGER.warning("LLM endpoint unavailable provider=%s code=%s", provider.id, code) except ValidationError as invalid: info={'validation_keyword':invalid.validator} - if request['prompt_version'] in {'investigation_bootstrap_v22','investigation_bootstrap_v21','investigation_project_v14','investigation_project_v15','investigation_project_v16','investigation_project_v17','investigation_project_v18','investigation_project_v19','investigation_project_v20'}: + if request['prompt_version'] in {REPAIR_PROMPT,'investigation_bootstrap_v22','investigation_bootstrap_v21','investigation_project_v14','investigation_project_v15','investigation_project_v16','investigation_project_v17','investigation_project_v18','investigation_project_v19','investigation_project_v20'}: from .diagnostics import schema_feedback info.update(schema_feedback(invalid,include_keyword=True)) error = LLMError("LLM_OUTPUT_SCHEMA",diagnostics=info) @@ -184,9 +185,11 @@ class LLMGateway: if enabled(request) and isinstance(response.document,dict) and len(json_bytes(response.document))<=LIMIT: error.rejected_document=copy.deepcopy(response.document) if response.normalization:error.normalization=copy.deepcopy(response.normalization) + self._retain_rejected_repair(error,request,response) self._failure_metadata(error,provider,request,started,response,attempts) raise error from None except LLMError as error: + self._retain_rejected_repair(error,request,response) self._failure_metadata(error,provider,request,started,response,attempts) raise except Exception: @@ -195,6 +198,19 @@ class LLMGateway: raise error from None raise GatewayUnavailable(attempts) + def _retain_rejected_repair(self,error,request,response): + from .response_recovery import REPAIR_PROMPT,LIMIT,sha + if request['prompt_version']!=REPAIR_PROMPT:return + if (request['prompt_version']==REPAIR_PROMPT and isinstance(response,ProviderResponse) + and isinstance(response.document,dict) + and len(json_bytes(response.document))<=min(LIMIT,self.config.max_response_bytes)): + error.rejected_repair={'version':1,'reason':error.code, + 'patch':copy.deepcopy(response.document),'sha256':sha(response.document)} + elif isinstance(getattr(error,'repair_response_text',None),str): + text=error.repair_response_text + if len(text.encode('utf-8'))<=min(LIMIT,self.config.max_response_bytes): + error.rejected_repair={'version':1,'reason':error.code,'original_text':text,'sha256':sha(text)} + @staticmethod def _failure_metadata(error,provider,request,started,response,attempts): details = sanitize(error.diagnostics) diff --git a/src/viano_bug_analysis/llm/response_recovery.py b/src/viano_bug_analysis/llm/response_recovery.py index 4363d04..85d3992 100644 --- a/src/viano_bug_analysis/llm/response_recovery.py +++ b/src/viano_bug_analysis/llm/response_recovery.py @@ -12,6 +12,8 @@ from .errors import LLMError MARKER='x-viano-response-recovery' LIMIT=65536 REPAIR_PROMPT='investigation_format_repair_v1' +OPERATORS=frozenset({'eq','ne','lt','le','gt','ge','contains','in'}) +FILTER_PATH=['next','arguments','filters'] OBJECT_PATH=re.compile(r'(?:/assessment|/update|/next|/next/arguments|/update/working_brief|/update/next_step|/update/working_brief/findings/[0-9]+|/next/arguments/filters/[0-9]+)') REPAIR_PATH=re.compile(r'(?:/update/working_brief/findings(?:/[0-9]+)?|/next/arguments/filters(?:/[0-9]+)?)') @@ -26,6 +28,14 @@ def sha(document):return hashlib.sha256(json_bytes(document)).hexdigest() def parse_text(text,request): + try:return _parse_text(text,request) + except LLMError as error: + if request.get('prompt_version')==REPAIR_PROMPT and len(text.encode('utf-8'))<=LIMIT: + error.repair_response_text=text + raise + + +def _parse_text(text,request): from .diagnostics import parse_response_object if enabled(request) and len(text.encode('utf-8'))<=LIMIT: match=re.fullmatch(r'\s*```(?:json)?\s*\n(.*?)\n```\s*',text,re.S) @@ -89,6 +99,86 @@ def target_schema(schema,path): return copy.deepcopy(schema) +def filter_records(value,depth=0): + """Recognize complete records only, including positional container encodings.""" + if depth>8:return None + if isinstance(value,dict): + if set(value)=={'column','operator','value'}:return [value] + if value and all(isinstance(k,str) and k.isascii() and k.isdigit() for k in value): + if set(value)!={str(i) for i in range(len(value))}:return None + value=[value[str(i)] for i in range(len(value))] + else:return None + if not isinstance(value,list) or not value:return None + if len(value)==3 and isinstance(value[0],str) and isinstance(value[1],str): + return [dict(zip(('column','operator','value'),value))] + if len(value)==6 and all(isinstance(k,str) for k in value[::2]) and set(value[::2])=={'column','operator','value'}: + return [dict(zip(value[::2],value[1::2]))] + records=[] + for item in value: + group=filter_records(item,depth+1) + if group is None:return None + records.extend(group) + if len(records)>50:return None + return records + + +def filter_contract_feedback(document): + try: + action=document['next'];args=action['arguments'] + if action['kind']!='tool' or action['operation']!='project.query' or args['kind'] not in {'database.rows','database.count'}:return None + records=filter_records(args['filters']) + if records and any(not isinstance(r['operator'],str) or r['operator'] not in OPERATORS for r in records): + return {'validation_path':'/next/arguments/filters','validation_fields':['operator'], + 'requirement':'Unsupported operator. Allowed: eq,ne,lt,le,gt,ge,contains,in. No automatic SQL LIKE translation or value changes.'} + except (KeyError,TypeError,IndexError):pass + return None + + +def resolved_schema(root,path,document): + """Resolve the selected container across matching action/query alternatives.""" + def walk(rule,remaining,value,depth): + if depth>24:return [] + if '$ref' in rule: + ref=rule['$ref'] + if not ref.startswith('#/'):return [] + target=root + for key in ref[2:].split('/'):target=target[key.replace('~1','/').replace('~0','~')] + return walk(target,remaining,value,depth+1) + for discriminator in ('kind','operation'): + constraint=rule.get('properties',{}).get(discriminator,{}) + if isinstance(value,dict) and discriminator in value: + if 'const' in constraint and value[discriminator]!=constraint['const']:return [] + if 'enum' in constraint and value[discriminator] not in constraint['enum']:return [] + if not remaining:return [rule] + key=remaining[0] + if type(key) is int and 'items' in rule:return walk(rule['items'],remaining[1:],value[key],depth+1) + if isinstance(key,str) and key in rule.get('properties',{}): + return walk(rule['properties'][key],remaining[1:],value[key],depth+1) + found=[] + for branch in rule.get('oneOf',rule.get('anyOf',[])):found.extend(walk(branch,remaining,value,depth+1)) + return found + found={json_bytes(r):r for r in walk(root,path,document,0)} + return copy.deepcopy(next(iter(found.values()))) if len(found)==1 else None + + +def admission_reason(document,path,rule): + """Reject impossible/value-changing repairs before spending a model call.""" + if not REPAIR_PATH.fullmatch('/'+'/'.join(map(str,path))):return 'invalid_target' + before=at(document,path) + if not isinstance(before,(dict,list)):return 'scalar_target' + expected=rule.get('type') + if expected not in ('object','array'):return 'unsupported_target_type' + if path[:3]==FILTER_PATH: + records=filter_records(before) + if not records:return 'incomplete_filter_records' + if any(not isinstance(r['operator'],str) or r['operator'] not in OPERATORS for r in records):return 'unknown_operator' + if any(not isinstance(r['column'],str) or not re.fullmatch(r'[A-Za-z_][A-Za-z0-9_]*',r['column']) for r in records):return 'invalid_column' + if expected=='object': + needed=sum(1 for key in rule.get('required',[]) if rule.get('properties',{}).get(key,{}).get('type') not in ('object','array')) + if sum(atoms(before).values())LIMIT @@ -116,6 +206,11 @@ def apply_repair(request,patch): if not REPAIR_PATH.fullmatch('/'+'/'.join(map(str,path))):raise LLMError('LLM_REPAIR_BINDING') if sha(original)!=data['document_sha256']:raise LLMError('LLM_REPAIR_BINDING') before=at(original,path);after=patch['replacement'] + if not isinstance(before,(dict,list)):raise LLMError('LLM_REPAIR_INELIGIBLE') + if path[:3]==FILTER_PATH: + records=filter_records(before) + if not records or any(not isinstance(r['operator'],str) or r['operator'] not in OPERATORS for r in records): + raise LLMError('LLM_REPAIR_INELIGIBLE') if atoms(before)!=atoms(after):raise LLMError('LLM_REPAIR_CONTENT_CHANGED') def named(value): found=[] @@ -172,12 +267,18 @@ def _make_repair(context,call,call_ref): if details.get('expected_types'):details.setdefault('validation_keyword','type') path=eligible(context,document,details) if path is None:return None + if filter_contract_feedback(document):return None + if path[:3]==FILTER_PATH and not isinstance(at(document,path),(dict,list)): + # A type error at filters[0] can describe one field of a flat record. + # Repair the complete record/list, never that scalar in isolation. + path=FILTER_PATH.copy() # A prose finding is missing semantic fields, not merely JSON structure. # Do not spend a repair call inventing ID/direction/scope/reference values. if path[:3]==['update','working_brief','findings'] and not isinstance(at(document,path),dict): return None schema=output_schema(context,strict=True) - try:rule=target_schema(schema,path) + rule=resolved_schema(schema,path,document) + try:rule=rule or target_schema(schema,path) except (KeyError,TypeError): def errors(error): yield error @@ -187,6 +288,23 @@ def _make_repair(context,call,call_ref): if not candidates:return None rule=copy.deepcopy(candidates[0]) if rule.get('type') not in ('object','array'):return None + if admission_reason(document,path,rule):return None + # A bounded structural witness must already satisfy the entire contract. + # This excludes other unresolved errors and missing semantic values. + before=at(document,path) + if path[:3]==FILTER_PATH: + action=document['next'];args=action['arguments'] + if action.get('kind')!='tool' or action.get('operation')!='project.query' or args.get('kind') not in {'database.rows','database.count'}:return None + records=filter_records(before) + if rule['type']=='object' and len(records)!=1:return None + replacement=records[0] if rule['type']=='object' else records + elif rule['type']=='array' and isinstance(before,dict):replacement=[before] + else:return None + witness={'messages':[{}, {'content':json_bytes({'repair':{ + 'document':document,'document_sha256':sha(document),'path':path}}).decode('utf-8')}]} + try:reconstructed=apply_repair(witness,{'replacement':replacement}) + except LLMError:return None + if not Draft202012Validator(schema).is_valid(reconstructed):return None base=build_prompt(context) live=json.loads(base['messages'][1]['content']).get('input',{}) binding={'hypothesis':{'id':context['selected_hypothesis']}, diff --git a/src/viano_bug_analysis/llm/runner.py b/src/viano_bug_analysis/llm/runner.py index 7e4cf10..96b7d41 100644 --- a/src/viano_bug_analysis/llm/runner.py +++ b/src/viano_bug_analysis/llm/runner.py @@ -158,7 +158,7 @@ class CaseLLMRunner: outcome['diagnostics'] = error.diagnostics if error.http_status is not None: outcome["http_status"] = error.http_status - for field in ('rejected_document','normalization'): + for field in ('rejected_document','normalization','rejected_repair'): if hasattr(error,field):outcome[field]=getattr(error,field) result_ref = base + f"/round-{rounds:04d}.json" details["result_ref"] = result_ref diff --git a/src/viano_bug_analysis/schemas/llm-call-v1.schema.json b/src/viano_bug_analysis/schemas/llm-call-v1.schema.json index f256e55..31f10b0 100644 --- a/src/viano_bug_analysis/schemas/llm-call-v1.schema.json +++ b/src/viano_bug_analysis/schemas/llm-call-v1.schema.json @@ -308,6 +308,20 @@ "boolean", "null" ] + }, + "json_error_kind": { + "enum": [ + "missing_delimiter", + "missing_key", + "unterminated_string", + "invalid_escape", + "extra_data", + "unexpected_value", + "other" + ] + }, + "json_error_at_end": { + "type": "boolean" } }, "additionalProperties": false @@ -359,6 +373,48 @@ "type": "object" } } + }, + "rejected_repair": { + "type": "object", + "additionalProperties": false, + "required": [ + "version", + "reason", + "sha256" + ], + "properties": { + "version": { + "const": 1 + }, + "reason": { + "type": "string", + "maxLength": 64, + "pattern": "^[A-Z][A-Z0-9_]*$" + }, + "patch": { + "type": "object" + }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "original_text": { + "type": "string", + "maxLength": 65536 + } + }, + "oneOf": [ + { + "required": [ + "patch" + ] + }, + { + "required": [ + "original_text" + ] + } + ] } } } diff --git a/src/viano_bug_analysis/schemas/llm-failure-diagnostics-v1.schema.json b/src/viano_bug_analysis/schemas/llm-failure-diagnostics-v1.schema.json index bd34ad6..b1273a7 100644 --- a/src/viano_bug_analysis/schemas/llm-failure-diagnostics-v1.schema.json +++ b/src/viano_bug_analysis/schemas/llm-failure-diagnostics-v1.schema.json @@ -155,6 +155,20 @@ "type": "string", "maxLength": 64 } + }, + "json_error_kind": { + "enum": [ + "missing_delimiter", + "missing_key", + "unterminated_string", + "invalid_escape", + "extra_data", + "unexpected_value", + "other" + ] + }, + "json_error_at_end": { + "type": "boolean" } }, "additionalProperties": false diff --git a/tests/test_repair_admission.py b/tests/test_repair_admission.py new file mode 100644 index 0000000..102a426 --- /dev/null +++ b/tests/test_repair_admission.py @@ -0,0 +1,160 @@ +import asyncio +import copy +import json + +import pytest +from jsonschema import Draft202012Validator + +from test_case_builder import case_setup +from test_investigation_tools import tool_case +from test_case_interpreter import gateway +from test_compact_investigation import context +from test_lean_investigation_decision import reply +from test_investigator import LoopAdapter,CASE,state +from test_project_research import rows +from test_response_recovery import loop,RepairAdapter +from viano_bug_analysis.analysis.investigation_prompt import output_schema +from viano_bug_analysis.budget import BudgetManager +from viano_bug_analysis.llm import ProviderResponse +from viano_bug_analysis.llm.response_recovery import (make_repair,repair_request,apply_repair, + filter_contract_feedback,sha,admission_reason,REPAIR_PROMPT) +from viano_bug_analysis.package import PackageReader,PackageValidator + + +def setup_case(filters): + c=context();r=reply() + r['next']={'kind':'tool','operation':'project.query','arguments':rows(filters=filters)} + c['previous_rejection']='INVESTIGATION_OUTPUT' + c['previous_rejection_details']={'validation_path':'/next/arguments/filters/0', + 'validation_keyword':'type','expected_types':['object'],'actual_type':'string'} + return c,{'outcome':{'document':r}} + + +def test_flat_filter_is_repaired_as_whole_list_not_first_scalar(): + c,call=setup_case(['id','eq',1]) + plan,binding=make_repair(c,call,'source-call.json') + assert plan['path']==['next','arguments','filters'] and plan['schema']['type']=='array' + c.update(response_repair=plan,repair_binding=binding) + fixed=apply_repair(repair_request(c),{'replacement':[{'column':'id','operator':'eq','value':1}]}) + assert fixed['next']['arguments']['filters']==[{'column':'id','operator':'eq','value':1}] + assert fixed['update']==call['outcome']['document']['update'] + Draft202012Validator(output_schema(c,strict=True)).validate(fixed) + + +@pytest.mark.parametrize('filters',[ + ['LogTitle','like','%Wolpertinger%'], + [['LogTitle','like','%Wolpertinger%']], + [{'column':'LogTitle','operator':'like','value':'%Wolpertinger%'}], + ['column','LogTitle','operator','like','value','%Wolpertinger%'], +]) +def test_unknown_operator_has_no_format_repair_or_translation(filters): + c,call=setup_case(filters);before=copy.deepcopy(call) + feedback=filter_contract_feedback(call['outcome']['document']) + assert feedback['validation_fields']==['operator'] + assert 'Allowed: eq,ne,lt,le,gt,ge,contains,in' in feedback['requirement'] + assert make_repair(c,call,'source-call.json') is None + assert call==before + + +@pytest.mark.parametrize('filters',[['id'],['id','eq'],[{'column':'id','operator':'eq'}],{'0':{'column':'id'}}]) +def test_incomplete_filter_cannot_invent_missing_values(filters): + c,call=setup_case(filters) + assert make_repair(c,call,'source-call.json') is None + + +def test_direct_scalar_target_is_rejected_before_api(): + doc={'next':{'arguments':{'filters':['LogTitle','like','%Wolpertinger%']}}} + assert admission_reason(doc,['next','arguments','filters',0],{'type':'object'})=='scalar_target' + + +def test_operator_contract_feedback_reaches_next_regular_call(tool_case): + def change(r,c): + filters=['id','like','%1%'] if not c.get('previous_rejection') else [{'column':'id','operator':'eq','value':1}] + r['next']={'kind':'tool','operation':'project.query','arguments':rows(filters=filters)} + adapter=LoopAdapter(change);worker=loop(tool_case,adapter) + first=asyncio.run(worker.step(*CASE));assert first.status=='REJECTED' + assert not BudgetManager(tool_case[0]).snapshot(*CASE).state['accounts']['H1']['actions'] + with PackageReader(first.package) as r:c=json.loads(r.read_bytes(state(first)['active']['context_ref'])) + assert 'response_repair' not in c and c['previous_rejection_details']['validation_fields']==['operator'] + second=asyncio.run(worker.step(*CASE));assert second.status=='PROGRESSED' + assert len(adapter.contexts)==2 and PackageValidator().validate(second.package).valid + + +def test_rejected_patch_is_saved_but_not_logged(tool_case,caplog): + adapter=RepairAdapter(tamper=True);worker=loop(tool_case,adapter) + assert asyncio.run(worker.step(*CASE)).status=='REJECTED' + second=asyncio.run(worker.step(*CASE));assert second.code=='LLM_REPAIR_CONTENT_CHANGED' + with PackageReader(second.package) as r: + call=json.loads(r.read_bytes(state(second)['hypotheses']['H1']['notes'][-1]['llm_ref'])) + saved=call['outcome']['rejected_repair'] + assert saved['patch']=={'replacement':{'column':'id','operator':'eq','value':999}} + assert saved['sha256']==sha(saved['patch']) and saved['reason']==second.code + assert 'replacement' not in caplog.text + assert PackageValidator().validate(second.package).valid + assert not BudgetManager(tool_case[0]).snapshot(*CASE).state['accounts']['H1']['actions'] + + +def test_schema_rejected_patch_is_also_retained(tool_case): + class Adapter(RepairAdapter): + async def generate(self,config,request,**kwargs): + if request['prompt_version']==REPAIR_PROMPT: + return ProviderResponse({'replacement':{'column':'id','operator':'unknown','value':1}},'fixture',100,20) + return await super().generate(config,request,**kwargs) + worker=loop(tool_case,Adapter());assert asyncio.run(worker.step(*CASE)).status=='REJECTED' + second=asyncio.run(worker.step(*CASE));assert second.code=='LLM_OUTPUT_SCHEMA' + with PackageReader(second.package) as r: + call=json.loads(r.read_bytes(state(second)['hypotheses']['H1']['notes'][-1]['llm_ref'])) + assert call['outcome']['rejected_repair']['patch']['replacement']['operator']=='unknown' + assert call['outcome']['diagnostics']['validation_path']=='/replacement/operator' + + +def test_malformed_repair_json_is_retained_without_log_echo(tool_case,caplog): + from viano_bug_analysis.llm.response_recovery import parse_text + raw='{"replacement":"PRIVATE_REPAIR_TEXT"' + class Adapter(RepairAdapter): + async def generate(self,config,request,**kwargs): + if request['prompt_version']==REPAIR_PROMPT:parse_text(raw,request) + return await super().generate(config,request,**kwargs) + worker=loop(tool_case,Adapter());assert asyncio.run(worker.step(*CASE)).status=='REJECTED' + second=asyncio.run(worker.step(*CASE));assert second.code=='LLM_RESPONSE' + with PackageReader(second.package) as r: + call=json.loads(r.read_bytes(state(second)['hypotheses']['H1']['notes'][-1]['llm_ref'])) + saved=call['outcome']['rejected_repair'] + assert saved['original_text']==raw and saved['sha256']==sha(raw) + assert 'PRIVATE_REPAIR_TEXT' not in caplog.text + assert PackageValidator().validate(second.package).valid + + +def test_another_invalid_field_prevents_format_repair(): + c,call=setup_case(['id','eq',1]);call['outcome']['document']['next']['kind']='invented_action' + assert make_repair(c,call,'source-call.json') is None + + +def test_ineligible_saved_plan_blocks_without_another_provider_call(tool_case): + adapter=RepairAdapter();worker=loop(tool_case,adapter) + first=asyncio.run(worker.step(*CASE));assert first.status=='REJECTED' + s=state(first) + with PackageReader(first.package) as r:c=json.loads(r.read_bytes(s['active']['context_ref'])) + plan=c['response_repair'];plan['document']['next']['arguments']['filters']=['id','eq',1] + plan['document_sha256']=sha(plan['document']) + assert plan['path']==['next','arguments','filters',0] # now points to a scalar + s['active']['context_ref']=s['active']['context_ref'].replace('.json','-ineligible.json') + worker._save(s,context=c) + blocked=asyncio.run(worker.step(*CASE)) + assert blocked.status=='BLOCKED' and blocked.code=='LLM_REPAIR_INELIGIBLE' + assert len(adapter.requests)==1 + assert not BudgetManager(tool_case[0]).snapshot(*CASE).state['accounts']['H1']['actions'] + + +def test_oversized_rejected_patch_is_not_retained(tool_case): + class Adapter(RepairAdapter): + async def generate(self,config,request,**kwargs): + if request['prompt_version']==REPAIR_PROMPT: + return ProviderResponse({'replacement':{'column':'id','operator':'eq','value':'x'*70000}},'fixture',100,20) + return await super().generate(config,request,**kwargs) + worker=loop(tool_case,Adapter());assert asyncio.run(worker.step(*CASE)).status=='REJECTED' + second=asyncio.run(worker.step(*CASE));assert second.status=='REJECTED' + with PackageReader(second.package) as r: + call=json.loads(r.read_bytes(state(second)['hypotheses']['H1']['notes'][-1]['llm_ref'])) + assert 'rejected_repair' not in call['outcome'] + assert PackageValidator().validate(second.package).valid