Reject impossible format repairs and retain rejected proposals
This commit is contained in:
1 parent
bfb353eca2
commit
2fe1e478a5
9 files changed
+416
-6
No files matched your search
@@ -77,3 +77,41 @@ Extraaufruf, Wiederaufnahme und die Ablehnung geänderter Abfragewerte sowie
|
||||
unbekannter Aktionen/Belege ab. Ein gezielter Reparaturprompt bleibt im geprüften
|
||||
Fixture bei 2.199 statt 10.353 Zeichen des normalen Untersuchungsaufrufs. Das ist keine
|
||||
allgemeine Kosten- oder Qualitätsgarantie; bezahlte Tests erfolgen separat.
|
||||
|
||||
## Zulassung und Ablehnungs-Audit nach dem ersten echten Lauf
|
||||
|
||||
Der erste echte Lauf normalisierte einen `name/contains/Wolpertinger`-Filter
|
||||
erfolgreich. Ein weiterer Filter enthielt dagegen
|
||||
`["LogTitle","like","%Wolpertinger%"]`. Das ist nicht rein strukturell: `like`
|
||||
gehört nicht zum Werkzeugvertrag. Die damalige Reparatur zielte zusätzlich
|
||||
fälschlich auf den einzelnen String `filters[0]`. Ein vollständiges Objekt mit
|
||||
Spalte, Operator und Wert kann daraus nicht werterhaltend erzeugt werden.
|
||||
|
||||
Die Zulassung berücksichtigt jetzt die vollständige Filtereinheit und löst den
|
||||
zugehörigen Schemaausschnitt über die passende Aktions-/Query-Alternative auf.
|
||||
Ein Fehlerpfad innerhalb einer flachen Liste wird, soweit überhaupt reparierbar,
|
||||
auf deren vollständigen Filterbereich angehoben. Skalare Ziele, unvollständige
|
||||
Filter, fehlende Werte und unbekannte Operatoren erhalten keinen Format-Reparaturaufruf.
|
||||
Es wird weder SQL `LIKE` in `contains` übersetzt noch werden Prozentzeichen entfernt.
|
||||
Stattdessen bekommt der normale Ablehnungspfad einen kurzen, ausdrücklichen
|
||||
Operator-Vertragshinweis. Eine neue fachliche Entscheidung bleibt Sache der LLM.
|
||||
|
||||
Vor der Zulassung wird mit den bereits vorhandenen Filterwerten eine rein
|
||||
strukturelle Prüffassung gebildet. Sie muss die gesamte ursprüngliche Ausgabestruktur
|
||||
erfüllen und die Erhaltungsregeln einhalten. Weitere unveränderte Strukturfehler
|
||||
verhindern so ebenfalls einen aussichtslosen Reparaturaufruf. Diese Prüffassung
|
||||
wird nicht ausgeführt; eine tatsächliche Modellreparatur muss später nochmals
|
||||
alle Kontrollen einschließlich fachlicher Beleg- und Budgetprüfung bestehen.
|
||||
Alte gespeicherte Reparaturpläne werden vor einem erneuten Anbieteraufruf ebenfalls
|
||||
auf ungeeignete Zieltypen und Operatoren geprüft, ohne ihre historischen Prompts
|
||||
umzuschreiben.
|
||||
|
||||
Abgelehnte Reparaturantworten stehen separat in `outcome.rejected_repair`, mit
|
||||
Version, Ablehnungsgrund und SHA-256 über die kanonische JSON-Darstellung des
|
||||
gespeicherten Inhalts. Ein geparster Vorschlag wird als `patch`, kaputtes JSON
|
||||
als `original_text` gespeichert. Die Grenze liegt bei höchstens 65.536 UTF-8-Bytes
|
||||
und zusätzlich innerhalb des konfigurierten Response-Limits. Größere Vorschläge
|
||||
werden nicht gekürzt oder teilweise übernommen. HTTP-Fehlerkörper und Header
|
||||
werden dadurch nicht aufbewahrt. Logs und bereinigte Diagnosen enthalten weiterhin
|
||||
keinen Antworttext. Das Aufrufprotokoll erlaubt die zusätzlichen sicheren
|
||||
JSON-Parserdiagnosen, damit deren Speicherung selbst nicht mit `INPUT_SCHEMA` scheitert.
|
||||
@@ -410,6 +410,11 @@ class Investigator:
|
||||
if budget.snapshot(case_id,package_id).state!=context["budget"]:
|
||||
raise InvestigationError("INVESTIGATION_CONTEXT_CHANGED")
|
||||
operation=f"investigation:v1:{active['hypothesis_id']}:{active['iteration']}:{active['attempt']}:{active['retry_token']}"
|
||||
if context.get('response_repair'):
|
||||
from viano_bug_analysis.llm.response_recovery import admission_reason
|
||||
plan=context['response_repair']
|
||||
if admission_reason(plan['document'],plan['path'],plan['schema']):
|
||||
return self._research_blocked(state,'LLM_REPAIR_INELIGIBLE','INVALID_LLM_RESPONSES')
|
||||
try:
|
||||
request=build_prompt(context)
|
||||
if context.get('prompt_version') in {'investigation_project_v15','investigation_project_v16','investigation_project_v17','investigation_project_v18'}:
|
||||
@@ -651,7 +656,10 @@ class Investigator:
|
||||
if context['policy'].get('response_recovery_version')==1 and not context.get('response_repair'):
|
||||
with PackageReader(self._latest(state['case_id'],state['package_id']),self.store.limits) as reader:
|
||||
call=self._read(reader,llm_ref,'llm-call')
|
||||
from viano_bug_analysis.llm.response_recovery import make_repair
|
||||
from viano_bug_analysis.llm.response_recovery import make_repair,filter_contract_feedback
|
||||
document=call['outcome'].get('rejected_document',call['outcome'].get('document'))
|
||||
contract_feedback=filter_contract_feedback(document)
|
||||
if contract_feedback:context['previous_rejection_details']=contract_feedback
|
||||
prepared=make_repair(context,call,llm_ref)
|
||||
if prepared:
|
||||
context['response_repair'],context['repair_binding']=prepared
|
||||
|
||||
@@ -21,7 +21,7 @@ VALIDATION_FIELDS = frozenset({'retire','finding_id','intent','hypothesis_id','a
|
||||
'prerequisites','requested_observations','environment','countercheck_refs','path',
|
||||
'start_line','end_line','limit','offset','table','columns','filters','column','operator',
|
||||
'value','text','case_sensitive','path_prefix','source_ref','source_refs','note','causal_review',
|
||||
'next_step','goal','supports_if','weakens_if','investigate_if','profile','entity_refs'})
|
||||
'next_step','goal','supports_if','weakens_if','investigate_if','profile','entity_refs','replacement'})
|
||||
|
||||
|
||||
def schema_feedback(error, _depth=0, *, include_keyword=False):
|
||||
|
||||
@@ -17,6 +17,7 @@ from .provider_base import ProviderResponse
|
||||
from .diagnostics import sanitize, count
|
||||
from .gemini import GeminiProvider
|
||||
from .usage import usage_metadata
|
||||
from .response_recovery import REPAIR_PROMPT
|
||||
|
||||
LOGGER = logging.getLogger("viano_bug_analysis.llm")
|
||||
PRIVACY = {"local_only": {"local"}, "self_hosted": {"local", "self_hosted"}, "cloud_allowed": {"local", "self_hosted", "cloud"}}
|
||||
@@ -116,7 +117,7 @@ class LLMGateway:
|
||||
if (response.cached_input_tokens is None or response.cached_input_tokens <= 0
|
||||
or response.input_tokens is None or response.cached_input_tokens > response.input_tokens):
|
||||
raise LLMError('LLM_CACHE_USAGE_UNCONFIRMED')
|
||||
from .response_recovery import enabled,normalize,REPAIR_PROMPT,apply_repair
|
||||
from .response_recovery import enabled,normalize,apply_repair
|
||||
if enabled(request):
|
||||
document,audit=normalize(response.document)
|
||||
if response.normalization:
|
||||
@@ -176,7 +177,7 @@ class LLMGateway:
|
||||
LOGGER.warning("LLM endpoint unavailable provider=%s code=%s", provider.id, code)
|
||||
except ValidationError as invalid:
|
||||
info={'validation_keyword':invalid.validator}
|
||||
if request['prompt_version'] in {'investigation_bootstrap_v22','investigation_bootstrap_v21','investigation_project_v14','investigation_project_v15','investigation_project_v16','investigation_project_v17','investigation_project_v18','investigation_project_v19','investigation_project_v20'}:
|
||||
if request['prompt_version'] in {REPAIR_PROMPT,'investigation_bootstrap_v22','investigation_bootstrap_v21','investigation_project_v14','investigation_project_v15','investigation_project_v16','investigation_project_v17','investigation_project_v18','investigation_project_v19','investigation_project_v20'}:
|
||||
from .diagnostics import schema_feedback
|
||||
info.update(schema_feedback(invalid,include_keyword=True))
|
||||
error = LLMError("LLM_OUTPUT_SCHEMA",diagnostics=info)
|
||||
@@ -184,9 +185,11 @@ class LLMGateway:
|
||||
if enabled(request) and isinstance(response.document,dict) and len(json_bytes(response.document))<=LIMIT:
|
||||
error.rejected_document=copy.deepcopy(response.document)
|
||||
if response.normalization:error.normalization=copy.deepcopy(response.normalization)
|
||||
self._retain_rejected_repair(error,request,response)
|
||||
self._failure_metadata(error,provider,request,started,response,attempts)
|
||||
raise error from None
|
||||
except LLMError as error:
|
||||
self._retain_rejected_repair(error,request,response)
|
||||
self._failure_metadata(error,provider,request,started,response,attempts)
|
||||
raise
|
||||
except Exception:
|
||||
@@ -195,6 +198,19 @@ class LLMGateway:
|
||||
raise error from None
|
||||
raise GatewayUnavailable(attempts)
|
||||
|
||||
def _retain_rejected_repair(self,error,request,response):
|
||||
from .response_recovery import REPAIR_PROMPT,LIMIT,sha
|
||||
if request['prompt_version']!=REPAIR_PROMPT:return
|
||||
if (request['prompt_version']==REPAIR_PROMPT and isinstance(response,ProviderResponse)
|
||||
and isinstance(response.document,dict)
|
||||
and len(json_bytes(response.document))<=min(LIMIT,self.config.max_response_bytes)):
|
||||
error.rejected_repair={'version':1,'reason':error.code,
|
||||
'patch':copy.deepcopy(response.document),'sha256':sha(response.document)}
|
||||
elif isinstance(getattr(error,'repair_response_text',None),str):
|
||||
text=error.repair_response_text
|
||||
if len(text.encode('utf-8'))<=min(LIMIT,self.config.max_response_bytes):
|
||||
error.rejected_repair={'version':1,'reason':error.code,'original_text':text,'sha256':sha(text)}
|
||||
|
||||
@staticmethod
|
||||
def _failure_metadata(error,provider,request,started,response,attempts):
|
||||
details = sanitize(error.diagnostics)
|
||||
|
||||
@@ -12,6 +12,8 @@ from .errors import LLMError
|
||||
MARKER='x-viano-response-recovery'
|
||||
LIMIT=65536
|
||||
REPAIR_PROMPT='investigation_format_repair_v1'
|
||||
OPERATORS=frozenset({'eq','ne','lt','le','gt','ge','contains','in'})
|
||||
FILTER_PATH=['next','arguments','filters']
|
||||
OBJECT_PATH=re.compile(r'(?:/assessment|/update|/next|/next/arguments|/update/working_brief|/update/next_step|/update/working_brief/findings/[0-9]+|/next/arguments/filters/[0-9]+)')
|
||||
REPAIR_PATH=re.compile(r'(?:/update/working_brief/findings(?:/[0-9]+)?|/next/arguments/filters(?:/[0-9]+)?)')
|
||||
|
||||
@@ -26,6 +28,14 @@ def sha(document):return hashlib.sha256(json_bytes(document)).hexdigest()
|
||||
|
||||
|
||||
def parse_text(text,request):
|
||||
try:return _parse_text(text,request)
|
||||
except LLMError as error:
|
||||
if request.get('prompt_version')==REPAIR_PROMPT and len(text.encode('utf-8'))<=LIMIT:
|
||||
error.repair_response_text=text
|
||||
raise
|
||||
|
||||
|
||||
def _parse_text(text,request):
|
||||
from .diagnostics import parse_response_object
|
||||
if enabled(request) and len(text.encode('utf-8'))<=LIMIT:
|
||||
match=re.fullmatch(r'\s*```(?:json)?\s*\n(.*?)\n```\s*',text,re.S)
|
||||
@@ -89,6 +99,86 @@ def target_schema(schema,path):
|
||||
return copy.deepcopy(schema)
|
||||
|
||||
|
||||
def filter_records(value,depth=0):
|
||||
"""Recognize complete records only, including positional container encodings."""
|
||||
if depth>8:return None
|
||||
if isinstance(value,dict):
|
||||
if set(value)=={'column','operator','value'}:return [value]
|
||||
if value and all(isinstance(k,str) and k.isascii() and k.isdigit() for k in value):
|
||||
if set(value)!={str(i) for i in range(len(value))}:return None
|
||||
value=[value[str(i)] for i in range(len(value))]
|
||||
else:return None
|
||||
if not isinstance(value,list) or not value:return None
|
||||
if len(value)==3 and isinstance(value[0],str) and isinstance(value[1],str):
|
||||
return [dict(zip(('column','operator','value'),value))]
|
||||
if len(value)==6 and all(isinstance(k,str) for k in value[::2]) and set(value[::2])=={'column','operator','value'}:
|
||||
return [dict(zip(value[::2],value[1::2]))]
|
||||
records=[]
|
||||
for item in value:
|
||||
group=filter_records(item,depth+1)
|
||||
if group is None:return None
|
||||
records.extend(group)
|
||||
if len(records)>50:return None
|
||||
return records
|
||||
|
||||
|
||||
def filter_contract_feedback(document):
|
||||
try:
|
||||
action=document['next'];args=action['arguments']
|
||||
if action['kind']!='tool' or action['operation']!='project.query' or args['kind'] not in {'database.rows','database.count'}:return None
|
||||
records=filter_records(args['filters'])
|
||||
if records and any(not isinstance(r['operator'],str) or r['operator'] not in OPERATORS for r in records):
|
||||
return {'validation_path':'/next/arguments/filters','validation_fields':['operator'],
|
||||
'requirement':'Unsupported operator. Allowed: eq,ne,lt,le,gt,ge,contains,in. No automatic SQL LIKE translation or value changes.'}
|
||||
except (KeyError,TypeError,IndexError):pass
|
||||
return None
|
||||
|
||||
|
||||
def resolved_schema(root,path,document):
|
||||
"""Resolve the selected container across matching action/query alternatives."""
|
||||
def walk(rule,remaining,value,depth):
|
||||
if depth>24:return []
|
||||
if '$ref' in rule:
|
||||
ref=rule['$ref']
|
||||
if not ref.startswith('#/'):return []
|
||||
target=root
|
||||
for key in ref[2:].split('/'):target=target[key.replace('~1','/').replace('~0','~')]
|
||||
return walk(target,remaining,value,depth+1)
|
||||
for discriminator in ('kind','operation'):
|
||||
constraint=rule.get('properties',{}).get(discriminator,{})
|
||||
if isinstance(value,dict) and discriminator in value:
|
||||
if 'const' in constraint and value[discriminator]!=constraint['const']:return []
|
||||
if 'enum' in constraint and value[discriminator] not in constraint['enum']:return []
|
||||
if not remaining:return [rule]
|
||||
key=remaining[0]
|
||||
if type(key) is int and 'items' in rule:return walk(rule['items'],remaining[1:],value[key],depth+1)
|
||||
if isinstance(key,str) and key in rule.get('properties',{}):
|
||||
return walk(rule['properties'][key],remaining[1:],value[key],depth+1)
|
||||
found=[]
|
||||
for branch in rule.get('oneOf',rule.get('anyOf',[])):found.extend(walk(branch,remaining,value,depth+1))
|
||||
return found
|
||||
found={json_bytes(r):r for r in walk(root,path,document,0)}
|
||||
return copy.deepcopy(next(iter(found.values()))) if len(found)==1 else None
|
||||
|
||||
|
||||
def admission_reason(document,path,rule):
|
||||
"""Reject impossible/value-changing repairs before spending a model call."""
|
||||
if not REPAIR_PATH.fullmatch('/'+'/'.join(map(str,path))):return 'invalid_target'
|
||||
before=at(document,path)
|
||||
if not isinstance(before,(dict,list)):return 'scalar_target'
|
||||
expected=rule.get('type')
|
||||
if expected not in ('object','array'):return 'unsupported_target_type'
|
||||
if path[:3]==FILTER_PATH:
|
||||
records=filter_records(before)
|
||||
if not records:return 'incomplete_filter_records'
|
||||
if any(not isinstance(r['operator'],str) or r['operator'] not in OPERATORS for r in records):return 'unknown_operator'
|
||||
if any(not isinstance(r['column'],str) or not re.fullmatch(r'[A-Za-z_][A-Za-z0-9_]*',r['column']) for r in records):return 'invalid_column'
|
||||
if expected=='object':
|
||||
needed=sum(1 for key in rule.get('required',[]) if rule.get('properties',{}).get(key,{}).get('type') not in ('object','array'))
|
||||
if sum(atoms(before).values())<needed:return 'insufficient_values'
|
||||
return None
|
||||
|
||||
|
||||
def eligible(context,document,details):
|
||||
path=details.get('validation_path','')
|
||||
if (not isinstance(document,dict) or len(json_bytes(document))>LIMIT
|
||||
@@ -116,6 +206,11 @@ def apply_repair(request,patch):
|
||||
if not REPAIR_PATH.fullmatch('/'+'/'.join(map(str,path))):raise LLMError('LLM_REPAIR_BINDING')
|
||||
if sha(original)!=data['document_sha256']:raise LLMError('LLM_REPAIR_BINDING')
|
||||
before=at(original,path);after=patch['replacement']
|
||||
if not isinstance(before,(dict,list)):raise LLMError('LLM_REPAIR_INELIGIBLE')
|
||||
if path[:3]==FILTER_PATH:
|
||||
records=filter_records(before)
|
||||
if not records or any(not isinstance(r['operator'],str) or r['operator'] not in OPERATORS for r in records):
|
||||
raise LLMError('LLM_REPAIR_INELIGIBLE')
|
||||
if atoms(before)!=atoms(after):raise LLMError('LLM_REPAIR_CONTENT_CHANGED')
|
||||
def named(value):
|
||||
found=[]
|
||||
@@ -172,12 +267,18 @@ def _make_repair(context,call,call_ref):
|
||||
if details.get('expected_types'):details.setdefault('validation_keyword','type')
|
||||
path=eligible(context,document,details)
|
||||
if path is None:return None
|
||||
if filter_contract_feedback(document):return None
|
||||
if path[:3]==FILTER_PATH and not isinstance(at(document,path),(dict,list)):
|
||||
# A type error at filters[0] can describe one field of a flat record.
|
||||
# Repair the complete record/list, never that scalar in isolation.
|
||||
path=FILTER_PATH.copy()
|
||||
# A prose finding is missing semantic fields, not merely JSON structure.
|
||||
# Do not spend a repair call inventing ID/direction/scope/reference values.
|
||||
if path[:3]==['update','working_brief','findings'] and not isinstance(at(document,path),dict):
|
||||
return None
|
||||
schema=output_schema(context,strict=True)
|
||||
try:rule=target_schema(schema,path)
|
||||
rule=resolved_schema(schema,path,document)
|
||||
try:rule=rule or target_schema(schema,path)
|
||||
except (KeyError,TypeError):
|
||||
def errors(error):
|
||||
yield error
|
||||
@@ -187,6 +288,23 @@ def _make_repair(context,call,call_ref):
|
||||
if not candidates:return None
|
||||
rule=copy.deepcopy(candidates[0])
|
||||
if rule.get('type') not in ('object','array'):return None
|
||||
if admission_reason(document,path,rule):return None
|
||||
# A bounded structural witness must already satisfy the entire contract.
|
||||
# This excludes other unresolved errors and missing semantic values.
|
||||
before=at(document,path)
|
||||
if path[:3]==FILTER_PATH:
|
||||
action=document['next'];args=action['arguments']
|
||||
if action.get('kind')!='tool' or action.get('operation')!='project.query' or args.get('kind') not in {'database.rows','database.count'}:return None
|
||||
records=filter_records(before)
|
||||
if rule['type']=='object' and len(records)!=1:return None
|
||||
replacement=records[0] if rule['type']=='object' else records
|
||||
elif rule['type']=='array' and isinstance(before,dict):replacement=[before]
|
||||
else:return None
|
||||
witness={'messages':[{}, {'content':json_bytes({'repair':{
|
||||
'document':document,'document_sha256':sha(document),'path':path}}).decode('utf-8')}]}
|
||||
try:reconstructed=apply_repair(witness,{'replacement':replacement})
|
||||
except LLMError:return None
|
||||
if not Draft202012Validator(schema).is_valid(reconstructed):return None
|
||||
base=build_prompt(context)
|
||||
live=json.loads(base['messages'][1]['content']).get('input',{})
|
||||
binding={'hypothesis':{'id':context['selected_hypothesis']},
|
||||
|
||||
@@ -158,7 +158,7 @@ class CaseLLMRunner:
|
||||
outcome['diagnostics'] = error.diagnostics
|
||||
if error.http_status is not None:
|
||||
outcome["http_status"] = error.http_status
|
||||
for field in ('rejected_document','normalization'):
|
||||
for field in ('rejected_document','normalization','rejected_repair'):
|
||||
if hasattr(error,field):outcome[field]=getattr(error,field)
|
||||
result_ref = base + f"/round-{rounds:04d}.json"
|
||||
details["result_ref"] = result_ref
|
||||
|
||||
@@ -308,6 +308,20 @@
|
||||
"boolean",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"json_error_kind": {
|
||||
"enum": [
|
||||
"missing_delimiter",
|
||||
"missing_key",
|
||||
"unterminated_string",
|
||||
"invalid_escape",
|
||||
"extra_data",
|
||||
"unexpected_value",
|
||||
"other"
|
||||
]
|
||||
},
|
||||
"json_error_at_end": {
|
||||
"type": "boolean"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
@@ -359,6 +373,48 @@
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
},
|
||||
"rejected_repair": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"version",
|
||||
"reason",
|
||||
"sha256"
|
||||
],
|
||||
"properties": {
|
||||
"version": {
|
||||
"const": 1
|
||||
},
|
||||
"reason": {
|
||||
"type": "string",
|
||||
"maxLength": 64,
|
||||
"pattern": "^[A-Z][A-Z0-9_]*$"
|
||||
},
|
||||
"patch": {
|
||||
"type": "object"
|
||||
},
|
||||
"sha256": {
|
||||
"type": "string",
|
||||
"pattern": "^[a-f0-9]{64}$"
|
||||
},
|
||||
"original_text": {
|
||||
"type": "string",
|
||||
"maxLength": 65536
|
||||
}
|
||||
},
|
||||
"oneOf": [
|
||||
{
|
||||
"required": [
|
||||
"patch"
|
||||
]
|
||||
},
|
||||
{
|
||||
"required": [
|
||||
"original_text"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -155,6 +155,20 @@
|
||||
"type": "string",
|
||||
"maxLength": 64
|
||||
}
|
||||
},
|
||||
"json_error_kind": {
|
||||
"enum": [
|
||||
"missing_delimiter",
|
||||
"missing_key",
|
||||
"unterminated_string",
|
||||
"invalid_escape",
|
||||
"extra_data",
|
||||
"unexpected_value",
|
||||
"other"
|
||||
]
|
||||
},
|
||||
"json_error_at_end": {
|
||||
"type": "boolean"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
import asyncio
|
||||
import copy
|
||||
import json
|
||||
|
||||
import pytest
|
||||
from jsonschema import Draft202012Validator
|
||||
|
||||
from test_case_builder import case_setup
|
||||
from test_investigation_tools import tool_case
|
||||
from test_case_interpreter import gateway
|
||||
from test_compact_investigation import context
|
||||
from test_lean_investigation_decision import reply
|
||||
from test_investigator import LoopAdapter,CASE,state
|
||||
from test_project_research import rows
|
||||
from test_response_recovery import loop,RepairAdapter
|
||||
from viano_bug_analysis.analysis.investigation_prompt import output_schema
|
||||
from viano_bug_analysis.budget import BudgetManager
|
||||
from viano_bug_analysis.llm import ProviderResponse
|
||||
from viano_bug_analysis.llm.response_recovery import (make_repair,repair_request,apply_repair,
|
||||
filter_contract_feedback,sha,admission_reason,REPAIR_PROMPT)
|
||||
from viano_bug_analysis.package import PackageReader,PackageValidator
|
||||
|
||||
|
||||
def setup_case(filters):
|
||||
c=context();r=reply()
|
||||
r['next']={'kind':'tool','operation':'project.query','arguments':rows(filters=filters)}
|
||||
c['previous_rejection']='INVESTIGATION_OUTPUT'
|
||||
c['previous_rejection_details']={'validation_path':'/next/arguments/filters/0',
|
||||
'validation_keyword':'type','expected_types':['object'],'actual_type':'string'}
|
||||
return c,{'outcome':{'document':r}}
|
||||
|
||||
|
||||
def test_flat_filter_is_repaired_as_whole_list_not_first_scalar():
|
||||
c,call=setup_case(['id','eq',1])
|
||||
plan,binding=make_repair(c,call,'source-call.json')
|
||||
assert plan['path']==['next','arguments','filters'] and plan['schema']['type']=='array'
|
||||
c.update(response_repair=plan,repair_binding=binding)
|
||||
fixed=apply_repair(repair_request(c),{'replacement':[{'column':'id','operator':'eq','value':1}]})
|
||||
assert fixed['next']['arguments']['filters']==[{'column':'id','operator':'eq','value':1}]
|
||||
assert fixed['update']==call['outcome']['document']['update']
|
||||
Draft202012Validator(output_schema(c,strict=True)).validate(fixed)
|
||||
|
||||
|
||||
@pytest.mark.parametrize('filters',[
|
||||
['LogTitle','like','%Wolpertinger%'],
|
||||
[['LogTitle','like','%Wolpertinger%']],
|
||||
[{'column':'LogTitle','operator':'like','value':'%Wolpertinger%'}],
|
||||
['column','LogTitle','operator','like','value','%Wolpertinger%'],
|
||||
])
|
||||
def test_unknown_operator_has_no_format_repair_or_translation(filters):
|
||||
c,call=setup_case(filters);before=copy.deepcopy(call)
|
||||
feedback=filter_contract_feedback(call['outcome']['document'])
|
||||
assert feedback['validation_fields']==['operator']
|
||||
assert 'Allowed: eq,ne,lt,le,gt,ge,contains,in' in feedback['requirement']
|
||||
assert make_repair(c,call,'source-call.json') is None
|
||||
assert call==before
|
||||
|
||||
|
||||
@pytest.mark.parametrize('filters',[['id'],['id','eq'],[{'column':'id','operator':'eq'}],{'0':{'column':'id'}}])
|
||||
def test_incomplete_filter_cannot_invent_missing_values(filters):
|
||||
c,call=setup_case(filters)
|
||||
assert make_repair(c,call,'source-call.json') is None
|
||||
|
||||
|
||||
def test_direct_scalar_target_is_rejected_before_api():
|
||||
doc={'next':{'arguments':{'filters':['LogTitle','like','%Wolpertinger%']}}}
|
||||
assert admission_reason(doc,['next','arguments','filters',0],{'type':'object'})=='scalar_target'
|
||||
|
||||
|
||||
def test_operator_contract_feedback_reaches_next_regular_call(tool_case):
|
||||
def change(r,c):
|
||||
filters=['id','like','%1%'] if not c.get('previous_rejection') else [{'column':'id','operator':'eq','value':1}]
|
||||
r['next']={'kind':'tool','operation':'project.query','arguments':rows(filters=filters)}
|
||||
adapter=LoopAdapter(change);worker=loop(tool_case,adapter)
|
||||
first=asyncio.run(worker.step(*CASE));assert first.status=='REJECTED'
|
||||
assert not BudgetManager(tool_case[0]).snapshot(*CASE).state['accounts']['H1']['actions']
|
||||
with PackageReader(first.package) as r:c=json.loads(r.read_bytes(state(first)['active']['context_ref']))
|
||||
assert 'response_repair' not in c and c['previous_rejection_details']['validation_fields']==['operator']
|
||||
second=asyncio.run(worker.step(*CASE));assert second.status=='PROGRESSED'
|
||||
assert len(adapter.contexts)==2 and PackageValidator().validate(second.package).valid
|
||||
|
||||
|
||||
def test_rejected_patch_is_saved_but_not_logged(tool_case,caplog):
|
||||
adapter=RepairAdapter(tamper=True);worker=loop(tool_case,adapter)
|
||||
assert asyncio.run(worker.step(*CASE)).status=='REJECTED'
|
||||
second=asyncio.run(worker.step(*CASE));assert second.code=='LLM_REPAIR_CONTENT_CHANGED'
|
||||
with PackageReader(second.package) as r:
|
||||
call=json.loads(r.read_bytes(state(second)['hypotheses']['H1']['notes'][-1]['llm_ref']))
|
||||
saved=call['outcome']['rejected_repair']
|
||||
assert saved['patch']=={'replacement':{'column':'id','operator':'eq','value':999}}
|
||||
assert saved['sha256']==sha(saved['patch']) and saved['reason']==second.code
|
||||
assert 'replacement' not in caplog.text
|
||||
assert PackageValidator().validate(second.package).valid
|
||||
assert not BudgetManager(tool_case[0]).snapshot(*CASE).state['accounts']['H1']['actions']
|
||||
|
||||
|
||||
def test_schema_rejected_patch_is_also_retained(tool_case):
|
||||
class Adapter(RepairAdapter):
|
||||
async def generate(self,config,request,**kwargs):
|
||||
if request['prompt_version']==REPAIR_PROMPT:
|
||||
return ProviderResponse({'replacement':{'column':'id','operator':'unknown','value':1}},'fixture',100,20)
|
||||
return await super().generate(config,request,**kwargs)
|
||||
worker=loop(tool_case,Adapter());assert asyncio.run(worker.step(*CASE)).status=='REJECTED'
|
||||
second=asyncio.run(worker.step(*CASE));assert second.code=='LLM_OUTPUT_SCHEMA'
|
||||
with PackageReader(second.package) as r:
|
||||
call=json.loads(r.read_bytes(state(second)['hypotheses']['H1']['notes'][-1]['llm_ref']))
|
||||
assert call['outcome']['rejected_repair']['patch']['replacement']['operator']=='unknown'
|
||||
assert call['outcome']['diagnostics']['validation_path']=='/replacement/operator'
|
||||
|
||||
|
||||
def test_malformed_repair_json_is_retained_without_log_echo(tool_case,caplog):
|
||||
from viano_bug_analysis.llm.response_recovery import parse_text
|
||||
raw='{"replacement":"PRIVATE_REPAIR_TEXT"'
|
||||
class Adapter(RepairAdapter):
|
||||
async def generate(self,config,request,**kwargs):
|
||||
if request['prompt_version']==REPAIR_PROMPT:parse_text(raw,request)
|
||||
return await super().generate(config,request,**kwargs)
|
||||
worker=loop(tool_case,Adapter());assert asyncio.run(worker.step(*CASE)).status=='REJECTED'
|
||||
second=asyncio.run(worker.step(*CASE));assert second.code=='LLM_RESPONSE'
|
||||
with PackageReader(second.package) as r:
|
||||
call=json.loads(r.read_bytes(state(second)['hypotheses']['H1']['notes'][-1]['llm_ref']))
|
||||
saved=call['outcome']['rejected_repair']
|
||||
assert saved['original_text']==raw and saved['sha256']==sha(raw)
|
||||
assert 'PRIVATE_REPAIR_TEXT' not in caplog.text
|
||||
assert PackageValidator().validate(second.package).valid
|
||||
|
||||
|
||||
def test_another_invalid_field_prevents_format_repair():
|
||||
c,call=setup_case(['id','eq',1]);call['outcome']['document']['next']['kind']='invented_action'
|
||||
assert make_repair(c,call,'source-call.json') is None
|
||||
|
||||
|
||||
def test_ineligible_saved_plan_blocks_without_another_provider_call(tool_case):
|
||||
adapter=RepairAdapter();worker=loop(tool_case,adapter)
|
||||
first=asyncio.run(worker.step(*CASE));assert first.status=='REJECTED'
|
||||
s=state(first)
|
||||
with PackageReader(first.package) as r:c=json.loads(r.read_bytes(s['active']['context_ref']))
|
||||
plan=c['response_repair'];plan['document']['next']['arguments']['filters']=['id','eq',1]
|
||||
plan['document_sha256']=sha(plan['document'])
|
||||
assert plan['path']==['next','arguments','filters',0] # now points to a scalar
|
||||
s['active']['context_ref']=s['active']['context_ref'].replace('.json','-ineligible.json')
|
||||
worker._save(s,context=c)
|
||||
blocked=asyncio.run(worker.step(*CASE))
|
||||
assert blocked.status=='BLOCKED' and blocked.code=='LLM_REPAIR_INELIGIBLE'
|
||||
assert len(adapter.requests)==1
|
||||
assert not BudgetManager(tool_case[0]).snapshot(*CASE).state['accounts']['H1']['actions']
|
||||
|
||||
|
||||
def test_oversized_rejected_patch_is_not_retained(tool_case):
|
||||
class Adapter(RepairAdapter):
|
||||
async def generate(self,config,request,**kwargs):
|
||||
if request['prompt_version']==REPAIR_PROMPT:
|
||||
return ProviderResponse({'replacement':{'column':'id','operator':'eq','value':'x'*70000}},'fixture',100,20)
|
||||
return await super().generate(config,request,**kwargs)
|
||||
worker=loop(tool_case,Adapter());assert asyncio.run(worker.step(*CASE)).status=='REJECTED'
|
||||
second=asyncio.run(worker.step(*CASE));assert second.status=='REJECTED'
|
||||
with PackageReader(second.package) as r:
|
||||
call=json.loads(r.read_bytes(state(second)['hypotheses']['H1']['notes'][-1]['llm_ref']))
|
||||
assert 'rejected_repair' not in call['outcome']
|
||||
assert PackageValidator().validate(second.package).valid
|
||||
Reference in new issue
Block a user