diff --git a/docs/assessment-scope-recovery.md b/docs/assessment-scope-recovery.md new file mode 100644 index 0000000..69b5da2 --- /dev/null +++ b/docs/assessment-scope-recovery.md @@ -0,0 +1,56 @@ +# Budgetbewertung nach erneutem Belegabruf + +Bug17 wurde an Generation 103 blockiert, weil drei Antworten nach Recall eine +Bewertung enthielten, obwohl `previous_result` null war. Die nächste Source-Abfrage +war plausibel und vollständig. `assessment` betrifft ausschließlich die einmalige +Budgetabrechnung eines neuen Rechercheergebnisses, nicht eine erneute Bewertung +alten Inhalts. Recall selbst erzeugt keinen neuen abrechenbaren Auftrag. + +## Kontextabhängiger Vertrag + +Neue begrenzte Untersuchungskontexte enthalten `assessment_scope_version: 1`. +Damit erlaubt ihr Antwortschema bei null `previous_result` nur null `assessment`, +bei einem offenen Ergebnis dagegen nur ein vollständiges Bewertungsobjekt. +Das optionale Kontextfeld erhält bestehende gespeicherte Anfragen unverändert. +Bereits eingefrorene Kontexte werden nicht nachträglich umgeschrieben; die +Controllerprüfung und neue Rückmeldungen gelten trotzdem auch bei ihnen. +Fehlerhinweise nennen `/assessment` und erklären das erforderliche Verhalten. + +## Eng begrenzte Übernahme + +Bei aktivierter Antwortreparatur und begrenztem Fenster v2 kann ein normaler +`investigate_one_step` mit ausdrücklich null `previous_result` eine überflüssige +Budgetbewertung neutralisieren. Voraussetzung sind: + +- Ausschließlich `assessment`, `observation`, `next` an der Antwortwurzel. +- Vollständige Bewertung mit Integer-Score 0–100 und nicht leerem Grund. +- Keine zusätzlichen Belegfelder und keine unbekannten E-Referenzen im Grund. +- Eindeutige Aktion `tool`/`project.query` mit einem bekannten lesenden Query-Kind. +- Die Antwort besteht nach Setzen von `assessment: null` das Antwortschema. + +Beobachtung und Auftrag bleiben unverändert. Original und Regel +`non_applicable_budget_assessment:/assessment` werden im Aufruf gespeichert; +die entfernte Bewertung wird weder als Erkenntnis noch als Gutschrift übernommen. +Anschließend gelten weiterhin sämtliche Controller-, Beleg-, Projekt- und +Budgetprüfungen vor der Ausführung. Unbekannte Aktionen, fehlerhafte Scores, +zusätzliche Findings, unklare Belegangaben und ausstehende neue Ergebnisse fallen +nicht unter diese Reparatur. Fehlende Bewertungen werden nicht erfunden. + +## Nachweis + +`test-results/token-usage/bug17-assessment-recovery-offline-20261020.json` +protokolliert die Übernahme aller drei Originalantworten gegen die exakten +ursprünglichen Kontexte und die konfigurierte Source-Prüfung. Keine Ausführung, +keine Budgetänderung, keine neuen bezahlten API-Anfragen. + +Tests prüfen kontextabhängige Schemas, historische Kontextkompatibilität, +Originalantworten, Idempotenz, Audit, Ausschluss mehrdeutiger Fälle und einen +Controllerdurchlauf nach Recall: Ein gültiger Rechercheauftrag kostet genau seine +regulären Einheiten, ohne zusätzliche Rückerstattung für den bereits bekannten +Beleg und ohne zusätzlichen LLM-Korrekturaufruf. + +Prüfergebnis: 212 Tests der breiteren Prüfung bestanden (191,46 Sekunden). +Die gezielten Tests wurden nach der letzten Präzisierung erneut ausgeführt: +24 bestanden (6,48 Sekunden); die Mengen überlappen. `git diff --check` ohne Befund. +Der bestehende Fall ist ohne Budget-/Artefaktänderung von Generation 103 auf 104 +manuell wieder freigegeben. Keine kostenpflichtigen API-Aufrufe. diff --git a/docs/test-results/token-usage/bug17-assessment-recovery-offline-20261020.json b/docs/test-results/token-usage/bug17-assessment-recovery-offline-20261020.json new file mode 100644 index 0000000..6aca1e4 --- /dev/null +++ b/docs/test-results/token-usage/bug17-assessment-recovery-offline-20261020.json @@ -0,0 +1,63 @@ +{ + "format": "viano-offline-recovery-regression", + "date": "2026-10-20", + "source_run": "bug17-bounded-full-20261020", + "source_generation": 103, + "cases": [ + { + "call_ref": "llm/operations/8f58f73379896f4edda417f564668be151dea6f1dd75d1872c8aeb64d0cef5ce/round-0001.json", + "normalization_rules": [ + "non_applicable_budget_assessment:/assessment" + ], + "query_preserved": true, + "observation_preserved": true, + "full_controller_validation": "passed" + }, + { + "call_ref": "llm/operations/6ccfd3edde468aa5c42f40ec7c75c2117ae13bf79c3eddd91eb4335ab02d4482/round-0001.json", + "normalization_rules": [ + "non_applicable_budget_assessment:/assessment" + ], + "query_preserved": true, + "observation_preserved": true, + "full_controller_validation": "passed" + }, + { + "call_ref": "llm/operations/4115b3b2b3e209ed778e9ec0c51f6e5ce3a4249e00f87c89aeeedead0841f6a2/round-0001.json", + "normalization_rules": [ + "non_applicable_budget_assessment:/assessment" + ], + "query_preserved": true, + "observation_preserved": true, + "full_controller_validation": "passed" + } + ], + "budgets_unchanged": true, + "paid_api_calls": 0, + "scope": "Exact saved request and context; recovery followed by full controller validation including configured project source validation. No action execution or result accounting.", + "tests": { + "broad_suite_passed": 212, + "broad_elapsed_seconds": 191.46, + "final_targeted_passed": 24, + "final_targeted_elapsed_seconds": 6.48, + "note": "Targeted suite rerun after tightening query-kind whitelist and pending-result feedback. Counts overlap; do not add." + }, + "manual_resume": { + "from_generation": 103, + "to_generation": 104, + "state": "INVESTIGATING_HYPOTHESES", + "unchanged_artifact_sha256": { + "investigation_state": "1c71126a1460f71c51a1d2866aab07b37797aaf835f6891f1df46529d2bfc476", + "budget_ledger": "68e74e2f64e0d0b43f002a8c6bfe7ba4d765a72a568012e7dd19a939f55f80eb", + "case_interpretation": "0fd3c2a1efe0f229592b3038b66db26e704f2fb1a03b7321ceb76450e3f98cb4", + "hypotheses": "30494a06ce373f4c497b3964524cee21bdf5350eac515f7a0022619879d125c8" + }, + "budgets_reset": false, + "invalid_responses_reset": false + }, + "source_sha256": { + "src\\viano_bug_analysis\\analysis\\investigation_lean_v20.py": "f58df59fdd423f515dd439dc90479c815918a42856e9c1cb2ae96f1c572a04f2", + "src\\viano_bug_analysis\\analysis\\investigator.py": "ee3db3cb9bfa9a5f6eedf255283a620a77a7d29ee950d4d6adac263e05400d03", + "src\\viano_bug_analysis\\llm\\response_recovery.py": "faabf4e34542375f9d881303bf4c95ba1f0af10e3fab7236b5786a2d99eb222f" + } +} diff --git a/src/viano_bug_analysis/analysis/investigation_contract.py b/src/viano_bug_analysis/analysis/investigation_contract.py index f06d1a8..2b4cfb4 100644 --- a/src/viano_bug_analysis/analysis/investigation_contract.py +++ b/src/viano_bug_analysis/analysis/investigation_contract.py @@ -98,6 +98,10 @@ def feedback(error, document): result = filter_feedback(repair_feedback(schema_feedback(error, include_keyword=True))) path = result.get('validation_path', '') action = document.get('next') if isinstance(document,dict) else None + if path == '/assessment' and error.schema.get('type') == 'null': + result['requirement'] = 'assessment must be null; no pending new research result exists, and recalled evidence cannot receive another budget refund' + elif path == '/assessment' and error.schema.get('type') == 'object': + result['requirement'] = 'provide assessment with integer semantic_score 0..100 and reason for the pending new previous_result' if path in {'/next','/next/kind'}: def permitted(rule): if not isinstance(rule,dict): return set() diff --git a/src/viano_bug_analysis/analysis/investigation_lean_v20.py b/src/viano_bug_analysis/analysis/investigation_lean_v20.py index 79652a1..2af8137 100644 --- a/src/viano_bug_analysis/analysis/investigation_lean_v20.py +++ b/src/viano_bug_analysis/analysis/investigation_lean_v20.py @@ -16,6 +16,12 @@ def legacy(context): def schema(context, *, strict=False): result = _schema(context, strict=strict) + if context.get('assessment_scope_version') == 1: + if context.get('previous_result') is None: + result['properties']['assessment'] = {'type':'null'} + else: + result['properties']['assessment'] = next(copy.deepcopy(v) for v in + result['properties']['assessment']['anyOf'] if v.get('type') == 'object') from .investigation_window import enabled, extend_recall if enabled(context): result = extend_recall(result) diff --git a/src/viano_bug_analysis/analysis/investigator.py b/src/viano_bug_analysis/analysis/investigator.py index c19bbac..0aa94a4 100644 --- a/src/viano_bug_analysis/analysis/investigator.py +++ b/src/viano_bug_analysis/analysis/investigator.py @@ -246,6 +246,9 @@ class Investigator: "interpretation":interpretation["interpretation"],"hypotheses":state["hypotheses"],"selected_hypothesis":hid, "budget":snapshot.state,"budget_policy":ledger["policy"],"previous_result":previous, "evidence":evidence,"policy":state["policy"],"tool_schema":self.router.request_schema()} + if state['policy'].get('prompt_window_version') == 2: + # Frozen per context: historical requests remain reconstructable. + context['assessment_scope_version'] = 1 if state.get("prompt_protocol") == 2: context["prompt_version"] = "investigation_project_v2" context["case_sources"] = interpretation_context["sources"] @@ -583,7 +586,11 @@ class Investigator: decision = normalize(context, decision) hid=state["active"]["hypothesis_id"];h=state["hypotheses"][hid];update=decision["update"];next_action=decision["next"] previous=context["previous_result"] - if (decision["assessment"] is None)!=(previous is None):raise InvestigationError("INVESTIGATION_ASSESSMENT_REQUIRED") + if (decision["assessment"] is None)!=(previous is None): + raise InvestigationError("INVESTIGATION_ASSESSMENT_REQUIRED", feedback={ + 'validation_path':'/assessment','validation_fields':['assessment'], + 'requirement':('assessment must be null: previous_result is null; recalled evidence is already accounted for and cannot receive another budget refund' + if previous is None else 'provide assessment with integer semantic_score 0..100 and reason for the pending previous_result')}) evidence=context["evidence"] from .confidence_review import effective_confidence, validate_review if context.get('prompt_version') in {'investigation_project_v12','investigation_project_v13','investigation_project_v14','investigation_project_v15','investigation_project_v16','investigation_project_v17','investigation_project_v18'}: diff --git a/src/viano_bug_analysis/llm/response_recovery.py b/src/viano_bug_analysis/llm/response_recovery.py index 450dc55..7b031ea 100644 --- a/src/viano_bug_analysis/llm/response_recovery.py +++ b/src/viano_bug_analysis/llm/response_recovery.py @@ -103,6 +103,37 @@ def normalize(document, *, request=None): and isinstance(filters[1],str) and filters[1] in {'eq','ne','lt','le','gt','ge','contains','in'}): args['filters']=[dict(zip(('column','operator','value'),filters))] rules.append('flat_filter:/next/arguments/filters') + # A recall is not a newly charged research action. Discard only a well-formed + # but non-applicable budget score, never causal findings, evidence references, + # action intent or missing required values. The original score stays audited. + if request and enabled(request) and request.get('output_schema',{}).get('x-viano-investigation-window') == 2: + try: + body = json.loads(request['messages'][1]['content']) + except (ValueError,KeyError,IndexError,TypeError): + body = None + assessment = result.get('assessment') + if (isinstance(body,dict) and body.get('task') == 'investigate_one_step' + and isinstance(body.get('input'),dict) and 'previous_result' in body['input'] + and body['input']['previous_result'] is None + and set(result) == {'assessment','observation','next'} + and isinstance(assessment,dict) and set(assessment) == {'semantic_score','reason'} + and type(assessment['semantic_score']) is int and 0 <= assessment['semantic_score'] <= 100 + and isinstance(assessment['reason'],str) and assessment['reason'].strip() + and len(assessment['reason']) <= 4000 + and isinstance(action,dict) and action.get('kind') == 'tool' + and action.get('operation') == 'project.query' + and isinstance(args,dict) and args.get('kind') in { + 'source.files','source.read','source.search','source.symbol','source.registration','source.schema','source.context', + 'database.tables','database.schema','database.rows','database.count','database.relations'}): + visible = set(body['input'].get('evidence',{})) + cited = set(re.findall(r'\bE[0-9]{6,}\b',assessment['reason'])) + if cited <= visible: + from jsonschema import Draft202012Validator + candidate = copy.deepcopy(result) + candidate['assessment'] = None + if Draft202012Validator(request['output_schema']).is_valid(candidate): + result = candidate + rules.append('non_applicable_budget_assessment:/assessment') if len(rules)>16:return original,None return result,({'version':1,'original_document':original,'rules':rules} if rules else None) diff --git a/src/viano_bug_analysis/schemas/investigation-context-v1.schema.json b/src/viano_bug_analysis/schemas/investigation-context-v1.schema.json index ca5ce58..b2c8b83 100644 --- a/src/viano_bug_analysis/schemas/investigation-context-v1.schema.json +++ b/src/viano_bug_analysis/schemas/investigation-context-v1.schema.json @@ -901,6 +901,9 @@ "type": "integer", "minimum": 0 } + }, + "assessment_scope_version": { + "const": 1 } } } diff --git a/tests/test_assessment_scope.py b/tests/test_assessment_scope.py new file mode 100644 index 0000000..b62f809 --- /dev/null +++ b/tests/test_assessment_scope.py @@ -0,0 +1,129 @@ +import copy +import json +from pathlib import Path + +import pytest +from jsonschema import Draft202012Validator + +from test_bounded_contract_v2 import context, loop_for +from test_case_builder import case_setup +from test_investigation_tools import tool_case +from test_investigation_startup import Adapter +from test_investigator import CASE, run, state +from test_project_research import rows +from viano_bug_analysis.analysis.investigation_prompt import build_prompt, output_schema +from viano_bug_analysis.analysis.investigation_contract import feedback +from viano_bug_analysis.budget import BudgetManager +from viano_bug_analysis.llm import ProviderResponse +from viano_bug_analysis.llm.response_recovery import normalize +from viano_bug_analysis.package import PackageReader, PackageValidator + +CORPUS = Path(__file__).parent/'fixtures/llm_rejections/20261020/bug17-recall-assessment' + + +@pytest.mark.parametrize('pending',[False,True]) +def test_versioned_schema_only_allows_applicable_assessment(pending): + c = context() + if not pending: c['previous_result'] = None + legacy_schema = output_schema(c,strict=True) + assert 'anyOf' in legacy_schema['properties']['assessment'] + c['assessment_scope_version'] = 1 + schema = output_schema(c,strict=True)['properties']['assessment'] + assert Draft202012Validator(schema).is_valid(None) == (not pending) + assert Draft202012Validator(schema).is_valid({'semantic_score':50,'reason':'New result.'}) == pending + + +@pytest.mark.parametrize('name',['8f58f733','6ccfd3ed','4115b3b2']) +def test_actual_rejections_keep_query_and_observation_without_budget_score(name): + folder = CORPUS/name + request = json.loads((folder/'request.json').read_text(encoding='utf-8')) + answer = json.loads((folder/'parsed-answer.json').read_text(encoding='utf-8')) + original = copy.deepcopy(answer) + result,audit = normalize(answer,request=request) + assert result['assessment'] is None + assert result['next'] == answer['next'] and result['observation'] == answer['observation'] + assert answer == original and audit['original_document'] == original + assert audit['rules'] == ['non_applicable_budget_assessment:/assessment'] + c = context();c['previous_result'] = None;c['assessment_scope_version'] = 1 + assert Draft202012Validator(output_schema(c,strict=True)).is_valid(result) + assert normalize(result,request=request) == (result,None) + + +@pytest.mark.parametrize('change', ['pending','missing_previous','unknown_evidence','unknown_action', + 'unknown_query','extra_evidence_refs','missing_score','bad_score','float_score','bool_score','empty_reason', + 'causal_findings','checkpoint','legacy_window','disabled','missing_assessment']) +def test_ambiguous_or_out_of_scope_scores_are_not_neutralized(change): + folder = CORPUS/'8f58f733' + request = json.loads((folder/'request.json').read_text(encoding='utf-8')) + answer = json.loads((folder/'parsed-answer.json').read_text(encoding='utf-8')) + body = json.loads(request['messages'][1]['content']) + if change == 'pending': body['input']['previous_result'] = {'evidence_id':'E000005'} + if change == 'missing_previous': body['input'].pop('previous_result') + if change == 'unknown_evidence': answer['assessment']['reason'] = 'E999999 proves it.' + if change == 'unknown_action': answer['next']['operation'] = 'shell.exec' + if change == 'unknown_query': answer['next']['arguments']['kind'] = 'invented.query' + if change == 'extra_evidence_refs': answer['assessment']['evidence_refs'] = ['E999999'] + if change == 'missing_score': answer['assessment'].pop('semantic_score') + if change == 'bad_score': answer['assessment']['semantic_score'] = 101 + if change == 'float_score': answer['assessment']['semantic_score'] = 1.0 + if change == 'bool_score': answer['assessment']['semantic_score'] = True + if change == 'empty_reason': answer['assessment']['reason'] = ' ' + if change == 'causal_findings': answer['findings'] = [{'text':'Unverified.'}] + if change == 'checkpoint': body['task'] = 'consolidate_investigation_session' + if change == 'legacy_window': request['output_schema']['x-viano-investigation-window'] = 1 + if change == 'disabled': request['output_schema'].pop('x-viano-response-recovery') + if change == 'missing_assessment': answer.pop('assessment') + request['messages'][1]['content'] = json.dumps(body) + result,audit = normalize(answer,request=request) + assert result == answer and audit is None + + +def test_null_assessment_error_has_actionable_feedback(): + c = context();c['assessment_scope_version'] = 1;c['previous_result'] = None + rule = output_schema(c,strict=True)['properties']['assessment'] + validator = Draft202012Validator({'properties':{'assessment':rule}}) + error = next(validator.iter_errors({'assessment':{'semantic_score':50,'reason':'Old evidence.'}})) + info = feedback(error,{'assessment':{'semantic_score':50,'reason':'Old evidence.'}}) + assert info['validation_path'] == '/assessment' + assert 'must be null' in info['requirement'] and 'refund' in info['requirement'] + + +def test_valid_query_after_recall_is_recovered_without_refund_or_extra_llm(tool_case): + class Read(Adapter): + async def generate(self,config,request,**kwargs): + body = json.loads(request['messages'][1]['content']) + if body['task'] != 'investigate_one_step': + return await super().generate(config,request,**kwargs) + self.calls.append(body) + assert request['output_schema']['properties']['assessment']['type'] == 'null' + if len(self.calls) == 5: + out = {'assessment':None,'observation':'Read existing evidence.', + 'next':{'kind':'recall','evidence_refs':['E000001'],'page':0}} + else: + out = {'assessment':{'semantic_score':90,'reason':'E000001 is useful old evidence.'}, + 'observation':'Query a bounded table.', + 'next':{'kind':'tool','operation':'project.query','arguments':rows()}} + return ProviderResponse(out,'fixture',100,50) + adapter = Read();loop = loop_for(tool_case,adapter) + run(loop,steps=5) + before = BudgetManager(tool_case[0]).snapshot(*CASE).state + result = run(loop,steps=1) + saved = state(result) + assert len(adapter.calls) == 6 and saved['hypotheses']['H1']['invalid_responses'] == 0 + after = BudgetManager(tool_case[0]).snapshot(*CASE).state + assert after['accounts']['H1']['rewarded'] == before['accounts']['H1']['rewarded'] + assert after['accounts']['H1']['spent'] == before['accounts']['H1']['spent'] + 10 + with PackageReader(result.package) as reader: + ref = saved['hypotheses']['H1']['notes'][-1]['llm_ref'] + call = json.loads(reader.read_bytes(ref)) + assert call['outcome']['document']['assessment'] is None + assert call['outcome']['normalization']['original_document']['assessment']['semantic_score'] == 90 + assert PackageValidator().validate(result.package).valid + + +def test_pending_result_feedback_requires_a_real_assessment(): + c = context();c['assessment_scope_version'] = 1 + rule = output_schema(c,strict=True)['properties']['assessment'] + error = next(Draft202012Validator({'properties':{'assessment':rule}}).iter_errors({'assessment':None})) + info = feedback(error,{'assessment':None}) + assert 'semantic_score' in info['requirement'] and 'pending new' in info['requirement']