From 7b7779b7690a82b4c94fdca7cd95380ede723c89 Mon Sep 17 00:00:00 2001 From: Nitsu45 Date: Tue, 20 Oct 2026 21:36:32 +0200 Subject: [PATCH] Align checkpoint recall validation and resume lean investigation --- docs/checkpoint-recall-fix.md | 38 ++++++ ...heckpoint-recall-offline-fix-20261020.json | 51 ++++++++ .../analysis/investigation_compact.py | 13 +- .../analysis/investigation_contract.py | 15 +++ .../analysis/investigation_lean_v20.py | 7 +- .../analysis/investigation_window.py | 11 ++ .../analysis/investigator.py | 2 +- tests/test_checkpoint_recall.py | 116 ++++++++++++++++++ 8 files changed, 246 insertions(+), 7 deletions(-) create mode 100644 docs/checkpoint-recall-fix.md create mode 100644 docs/test-results/token-usage/bug17-checkpoint-recall-offline-fix-20261020.json create mode 100644 tests/test_checkpoint_recall.py diff --git a/docs/checkpoint-recall-fix.md b/docs/checkpoint-recall-fix.md new file mode 100644 index 0000000..6ec3c11 --- /dev/null +++ b/docs/checkpoint-recall-fix.md @@ -0,0 +1,38 @@ +# Konsolidierung mit anschließendem Belegabruf + +Der Bug17-Lauf wurde an Generation 84 technisch blockiert. Entgegen der ersten +Auswertung war `recall` im äußeren begrenzten v20-Checkpoint-Vertrag bereits erlaubt. +Die zweite Prüfung der normalisierten kanonischen Antwort verwendete jedoch den +alten Vertrag ohne Recall-Aktion und verwarf alle drei Antworten an `/next`. +Die Erklärung in den bisherigen Testberichten bleibt als historischer Befund +stehen; dieser Nachtrag präzisiert die tatsächliche Ursache. + +Beide Prüfschichten verwenden jetzt dieselbe Aktion für begrenzte Checkpoints: +eine bekannte E-Referenz und eine nichtnegative Seitennummer. Unbekannte Belege +und nicht vorhandene Seiten werden weiter vom Controller abgelehnt. Die gesonderte +frühe Diagnosebewertung bleibt auf ihre erlaubten Aktionen beschränkt. + +Bei erfolgreicher Konsolidierung mit Recall wird unter Fenster-Version 2 der +aktuelle Arbeitsstand gespeichert, die Session konsolidiert und der Checkpoint +beendet. Die angeforderte Seite bleibt im Zustand erhalten; der nächste Aufruf ist +wieder `investigate_one_step`. Es erfolgt keine automatische Tool-Recherche und +keine Umdeutung der Aktionsabsicht. Bereits gespeicherte Antworten werden nicht +verändert. Budgetabrechnung, Gegenbelege, Hypothesenwahl und Zeichenobergrenze +bleiben erhalten. Fenster-Version 1 behält ihr bisheriges Sessionverhalten. + +Bei ungültigen Next-Aktionen nennt das Feedback jetzt `/next/kind` und die aus dem +maßgeblichen Schema abgeleiteten zulässigen Aktionen. Keine ungeprüften Nutzwerte +werden in den Hinweis übernommen. + +Der Offline-Bericht `test-results/token-usage/bug17-checkpoint-recall-offline-fix-20261020.json` +belegt, dass sämtliche drei kuratierten Antworten gegen ihre exakten ursprünglichen +Vollkontexte bestehen. Messages, Antwortschema und Prompt-Version der ursprünglichen +Anfragen bleiben identisch. Kein neuer API-Aufruf war dafür notwendig. + +Regressionstests prüfen Originalantworten, beide Prüfschichten, gespeicherten +Arbeitsstand, Rückkehr zur schlanken Schleife, portable Wiederaufnahme, unveränderte +Budgets beim reinen Recall sowie Ablehnung unbekannter Belegreferenzen und Seiten. + +Prüfergebnis: 89 lokale Tests bestanden (158,35 Sekunden). Der bestehende +Bug17-Fall wurde ohne Budget- oder Artefaktänderung von Generation 84 auf 85 +manuell wieder freigegeben. Keine kostenpflichtigen API-Aufrufe. diff --git a/docs/test-results/token-usage/bug17-checkpoint-recall-offline-fix-20261020.json b/docs/test-results/token-usage/bug17-checkpoint-recall-offline-fix-20261020.json new file mode 100644 index 0000000..c61eb45 --- /dev/null +++ b/docs/test-results/token-usage/bug17-checkpoint-recall-offline-fix-20261020.json @@ -0,0 +1,51 @@ +{ + "format": "viano-offline-regression", + "date": "2026-10-20", + "run": "bug17-bounded-full-20261020", + "source_generation": 84, + "scope": "All three rejected checkpoint replies validated unchanged against their exact saved full contexts; no tool execution, budget mutation or paid LLM request.", + "results": [ + { + "call_ref": "llm/operations/6849c974442865e5ca3e8f146ef4520c275f2e37a25fb7caa498f17089ed6dc1/round-0001.json", + "reply_sha256": "44b25b401d7019b6da566658147d28c93e7aeaf22644497200afdb7b3ceae386", + "full_saved_context_validation": "passed", + "request_messages_schema_version_unchanged": true + }, + { + "call_ref": "llm/operations/dfceb150480e764106de4d51501b6ed0b2daa32533888a9dabcc88114203726d/round-0001.json", + "reply_sha256": "7ab36e7d2604dc8c4537eef2516f2ed68cdeaa6864d1f4a98b5cd41d16933250", + "full_saved_context_validation": "passed", + "request_messages_schema_version_unchanged": true + }, + { + "call_ref": "llm/operations/a641cb5c51a47e1e2228d4a99c508446cfa14e6d4f1c8de3f639b6acb4851fb7/round-0001.json", + "reply_sha256": "5546ff27268c2a0a699280edd3b8d713f0387730499db12ea94e97051e4cdf03", + "full_saved_context_validation": "passed", + "request_messages_schema_version_unchanged": true + } + ], + "local_tests": { + "passed": 89, + "elapsed_seconds": 158.35, + "scope": "checkpoint recall, bounded v1/v2, lean loop, canonical decisions and startup" + }, + "source_sha256": { + "src\\viano_bug_analysis\\analysis\\investigation_compact.py": "2522b63dd5b4943213afa659b1531aea64231a7a4777f5113f5f8f17f0f7290f", + "src\\viano_bug_analysis\\analysis\\investigation_lean_v20.py": "f6373c0cf6c619129c127784c0ef088f14480f892c9817c12ce6a48e7ffc6d09", + "src\\viano_bug_analysis\\analysis\\investigator.py": "5993ad05bc8a971f167e7056ba66fc2b928f26c7299da744a832d1dd4069cabd" + }, + "manual_resume": { + "from_generation": 84, + "to_generation": 85, + "state": "INVESTIGATING_HYPOTHESES", + "unchanged_artifact_sha256": { + "investigation_state": "f469f7431a46880da4ee34ae8912d20977607665bc6985607903086718a48a68", + "budget_ledger": "c501e306fafc85bc63f443c72376d99f5e77311d3fb15cd8b7b3cedcca66282d", + "case_interpretation": "0fd3c2a1efe0f229592b3038b66db26e704f2fb1a03b7321ceb76450e3f98cb4", + "hypotheses": "30494a06ce373f4c497b3964524cee21bdf5350eac515f7a0022619879d125c8" + }, + "budgets_reset": false, + "invalid_responses_reset": false, + "paid_api_calls": 0 + } +} diff --git a/src/viano_bug_analysis/analysis/investigation_compact.py b/src/viano_bug_analysis/analysis/investigation_compact.py index 9da84e9..97fcb84 100644 --- a/src/viano_bug_analysis/analysis/investigation_compact.py +++ b/src/viano_bug_analysis/analysis/investigation_compact.py @@ -238,8 +238,19 @@ def validate_tool_shape(context,decision): if 'lean_session' in context: from .investigation_lean import schema from jsonschema import Draft202012Validator - error=next(Draft202012Validator(schema(context,strict=True)).iter_errors(decision),None) + contract = schema(context,strict=True) + from .investigation_window import enabled, checkpoint_recall + from .investigation_lean import checkpoint + if enabled(context) and checkpoint(context) and not preparing_review(context): + # Validate the canonical checkpoint wire with the same recall action + # already advertised by the bounded v20 model-facing contract. + contract = checkpoint_recall(contract) + error=next(Draft202012Validator(contract).iter_errors(decision),None) if error: + if context.get('policy',{}).get('prompt_window_version') == 2: + from .investigation_contract import feedback + from .investigation_config import InvestigationError + raise InvestigationError('INVESTIGATION_OUTPUT',feedback=feedback(error,decision)) from viano_bug_analysis.llm.diagnostics import schema_feedback from .investigation_config import InvestigationError from .investigation_lean import repair_feedback diff --git a/src/viano_bug_analysis/analysis/investigation_contract.py b/src/viano_bug_analysis/analysis/investigation_contract.py index 238936a..f06d1a8 100644 --- a/src/viano_bug_analysis/analysis/investigation_contract.py +++ b/src/viano_bug_analysis/analysis/investigation_contract.py @@ -98,6 +98,21 @@ def feedback(error, document): result = filter_feedback(repair_feedback(schema_feedback(error, include_keyword=True))) path = result.get('validation_path', '') action = document.get('next') if isinstance(document,dict) else None + if path in {'/next','/next/kind'}: + def permitted(rule): + if not isinstance(rule,dict): return set() + kind = rule.get('properties',{}).get('kind',{}) + names = {kind['const']} if isinstance(kind.get('const'),str) else set(kind.get('enum',[])) + for key in ('oneOf','anyOf'): + for branch in rule.get(key,[]): names.update(permitted(branch)) + return names + names = permitted(error.schema) + if path == '/next/kind': + names.update(error.schema.get('enum',[])) + if isinstance(error.schema.get('const'),str): names.add(error.schema['const']) + if names: + result.update(validation_path='/next/kind',validation_fields=['kind'], + requirement='choose one permitted action with its required fields: '+', '.join(sorted(names))) if isinstance(action,dict) and isinstance(action.get('kind'),dict): result.update(validation_path='/next/kind', validation_fields=['kind'], requirement='next.kind must be a permitted literal string, never a const/enum object; do not invent actions') diff --git a/src/viano_bug_analysis/analysis/investigation_lean_v20.py b/src/viano_bug_analysis/analysis/investigation_lean_v20.py index bc0714e..79652a1 100644 --- a/src/viano_bug_analysis/analysis/investigation_lean_v20.py +++ b/src/viano_bug_analysis/analysis/investigation_lean_v20.py @@ -20,11 +20,8 @@ def schema(context, *, strict=False): if enabled(context): result = extend_recall(result) if previous.checkpoint(context): - result['properties']['next'] = {'oneOf': [result['properties']['next'], { - 'type':'object','additionalProperties':False,'required':['kind','evidence_refs'], - 'properties':{'kind':{'const':'recall'},'page':{'type':'integer','minimum':0}, - 'evidence_refs':{'type':'array','minItems':1,'maxItems':1, - 'items':{'type':'string','pattern':'^E[0-9]{6,}$'}}}}]} + from .investigation_window import checkpoint_recall + result = checkpoint_recall(result) return result diff --git a/src/viano_bug_analysis/analysis/investigation_window.py b/src/viano_bug_analysis/analysis/investigation_window.py index 06c1189..6409461 100644 --- a/src/viano_bug_analysis/analysis/investigation_window.py +++ b/src/viano_bug_analysis/analysis/investigation_window.py @@ -127,6 +127,17 @@ def evidence_page(entry, index=0): 'fragment': raw[start:start + PAGE_CHARACTERS]} +def checkpoint_recall(schema): + """Same literal action for model-facing and canonical checkpoint validation.""" + result = copy.deepcopy(schema) + result['properties']['next'] = {'oneOf': [result['properties']['next'], { + 'type':'object','additionalProperties':False,'required':['kind','evidence_refs'], + 'properties':{'kind':{'const':'recall'},'page':{'type':'integer','minimum':0}, + 'evidence_refs':{'type':'array','minItems':1,'maxItems':1, + 'items':{'type':'string','pattern':'^E[0-9]{6,}$'}}}}]} + return result + + def extend_recall(schema): schema = copy.deepcopy(schema) def visit(value): diff --git a/src/viano_bug_analysis/analysis/investigator.py b/src/viano_bug_analysis/analysis/investigator.py index 46aaab5..c19bbac 100644 --- a/src/viano_bug_analysis/analysis/investigator.py +++ b/src/viano_bug_analysis/analysis/investigator.py @@ -872,7 +872,7 @@ class Investigator: if 'lean_session' in context: from .investigation_lean import checkpoint if checkpoint(context): - if kind != 'recall': + if kind != 'recall' or context['policy'].get('prompt_window_version') == 2: h['lean_session']={'turns':[],'checkpoint':False} if call['outcome']['document']['next']['kind']=='switch': state['lean_preferred_hypothesis']=call['outcome']['document']['next']['hypothesis_id'] diff --git a/tests/test_checkpoint_recall.py b/tests/test_checkpoint_recall.py new file mode 100644 index 0000000..21d4ef6 --- /dev/null +++ b/tests/test_checkpoint_recall.py @@ -0,0 +1,116 @@ +import copy +import json +from pathlib import Path + +import pytest + +from test_bounded_contract_v2 import context, loop_for +from test_case_builder import case_setup +from test_investigation_tools import tool_case +from test_investigation_startup import Adapter +from test_investigator import CASE, run, state +from viano_bug_analysis.analysis import Investigator, InvestigationError +from viano_bug_analysis.analysis.investigation_lean_v20 import validate_reply, normalize +from viano_bug_analysis.analysis.investigation_prompt import build_prompt +from viano_bug_analysis.budget import BudgetManager +from viano_bug_analysis.lifecycle import CheckpointStore, Workspace +from viano_bug_analysis.llm import ProviderResponse +from viano_bug_analysis.package import PackageReader, PackageValidator +from test_case_interpreter import gateway + + +CORPUS = Path(__file__).parent/'fixtures/llm_rejections/20261020/bug17-checkpoint-recall' + + +def checkpoint_context(): + c = context() + c['lean_session'] = {'checkpoint':True,'intent':'review','turns':[{ + 'evidence_id':None,'observation':'Read the next page.','llm_ref':'fixture', + 'read_ids':list(c['evidence'])}]} + return c + + +@pytest.mark.parametrize('name',['6849c974','dfceb150','a641cb5c']) +def test_actual_checkpoint_answers_pass_both_validation_layers(name): + c = checkpoint_context() + answer = json.loads((CORPUS/name/'parsed-answer.json').read_text(encoding='utf-8')) + before = copy.deepcopy(answer) + validate_reply(c,answer) + normalized = normalize(c,answer) + assert normalized['next'] == answer['next'] + assert answer == before + assert normalized['update']['working_brief']['findings'] + + +def test_invalid_action_feedback_names_allowed_checkpoint_actions(): + c = checkpoint_context() + answer = json.loads((CORPUS/'6849c974/parsed-answer.json').read_text(encoding='utf-8')) + answer['next'] = {'kind':'invented'} + with pytest.raises(InvestigationError) as error: + validate_reply(c,answer) + info = error.value.feedback + assert info['validation_path'] == '/next/kind' + assert 'recall' in info['requirement'] and 'finish' in info['requirement'] + assert len(json.dumps(info)) < 500 + + +class RecallCheckpoint(Adapter): + async def generate(self,config,request,**kwargs): + body = json.loads(request['messages'][1]['content']) + if body['task'] == 'investigate_one_step': + self.calls.append(body) + return ProviderResponse({'assessment':None,'observation':'Consolidate before reading another page.', + 'next':{'kind':'checkpoint','intent':'review'}},'fixture',100,50) + if body['task'] == 'consolidate_investigation_session': + self.calls.append(body) + return ProviderResponse({'assessment':None,'confidence':'plausible','findings':[], + 'uncertainties':['More pages needed.'],'next_question':'Read the next evidence page.', + 'retire':[],'next':{'kind':'recall','evidence_refs':['E000001'],'page':0}},'fixture',100,50) + return await super().generate(config,request,**kwargs) + + +def test_checkpoint_recall_persists_brief_then_returns_to_short_loop(tool_case,tmp_path): + adapter = RecallCheckpoint() + loop = loop_for(tool_case,adapter) + run(loop,steps=5) + before = BudgetManager(tool_case[0]).snapshot(*CASE).state + result = run(loop,steps=1) + saved = state(result) + assert result.status != 'BLOCKED' + assert saved['hypotheses']['H1']['lean_session'] == {'turns':[],'checkpoint':False} + assert saved['recalled_pages']['H1'] == {'E000001':0} + assert saved['hypotheses']['H1']['working_brief_ref'] + assert BudgetManager(tool_case[0]).snapshot(*CASE).state == before + assert PackageValidator().validate(result.package).valid + recipient = CheckpointStore(Workspace(tmp_path/'recipient')) + recipient.intake(result.package) + restored = Investigator(recipient,gateway(adapter),router=loop.router) + run(restored,steps=1) + assert adapter.calls[-1]['task'] == 'investigate_one_step' + assert 'E000001' in adapter.calls[-1]['input']['evidence'] + + +@pytest.mark.parametrize('change,code',[('unknown','INVESTIGATION_EVIDENCE_UNKNOWN'),('page','INVESTIGATION_EVIDENCE_PAGE')]) +def test_checkpoint_recall_cannot_read_unknown_evidence_or_page(tool_case,change,code): + class Invalid(RecallCheckpoint): + async def generate(self,config,request,**kwargs): + response = await super().generate(config,request,**kwargs) + if json.loads(request['messages'][1]['content'])['task'] == 'consolidate_investigation_session': + response.document['next']['evidence_refs'] = ['E999999'] if change == 'unknown' else ['E000001'] + response.document['next']['page'] = 99999 if change == 'page' else 0 + return response + loop = loop_for(tool_case,Invalid()) + run(loop,steps=5) + before = BudgetManager(tool_case[0]).snapshot(*CASE).state + result = run(loop,steps=1) + assert state(result)['hypotheses']['H1']['notes'][-1]['code'] == code + assert BudgetManager(tool_case[0]).snapshot(*CASE).state == before + assert 'H1' not in state(result).get('recalled_pages',{}) + + +def test_early_diagnosis_review_still_forbids_recall(): + c = checkpoint_context() + c['early_review'] = {'request':{'status':'preparing'}} + answer = json.loads((CORPUS/'6849c974/parsed-answer.json').read_text(encoding='utf-8')) + with pytest.raises(InvestigationError): + validate_reply(c,answer)