From f5a87dfca37a3261e327d9944c98e4ab092b228c Mon Sep 17 00:00:00 2001 From: Nitsu45 Date: Sun, 18 Oct 2026 23:55:16 +0200 Subject: [PATCH] Constrain nested bootstrap findings with versioned native output shapes --- config/investigation.lean.toml | 3 + docs/bootstrap-schema-diagnostics.md | 35 +++++++++ .../analysis/investigation_config.py | 3 + .../analysis/investigation_prompt.py | 3 +- .../analysis/investigator.py | 10 ++- .../llm/cache_measurement.py | 2 +- src/viano_bug_analysis/llm/gateway.py | 2 +- src/viano_bug_analysis/llm/gemini.py | 21 +++++- src/viano_bug_analysis/llm/stable_context.py | 2 +- .../investigation-context-v1.schema.json | 10 +++ .../investigation-state-v1.schema.json | 7 ++ tests/test_bootstrap_nested_shape.py | 71 +++++++++++++++++++ tests/test_bootstrap_schema_diagnostics.py | 2 +- tests/test_parameter_and_bootstrap_guard.py | 14 ++-- 14 files changed, 173 insertions(+), 12 deletions(-) create mode 100644 tests/test_bootstrap_nested_shape.py diff --git a/config/investigation.lean.toml b/config/investigation.lean.toml index 8ed6ca5..98a55b3 100644 --- a/config/investigation.lean.toml +++ b/config/investigation.lean.toml @@ -4,3 +4,6 @@ lean_protocol = 2 # Temporary experiment: no periodic consolidation. lean_checkpoint_steps = 0 bootstrap_json_framing = true + +# Versioned native object shapes for nonempty bootstrap findings. +bootstrap_framing_version = 2 diff --git a/docs/bootstrap-schema-diagnostics.md b/docs/bootstrap-schema-diagnostics.md index 69035fa..6f8ad1f 100644 --- a/docs/bootstrap-schema-diagnostics.md +++ b/docs/bootstrap-schema-diagnostics.md @@ -50,3 +50,38 @@ Typangaben erhalten keinen zusätzlichen Prompttext. Frühere Prompts bleiben so reproduzierbar. Die lokal simulierte Paketwiederaufnahme prüft denselben Fehlerpfad wie der echte Lauf und anschließend einen korrigierten Aufruf. Dies ersetzt keinen Erfolgsnachweis mit dem echten Modell. + +## Nativer verschachtelter Vertrag v22 + +Neue Untersuchungen verwenden bei aktiviertem Bootstrap-Framing standardmäßig +`bootstrap_framing_version = 2`, gespeichert in der Untersuchungspolicy. Der daraus +gebildete Kontext trägt denselben Versionsmarker; der Aufruf heißt +`investigation_bootstrap_v22`. Die Konfiguration im schlanken Testprofil setzt +diese Version ausdrücklich. + +Für Gemini im Modus `json_object` erhält `responseJsonSchema` jetzt auch die +verschachtelten Typen und Pflichtfelder von `assessment` und `update`, insbesondere +`working_brief.findings.items`. Jedes Belegelement muss ein Objekt mit `id`, +`direction`, `text`, `scope` und `evidence_refs` sein. Referenzen müssen als Liste +von Strings vorliegen. Auch `retired` und `next_step` behalten ihre Struktur. +Die Projektion wird aus dem lokalen Ausgabeschema erzeugt; sie enthält keine +zusätzliche fachliche Zusammenfassung und verändert das Schema nicht. + +Der native Vertrag ist bewusst eine Strukturprüfung, ohne die komplexen +Werkzeugalternativen von `next`. Wertebereiche, ID-Muster, zulässige Aktionen, +Provenienz und fachliche Belegprüfung bleiben Aufgaben der vollständigen lokalen +Validierung. Das vollständige Schema bleibt im Cachepräfix enthalten. Explizites +Caching, Cache-Messung und sichere Fehlerdiagnosen unterstützen v22. + +Version 1 und alte Policies ohne Versionsfeld behalten v21 und dessen äußeren +Vertrag. Gespeicherte Kontexte werden nicht migriert. Eine explizite abweichende +Policy wird abgelehnt. Für den nächsten Test muss daher ein Checkpoint vor der +Initialisierung der Untersuchung oder ein neues Eingabepaket verwendet werden; +der zuvor vorbereitete H3-Checkpoint würde weiterhin v21 nutzen. + +Lokale Transporttests prüfen den tatsächlich gesendeten Vertrag, nicht nur den +Prompt: String-, Array-, Null-, Zahlen- und Bool-Einträge als Belege werden +abgelehnt; ausgefüllte Objektbelege sind zulässig. Weitere Prüfungen decken +verschachtelte Feldtypen, Cache-Erzeugung/-Wiederverwendung sowie Paketwiederaufnahme +mit beiden Versionen ab. Ob der Anbieter den neuen Vertrag im echten API-Aufruf +akzeptiert und zuverlässig befolgt, muss weiterhin im Nutzer-Test geprüft werden. diff --git a/src/viano_bug_analysis/analysis/investigation_config.py b/src/viano_bug_analysis/analysis/investigation_config.py index fb364e7..5fa8e5f 100644 --- a/src/viano_bug_analysis/analysis/investigation_config.py +++ b/src/viano_bug_analysis/analysis/investigation_config.py @@ -24,6 +24,7 @@ class InvestigationConfig: lean_checkpoint_steps: int = 4 lean_protocol: int = 1 bootstrap_json_framing: bool = False + bootstrap_framing_version: int = 2 def __post_init__(self): if any(type(value) is not int or value <= 0 for name,value in asdict(self).items() if name not in {'lean_loop','lean_checkpoint_steps','bootstrap_json_framing'}): @@ -33,6 +34,8 @@ class InvestigationConfig: raise InvestigationError('INVESTIGATION_CONFIG') if type(self.lean_loop) is not bool or type(self.bootstrap_json_framing) is not bool: raise InvestigationError('INVESTIGATION_CONFIG') + if self.bootstrap_framing_version not in {1,2}: + raise InvestigationError('INVESTIGATION_CONFIG') @classmethod def load(cls, path): diff --git a/src/viano_bug_analysis/analysis/investigation_prompt.py b/src/viano_bug_analysis/analysis/investigation_prompt.py index 249bd12..fa16504 100644 --- a/src/viano_bug_analysis/analysis/investigation_prompt.py +++ b/src/viano_bug_analysis/analysis/investigation_prompt.py @@ -150,7 +150,8 @@ def _build_prompt(context): from .investigation_plan import framed_view visible = framed_view(context,visible) task = 'advance_investigation_goal' - return {"prompt_version":"investigation_bootstrap_v21" if context.get("bootstrap_json_framing") else version,"privacy":"configured","output_schema":schema, + bootstrap_version='investigation_bootstrap_v22' if context.get('bootstrap_framing_version')==2 else 'investigation_bootstrap_v21' + return {"prompt_version":bootstrap_version if context.get("bootstrap_json_framing") else version,"privacy":"configured","output_schema":schema, "messages":[{"role":"system","content":files("viano_bug_analysis.llm.prompts").joinpath(prompt_file+".txt").read_text(encoding="utf-8")}, {"role":"user","content":encode({"task":task,"input":visible,"output_schema":schema})}]} diff --git a/src/viano_bug_analysis/analysis/investigator.py b/src/viano_bug_analysis/analysis/investigator.py index 66f835a..44c4d38 100644 --- a/src/viano_bug_analysis/analysis/investigator.py +++ b/src/viano_bug_analysis/analysis/investigator.py @@ -291,7 +291,10 @@ class Investigator: if state['policy'].get('lean_protocol',1)==1 or state['hypotheses'][hid]['iterations']>0: context['lean_session']=copy.deepcopy(state['hypotheses'][hid].get('lean_session',{'turns':[],'checkpoint':False})) if state['policy'].get('lean_protocol',1)==2:context['lean_protocol']=2 - elif state['policy'].get('bootstrap_json_framing'):context['bootstrap_json_framing']=True + elif state['policy'].get('bootstrap_json_framing'): + context['bootstrap_json_framing']=True + if state['policy'].get('bootstrap_framing_version',1)==2: + context['bootstrap_framing_version']=2 if len(json_bytes(context))>self.limits.max_context_bytes:raise InvestigationError("INVESTIGATION_CONTEXT_LIMIT") validate(context,"investigation-context") return context @@ -318,7 +321,10 @@ class Investigator: if state is None: with PackageReader(self._latest(case_id,package_id),self.store.limits) as reader:state=self._begin(reader) state,_=self._save(state) - if self.config_explicit and {**asdict(InvestigationConfig()),**state["policy"]}!=asdict(self.config):raise InvestigationError("INVESTIGATION_POLICY_CHANGED") + effective_policy={**asdict(InvestigationConfig()),**state['policy']} + if state['policy'].get('bootstrap_json_framing') and 'bootstrap_framing_version' not in state['policy']: + effective_policy['bootstrap_framing_version']=1 + if self.config_explicit and effective_policy!=asdict(self.config):raise InvestigationError("INVESTIGATION_POLICY_CHANGED") if state["active"] and state["active"]["phase"]=="executing":return self._execute_plan(state,budget) if state["active"] is None: state.setdefault('early_stop_protocol',1) diff --git a/src/viano_bug_analysis/llm/cache_measurement.py b/src/viano_bug_analysis/llm/cache_measurement.py index ff902d2..20494e4 100644 --- a/src/viano_bug_analysis/llm/cache_measurement.py +++ b/src/viano_bug_analysis/llm/cache_measurement.py @@ -28,7 +28,7 @@ def measure_package(path): contexts[hashlib.sha256(json_bytes(original['messages'])).hexdigest()] = context elif name.startswith('llm/operations/') and name.endswith('/request.json'): req = json.loads(reader.read_bytes(name)) - if req.get('prompt_version','').startswith('investigation_project_'): + if req.get('prompt_version','').startswith(('investigation_project_','investigation_bootstrap_')): requests.append(req) samples, unmatched = [], 0 for req in requests: diff --git a/src/viano_bug_analysis/llm/gateway.py b/src/viano_bug_analysis/llm/gateway.py index 095ab3c..8cb48db 100644 --- a/src/viano_bug_analysis/llm/gateway.py +++ b/src/viano_bug_analysis/llm/gateway.py @@ -155,7 +155,7 @@ class LLMGateway: LOGGER.warning("LLM endpoint unavailable provider=%s code=%s", provider.id, code) except ValidationError as invalid: info={'validation_keyword':invalid.validator} - if request['prompt_version'] in {'investigation_bootstrap_v21','investigation_project_v14','investigation_project_v15','investigation_project_v16','investigation_project_v17','investigation_project_v18','investigation_project_v19','investigation_project_v20'}: + if request['prompt_version'] in {'investigation_bootstrap_v22','investigation_bootstrap_v21','investigation_project_v14','investigation_project_v15','investigation_project_v16','investigation_project_v17','investigation_project_v18','investigation_project_v19','investigation_project_v20'}: from .diagnostics import schema_feedback info.update(schema_feedback(invalid,include_keyword=True)) error = LLMError("LLM_OUTPUT_SCHEMA",diagnostics=info) diff --git a/src/viano_bug_analysis/llm/gemini.py b/src/viano_bug_analysis/llm/gemini.py index b099981..b119ff3 100644 --- a/src/viano_bug_analysis/llm/gemini.py +++ b/src/viano_bug_analysis/llm/gemini.py @@ -21,6 +21,23 @@ from .provider_base import ProviderResponse from .diagnostics import count, sanitize, parse_response_object +def bootstrap_shape(schema): + """Nested syntax contract only; full controller validation remains authoritative.""" + def shape(rule): + result={k:rule[k] for k in ('type','required','additionalProperties') if k in rule} + if 'properties' in rule:result['properties']={k:shape(v) for k,v in rule['properties'].items()} + if 'items' in rule:result['items']=shape(rule['items']) + if 'anyOf' in rule:result['anyOf']=[shape(v) for v in rule['anyOf']] + if 'const' in rule: + from .diagnostics import json_type + result['type']=json_type(rule['const']) + return result + return {'type':'object','additionalProperties':False,'required':['assessment','update','next'], + 'properties':{'assessment':shape(schema['properties']['assessment']), + 'update':shape(schema['properties']['update']), + 'next':{'type':'object','additionalProperties':True}}} + + class GeminiProvider: def __init__(self): self._cache = {} @@ -132,6 +149,8 @@ class GeminiProvider: 'update':{'type':'object','additionalProperties':True}, 'next':{'type':'object','additionalProperties':True}}, 'required':['assessment','update','next'],'additionalProperties':False} + if request['prompt_version']=='investigation_bootstrap_v22' and config.response_format=='json_object': + generation['responseJsonSchema']=bootstrap_shape(request['output_schema']) full = {'contents': prefix + contents, 'generationConfig': generation} if system['parts']: full['systemInstruction'] = system @@ -142,7 +161,7 @@ class GeminiProvider: if config.use_context_cache: status = 'below_minimum' if (len(json_bytes([system, prefix]).decode('utf-8')) >= config.cache_min_prefix_chars - or request['prompt_version']=='investigation_bootstrap_v21' or request['prompt_version'] in {'investigation_project_v15','investigation_project_v16','investigation_project_v17','investigation_project_v18','investigation_project_v19','investigation_project_v20'}): + or request['prompt_version'] in {'investigation_bootstrap_v21','investigation_bootstrap_v22'} or request['prompt_version'] in {'investigation_project_v15','investigation_project_v16','investigation_project_v17','investigation_project_v18','investigation_project_v19','investigation_project_v20'}): with self._lock: now = time.monotonic() self._cache = {k: v for k, v in self._cache.items() if v[1] > now} diff --git a/src/viano_bug_analysis/llm/stable_context.py b/src/viano_bug_analysis/llm/stable_context.py index 0b7a13f..2d1ae43 100644 --- a/src/viano_bug_analysis/llm/stable_context.py +++ b/src/viano_bug_analysis/llm/stable_context.py @@ -10,7 +10,7 @@ from viano_bug_analysis.lifecycle.storage import json_bytes VERSION = 'stable_case_v1' SUPPORTED_PROMPTS = {f'investigation_project_v{n}' for n in (4, 5, 6, 7, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20)} -SUPPORTED_PROMPTS.add('investigation_bootstrap_v21') +SUPPORTED_PROMPTS.update({'investigation_bootstrap_v21','investigation_bootstrap_v22'}) STABLE_FIELDS = frozenset({'format', 'format_version', 'prompt_version', 'case', 'sources', 'interpretation', 'budget_policy', 'policy', 'tool_schema', 'unread_sources', 'unread_inventory_scope'}) diff --git a/src/viano_bug_analysis/schemas/investigation-context-v1.schema.json b/src/viano_bug_analysis/schemas/investigation-context-v1.schema.json index f775013..2e105a9 100644 --- a/src/viano_bug_analysis/schemas/investigation-context-v1.schema.json +++ b/src/viano_bug_analysis/schemas/investigation-context-v1.schema.json @@ -110,6 +110,13 @@ }, "bootstrap_json_framing": { "type": "boolean" + }, + "bootstrap_framing_version": { + "type": "integer", + "enum": [ + 1, + 2 + ] } } }, @@ -802,6 +809,9 @@ }, "tool_guidance_version": { "const": 1 + }, + "bootstrap_framing_version": { + "const": 2 } } } diff --git a/src/viano_bug_analysis/schemas/investigation-state-v1.schema.json b/src/viano_bug_analysis/schemas/investigation-state-v1.schema.json index dc37161..e953f06 100644 --- a/src/viano_bug_analysis/schemas/investigation-state-v1.schema.json +++ b/src/viano_bug_analysis/schemas/investigation-state-v1.schema.json @@ -106,6 +106,13 @@ }, "bootstrap_json_framing": { "type": "boolean" + }, + "bootstrap_framing_version": { + "type": "integer", + "enum": [ + 1, + 2 + ] } } }, diff --git a/tests/test_bootstrap_nested_shape.py b/tests/test_bootstrap_nested_shape.py new file mode 100644 index 0000000..2bef708 --- /dev/null +++ b/tests/test_bootstrap_nested_shape.py @@ -0,0 +1,71 @@ +import copy +import json +from dataclasses import replace + +import pytest +from jsonschema import Draft202012Validator + +from test_parameter_and_bootstrap_guard import bootstrap +from test_lean_investigation_decision import reply +from test_llm_caching import config,native,scripted,CACHE +from viano_bug_analysis.analysis.investigation_prompt import build_prompt +from viano_bug_analysis.analysis import InvestigationConfig,InvestigationError +from viano_bug_analysis.llm.gemini import GeminiProvider,bootstrap_shape + + +def nested_request(): + from test_investigation_lean import lean_context + c=lean_context();c.pop('lean_session');c.update(bootstrap_json_framing=True,bootstrap_framing_version=2) + return build_prompt(c) + + +def nonempty_reply(): + r=reply();r['update']['next_step']=None + r['update']['working_brief']['findings']=[{'id':'B0001','direction':'neutral', + 'text':'Observed bounded result.','scope':'Examined snapshot.','evidence_refs':['E000001']}] + return r + + +@pytest.mark.parametrize('bad',['plain finding',['B0001','neutral'],None,42,True]) +def test_native_nested_shape_rejects_wrong_finding_type(bad): + r=nonempty_reply();request=nested_request();before=copy.deepcopy(request['output_schema']) + frame=bootstrap_shape(request['output_schema']);Draft202012Validator.check_schema(frame) + assert request['output_schema']==before + assert Draft202012Validator(frame).is_valid(r) + r['update']['working_brief']['findings'][0]=bad + assert not Draft202012Validator(frame).is_valid(r) + + +@pytest.mark.parametrize('field,value',[('evidence_refs','E000001'),('id',42),('text',[]),('scope',None)]) +def test_native_shape_covers_nested_finding_fields(field,value): + r=nonempty_reply();r['update']['working_brief']['findings'][0][field]=value + assert not Draft202012Validator(bootstrap_shape(nested_request()['output_schema'])).is_valid(r) + + +def test_new_native_contract_sent_with_cache_and_full_schema_retained(monkeypatch): + request=nested_request();assert request['prompt_version']=='investigation_bootstrap_v22' + response=native();response['candidates'][0]['content']['parts'][0]['text']=json.dumps(nonempty_reply()) + adapter=GeminiProvider();seen=scripted(adapter,monkeypatch,[CACHE,response,response]) + settings=replace(config(),cache_min_prefix_chars=10**9,response_format='json_object') + first=adapter._generate(settings,request,1000000);second=adapter._generate(settings,request,1000000) + assert first.document==second.document==nonempty_reply() + assert len(seen)==3 and seen[1][1]['cachedContent']==seen[2][1]['cachedContent'] + frame=seen[1][1]['generationConfig']['responseJsonSchema'] + item=frame['properties']['update']['properties']['working_brief']['properties']['findings']['items'] + assert item['type']=='object' and set(item['required'])=={'id','direction','text','scope','evidence_refs'} + assert 'oneOf' not in json.dumps(frame) and len(json.dumps(frame))<5000 + # The complete local/cached contract still constrains enum values and IDs. + bad=nonempty_reply();bad['update']['working_brief']['findings'][0]['direction']='invented' + assert not Draft202012Validator(request['output_schema']).is_valid(bad) + assert 'output_schema' in str(seen[0][1]['contents']) + + +def test_legacy_request_remains_v21_and_schema_identical(): + old=bootstrap();new=nested_request() + assert old['prompt_version']=='investigation_bootstrap_v21' + assert new['output_schema']==old['output_schema'] + + +@pytest.mark.parametrize('version',[0,3,True,'2']) +def test_invalid_bootstrap_version_rejected(version): + with pytest.raises(InvestigationError):InvestigationConfig(bootstrap_framing_version=version) diff --git a/tests/test_bootstrap_schema_diagnostics.py b/tests/test_bootstrap_schema_diagnostics.py index 4d148da..612a583 100644 --- a/tests/test_bootstrap_schema_diagnostics.py +++ b/tests/test_bootstrap_schema_diagnostics.py @@ -78,7 +78,7 @@ def test_bootstrap_schema_feedback_survives_package_resume_and_repairs_once(tool def __init__(self):super().__init__();self.sent=[] async def generate(self,config,request,**kwargs): self.sent.append(copy.deepcopy(request)) - assert request['prompt_version']=='investigation_bootstrap_v21' + assert request['prompt_version']=='investigation_bootstrap_v22' response=await super().generate(config,dict(request,prompt_version='investigation_project_v18'),**kwargs) if len(self.sent)==1: response.document['update']['working_brief']['findings']=['PRIVATE_RESPONSE_VALUE'] diff --git a/tests/test_parameter_and_bootstrap_guard.py b/tests/test_parameter_and_bootstrap_guard.py index d472725..d5349e6 100644 --- a/tests/test_parameter_and_bootstrap_guard.py +++ b/tests/test_parameter_and_bootstrap_guard.py @@ -90,7 +90,8 @@ def test_coding_contract_rejects_wrong_column_value_but_accepts_placeholder(): validate_request(payload,c) -def test_bootstrap_marker_is_persisted_and_replayable(tool_case,tmp_path): +@pytest.mark.parametrize('framing_version',[1,2]) +def test_bootstrap_marker_is_persisted_and_replayable(tool_case,tmp_path,framing_version): import asyncio from test_investigation_hybrid import HybridAdapter from test_case_interpreter import gateway @@ -103,14 +104,19 @@ def test_bootstrap_marker_is_persisted_and_replayable(tool_case,tmp_path): versions=[] async def generate(self,config,request,**kwargs): self.versions.append(request['prompt_version']) - if request['prompt_version']=='investigation_bootstrap_v21': + if request['prompt_version'] in {'investigation_bootstrap_v21','investigation_bootstrap_v22'}: request=dict(request,prompt_version='investigation_project_v18') return await super().generate(config,request,**kwargs) adapter=Framed();router=InvestigationToolRouter(project_research=service_for(tool_case)) loop=Investigator(tool_case[0],gateway(adapter),InvestigationConfig(lean_loop=True,lean_protocol=2, - lean_checkpoint_steps=0,bootstrap_json_framing=True),router=router) + lean_checkpoint_steps=0,bootstrap_json_framing=True,bootstrap_framing_version=framing_version),router=router) first=run(loop,steps=3) - assert adapter.versions==['investigation_bootstrap_v21']*3 + expected='investigation_bootstrap_v21' if framing_version==1 else 'investigation_bootstrap_v22' + assert adapter.versions==[expected]*3 portable=CheckpointStore(Workspace(tmp_path/'portable'));portable.intake(first.package) result=run(Investigator(portable,gateway(adapter),router=router)) assert result.status=='COMPLETED' and adapter.versions[3]=='investigation_project_v20' + from viano_bug_analysis.llm.cache_measurement import measure_package + measurement=measure_package(result.package) + assert measurement['all_exactly_reconstructed'] and measurement['unmatched_requests']==0 + assert sum(s['historical_prompt']==expected for s in measurement['samples'])==3