Constrain nested bootstrap findings with versioned native output shapes

This commit is contained in:
Nico committed 2026-10-18 23:55:16 +02:00
1 parent 67843454e4
commit f5a87dfca3
14 files changed
+173 -12

No files matched your search

+3
View File
@@ -4,3 +4,6 @@ lean_protocol = 2
# Temporary experiment: no periodic consolidation.
lean_checkpoint_steps = 0
bootstrap_json_framing = true
# Versioned native object shapes for nonempty bootstrap findings.
bootstrap_framing_version = 2
+35
View File
@@ -50,3 +50,38 @@ Typangaben erhalten keinen zusätzlichen Prompttext. Frühere Prompts bleiben so
reproduzierbar. Die lokal simulierte Paketwiederaufnahme prüft denselben Fehlerpfad
wie der echte Lauf und anschließend einen korrigierten Aufruf. Dies ersetzt keinen
Erfolgsnachweis mit dem echten Modell.
## Nativer verschachtelter Vertrag v22
Neue Untersuchungen verwenden bei aktiviertem Bootstrap-Framing standardmäßig
`bootstrap_framing_version = 2`, gespeichert in der Untersuchungspolicy. Der daraus
gebildete Kontext trägt denselben Versionsmarker; der Aufruf heißt
`investigation_bootstrap_v22`. Die Konfiguration im schlanken Testprofil setzt
diese Version ausdrücklich.
Für Gemini im Modus `json_object` erhält `responseJsonSchema` jetzt auch die
verschachtelten Typen und Pflichtfelder von `assessment` und `update`, insbesondere
`working_brief.findings.items`. Jedes Belegelement muss ein Objekt mit `id`,
`direction`, `text`, `scope` und `evidence_refs` sein. Referenzen müssen als Liste
von Strings vorliegen. Auch `retired` und `next_step` behalten ihre Struktur.
Die Projektion wird aus dem lokalen Ausgabeschema erzeugt; sie enthält keine
zusätzliche fachliche Zusammenfassung und verändert das Schema nicht.
Der native Vertrag ist bewusst eine Strukturprüfung, ohne die komplexen
Werkzeugalternativen von `next`. Wertebereiche, ID-Muster, zulässige Aktionen,
Provenienz und fachliche Belegprüfung bleiben Aufgaben der vollständigen lokalen
Validierung. Das vollständige Schema bleibt im Cachepräfix enthalten. Explizites
Caching, Cache-Messung und sichere Fehlerdiagnosen unterstützen v22.
Version 1 und alte Policies ohne Versionsfeld behalten v21 und dessen äußeren
Vertrag. Gespeicherte Kontexte werden nicht migriert. Eine explizite abweichende
Policy wird abgelehnt. Für den nächsten Test muss daher ein Checkpoint vor der
Initialisierung der Untersuchung oder ein neues Eingabepaket verwendet werden;
der zuvor vorbereitete H3-Checkpoint würde weiterhin v21 nutzen.
Lokale Transporttests prüfen den tatsächlich gesendeten Vertrag, nicht nur den
Prompt: String-, Array-, Null-, Zahlen- und Bool-Einträge als Belege werden
abgelehnt; ausgefüllte Objektbelege sind zulässig. Weitere Prüfungen decken
verschachtelte Feldtypen, Cache-Erzeugung/-Wiederverwendung sowie Paketwiederaufnahme
mit beiden Versionen ab. Ob der Anbieter den neuen Vertrag im echten API-Aufruf
akzeptiert und zuverlässig befolgt, muss weiterhin im Nutzer-Test geprüft werden.
@@ -24,6 +24,7 @@ class InvestigationConfig:
lean_checkpoint_steps: int = 4
lean_protocol: int = 1
bootstrap_json_framing: bool = False
bootstrap_framing_version: int = 2
def __post_init__(self):
if any(type(value) is not int or value <= 0 for name,value in asdict(self).items() if name not in {'lean_loop','lean_checkpoint_steps','bootstrap_json_framing'}):
@@ -33,6 +34,8 @@ class InvestigationConfig:
raise InvestigationError('INVESTIGATION_CONFIG')
if type(self.lean_loop) is not bool or type(self.bootstrap_json_framing) is not bool:
raise InvestigationError('INVESTIGATION_CONFIG')
if self.bootstrap_framing_version not in {1,2}:
raise InvestigationError('INVESTIGATION_CONFIG')
@classmethod
def load(cls, path):
@@ -150,7 +150,8 @@ def _build_prompt(context):
from .investigation_plan import framed_view
visible = framed_view(context,visible)
task = 'advance_investigation_goal'
return {"prompt_version":"investigation_bootstrap_v21" if context.get("bootstrap_json_framing") else version,"privacy":"configured","output_schema":schema,
bootstrap_version='investigation_bootstrap_v22' if context.get('bootstrap_framing_version')==2 else 'investigation_bootstrap_v21'
return {"prompt_version":bootstrap_version if context.get("bootstrap_json_framing") else version,"privacy":"configured","output_schema":schema,
"messages":[{"role":"system","content":files("viano_bug_analysis.llm.prompts").joinpath(prompt_file+".txt").read_text(encoding="utf-8")},
{"role":"user","content":encode({"task":task,"input":visible,"output_schema":schema})}]}
@@ -291,7 +291,10 @@ class Investigator:
if state['policy'].get('lean_protocol',1)==1 or state['hypotheses'][hid]['iterations']>0:
context['lean_session']=copy.deepcopy(state['hypotheses'][hid].get('lean_session',{'turns':[],'checkpoint':False}))
if state['policy'].get('lean_protocol',1)==2:context['lean_protocol']=2
elif state['policy'].get('bootstrap_json_framing'):context['bootstrap_json_framing']=True
elif state['policy'].get('bootstrap_json_framing'):
context['bootstrap_json_framing']=True
if state['policy'].get('bootstrap_framing_version',1)==2:
context['bootstrap_framing_version']=2
if len(json_bytes(context))>self.limits.max_context_bytes:raise InvestigationError("INVESTIGATION_CONTEXT_LIMIT")
validate(context,"investigation-context")
return context
@@ -318,7 +321,10 @@ class Investigator:
if state is None:
with PackageReader(self._latest(case_id,package_id),self.store.limits) as reader:state=self._begin(reader)
state,_=self._save(state)
if self.config_explicit and {**asdict(InvestigationConfig()),**state["policy"]}!=asdict(self.config):raise InvestigationError("INVESTIGATION_POLICY_CHANGED")
effective_policy={**asdict(InvestigationConfig()),**state['policy']}
if state['policy'].get('bootstrap_json_framing') and 'bootstrap_framing_version' not in state['policy']:
effective_policy['bootstrap_framing_version']=1
if self.config_explicit and effective_policy!=asdict(self.config):raise InvestigationError("INVESTIGATION_POLICY_CHANGED")
if state["active"] and state["active"]["phase"]=="executing":return self._execute_plan(state,budget)
if state["active"] is None:
state.setdefault('early_stop_protocol',1)
@@ -28,7 +28,7 @@ def measure_package(path):
contexts[hashlib.sha256(json_bytes(original['messages'])).hexdigest()] = context
elif name.startswith('llm/operations/') and name.endswith('/request.json'):
req = json.loads(reader.read_bytes(name))
if req.get('prompt_version','').startswith('investigation_project_'):
if req.get('prompt_version','').startswith(('investigation_project_','investigation_bootstrap_')):
requests.append(req)
samples, unmatched = [], 0
for req in requests:
+1 -1
View File
@@ -155,7 +155,7 @@ class LLMGateway:
LOGGER.warning("LLM endpoint unavailable provider=%s code=%s", provider.id, code)
except ValidationError as invalid:
info={'validation_keyword':invalid.validator}
if request['prompt_version'] in {'investigation_bootstrap_v21','investigation_project_v14','investigation_project_v15','investigation_project_v16','investigation_project_v17','investigation_project_v18','investigation_project_v19','investigation_project_v20'}:
if request['prompt_version'] in {'investigation_bootstrap_v22','investigation_bootstrap_v21','investigation_project_v14','investigation_project_v15','investigation_project_v16','investigation_project_v17','investigation_project_v18','investigation_project_v19','investigation_project_v20'}:
from .diagnostics import schema_feedback
info.update(schema_feedback(invalid,include_keyword=True))
error = LLMError("LLM_OUTPUT_SCHEMA",diagnostics=info)
+20 -1
View File
@@ -21,6 +21,23 @@ from .provider_base import ProviderResponse
from .diagnostics import count, sanitize, parse_response_object
def bootstrap_shape(schema):
"""Nested syntax contract only; full controller validation remains authoritative."""
def shape(rule):
result={k:rule[k] for k in ('type','required','additionalProperties') if k in rule}
if 'properties' in rule:result['properties']={k:shape(v) for k,v in rule['properties'].items()}
if 'items' in rule:result['items']=shape(rule['items'])
if 'anyOf' in rule:result['anyOf']=[shape(v) for v in rule['anyOf']]
if 'const' in rule:
from .diagnostics import json_type
result['type']=json_type(rule['const'])
return result
return {'type':'object','additionalProperties':False,'required':['assessment','update','next'],
'properties':{'assessment':shape(schema['properties']['assessment']),
'update':shape(schema['properties']['update']),
'next':{'type':'object','additionalProperties':True}}}
class GeminiProvider:
def __init__(self):
self._cache = {}
@@ -132,6 +149,8 @@ class GeminiProvider:
'update':{'type':'object','additionalProperties':True},
'next':{'type':'object','additionalProperties':True}},
'required':['assessment','update','next'],'additionalProperties':False}
if request['prompt_version']=='investigation_bootstrap_v22' and config.response_format=='json_object':
generation['responseJsonSchema']=bootstrap_shape(request['output_schema'])
full = {'contents': prefix + contents, 'generationConfig': generation}
if system['parts']:
full['systemInstruction'] = system
@@ -142,7 +161,7 @@ class GeminiProvider:
if config.use_context_cache:
status = 'below_minimum'
if (len(json_bytes([system, prefix]).decode('utf-8')) >= config.cache_min_prefix_chars
or request['prompt_version']=='investigation_bootstrap_v21' or request['prompt_version'] in {'investigation_project_v15','investigation_project_v16','investigation_project_v17','investigation_project_v18','investigation_project_v19','investigation_project_v20'}):
or request['prompt_version'] in {'investigation_bootstrap_v21','investigation_bootstrap_v22'} or request['prompt_version'] in {'investigation_project_v15','investigation_project_v16','investigation_project_v17','investigation_project_v18','investigation_project_v19','investigation_project_v20'}):
with self._lock:
now = time.monotonic()
self._cache = {k: v for k, v in self._cache.items() if v[1] > now}
+1 -1
View File
@@ -10,7 +10,7 @@ from viano_bug_analysis.lifecycle.storage import json_bytes
VERSION = 'stable_case_v1'
SUPPORTED_PROMPTS = {f'investigation_project_v{n}' for n in (4, 5, 6, 7, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20)}
SUPPORTED_PROMPTS.add('investigation_bootstrap_v21')
SUPPORTED_PROMPTS.update({'investigation_bootstrap_v21','investigation_bootstrap_v22'})
STABLE_FIELDS = frozenset({'format', 'format_version', 'prompt_version', 'case',
'sources', 'interpretation', 'budget_policy', 'policy', 'tool_schema',
'unread_sources', 'unread_inventory_scope'})
@@ -110,6 +110,13 @@
},
"bootstrap_json_framing": {
"type": "boolean"
},
"bootstrap_framing_version": {
"type": "integer",
"enum": [
1,
2
]
}
}
},
@@ -802,6 +809,9 @@
},
"tool_guidance_version": {
"const": 1
},
"bootstrap_framing_version": {
"const": 2
}
}
}
@@ -106,6 +106,13 @@
},
"bootstrap_json_framing": {
"type": "boolean"
},
"bootstrap_framing_version": {
"type": "integer",
"enum": [
1,
2
]
}
}
},
+71
View File
@@ -0,0 +1,71 @@
import copy
import json
from dataclasses import replace
import pytest
from jsonschema import Draft202012Validator
from test_parameter_and_bootstrap_guard import bootstrap
from test_lean_investigation_decision import reply
from test_llm_caching import config,native,scripted,CACHE
from viano_bug_analysis.analysis.investigation_prompt import build_prompt
from viano_bug_analysis.analysis import InvestigationConfig,InvestigationError
from viano_bug_analysis.llm.gemini import GeminiProvider,bootstrap_shape
def nested_request():
from test_investigation_lean import lean_context
c=lean_context();c.pop('lean_session');c.update(bootstrap_json_framing=True,bootstrap_framing_version=2)
return build_prompt(c)
def nonempty_reply():
r=reply();r['update']['next_step']=None
r['update']['working_brief']['findings']=[{'id':'B0001','direction':'neutral',
'text':'Observed bounded result.','scope':'Examined snapshot.','evidence_refs':['E000001']}]
return r
@pytest.mark.parametrize('bad',['plain finding',['B0001','neutral'],None,42,True])
def test_native_nested_shape_rejects_wrong_finding_type(bad):
r=nonempty_reply();request=nested_request();before=copy.deepcopy(request['output_schema'])
frame=bootstrap_shape(request['output_schema']);Draft202012Validator.check_schema(frame)
assert request['output_schema']==before
assert Draft202012Validator(frame).is_valid(r)
r['update']['working_brief']['findings'][0]=bad
assert not Draft202012Validator(frame).is_valid(r)
@pytest.mark.parametrize('field,value',[('evidence_refs','E000001'),('id',42),('text',[]),('scope',None)])
def test_native_shape_covers_nested_finding_fields(field,value):
r=nonempty_reply();r['update']['working_brief']['findings'][0][field]=value
assert not Draft202012Validator(bootstrap_shape(nested_request()['output_schema'])).is_valid(r)
def test_new_native_contract_sent_with_cache_and_full_schema_retained(monkeypatch):
request=nested_request();assert request['prompt_version']=='investigation_bootstrap_v22'
response=native();response['candidates'][0]['content']['parts'][0]['text']=json.dumps(nonempty_reply())
adapter=GeminiProvider();seen=scripted(adapter,monkeypatch,[CACHE,response,response])
settings=replace(config(),cache_min_prefix_chars=10**9,response_format='json_object')
first=adapter._generate(settings,request,1000000);second=adapter._generate(settings,request,1000000)
assert first.document==second.document==nonempty_reply()
assert len(seen)==3 and seen[1][1]['cachedContent']==seen[2][1]['cachedContent']
frame=seen[1][1]['generationConfig']['responseJsonSchema']
item=frame['properties']['update']['properties']['working_brief']['properties']['findings']['items']
assert item['type']=='object' and set(item['required'])=={'id','direction','text','scope','evidence_refs'}
assert 'oneOf' not in json.dumps(frame) and len(json.dumps(frame))<5000
# The complete local/cached contract still constrains enum values and IDs.
bad=nonempty_reply();bad['update']['working_brief']['findings'][0]['direction']='invented'
assert not Draft202012Validator(request['output_schema']).is_valid(bad)
assert 'output_schema' in str(seen[0][1]['contents'])
def test_legacy_request_remains_v21_and_schema_identical():
old=bootstrap();new=nested_request()
assert old['prompt_version']=='investigation_bootstrap_v21'
assert new['output_schema']==old['output_schema']
@pytest.mark.parametrize('version',[0,3,True,'2'])
def test_invalid_bootstrap_version_rejected(version):
with pytest.raises(InvestigationError):InvestigationConfig(bootstrap_framing_version=version)
+1 -1
View File
@@ -78,7 +78,7 @@ def test_bootstrap_schema_feedback_survives_package_resume_and_repairs_once(tool
def __init__(self):super().__init__();self.sent=[]
async def generate(self,config,request,**kwargs):
self.sent.append(copy.deepcopy(request))
assert request['prompt_version']=='investigation_bootstrap_v21'
assert request['prompt_version']=='investigation_bootstrap_v22'
response=await super().generate(config,dict(request,prompt_version='investigation_project_v18'),**kwargs)
if len(self.sent)==1:
response.document['update']['working_brief']['findings']=['PRIVATE_RESPONSE_VALUE']
+10 -4
View File
@@ -90,7 +90,8 @@ def test_coding_contract_rejects_wrong_column_value_but_accepts_placeholder():
validate_request(payload,c)
def test_bootstrap_marker_is_persisted_and_replayable(tool_case,tmp_path):
@pytest.mark.parametrize('framing_version',[1,2])
def test_bootstrap_marker_is_persisted_and_replayable(tool_case,tmp_path,framing_version):
import asyncio
from test_investigation_hybrid import HybridAdapter
from test_case_interpreter import gateway
@@ -103,14 +104,19 @@ def test_bootstrap_marker_is_persisted_and_replayable(tool_case,tmp_path):
versions=[]
async def generate(self,config,request,**kwargs):
self.versions.append(request['prompt_version'])
if request['prompt_version']=='investigation_bootstrap_v21':
if request['prompt_version'] in {'investigation_bootstrap_v21','investigation_bootstrap_v22'}:
request=dict(request,prompt_version='investigation_project_v18')
return await super().generate(config,request,**kwargs)
adapter=Framed();router=InvestigationToolRouter(project_research=service_for(tool_case))
loop=Investigator(tool_case[0],gateway(adapter),InvestigationConfig(lean_loop=True,lean_protocol=2,
lean_checkpoint_steps=0,bootstrap_json_framing=True),router=router)
lean_checkpoint_steps=0,bootstrap_json_framing=True,bootstrap_framing_version=framing_version),router=router)
first=run(loop,steps=3)
assert adapter.versions==['investigation_bootstrap_v21']*3
expected='investigation_bootstrap_v21' if framing_version==1 else 'investigation_bootstrap_v22'
assert adapter.versions==[expected]*3
portable=CheckpointStore(Workspace(tmp_path/'portable'));portable.intake(first.package)
result=run(Investigator(portable,gateway(adapter),router=router))
assert result.status=='COMPLETED' and adapter.versions[3]=='investigation_project_v20'
from viano_bug_analysis.llm.cache_measurement import measure_package
measurement=measure_package(result.package)
assert measurement['all_exactly_reconstructed'] and measurement['unmatched_requests']==0
assert sum(s['historical_prompt']==expected for s in measurement['samples'])==3