Constrain nested bootstrap findings with versioned native output shapes
This commit is contained in:
1 parent
67843454e4
commit
f5a87dfca3
14 files changed
+173
-12
No files matched your search
@@ -4,3 +4,6 @@ lean_protocol = 2
|
||||
# Temporary experiment: no periodic consolidation.
|
||||
lean_checkpoint_steps = 0
|
||||
bootstrap_json_framing = true
|
||||
|
||||
# Versioned native object shapes for nonempty bootstrap findings.
|
||||
bootstrap_framing_version = 2
|
||||
@@ -50,3 +50,38 @@ Typangaben erhalten keinen zusätzlichen Prompttext. Frühere Prompts bleiben so
|
||||
reproduzierbar. Die lokal simulierte Paketwiederaufnahme prüft denselben Fehlerpfad
|
||||
wie der echte Lauf und anschließend einen korrigierten Aufruf. Dies ersetzt keinen
|
||||
Erfolgsnachweis mit dem echten Modell.
|
||||
|
||||
## Nativer verschachtelter Vertrag v22
|
||||
|
||||
Neue Untersuchungen verwenden bei aktiviertem Bootstrap-Framing standardmäßig
|
||||
`bootstrap_framing_version = 2`, gespeichert in der Untersuchungspolicy. Der daraus
|
||||
gebildete Kontext trägt denselben Versionsmarker; der Aufruf heißt
|
||||
`investigation_bootstrap_v22`. Die Konfiguration im schlanken Testprofil setzt
|
||||
diese Version ausdrücklich.
|
||||
|
||||
Für Gemini im Modus `json_object` erhält `responseJsonSchema` jetzt auch die
|
||||
verschachtelten Typen und Pflichtfelder von `assessment` und `update`, insbesondere
|
||||
`working_brief.findings.items`. Jedes Belegelement muss ein Objekt mit `id`,
|
||||
`direction`, `text`, `scope` und `evidence_refs` sein. Referenzen müssen als Liste
|
||||
von Strings vorliegen. Auch `retired` und `next_step` behalten ihre Struktur.
|
||||
Die Projektion wird aus dem lokalen Ausgabeschema erzeugt; sie enthält keine
|
||||
zusätzliche fachliche Zusammenfassung und verändert das Schema nicht.
|
||||
|
||||
Der native Vertrag ist bewusst eine Strukturprüfung, ohne die komplexen
|
||||
Werkzeugalternativen von `next`. Wertebereiche, ID-Muster, zulässige Aktionen,
|
||||
Provenienz und fachliche Belegprüfung bleiben Aufgaben der vollständigen lokalen
|
||||
Validierung. Das vollständige Schema bleibt im Cachepräfix enthalten. Explizites
|
||||
Caching, Cache-Messung und sichere Fehlerdiagnosen unterstützen v22.
|
||||
|
||||
Version 1 und alte Policies ohne Versionsfeld behalten v21 und dessen äußeren
|
||||
Vertrag. Gespeicherte Kontexte werden nicht migriert. Eine explizite abweichende
|
||||
Policy wird abgelehnt. Für den nächsten Test muss daher ein Checkpoint vor der
|
||||
Initialisierung der Untersuchung oder ein neues Eingabepaket verwendet werden;
|
||||
der zuvor vorbereitete H3-Checkpoint würde weiterhin v21 nutzen.
|
||||
|
||||
Lokale Transporttests prüfen den tatsächlich gesendeten Vertrag, nicht nur den
|
||||
Prompt: String-, Array-, Null-, Zahlen- und Bool-Einträge als Belege werden
|
||||
abgelehnt; ausgefüllte Objektbelege sind zulässig. Weitere Prüfungen decken
|
||||
verschachtelte Feldtypen, Cache-Erzeugung/-Wiederverwendung sowie Paketwiederaufnahme
|
||||
mit beiden Versionen ab. Ob der Anbieter den neuen Vertrag im echten API-Aufruf
|
||||
akzeptiert und zuverlässig befolgt, muss weiterhin im Nutzer-Test geprüft werden.
|
||||
@@ -24,6 +24,7 @@ class InvestigationConfig:
|
||||
lean_checkpoint_steps: int = 4
|
||||
lean_protocol: int = 1
|
||||
bootstrap_json_framing: bool = False
|
||||
bootstrap_framing_version: int = 2
|
||||
|
||||
def __post_init__(self):
|
||||
if any(type(value) is not int or value <= 0 for name,value in asdict(self).items() if name not in {'lean_loop','lean_checkpoint_steps','bootstrap_json_framing'}):
|
||||
@@ -33,6 +34,8 @@ class InvestigationConfig:
|
||||
raise InvestigationError('INVESTIGATION_CONFIG')
|
||||
if type(self.lean_loop) is not bool or type(self.bootstrap_json_framing) is not bool:
|
||||
raise InvestigationError('INVESTIGATION_CONFIG')
|
||||
if self.bootstrap_framing_version not in {1,2}:
|
||||
raise InvestigationError('INVESTIGATION_CONFIG')
|
||||
|
||||
@classmethod
|
||||
def load(cls, path):
|
||||
|
||||
@@ -150,7 +150,8 @@ def _build_prompt(context):
|
||||
from .investigation_plan import framed_view
|
||||
visible = framed_view(context,visible)
|
||||
task = 'advance_investigation_goal'
|
||||
return {"prompt_version":"investigation_bootstrap_v21" if context.get("bootstrap_json_framing") else version,"privacy":"configured","output_schema":schema,
|
||||
bootstrap_version='investigation_bootstrap_v22' if context.get('bootstrap_framing_version')==2 else 'investigation_bootstrap_v21'
|
||||
return {"prompt_version":bootstrap_version if context.get("bootstrap_json_framing") else version,"privacy":"configured","output_schema":schema,
|
||||
"messages":[{"role":"system","content":files("viano_bug_analysis.llm.prompts").joinpath(prompt_file+".txt").read_text(encoding="utf-8")},
|
||||
{"role":"user","content":encode({"task":task,"input":visible,"output_schema":schema})}]}
|
||||
|
||||
|
||||
@@ -291,7 +291,10 @@ class Investigator:
|
||||
if state['policy'].get('lean_protocol',1)==1 or state['hypotheses'][hid]['iterations']>0:
|
||||
context['lean_session']=copy.deepcopy(state['hypotheses'][hid].get('lean_session',{'turns':[],'checkpoint':False}))
|
||||
if state['policy'].get('lean_protocol',1)==2:context['lean_protocol']=2
|
||||
elif state['policy'].get('bootstrap_json_framing'):context['bootstrap_json_framing']=True
|
||||
elif state['policy'].get('bootstrap_json_framing'):
|
||||
context['bootstrap_json_framing']=True
|
||||
if state['policy'].get('bootstrap_framing_version',1)==2:
|
||||
context['bootstrap_framing_version']=2
|
||||
if len(json_bytes(context))>self.limits.max_context_bytes:raise InvestigationError("INVESTIGATION_CONTEXT_LIMIT")
|
||||
validate(context,"investigation-context")
|
||||
return context
|
||||
@@ -318,7 +321,10 @@ class Investigator:
|
||||
if state is None:
|
||||
with PackageReader(self._latest(case_id,package_id),self.store.limits) as reader:state=self._begin(reader)
|
||||
state,_=self._save(state)
|
||||
if self.config_explicit and {**asdict(InvestigationConfig()),**state["policy"]}!=asdict(self.config):raise InvestigationError("INVESTIGATION_POLICY_CHANGED")
|
||||
effective_policy={**asdict(InvestigationConfig()),**state['policy']}
|
||||
if state['policy'].get('bootstrap_json_framing') and 'bootstrap_framing_version' not in state['policy']:
|
||||
effective_policy['bootstrap_framing_version']=1
|
||||
if self.config_explicit and effective_policy!=asdict(self.config):raise InvestigationError("INVESTIGATION_POLICY_CHANGED")
|
||||
if state["active"] and state["active"]["phase"]=="executing":return self._execute_plan(state,budget)
|
||||
if state["active"] is None:
|
||||
state.setdefault('early_stop_protocol',1)
|
||||
|
||||
@@ -28,7 +28,7 @@ def measure_package(path):
|
||||
contexts[hashlib.sha256(json_bytes(original['messages'])).hexdigest()] = context
|
||||
elif name.startswith('llm/operations/') and name.endswith('/request.json'):
|
||||
req = json.loads(reader.read_bytes(name))
|
||||
if req.get('prompt_version','').startswith('investigation_project_'):
|
||||
if req.get('prompt_version','').startswith(('investigation_project_','investigation_bootstrap_')):
|
||||
requests.append(req)
|
||||
samples, unmatched = [], 0
|
||||
for req in requests:
|
||||
|
||||
@@ -155,7 +155,7 @@ class LLMGateway:
|
||||
LOGGER.warning("LLM endpoint unavailable provider=%s code=%s", provider.id, code)
|
||||
except ValidationError as invalid:
|
||||
info={'validation_keyword':invalid.validator}
|
||||
if request['prompt_version'] in {'investigation_bootstrap_v21','investigation_project_v14','investigation_project_v15','investigation_project_v16','investigation_project_v17','investigation_project_v18','investigation_project_v19','investigation_project_v20'}:
|
||||
if request['prompt_version'] in {'investigation_bootstrap_v22','investigation_bootstrap_v21','investigation_project_v14','investigation_project_v15','investigation_project_v16','investigation_project_v17','investigation_project_v18','investigation_project_v19','investigation_project_v20'}:
|
||||
from .diagnostics import schema_feedback
|
||||
info.update(schema_feedback(invalid,include_keyword=True))
|
||||
error = LLMError("LLM_OUTPUT_SCHEMA",diagnostics=info)
|
||||
|
||||
@@ -21,6 +21,23 @@ from .provider_base import ProviderResponse
|
||||
from .diagnostics import count, sanitize, parse_response_object
|
||||
|
||||
|
||||
def bootstrap_shape(schema):
|
||||
"""Nested syntax contract only; full controller validation remains authoritative."""
|
||||
def shape(rule):
|
||||
result={k:rule[k] for k in ('type','required','additionalProperties') if k in rule}
|
||||
if 'properties' in rule:result['properties']={k:shape(v) for k,v in rule['properties'].items()}
|
||||
if 'items' in rule:result['items']=shape(rule['items'])
|
||||
if 'anyOf' in rule:result['anyOf']=[shape(v) for v in rule['anyOf']]
|
||||
if 'const' in rule:
|
||||
from .diagnostics import json_type
|
||||
result['type']=json_type(rule['const'])
|
||||
return result
|
||||
return {'type':'object','additionalProperties':False,'required':['assessment','update','next'],
|
||||
'properties':{'assessment':shape(schema['properties']['assessment']),
|
||||
'update':shape(schema['properties']['update']),
|
||||
'next':{'type':'object','additionalProperties':True}}}
|
||||
|
||||
|
||||
class GeminiProvider:
|
||||
def __init__(self):
|
||||
self._cache = {}
|
||||
@@ -132,6 +149,8 @@ class GeminiProvider:
|
||||
'update':{'type':'object','additionalProperties':True},
|
||||
'next':{'type':'object','additionalProperties':True}},
|
||||
'required':['assessment','update','next'],'additionalProperties':False}
|
||||
if request['prompt_version']=='investigation_bootstrap_v22' and config.response_format=='json_object':
|
||||
generation['responseJsonSchema']=bootstrap_shape(request['output_schema'])
|
||||
full = {'contents': prefix + contents, 'generationConfig': generation}
|
||||
if system['parts']:
|
||||
full['systemInstruction'] = system
|
||||
@@ -142,7 +161,7 @@ class GeminiProvider:
|
||||
if config.use_context_cache:
|
||||
status = 'below_minimum'
|
||||
if (len(json_bytes([system, prefix]).decode('utf-8')) >= config.cache_min_prefix_chars
|
||||
or request['prompt_version']=='investigation_bootstrap_v21' or request['prompt_version'] in {'investigation_project_v15','investigation_project_v16','investigation_project_v17','investigation_project_v18','investigation_project_v19','investigation_project_v20'}):
|
||||
or request['prompt_version'] in {'investigation_bootstrap_v21','investigation_bootstrap_v22'} or request['prompt_version'] in {'investigation_project_v15','investigation_project_v16','investigation_project_v17','investigation_project_v18','investigation_project_v19','investigation_project_v20'}):
|
||||
with self._lock:
|
||||
now = time.monotonic()
|
||||
self._cache = {k: v for k, v in self._cache.items() if v[1] > now}
|
||||
|
||||
@@ -10,7 +10,7 @@ from viano_bug_analysis.lifecycle.storage import json_bytes
|
||||
|
||||
VERSION = 'stable_case_v1'
|
||||
SUPPORTED_PROMPTS = {f'investigation_project_v{n}' for n in (4, 5, 6, 7, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20)}
|
||||
SUPPORTED_PROMPTS.add('investigation_bootstrap_v21')
|
||||
SUPPORTED_PROMPTS.update({'investigation_bootstrap_v21','investigation_bootstrap_v22'})
|
||||
STABLE_FIELDS = frozenset({'format', 'format_version', 'prompt_version', 'case',
|
||||
'sources', 'interpretation', 'budget_policy', 'policy', 'tool_schema',
|
||||
'unread_sources', 'unread_inventory_scope'})
|
||||
|
||||
@@ -110,6 +110,13 @@
|
||||
},
|
||||
"bootstrap_json_framing": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"bootstrap_framing_version": {
|
||||
"type": "integer",
|
||||
"enum": [
|
||||
1,
|
||||
2
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -802,6 +809,9 @@
|
||||
},
|
||||
"tool_guidance_version": {
|
||||
"const": 1
|
||||
},
|
||||
"bootstrap_framing_version": {
|
||||
"const": 2
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -106,6 +106,13 @@
|
||||
},
|
||||
"bootstrap_json_framing": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"bootstrap_framing_version": {
|
||||
"type": "integer",
|
||||
"enum": [
|
||||
1,
|
||||
2
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
import copy
|
||||
import json
|
||||
from dataclasses import replace
|
||||
|
||||
import pytest
|
||||
from jsonschema import Draft202012Validator
|
||||
|
||||
from test_parameter_and_bootstrap_guard import bootstrap
|
||||
from test_lean_investigation_decision import reply
|
||||
from test_llm_caching import config,native,scripted,CACHE
|
||||
from viano_bug_analysis.analysis.investigation_prompt import build_prompt
|
||||
from viano_bug_analysis.analysis import InvestigationConfig,InvestigationError
|
||||
from viano_bug_analysis.llm.gemini import GeminiProvider,bootstrap_shape
|
||||
|
||||
|
||||
def nested_request():
|
||||
from test_investigation_lean import lean_context
|
||||
c=lean_context();c.pop('lean_session');c.update(bootstrap_json_framing=True,bootstrap_framing_version=2)
|
||||
return build_prompt(c)
|
||||
|
||||
|
||||
def nonempty_reply():
|
||||
r=reply();r['update']['next_step']=None
|
||||
r['update']['working_brief']['findings']=[{'id':'B0001','direction':'neutral',
|
||||
'text':'Observed bounded result.','scope':'Examined snapshot.','evidence_refs':['E000001']}]
|
||||
return r
|
||||
|
||||
|
||||
@pytest.mark.parametrize('bad',['plain finding',['B0001','neutral'],None,42,True])
|
||||
def test_native_nested_shape_rejects_wrong_finding_type(bad):
|
||||
r=nonempty_reply();request=nested_request();before=copy.deepcopy(request['output_schema'])
|
||||
frame=bootstrap_shape(request['output_schema']);Draft202012Validator.check_schema(frame)
|
||||
assert request['output_schema']==before
|
||||
assert Draft202012Validator(frame).is_valid(r)
|
||||
r['update']['working_brief']['findings'][0]=bad
|
||||
assert not Draft202012Validator(frame).is_valid(r)
|
||||
|
||||
|
||||
@pytest.mark.parametrize('field,value',[('evidence_refs','E000001'),('id',42),('text',[]),('scope',None)])
|
||||
def test_native_shape_covers_nested_finding_fields(field,value):
|
||||
r=nonempty_reply();r['update']['working_brief']['findings'][0][field]=value
|
||||
assert not Draft202012Validator(bootstrap_shape(nested_request()['output_schema'])).is_valid(r)
|
||||
|
||||
|
||||
def test_new_native_contract_sent_with_cache_and_full_schema_retained(monkeypatch):
|
||||
request=nested_request();assert request['prompt_version']=='investigation_bootstrap_v22'
|
||||
response=native();response['candidates'][0]['content']['parts'][0]['text']=json.dumps(nonempty_reply())
|
||||
adapter=GeminiProvider();seen=scripted(adapter,monkeypatch,[CACHE,response,response])
|
||||
settings=replace(config(),cache_min_prefix_chars=10**9,response_format='json_object')
|
||||
first=adapter._generate(settings,request,1000000);second=adapter._generate(settings,request,1000000)
|
||||
assert first.document==second.document==nonempty_reply()
|
||||
assert len(seen)==3 and seen[1][1]['cachedContent']==seen[2][1]['cachedContent']
|
||||
frame=seen[1][1]['generationConfig']['responseJsonSchema']
|
||||
item=frame['properties']['update']['properties']['working_brief']['properties']['findings']['items']
|
||||
assert item['type']=='object' and set(item['required'])=={'id','direction','text','scope','evidence_refs'}
|
||||
assert 'oneOf' not in json.dumps(frame) and len(json.dumps(frame))<5000
|
||||
# The complete local/cached contract still constrains enum values and IDs.
|
||||
bad=nonempty_reply();bad['update']['working_brief']['findings'][0]['direction']='invented'
|
||||
assert not Draft202012Validator(request['output_schema']).is_valid(bad)
|
||||
assert 'output_schema' in str(seen[0][1]['contents'])
|
||||
|
||||
|
||||
def test_legacy_request_remains_v21_and_schema_identical():
|
||||
old=bootstrap();new=nested_request()
|
||||
assert old['prompt_version']=='investigation_bootstrap_v21'
|
||||
assert new['output_schema']==old['output_schema']
|
||||
|
||||
|
||||
@pytest.mark.parametrize('version',[0,3,True,'2'])
|
||||
def test_invalid_bootstrap_version_rejected(version):
|
||||
with pytest.raises(InvestigationError):InvestigationConfig(bootstrap_framing_version=version)
|
||||
@@ -78,7 +78,7 @@ def test_bootstrap_schema_feedback_survives_package_resume_and_repairs_once(tool
|
||||
def __init__(self):super().__init__();self.sent=[]
|
||||
async def generate(self,config,request,**kwargs):
|
||||
self.sent.append(copy.deepcopy(request))
|
||||
assert request['prompt_version']=='investigation_bootstrap_v21'
|
||||
assert request['prompt_version']=='investigation_bootstrap_v22'
|
||||
response=await super().generate(config,dict(request,prompt_version='investigation_project_v18'),**kwargs)
|
||||
if len(self.sent)==1:
|
||||
response.document['update']['working_brief']['findings']=['PRIVATE_RESPONSE_VALUE']
|
||||
|
||||
@@ -90,7 +90,8 @@ def test_coding_contract_rejects_wrong_column_value_but_accepts_placeholder():
|
||||
validate_request(payload,c)
|
||||
|
||||
|
||||
def test_bootstrap_marker_is_persisted_and_replayable(tool_case,tmp_path):
|
||||
@pytest.mark.parametrize('framing_version',[1,2])
|
||||
def test_bootstrap_marker_is_persisted_and_replayable(tool_case,tmp_path,framing_version):
|
||||
import asyncio
|
||||
from test_investigation_hybrid import HybridAdapter
|
||||
from test_case_interpreter import gateway
|
||||
@@ -103,14 +104,19 @@ def test_bootstrap_marker_is_persisted_and_replayable(tool_case,tmp_path):
|
||||
versions=[]
|
||||
async def generate(self,config,request,**kwargs):
|
||||
self.versions.append(request['prompt_version'])
|
||||
if request['prompt_version']=='investigation_bootstrap_v21':
|
||||
if request['prompt_version'] in {'investigation_bootstrap_v21','investigation_bootstrap_v22'}:
|
||||
request=dict(request,prompt_version='investigation_project_v18')
|
||||
return await super().generate(config,request,**kwargs)
|
||||
adapter=Framed();router=InvestigationToolRouter(project_research=service_for(tool_case))
|
||||
loop=Investigator(tool_case[0],gateway(adapter),InvestigationConfig(lean_loop=True,lean_protocol=2,
|
||||
lean_checkpoint_steps=0,bootstrap_json_framing=True),router=router)
|
||||
lean_checkpoint_steps=0,bootstrap_json_framing=True,bootstrap_framing_version=framing_version),router=router)
|
||||
first=run(loop,steps=3)
|
||||
assert adapter.versions==['investigation_bootstrap_v21']*3
|
||||
expected='investigation_bootstrap_v21' if framing_version==1 else 'investigation_bootstrap_v22'
|
||||
assert adapter.versions==[expected]*3
|
||||
portable=CheckpointStore(Workspace(tmp_path/'portable'));portable.intake(first.package)
|
||||
result=run(Investigator(portable,gateway(adapter),router=router))
|
||||
assert result.status=='COMPLETED' and adapter.versions[3]=='investigation_project_v20'
|
||||
from viano_bug_analysis.llm.cache_measurement import measure_package
|
||||
measurement=measure_package(result.package)
|
||||
assert measurement['all_exactly_reconstructed'] and measurement['unmatched_requests']==0
|
||||
assert sum(s['historical_prompt']==expected for s in measurement['samples'])==3
|
||||
Reference in new issue
Block a user