Fix durable lean investigation repair feedback and context validation
This commit is contained in:
1 parent
ca14271f20
commit
dd666026d3
6 files changed
+89
-2
No files matched your search
@@ -33,3 +33,7 @@ Das Paket speichert kurze Beobachtungen samt LLM-Referenz, bewertetem Ergebnis u
|
||||
Lokale Fixture-Tests prüfen Schema/Cache-Roundtrip, verzögerte fachliche Updates, Wechsel-/periodische Konsolidierung, Fortsetzung per defer, Iterations- und Budgetende, ungültige Tools, ungelesene Referenzen, portable Wiederaufnahme und Unterbrechung nach Evidenzpublikation. Bestehende Investigation-/Gateway-/Early-Diagnosis-Tests sichern den vorherigen Modus ab.
|
||||
|
||||
Die Offline-Messdatei replayt alte v18-Kontexte mit leerer v19-Sitzung. Sie misst Zeichen, keine Tokens oder tatsächliche neue Untersuchung. Laufkosten und Denkarbeit müssen einschließlich der Konsolidierungen in einem neuen kostenpflichtigen Lauf gemessen werden; während der Implementierung wurde kein LLM-Dienst aufgerufen.
|
||||
|
||||
## Korrekturrückmeldungen
|
||||
|
||||
Schemafehler werden vor dem Speichern auf den begrenzten Korrekturvertrag abgebildet. Ungültige Befund-IDs erhalten den konkreten Hinweis auf `B0001` (B plus mindestens vier Ziffern). Bereits gespeicherte Kontexte mit dem früheren `validation_keyword` bleiben lesbar; für den nächsten Prompt wird daraus ein sicherer Korrekturhinweis, ohne das alte Paket zu verändern. Neue Promptkontexte werden vor ihrer Veröffentlichung validiert.
|
||||
@@ -239,7 +239,8 @@ def validate_tool_shape(context,decision):
|
||||
if error:
|
||||
from viano_bug_analysis.llm.diagnostics import schema_feedback
|
||||
from .investigation_config import InvestigationError
|
||||
raise InvestigationError('INVESTIGATION_OUTPUT',feedback=schema_feedback(error,include_keyword=True))
|
||||
from .investigation_lean import repair_feedback
|
||||
raise InvestigationError('INVESTIGATION_OUTPUT',feedback=repair_feedback(schema_feedback(error,include_keyword=True)))
|
||||
from .investigation_lean import checkpoint,build
|
||||
if checkpoint(context):
|
||||
import json
|
||||
|
||||
@@ -93,6 +93,8 @@ def build(context):
|
||||
document['input']['tool_schema']['request_contract']=schema(context,strict=True)['properties']['next']
|
||||
full['messages'][0]['content']=files('viano_bug_analysis.llm.prompts').joinpath('investigation_lean_v19.txt').read_text(encoding='utf-8')
|
||||
full.update(prompt_version=VERSION,output_schema=contract)
|
||||
if 'previous_rejection_details' in document['input']:
|
||||
document['input']['previous_rejection_details']=repair_feedback(document['input']['previous_rejection_details'])
|
||||
full['messages'][1]['content']=model_json(document)
|
||||
return full
|
||||
|
||||
@@ -121,3 +123,21 @@ def normalize(context, reply):
|
||||
result['update'].pop('next_step',None)
|
||||
result['update']['confidence']=context['hypotheses'][context['selected_hypothesis']]['confidence']
|
||||
return result
|
||||
|
||||
|
||||
def repair_feedback(details):
|
||||
"""Project schema failures to the durable, value-free repair contract."""
|
||||
result=copy.deepcopy(details)
|
||||
keyword=result.pop('validation_keyword',None)
|
||||
path=result.get('validation_path','')
|
||||
import re
|
||||
if keyword=='pattern' and re.fullmatch(r'/update/working_brief/findings/[0-9]+/id',path):
|
||||
result['requirement']='use_stable_finding_ids_B_followed_by_at_least_four_digits_example_B0001_not_case_fact_ids'
|
||||
elif keyword=='required' and 'path_prefix' in result.get('validation_fields',[]):
|
||||
result['requirement']='provide_missing_required_fields; path_prefix="" searches_repository_root'
|
||||
elif keyword is not None:
|
||||
result.setdefault('requirement',{
|
||||
'required':'provide_missing_required_fields','additionalProperties':'remove_unrecognized_fields',
|
||||
'type':'match_expected_type_at_validation_path','enum':'use_allowed_value_at_validation_path',
|
||||
'const':'use_allowed_value_at_validation_path'}.get(keyword,'match_full_controller_contract_at_validation_path'))
|
||||
return result
|
||||
@@ -165,7 +165,9 @@ class Investigator:
|
||||
raise InvestigationError("INVESTIGATION_GENERATION_CHANGED")
|
||||
state["parent_ref"]=reader.manifest["artifacts"].get("investigation_state") if previous else None
|
||||
state["sequence"]+=1
|
||||
if context is not None:state["active"]["context_sha256"]=digest(context)
|
||||
if context is not None:
|
||||
validate(context,"investigation-context")
|
||||
state["active"]["context_sha256"]=digest(context)
|
||||
if state["sequence"]>self.limits.max_state_checkpoints:raise InvestigationError("INVESTIGATION_HISTORY_LIMIT")
|
||||
ref=f"analysis/investigation/{state['analysis_revision']:04d}/states/{state['sequence']:08d}.json"
|
||||
validate(state,"investigation-state")
|
||||
|
||||
@@ -464,6 +464,11 @@
|
||||
"type": "integer",
|
||||
"minimum": 0,
|
||||
"maximum": 1000000000000
|
||||
},
|
||||
"validation_keyword": {
|
||||
"type": "string",
|
||||
"maxLength": 64,
|
||||
"pattern": "^[A-Za-z][A-Za-z0-9]*$"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
@@ -183,3 +183,58 @@ def test_replay_after_tool_publication_does_not_repeat_model_or_budget(tool_case
|
||||
assert result.status=='PROGRESSED' and len(adapter.calls)==1
|
||||
assert len(state(result)['hypotheses']['H1']['lean_session']['turns'])==1
|
||||
assert len(BudgetManager(tool_case[0]).snapshot(*CASE).state['accounts']['H1']['actions'])==1
|
||||
|
||||
|
||||
|
||||
def test_bad_consolidation_finding_id_repairs_without_corrupting_context(tool_case,tmp_path):
|
||||
from viano_bug_analysis.case.contracts import validate
|
||||
class BadIdOnce(LeanAdapter):
|
||||
failed=False
|
||||
async def generate(self,config,request,*,max_response_bytes):
|
||||
response=await super().generate(config,request,max_response_bytes=max_response_bytes)
|
||||
c=self.calls[-1]['input']
|
||||
if self.calls[-1]['task']=='consolidate_investigation_session' and not self.failed:
|
||||
self.failed=True
|
||||
eid=next(iter(c['evidence']))
|
||||
response.document['update']['working_brief']['findings']=[{'id':'F01','direction':'neutral',
|
||||
'text':'Schema available.','scope':'Captured schema.','evidence_refs':[eid]}]
|
||||
return response
|
||||
adapter=BadIdOnce();router=InvestigationToolRouter(project_research=service_for(tool_case))
|
||||
loop=Investigator(tool_case[0],gateway(adapter),InvestigationConfig(lean_loop=True),router=router)
|
||||
first=run(loop,steps=1)
|
||||
before=BudgetManager(tool_case[0]).snapshot(*CASE).state
|
||||
rejected=asyncio.run(loop.step(*CASE))
|
||||
assert rejected.status=='REJECTED'
|
||||
assert BudgetManager(tool_case[0]).snapshot(*CASE).state==before
|
||||
with PackageReader(rejected.package) as reader:
|
||||
c=json.loads(reader.read_bytes(state(rejected)['active']['context_ref']))
|
||||
validate(c,'investigation-context')
|
||||
detail=c['previous_rejection_details']
|
||||
assert 'validation_keyword' not in detail
|
||||
assert 'B0001' in detail['requirement']
|
||||
# The original released version persisted this extra keyword. Its
|
||||
# immutable context must remain loadable without a package rewrite.
|
||||
c['previous_rejection_details']={'validation_path':'/update/working_brief/findings/0/id',
|
||||
'validation_fields':[],'validation_keyword':'pattern'}
|
||||
validate(c,'investigation-context')
|
||||
prompt=build_prompt(c)
|
||||
assert 'B0001' in json.loads(prompt['messages'][1]['content'])['input']['previous_rejection_details']['requirement']
|
||||
portable=CheckpointStore(Workspace(tmp_path/'portable'));portable.intake(rejected.package)
|
||||
final=run(Investigator(portable,gateway(adapter),router=router))
|
||||
assert final.status=='COMPLETED' and PackageValidator().validate(final.package).valid
|
||||
|
||||
|
||||
|
||||
def test_invalid_context_is_not_published(tool_case):
|
||||
from viano_bug_analysis.case.errors import CaseBuildError
|
||||
adapter=LeanAdapter();loop=Investigator(tool_case[0],gateway(adapter),InvestigationConfig(lean_loop=True),
|
||||
router=InvestigationToolRouter(project_research=service_for(tool_case)))
|
||||
result=run(loop,steps=1);saved=state(result)
|
||||
# A new prompt context is validated before a checkpoint can reference it.
|
||||
saved['active']={'hypothesis_id':'H1','iteration':2,'attempt':0,'phase':'prompt',
|
||||
'context_ref':'analysis/investigation/0001/contexts/invalid.json','context_sha256':'0'*64,
|
||||
'decision':None,'llm_ref':None,'test_request':None,'test_deadline':None,'retry_token':'initial'}
|
||||
before=tool_case[0].latest(*CASE)
|
||||
with pytest.raises(CaseBuildError) as caught:loop._save(saved,context={'unexpected':'invalid'})
|
||||
assert caught.value.code=='INPUT_SCHEMA'
|
||||
assert tool_case[0].latest(*CASE)==before
|
||||
Reference in new issue
Block a user