Select phase-specific working context and retain auditable source evidence
This commit is contained in:
1 parent
3ea7cd5000
commit
d6749001bb
36 files changed
+599
-60
No files matched your search
@@ -70,7 +70,7 @@ def test_new_diagnosis_keeps_frozen_context_and_uses_compact_request(investigate
|
||||
adapter = FinalAdapter(True)
|
||||
result = build(DiagnosisBuilder(store, gateway(adapter)))
|
||||
request = adapter.requests[0]
|
||||
assert request['prompt_version'] == 'diagnosis_v4'
|
||||
assert request['prompt_version'] == 'diagnosis_v5'
|
||||
visible = restore(json.loads(request['messages'][1]['content'])['input'])
|
||||
with PackageReader(result.package) as reader:
|
||||
doc = json.loads(reader.read_bytes(result.diagnosis_ref))
|
||||
|
||||
@@ -65,7 +65,7 @@ def test_retry_gets_actionable_feedback_and_portable_replay(diagnosed, tmp_path,
|
||||
class Correcting(RequestAdapter):
|
||||
async def generate(self, config, request, **kwargs):
|
||||
response = await super().generate(config, request, **kwargs)
|
||||
assert request['prompt_version'] == (first_version if self.calls == 1 else 'change_request_v5')
|
||||
assert request['prompt_version'] == (first_version if self.calls == 1 else 'change_request_v6')
|
||||
if self.calls == 1:
|
||||
response.document['affected_scope'][0]['source_ref'] = 'wrong.cpp'
|
||||
else:
|
||||
|
||||
@@ -218,7 +218,7 @@ def test_legacy_rejection_upgrades_next_attempt_without_rewriting_context(invest
|
||||
assert build(builder).status=='REJECTED'
|
||||
assert adapter.requests[0]['prompt_version']=='diagnosis_v1'
|
||||
assert build(builder).status=='COMPLETED'
|
||||
assert adapter.requests[1]['prompt_version']=='diagnosis_v4'
|
||||
assert adapter.requests[1]['prompt_version']=='diagnosis_v5'
|
||||
received=restore(json.loads(adapter.requests[1]['messages'][1]['content'])['input'])
|
||||
assert received['validation_feedback'][0]['code']=='DIAGNOSIS_SELECTION'
|
||||
assert build(builder).reused and adapter.calls==2
|
||||
|
||||
@@ -82,7 +82,7 @@ class Model:
|
||||
document["resolutions"][0]["candidate_id"] = "K9999"
|
||||
else:
|
||||
document = payload()
|
||||
if version in {"case_interpretation_resolved_v3", "case_interpretation_resolved_v4", "case_interpretation_resolved_v5", "case_interpretation_resolved_v6", "case_interpretation_resolved_v7"}:
|
||||
if version in {"case_interpretation_resolved_v3", "case_interpretation_resolved_v4", "case_interpretation_resolved_v5", "case_interpretation_resolved_v6", "case_interpretation_resolved_v7","case_interpretation_resolved_v8"}:
|
||||
document.update(described_behavior=[],provisional_behavior_gap=None)
|
||||
data = restore(json.loads(request["messages"][1]["content"])["input"])
|
||||
selected = [(k,v) for k,v in data["sources"].items() if v["kind"] == "entity" and v["case_pointer"].startswith("/entity_resolution/")]
|
||||
@@ -96,7 +96,7 @@ class Model:
|
||||
if self.omit_resolved:
|
||||
self.omit_resolved -= 1
|
||||
document["relevant_entities"] = payload()["relevant_entities"]
|
||||
if version in {'case_interpretation_resolved_v4','case_interpretation_resolved_v5','case_interpretation_resolved_v6','case_interpretation_resolved_v7'}:
|
||||
if version in {'case_interpretation_resolved_v4','case_interpretation_resolved_v5','case_interpretation_resolved_v6','case_interpretation_resolved_v7','case_interpretation_resolved_v8'}:
|
||||
from viano_bug_analysis.analysis.quality import unnamed_entity_reasons
|
||||
reasons = unnamed_entity_reasons(data)
|
||||
for entity in document['relevant_entities']:
|
||||
@@ -133,7 +133,7 @@ def test_resolves_unrecorded_subject_preserves_case_and_portable_replay(built_ca
|
||||
model = Model(); service = service_for(store)
|
||||
result = run(store, model, service)
|
||||
assert result.status == "COMPLETED" and PackageValidator().validate(result.package).valid
|
||||
assert [r["prompt_version"] for r in model.requests] == ["entity_resolution_plan_v4","entity_resolution_assessment_v4","case_interpretation_resolved_v7"]
|
||||
assert [r["prompt_version"] for r in model.requests] == ["entity_resolution_plan_v4","entity_resolution_assessment_v4","case_interpretation_resolved_v8"]
|
||||
assert len(service.database.calls) == 1
|
||||
assert service.database.calls[0]["filters"] == [{"column":"name","operator":"contains","value":"switch"}]
|
||||
with PackageReader(result.package) as reader:
|
||||
@@ -248,7 +248,7 @@ def test_search_cannot_be_grounded_in_a_capture_filename(built_case):
|
||||
def test_empty_plan_still_supports_descriptive_quality_contract(built_case):
|
||||
store=built_case[0];model=Model(plan={'searches':[]});service=service_for(store)
|
||||
assert run(store,model,service).status=='COMPLETED'
|
||||
assert model.requests[-1]['prompt_version']=='case_interpretation_resolved_v7' and not service.database.calls
|
||||
assert model.requests[-1]['prompt_version']=='case_interpretation_resolved_v8' and not service.database.calls
|
||||
|
||||
|
||||
def test_trial_phase_stop_includes_resolution_and_is_idempotent(built_case):
|
||||
@@ -409,7 +409,7 @@ def test_third_assessment_can_use_lower_reasoning_without_resetting_history(buil
|
||||
assert len(service.database.calls) == 1
|
||||
assert model.efforts == [('entity_resolution_plan_v4','medium'),
|
||||
('entity_resolution_assessment_v4','medium'),('entity_resolution_assessment_v4','medium'),
|
||||
('entity_resolution_assessment_v4','low'),('case_interpretation_resolved_v7','medium')]
|
||||
('entity_resolution_assessment_v4','low'),('case_interpretation_resolved_v8','medium')]
|
||||
with PackageReader(result.package) as reader:
|
||||
saved = json.loads(reader.read_bytes('analysis/interpretations/0001/entity-resolution/assessment.json'))
|
||||
assert saved['payload']['attempt'] == 3
|
||||
|
||||
@@ -94,7 +94,7 @@ def test_generation_budget_registry_and_preservation(interpreted_case):
|
||||
if name != "manifest.json": assert before.read_bytes(name) == after.read_bytes(name)
|
||||
assert original.read_bytes() == original_bytes
|
||||
assert len(adapter.requests) == 1
|
||||
assert adapter.requests[0]["prompt_version"] == "hypotheses_v3"
|
||||
assert adapter.requests[0]["prompt_version"] == "hypotheses_v4"
|
||||
assert case_builder.build("case-A","stable-package").reused
|
||||
assert asyncio.run(CaseInterpreter(store,gateway(adapter)).interpret("case-A","stable-package")).reused
|
||||
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
import copy
|
||||
import json
|
||||
from viano_bug_analysis.analysis.phase_selection import select_context
|
||||
from viano_bug_analysis.analysis.investigation_focus import focused_view
|
||||
from viano_bug_analysis.analysis.investigation_blocks import restore_blocks
|
||||
from test_investigation_consolidation import populated
|
||||
|
||||
|
||||
def sources():
|
||||
return {'sources':{
|
||||
'O0001':{'kind':'observation','case_pointer':'/human_observations/0','data':{'text':'Nothing happened','provenance':{'author':'Tester','observed_at':'old-build','source_ref':'observation.json','json_pointer':'/0'}}},
|
||||
'F0001':{'kind':'fact','case_pointer':'/runtime_facts/0','data':{'value':{'row':{'path':'a','sha256':'b','value':None,'flag':False},'coverage':'complete_filtered_rows','truncated':False},'provenance':{'snapshot_id':'snapshot-A','source_ref':'capture.json'}}},
|
||||
'N0001':{'kind':'entity','case_pointer':'/entity_resolution/entities/0','data':{'id':'17','type':'item','provenance':{'source_ref':'capture.json'}}},
|
||||
'X0001':{'kind':'unread_reference','case_pointer':'/source_context/refs/0','data':{'path':'unread.cpp','sha256':'a'*64}},
|
||||
'D0001':{'kind':'data_quality','case_pointer':'/quality/0','data':{'text':'Observation predates source revision'}}},
|
||||
'case_ref':'case.json','case_sha256':'x'*64,'case':{'case_id':'C'}}
|
||||
|
||||
|
||||
def test_interpretation_omits_unread_catalog_without_removing_facts_or_identity_links():
|
||||
c=sources();before=copy.deepcopy(c);audit=[]
|
||||
v=select_context(c,'interpretation',audit_out=audit)
|
||||
assert 'X0001' not in v['sources'] and '/sources/X0001' in audit
|
||||
assert v['unread_inventory']['counts']=={'unread_reference':1}
|
||||
assert v['sources']['F0001']['data'] == c['sources']['F0001']['data']
|
||||
assert v['sources']['N0001']==c['sources']['N0001']
|
||||
assert v['sources']['O0001']['data']['provenance']['observed_at']=='old-build'
|
||||
assert v['sources']['O0001']['data']['provenance']['source_ref']=='observation.json'
|
||||
assert c==before and select_context(c,'interpretation')==v
|
||||
|
||||
|
||||
def test_hypotheses_keep_all_interpretations_facts_prior_feedback_and_observations():
|
||||
c={'case_context':sources(),'interpretation_sources':{'I0001':{'kind':'unknowns','interpretation_pointer':'/unknowns/0','data':{'text':'Deployment is unknown','source_ids':['D0001']}}},'policy':{},'prior_generations':[{'failed_patch':'Did not change behavior'}]}
|
||||
v=select_context(c,'hypotheses')
|
||||
assert v['interpretation_sources']['I0001']['data']==c['interpretation_sources']['I0001']['data']
|
||||
assert v['prior_generations']==c['prior_generations']
|
||||
assert set(v['case_context']['sources'])=={'O0001','F0001','N0001','D0001'}
|
||||
|
||||
|
||||
def test_diagnosis_omits_plans_and_bookkeeping_retains_every_evidence_and_uncertainty():
|
||||
h={'proposal':{'title':'Missing hook','mechanism':'May be missing','falsifying_evidence':['Hook exists'],'rationale':'Early speculation','open_questions':['Hook present?']},'notes':[{'text':'May be missing'},{'text':'Hook found; old deployment unknown'}],'questions':[{'id':'Q1','text':'Hook present?','resolved':True},{'id':'Q2','text':'Which deployment?','resolved':False}],'evidence_refs':['E1'],'confidence':'weakened','disposition':'active'}
|
||||
c={'case':{'case':{},'human_observations':['Nothing moved'],'runtime_facts':{'facts':[{'id':'F1','value':False}],'raw_refs':['unread']},'inputs':['huge catalog']},'investigation':{'hypotheses':{'H1':h},'active':None},'evidence':{'E1':{'content':{'items':[],'coverage':'complete_filtered_rows','truncated':False}}},'interpretation':{'contradictions':['Old observation versus new code'],'unknowns':['Deployment']},'budget':{'seen_evidence':['hash'],'accounts':{'H1':{'remaining':80,'status':'investigating','actions':{'old':'history'}}}}}
|
||||
before=copy.deepcopy(c);audit=[];v=select_context(c,'diagnosis',audit_out=audit)
|
||||
assert v['evidence']==c['evidence'] and v['interpretation']==c['interpretation']
|
||||
assert v['case']['runtime_facts']['facts']==c['case']['runtime_facts']['facts']
|
||||
assert v['investigation']['hypotheses']['H1']['notes']==h['notes'][-1:]
|
||||
assert v['investigation']['hypotheses']['H1']['closed_questions']=={'Q1':'Hook present?'}
|
||||
assert '/investigation/hypotheses/H1/notes/0' in audit
|
||||
assert c==before
|
||||
|
||||
|
||||
def test_investigation_keeps_latest_note_and_can_recall_older_counterevidence():
|
||||
c=populated();c['prompt_version']='investigation_project_v9';c['budget']['accounts']['H1']['actions']={}
|
||||
h=c['hypotheses']['H1'];h['evidence_refs']=['E000002'];h['notes']=['old speculation','current counterevidence']
|
||||
audit=[]
|
||||
assert set(focused_view(c,audit_out=audit)['evidence'])=={'E000001'}
|
||||
assert audit==['/hypotheses/H1/notes/0']
|
||||
c['recalled_evidence']=['E000002']
|
||||
v=restore_blocks(focused_view(c))
|
||||
assert set(v['evidence'])=={'E000001','E000002'}
|
||||
assert v['hypotheses']['H1']['notes']==['current counterevidence']
|
||||
assert c['hypotheses']['H1']['notes']==['old speculation','current counterevidence']
|
||||
c['prompt_version']='investigation_project_v8'
|
||||
assert len(focused_view(c)['hypotheses']['H1']['notes'])==2
|
||||
|
||||
|
||||
def test_handoff_keeps_repair_constraints_and_evidence_without_search_direction():
|
||||
c={'case':{'human_observations':['Absent'],'data_quality':{'warnings':['Snapshot only']}},'interpretation':{'investigation_direction':'search more','contradictions':['different versions'],'unknowns':['Placement coordinates'],'relevant_entities':['typed id']},'evidence':{'E1':{'content':{'items':[],'coverage':'complete_filtered_rows'},'content_sha256':'hash'}},'local_context':{'capture':{'allowed_evidence_refs':['E1']}},'policy':{'rules':['No invented coordinates']}}
|
||||
v=select_context(c,'handoff')
|
||||
assert 'investigation_direction' not in v['interpretation']
|
||||
assert v['interpretation']['unknowns']==c['interpretation']['unknowns']
|
||||
assert v['evidence']['E1']['content']==c['evidence']['E1']['content']
|
||||
assert v['local_context']==c['local_context'] and v['policy']==c['policy']
|
||||
@@ -94,7 +94,7 @@ def test_real_contextual_identity_resolution_with_fixture_llm(service, tmp_path)
|
||||
assert any(e['id']=='23486' for e in entities.values())
|
||||
resolved=next(k for k,e in entities.items() if e['id']=='23487')
|
||||
doc=missing_payload()
|
||||
if request["prompt_version"]in {"case_interpretation_resolved_v3","case_interpretation_resolved_v4","case_interpretation_resolved_v5","case_interpretation_resolved_v6","case_interpretation_resolved_v7"}:
|
||||
if request["prompt_version"]in {"case_interpretation_resolved_v3","case_interpretation_resolved_v4","case_interpretation_resolved_v5","case_interpretation_resolved_v6","case_interpretation_resolved_v7","case_interpretation_resolved_v8"}:
|
||||
doc.update(described_behavior=[{'text':'The quest describes catching wild creatures using a net while intoxicated or wearing goggles.',
|
||||
'source_ids':[self.mention],'basis':'local_description','verification':'not_independently_verified'}],
|
||||
provisional_behavior_gap={'text':'The tester does not see these creatures despite wearing goggles.',
|
||||
|
||||
@@ -204,7 +204,7 @@ def test_model_loop_uses_project_tools_and_compact_previous_result(tool_case, re
|
||||
if previous:
|
||||
assert context["evidence"][eid]["content"]["items"]
|
||||
assert "inputs" not in context["case"]
|
||||
assert request["prompt_version"]=="investigation_project_v8"
|
||||
assert request["prompt_version"]=="investigation_project_v9"
|
||||
reply={"assessment":{"semantic_score":50,"reason":"Scoped fact","evidence_refs":[eid]} if previous else None,
|
||||
"update":{"confidence":"plausible","evidence_refs":[eid] if previous else [],"resolve_questions":[],"add_questions":[],"note":"Query typed identity"},
|
||||
"next":{"kind":"finish","reason":"Check complete"} if previous else {"kind":"tool","operation":"project.query","arguments":rows()}}
|
||||
|
||||
+3
-3
@@ -27,9 +27,9 @@ class PipelineAdapter:
|
||||
|
||||
async def generate(self, config, request, **kwargs):
|
||||
self.calls += 1
|
||||
phase = {"case_interpretation_v1": "case", "hypotheses_v1": "hypotheses", "hypotheses_v2": "hypotheses", "hypotheses_v3": "hypotheses",
|
||||
"investigation_v1": "loop", "investigation_v2": "loop", "investigation_v3": "loop", "diagnosis_v1": "diagnosis", "diagnosis_v2": "diagnosis", "diagnosis_v3": "diagnosis", "diagnosis_v4": "diagnosis",
|
||||
"change_request_v1": "request", "change_request_v2": "request", "change_request_v3": "request", "change_request_v4": "request", "change_request_v5": "request"}[request["prompt_version"]]
|
||||
phase = {"case_interpretation_v1": "case", "hypotheses_v1": "hypotheses", "hypotheses_v2": "hypotheses", "hypotheses_v3": "hypotheses", "hypotheses_v4": "hypotheses",
|
||||
"investigation_v1": "loop", "investigation_v2": "loop", "investigation_v3": "loop", "diagnosis_v1": "diagnosis", "diagnosis_v2": "diagnosis", "diagnosis_v3": "diagnosis", "diagnosis_v4": "diagnosis", "diagnosis_v5": "diagnosis",
|
||||
"change_request_v1": "request", "change_request_v2": "request", "change_request_v3": "request", "change_request_v4": "request", "change_request_v5": "request", "change_request_v6": "request"}[request["prompt_version"]]
|
||||
adapter = self.adapters[phase]
|
||||
return await adapter.generate(config, request, **kwargs)
|
||||
|
||||
|
||||
Reference in new issue
Block a user