Narrow early investigation review and improve safe response repair diagnostics

tests/test_investigation_review.py

@@ -0,0 +1,159 @@
+"""Offline regressions for phase-specific review and actionable safe rejection."""
+import asyncio
+import copy
+import json
+from dataclasses import replace
+
+import pytest
+from jsonschema import Draft202012Validator
+
+from test_compact_investigation import context, entry
+from test_lean_investigation_decision import reply
+from test_llm_caching import config, native, scripted
+from test_llm_gateway import request
+from test_investigation_tools import tool_case
+from test_case_builder import case_setup
+from viano_bug_analysis.analysis.investigation_compact import model_view, review_view, validate_tool_shape
+from viano_bug_analysis.analysis.investigation_config import InvestigationError
+from viano_bug_analysis.analysis.investigation_prompt import build_prompt, output_schema
+from viano_bug_analysis.analysis.investigation_decision import normalize
+from viano_bug_analysis.llm import LLMError
+from viano_bug_analysis.llm.diagnostics import parse_response_object, sanitize
+from viano_bug_analysis.llm.gemini import GeminiProvider
+from viano_bug_analysis.llm.stable_context import split_document, restore_document
+
+
+def review_context():
+    c=context();c['prompt_version']='investigation_project_v16'
+    c['early_review']={'attempts':1,'limit':3,'request':{'status':'preparing','hypothesis_id':'H2',
+        'reason':'Other mechanism has scoped support, independent diagnosis pending.'}}
+    return c
+
+
+def test_review_is_narrower_but_keeps_pending_results_and_counterfindings():
+    c=review_context();before=copy.deepcopy(c)
+    r=build_prompt(c);old=build_prompt(dict(c,prompt_version='investigation_project_v15'))
+    view=json.loads(r['messages'][1]['content'])['input']
+    assert 'project.query' not in r['messages'][0]['content']
+    assert all(k not in view for k in ('known_schemas','tool_schema','evidence_catalog','budget'))
+    assert view['evidence']==model_view(c)['evidence']
+    assert view['working_brief']['findings'][0]['direction']=='against'
+    assert view['other_working_briefs']==model_view(c)['other_working_briefs']
+    assert c==before
+    assert sum(len(m['content']) for m in r['messages']) < sum(len(m['content']) for m in old['messages'])
+    stable,live=split_document(json.loads(r['messages'][1]['content']),r['prompt_version'])
+    assert restore_document(stable,live)==json.loads(r['messages'][1]['content'])
+
+
+@pytest.mark.parametrize('kind',['tool','recall','test','finish','request_diagnosis'])
+def test_review_contract_rejects_new_research_or_diagnosis_actions(kind):
+    c=review_context()
+    for strict in (False,True):
+        schema=output_schema(c,strict=strict)['properties']['next']
+        assert not Draft202012Validator(schema).is_valid({'kind':kind,'reason':'New work'})
+        assert Draft202012Validator(schema).is_valid({'kind':'defer','reason':'Pending result assessed'})
+        assert Draft202012Validator(schema).is_valid({'kind':'reject','reason':'Direct counterevidence'})
+
+
+def test_review_strict_validation_and_normalization_preserve_assessment():
+    c=review_context();r=reply();r['next']={'kind':'defer','reason':'Ready for independent review'}
+    validate_tool_shape(c,r)
+    normalized=normalize(c,r)
+    assert normalized['assessment']['evidence_refs']==['E000020']
+    assert normalized['update']['working_brief']['findings']==c['working_brief']['brief']['findings']
+    assert normalized['update']['causal_review']['effect']=='contradicts'
+    r['next']={'kind':'tool','operation':'project.query','arguments':{'kind':'database.schema','table':'creature'}}
+    with pytest.raises(InvestigationError,match='INVESTIGATION_OUTPUT'):validate_tool_shape(c,r)
+
+
+def test_reference_namespace_error_is_actionable_without_echoing_values():
+    c=review_context();r=reply()
+    r['update']['working_brief']['findings'][0]['evidence_refs']=['F0001']
+    with pytest.raises(InvestigationError) as error:validate_tool_shape(c,r)
+    assert 'research_E_ids_only_not_case_F_ids' in error.value.feedback['requirement']
+    assert 'F0001' not in json.dumps(error.value.feedback)
+    assert '/evidence_refs/0' in error.value.feedback['validation_path']
+
+
+def test_origin_consolidation_is_lossless_and_does_not_touch_db_cells():
+    c=context();c['prompt_version']='investigation_project_v16';c['recalled_evidence']=['E000019']
+    origin={'snapshot_id':'snapshot-'+('a'*150),'database':'world','revision':'fixed'}
+    for eid in ('E000019','E000020'):
+        capture=c['evidence'][eid]['content']['items'][0]
+        capture['origin']=copy.deepcopy(origin)
+        capture['result']['items']=[{'row':{'origin':copy.deepcopy(origin),'metadata_ref':'literal'}}]
+    original=copy.deepcopy(c);v=model_view(c)
+    assert len(v['shared_metadata'])==1
+    for e in v['evidence'].values():
+        capture=e['content']['items'][0]
+        assert v['shared_metadata'][capture['origin']['metadata_ref']]==origin
+        assert capture['result']['items'][0]['row']=={'origin':origin,'metadata_ref':'literal'}
+    assert original==c
+
+
+@pytest.mark.parametrize('raw,category',[(b'{"private":', 'invalid_json'),
+    (b'{"private":1,"private":2}','json_policy'),(b'[]','root_not_object'),
+    (b'{"private":NaN}','json_policy')])
+def test_json_diagnostics_are_precise_but_never_echo_content(raw,category):
+    with pytest.raises(LLMError) as caught:parse_response_object(raw,stage='output_json')
+    info=caught.value.diagnostics
+    assert info['response_error']==category and info['response_stage']=='output_json'
+    assert 'private' not in json.dumps(info)
+    assert sanitize(info)==info
+
+
+@pytest.mark.parametrize('broken,stage,category',[
+    ({'finishReason':'STOP','content':{'parts':[{'text':'{"secret":'}]}},'output_json','invalid_json'),
+    ({'finishReason':'STOP'},'candidate','invalid_structure')])
+def test_gemini_failed_output_keeps_usage_and_safe_category(monkeypatch,broken,stage,category):
+    adapter=GeminiProvider();envelope=native();envelope['candidates']=[broken]
+    scripted(adapter,monkeypatch,[envelope])
+    with pytest.raises(LLMError) as caught:
+        asyncio.run(adapter.generate(replace(config(),context_cache=False),request(),max_response_bytes=10000))
+    info=caught.value.diagnostics
+    assert info['response_stage']==stage and info['response_error']==category
+    assert info['input_tokens']==100 and info['reasoning_tokens']==5
+    assert 'secret' not in json.dumps(info)
+
+
+def test_unrecognized_diagnostic_values_are_removed():
+    assert sanitize({'response_stage':'secret','response_error':'secret','parse_line':-1})=={'parse_line':None}
+
+
+def test_v15_uses_frozen_general_prompt_even_when_review_pending():
+    c=review_context();c['prompt_version']='investigation_project_v15'
+    prompt=build_prompt(c)
+    assert json.loads(prompt['messages'][1]['content'])['task']=='investigate_hypothesis'
+    assert Draft202012Validator(output_schema(c)['properties']['next']).is_valid({'kind':'recall','evidence_refs':['E000001']})
+
+
+def test_provider_parse_details_survive_checkpoint_and_repair_without_extra_queries(tool_case):
+    from test_investigator import LoopAdapter, run, CASE
+    from test_working_brief import service_for
+    from test_project_research import rows
+    from test_case_interpreter import gateway
+    from viano_bug_analysis.analysis import Investigator
+    from viano_bug_analysis.tools import InvestigationToolRouter
+    from viano_bug_analysis.budget import BudgetManager
+    from viano_bug_analysis.package import PackageReader, PackageValidator
+    class RepairAdapter(LoopAdapter):
+        async def generate(self, config, req, *, max_response_bytes):
+            if not hasattr(self, 'failed'):
+                self.failed=True
+                raise LLMError('LLM_RESPONSE',diagnostics={'response_stage':'output_json',
+                    'response_error':'invalid_json','parse_line':3,'parse_column':8})
+            return await super().generate(config,req,max_response_bytes=max_response_bytes)
+    def change(reply,c):
+        reply['next']={'kind':'tool','operation':'project.query','arguments':rows()}
+    store=tool_case[0];adapter=RepairAdapter(change)
+    result=run(Investigator(store,gateway(adapter),router=InvestigationToolRouter(
+        project_research=service_for(tool_case))),steps=2)
+    info=adapter.contexts[0]['previous_rejection_details']
+    assert info['response_error']=='invalid_json' and info['parse_line']==3
+    assert info['requirement']=='return_one_valid_JSON_object_no_fences_or_duplicate_keys'
+    assert len(BudgetManager(store).snapshot(*CASE).state['accounts']['H1']['actions'])==1
+    assert PackageValidator().validate(result.package).valid
+    with PackageReader(result.package) as reader:
+        failed=[json.loads(reader.read_bytes(n)) for n in reader.files
+            if n.startswith('llm/operations/') and '/round-' in n]
+    assert next(r for r in failed if r.get('code')=='LLM_RESPONSE')['outcome']['diagnostics']['parse_column']==8

tests/test_investigator.py

@@ -45,10 +45,10 @@ class LoopAdapter:
             result["next"]={"kind":"test","executor":"human","question":"What occurs after interaction?","steps":["Interact once"],
                             "prerequisites":["Use the specified build"],"requested_observations":["Record the observed state"],"environment":"build-A"}
         if self.change:self.change(result,context)
-        if request['prompt_version'] in {'investigation_project_v12','investigation_project_v13','investigation_project_v14','investigation_project_v15'}:
+        if request['prompt_version'] in {'investigation_project_v12','investigation_project_v13','investigation_project_v14','investigation_project_v15','investigation_project_v16'}:
             result['update'].setdefault('causal_review',{'effect':'neutral','support':[],'against':[],
                 'reason':'This fixture gathers context without establishing a causal defect.'})
-        if request['prompt_version'] in {'investigation_project_v13','investigation_project_v14','investigation_project_v15'}:
+        if request['prompt_version'] in {'investigation_project_v13','investigation_project_v14','investigation_project_v15','investigation_project_v16'}:
             existing=context.get('working_brief')
             review=result['update']['causal_review']
             findings=[]
@@ -62,7 +62,7 @@ class LoopAdapter:
                 'uncertainties':['Causal defect not established.'],'next_question':'Check the selected mechanism.',
                 'changes':[{'id':fid,'reason':'Fixture revises the finding.',
                     'evidence_refs':result['update']['evidence_refs']} for fid,f in old.items() if new.get(fid)!=f]})
-        if request['prompt_version'] in {'investigation_project_v14','investigation_project_v15'}:
+        if request['prompt_version'] in {'investigation_project_v14','investigation_project_v15','investigation_project_v16'}:
             b=result['update']['working_brief']
             if b['format_version']==1:
                 reasons={v['id']:v['reason'] for v in b.pop('changes')}