Simplify investigation replies and derive controller bookkeeping

scripts/measure_investigation_contract.py

@@ -0,0 +1,103 @@
+"""Offline projection of archived v13 replies onto v14, without new reasoning."""
+import argparse
+import copy
+import hashlib
+import json
+from pathlib import Path
+
+from jsonschema import Draft202012Validator
+from viano_bug_analysis.analysis.investigation_prompt import build_prompt, output_schema
+from viano_bug_analysis.analysis.investigation_decision import normalize
+from viano_bug_analysis.analysis.confidence_review import effective_confidence
+from viano_bug_analysis.analysis.model_context import model_json
+from viano_bug_analysis.analysis.investigation_config import InvestigationError
+from viano_bug_analysis.package import PackageReader
+from viano_bug_analysis.llm.gemini import GeminiProvider
+
+
+def signature(messages):
+    return hashlib.sha256(model_json(messages).encode('utf8')).hexdigest()
+
+
+def project(context, document):
+    """Copy semantic fields exactly; reuse only already written change reasons."""
+    result=copy.deepcopy(document);update=result['update'];brief=update['working_brief']
+    old={f['id']:f for f in context.get('working_brief',{}).get('brief',{}).get('findings',[])}
+    reasons={c['id']:c['reason'] for c in brief.pop('changes')}
+    brief['format_version']=2
+    for f in brief['findings']:
+        previous=old.get(f['id'])
+        critical=previous and (f['direction']!=previous['direction'] or set(previous['evidence_refs'])-set(f['evidence_refs']))
+        if critical and f['id'] in reasons:f['revision_reason']=reasons[f['id']]
+    present={f['id'] for f in brief['findings']}
+    brief['retired']=[{'id':fid,'reason':reasons[fid]} for fid in old if fid not in present and fid in reasons]
+    for key in ('note','evidence_refs','causal_review'):update.pop(key)
+    if result['assessment'] is not None:result['assessment'].pop('evidence_refs')
+    return result
+
+
+def measure(path):
+    with PackageReader(path) as reader:
+        contexts={};calls=[]
+        for ref in reader.files:
+            if ref.startswith('analysis/investigation/') and '/contexts/' in ref:
+                c=json.loads(reader.read_bytes(ref))
+                if c.get('prompt_version')=='investigation_project_v13':
+                    contexts[signature(build_prompt(c)['messages'])]=c
+            elif ref.startswith('llm/operations/') and '/round-' in ref and ref.endswith('.json'):
+                call=json.loads(reader.read_bytes(ref))
+                if call['phase']=='INVESTIGATING_HYPOTHESES':calls.append((ref,call))
+        state=json.loads(reader.read_bytes(reader.manifest['artifacts']['investigation_state']))
+        rejected={n['llm_ref']:n['code'] for h in state['hypotheses'].values() for n in h['notes'] if n['kind']=='invalid_response'}
+        rows=[];requests=[];unavailable=0
+        for ref,call in calls:
+            request=json.loads(reader.read_bytes(call['request_ref']))
+            original=contexts[signature(request['messages'])]
+            context=dict(original,prompt_version='investigation_project_v14')
+            def request_sizes(req):
+                system,prefix,live=GeminiProvider._messages(req,stable_case_context=True)
+                return {'message_bytes':sum(len(m['content'].encode('utf8')) for m in req['messages']),
+                        'prefix_bytes':len(model_json([system,prefix]).encode('utf8')),
+                        'live_bytes':len(model_json(live).encode('utf8'))}
+            requests.append({'call_ref':ref,'old':request_sizes(request),
+                             'new':request_sizes(build_prompt(context))})
+            document=call.get('outcome',{}).get('document')
+            if document is None:
+                unavailable+=1;continue
+            wire=project(original,document)
+            errors=list(Draft202012Validator(output_schema(context)).iter_errors(wire))
+            code='WIRE_SCHEMA' if errors else None
+            normalized=None
+            if code is None:
+                try:normalized=normalize(context,wire)
+                except InvestigationError as error:code=error.code
+            rows.append({'call_ref':ref,'original_rejection':rejected.get(ref),
+                'projected_rejection':code,'old_document_bytes':len(model_json(document).encode('utf8')),
+                'new_document_bytes':len(model_json(wire).encode('utf8')),
+                'projected_effect':normalized['update']['causal_review']['effect'] if normalized else None,
+                'projected_confidence':effective_confidence(normalized['update']) if normalized else None})
+        return {'case_id':reader.manifest['case_id'],'package_generation':reader.manifest['package_generation'],
+                'calls':len(calls),'unavailable_schema_replies':unavailable,'samples':rows,'requests':requests}
+
+
+if __name__=='__main__':
+    parser=argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('packages',nargs='+',type=Path);parser.add_argument('--output',type=Path,required=True)
+    args=parser.parse_args();results=[measure(p) for p in args.packages]
+    rows=[r for result in results for r in result['samples']]
+    old=sum(r['old_document_bytes'] for r in rows);new=sum(r['new_document_bytes'] for r in rows)
+    requests=[r for result in results for r in result['requests']]
+    report={'measurement':'investigation_v14_reply_projection','old_document_bytes':old,'new_document_bytes':new,
+        'visible_json_reduction_percent':round(100*(1-new/old),2),
+        'rejected_v13_structured':sum(bool(r['original_rejection']) for r in rows),
+        'rejected_v14_projection':sum(bool(r['projected_rejection']) for r in rows),
+        'request_totals':{version:{unit:sum(r[version][unit] for r in requests)
+            for unit in ('message_bytes','prefix_bytes','live_bytes')} for version in ('old','new')},
+        'limitations':['Canonical visible JSON bytes only, not provider output/thinking tokens.',
+            'No new model replies; semantic claims copied exactly, no proof of correctness.',
+            'Schema-failing replies without stored documents cannot be projected.',
+            'Projection validates wire and brief normalization, not external tool availability or final diagnosis.'],
+        'packages':results}
+    args.output.parent.mkdir(parents=True,exist_ok=True)
+    args.output.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n',encoding='utf8')
+    print(json.dumps({k:v for k,v in report.items() if k not in {'packages','limitations'}}))