104 lines
6.3 KiB
Python
104 lines
6.3 KiB
Python
"""Offline projection of archived v13 replies onto v14, without new reasoning."""
|
|
import argparse
|
|
import copy
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
|
|
from jsonschema import Draft202012Validator
|
|
from viano_bug_analysis.analysis.investigation_prompt import build_prompt, output_schema
|
|
from viano_bug_analysis.analysis.investigation_decision import normalize
|
|
from viano_bug_analysis.analysis.confidence_review import effective_confidence
|
|
from viano_bug_analysis.analysis.model_context import model_json
|
|
from viano_bug_analysis.analysis.investigation_config import InvestigationError
|
|
from viano_bug_analysis.package import PackageReader
|
|
from viano_bug_analysis.llm.gemini import GeminiProvider
|
|
|
|
|
|
def signature(messages):
|
|
return hashlib.sha256(model_json(messages).encode('utf8')).hexdigest()
|
|
|
|
|
|
def project(context, document):
|
|
"""Copy semantic fields exactly; reuse only already written change reasons."""
|
|
result=copy.deepcopy(document);update=result['update'];brief=update['working_brief']
|
|
old={f['id']:f for f in context.get('working_brief',{}).get('brief',{}).get('findings',[])}
|
|
reasons={c['id']:c['reason'] for c in brief.pop('changes')}
|
|
brief['format_version']=2
|
|
for f in brief['findings']:
|
|
previous=old.get(f['id'])
|
|
critical=previous and (f['direction']!=previous['direction'] or set(previous['evidence_refs'])-set(f['evidence_refs']))
|
|
if critical and f['id'] in reasons:f['revision_reason']=reasons[f['id']]
|
|
present={f['id'] for f in brief['findings']}
|
|
brief['retired']=[{'id':fid,'reason':reasons[fid]} for fid in old if fid not in present and fid in reasons]
|
|
for key in ('note','evidence_refs','causal_review'):update.pop(key)
|
|
if result['assessment'] is not None:result['assessment'].pop('evidence_refs')
|
|
return result
|
|
|
|
|
|
def measure(path):
|
|
with PackageReader(path) as reader:
|
|
contexts={};calls=[]
|
|
for ref in reader.files:
|
|
if ref.startswith('analysis/investigation/') and '/contexts/' in ref:
|
|
c=json.loads(reader.read_bytes(ref))
|
|
if c.get('prompt_version')=='investigation_project_v13':
|
|
contexts[signature(build_prompt(c)['messages'])]=c
|
|
elif ref.startswith('llm/operations/') and '/round-' in ref and ref.endswith('.json'):
|
|
call=json.loads(reader.read_bytes(ref))
|
|
if call['phase']=='INVESTIGATING_HYPOTHESES':calls.append((ref,call))
|
|
state=json.loads(reader.read_bytes(reader.manifest['artifacts']['investigation_state']))
|
|
rejected={n['llm_ref']:n['code'] for h in state['hypotheses'].values() for n in h['notes'] if n['kind']=='invalid_response'}
|
|
rows=[];requests=[];unavailable=0
|
|
for ref,call in calls:
|
|
request=json.loads(reader.read_bytes(call['request_ref']))
|
|
original=contexts[signature(request['messages'])]
|
|
context=dict(original,prompt_version='investigation_project_v14')
|
|
def request_sizes(req):
|
|
system,prefix,live=GeminiProvider._messages(req,stable_case_context=True)
|
|
return {'message_bytes':sum(len(m['content'].encode('utf8')) for m in req['messages']),
|
|
'prefix_bytes':len(model_json([system,prefix]).encode('utf8')),
|
|
'live_bytes':len(model_json(live).encode('utf8'))}
|
|
requests.append({'call_ref':ref,'old':request_sizes(request),
|
|
'new':request_sizes(build_prompt(context))})
|
|
document=call.get('outcome',{}).get('document')
|
|
if document is None:
|
|
unavailable+=1;continue
|
|
wire=project(original,document)
|
|
errors=list(Draft202012Validator(output_schema(context)).iter_errors(wire))
|
|
code='WIRE_SCHEMA' if errors else None
|
|
normalized=None
|
|
if code is None:
|
|
try:normalized=normalize(context,wire)
|
|
except InvestigationError as error:code=error.code
|
|
rows.append({'call_ref':ref,'original_rejection':rejected.get(ref),
|
|
'projected_rejection':code,'old_document_bytes':len(model_json(document).encode('utf8')),
|
|
'new_document_bytes':len(model_json(wire).encode('utf8')),
|
|
'projected_effect':normalized['update']['causal_review']['effect'] if normalized else None,
|
|
'projected_confidence':effective_confidence(normalized['update']) if normalized else None})
|
|
return {'case_id':reader.manifest['case_id'],'package_generation':reader.manifest['package_generation'],
|
|
'calls':len(calls),'unavailable_schema_replies':unavailable,'samples':rows,'requests':requests}
|
|
|
|
|
|
if __name__=='__main__':
|
|
parser=argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('packages',nargs='+',type=Path);parser.add_argument('--output',type=Path,required=True)
|
|
args=parser.parse_args();results=[measure(p) for p in args.packages]
|
|
rows=[r for result in results for r in result['samples']]
|
|
old=sum(r['old_document_bytes'] for r in rows);new=sum(r['new_document_bytes'] for r in rows)
|
|
requests=[r for result in results for r in result['requests']]
|
|
report={'measurement':'investigation_v14_reply_projection','old_document_bytes':old,'new_document_bytes':new,
|
|
'visible_json_reduction_percent':round(100*(1-new/old),2),
|
|
'rejected_v13_structured':sum(bool(r['original_rejection']) for r in rows),
|
|
'rejected_v14_projection':sum(bool(r['projected_rejection']) for r in rows),
|
|
'request_totals':{version:{unit:sum(r[version][unit] for r in requests)
|
|
for unit in ('message_bytes','prefix_bytes','live_bytes')} for version in ('old','new')},
|
|
'limitations':['Canonical visible JSON bytes only, not provider output/thinking tokens.',
|
|
'No new model replies; semantic claims copied exactly, no proof of correctness.',
|
|
'Schema-failing replies without stored documents cannot be projected.',
|
|
'Projection validates wire and brief normalization, not external tool availability or final diagnosis.'],
|
|
'packages':results}
|
|
args.output.parent.mkdir(parents=True,exist_ok=True)
|
|
args.output.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n',encoding='utf8')
|
|
print(json.dumps({k:v for k,v in report.items() if k not in {'packages','limitations'}}))
|