Select phase-specific working context and retain auditable source evidence

scripts/measure_phase_selection.py

@@ -0,0 +1,47 @@
+"""Audit phase selection on stored model views; no API calls or package writes."""
+import argparse
+import json
+from pathlib import Path
+import zipfile
+from viano_bug_analysis.analysis.json_consolidation import consolidate, restore, INSTRUCTIONS
+from viano_bug_analysis.analysis.phase_selection import select_context
+from viano_bug_analysis.analysis.model_context import model_json
+
+VERSIONS={
+    'case_interpretation_resolved_v6':('interpretation','case_interpretation_resolved_v7','case_interpretation_resolved_v8'),
+    'case_interpretation_resolved_v7':('interpretation','case_interpretation_resolved_v7','case_interpretation_resolved_v8'),
+    'hypotheses_v2':('hypotheses','hypotheses_v3','hypotheses_v4'),
+    'hypotheses_v3':('hypotheses','hypotheses_v3','hypotheses_v4'),
+    'diagnosis_v3':('diagnosis','diagnosis_v4','diagnosis_v5'),
+    'diagnosis_v4':('diagnosis','diagnosis_v4','diagnosis_v5'),
+    'diagnosis_early_v3':('diagnosis','diagnosis_early_v4','diagnosis_early_v5'),
+    'diagnosis_early_v4':('diagnosis','diagnosis_early_v4','diagnosis_early_v5'),
+    'change_request_v4':('handoff','change_request_v5','change_request_v6'),
+    'change_request_v5':('handoff','change_request_v5','change_request_v6')}
+
+
+def measure(package):
+    requests=[]
+    prompts=Path(__file__).resolve().parents[1]/'src/viano_bug_analysis/llm/prompts'
+    with zipfile.ZipFile(package) as archive:
+        for name in archive.namelist():
+            if not name.startswith('llm/operations/') or not name.endswith('/request.json'):continue
+            request=json.loads(archive.read(name));version=request['prompt_version']
+            if version not in VERSIONS:continue
+            phase,baseline_version,target=VERSIONS[version]
+            user=json.loads(request['messages'][1]['content']);original=restore(user['input']);excluded=[]
+            selected=select_context(original,phase,audit_out=excluded)
+            before=consolidate(original);after=consolidate(selected)
+            assert model_json(restore(after)) == model_json(selected)
+            extra=sum(len(m['content']) for m in request['messages'][2:])
+            baseline=len(model_json({**user,'input':before}))+len((prompts/(baseline_version+'.txt')).read_text(encoding='utf-8'))+extra
+            current=len(model_json({**user,'input':after}))+len((prompts/(target+'.txt')).read_text(encoding='utf-8'))+extra
+            requests.append({'request':name,'phase':phase,'baseline':baseline_version,'target':target,
+                'messages_before':baseline,'messages_after':current,'excluded_paths':excluded})
+    return {'paid_api_calls':0,'measurement':'projection of stored model views, including system instructions; not billed tokens',
+            'canonical_archive_modified':False,'requests':requests}
+
+
+if __name__=='__main__':
+    parser=argparse.ArgumentParser(description=__doc__);parser.add_argument('package',type=Path)
+    print(json.dumps(measure(parser.parse_args().package),ensure_ascii=False,indent=2))