48 lines
2.9 KiB
Python
48 lines
2.9 KiB
Python
"""Audit phase selection on stored model views; no API calls or package writes."""
|
|
import argparse
|
|
import json
|
|
from pathlib import Path
|
|
import zipfile
|
|
from viano_bug_analysis.analysis.json_consolidation import consolidate, restore, INSTRUCTIONS
|
|
from viano_bug_analysis.analysis.phase_selection import select_context
|
|
from viano_bug_analysis.analysis.model_context import model_json
|
|
|
|
VERSIONS={
|
|
'case_interpretation_resolved_v6':('interpretation','case_interpretation_resolved_v7','case_interpretation_resolved_v8'),
|
|
'case_interpretation_resolved_v7':('interpretation','case_interpretation_resolved_v7','case_interpretation_resolved_v8'),
|
|
'hypotheses_v2':('hypotheses','hypotheses_v3','hypotheses_v4'),
|
|
'hypotheses_v3':('hypotheses','hypotheses_v3','hypotheses_v4'),
|
|
'diagnosis_v3':('diagnosis','diagnosis_v4','diagnosis_v5'),
|
|
'diagnosis_v4':('diagnosis','diagnosis_v4','diagnosis_v5'),
|
|
'diagnosis_early_v3':('diagnosis','diagnosis_early_v4','diagnosis_early_v5'),
|
|
'diagnosis_early_v4':('diagnosis','diagnosis_early_v4','diagnosis_early_v5'),
|
|
'change_request_v4':('handoff','change_request_v5','change_request_v6'),
|
|
'change_request_v5':('handoff','change_request_v5','change_request_v6')}
|
|
|
|
|
|
def measure(package):
|
|
requests=[]
|
|
prompts=Path(__file__).resolve().parents[1]/'src/viano_bug_analysis/llm/prompts'
|
|
with zipfile.ZipFile(package) as archive:
|
|
for name in archive.namelist():
|
|
if not name.startswith('llm/operations/') or not name.endswith('/request.json'):continue
|
|
request=json.loads(archive.read(name));version=request['prompt_version']
|
|
if version not in VERSIONS:continue
|
|
phase,baseline_version,target=VERSIONS[version]
|
|
user=json.loads(request['messages'][1]['content']);original=restore(user['input']);excluded=[]
|
|
selected=select_context(original,phase,audit_out=excluded)
|
|
before=consolidate(original);after=consolidate(selected)
|
|
assert model_json(restore(after)) == model_json(selected)
|
|
extra=sum(len(m['content']) for m in request['messages'][2:])
|
|
baseline=len(model_json({**user,'input':before}))+len((prompts/(baseline_version+'.txt')).read_text(encoding='utf-8'))+extra
|
|
current=len(model_json({**user,'input':after}))+len((prompts/(target+'.txt')).read_text(encoding='utf-8'))+extra
|
|
requests.append({'request':name,'phase':phase,'baseline':baseline_version,'target':target,
|
|
'messages_before':baseline,'messages_after':current,'excluded_paths':excluded})
|
|
return {'paid_api_calls':0,'measurement':'projection of stored model views, including system instructions; not billed tokens',
|
|
'canonical_archive_modified':False,'requests':requests}
|
|
|
|
|
|
if __name__=='__main__':
|
|
parser=argparse.ArgumentParser(description=__doc__);parser.add_argument('package',type=Path)
|
|
print(json.dumps(measure(parser.parse_args().package),ensure_ascii=False,indent=2))
|