43 lines
2.1 KiB
Python
43 lines
2.1 KiB
Python
"""Measure archived LLM requests offline; no provider, package writes or secrets."""
|
|
import argparse
|
|
from collections import defaultdict
|
|
import json
|
|
from pathlib import Path
|
|
import zipfile
|
|
from viano_bug_analysis.analysis.json_consolidation import consolidate, restore, INSTRUCTIONS
|
|
from viano_bug_analysis.analysis.model_context import model_json
|
|
|
|
VERSIONS = {'case_interpretation_resolved_v6':'case_interpretation_resolved_v7',
|
|
'hypotheses_v2':'hypotheses_v3', 'diagnosis_v3':'diagnosis_v4',
|
|
'diagnosis_early_v3':'diagnosis_early_v4', 'change_request_v4':'change_request_v5'}
|
|
|
|
|
|
def measure(package):
|
|
groups=defaultdict(list)
|
|
with zipfile.ZipFile(package) as archive:
|
|
for name in archive.namelist():
|
|
if not name.startswith('llm/operations/') or not name.endswith('/request.json'):continue
|
|
request=json.loads(archive.read(name))
|
|
version=request['prompt_version']
|
|
if version not in VERSIONS:continue
|
|
user=json.loads(request['messages'][1]['content'])
|
|
encoded=consolidate(user['input'])
|
|
assert model_json(restore(encoded)) == model_json(user['input']), name
|
|
old_user=len(request['messages'][1]['content'])
|
|
new_user=len(model_json({**user,'input':encoded}))
|
|
old_total=sum(len(m['content']) for m in request['messages'])
|
|
new_total=old_total-old_user+new_user+len(INSTRUCTIONS)+2
|
|
groups[version].append({'request':name,'user_before':old_user,'user_after':new_user,
|
|
'messages_before':old_total,'messages_after':new_total,'roundtrip':'exact'})
|
|
return {'package':str(package),'paid_api_calls':0,'phases':{
|
|
version:{'target':VERSIONS[version],'requests':rows,
|
|
**{key:sum(row[key] for row in rows) for key in ('user_before','user_after','messages_before','messages_after')}}
|
|
for version,rows in groups.items()}}
|
|
|
|
|
|
if __name__ == '__main__':
|
|
parser=argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('package',type=Path)
|
|
args=parser.parse_args()
|
|
print(json.dumps(measure(args.package),ensure_ascii=False,indent=2))
|