"""Measure archived LLM requests offline; no provider, package writes or secrets.""" import argparse from collections import defaultdict import json from pathlib import Path import zipfile from viano_bug_analysis.analysis.json_consolidation import consolidate, restore, INSTRUCTIONS from viano_bug_analysis.analysis.model_context import model_json VERSIONS = {'case_interpretation_resolved_v6':'case_interpretation_resolved_v7', 'hypotheses_v2':'hypotheses_v3', 'diagnosis_v3':'diagnosis_v4', 'diagnosis_early_v3':'diagnosis_early_v4', 'change_request_v4':'change_request_v5'} def measure(package): groups=defaultdict(list) with zipfile.ZipFile(package) as archive: for name in archive.namelist(): if not name.startswith('llm/operations/') or not name.endswith('/request.json'):continue request=json.loads(archive.read(name)) version=request['prompt_version'] if version not in VERSIONS:continue user=json.loads(request['messages'][1]['content']) encoded=consolidate(user['input']) assert model_json(restore(encoded)) == model_json(user['input']), name old_user=len(request['messages'][1]['content']) new_user=len(model_json({**user,'input':encoded})) old_total=sum(len(m['content']) for m in request['messages']) new_total=old_total-old_user+new_user+len(INSTRUCTIONS)+2 groups[version].append({'request':name,'user_before':old_user,'user_after':new_user, 'messages_before':old_total,'messages_after':new_total,'roundtrip':'exact'}) return {'package':str(package),'paid_api_calls':0,'phases':{ version:{'target':VERSIONS[version],'requests':rows, **{key:sum(row[key] for row in rows) for key in ('user_before','user_after','messages_before','messages_after')}} for version,rows in groups.items()}} if __name__ == '__main__': parser=argparse.ArgumentParser(description=__doc__) parser.add_argument('package',type=Path) args=parser.parse_args() print(json.dumps(measure(args.package),ensure_ascii=False,indent=2))