89 lines
4.5 KiB
Python
89 lines
4.5 KiB
Python
"""Offline cache-view measurement. Never instantiates or contacts an LLM provider."""
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
|
|
from viano_bug_analysis.package import PackageReader
|
|
from viano_bug_analysis.lifecycle.storage import json_bytes
|
|
from .gemini import GeminiProvider
|
|
from .stable_context import restore_document
|
|
|
|
|
|
def _size(value):
|
|
return len(json_bytes(value).decode('utf-8'))
|
|
|
|
|
|
def measure_package(path):
|
|
from viano_bug_analysis.analysis.investigation_prompt import build_prompt
|
|
contexts, requests = {}, []
|
|
with PackageReader(path) as reader:
|
|
identity = {k: reader.manifest[k] for k in ('case_id','package_id','analysis_revision','package_generation')}
|
|
for name in reader.files:
|
|
if name.startswith('analysis/investigation/') and '/contexts/' in name and name.endswith('.json'):
|
|
context = json.loads(reader.read_bytes(name))
|
|
if context.get('format') != 'viano-investigation-context':
|
|
continue
|
|
original = build_prompt(context)
|
|
contexts[hashlib.sha256(json_bytes(original['messages'])).hexdigest()] = context
|
|
elif name.startswith('llm/operations/') and name.endswith('/request.json'):
|
|
req = json.loads(reader.read_bytes(name))
|
|
if req.get('prompt_version','').startswith('investigation_project_'):
|
|
requests.append(req)
|
|
samples, unmatched = [], 0
|
|
for req in requests:
|
|
context = contexts.get(hashlib.sha256(json_bytes(req['messages'])).hexdigest())
|
|
if context is None:
|
|
unmatched += 1
|
|
continue
|
|
# Historical contexts are immutable. Rebuild a detached v11 model view
|
|
# to measure stable aliases; the original request remains the baseline.
|
|
target = context['prompt_version'] if context.get('prompt_version') in {'investigation_project_v12','investigation_project_v13'} else 'investigation_project_v11'
|
|
context = dict(context, prompt_version=target)
|
|
upgraded = build_prompt(context)
|
|
old_system, old_prefix, old_live = GeminiProvider._messages(req)
|
|
system, prefix, live = GeminiProvider._messages(upgraded, stable_case_context=True)
|
|
prefix_doc = json.loads(prefix[0]['parts'][0]['text'])
|
|
current = json.loads(live[0]['parts'][0]['text'])
|
|
current['output_schema'] = prefix_doc['output_schema']
|
|
canonical = json.loads(upgraded['messages'][1]['content'])
|
|
assert restore_document(prefix_doc['stable_case_context'], current) == canonical
|
|
samples.append({'prefix_id':hashlib.sha256(json_bytes([system,prefix])).hexdigest(),
|
|
'historical_prompt':req['prompt_version'],
|
|
'rebuilt_prompt':target,
|
|
'old_prefix_characters':_size([old_system,old_prefix]),
|
|
'new_prefix_characters':_size([system,prefix]),
|
|
'old_live_characters':_size(old_live), 'new_live_characters':_size(live),
|
|
'full_v11_characters':_size(upgraded['messages']), 'exact_reconstruction':True})
|
|
old_live = sum(s['old_live_characters'] for s in samples)
|
|
new_live = sum(s['new_live_characters'] for s in samples)
|
|
return {'measurement':'stable_case_cache_offline_v1', **identity,
|
|
'requests':len(samples),'unmatched_requests':unmatched,
|
|
'unique_prefixes':len({s['prefix_id'] for s in samples}),
|
|
'old_live_characters':old_live, 'new_live_characters':new_live,
|
|
'live_character_reduction_percent':round(100*(old_live-new_live)/old_live,2) if old_live else None,
|
|
'all_exactly_reconstructed':all(s['exact_reconstruction'] for s in samples) and bool(samples),
|
|
'limitations':['Characters are not provider token counts or a paid quality benchmark.',
|
|
'Historical requests are compared to reconstructed v11 views; earlier view policies may differ.',
|
|
'Prefix counts ignore remote TTL expiry, credentials, latency and storage billing.'],
|
|
'samples':samples}
|
|
|
|
|
|
def main(argv=None):
|
|
parser=argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--package',type=Path,required=True)
|
|
parser.add_argument('--output',type=Path)
|
|
args=parser.parse_args(argv)
|
|
report=measure_package(args.package)
|
|
data=json.dumps(report,ensure_ascii=False,indent=2)+'\n'
|
|
if args.output:
|
|
args.output.parent.mkdir(parents=True,exist_ok=True)
|
|
args.output.write_text(data,encoding='utf-8',newline='\n')
|
|
else:
|
|
print(data,end='')
|
|
return 0
|
|
|
|
|
|
if __name__=='__main__':
|
|
raise SystemExit(main())
|