193 lines
9.7 KiB
Python
193 lines
9.7 KiB
Python
"""Scoped read models. No prompts, raw responses, credentials or host paths leave here."""
|
|
import json
|
|
import threading
|
|
from datetime import datetime, timezone
|
|
|
|
from viano_bug_analysis.lifecycle.checkpoints import checkpoint_chain
|
|
from viano_bug_analysis.lifecycle.storage import key
|
|
from viano_bug_analysis.package import PackageReader
|
|
from viano_bug_analysis.telemetry.inspection import read_status
|
|
from viano_bug_analysis.telemetry.config import TelemetryError
|
|
|
|
FINISHED = {'OUTPUT_READY', 'HANDED_OFF', 'FAILED_TO_ANALYZE'}
|
|
LABELS = {'building': 'Fallaufbau', 'planning': 'Fallplanung', 'hypotheses': 'Hypothesen',
|
|
'investigation': 'Untersuchung', 'diagnosis': 'Diagnose', 'coding': 'Codingauftrag',
|
|
'correction': 'Korrekturen / Wiederholungen', 'unknown': 'Nicht zugeordnet'}
|
|
|
|
|
|
def read_json(reader, ref, limit=8*1024**2):
|
|
if not ref or ref not in reader.files or reader._archive.getinfo(ref).file_size > limit:
|
|
return {}
|
|
value = json.loads(reader.read_bytes(ref))
|
|
return value if isinstance(value, dict) else {}
|
|
|
|
|
|
def phase(call, prompt):
|
|
if prompt.startswith('change_request'):
|
|
return 'coding'
|
|
return {'BUILDING_CASE': 'building', 'INTERPRETING_CASE': 'planning',
|
|
'GENERATING_HYPOTHESES': 'hypotheses', 'INVESTIGATING_HYPOTHESES': 'investigation',
|
|
'BUILDING_DIAGNOSIS': 'diagnosis'}.get(call['phase'], 'unknown')
|
|
|
|
|
|
def historical_classification(call, request):
|
|
prompt = request.get('prompt_version', '')
|
|
if call['rounds'] > 1 or 'repair' in prompt:
|
|
return 'correction'
|
|
for message in request.get('messages', []):
|
|
try:
|
|
data = json.loads(message.get('content', ''))
|
|
except (ValueError, TypeError):
|
|
continue
|
|
if not isinstance(data, dict):
|
|
continue
|
|
if data.get('task') == 'correct_rejected_output' or data.get('validation_feedback'):
|
|
return 'correction'
|
|
context = data.get('input', data)
|
|
if isinstance(context, dict):
|
|
if any(context.get(k) for k in ('response_repair', 'validation_feedback', 'correction_feedback')):
|
|
return 'correction'
|
|
if call['phase'] == 'INVESTIGATING_HYPOTHESES' and type(context.get('attempt')) is int:
|
|
return 'correction' if context['attempt'] > 0 else 'regular'
|
|
# Older records are not silently counted as regular when attribution is unclear.
|
|
return 'unknown'
|
|
|
|
|
|
def case_view(reader):
|
|
manifest = reader.manifest
|
|
revision = manifest['analysis_revision']
|
|
artifacts = manifest['artifacts']
|
|
docs = {p: read_json(reader, artifacts.get(p)) for p in (
|
|
'case', 'case_interpretation', 'hypotheses', 'investigation_state', 'diagnosis', 'change_request')}
|
|
def current(pointer):
|
|
doc = docs[pointer]
|
|
return doc.get('case', doc).get('analysis_revision') == revision
|
|
chain = checkpoint_chain(reader)
|
|
complete = [manifest['state'] != 'NEW', current('case'), current('case_interpretation'),
|
|
current('hypotheses'), (current('investigation_state') and docs['investigation_state'].get('completed') is True)
|
|
or manifest['state'] == 'BUILDING_DIAGNOSIS' or manifest['state'] in FINISHED,
|
|
current('diagnosis'), current('change_request')]
|
|
title = manifest['case_id']
|
|
reports = docs['case'].get('bug', {}).get('reports', [])
|
|
if not reports:
|
|
index = read_json(reader, artifacts.get('case_inputs'))
|
|
reports = [read_json(reader, ref) for ref in index.get('reports', [])[:20]]
|
|
for report in reports:
|
|
if isinstance(report.get('title'), str) and report['title'].strip():
|
|
title = report['title'][:240]
|
|
break
|
|
calls = []
|
|
seen = set()
|
|
for _, cp in reversed(chain):
|
|
detail = cp['details']
|
|
ref = detail.get('result_ref')
|
|
if detail.get('operation') != 'llm-round' or not ref or ref in seen:
|
|
continue
|
|
seen.add(ref)
|
|
call = read_json(reader, ref, 16*1024**2)
|
|
if not call or call.get('derived'):
|
|
continue # derived repaired documents are not additional provider calls
|
|
outcome = call.get('outcome', {})
|
|
meta = outcome.get('metadata', outcome.get('diagnostics', {}))
|
|
request = read_json(reader, call.get('request_ref'), 8*1024**2)
|
|
prompt = meta.get('prompt_version', request.get('prompt_version', ''))
|
|
category = outcome.get('usage_classification') or historical_classification(call, request)
|
|
bucket = phase(call, prompt) if category == 'regular' else category
|
|
usage = {}
|
|
for field in ('input_tokens', 'output_tokens', 'reasoning_tokens', 'cached_input_tokens'):
|
|
value = meta.get(field)
|
|
usage[field] = value if type(value) is int and value >= 0 else None
|
|
calls.append({'bucket': bucket if bucket in LABELS else 'unknown', 'phase': phase(call, prompt),
|
|
'category': category, 'day': cp['created_at'][:10], **usage})
|
|
return {k: manifest[k] for k in ('case_id', 'package_id', 'analysis_revision', 'package_generation', 'state')} | {
|
|
'key':key(manifest['case_id']) + '.' + key(manifest['package_id']),
|
|
'project_id': manifest['project'].get('id'), 'title': title,
|
|
'progress': round(sum(complete) / len(complete) * 100), 'calls': calls,
|
|
'updated_at': manifest['updated_at'],
|
|
'destination':manifest.get('routing',{}).get('destination'),
|
|
'reason':manifest.get('routing',{}).get('reason')}
|
|
|
|
|
|
def aggregate(cases):
|
|
buckets = {name: {'id': name, 'label': label, 'input_tokens': 0, 'output_tokens': 0,
|
|
'reasoning_tokens': 0, 'cached_input_tokens': 0, 'calls': 0, 'unknown_usage': 0}
|
|
for name, label in LABELS.items()}
|
|
days = {}
|
|
for case in cases:
|
|
for call in case['calls']:
|
|
bucket = buckets[call['bucket']]
|
|
bucket['calls'] += 1
|
|
bucket['unknown_usage'] += int(call['input_tokens'] is None or call['output_tokens'] is None)
|
|
day = days.setdefault(call['day'], {'day': call['day'], 'regular': 0, 'correction': 0, 'unknown': 0})
|
|
total = (call['input_tokens'] or 0) + (call['output_tokens'] or 0)
|
|
day[call['category'] if call['category'] in {'regular', 'correction'} else 'unknown'] += total
|
|
for field in ('input_tokens', 'output_tokens', 'reasoning_tokens', 'cached_input_tokens'):
|
|
bucket[field] += call[field] or 0
|
|
return {'buckets': list(buckets.values()), 'days': sorted(days.values(), key=lambda d: d['day']),
|
|
'scope': 'retained_cases_all_revisions', 'reasoning_included_in_output': True,
|
|
'unknown_usage_calls': sum(b['unknown_usage'] for b in buckets.values())}
|
|
|
|
|
|
class Dashboard:
|
|
def __init__(self, service):
|
|
self.service = service
|
|
self.lock = threading.Lock()
|
|
self.cache = {}
|
|
|
|
def overview(self, client):
|
|
with self.lock:
|
|
views, errors, present = [], 0, set()
|
|
for folder in self.service.workspace.cases.glob('*/*'):
|
|
try:
|
|
if folder.is_symlink() or folder.parent.is_symlink():
|
|
continue
|
|
paths = sorted(folder.glob('g*.zip'), key=lambda p: int(p.stem[1:]))
|
|
if not paths:
|
|
continue
|
|
path = paths[-1]
|
|
if path.is_symlink():
|
|
continue
|
|
stat = path.stat()
|
|
signature = (str(path), stat.st_size, stat.st_mtime_ns)
|
|
present.add(str(folder))
|
|
cached = self.cache.get(str(folder))
|
|
if not cached or cached[0] != signature:
|
|
with PackageReader(path) as reader:
|
|
m = reader.manifest
|
|
if folder.name != key(m['package_id']) or folder.parent.name != key(m['case_id']):
|
|
continue
|
|
view = case_view(reader)
|
|
self.cache[str(folder)] = signature, view
|
|
else:
|
|
view = cached[1]
|
|
if view['project_id'] in client.projects:
|
|
views.append(view)
|
|
except Exception:
|
|
errors += 1
|
|
self.cache = {k: v for k, v in self.cache.items() if k in present}
|
|
worker = self._worker(views)
|
|
cases = [{k: v for k, v in view.items() if k != 'calls'} | {'usage': aggregate([view])['buckets']}
|
|
for view in views]
|
|
return {'workers': worker, 'cases': cases, 'usage': aggregate(views),
|
|
'read_errors': errors, 'observed_at': datetime.now(timezone.utc).isoformat()}
|
|
|
|
def _worker(self, cases):
|
|
try:
|
|
path = self.service.workspace.runtime / 'worker/status.json'
|
|
status = json.loads(path.read_bytes())
|
|
if status.get('status') == 'STOPPED':
|
|
return []
|
|
telemetry = read_status(self.service.workspace.root)
|
|
if (telemetry.get('observer') != 'LIVE' or not status.get('service_id')
|
|
or status['service_id'] != telemetry.get('service_id')):
|
|
return [] # never present an old snapshot as a live worker
|
|
identity = status.get('case')
|
|
case = next((c for c in cases if identity and c['case_id'] == identity.get('case_id')
|
|
and c['package_id'] == identity.get('package_id')), None)
|
|
if identity and (not case or case['state'] in FINISHED):
|
|
return []
|
|
return [{'id': telemetry.get('service_id', 'analysis-worker'), 'name': 'Analyseworker 1',
|
|
'busy': bool(case), 'case': {k: v for k, v in case.items() if k != 'calls'} if case else None}]
|
|
except (OSError, ValueError, TypeError, AttributeError, TelemetryError):
|
|
return []
|