From 58fc00601e3e3eb96c8cbb34fb0d9a67e85c9f6f Mon Sep 17 00:00:00 2001 From: Nitsu45 Date: Wed, 21 Oct 2026 21:25:23 +0200 Subject: [PATCH] Add web overview with worker progress and separate correction token charts --- README.md | 2 + docs/intake-api.md | 1 + docs/test-results/web-console-20261021.md | 40 ++++ docs/web-console.md | 101 ++++++++++ pyproject.toml | 1 + src/viano_bug_analysis/api/app.py | 24 ++- src/viano_bug_analysis/api/dashboard.py | 189 ++++++++++++++++++ src/viano_bug_analysis/llm/accounting.py | 16 ++ src/viano_bug_analysis/llm/runner.py | 2 + .../schemas/llm-call-v1.schema.json | 1 + src/viano_bug_analysis/web/__init__.py | 1 + src/viano_bug_analysis/web/dashboard.css | 3 + src/viano_bug_analysis/web/dashboard.js | 140 +++++++++++++ src/viano_bug_analysis/web/index.html | 39 ++++ src/viano_bug_analysis/worker/service.py | 3 +- tests/dashboard-smoke.mjs | 55 +++++ tests/test_dashboard.py | 122 +++++++++++ tests/ui_dashboard_server.py | 66 ++++++ 18 files changed, 804 insertions(+), 2 deletions(-) create mode 100644 docs/test-results/web-console-20261021.md create mode 100644 docs/web-console.md create mode 100644 src/viano_bug_analysis/api/dashboard.py create mode 100644 src/viano_bug_analysis/llm/accounting.py create mode 100644 src/viano_bug_analysis/web/__init__.py create mode 100644 src/viano_bug_analysis/web/dashboard.css create mode 100644 src/viano_bug_analysis/web/dashboard.js create mode 100644 src/viano_bug_analysis/web/index.html create mode 100644 tests/dashboard-smoke.mjs create mode 100644 tests/test_dashboard.py create mode 100644 tests/ui_dashboard_server.py diff --git a/README.md b/README.md index bc9a832..58c5084 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,8 @@ Authenticated submission of new cases and completed-case feedback is documented external callers do not need to implement the internal package format. The [continuous analysis worker](docs/analysis-worker.md) consumes that inbox, routes cases to configured projects and resumes checkpointed analysis after restart. +The [web console](docs/web-console.md) shows active workers, case progress and token +charts, with corrections accounted for separately, and accepts new/return submissions. Requires Python 3.11 or newer. diff --git a/docs/intake-api.md b/docs/intake-api.md index e9231aa..6b7a08d 100644 --- a/docs/intake-api.md +++ b/docs/intake-api.md @@ -39,6 +39,7 @@ Weitere Projekte benötigen passende Adapter/Profile, keine separat gepflegten F | `GET /healthz` | Minimale öffentliche Erreichbarkeitsprüfung | | `GET /v1/profiles` | Für diesen Client freigegebene Projekte/Profile | | `GET /v1/schema` | Öffentliches JSON-Schema der Übergabemetadaten | +| `GET /v1/overview` | Projektbezogene Worker-/Fallübersicht und Tokenstatistik | | `POST /v1/submissions` | Multipart-Upload, `201` bei neuer Einreihung, `200` bei Wiederholung | | `GET /v1/submissions/{id}` | Quittung und aktueller verwalteter Zustand | | `GET /v1/submissions/{id}/package` | Abgeschlossene Akte herunterladen; sonst `409` | diff --git a/docs/test-results/web-console-20261021.md b/docs/test-results/web-console-20261021.md new file mode 100644 index 0000000..40cd89e --- /dev/null +++ b/docs/test-results/web-console-20261021.md @@ -0,0 +1,40 @@ +# Webübersicht – lokale Prüfung vom 2026-10-21 + +Ausgangspunkt `012a8d8`. Keine kostenpflichtigen Modellanfragen und kein echter +Analyseworker gegen Nutzeraufträge gestartet. + +224 automatisierte Tests bestanden: Dashboard/Zuordnung, Worker/Intake, +Antwortreparatur, Response-Recovery, Gateway, Fehlerdiagnostik und Cache. +Die Verbrauchszuordnung ergänzt ein optionales Feld im gespeicherten LLM-Aufruf; +vorhandene Aufrufe bleiben unverändert und bestehende Schemas kompatibel. + +Geprüft wurden insbesondere: + +- Zugriffsschutz und Projektfreigaben des Overview-Endpunkts; +- Fallname, Fortschritt nach abgeschlossenen Stufen und Ausblenden abgeschlossener Fälle; +- keine Anzeige alter Worker-Snapshots als lebender Worker; +- regulärer erster Aufruf und zusätzliche Korrektur als disjunkte Kategorien; +- abgeleitete Dokumente und doppelte Checkpoint-Verweise nicht erneut als Aufrufe gezählt; +- Denktokens nicht erneut auf die Ausgabe addiert; +- unbekannte Messwerte/Zuordnungen kenntlich gemacht; +- Auslieferung lokaler Assets mit Content Security Policy. + +Zusätzlich realer Browser-Smoke-Test mit unsichtbarem Edge und synthetischer +temporärer API: Desktop 1440×1200 und Mobil 390×1200, keine horizontale Überbreite, +keine JavaScript-/CSP-Konsolenfehler. Projektfilter liefert dieselbe Summe, +Rückgabeformular schaltet korrekt um, Abmeldung entfernt angezeigte Daten. Kein +Token wurde im Local Storage abgelegt. Die erste Wiederholung des Browsertests +traf eine zu frühe Testinteraktion vor Ausführung des defer-Scripts; die +Wartebedingung wurde auf vollständige Dokument-/Scriptbereitschaft korrigiert. + +Die Vorschau zeigte genau einen aktiven Worker/Fall bei 57 %, einen bereits +abgeschlossenen Fall ausschließlich in der Verbrauchsbilanz und 37.800 synthetische +Tokens mit 20,6 % Korrekturanteil. Diese Werte sind **UI-Fixtures, keine gemessenen +Analyse- oder Abrechnungskosten**. Screenshots liegen als lokale Prüfergebnisse +unter `service-data/ui-preview/desktop.png` und `mobile.png`. Vollständige +Testpakete und Browserprofile werden nicht als dauerhaftes Ergebnis archiviert. + +Speicheraudit vor den Tests: 16.939.129.428 Bytes Trial-Workspaces, +2.471.041 Bytes Fehlerkorpus und 11.156.187 Bytes vorhandene pytest-Temporärdaten; +die neuen synthetischen Tests/Screenshots sind klein und bleiben innerhalb der +20-GB-Grenze. Temporäre Vorschauakten wurden entfernt. diff --git a/docs/web-console.md b/docs/web-console.md new file mode 100644 index 0000000..8e490a3 --- /dev/null +++ b/docs/web-console.md @@ -0,0 +1,101 @@ +# Webübersicht + +Die Betriebsübersicht wird durch die bestehende Übergabe-API ausgeliefert: + +```powershell +.\.venv\Scripts\python.exe -m viano_bug_analysis.api serve --config config/api.local.toml +``` + +Anschließend `http://127.0.0.1:8080/` öffnen und mit einem freigegebenen **API-Token** +anmelden. Dies ist nicht der Gemini-Schlüssel. Der Token bleibt ausschließlich im +Arbeitsspeicher des Tabs, nicht in Cookies, Local Storage oder URLs. Für entfernte +Zugriffe HTTPS verwenden. Der Analyseworker läuft unabhängig davon wie unter +[Analyseworker](analysis-worker.md) beschrieben. Das Öffnen der Übersicht löst +keine Modellanfragen aus. + +## Worker und Fortschritt + +Die Startseite listet lebende Worker und darunter den jeweils bearbeiteten Fall +mit seinem Namen und einem Ladebalken. Der bestehende Dienst erlaubt einen Worker +je Workspace; die Darstellung ist bereits als Liste aufgebaut. Die Einführung +eines parallelen Worker-Pools ist damit nicht verbunden. + +Der Balken folgt abgeschlossenen Arbeitsschritten, nicht vergangener Zeit. Seine +sieben Anteile entsprechen intern Aufnahme, Fallaufbau, Interpretation, +Hypothesenbildung, Untersuchung, Diagnose und Codingauftrag. Die Oberfläche führt +diese Stufen nicht einzeln auf. Der Codingauftrag ist die letzte Stufe, auch wenn +sein Dokument fachlich erklärt, dass kein Codingauftrag benötigt wird. Bei +gemeinsamer Interpretation/Hypothesengenerierung können zwei Anteile gleichzeitig +fertig werden. Eine lang laufende Untersuchung füllt den Balken nicht künstlich. + +Der aktive Fall verschwindet bei `OUTPUT_READY`, `HANDED_OFF` oder +`FAILED_TO_ANALYZE` aus der Workeransicht und der Tabelle laufender Fälle. Er bleibt +in der Verbrauchsstatistik enthalten. Wartende Fälle bleiben in der Fallübersicht, +belegen aber keinen arbeitenden Worker. Ein alter Dienstsnapshot ohne lebenden +Telemetrie-Heartbeat wird nicht als aktiver Worker dargestellt. + +## Verbrauch + +- Kreisdiagramm: bekannte Eingabe- und Ausgabetokens je Phase; zusätzliche + Korrektur-/Reparatur-/Wiederholungsaufrufe haben ein eigenes orangefarbenes Segment. +- Balkendiagramm: Eingabe und Ausgabe einschließlich Denken je Kategorie. +- Der erste reguläre Aufruf bleibt bei seiner Phase, auch wenn sein Ergebnis später + zurückgewiesen wurde. Seine Korrektur wird ausschließlich dem Korrektursegment + zugeordnet. Abgeleitete, deterministisch reparierte Antwortdokumente sind keine + zusätzlichen Anbieteraufrufe und werden nicht erneut gezählt. +- Denktokens sind bereits Bestandteil der Ausgabetokens. Sie werden weder zum + Kreisdiagramm noch zum Gesamtverbrauch noch einmal hinzuaddiert. +- Neue LLM-Runden speichern ihre Verbrauchszuordnung deterministisch aus der + Operationskennung. Das verändert keine Prompts, Modellparameter oder Budgets. + Bereits gespeicherte Aufrufe bleiben unverändert. Bei älteren Akten wird nur + eindeutig erkennbare Korrekturinformation übernommen; sonst erscheint der Anteil + als „Nicht zugeordnet“. +- Fehlende Messwerte bleiben unbekannt. Die Diagramme summieren nur bekannte Werte + und weisen unvollständige Aufrufe aus. Bei vollständig fehlender Tokennutzung + lässt sich deren Größe nicht als Kreisanteil darstellen. + +Der Messumfang umfasst **alle erhaltenen, für den Client freigegebenen Akten und +deren Revisionen** im konfigurierten Workspace. Er ist keine Google-Abrechnung und +keine lebenslange Statistik bereits gelöschter Akten. Dauerhafte Testbilanzen unter +`docs/test-results/token-usage/` bleiben davon unabhängig. Ein Projektfilter wendet +sich auf Fallliste, Worker und Diagramme gleichzeitig an. Gemeinsame Planungsaufrufe +werden einmal als Fallplanung gezählt, nicht zusätzlich als Hypothesengenerierung. + +## Übergabe + +Die Oberfläche bietet ein einfaches Formular für neue wowdiag-/wowsrc-Aufträge und +für abgeschlossene ZIP-Akten mit Testerfeedback. Es verwendet dieselbe Multipart- +API und denselben Idempotenzmechanismus wie externe Programme. Bei unveränderten +Eingaben verwendet eine Wiederholung denselben Schlüssel. Anhänge und Fallakten +werden nicht clientseitig bearbeitet. Vollständige Falldetailansichten und eine +Bedienoberfläche für administrative Worker-Eingriffe bleiben weitere Ausbauschritte. + +## Datenschutz und Betrieb + +`GET /v1/overview` benötigt einen Bearer-Token und liefert nur freigegebene Projekte. +Prompts, vollständige Modellantworten, lokale Pfade und Zugangsdaten werden nicht +an den Browser übergeben. Fallnamen werden als Text dargestellt. Die Seite lädt +keine externen Fonts, Chartbibliotheken oder Analysedienste; Diagramme werden lokal +mit Canvas/CSS gezeichnet. Eine Content Security Policy begrenzt Ressourcen auf +denselben Ursprung. Abmeldung entfernt dargestellte Daten und Formularinhalte; +verzögert eintreffende Antworten aus einer vorherigen Anmeldung werden ignoriert. + +Die Ansicht aktualisiert sich alle vier Sekunden. Validierte Darstellungen werden +für unveränderte Paketgenerationen im API-Prozess zwischengespeichert; dies ist +eine Anzeigeoptimierung, kein fachlicher Zustand und kein Analysecache. + +## Lokale Prüfung + +```powershell +.\.venv\Scripts\python.exe -m pytest tests/test_dashboard.py -q +node tests/dashboard-smoke.mjs +``` + +Der Browser-Smoke-Test verwendet einen lokalen temporären API-Server, einen +unsichtbaren Edge-Browser und ausdrücklich künstliche Fall-/Verbrauchsdaten. Er +prüft Desktop, Mobilansicht, Projektfilter, Formularumschaltung, Abmeldung, +Tokenanzeige, Fortschritt und Browserfehler. Unter Windows wird der installierte +Edge erwartet; über `BROWSER_PATH` lässt sich ein anderes Chromium angeben. +Die Testserver/-browser werden beendet, ihre erzeugten temporären Verzeichnisse +nach Pfadprüfung entfernt. Screenshots als Prüfergebnis liegen unter +`service-data/ui-preview/`. Es werden keine LLM- oder Datenbankanbieter aufgerufen. diff --git a/pyproject.toml b/pyproject.toml index c528867..2026734 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ where = ["src"] [tool.setuptools.package-data] "viano_bug_analysis.schemas" = ["*.json"] "viano_bug_analysis.llm.prompts" = ["*.txt"] +"viano_bug_analysis.web" = ["*.html", "*.css", "*.js"] [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/src/viano_bug_analysis/api/app.py b/src/viano_bug_analysis/api/app.py index ef30074..2f36dcb 100644 --- a/src/viano_bug_analysis/api/app.py +++ b/src/viano_bug_analysis/api/app.py @@ -17,10 +17,13 @@ from starlette.formparsers import MultiPartParser, MultiPartException from .config import APIError from .contracts import schema from .service import IntakeService +from .dashboard import Dashboard +from importlib.resources import files def create_app(config): service = IntakeService(config) + dashboard = Dashboard(service) tokens = config.tokens() service.recover() active = 0 @@ -46,6 +49,24 @@ def create_app(config): authenticate(request) return JSONResponse(schema()) + async def overview(request): + client = authenticate(request) + response = JSONResponse(await run_in_threadpool(dashboard.overview, client)) + response.headers['Cache-Control'] = 'no-store' + return response + + async def web(request): + from starlette.responses import Response + name = request.path_params.get('asset', 'index.html') + if name not in {'index.html', 'dashboard.js', 'dashboard.css'}: + raise APIError('API_NOT_FOUND', 404) + data = files('viano_bug_analysis.web').joinpath(name).read_bytes() + response = Response(data, media_type={'index.html': 'text/html', 'dashboard.js': 'text/javascript', + 'dashboard.css': 'text/css'}[name]) + response.headers.update({'Content-Security-Policy': "default-src 'self'; script-src 'self'; style-src 'self'; connect-src 'self'; object-src 'none'; base-uri 'none'; frame-ancestors 'none'", + 'X-Content-Type-Options': 'nosniff', 'Referrer-Policy': 'no-referrer', 'Cache-Control': 'no-store'}) + return response + async def status(request): client = authenticate(request) return JSONResponse(await run_in_threadpool(service.status, request.path_params['submission'], client)) @@ -113,7 +134,8 @@ def create_app(config): async def storage_error(request, exc): return JSONResponse({'status': 'ERROR', 'code': 'API_STORAGE_ERROR'}, status_code=503) - app = Starlette(routes=[Route('/healthz', health), Route('/v1/profiles', profiles), + app = Starlette(routes=[Route('/', web), Route('/assets/{asset}', web), + Route('/v1/overview', overview), Route('/healthz', health), Route('/v1/profiles', profiles), Route('/v1/schema', contract), Route('/v1/submissions', submit, methods=['POST']), Route('/v1/submissions/{submission}', status), Route('/v1/submissions/{submission}/package', package)], exception_handlers={APIError: error, HTTPException: http_error, OSError: storage_error}) diff --git a/src/viano_bug_analysis/api/dashboard.py b/src/viano_bug_analysis/api/dashboard.py new file mode 100644 index 0000000..a37f378 --- /dev/null +++ b/src/viano_bug_analysis/api/dashboard.py @@ -0,0 +1,189 @@ +"""Scoped read models. No prompts, raw responses, credentials or host paths leave here.""" +import json +import threading +from datetime import datetime, timezone + +from viano_bug_analysis.lifecycle.checkpoints import checkpoint_chain +from viano_bug_analysis.lifecycle.storage import key +from viano_bug_analysis.package import PackageReader +from viano_bug_analysis.telemetry.inspection import read_status +from viano_bug_analysis.telemetry.config import TelemetryError + +FINISHED = {'OUTPUT_READY', 'HANDED_OFF', 'FAILED_TO_ANALYZE'} +LABELS = {'building': 'Fallaufbau', 'planning': 'Fallplanung', 'hypotheses': 'Hypothesen', + 'investigation': 'Untersuchung', 'diagnosis': 'Diagnose', 'coding': 'Codingauftrag', + 'correction': 'Korrekturen / Wiederholungen', 'unknown': 'Nicht zugeordnet'} + + +def read_json(reader, ref, limit=8*1024**2): + if not ref or ref not in reader.files or reader._archive.getinfo(ref).file_size > limit: + return {} + value = json.loads(reader.read_bytes(ref)) + return value if isinstance(value, dict) else {} + + +def phase(call, prompt): + if prompt.startswith('change_request'): + return 'coding' + return {'BUILDING_CASE': 'building', 'INTERPRETING_CASE': 'planning', + 'GENERATING_HYPOTHESES': 'hypotheses', 'INVESTIGATING_HYPOTHESES': 'investigation', + 'BUILDING_DIAGNOSIS': 'diagnosis'}.get(call['phase'], 'unknown') + + +def historical_classification(call, request): + prompt = request.get('prompt_version', '') + if call['rounds'] > 1 or 'repair' in prompt: + return 'correction' + for message in request.get('messages', []): + try: + data = json.loads(message.get('content', '')) + except (ValueError, TypeError): + continue + if not isinstance(data, dict): + continue + if data.get('task') == 'correct_rejected_output' or data.get('validation_feedback'): + return 'correction' + context = data.get('input', data) + if isinstance(context, dict): + if any(context.get(k) for k in ('response_repair', 'validation_feedback', 'correction_feedback')): + return 'correction' + if call['phase'] == 'INVESTIGATING_HYPOTHESES' and type(context.get('attempt')) is int: + return 'correction' if context['attempt'] > 0 else 'regular' + # Older records are not silently counted as regular when attribution is unclear. + return 'unknown' + + +def case_view(reader): + manifest = reader.manifest + revision = manifest['analysis_revision'] + artifacts = manifest['artifacts'] + docs = {p: read_json(reader, artifacts.get(p)) for p in ( + 'case', 'case_interpretation', 'hypotheses', 'investigation_state', 'diagnosis', 'change_request')} + def current(pointer): + doc = docs[pointer] + return doc.get('case', doc).get('analysis_revision') == revision + chain = checkpoint_chain(reader) + complete = [manifest['state'] != 'NEW', current('case'), current('case_interpretation'), + current('hypotheses'), (current('investigation_state') and docs['investigation_state'].get('completed') is True) + or manifest['state'] == 'BUILDING_DIAGNOSIS' or manifest['state'] in FINISHED, + current('diagnosis'), current('change_request')] + title = manifest['case_id'] + reports = docs['case'].get('bug', {}).get('reports', []) + if not reports: + index = read_json(reader, artifacts.get('case_inputs')) + reports = [read_json(reader, ref) for ref in index.get('reports', [])[:20]] + for report in reports: + if isinstance(report.get('title'), str) and report['title'].strip(): + title = report['title'][:240] + break + calls = [] + seen = set() + for _, cp in reversed(chain): + detail = cp['details'] + ref = detail.get('result_ref') + if detail.get('operation') != 'llm-round' or not ref or ref in seen: + continue + seen.add(ref) + call = read_json(reader, ref, 16*1024**2) + if not call or call.get('derived'): + continue # derived repaired documents are not additional provider calls + outcome = call.get('outcome', {}) + meta = outcome.get('metadata', outcome.get('diagnostics', {})) + request = read_json(reader, call.get('request_ref'), 8*1024**2) + prompt = meta.get('prompt_version', request.get('prompt_version', '')) + category = outcome.get('usage_classification') or historical_classification(call, request) + bucket = phase(call, prompt) if category == 'regular' else category + usage = {} + for field in ('input_tokens', 'output_tokens', 'reasoning_tokens', 'cached_input_tokens'): + value = meta.get(field) + usage[field] = value if type(value) is int and value >= 0 else None + calls.append({'bucket': bucket if bucket in LABELS else 'unknown', 'phase': phase(call, prompt), + 'category': category, 'day': cp['created_at'][:10], **usage}) + return {k: manifest[k] for k in ('case_id', 'package_id', 'analysis_revision', 'package_generation', 'state')} | { + 'project_id': manifest['project'].get('id'), 'title': title, + 'progress': round(sum(complete) / len(complete) * 100), 'calls': calls, + 'updated_at': manifest['updated_at']} + + +def aggregate(cases): + buckets = {name: {'id': name, 'label': label, 'input_tokens': 0, 'output_tokens': 0, + 'reasoning_tokens': 0, 'cached_input_tokens': 0, 'calls': 0, 'unknown_usage': 0} + for name, label in LABELS.items()} + days = {} + for case in cases: + for call in case['calls']: + bucket = buckets[call['bucket']] + bucket['calls'] += 1 + bucket['unknown_usage'] += int(call['input_tokens'] is None or call['output_tokens'] is None) + day = days.setdefault(call['day'], {'day': call['day'], 'regular': 0, 'correction': 0, 'unknown': 0}) + total = (call['input_tokens'] or 0) + (call['output_tokens'] or 0) + day[call['category'] if call['category'] in {'regular', 'correction'} else 'unknown'] += total + for field in ('input_tokens', 'output_tokens', 'reasoning_tokens', 'cached_input_tokens'): + bucket[field] += call[field] or 0 + return {'buckets': list(buckets.values()), 'days': sorted(days.values(), key=lambda d: d['day']), + 'scope': 'retained_cases_all_revisions', 'reasoning_included_in_output': True, + 'unknown_usage_calls': sum(b['unknown_usage'] for b in buckets.values())} + + +class Dashboard: + def __init__(self, service): + self.service = service + self.lock = threading.Lock() + self.cache = {} + + def overview(self, client): + with self.lock: + views, errors, present = [], 0, set() + for folder in self.service.workspace.cases.glob('*/*'): + try: + if folder.is_symlink() or folder.parent.is_symlink(): + continue + paths = sorted(folder.glob('g*.zip'), key=lambda p: int(p.stem[1:])) + if not paths: + continue + path = paths[-1] + if path.is_symlink(): + continue + stat = path.stat() + signature = (str(path), stat.st_size, stat.st_mtime_ns) + present.add(str(folder)) + cached = self.cache.get(str(folder)) + if not cached or cached[0] != signature: + with PackageReader(path) as reader: + m = reader.manifest + if folder.name != key(m['package_id']) or folder.parent.name != key(m['case_id']): + continue + view = case_view(reader) + self.cache[str(folder)] = signature, view + else: + view = cached[1] + if view['project_id'] in client.projects: + views.append(view) + except Exception: + errors += 1 + self.cache = {k: v for k, v in self.cache.items() if k in present} + worker = self._worker(views) + cases = [{k: v for k, v in view.items() if k != 'calls'} | {'usage': aggregate([view])['buckets']} + for view in views] + return {'workers': worker, 'cases': cases, 'usage': aggregate(views), + 'read_errors': errors, 'observed_at': datetime.now(timezone.utc).isoformat()} + + def _worker(self, cases): + try: + path = self.service.workspace.runtime / 'worker/status.json' + status = json.loads(path.read_bytes()) + if status.get('status') == 'STOPPED': + return [] + telemetry = read_status(self.service.workspace.root) + if (telemetry.get('observer') != 'LIVE' or not status.get('service_id') + or status['service_id'] != telemetry.get('service_id')): + return [] # never present an old snapshot as a live worker + identity = status.get('case') + case = next((c for c in cases if identity and c['case_id'] == identity.get('case_id') + and c['package_id'] == identity.get('package_id')), None) + if identity and (not case or case['state'] in FINISHED): + return [] + return [{'id': telemetry.get('service_id', 'analysis-worker'), 'name': 'Analyseworker 1', + 'busy': bool(case), 'case': {k: v for k, v in case.items() if k != 'calls'} if case else None}] + except (OSError, ValueError, TypeError, AttributeError, TelemetryError): + return [] diff --git a/src/viano_bug_analysis/llm/accounting.py b/src/viano_bug_analysis/llm/accounting.py new file mode 100644 index 0000000..e775436 --- /dev/null +++ b/src/viano_bug_analysis/llm/accounting.py @@ -0,0 +1,16 @@ +"""Classify additional repair calls without moving the first rejected call.""" +import re + + +def classification(operation, rounds, prompt): + if rounds > 1 or any(word in operation for word in (':answer-repair:', ':retry:', 'correction-')) or 'repair' in prompt: + return 'correction' + for pattern, first in ((r'^investigation:v1:[^:]+:[0-9]+:([0-9]+):', 0), + (r'^investigation-startup-review:v1:([0-9]+):', 0), + (r'^entity-resolution:v1:[^:]+:([0-9]+)$', 1), + (r'^(?:diagnosis|change-request):v1:.*?attempt:([0-9]+)', 1)): + if match := re.search(pattern, operation): + return 'correction' if int(match[1]) > first else 'regular' + if re.fullmatch(r'(?:case-interpretation|hypotheses):v1:default', operation): + return 'regular' + return 'unknown' diff --git a/src/viano_bug_analysis/llm/runner.py b/src/viano_bug_analysis/llm/runner.py index a40cea1..43cbf67 100644 --- a/src/viano_bug_analysis/llm/runner.py +++ b/src/viano_bug_analysis/llm/runner.py @@ -173,6 +173,8 @@ class CaseLLMRunner: record = {"format": "viano-llm-call", "format_version": 1, "case_id": case_id, "package_id": package_id, "analysis_revision": manifest["analysis_revision"], **details, "outcome": outcome} + from .accounting import classification + record['outcome']['usage_classification'] = classification(operation_id, rounds, request['prompt_version']) if phase == State.INVESTIGATING_HYPOTHESES: from .investigation_size import characters, TARGET, LIMIT record['outcome']['prompt_size'] = { diff --git a/src/viano_bug_analysis/schemas/llm-call-v1.schema.json b/src/viano_bug_analysis/schemas/llm-call-v1.schema.json index 4a77684..0e48d15 100644 --- a/src/viano_bug_analysis/schemas/llm-call-v1.schema.json +++ b/src/viano_bug_analysis/schemas/llm-call-v1.schema.json @@ -85,6 +85,7 @@ "outcome": { "type": "object", "properties": { + "usage_classification": {"enum": ["regular", "correction", "unknown"]}, "http_status": { "type": "integer", "minimum": 100, diff --git a/src/viano_bug_analysis/web/__init__.py b/src/viano_bug_analysis/web/__init__.py new file mode 100644 index 0000000..a13f57a --- /dev/null +++ b/src/viano_bug_analysis/web/__init__.py @@ -0,0 +1 @@ +"""Local web console assets; no external analytics, fonts or chart dependencies.""" diff --git a/src/viano_bug_analysis/web/dashboard.css b/src/viano_bug_analysis/web/dashboard.css new file mode 100644 index 0000000..7b404f2 --- /dev/null +++ b/src/viano_bug_analysis/web/dashboard.css @@ -0,0 +1,3 @@ +:root{--bg:#f4f5f8;--ink:#202536;--muted:#727a8c;--line:#e7e9f0;--accent:#5364de;--orange:#e9954c}*{box-sizing:border-box}html{scroll-behavior:smooth}body{margin:0;background:var(--bg);color:var(--ink);font:14px/1.55 system-ui,-apple-system,Segoe UI,sans-serif}.rail{position:fixed;inset:0 auto 0 0;width:210px;background:#202637;color:#a9b1c6;padding:36px 24px;display:flex;flex-direction:column}.brand{color:white;font-size:30px;font-weight:800;letter-spacing:-2px;text-decoration:none}.brand span{color:#a6afff}.brand small{display:block;font-size:12px;letter-spacing:2px;color:#9aa4bf;font-weight:500}nav{display:grid;gap:10px;margin-top:58px}nav a{color:#aeb7cc;text-decoration:none;padding:12px;border-radius:8px}nav a:hover,nav .selected{background:#323b56;color:white}.rail-bottom{margin-top:auto;font-size:10px;letter-spacing:1px;color:#7f8aa5}.rail-bottom span{font-size:12px;color:#c2cae0;letter-spacing:0}main{margin-left:210px;padding:42px 42px 30px;max-width:1660px}header{display:flex;align-items:center;justify-content:space-between;margin-bottom:30px}h1{font-size:36px;letter-spacing:-1.4px;line-height:1.2;margin:8px 0}h2{font-size:19px;letter-spacing:-.4px;margin:4px 0}p{color:var(--muted);margin:6px 0 12px}.eyebrow{font-size:10px;letter-spacing:1.8px;font-weight:750;color:#8b93a7}.live{font-size:12px;color:var(--muted);display:flex;align-items:center;gap:8px}.dot{width:7px;height:7px;background:#8e98ad;border-radius:50%}.connected .dot{background:#3db28a}.panel{background:#fff;border:1px solid var(--line);border-radius:14px;padding:25px;margin-bottom:24px;box-shadow:0 3px 14px #17243c03}.section-head{display:flex;align-items:center;justify-content:space-between;gap:15px;margin-bottom:22px}.stats{display:grid;grid-template-columns:repeat(4,1fr);gap:18px;margin-bottom:24px}.stats article{background:#fff;border:1px solid var(--line);padding:22px;border-radius:12px}.stats span{color:var(--muted);font-size:12px}.stats strong{display:block;font-size:30px;font-weight:650;letter-spacing:-1px;margin:8px 0}.stats small{color:#969daf;font-size:10px}.badge{background:#eef0fd;color:#6572cb;border-radius:20px;padding:5px 12px;font-size:11px}.worker{border-top:1px solid var(--line);padding:18px 0}.worker:first-child{border-top:0;padding-top:0}.worker-top{display:flex;align-items:center;gap:9px;font-weight:600}.worker-top i{width:8px;height:8px;background:#42b98e;border-radius:50%}.worker-case{display:flex;justify-content:space-between;gap:15px;margin:12px 0 8px;color:#646d82}.progress{height:8px;background:#edf0f7;border-radius:8px;overflow:hidden}.progress-fill{height:100%;background:var(--accent);border-radius:8px;transition:width .5s}.hint{font-size:11px;color:#929aab;margin:14px 0 0}.charts{display:grid;grid-template-columns:1fr 1fr;gap:24px}.charts .panel{min-width:0}.pie-layout{display:grid;grid-template-columns:minmax(170px,1fr) 1fr;align-items:center;gap:20px}canvas{display:block;width:100%;max-width:250px;height:auto;margin:auto}.legend{padding:0;margin:0;list-style:none}.legend li{display:grid;grid-template-columns:9px 1fr auto;gap:8px;font-size:11px;align-items:center;margin:10px 0}.swatch{height:8px;width:8px;border-radius:2px}.legend b{font-size:11px;font-weight:600}.keys{display:flex;gap:20px;color:var(--muted);font-size:11px;margin-bottom:18px}.keys span:before{content:'';display:inline-block;width:8px;height:8px;background:#8e9aed;border-radius:2px;margin-right:6px}.keys .output-key:before{background:#303f96}.bar-row{margin:14px 0}.bar-label{display:flex;justify-content:space-between;font-size:11px;margin-bottom:5px;color:#68728a}.bar-track{height:12px;background:#f1f3f8;border-radius:3px;display:flex;overflow:hidden}.bar-input{background:#8e9aed}.bar-output{background:#303f96}.bar-row.correction .bar-input{background:#f2c397}.bar-row.correction .bar-output{background:var(--orange)}table{width:100%;border-collapse:collapse;font-size:12px;text-align:left}th{color:#929aaa;font-size:10px;letter-spacing:.5px;font-weight:600;padding:12px}td{padding:15px 12px;border-top:1px solid var(--line)}td small{display:block;color:var(--muted)}td .progress{width:110px;height:5px}label{display:block;color:#788297;font-size:11px;margin-bottom:14px}input,select,textarea{display:block;width:100%;font:inherit;color:var(--ink);border:1px solid #dfe3ed;background:white;border-radius:7px;padding:10px 12px;margin-top:5px}input:focus,textarea:focus,select:focus{outline:2px solid #aab3fa;outline-offset:1px}input[type=file]{font-size:11px;padding:8px}.filter{display:flex;align-items:center;gap:9px;margin:0}.filter select{width:auto;max-width:240px;margin:0}.form-grid{display:grid;grid-template-columns:1fr 1fr;gap:0 22px}button{font:inherit;background:var(--accent);color:white;border:0;border-radius:7px;padding:11px 18px;cursor:pointer}button:hover{background:#4654be}button:disabled{opacity:.5;cursor:wait}#logout{font-size:11px;padding:6px 10px;margin-top:12px;background:#353f58}.inline{display:flex;gap:10px}.inline input{margin:0}.login{display:flex;gap:40px;align-items:center}.login form{flex:1;min-width:240px}.error{background:#fff0eb;color:#b04c34;padding:12px 16px;border-radius:8px;font-size:12px}.empty{padding:20px 0;color:#9099ad;font-size:13px}.table-wrap{overflow:auto}footer{font-size:10px;color:#9299aa;padding:0 0 20px}[hidden]{display:none!important}@media(min-width:1550px){main{margin-right:auto}}@media(max-width:1150px){main{padding:28px 24px}.rail{width:170px;padding:28px 18px}main{margin-left:170px}.charts{grid-template-columns:1fr}.stats{grid-template-columns:1fr 1fr}}@media(max-width:700px){.rail{position:static;width:100%;padding:16px 20px;flex-direction:row;align-items:center;gap:20px}.brand{font-size:23px}.brand small,.rail-bottom{display:none}nav{display:flex;margin:0;gap:0;font-size:11px}nav a{padding:7px}main{margin:0;padding:24px 16px}header{align-items:flex-start;gap:12px}h1{font-size:29px}.live{font-size:10px;max-width:100px}.stats{gap:10px}.stats article{padding:15px}.stats strong{font-size:25px}.panel{padding:19px}.section-head{flex-wrap:wrap}.form-grid{grid-template-columns:1fr}.pie-layout{gap:12px;grid-template-columns:1fr 1fr}.login{display:block}.legend li{font-size:10px}.inline{flex-wrap:wrap}} + +.worker-case span:last-child{white-space:nowrap;flex-shrink:0} diff --git a/src/viano_bug_analysis/web/dashboard.js b/src/viano_bug_analysis/web/dashboard.js new file mode 100644 index 0000000..d5a044f --- /dev/null +++ b/src/viano_bug_analysis/web/dashboard.js @@ -0,0 +1,140 @@ +'use strict'; +const $ = id => document.getElementById(id); +const colors = ['#6977db','#8d9ae7','#9e83c9','#50a994','#548ba9','#344779','#e9954c','#b6bdca']; +const names = {NEW:'Eingang',VALIDATED:'Angenommen',BUILDING_CASE:'Fallaufbau',INTERPRETING_CASE:'Fallplanung',GENERATING_HYPOTHESES:'Hypothesen',INVESTIGATING_HYPOTHESES:'Untersuchung',BUILDING_DIAGNOSIS:'Abschluss',BLOCKED:'Wartet',REOPENED:'Wiedereröffnet'}; +const finished = new Set(['OUTPUT_READY','FAILED_TO_ANALYZE','HANDED_OFF']); +const number = n => new Intl.NumberFormat('de-DE').format(n); +let token = '', timer = null, overview = null, projects = [], sending = false, submission = null, session = 0; +const text = (tag, value, cls) => { const e=document.createElement(tag); e.textContent=value; if(cls)e.className=cls; return e; }; +function error(message='') { $('error').hidden=!message; $('error').textContent=message; } +async function api(path, options={}) { + const credential=token; + const response=await fetch(path,{...options,headers:{...options.headers,Authorization:`Bearer ${credential}`},cache:'no-store'}); + const data=await response.json(); + if(!response.ok) { if(response.status===401 && credential===token) logout(); throw new Error(data.code || `HTTP ${response.status}`); } + return data; +} +function progress(value, label) { + const outer=text('div','','progress'); outer.setAttribute('role','progressbar'); + outer.setAttribute('aria-label',label); outer.setAttribute('aria-valuemin','0'); outer.setAttribute('aria-valuemax','100'); outer.setAttribute('aria-valuenow',String(value)); + const inner=text('div','','progress-fill'); inner.style.width=`${value}%`; outer.append(inner); return outer; +} +function logout() { + session++;token=''; clearTimeout(timer); $('token').value=''; $('content').hidden=true; $('login').hidden=false; + $('logout').hidden=true; $('connection').textContent='Nicht verbunden'; document.body.classList.remove('connected'); + $('updated').textContent='Warte auf Verbindung'; overview=null;projects=[];submission=null; + $('submit-form').reset();$('receipt').textContent='';$('new-files').hidden=false;$('return-files').hidden=true; + ['workers','case-rows','bars','legend','project','project-filter'].forEach(id=>$(id).replaceChildren()); + ['active-count','token-count','correction-count','finished-count'].forEach(id=>$(id).textContent='—'); + $('pie').getContext('2d').clearRect(0,0,360,360); +} +function renderWorkers(workers, project) { + const root=$('workers'); root.replaceChildren(); + const visible=workers.filter(w=>!project || (w.case && w.case.project_id===project)); + $('worker-count').textContent=`${visible.length} online`; + if(!visible.length)root.append(text('div','Kein aktiver Worker für diese Auswahl.','empty')); + visible.forEach(worker=>{ + const row=text('article','','worker'); const head=text('div','','worker-top'); + head.append(document.createElement('i'),text('span',worker.name)); row.append(head); + if(worker.case) { + const c=worker.case, line=text('div','','worker-case'); + line.append(text('span',`${c.case_id} · ${c.title}`),text('span',`${c.progress} %`)); + row.append(line,progress(c.progress,`Fortschritt ${c.case_id}`)); + } else row.append(text('p','Bereit für den nächsten Auftrag.','hint')); + root.append(row); + }); +} +function sumBuckets(cases) { + // Per-project aggregation comes from server-filtered individual usage summaries. + return cases.reduce((map,c)=>{ + (c.usage || []).forEach(b=>{const a=map.get(b.id)||{...b,input_tokens:0,output_tokens:0,calls:0,unknown_usage:0}; + ['input_tokens','output_tokens','calls','unknown_usage'].forEach(k=>a[k]+=b[k]||0);map.set(b.id,a);});return map; + },new Map()); +} +function renderPie(buckets) { + const canvas=$('pie'), ctx=canvas.getContext('2d'), size=360; + ctx.clearRect(0,0,size,size); const total=buckets.reduce((n,b)=>n+b.input_tokens+b.output_tokens,0); + const legend=$('legend'); legend.replaceChildren(); let angle=-Math.PI/2; + if(!total) {ctx.beginPath();ctx.arc(180,180,140,0,Math.PI*2);ctx.fillStyle='#edf0f7';ctx.fill();} + buckets.forEach((b,i)=>{ + const count=b.input_tokens+b.output_tokens; if(!count&&!b.calls)return; + const color=colors[i%colors.length]; if(total&&count) {const end=angle+count/total*Math.PI*2; + ctx.beginPath();ctx.moveTo(180,180);ctx.arc(180,180,140,angle,end);ctx.closePath();ctx.fillStyle=color;ctx.fill();angle=end;} + const item=document.createElement('li'), swatch=text('span','','swatch');swatch.style.background=color; + const name=text('span',b.label), val=text('b',total?`${(count/total*100).toFixed(1).replace('.',',')} %`:'—'); + item.title=`${b.label}: ${number(count)} Tokens, ${b.calls} Aufrufe${b.unknown_usage?' · Messwerte unvollständig':''}`; + item.append(swatch,name,val);legend.append(item); + }); + canvas.setAttribute('aria-label',total?'Tokenanteile: '+buckets.filter(b=>b.input_tokens+b.output_tokens).map(b=>`${b.label}: ${number(b.input_tokens+b.output_tokens)}`).join(', '):'Noch keine gemessene Tokennutzung'); +} +function renderBars(buckets) { + const root=$('bars');root.replaceChildren(); const max=Math.max(1,...buckets.map(b=>b.input_tokens+b.output_tokens)); + buckets.filter(b=>b.calls).forEach(b=>{ + const row=text('div','',`bar-row ${b.id}`), label=text('div','','bar-label'); + label.append(text('span',b.label),text('span',number(b.input_tokens+b.output_tokens))); + const track=text('div','','bar-track'), input=text('div','','bar-input'), output=text('div','','bar-output'); + input.style.width=`${b.input_tokens/max*100}%`;output.style.width=`${b.output_tokens/max*100}%`; + row.title=`Eingabe: ${number(b.input_tokens)} · Ausgabe inkl. Denken: ${number(b.output_tokens)}`; + track.append(input,output);row.append(label,track);root.append(row); + }); + if(!root.children.length)root.append(text('div','Noch keine gemessene Tokennutzung.','empty')); +} +function render() { + if(!overview)return; + const project=$('project-filter').value; + const cases=overview.cases.filter(c=>!project || c.project_id===project), active=cases.filter(c=>!finished.has(c.state)); + const buckets=project?[...sumBuckets(cases).values()]:overview.usage.buckets; + const total=buckets.reduce((n,b)=>n+b.input_tokens+b.output_tokens,0), corrections=buckets.find(b=>b.id==='correction'); + const corrected=corrections?corrections.input_tokens+corrections.output_tokens:0; + $('active-count').textContent=number(active.length);$('finished-count').textContent=number(cases.length-active.length); + $('token-count').textContent=number(total);$('correction-count').textContent=total?`${(corrected/total*100).toFixed(1).replace('.',',')} %`:'—'; + renderWorkers(overview.workers,project);renderPie(buckets);renderBars(buckets); + const missing=buckets.reduce((n,b)=>n+b.unknown_usage,0), unassigned=buckets.find(b=>b.id==='unknown'); + $('coverage').textContent=`${buckets.reduce((n,b)=>n+b.calls,0)} erfasste Aufrufe. ${missing?`${missing} Aufrufe mit fehlenden Tokenmesswerten; die Diagramme zeigen nur bekannte Werte.`:'Alle erfassten Aufrufe mit vollständigen Eingabe-/Ausgabewerten.'}${unassigned?.calls?` ${unassigned.calls} ältere Aufrufe ohne sichere Korrekturzuordnung.`:''}`; + const root=$('case-rows');root.replaceChildren(); + active.forEach(c=>{const row=document.createElement('tr'), title=document.createElement('td'); + title.append(text('strong',c.case_id),text('small',c.title));row.append(title,text('td',c.project_id),text('td',names[c.state]||c.state),text('td',String(c.analysis_revision))); + const cell=document.createElement('td');cell.append(progress(c.progress,`Fortschritt ${c.case_id}`));row.append(cell);root.append(row);}); + if(!active.length){const row=document.createElement('tr'),cell=text('td','Keine offenen Fälle.','empty');cell.colSpan=5;row.append(cell);root.append(row);} +} +async function refresh() { + clearTimeout(timer); if(!token)return;const generation=session; + try {const response=await api('/v1/overview');if(generation!==session)return;overview=response;render();$('updated').textContent=`Aktualisiert ${new Date().toLocaleTimeString('de-DE')}`; + error(overview.read_errors?'Einige Akten konnten nicht gelesen werden. Die Verbrauchsdaten können unvollständig sein.':''); + } catch(e) {if(generation===session)error(`Aktualisierung fehlgeschlagen: ${e.message}`);} + if(token && generation===session)timer=setTimeout(refresh,4000); +} +$('login-form').addEventListener('submit',async e=>{ + e.preventDefault();const generation=++session;token=$('token').value.trim();error(); + try {const response=await api('/v1/profiles');if(generation!==session)return;projects=response.projects; + $('project').replaceChildren();$('project-filter').replaceChildren(new Option('Alle freigegebenen Projekte','')); + projects.forEach(p=>{$('project').add(new Option(p.project_id,p.project_id));$('project-filter').add(new Option(p.project_id,p.project_id));}); + $('login').hidden=true;$('content').hidden=false;$('logout').hidden=false;$('token').value=''; + $('connection').textContent='Verbunden';document.body.classList.add('connected');await refresh(); + } catch(e) {if(generation===session){token='';error(`Verbindung fehlgeschlagen: ${e.message}`);}else if(!token)error('Verbindung fehlgeschlagen: API-Zugangsschlüssel prüfen.');} +}); +$('logout').addEventListener('click',logout);$('project-filter').addEventListener('change',render); +$('kind').addEventListener('change',()=>{$('new-files').hidden=$('kind').value!=='new_case';$('return-files').hidden=$('kind').value!=='case_return';}); +$('submit-form').addEventListener('submit',async e=>{ + e.preventDefault();if(sending)return;error(); + const project=projects.find(p=>p.project_id===$('project').value), kind=$('kind').value; + if(!project)return; + const observations=$('observations').value.split('\n').map(v=>v.trim()).filter(Boolean); + if(!observations.length)return error('Bitte mindestens eine Beobachtung angeben.'); + const metadata={kind,project_id:project.project_id,profile_id:project.profile_id,case_id:$('case-id').value.trim()}; + const form=new FormData(), chosen=[]; + function append(role, files){for(const file of files){form.append(role,file);chosen.push([role,file.name,file.size,file.lastModified]);}} + if(kind==='new_case') {if(!$('wowdiag').files.length)return error('Bitte das wowdiag-Archiv auswählen.'); + metadata.report={title:$('title').value.trim(),observations};append('wowdiag',$('wowdiag').files);append('wowsrc',$('wowsrc').files); + } else {if(!$('case-package').files.length)return error('Bitte die abgeschlossene ZIP-Akte auswählen.'); + metadata.feedback={outcome:$('outcome').value,summary:$('title').value.trim(),observations};append('case_package',$('case-package').files);} + append('attachments',$('attachments').files);form.append('metadata',JSON.stringify(metadata)); + const signature=JSON.stringify([metadata,chosen]); + if(!submission || submission.signature!==signature)submission={signature,key:crypto.randomUUID()}; + const generation=session;sending=true;$('send').disabled=true;$('receipt').textContent='Auftrag wird übertragen …'; + try {const receipt=await api('/v1/submissions',{method:'POST',headers:{'Idempotency-Key':submission.key},body:form}); + if(generation!==session)return; + $('receipt').textContent=`Angenommen: ${receipt.case_id} · ${receipt.status} · Auftrags-ID ${receipt.submission_id}`;await refresh(); + } catch(e) {if(generation===session){$('receipt').textContent='';error(`Übergabe fehlgeschlagen: ${e.message}. Bei unveränderten Eingaben wird derselbe Idempotenzschlüssel wiederverwendet.`);}} + finally{sending=false;$('send').disabled=false;} +}); diff --git a/src/viano_bug_analysis/web/index.html b/src/viano_bug_analysis/web/index.html new file mode 100644 index 0000000..e9332e4 --- /dev/null +++ b/src/viano_bug_analysis/web/index.html @@ -0,0 +1,39 @@ + + + +BugAnalysis · Übersicht + + + +
+
BETRIEBSÜBERSICHT

Analyse im Blick.

Aktuelle Arbeit und messbarer Verbrauch.

Warte auf Verbindung
+ + +
diff --git a/src/viano_bug_analysis/worker/service.py b/src/viano_bug_analysis/worker/service.py index 94ea74a..beaefa9 100644 --- a/src/viano_bug_analysis/worker/service.py +++ b/src/viano_bug_analysis/worker/service.py @@ -55,7 +55,8 @@ class AnalysisWorker: def _status(self, status, *, code=None, case=None): record = {'status': status, 'updated_at': timestamp(), 'pid': os.getpid(), - 'code': safe_code(code) if code else None, 'case': case} + 'code': safe_code(code) if code else None, 'case': case, + 'service_id': getattr(self.telemetry, 'service_id', None)} atomic_json(self.root / 'status.json', record) def _due(self, value): diff --git a/tests/dashboard-smoke.mjs b/tests/dashboard-smoke.mjs new file mode 100644 index 0000000..4413690 --- /dev/null +++ b/tests/dashboard-smoke.mjs @@ -0,0 +1,55 @@ +// Real Chromium smoke test; Node built-ins only. No paid model calls. +import {spawn} from 'node:child_process'; +import {mkdtemp, mkdir, writeFile, rm} from 'node:fs/promises'; +import {tmpdir} from 'node:os'; +import path from 'node:path'; +const port=18765, debugPort=18766; +const pause=ms=>new Promise(r=>setTimeout(r,ms)); +const output=path.resolve('service-data/ui-preview');await mkdir(output,{recursive:true}); +const profile=await mkdtemp(path.join(tmpdir(),'vba-browser-')); +const fixtureRoot=await mkdtemp(path.join(tmpdir(),'vba-ui-data-')); +let browser,server,ws; const errors=[]; +async function until(fn, milliseconds=15000){const start=Date.now();while(Date.now()-start (await fetch(`http://127.0.0.1:${port}/healthz`)).ok); + browser=spawn(process.env.BROWSER_PATH||'C:/Program Files (x86)/Microsoft/Edge/Application/msedge.exe',[ + '--headless=new','--disable-gpu','--no-first-run','--no-default-browser-check', + `--user-data-dir=${profile}`,`--remote-debugging-port=${debugPort}`,'about:blank'],{windowsHide:true,stdio:'ignore'}); + const tabs=await until(async()=>await (await fetch(`http://127.0.0.1:${debugPort}/json`)).json()); + ws=new WebSocket(tabs.find(t=>t.type==='page').webSocketDebuggerUrl); + await new Promise((resolve,reject)=>{ws.onopen=resolve;ws.onerror=reject;}); + let next=0;const pending=new Map(); + ws.onmessage=e=>{const m=JSON.parse(e.data);if(m.id){const p=pending.get(m.id);pending.delete(m.id);m.error?p.reject(new Error(m.error.message)):p.resolve(m.result);} + if(m.method==='Runtime.exceptionThrown')errors.push(m.params.exceptionDetails.text); + if(m.method==='Log.entryAdded'&&m.params.entry.level==='error')errors.push(m.params.entry.text);}; + const send=(method,params={})=>new Promise((resolve,reject)=>{const id=++next;pending.set(id,{resolve,reject});ws.send(JSON.stringify({id,method,params}));}); + const evaluate=async expression=>{const r=await send('Runtime.evaluate',{expression,returnByValue:true,awaitPromise:true});if(r.exceptionDetails)throw new Error(r.exceptionDetails.text);return r.result.value;}; + await send('Runtime.enable');await send('Log.enable');await send('Page.enable'); + await send('Emulation.setDeviceMetricsOverride',{width:1440,height:1200,deviceScaleFactor:1,mobile:false}); + await send('Page.navigate',{url:`http://127.0.0.1:${port}/`}); + await until(()=>evaluate("document.readyState === 'complete' && typeof render === 'function'")); + await evaluate("document.getElementById('token').value='ui_fixture_"+'x'.repeat(40)+"';document.getElementById('login-form').dispatchEvent(new Event('submit',{bubbles:true,cancelable:true}));"); + await until(()=>evaluate("document.getElementById('token-count').textContent==='37.800'")); + const state=await evaluate("({workers:document.querySelectorAll('.worker').length,cases:document.querySelectorAll('#case-rows tr').length,active:document.getElementById('active-count').textContent,correction:document.getElementById('correction-count').textContent,overflow:document.documentElement.scrollWidth>innerWidth,stored:localStorage.length,progress:document.querySelector('.worker [role=progressbar]').getAttribute('aria-valuenow')})"); + if(state.workers!==1||state.cases!==1||state.active!=='1'||state.stored!==0||state.overflow||state.progress!=='57')throw new Error(JSON.stringify(state)); + await writeFile(path.join(output,'desktop.png'),Buffer.from((await send('Page.captureScreenshot',{format:'png'})).data,'base64')); + await send('Emulation.setDeviceMetricsOverride',{width:390,height:1200,deviceScaleFactor:1,mobile:true});await pause(200); + if(await evaluate('document.documentElement.scrollWidth>innerWidth'))throw new Error('Mobile horizontal overflow'); + await writeFile(path.join(output,'mobile.png'),Buffer.from((await send('Page.captureScreenshot',{format:'png'})).data,'base64')); + await evaluate("document.getElementById('project-filter').value='viano-core';document.getElementById('project-filter').dispatchEvent(new Event('change')); "); + if(await evaluate("document.getElementById('token-count').textContent")!=='37.800')throw new Error('Project aggregation differs'); + await evaluate("document.getElementById('kind').value='case_return';document.getElementById('kind').dispatchEvent(new Event('change'));"); + if(!await evaluate("document.getElementById('new-files').hidden&&!document.getElementById('return-files').hidden"))throw new Error('Return form not selected'); + await evaluate("document.getElementById('logout').click()"); + if(!await evaluate("document.getElementById('content').hidden&&!document.getElementById('login').hidden&&document.getElementById('token-count').textContent==='—'"))throw new Error('Logout retained visible case data'); + if(errors.filter(e=>!e.includes('404')).length)throw new Error(errors.join('\n')); + console.log(JSON.stringify({status:'PASS',...state,console_errors:errors.filter(e=>!e.includes('404'))})); +} finally { + ws?.close();browser?.kill();server?.kill();await pause(400); + for(const [folder,prefix] of [[profile,'vba-browser-'],[fixtureRoot,'vba-ui-data-']]) { + const target=path.resolve(folder),relative=path.relative(path.resolve(tmpdir()),target); + if(!relative||relative.startsWith('..')||path.isAbsolute(relative)||!path.basename(target).startsWith(prefix))throw new Error('Unsafe temporary cleanup'); + await rm(target,{recursive:true,force:true,maxRetries:5,retryDelay:200}); + } +} diff --git a/tests/test_dashboard.py b/tests/test_dashboard.py new file mode 100644 index 0000000..dc7a219 --- /dev/null +++ b/tests/test_dashboard.py @@ -0,0 +1,122 @@ +import json +from types import SimpleNamespace + +import pytest +pytest.importorskip('starlette') +pytest.importorskip('httpx') +pytest.importorskip('multipart') +from starlette.testclient import TestClient + +from viano_bug_analysis.api import APIConfig, Client, Project +from viano_bug_analysis.api.app import create_app +from viano_bug_analysis.api.dashboard import aggregate, case_view +from viano_bug_analysis.llm.accounting import classification +from viano_bug_analysis.lifecycle.storage import atomic_json +from test_case_builder import case_setup + + +@pytest.mark.parametrize('operation,rounds,prompt,expected', [ + ('case-interpretation:v1:default',1,'case_plan_v2','regular'), + ('case-interpretation:v1:correction-interpretation-1',1,'case_plan_v2','correction'), + ('hypotheses:v1:default',2,'hypotheses_v4','correction'), + ('investigation:v1:H1:1:0:token',1,'investigation_lean_v21','regular'), + ('investigation:v1:H1:1:1:token',1,'investigation_lean_v21','correction'), + ('investigation-startup-review:v1:0:token',1,'startup','regular'), + ('investigation-startup-review:v1:1:token',1,'startup','correction'), + ('entity-resolution:v1:plan:1',1,'entity_resolution_plan_v4','regular'), + ('entity-resolution:v1:assessment:2',1,'entity_resolution_assessment_v4','correction'), + ('diagnosis:v1:early:2:attempt:1',1,'diagnosis_v8','regular'), + ('change-request:v1:attempt:2',1,'change_request_v9','correction'), + ('other',1,'unknown','unknown'), +]) +def test_call_classification(operation, rounds, prompt, expected): + assert classification(operation,rounds,prompt) == expected + + +class Reader: + def __init__(self, docs, manifest): + self.docs, self.manifest = docs, manifest + self.files = docs + self._archive = SimpleNamespace(getinfo=lambda ref: SimpleNamespace(file_size=len(json.dumps(docs[ref])))) + def read_bytes(self, ref): + return json.dumps(self.docs[ref]).encode() + + +def test_correction_slice_disjoint_derived_excluded_and_thinking_not_doubled(monkeypatch): + import viano_bug_analysis.api.dashboard as module + docs = {'case': {'case': {'analysis_revision': 1}, 'bug': {'reports':[{'title':'Testfall'}]}}, + 'request': {'prompt_version':'investigation_lean_v21'}} + chain = [] + for i, category in enumerate(('regular', 'correction', 'regular')): + ref = f'call{i}' + docs[ref] = {'rounds':1, 'phase':'INVESTIGATING_HYPOTHESES', 'request_ref':'request', + 'derived': i==2, 'outcome': {'usage_classification':category, + 'metadata':{'input_tokens':100,'output_tokens':40,'reasoning_tokens':30,'cached_input_tokens':20}}} + chain.append(('cp'+str(i), {'details':{'operation':'llm-round','result_ref':ref}, 'created_at':'2026-10-21T10:00:00Z'})) + # Duplicate checkpoint provenance does not charge a provider call twice. + chain.append(chain[0]) + monkeypatch.setattr(module,'checkpoint_chain',lambda reader:chain) + reader=Reader(docs,{'case_id':'BUG-1','package_id':'one','analysis_revision':1,'package_generation':10, + 'state':'INVESTIGATING_HYPOTHESES','artifacts':{'case':'case'},'project':{'id':'p'},'updated_at':'now'}) + view=case_view(reader) + assert len(view['calls'])==2 + usage=aggregate([view]);buckets={b['id']:b for b in usage['buckets']} + assert buckets['investigation']['input_tokens']==100 + assert buckets['correction']['input_tokens']==100 + assert sum(b['input_tokens']+b['output_tokens'] for b in buckets.values())==280 + assert sum(b['reasoning_tokens'] for b in buckets.values())==60 + + +def test_unknown_usage_is_not_reported_as_complete(): + call={'bucket':'unknown','category':'unknown','phase':'diagnosis','day':'2026-10-21', + 'input_tokens':None,'output_tokens':5,'reasoning_tokens':None,'cached_input_tokens':None} + result=aggregate([{'calls':[call]}]) + assert result['unknown_usage_calls']==1 + assert next(b for b in result['buckets'] if b['id']=='unknown')['output_tokens']==5 + + +@pytest.fixture(autouse=True) +def project_manifest(manifest): + manifest['project']={'id':'p','type':'wow','profile':'vcore'} + + +def test_overview_scope_assets_progress_and_completed_case_hidden(case_setup, monkeypatch): + import viano_bug_analysis.api.dashboard as module + store,builder,*_=case_setup + built=builder.build('case-A','stable-package') + token='dashboard_test_'+'x'*40;monkeypatch.setenv('DASHBOARD_TOKEN',token) + config=APIConfig(store.workspace.root,(Project('p','wow','vcore'),Project('other','wow','vcore')), + (Client('allowed','DASHBOARD_TOKEN',('p',)),)) + app=create_app(config);client=TestClient(app);auth={'Authorization':'Bearer '+token} + assert client.get('/v1/overview').status_code==401 + html=client.get('/') + assert html.status_code==200 and 'Content-Security-Policy' in html.headers + assert 'renderPie' in client.get('/assets/dashboard.js').text + assert client.get('/assets/missing.js').status_code==404 + monkeypatch.setattr(module,'read_status',lambda _: {'observer':'LIVE','service_id':'test-service'}) + directory=store.workspace.runtime/'worker';directory.mkdir(exist_ok=True) + atomic_json(directory/'status.json',{'status':'ANALYZING','service_id':'test-service','case':{'case_id':'case-A','package_id':'stable-package'}}) + data=client.get('/v1/overview',headers=auth).json() + assert data['workers'][0]['case']['title']=='Generic problem' + assert data['workers'][0]['case']['progress']==29 # intake + completed casebuild / seven stages + assert data['cases'][0]['project_id']=='p' + assert token not in json.dumps(data) and str(store.workspace.root) not in json.dumps(data) + monkeypatch.setattr(module,'read_status',lambda _: {'observer':'LIVE','service_id':'unrelated-trial'}) + assert client.get('/v1/overview',headers=auth).json()['workers']==[] + monkeypatch.setattr(module,'read_status',lambda _: {'observer':'LIVE','service_id':'test-service'}) + stranger=Client('other','DASHBOARD_TOKEN',('other',)) + hidden=module.Dashboard(app.state.intake).overview(stranger) + assert hidden['cases']==[] and hidden['workers']==[] + assert sum(b['calls'] for b in hidden['usage']['buckets'])==0 + for state in ('INTERPRETING_CASE','GENERATING_HYPOTHESES','INVESTIGATING_HYPOTHESES','BUILDING_DIAGNOSIS','OUTPUT_READY'): + store.advance('case-A','stable-package',state,step_id='test:'+state) + completed=client.get('/v1/overview',headers=auth).json() + assert completed['workers']==[] + monkeypatch.setattr(module,'read_status',lambda _: {'observer':'HEARTBEAT_STALE'}) + assert client.get('/v1/overview',headers=auth).json()['workers']==[] + def corrupted(_): + from viano_bug_analysis.telemetry.config import TelemetryError + raise TelemetryError('TELEMETRY_READ') + monkeypatch.setattr(module,'read_status',corrupted) + response=client.get('/v1/overview',headers=auth) + assert response.status_code==200 and response.json()['workers']==[] diff --git a/tests/ui_dashboard_server.py b/tests/ui_dashboard_server.py new file mode 100644 index 0000000..8702821 --- /dev/null +++ b/tests/ui_dashboard_server.py @@ -0,0 +1,66 @@ +"""Synthetic, disposable UI fixture; never uses model or database providers.""" +import json +import os +import tempfile +from pathlib import Path + +import uvicorn +from viano_bug_analysis.api import APIConfig, Client, Project +from viano_bug_analysis.api.app import create_app +from viano_bug_analysis.lifecycle import Workspace, CheckpointStore +from viano_bug_analysis.lifecycle.job import timestamp +from viano_bug_analysis.lifecycle.storage import atomic_json, json_bytes +from viano_bug_analysis.package.factory import PackageFactory +from viano_bug_analysis.case import CaseBuilder +import viano_bug_analysis.api.dashboard as dashboard + +TOKEN = 'ui_fixture_' + 'x' * 40 + + +def seed(root, case_id, title, completed=False): + workspace = Workspace(root) + store = CheckpointStore(workspace) + now = timestamp() + manifest = {'format':'viano-bug-case-package','format_version':1,'case_id':case_id, + 'package_id':'preview-'+case_id,'package_generation':0,'analysis_revision':0, + 'project':{'id':'viano-core','type':'wow','profile':'vcore'},'created_at':now,'updated_at':now, + 'state':'NEW','routing':{'destination':'inbox','reason':None,'retry_count':0,'retry_not_before':None}, + 'artifacts':{'case':None,'diagnosis':None,'change_request':None,'case_inputs':'inputs.json'}, + 'history':{'analysis_revisions':0,'reproductions':0,'evidence':0,'handoffs':0,'feedback':0,'events_ref':None},'integrity':{}} + contents={'inputs.json':json_bytes({'format':'viano-case-inputs','format_version':1,'reports':['report.json']}), + 'report.json':json_bytes({'format':'viano-bug-report','format_version':1,'title':title, + 'observations':[{'text':'Synthetic UI fixture observation'}]})} + path=workspace.inbox/(case_id+'.zip');PackageFactory().create(path,manifest,contents) + store.intake(path);path.unlink() + identity=case_id,manifest['package_id'];CaseBuilder(store).build(*identity) + for state,pointer in [('INTERPRETING_CASE','case_interpretation'),('GENERATING_HYPOTHESES','hypotheses')]: + ref=pointer+'.json';store.advance(*identity,state,step_id=state, + artifacts={pointer:ref},additions={ref:json_bytes({'analysis_revision':1})}) + store.advance(*identity,'INVESTIGATING_HYPOTHESES',step_id='investigate') + for i,(category,inp,out) in enumerate([('regular',12000,3000),('correction',3000,900)]): + ref=f'call{i}.json';request=f'request{i}.json' + call={'rounds':1,'phase':'INVESTIGATING_HYPOTHESES','request_ref':request,'outcome':{ + 'usage_classification':category,'metadata':{'input_tokens':inp,'output_tokens':out, + 'reasoning_tokens':out//2,'cached_input_tokens':1000}}} + store.advance(*identity,'INVESTIGATING_HYPOTHESES',step_id=f'call{i}', + details={'operation':'llm-round','result_ref':ref}, + additions={ref:json_bytes(call),request:json_bytes({'prompt_version':'investigation_lean_v21'})}) + if completed: + store.advance(*identity,'BUILDING_DIAGNOSIS',step_id='diagnosis') + store.advance(*identity,'OUTPUT_READY',step_id='done') + return identity + + +if __name__ == '__main__': + with tempfile.TemporaryDirectory(prefix='vba-dashboard-', dir=os.environ.get('VBA_UI_TEST_WORKSPACE')) as temporary: + root=Path(temporary) + identity=seed(root,'BUG-000020','Wolpertinger fehlen in Dun Morogh') + seed(root,'BUG-000017','Leuchtbrille – Kontrolllauf',completed=True) + config=APIConfig(root,(Project('viano-core','wow','vcore'),), + (Client('preview','VBA_UI_TEST_TOKEN',('viano-core',)),)) + os.environ['VBA_UI_TEST_TOKEN']=TOKEN + dashboard.read_status=lambda _: {'observer':'LIVE','service_id':'fixture-worker'} + folder=root/'runtime/worker';folder.mkdir(exist_ok=True) + atomic_json(folder/'status.json',{'status':'ANALYZING','service_id':'fixture-worker', + 'case':{'case_id':identity[0],'package_id':identity[1]}}) + uvicorn.run(create_app(config),host='127.0.0.1',port=int(os.environ.get('VBA_UI_TEST_PORT','8765')),access_log=False)