From 5826d8f730b8e34ae406bf162d5d93974b0b4fd9 Mon Sep 17 00:00:00 2001 From: Nitsu45 Date: Wed, 14 Oct 2026 15:41:14 +0200 Subject: [PATCH] Add resumable single-case CLI trial with telemetry and Vcore import --- .gitignore | 1 + README.md | 5 + docs/first-llm-trial.md | 117 ++++++++++++ src/viano_bug_analysis/telemetry/codes.py | 4 + src/viano_bug_analysis/trial.py | 221 ++++++++++++++++++++++ tests/test_trial.py | 179 ++++++++++++++++++ 6 files changed, 527 insertions(+) create mode 100644 docs/first-llm-trial.md create mode 100644 src/viano_bug_analysis/trial.py create mode 100644 tests/test_trial.py diff --git a/.gitignore b/.gitignore index c6f41bb..ea6e3b9 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ build/ dist/ config/*.local.toml .env +service-data/ diff --git a/README.md b/README.md index 7dc2bf6..5c1a258 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,11 @@ policy-bound Change Requests, with final output validation and atomic local outbox/review publication. Opt-in operational telemetry observes the existing APIs. A continuous worker remains a future milestone. +A bounded single-case command-line trial now connects import, analysis and local +delivery with telemetry: [first real LLM trial](docs/first-llm-trial.md). +Use `python -m viano_bug_analysis.trial --help`. Only its `run` command makes LLM +requests; `prepare` and `check` do not contact model services. + Expected-behavior resolution and the read-only provider contract: [expected-behavior-v1.md](docs/expected-behavior-v1.md). Configurable LLM transport, failover and durable retries: [llm-gateway-v1.md](docs/llm-gateway-v1.md), with [example configuration](config/llm.example.toml). diff --git a/docs/first-llm-trial.md b/docs/first-llm-trial.md new file mode 100644 index 0000000..26ee580 --- /dev/null +++ b/docs/first-llm-trial.md @@ -0,0 +1,117 @@ +# Erster LLM-Test + +Der einmalige Testtreiber verbindet vorhandene Komponenten. Er ist kein Daemon: +genau ein Fall, begrenzte Dispatches, keine Hintergrund-Wiederholungen, keine +automatische Wiederaufnahme blockierter Fälle. Architektur und Projektcode bleiben +unverändert. Die bekannte Fehlerlösung gehört nicht in Eingaben oder Prompts. + +## Vorbereitung in PowerShell + +Alle Befehle aus dem Repository-Hauptverzeichnis ausführen, nicht aus `config/`. +Python 3.11+ und die installierte Projektumgebung werden vorausgesetzt. Die lokale +Datei `config/llm.local.toml` muss existieren; API-Schlüssel kommen ausschließlich +aus den darin benannten Umgebungsvariablen. Das Programm muss aus der PowerShell +mit diesen Variablen gestartet werden. Es lädt keine `.env` automatisch. + +```powershell +.\.venv\Scripts\python.exe -m viano_bug_analysis.trial check +``` + +`CONFIG_READY` bestätigt gültige Konfiguration und vorhandene Schlüssel aller +aktivierten, zulässigen Anbieter. Es prüft keine Erreichbarkeit, Modellfreigabe, +Abrechnung oder JSON-Kompatibilität. Es sendet keine Netzwerkanfrage und gibt +keine Schlüssel aus. Ohne API-Key erscheint `LLM_CREDENTIALS`. + +Import des Beispielpaars ohne LLM: + +```powershell +.\.venv\Scripts\python.exe -m viano_bug_analysis.trial prepare --workspace service-data --wowdiag "C:/Users/User/Desktop/Cata Bug fixes/BUG-000020-20261008-130429.tar.gz" --wowsrc "C:/Users/User/Desktop/Cata Bug fixes/WOWSRC-BUG-000020-20261008-130446.tar.gz" --case-id BUG-000020 --package-id trial-BUG-000020 +``` + +Die Ausgabe `PREPARED` nennt den vollständigen `package`-Pfad in +`service-data/inbox/`. IDs werden explizit vergeben; Dateinamen liefern keine +Identität. Ohne `--package-id` entsteht eine UUID. Identische erneute Vorbereitung +überschreibt keine vorhandene Datei. `service-data/` und `config/*.local.toml` +sind durch Git ausgeschlossen. Originalarchive bleiben unverändert. + +## Echte Verarbeitung + +Nur dieser Schritt kann kostenpflichtige LLM-Aufrufe auslösen und bei +`cloud_allowed` zusammengestellte Falldaten an Cloud-Dienste senden: + +```powershell +.\.venv\Scripts\python.exe -m viano_bug_analysis.trial run --workspace service-data --package "VOLLSTAENDIGER_PACKAGE_PFAD_AUS_PREPARED" +``` + +Der erste Lauf validiert die Inbox-Datei, sichert den Original-ZIP im Cache und +übernimmt ihn in die verwalteten Generationen. Nach bestätigter Aufnahme wird +die Inbox-Datei quittiert. Andere Inbox-Dateien werden nicht verarbeitet. +Phase und Dispatchnummer erscheinen auf stderr; das finale JSON auf stdout +enthält Identität, Zustand, Generation, Routing und ggf. Export-/Retry-Pfad. + +Für den ersten Verbindungstest `--max-dispatches 2` ergänzen: Bei einem frischen +Paket wird der Case deterministisch gebaut und anschließend die Interpretation +angefragt. Bei Erfolg endet dieser kurze Lauf mit `YIELDED`. Anschließend mit +den stabilen IDs weiterarbeiten. Bereits bestätigte Schritte werden wiederverwendet. + +Nach Aufnahme mit der stabilen Identität fortsetzen: + +```powershell +.\.venv\Scripts\python.exe -m viano_bug_analysis.trial run --workspace service-data --case-id BUG-000020 --package-id trial-BUG-000020 +``` + +Ein Prozess führt standardmäßig höchstens 100 Dispatches aus. Das ist eine +konfigurierbare Testlaufgrenze (`--max-dispatches`), kein neues Hypothesenbudget +und keine Geldgrenze. Investigation macht einen kontrollierten Schritt pro +Dispatch. Die bestehenden Hypothesenbudgets und Iterationsgrenzen gelten weiter. +Bei `YIELDED` setzt derselbe Resume-Befehl die Analyse fort. + +`RETRY_PENDING`/`DEFERRED` beendet den Lauf sofort mit dem frühesten nächsten +Zeitpunkt. Nach diesem Zeitpunkt denselben Resume-Befehl ausführen. Kein Sleep +oder automatischer Neustart: zehn Provider-Runden benötigen wiederholte fällige +Dispatches. `BLOCKED` bleibt blockiert, auch wenn eine Frist vergeht. Externe +Testanfragen erhalten ihre Reservierungen; die spätere bestätigte Feedback- +Integration ist für deren Wiederaufnahme zuständig. + +Interpretation/Hypothesen mit ungültiger Antwort werden kontrolliert gestoppt. +Ein bewusst neuer Versuch kann mit `--attempt-id NEUE-STABILE-ID` erfolgen, +ohne eine gespeicherte alte Antwort unter derselben Operation zu überschreiben. +Dies ist kein automatischer Retry und kann erneut Kosten verursachen. Die +späteren Phasen behalten ihre bestehenden Grenzen für ungültige Antworten. +Keine automatische neue Analysis Revision oder Budgetzurücksetzung findet statt. + +Optionale Konfigurationspfade: `--analysis-config`, `--budget-config`, +`--investigation-config`, `--diagnosis-config`, `--change-request-config`, +`--telemetry-config`, `--llm-config`. Ohne Pfade gelten die bestehenden Defaults. +Bereits eingefrorene Policies nicht zwischen Resume-Läufen ändern; dieselben +Optionen erneut angeben. Der gesamte Gateway-Konfigurationsfingerprint ist Teil +der Operation; Änderungen können kontrolliert als Konflikt abgelehnt werden. + +## Ergebnis und Diagnose + +- `OUTPUT_READY`: validiertes ZIP mit Diagnose und Coding-Auftrag in `outbox/`. +- `REVIEW`: validiertes ZIP mit Untersuchungsbericht in `review/`; kein Coding-Auftrag. +- `BLOCKED`: langlebiger verwalteter Fall; Grund und Budgets bleiben erhalten. +- `YIELDED`, `RETRY_PENDING`, `DEFERRED`, `REJECTED`: kontrollierter Zwischenstand. +- `ERROR`/`QUARANTINED`: Fehlercode und ggf. Quarantäne-Diagnose prüfen. + +Exitcodes: 0 für erfolgreiche Vorbereitung/Prüfung/lokale Ausgabe, 3 für einen +Zwischen-/Wartezustand, 2 für Fehler, 130 für Abbruch mit Ctrl+C. Ein Review ist +ein erfolgreiches kontrolliertes Analyseergebnis, kein bewiesener Fix. + +```powershell +.\.venv\Scripts\python.exe -m viano_bug_analysis.telemetry status --workspace service-data +.\.venv\Scripts\python.exe -m viano_bug_analysis.telemetry errors --workspace service-data +.\.venv\Scripts\python.exe -m viano_bug_analysis.telemetry events --workspace service-data +``` + +Logs, Status und Prometheus-Text liegen in `service-data/runtime/telemetry/`. +`STOPPED` nach einem beendeten Testlauf ist normal. Stack-Diagnosen enthalten +keine freien Exceptiontexte, Prompts oder Schlüssel. Der Treiber bleibt bei +kontrollierten Fehlern ohne rohe Tracebacks. Fallartefakte enthalten hingegen +die fachlichen LLM-Anfragen/-Antworten und sind entsprechend vertraulich. + +Vergleiche anschließend Diagnose, Evidenz und ggf. Änderungsauftrag mit der +bekannten Fehlerlösung. Ein Live-Test beweist die Modellqualität noch nicht; +insbesondere fehlende Sollinformationen oder nicht angeschlossene externe +Research-/Tester-Dienste können zu Wartezuständen oder Review führen. diff --git a/src/viano_bug_analysis/telemetry/codes.py b/src/viano_bug_analysis/telemetry/codes.py index 61cd624..1880e5b 100644 --- a/src/viano_bug_analysis/telemetry/codes.py +++ b/src/viano_bug_analysis/telemetry/codes.py @@ -2,3 +2,7 @@ KNOWN_CODES = frozenset(['ACCEPTED', 'ARCHIVE_ERROR', 'ARCHIVE_INVALID', 'ARCHIVE_LIMIT', 'ARTIFACT_EXISTS', 'ARTIFACT_INVALID', 'ARTIFACT_TYPE', 'ASYNC_TEST_ATTACHMENT', 'ASYNC_TEST_CONFLICT', 'ASYNC_TEST_COST', 'ASYNC_TEST_IDENTITY', 'ASYNC_TEST_INVALID', 'ASYNC_TEST_LIMIT', 'ASYNC_TEST_NOT_WAITING', 'ASYNC_TEST_PHASE', 'ASYNC_TEST_RESERVATION', 'AWAITING_TEST_FEEDBACK', 'BLOCKED', 'BUDGET_ACCOUNTING_CHANGED', 'BUDGET_ACTION_CONFLICT', 'BUDGET_ACTION_UNKNOWN', 'BUDGET_CASE_NOT_FOUND', 'BUDGET_CONFIG', 'BUDGET_EVENT_LIMIT', 'BUDGET_EVIDENCE', 'BUDGET_EVIDENCE_BINDING', 'BUDGET_GENERATION_CHANGED', 'BUDGET_HISTORY_INVALID', 'BUDGET_HISTORY_LIMIT', 'BUDGET_HISTORY_MISSING', 'BUDGET_HYPOTHESES_NOT_READY', 'BUDGET_HYPOTHESIS_UNKNOWN', 'BUDGET_INPUT', 'BUDGET_INSUFFICIENT', 'BUDGET_LIMIT_CONFIG', 'BUDGET_NOT_INITIALIZED', 'BUDGET_NOVELTY_CAP', 'BUDGET_OPERATION', 'BUDGET_ORIGIN_CHANGED', 'BUDGET_PACKAGE', 'BUDGET_PENDING_ACTION', 'BUDGET_PHASE', 'BUDGET_POINTER_CHANGED', 'BUDGET_POLICY_CHANGED', 'BUDGET_RESULT_LIMIT', 'BUDGET_RESULT_REQUIRED', 'BUDGET_SEMANTIC_SCORE', 'BUDGET_SOURCE_CHANGED', 'BUDGET_SOURCE_REFERENCES', 'BUDGET_STEP_CONFLICT', 'BUDGET_TERMINAL', 'BUG_ID_MISMATCH', 'BUILDING_CASE', 'BUILDING_DIAGNOSIS', 'BUSY', 'CACHE_CONFLICT', 'CANCELLED', 'CASE_BLOCKED', 'CASE_BUILD_ERROR', 'CASE_IDENTITY', 'CASE_LIMIT', 'CASE_NOT_FOUND', 'CASE_OUTPUT_CHANGED', 'CASE_SCHEMA', 'CHANGE_REQUEST_CONFIG', 'CHANGE_REQUEST_CONTEXT_CHANGED', 'CHANGE_REQUEST_CONTEXT_LIMIT', 'CHANGE_REQUEST_DIAGNOSIS_NOT_READY', 'CHANGE_REQUEST_EVIDENCE', 'CHANGE_REQUEST_INCOMPLETE', 'CHANGE_REQUEST_INPUT', 'CHANGE_REQUEST_INSUFFICIENT_CONTEXT', 'CHANGE_REQUEST_LIMIT_CONFIG', 'CHANGE_REQUEST_LLM_BINDING', 'CHANGE_REQUEST_LOCAL_CONTEXT_CHANGED', 'CHANGE_REQUEST_MISSING_INFORMATION', 'CHANGE_REQUEST_OUTPUT', 'CHANGE_REQUEST_OUTPUT_CHANGED', 'CHANGE_REQUEST_OUTPUT_LIMIT', 'CHANGE_REQUEST_PACKAGE', 'CHANGE_REQUEST_PHASE', 'CHANGE_REQUEST_POINTER_CHANGED', 'CHANGE_REQUEST_POLICY_CHANGED', 'CHANGE_REQUEST_REJECTION_LIMIT', 'CHANGE_REQUEST_RETRY_ID', 'CHANGE_REQUEST_ROOT_SCOPE', 'CHANGE_REQUEST_SCOPE', 'CHANGE_REQUEST_SCOPE_EVIDENCE', 'CHECKPOINT_HISTORY_LOST', 'CHECKPOINT_INVALID', 'CHECKPOINT_LIMIT', 'CLEANUP_ERROR', 'COMPLETED', 'CONFLICTING_CLAIMS', 'CORE_REVISION_MISSING', 'CREATE_ERROR', 'DEFERRED', 'DEGRADED', 'DIAGNOSIS_BUDGET_NOT_READY', 'DIAGNOSIS_CASE_NOT_FOUND', 'DIAGNOSIS_CONFIG', 'DIAGNOSIS_CONTEXT_CHANGED', 'DIAGNOSIS_CONTEXT_LIMIT', 'DIAGNOSIS_COUNTERCHECK', 'DIAGNOSIS_EVIDENCE_REFERENCE', 'DIAGNOSIS_HYPOTHESES', 'DIAGNOSIS_IDENTITY', 'DIAGNOSIS_INCONCLUSIVE', 'DIAGNOSIS_INPUT', 'DIAGNOSIS_INVESTIGATION_NOT_READY', 'DIAGNOSIS_LIMIT_CONFIG', 'DIAGNOSIS_LLM_BINDING', 'DIAGNOSIS_OUTPUT', 'DIAGNOSIS_OUTPUT_CHANGED', 'DIAGNOSIS_OUTPUT_LIMIT', 'DIAGNOSIS_PACKAGE', 'DIAGNOSIS_PHASE', 'DIAGNOSIS_POINTER_CHANGED', 'DIAGNOSIS_POLICY_CHANGED', 'DIAGNOSIS_REJECTED_SELECTION', 'DIAGNOSIS_REJECTION_LIMIT', 'DIAGNOSIS_RETRY_ID', 'DIAGNOSIS_SELECTION', 'DIAGNOSIS_STALE_CASE', 'DIAGNOSIS_UNGROUNDED', 'DUPLICATE', 'ENTRY_COLLISION', 'ENTRY_LIMIT', 'ERROR', 'EVIDENCE_CASE_NOT_FOUND', 'EVIDENCE_CONTENT_CHANGED', 'EVIDENCE_DUPLICATE_CHANGED', 'EVIDENCE_GENERATION_CHANGED', 'EVIDENCE_HISTORY_INVALID', 'EVIDENCE_HISTORY_LIMIT', 'EVIDENCE_IDENTITY', 'EVIDENCE_INDEX_CHANGED', 'EVIDENCE_INPUT', 'EVIDENCE_LIMIT', 'EVIDENCE_LIMIT_CONFIG', 'EVIDENCE_NOVELTY_CHANGED', 'EVIDENCE_OPERATION_ID', 'EVIDENCE_ORIGIN_CHANGED', 'EVIDENCE_PHASE', 'EVIDENCE_POINTER_CHANGED', 'EVIDENCE_RECORD_INVALID', 'EVIDENCE_RESERVATION', 'EVIDENCE_RESULT_CHANGED', 'EVIDENCE_SOURCE_CHANGED', 'EVIDENCE_STEP_CONFLICT', 'EXPANSION_LIMIT', 'EXPECTED_BEHAVIOR_', 'EXTERNAL_FEEDBACK_REQUIRED', 'FAILED_TO_ANALYZE', 'FILE_LIMIT', 'FILE_NOT_FOUND', 'GENERATING_HYPOTHESES', 'GENERATION_CHANGED', 'GENERATION_CONFLICT', 'HANDED_OFF', 'HEADER_LIMIT', 'HEARTBEAT_STALE', 'HISTORY_REGRESSION', 'HYPOTHESIS_ALLOCATION_CHANGED', 'HYPOTHESIS_ATTEMPT', 'HYPOTHESIS_CONFIG', 'HYPOTHESIS_CONTEXT_CHANGED', 'HYPOTHESIS_CONTEXT_LIMIT', 'HYPOTHESIS_COUNT', 'HYPOTHESIS_DUPLICATE', 'HYPOTHESIS_EXISTING_REGISTRY', 'HYPOTHESIS_GENERATION_CHANGED', 'HYPOTHESIS_GROUNDING', 'HYPOTHESIS_HISTORY_IDENTITY', 'HYPOTHESIS_HISTORY_LIMIT', 'HYPOTHESIS_IDENTITY', 'HYPOTHESIS_INPUT', 'HYPOTHESIS_INTERPRETATION_NOT_READY', 'HYPOTHESIS_LIMIT_CONFIG', 'HYPOTHESIS_LLM_BINDING', 'HYPOTHESIS_NO_OBSERVATIONS', 'HYPOTHESIS_OUTPUT_CHANGED', 'HYPOTHESIS_OUTPUT_LIMIT', 'HYPOTHESIS_PACKAGE', 'HYPOTHESIS_PHASE', 'HYPOTHESIS_POLICY_CHANGED', 'HYPOTHESIS_PRIMARY_FOCUS', 'HYPOTHESIS_REGISTRY_CHANGED', 'HYPOTHESIS_SCHEMA', 'HYPOTHESIS_SOURCE_UNKNOWN', 'HYPOTHESIS_SUMMARY_LIMIT', 'IDLE', 'IMPORT_COLLISION', 'IMPORT_ERROR', 'IMPORT_EXISTS', 'IMPORT_LIMIT', 'IMPORT_RECEIPT', 'INCONCLUSIVE', 'INDEX_UPDATE_FORBIDDEN', 'INFO', 'INITIAL_GENERATION', 'INPUT_DUPLICATE', 'INPUT_JSON', 'INPUT_LIMIT', 'INPUT_PENDING', 'INPUT_REFERENCE', 'INPUT_SCHEMA', 'INPUT_TARGET_ALIAS', 'INTEGRITY_MISMATCH', 'INTERPRETATION_ATTEMPT', 'INTERPRETATION_CASE_IDENTITY', 'INTERPRETATION_CASE_IDS', 'INTERPRETATION_CASE_NOT_READY', 'INTERPRETATION_CONTEXT_CHANGED', 'INTERPRETATION_CONTEXT_LIMIT', 'INTERPRETATION_CONTRADICTION_SOURCES', 'INTERPRETATION_ENTITY', 'INTERPRETATION_EXPECTATION_UNKNOWN', 'INTERPRETATION_GAP_SOURCES', 'INTERPRETATION_GENERATION_CHANGED', 'INTERPRETATION_INPUT', 'INTERPRETATION_LLM_BINDING', 'INTERPRETATION_OUTPUT_CHANGED', 'INTERPRETATION_OUTPUT_LIMIT', 'INTERPRETATION_PACKAGE', 'INTERPRETATION_PHASE', 'INTERPRETATION_SCHEMA', 'INTERPRETATION_SOURCE_KIND', 'INTERPRETATION_SOURCE_LIMIT', 'INTERPRETATION_SOURCE_UNKNOWN', 'INTERPRETATION_SUMMARY_LIMIT', 'INTERPRETING_CASE', 'INVALID_BUILD_STATE', 'INVALID_GENERATION', 'INVALID_LLM_RESPONSES', 'INVALID_RETRY', 'INVALID_REVISION', 'INVALID_STATE', 'INVALID_STEP', 'INVALID_TRANSITION', 'INVESTIGATING_HYPOTHESES', 'INVESTIGATION_ASSESSMENT_BINDING', 'INVESTIGATION_ASSESSMENT_REQUIRED', 'INVESTIGATION_CASE_NOT_FOUND', 'INVESTIGATION_CLOCK', 'INVESTIGATION_CONFIG', 'INVESTIGATION_CONTEXT_CHANGED', 'INVESTIGATION_CONTEXT_LIMIT', 'INVESTIGATION_CONTRADICTORY_UPDATE', 'INVESTIGATION_DISPATCH_LIMIT', 'INVESTIGATION_EVIDENCE_UNKNOWN', 'INVESTIGATION_GENERATION_CHANGED', 'INVESTIGATION_HISTORY_INVALID', 'INVESTIGATION_HISTORY_LIMIT', 'INVESTIGATION_HISTORY_MISSING', 'INVESTIGATION_INPUT', 'INVESTIGATION_ITERATION_LIMIT', 'INVESTIGATION_LIMIT_CONFIG', 'INVESTIGATION_LLM_BINDING', 'INVESTIGATION_OUTPUT', 'INVESTIGATION_PHASE', 'INVESTIGATION_PLAN_BINDING', 'INVESTIGATION_POINTER_CHANGED', 'INVESTIGATION_POLICY_CHANGED', 'INVESTIGATION_QUESTION', 'INVESTIGATION_QUESTION_DUPLICATE', 'INVESTIGATION_QUESTION_LIMIT', 'INVESTIGATION_RETRY_ID', 'INVESTIGATION_RETRY_NOT_EXHAUSTED', 'INVESTIGATION_SCORE', 'INVESTIGATION_STALE_CASE', 'INVESTIGATION_STATE_LIMIT', 'INVESTIGATION_SUMMARY_CHANGED', 'INVESTIGATION_SUPPORT_REQUIRED', 'INVESTIGATION_TEST_COST', 'INVESTIGATION_UNOWNED_PENDING_ACTION', 'LIFECYCLE_ERROR', 'LIVE', 'LLM_ADAPTER', 'LLM_CHECKPOINT', 'LLM_CLOCK', 'LLM_CONFIG', 'LLM_CONFIGURATION', 'LLM_CONNECTION', 'LLM_CREDENTIALS', 'LLM_ERROR', 'LLM_GENERATION_CHANGED', 'LLM_HTTP_CONFIGURATION', 'LLM_HTTP_TEMPORARY', 'LLM_INCOMPLETE_OUTPUT', 'LLM_LOG_PATH', 'LLM_OPERATION', 'LLM_OPERATION_CONFLICT', 'LLM_OUTPUT_SCHEMA', 'LLM_PHASE', 'LLM_PRIVACY_NO_PROVIDER', 'LLM_PROVIDER_ERROR', 'LLM_REQUEST', 'LLM_REQUEST_LIMIT', 'LLM_RESPONSE', 'LLM_RESPONSE_LIMIT', 'LLM_RESULT_MISSING', 'LLM_RETRY_EXHAUSTED', 'LLM_SCHEMA_REFERENCE', 'LLM_TIMEOUT', 'LLM_UNAVAILABLE', 'LOCK_BUSY', 'LOCK_ERROR', 'MANIFEST_AMBIGUOUS', 'MANIFEST_CHANGED', 'MANIFEST_MISSING', 'MANIFEST_SCHEMA', 'MANIFEST_TIME', 'MANIFEST_UPDATE_FORBIDDEN', 'MEMBER_COLLISION', 'MEMBER_TRUNCATED', 'MISSING_INPUT_INDEX', 'MISSING_OBSERVATIONS', 'MISSING_REPORT', 'MISSING_REPRODUCTION', 'NEW', 'NFKC', 'NORMALIZATION_ERROR', 'NOT_STARTED', 'NO_RECENT_PROGRESS', 'OUTPUT_CHANGED', 'OUTPUT_CHANGE_REQUEST_NOT_READY', 'OUTPUT_EXPORT_CONFLICT', 'OUTPUT_INCONCLUSIVE_CODING_TASK', 'OUTPUT_INPUT', 'OUTPUT_NOT_READY', 'OUTPUT_PACKAGE', 'OUTPUT_PHASE', 'OUTPUT_READY', 'OUTPUT_REPORT_CHANGED', 'OUTPUT_REVIEW_ROUTING', 'OUTPUT_ROUTING', 'OUTPUT_STORAGE', 'OUTPUT_UNSAFE_DESTINATION', 'OUTPUT_VALIDATION_MISSING', 'PACKAGE_ERROR', 'PACKAGE_INVALID', 'PACKAGE_LIMIT', 'PRESERVATION_FAILED', 'PREVIOUS_CASE', 'PREVIOUS_IDS', 'PREVIOUS_INPUT_CHANGED', 'PREVIOUS_RECORD_LOST', 'PROGRESSED', 'PROJECT_MISMATCH', 'Q0001', 'QUARANTINED', 'QUEUE_UNKNOWN', 'RATIO_LIMIT', 'READY_NOT_PROCESSING', 'READ_ERROR', 'REFERENCE_MISSING', 'REJECTED', 'REOPENED', 'RESEARCH_EXISTS', 'RESEARCH_IDENTITY', 'RESEARCH_JSON', 'RESEARCH_LIMIT', 'RESEARCH_NOT_REQUIRED', 'RESEARCH_PROVIDER', 'RESEARCH_REQUEST_MISMATCH', 'RESEARCH_SOURCE', 'RESEARCH_STATE', 'RESEARCH_UNAVAILABLE', 'RESERVED_ARTIFACT', 'RESUMED', 'RETRY_NOT_DUE', 'RETRY_PENDING', 'REVIEW', 'REVISION_IMMUTABLE', 'REVISION_REGRESSION', 'SIZE_MISMATCH', 'SOURCE_CHANGED', 'SOURCE_ENTITY', 'SOURCE_FIELD', 'SOURCE_FORMAT', 'SOURCE_ID', 'SOURCE_INDEX', 'SOURCE_JSON', 'SOURCE_MANIFEST', 'SOURCE_NOT_COPIED', 'SOURCE_PAIR_MISMATCH', 'SOURCE_REFERENCE', 'SOURCE_RELATION', 'SOURCE_REPORTS_MISSING', 'SOURCE_ROWS', 'SOURCE_SIZE', 'SOURCE_TIME_UNQUALIFIED', 'SOURCE_WARNING', 'SPECIAL_ENTRY', 'SPECIAL_MEMBER', 'STALE_CASE', 'STEP_CONFLICT', 'STOPPED', 'STORAGE_CHECKPOINT_INVALID', 'STORAGE_CONFLICT', 'STORAGE_ERROR', 'STORAGE_PACKAGE_CHANGED', 'STORAGE_PACKAGE_INVALID', 'STREAM_LIMIT', 'STRUCTURED_INPUT_LIMIT', 'SUCCESS', 'SUMMARY_LIMIT', 'TARGET_EXISTS', 'TARGET_SYMLINK', 'TELEMETRY_CALLBACK', 'TELEMETRY_CLOCK', 'TELEMETRY_CONFIG', 'TELEMETRY_EVENT', 'TELEMETRY_LIMIT', 'TELEMETRY_LOG_CORRUPT', 'TELEMETRY_OPERATION', 'TELEMETRY_PATH', 'TELEMETRY_READ', 'TELEMETRY_SCAN_CHECKPOINT', 'TELEMETRY_SCAN_IDENTITY', 'TELEMETRY_SCAN_PATH', 'TELEMETRY_SESSION', 'TELEMETRY_STORAGE', 'TOOL_BATCH_LIMIT', 'TOOL_CASE_IDENTITY', 'TOOL_CONTEXT_RANGE', 'TOOL_DATABASE_COLUMN_UNKNOWN', 'TOOL_DATABASE_DESCRIPTOR', 'TOOL_DATABASE_JSON', 'TOOL_DATABASE_ROWS', 'TOOL_DUPLICATE_REQUEST', 'TOOL_FILE_LIMIT', 'TOOL_HYPOTHESIS_IDENTITY', 'TOOL_HYPOTHESIS_UNKNOWN', 'TOOL_IDENTITY', 'TOOL_INPUT', 'TOOL_INPUT_LIMIT', 'TOOL_LIMIT_CONFIG', 'TOOL_MATCH_LIMIT', 'TOOL_PACKAGE', 'TOOL_PHASE', 'TOOL_REQUEST_INTEGER', 'TOOL_REQUEST_LIMIT', 'TOOL_RESULT_LIMIT', 'TOOL_SOURCE_BINARY', 'TOOL_SOURCE_CHANGED', 'TOOL_SOURCE_CONFLICT', 'TOOL_SOURCE_ENCODING', 'TOOL_SOURCE_NOT_DECLARED', 'TOOL_STALE_CASE', 'TOTAL_LIMIT', 'UNCLASSIFIED_ERROR', 'UNKNOWN', 'UNSAFE_INPUT', 'UNSAFE_LOCK', 'UNSAFE_PATH', 'UNSAFE_REFERENCE', 'UNSAFE_WORKSPACE', 'UNSUPPORTED_ENTRY', 'USD', 'VALID', 'VALIDATED', 'WAITING_EXTERNAL', 'WARNING', 'WORKING', 'WRITE_ERROR', 'YIELDED']) KNOWN_MODULES = frozenset(['viano_bug_analysis', 'viano_bug_analysis.analysis', 'viano_bug_analysis.analysis.change_request_builder', 'viano_bug_analysis.analysis.change_request_contracts', 'viano_bug_analysis.analysis.context', 'viano_bug_analysis.analysis.contracts', 'viano_bug_analysis.analysis.diagnosis_builder', 'viano_bug_analysis.analysis.errors', 'viano_bug_analysis.analysis.hypothesis_builder', 'viano_bug_analysis.analysis.hypothesis_config', 'viano_bug_analysis.analysis.hypothesis_contracts', 'viano_bug_analysis.analysis.hypothesis_prompt', 'viano_bug_analysis.analysis.hypothesis_summary', 'viano_bug_analysis.analysis.interpreter', 'viano_bug_analysis.analysis.investigation_config', 'viano_bug_analysis.analysis.investigation_prompt', 'viano_bug_analysis.analysis.investigation_summary', 'viano_bug_analysis.analysis.investigator', 'viano_bug_analysis.analysis.output_validator', 'viano_bug_analysis.analysis.prompt', 'viano_bug_analysis.analysis.summary', 'viano_bug_analysis.budget', 'viano_bug_analysis.budget.config', 'viano_bug_analysis.budget.engine', 'viano_bug_analysis.budget.manager', 'viano_bug_analysis.budget.scoring', 'viano_bug_analysis.case', 'viano_bug_analysis.case.assembler', 'viano_bug_analysis.case.builder', 'viano_bug_analysis.case.contracts', 'viano_bug_analysis.case.errors', 'viano_bug_analysis.case.expected_behavior', 'viano_bug_analysis.case.report_collector', 'viano_bug_analysis.case.reproduction_collector', 'viano_bug_analysis.case.summary', 'viano_bug_analysis.evidence', 'viano_bug_analysis.evidence.scoring', 'viano_bug_analysis.evidence.store', 'viano_bug_analysis.lifecycle', 'viano_bug_analysis.lifecycle.async_tests', 'viano_bug_analysis.lifecycle.checkpoints', 'viano_bug_analysis.lifecycle.errors', 'viano_bug_analysis.lifecycle.job', 'viano_bug_analysis.lifecycle.locking', 'viano_bug_analysis.lifecycle.queue', 'viano_bug_analysis.lifecycle.state', 'viano_bug_analysis.lifecycle.storage', 'viano_bug_analysis.llm', 'viano_bug_analysis.llm.config', 'viano_bug_analysis.llm.errors', 'viano_bug_analysis.llm.gateway', 'viano_bug_analysis.llm.logging', 'viano_bug_analysis.llm.openai_compatible', 'viano_bug_analysis.llm.prompt', 'viano_bug_analysis.llm.prompts', 'viano_bug_analysis.llm.provider_base', 'viano_bug_analysis.llm.runner', 'viano_bug_analysis.package', 'viano_bug_analysis.package.errors', 'viano_bug_analysis.package.factory', 'viano_bug_analysis.package.limits', 'viano_bug_analysis.package.manifest', 'viano_bug_analysis.package.reader', 'viano_bug_analysis.package.validator', 'viano_bug_analysis.package.writer', 'viano_bug_analysis.profiles', 'viano_bug_analysis.profiles.vcore', 'viano_bug_analysis.profiles.vcore.archive_reader', 'viano_bug_analysis.profiles.vcore.common', 'viano_bug_analysis.profiles.vcore.db_tools', 'viano_bug_analysis.profiles.vcore.errors', 'viano_bug_analysis.profiles.vcore.importer', 'viano_bug_analysis.profiles.vcore.wowdiag_adapter', 'viano_bug_analysis.profiles.vcore.wowsrc_adapter', 'viano_bug_analysis.research', 'viano_bug_analysis.research.provider', 'viano_bug_analysis.schemas', 'viano_bug_analysis.telemetry', 'viano_bug_analysis.telemetry.__main__', 'viano_bug_analysis.telemetry.config', 'viano_bug_analysis.telemetry.core', 'viano_bug_analysis.telemetry.decorators', 'viano_bug_analysis.telemetry.inspection', 'viano_bug_analysis.telemetry.safety', 'viano_bug_analysis.tools', 'viano_bug_analysis.tools.config', 'viano_bug_analysis.tools.context', 'viano_bug_analysis.tools.database_query', 'viano_bug_analysis.tools.router', 'viano_bug_analysis.tools.source_query']) KNOWN_CLASSES = frozenset(['AdaptedInputs', 'AdapterError', 'ArchiveLimits', 'ArchiveReader', 'AsyncTestCoordinator', 'AsyncTestResult', 'BoundedTarInfo', 'BudgetConfig', 'BudgetError', 'BudgetLimits', 'BudgetManager', 'BudgetResult', 'CaseAssembler', 'CaseBuildError', 'CaseBuildResult', 'CaseBuilder', 'CaseBuilderLimits', 'CaseInterpreter', 'CaseLLMRunner', 'ChangeRequestBuilder', 'ChangeRequestConfig', 'ChangeRequestError', 'ChangeRequestLimits', 'ChangeRequestResult', 'CheckpointStore', 'DatabaseQueryTool', 'DiagnosisBuilder', 'DiagnosisConfig', 'DiagnosisError', 'DiagnosisLimits', 'DiagnosisResult', 'EvidenceError', 'EvidenceLimits', 'EvidenceResult', 'EvidenceStore', 'ExpectedBehaviorResolver', 'ExternalResearchProvider', 'FileLock', 'GatewayConfig', 'GatewayUnavailable', 'HypothesisBuildResult', 'HypothesisBuilder', 'HypothesisConfig', 'HypothesisError', 'HypothesisLimits', 'HypothesisPromptBuilder', 'ImportPlan', 'InboxProcessor', 'IntakeResult', 'InterpretationError', 'InterpretationLimits', 'InterpretationPromptBuilder', 'InterpretationResult', 'InvestigationConfig', 'InvestigationError', 'InvestigationLimits', 'InvestigationResult', 'InvestigationToolRouter', 'Investigator', 'Job', 'LLMError', 'LLMGateway', 'LLMProvider', 'LLMRunResult', 'LifecycleError', 'LockBusy', 'OpenAICompatibleProvider', 'OutputError', 'OutputResult', 'OutputValidator', 'PackageFactory', 'PackageLimits', 'PackageReadError', 'PackageReader', 'PackageValidator', 'PackageWriteError', 'PackageWriter', 'PromptBuilder', 'ProviderConfig', 'ProviderResponse', 'ProviderUnavailable', 'QueryContext', 'ResearchError', 'ResearchLimits', 'ResearchService', 'SourceQueryTool', 'State', 'Telemetry', 'TelemetryConfig', 'TelemetryError', 'ToolError', 'ToolLimits', 'ValidationIssue', 'ValidationResult', 'VcoreImporter', 'Workspace', 'WowdiagAdapter', 'WowsrcAdapter', '_GatewayFilter', '_Handler', '_JSONFormatter', '_NoRedirect', '_PhaseLogger', '_Scope']) +# Command-line trial diagnostics use only static identifiers, never free text. +KNOWN_CODES = KNOWN_CODES | frozenset({"TRIAL_DISPATCH_LIMIT", "TRIAL_PHASE"}) +KNOWN_MODULES = KNOWN_MODULES | frozenset({"viano_bug_analysis.trial"}) +KNOWN_CLASSES = KNOWN_CLASSES | frozenset({"TrialError"}) diff --git a/src/viano_bug_analysis/trial.py b/src/viano_bug_analysis/trial.py new file mode 100644 index 0000000..734fc85 --- /dev/null +++ b/src/viano_bug_analysis/trial.py @@ -0,0 +1,221 @@ +"""Bounded, resumable single-case trial; no daemon or automatic unblocking.""" +import argparse +import asyncio +import json +import os +import sys +import uuid +from pathlib import Path + +from .analysis import (CaseInterpreter, HypothesisBuilder, HypothesisConfig, + Investigator, InvestigationConfig, DiagnosisBuilder, + DiagnosisConfig, ChangeRequestBuilder, ChangeRequestConfig, + OutputValidator) +from .budget import BudgetConfig +from .case import CaseBuilder +from .lifecycle import Workspace, InboxProcessor, State +from .lifecycle.locking import FileLock +from .lifecycle.storage import key +from .llm import GatewayConfig, LLMGateway +from .llm.gateway import PRIVACY +from .package import PackageReader +from .profiles.vcore import VcoreImporter +from .telemetry import Telemetry, TelemetryConfig +from .telemetry.safety import safe_code + + +class TrialError(Exception): + def __init__(self, code): + self.code = code + super().__init__(code) + + +def preflight(path): + """Validate transport configuration and credentials without sending requests.""" + config = GatewayConfig.load(path) + gateway = LLMGateway(config) + eligible = [p for p in config.providers + if p.enabled and p.privacy in PRIVACY[config.privacy_profile]] + if not eligible: + raise TrialError("LLM_PRIVACY_NO_PROVIDER") + for provider in eligible: + if provider.api_key_env: + value = os.environ.get(provider.api_key_env) + if not value or any(ord(c) < 32 or ord(c) == 127 for c in value): + raise TrialError("LLM_CREDENTIALS") + return gateway + + +def describe(path, status, *, code=None, exported=None, retry_not_before=None): + with PackageReader(path) as reader: + m = reader.manifest + return {"status": status, "code": code, "package": str(path), + "case_id": m["case_id"], "package_id": m["package_id"], + "package_generation": m["package_generation"], + "analysis_revision": m["analysis_revision"], "state": m["state"], + "routing": m["routing"], "exported": str(exported) if exported else None, + "retry_not_before": retry_not_before} + + +async def drive(store, gateway, case_id, package_id, *, max_dispatches=100, + attempt_id="default", policies=None, progress=None): + if type(max_dispatches) is not int or max_dispatches <= 0: + raise TrialError("TRIAL_DISPATCH_LIMIT") + policies = policies or {} + identity = (case_id, package_id) + interpreter = CaseInterpreter(store, gateway) + hypotheses = HypothesisBuilder(store, gateway, policies.get("analysis")) + loop = Investigator(store, gateway, policies.get("investigation"), + budget_config=policies.get("budget")) + diagnosis = DiagnosisBuilder(store, gateway, policies.get("diagnosis")) + request = ChangeRequestBuilder(store, gateway, policies.get("change_request")) + output = OutputValidator(store, gateway) + for dispatch in range(max_dispatches): + latest = store.latest(*identity) + if latest is None: + raise TrialError("CASE_NOT_FOUND") + with PackageReader(latest, store.limits) as reader: + state = reader.manifest["state"] + artifacts = reader.manifest["artifacts"] + routing = reader.manifest["routing"] + if state == State.BLOCKED: + return describe(latest, "BLOCKED", code=routing["reason"], + retry_not_before=routing["retry_not_before"]) + if progress: + progress({"dispatch": dispatch + 1, "state": state}) + early_phase = state in {State.BUILDING_CASE, State.INTERPRETING_CASE} or ( + state == State.GENERATING_HYPOTHESES and not artifacts.get("hypotheses")) + if state in {State.OUTPUT_READY, State.FAILED_TO_ANALYZE, State.HANDED_OFF}: + result = await output.publish(*identity) + elif state == State.BUILDING_CASE and artifacts.get("case"): + result = await interpreter.interpret(*identity, attempt_id=attempt_id) + elif state in {State.VALIDATED, State.REOPENED, State.BUILDING_CASE}: + result = CaseBuilder(store).build(*identity) + elif state == State.INTERPRETING_CASE: + if artifacts.get("case_interpretation"): + result = await hypotheses.build(*identity, attempt_id=attempt_id) + else: + result = await interpreter.interpret(*identity, attempt_id=attempt_id) + elif state == State.GENERATING_HYPOTHESES: + if artifacts.get("hypotheses"): + result = await loop.step(*identity) + else: + result = await hypotheses.build(*identity, attempt_id=attempt_id) + elif state == State.INVESTIGATING_HYPOTHESES: + result = await loop.step(*identity) + elif state == State.BUILDING_DIAGNOSIS: + if not artifacts.get("diagnosis"): + result = await diagnosis.build(*identity) + elif not artifacts.get("change_request"): + result = await request.build(*identity) + else: + result = await output.publish(*identity) + else: + raise TrialError("TRIAL_PHASE") + status = getattr(result, "status", "COMPLETED") + if status in {"BLOCKED", "RETRY_PENDING", "DEFERRED", "ERROR", "BUSY", + "OUTPUT_READY", "REVIEW", "HANDED_OFF"} or ( + status == "REJECTED" and early_phase): + return describe(result.package, status, code=getattr(result, "code", None), + exported=getattr(result, "exported", None), + retry_not_before=getattr(result, "retry_not_before", None)) + return describe(store.latest(*identity), "YIELDED") + + +def positive(text): + value = int(text) + if value <= 0: + raise argparse.ArgumentTypeError("Expected a positive integer") + return value + + +def parser(): + p = argparse.ArgumentParser(description=__doc__) + commands = p.add_subparsers(dest="command", required=True) + check = commands.add_parser("check", help="Validate configuration/key presence; no network") + check.add_argument("--llm-config", default="config/llm.local.toml") + prepare = commands.add_parser("prepare", help="Import wowdiag/wowsrc into the inbox; no LLM") + prepare.add_argument("--workspace", default="service-data") + prepare.add_argument("--wowdiag", required=True, type=Path) + prepare.add_argument("--wowsrc", type=Path) + prepare.add_argument("--case-id", required=True) + prepare.add_argument("--package-id") + run = commands.add_parser("run", help="Process one case; may make billable API calls") + run.add_argument("--workspace", default="service-data") + run.add_argument("--llm-config", default="config/llm.local.toml") + run.add_argument("--package", type=Path, help="Initial ZIP directly in this workspace's inbox") + run.add_argument("--case-id", help="Resume a managed case") + run.add_argument("--package-id", help="Stable managed package identity") + run.add_argument("--attempt-id", default="default") + run.add_argument("--max-dispatches", type=positive, default=100) + run.add_argument("--telemetry-config", type=Path) + for name in ("analysis", "budget", "investigation", "diagnosis", "change-request"): + run.add_argument(f"--{name}-config", type=Path) + return p + + +def main(argv=None): + p = parser() + args = p.parse_args(argv) + if args.command == "run" and (bool(args.package) == bool(args.case_id and args.package_id) + or bool(args.case_id) != bool(args.package_id)): + p.error("Use --package OR both --case-id and --package-id") + try: + if args.command == "check": + gateway = preflight(args.llm_config) + result = {"status": "CONFIG_READY", "network_tested": False, + "eligible_providers": sum(p.enabled and p.privacy in PRIVACY[gateway.config.privacy_profile] + for p in gateway.config.providers)} + elif args.command == "prepare": + workspace = Workspace(args.workspace) + package_id = args.package_id or str(uuid.uuid4()) + path = workspace.inbox / ("import-" + key(args.case_id + ":" + package_id) + ".zip") + VcoreImporter().create(args.wowdiag, path, case_id=args.case_id, + package_id=package_id, wowsrc=args.wowsrc) + result = describe(path, "PREPARED") + else: + gateway = preflight(args.llm_config) + policies = {} + for name, cls in (("analysis", HypothesisConfig), ("budget", BudgetConfig), + ("investigation", InvestigationConfig), ("diagnosis", DiagnosisConfig), + ("change_request", ChangeRequestConfig)): + path = getattr(args, name + "_config") + if path: + policies[name] = cls.load(path) + config = TelemetryConfig.load(args.telemetry_config) if args.telemetry_config else None + workspace = Workspace(args.workspace) + with Telemetry(workspace, config) as telemetry: + try: + queue = InboxProcessor(workspace, telemetry=telemetry) + if args.package: + admitted = queue.process(args.package) + if admitted.status in {"ERROR", "BUSY", "QUARANTINED", "DEFERRED"}: + print(json.dumps({"status": admitted.status, "code": safe_code(admitted.code)})) + return 2 if admitted.status in {"ERROR", "QUARANTINED"} else 3 + with PackageReader(admitted.package) as reader: + case_id, package_id = reader.manifest["case_id"], reader.manifest["package_id"] + else: + case_id, package_id = args.case_id, args.package_id + # Lock across this one trial; ordinary phase locks remain authoritative. + with FileLock(workspace.runtime / "locks" / ("trial-" + key(case_id) + ".lock")): + result = asyncio.run(drive(queue.store, gateway, case_id, package_id, + max_dispatches=args.max_dispatches, attempt_id=args.attempt_id, policies=policies, + progress=lambda event: print(json.dumps(event), file=sys.stderr, flush=True))) + except Exception as error: + telemetry.guard(telemetry.exception, error) + raise + print(json.dumps(result, ensure_ascii=True)) + return 0 if result["status"] in {"CONFIG_READY", "PREPARED", "OUTPUT_READY", "REVIEW", "HANDED_OFF"} else 3 + except KeyboardInterrupt: + print(json.dumps({"status": "INTERRUPTED"}), file=sys.stderr) + return 130 + except Exception as error: + code = getattr(error, "code", None) + if code not in {"TRIAL_DISPATCH_LIMIT", "TRIAL_PHASE"}: + code = safe_code(code) + print(json.dumps({"status": "ERROR", "code": code}), file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_trial.py b/tests/test_trial.py new file mode 100644 index 0000000..0bb33a9 --- /dev/null +++ b/tests/test_trial.py @@ -0,0 +1,179 @@ +import asyncio +import json +from dataclasses import replace + +import pytest + +from viano_bug_analysis.trial import drive, main, preflight, TrialError +from viano_bug_analysis.llm import ProviderUnavailable +from viano_bug_analysis.package import PackageReader, PackageValidator +from viano_bug_analysis.budget import BudgetManager, BudgetConfig +from viano_bug_analysis.lifecycle import State +from test_case_builder import case_setup +from test_investigation_tools import tool_case +from test_case_interpreter import gateway, FixtureAdapter +from test_hypothesis_builder import ProposalAdapter +from test_investigator import LoopAdapter, CASE +from test_diagnosis_builder import FinalAdapter +from test_change_request_builder import RequestAdapter + + +class PipelineAdapter: + def __init__(self, confirmed=True): + self.calls = 0 + self.adapters = {"case": FixtureAdapter(), "hypotheses": ProposalAdapter(), + "loop": LoopAdapter(), "diagnosis": FinalAdapter(confirmed), + "request": RequestAdapter()} + + async def generate(self, config, request, **kwargs): + self.calls += 1 + phase = {"case_interpretation_v1": "case", "hypotheses_v1": "hypotheses", + "investigation_v1": "loop", "diagnosis_v1": "diagnosis", + "change_request_v1": "request"}[request["prompt_version"]] + adapter = self.adapters[phase] + return await adapter.generate(config, request, **kwargs) + + +@pytest.mark.parametrize("confirmed,expected", [(True, "OUTPUT_READY"), (False, "REVIEW")]) +def test_trial_full_investigation_delivery_and_resume(tool_case, confirmed, expected): + store, original = tool_case + adapter = PipelineAdapter(confirmed) + g = gateway(adapter) + result = asyncio.run(drive(store, g, *CASE)) + assert result["status"] == expected + exported = result["exported"] + assert exported and PackageValidator().validate(exported).valid + with PackageReader(original) as old, PackageReader(exported) as new: + for name in old.files: + if name != "manifest.json": + assert old.read_bytes(name) == new.read_bytes(name) + assert new.manifest["package_id"] == old.manifest["package_id"] + assert bool(new.manifest["artifacts"].get("change_request")) == confirmed + budget = BudgetManager(store).snapshot(*CASE).state + calls = adapter.calls + replay = asyncio.run(drive(store, g, *CASE)) + assert replay["exported"] == exported and adapter.calls == calls + assert BudgetManager(store).snapshot(*CASE).state == budget + + +def test_initial_builder_interpretation_and_hypotheses(case_setup): + store = case_setup[0] + adapter = PipelineAdapter() + result = asyncio.run(drive(store, gateway(adapter), *CASE, max_dispatches=3)) + assert result["status"] == "YIELDED" + assert result["state"] == State.GENERATING_HYPOTHESES + assert adapter.calls == 2 + + +def test_retry_returns_without_sleep_and_blocked_budget_kept(tool_case): + store = tool_case[0] + adapter = LoopAdapter(unavailable=True) + g = gateway(adapter) + g.config = replace(g.config, max_attempts=1) + result = asyncio.run(drive(store, g, *CASE)) + assert result["status"] == "BLOCKED" + budget = BudgetManager(store).snapshot(*CASE).state + calls = len(adapter.contexts) + assert asyncio.run(drive(store, g, *CASE))["status"] == "BLOCKED" + assert len(adapter.contexts) == calls + assert BudgetManager(store).snapshot(*CASE).state == budget + + +def test_retry_pending_and_early_resume_are_nonblocking(case_setup): + store = case_setup[0] + result = asyncio.run(drive(store, gateway(FixtureAdapter(ProviderUnavailable("LLM_CONNECTION"))), *CASE)) + assert result["status"] == "RETRY_PENDING" and result["retry_not_before"] + result = asyncio.run(drive(store, gateway(FixtureAdapter()), *CASE)) + assert result["status"] == "DEFERRED" + + +def test_preflight_no_network_no_secret_echo(tmp_path, monkeypatch, capsys): + config = tmp_path / "llm.toml" + config.write_text('[llm]\nprivacy_profile="cloud_allowed"\n[[llm.providers]]\nid="test"\nmodel="test"\nprivacy="cloud"\nbase_url="https://example.invalid/v1"\napi_key_env="TRIAL_TEST_KEY"\n') + monkeypatch.delenv("TRIAL_TEST_KEY", raising=False) + assert main(["check", "--llm-config", str(config)]) == 2 + assert "LLM_CREDENTIALS" in capsys.readouterr().err + monkeypatch.setenv("TRIAL_TEST_KEY", "private-value") + assert main(["check", "--llm-config", str(config)]) == 0 + output = capsys.readouterr().out + assert json.loads(output)["network_tested"] is False + assert "private-value" not in output + + +def test_cli_rejects_ambiguous_identity(): + with pytest.raises(SystemExit): + main(["run", "--package", "one.zip", "--case-id", "a", "--package-id", "b"]) + + +def test_cli_run_connects_telemetry_and_publishes(tool_case, monkeypatch, capsys): + import viano_bug_analysis.trial as trial + store = tool_case[0] + monkeypatch.setattr(trial, "preflight", lambda path: gateway(PipelineAdapter())) + assert main(["run", "--workspace", str(store.workspace.root), + "--case-id", CASE[0], "--package-id", CASE[1]]) == 0 + captured = capsys.readouterr() + assert json.loads(captured.out)["status"] == "OUTPUT_READY" + assert '"state"' in captured.err + directory = store.workspace.runtime / "telemetry" + assert (directory / "service.jsonl").is_file() + assert 'vba_llm_rounds_total' in (directory / "metrics.prom").read_text() + assert json.loads((directory / "status.json").read_text())["active_operations"] == [] + + +def test_human_test_returns_blocked_with_pending_budget(tool_case): + store = tool_case[0] + adapter = LoopAdapter(test=True) + result = asyncio.run(drive(store, gateway(adapter), *CASE, + policies={"budget": BudgetConfig(action_costs={"testing.request_human": 50})})) + assert result["status"] == "BLOCKED" + assert result["routing"]["reason"] == "AWAITING_TEST_FEEDBACK" + before = BudgetManager(store).snapshot(*CASE).state + assert asyncio.run(drive(store, gateway(adapter), *CASE))["status"] == "BLOCKED" + assert before == BudgetManager(store).snapshot(*CASE).state + + +def test_dispatch_yield_resumes_without_repeating_first_model_call(tool_case): + store = tool_case[0] + adapter = PipelineAdapter() + g = gateway(adapter) + first = asyncio.run(drive(store, g, *CASE, max_dispatches=1)) + assert first["status"] == "YIELDED" and adapter.calls == 1 + result = asyncio.run(drive(store, g, *CASE)) + assert result["status"] == "OUTPUT_READY" and adapter.calls == 8 + + +def test_invalid_early_model_reply_stops_after_one_call(case_setup): + from test_case_interpreter import payload + adapter = FixtureAdapter({**payload(), "unexpected": "invalid"}) + result = asyncio.run(drive(case_setup[0], gateway(adapter), *CASE)) + assert result["status"] in {"ERROR", "REJECTED"} + assert len(adapter.requests) == 1 + + +def test_cli_corrupt_inbox_is_quarantined_without_model_calls(tmp_path, monkeypatch, capsys): + import viano_bug_analysis.trial as trial + from viano_bug_analysis.lifecycle import Workspace + workspace = Workspace(tmp_path / "work") + source = workspace.inbox / "arbitrary.bin" + source.write_bytes(b"corrupt zip") + adapter = PipelineAdapter() + monkeypatch.setattr(trial, "preflight", lambda path: gateway(adapter)) + assert main(["run", "--workspace", str(workspace.root), "--package", str(source)]) == 2 + assert json.loads(capsys.readouterr().out)["status"] == "QUARANTINED" + assert adapter.calls == 0 + assert list(workspace.quarantine.glob("*/validation_error.json")) + + +def test_missing_case_is_safe_cli_error(tmp_path, monkeypatch, capsys): + import viano_bug_analysis.trial as trial + monkeypatch.setattr(trial, "preflight", lambda path: gateway(PipelineAdapter())) + assert main(["run", "--workspace", str(tmp_path / "work"), + "--case-id", "missing", "--package-id", "missing"]) == 2 + output = capsys.readouterr().err + assert '"CASE_NOT_FOUND"' in output and "Traceback" not in output + + +@pytest.mark.parametrize("limit", [0, -1, True]) +def test_dispatch_limit_is_controlled(tool_case, limit): + with pytest.raises(TrialError, match="TRIAL_DISPATCH_LIMIT"): + asyncio.run(drive(tool_case[0], gateway(LoopAdapter()), *CASE, max_dispatches=limit))