Files
VianoBugAnalysis/src/viano_bug_analysis/analysis/output_validator.py
T

193 lines
12 KiB
Python

"""Validate portable final output and atomically export to a local delivery queue."""
from viano_bug_analysis.telemetry.decorators import observed
import hashlib
import os
import tempfile
from dataclasses import dataclass
from pathlib import Path
from viano_bug_analysis.case.contracts import read_json, require_file, validate
from viano_bug_analysis.case.errors import CaseBuildError
from viano_bug_analysis.evidence.store import digest
from viano_bug_analysis.lifecycle import State, LifecycleError
from viano_bug_analysis.lifecycle.checkpoints import checkpoint_chain
from viano_bug_analysis.lifecycle.errors import LockBusy
from viano_bug_analysis.lifecycle.locking import FileLock
from viano_bug_analysis.lifecycle.storage import json_bytes, key
from viano_bug_analysis.package import PackageReader, PackageReadError
from viano_bug_analysis.package.validator import ARCHIVE_ERRORS
from .change_request_builder import ChangeRequestBuilder, ChangeRequestError
from .diagnosis_builder import DiagnosisError
class OutputError(Exception):
def __init__(self, code):
self.code = code
super().__init__(code)
@dataclass(frozen=True)
class OutputResult:
status: str
package: Path
exported: Path | None = None
reused: bool = False
class OutputValidator:
def __init__(self, store, gateway, limits=None, *, router=None):
self.store = store
self.builder = ChangeRequestBuilder(store, gateway, limits=limits, router=router)
self.limits = self.builder.limits
@observed("output_validate")
async def validate(self, case_id, package_id):
"""Read-only validation: guards prohibit initiating missing model stages."""
try:
path = self.store.latest(case_id, package_id)
if path is None:
raise OutputError("OUTPUT_NOT_READY")
with PackageReader(path, self.store.limits) as reader:
m = reader.manifest
revision = m["analysis_revision"]
chain = checkpoint_chain(reader)
diagnosis = next((cp for _, cp in chain if cp["step_id"] == f"diagnosis:v1:{revision}:complete"), None)
cr = next((cp for _, cp in chain if cp["step_id"] == f"change-request:v1:{revision}:complete"), None)
if not diagnosis:
raise OutputError("OUTPUT_NOT_READY")
doc = read_json(reader, m["artifacts"]["diagnosis"], "diagnosis-result", self.limits.max_json_bytes)
if doc["diagnosis"]["status"] == "confirmed" and not cr:
raise OutputError("OUTPUT_CHANGE_REQUEST_NOT_READY")
if doc["diagnosis"]["status"] == "inconclusive" and cr:
raise OutputError("OUTPUT_INCONCLUSIVE_CODING_TASK")
is_review = doc["diagnosis"]["status"] == "inconclusive" or cr and cr["details"]["failure_ref"] is not None
if is_review:
if m["state"] != State.FAILED_TO_ANALYZE or m["routing"]["destination"] != "review" or m["artifacts"]["change_request"] is not None:
raise OutputError("OUTPUT_REVIEW_ROUTING")
elif m["state"] not in {State.BUILDING_DIAGNOSIS, State.OUTPUT_READY, State.HANDED_OFF}:
raise OutputError("OUTPUT_PHASE")
result = await self.builder.build(case_id, package_id) if cr else await self.builder.diagnosis.build(case_id, package_id)
if result.package != path or result.status not in {"COMPLETED", "INCONCLUSIVE"}:
raise OutputError("OUTPUT_CHANGED")
with PackageReader(path, self.store.limits) as reader:
m = reader.manifest
if not is_review:
request = read_json(reader, m["artifacts"]["change_request"], "change-request", self.limits.max_json_bytes)
if request["request"]["status"] != "ready":
raise OutputError("OUTPUT_CHANGE_REQUEST_NOT_READY")
published = next((cp for _, cp in checkpoint_chain(reader) if cp["step_id"] == f"output:v1:{revision}:validated"), None)
if published:
if m["state"] not in {State.OUTPUT_READY, State.HANDED_OFF}:
raise OutputError("OUTPUT_PHASE")
d = published["details"]
report = read_json(reader, d["validation_ref"], "output-validation", self.limits.max_json_bytes)
if (m["artifacts"].get("output_validation") != d["validation_ref"] or digest(report) != d["validation_sha256"]
or report["diagnosis_ref"] != m["artifacts"]["diagnosis"] or report["diagnosis_sha256"] != digest(doc)
or report["change_request_ref"] != m["artifacts"]["change_request"] or report["change_request_sha256"] != digest(request)
or report["source_generation"] != published["source_generation"]
or any(report[f] != m[f] for f in ("case_id", "package_id", "analysis_revision"))):
raise OutputError("OUTPUT_REPORT_CHANGED")
elif m["state"] in {State.OUTPUT_READY, State.HANDED_OFF} or m["artifacts"].get("output_validation"):
raise OutputError("OUTPUT_VALIDATION_MISSING")
if m["state"] in {State.OUTPUT_READY, State.HANDED_OFF} and m["routing"]["destination"] != "outbox":
raise OutputError("OUTPUT_ROUTING")
else:
require_file(reader, m["artifacts"].get("failure_report"))
read_json(reader, m["artifacts"]["failure_report"], "failure-report", self.limits.max_json_bytes)
return OutputResult("REVIEW" if is_review else "VALID", path)
except LockBusy:
raise
except (ChangeRequestError, DiagnosisError, CaseBuildError, LifecycleError, PackageReadError) as error:
raise OutputError(getattr(error, "code", "OUTPUT_PACKAGE")) from None
except ARCHIVE_ERRORS + (ValueError, TypeError, KeyError, IndexError, UnicodeError, RecursionError):
raise OutputError("OUTPUT_INPUT") from None
@observed("output_finalize")
async def finalize(self, case_id, package_id):
try:
with FileLock(self.store.workspace.runtime / "locks" / f"output-{key(case_id)}.lock"):
checked = await self.validate(case_id, package_id)
if checked.status == "REVIEW":
return checked
with PackageReader(checked.package, self.store.limits) as reader:
m = reader.manifest
if m["state"] in {State.OUTPUT_READY, State.HANDED_OFF}:
return OutputResult(m["state"], checked.package, reused=True)
diagnosis = read_json(reader, m["artifacts"]["diagnosis"], "diagnosis-result", self.limits.max_json_bytes)
request = read_json(reader, m["artifacts"]["change_request"], "change-request", self.limits.max_json_bytes)
report = {"format": "viano-output-validation", "format_version": 1, "status": "valid", "validator_version": "1",
**{f: m[f] for f in ("case_id", "package_id", "analysis_revision")}, "source_generation": m["package_generation"],
"diagnosis_ref": m["artifacts"]["diagnosis"], "diagnosis_sha256": digest(diagnosis),
"change_request_ref": m["artifacts"]["change_request"], "change_request_sha256": digest(request)}
validate(report, "output-validation")
ref = f"analysis/output/{m['analysis_revision']:04d}/validation.json"
path = self.store.advance(case_id, package_id, State.OUTPUT_READY, step_id=f"output:v1:{m['analysis_revision']}:validated",
details={"operation": "output-validated", "validation_ref": ref, "validation_sha256": digest(report)},
additions={ref: json_bytes(report)}, artifacts={"output_validation": ref}, expected_generation=m["package_generation"])
await self.validate(case_id, package_id)
return OutputResult("OUTPUT_READY", path)
except (LifecycleError, PackageReadError, CaseBuildError) as error:
raise OutputError(getattr(error, "code", "OUTPUT_PACKAGE")) from None
@observed("output_publish")
async def publish(self, case_id, package_id):
"""Copy a validated generation; local availability is not recipient ACK."""
checked = await self.finalize(case_id, package_id)
if checked.status == "HANDED_OFF":
return checked
try:
with FileLock(self.store.workspace.runtime / "locks" / f"output-{key(case_id)}.lock"):
verified = await self.validate(case_id, package_id)
if verified.package != checked.package:
raise OutputError("OUTPUT_CHANGED")
with PackageReader(checked.package, self.store.limits) as reader:
m = reader.manifest
folder = self.store.workspace.review if checked.status == "REVIEW" else self.store.workspace.outbox
if folder.is_symlink():
raise OutputError("OUTPUT_UNSAFE_DESTINATION")
name = f"{key(case_id)}-{key(package_id)}-r{m['analysis_revision']:04d}-g{m['package_generation']:08d}.zip"
target = folder / name
if target.is_symlink():
raise OutputError("OUTPUT_UNSAFE_DESTINATION")
sha = hashlib.sha256()
reader._source.seek(0)
while chunk := reader._source.read(65536):
sha.update(chunk)
if target.exists():
previous = hashlib.sha256()
total = 0
with target.open("rb") as stream:
while chunk := stream.read(65536):
total += len(chunk)
if total > self.store.limits.max_archive_bytes:
raise OutputError("OUTPUT_EXPORT_CONFLICT")
previous.update(chunk)
if previous.digest() != sha.digest():
raise OutputError("OUTPUT_EXPORT_CONFLICT")
return OutputResult(checked.status, checked.package, target, True)
self._copy(reader, target)
return OutputResult(checked.status, checked.package, target)
except (OSError, ValueError, PackageReadError, LifecycleError) as error:
raise OutputError(getattr(error, "code", "OUTPUT_STORAGE")) from None
@staticmethod
def _copy(reader, target):
temporary = None
try:
with tempfile.NamedTemporaryFile(dir=target.parent, prefix=".vbcase-", delete=False) as stream:
temporary = Path(stream.name)
reader._source.seek(0)
while chunk := reader._source.read(65536):
stream.write(chunk)
stream.flush()
os.fsync(stream.fileno())
# Same-directory hard-link publication is atomic and refuses to
# overwrite a destination created since our existence check.
try:
os.link(temporary, target)
except FileExistsError:
raise OutputError("OUTPUT_EXPORT_CONFLICT") from None
finally:
if temporary is not None:
temporary.unlink(missing_ok=True)