193 lines
12 KiB
Python
193 lines
12 KiB
Python
"""Validate portable final output and atomically export to a local delivery queue."""
|
|
from viano_bug_analysis.telemetry.decorators import observed
|
|
import hashlib
|
|
import os
|
|
import tempfile
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
|
|
from viano_bug_analysis.case.contracts import read_json, require_file, validate
|
|
from viano_bug_analysis.case.errors import CaseBuildError
|
|
from viano_bug_analysis.evidence.store import digest
|
|
from viano_bug_analysis.lifecycle import State, LifecycleError
|
|
from viano_bug_analysis.lifecycle.checkpoints import checkpoint_chain
|
|
from viano_bug_analysis.lifecycle.errors import LockBusy
|
|
from viano_bug_analysis.lifecycle.locking import FileLock
|
|
from viano_bug_analysis.lifecycle.storage import json_bytes, key
|
|
from viano_bug_analysis.package import PackageReader, PackageReadError
|
|
from viano_bug_analysis.package.validator import ARCHIVE_ERRORS
|
|
from .change_request_builder import ChangeRequestBuilder, ChangeRequestError
|
|
from .diagnosis_builder import DiagnosisError
|
|
|
|
|
|
class OutputError(Exception):
|
|
def __init__(self, code):
|
|
self.code = code
|
|
super().__init__(code)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class OutputResult:
|
|
status: str
|
|
package: Path
|
|
exported: Path | None = None
|
|
reused: bool = False
|
|
|
|
|
|
class OutputValidator:
|
|
def __init__(self, store, gateway, limits=None, *, router=None):
|
|
self.store = store
|
|
self.builder = ChangeRequestBuilder(store, gateway, limits=limits, router=router)
|
|
self.limits = self.builder.limits
|
|
|
|
@observed("output_validate")
|
|
async def validate(self, case_id, package_id):
|
|
"""Read-only validation: guards prohibit initiating missing model stages."""
|
|
try:
|
|
path = self.store.latest(case_id, package_id)
|
|
if path is None:
|
|
raise OutputError("OUTPUT_NOT_READY")
|
|
with PackageReader(path, self.store.limits) as reader:
|
|
m = reader.manifest
|
|
revision = m["analysis_revision"]
|
|
chain = checkpoint_chain(reader)
|
|
diagnosis = next((cp for _, cp in chain if cp["step_id"] == f"diagnosis:v1:{revision}:complete"), None)
|
|
cr = next((cp for _, cp in chain if cp["step_id"] == f"change-request:v1:{revision}:complete"), None)
|
|
if not diagnosis:
|
|
raise OutputError("OUTPUT_NOT_READY")
|
|
doc = read_json(reader, m["artifacts"]["diagnosis"], "diagnosis-result", self.limits.max_json_bytes)
|
|
if doc["diagnosis"]["status"] == "confirmed" and not cr:
|
|
raise OutputError("OUTPUT_CHANGE_REQUEST_NOT_READY")
|
|
if doc["diagnosis"]["status"] == "inconclusive" and cr:
|
|
raise OutputError("OUTPUT_INCONCLUSIVE_CODING_TASK")
|
|
is_review = doc["diagnosis"]["status"] == "inconclusive" or cr and cr["details"]["failure_ref"] is not None
|
|
if is_review:
|
|
if m["state"] != State.FAILED_TO_ANALYZE or m["routing"]["destination"] != "review" or m["artifacts"]["change_request"] is not None:
|
|
raise OutputError("OUTPUT_REVIEW_ROUTING")
|
|
elif m["state"] not in {State.BUILDING_DIAGNOSIS, State.OUTPUT_READY, State.HANDED_OFF}:
|
|
raise OutputError("OUTPUT_PHASE")
|
|
result = await self.builder.build(case_id, package_id) if cr else await self.builder.diagnosis.build(case_id, package_id)
|
|
if result.package != path or result.status not in {"COMPLETED", "INCONCLUSIVE"}:
|
|
raise OutputError("OUTPUT_CHANGED")
|
|
with PackageReader(path, self.store.limits) as reader:
|
|
m = reader.manifest
|
|
if not is_review:
|
|
request = read_json(reader, m["artifacts"]["change_request"], "change-request", self.limits.max_json_bytes)
|
|
if request["request"]["status"] != "ready":
|
|
raise OutputError("OUTPUT_CHANGE_REQUEST_NOT_READY")
|
|
published = next((cp for _, cp in checkpoint_chain(reader) if cp["step_id"] == f"output:v1:{revision}:validated"), None)
|
|
if published:
|
|
if m["state"] not in {State.OUTPUT_READY, State.HANDED_OFF}:
|
|
raise OutputError("OUTPUT_PHASE")
|
|
d = published["details"]
|
|
report = read_json(reader, d["validation_ref"], "output-validation", self.limits.max_json_bytes)
|
|
if (m["artifacts"].get("output_validation") != d["validation_ref"] or digest(report) != d["validation_sha256"]
|
|
or report["diagnosis_ref"] != m["artifacts"]["diagnosis"] or report["diagnosis_sha256"] != digest(doc)
|
|
or report["change_request_ref"] != m["artifacts"]["change_request"] or report["change_request_sha256"] != digest(request)
|
|
or report["source_generation"] != published["source_generation"]
|
|
or any(report[f] != m[f] for f in ("case_id", "package_id", "analysis_revision"))):
|
|
raise OutputError("OUTPUT_REPORT_CHANGED")
|
|
elif m["state"] in {State.OUTPUT_READY, State.HANDED_OFF} or m["artifacts"].get("output_validation"):
|
|
raise OutputError("OUTPUT_VALIDATION_MISSING")
|
|
if m["state"] in {State.OUTPUT_READY, State.HANDED_OFF} and m["routing"]["destination"] != "outbox":
|
|
raise OutputError("OUTPUT_ROUTING")
|
|
else:
|
|
require_file(reader, m["artifacts"].get("failure_report"))
|
|
read_json(reader, m["artifacts"]["failure_report"], "failure-report", self.limits.max_json_bytes)
|
|
return OutputResult("REVIEW" if is_review else "VALID", path)
|
|
except LockBusy:
|
|
raise
|
|
except (ChangeRequestError, DiagnosisError, CaseBuildError, LifecycleError, PackageReadError) as error:
|
|
raise OutputError(getattr(error, "code", "OUTPUT_PACKAGE")) from None
|
|
except ARCHIVE_ERRORS + (ValueError, TypeError, KeyError, IndexError, UnicodeError, RecursionError):
|
|
raise OutputError("OUTPUT_INPUT") from None
|
|
|
|
@observed("output_finalize")
|
|
async def finalize(self, case_id, package_id):
|
|
try:
|
|
with FileLock(self.store.workspace.runtime / "locks" / f"output-{key(case_id)}.lock"):
|
|
checked = await self.validate(case_id, package_id)
|
|
if checked.status == "REVIEW":
|
|
return checked
|
|
with PackageReader(checked.package, self.store.limits) as reader:
|
|
m = reader.manifest
|
|
if m["state"] in {State.OUTPUT_READY, State.HANDED_OFF}:
|
|
return OutputResult(m["state"], checked.package, reused=True)
|
|
diagnosis = read_json(reader, m["artifacts"]["diagnosis"], "diagnosis-result", self.limits.max_json_bytes)
|
|
request = read_json(reader, m["artifacts"]["change_request"], "change-request", self.limits.max_json_bytes)
|
|
report = {"format": "viano-output-validation", "format_version": 1, "status": "valid", "validator_version": "1",
|
|
**{f: m[f] for f in ("case_id", "package_id", "analysis_revision")}, "source_generation": m["package_generation"],
|
|
"diagnosis_ref": m["artifacts"]["diagnosis"], "diagnosis_sha256": digest(diagnosis),
|
|
"change_request_ref": m["artifacts"]["change_request"], "change_request_sha256": digest(request)}
|
|
validate(report, "output-validation")
|
|
ref = f"analysis/output/{m['analysis_revision']:04d}/validation.json"
|
|
path = self.store.advance(case_id, package_id, State.OUTPUT_READY, step_id=f"output:v1:{m['analysis_revision']}:validated",
|
|
details={"operation": "output-validated", "validation_ref": ref, "validation_sha256": digest(report)},
|
|
additions={ref: json_bytes(report)}, artifacts={"output_validation": ref}, expected_generation=m["package_generation"])
|
|
await self.validate(case_id, package_id)
|
|
return OutputResult("OUTPUT_READY", path)
|
|
except (LifecycleError, PackageReadError, CaseBuildError) as error:
|
|
raise OutputError(getattr(error, "code", "OUTPUT_PACKAGE")) from None
|
|
|
|
@observed("output_publish")
|
|
async def publish(self, case_id, package_id):
|
|
"""Copy a validated generation; local availability is not recipient ACK."""
|
|
checked = await self.finalize(case_id, package_id)
|
|
if checked.status == "HANDED_OFF":
|
|
return checked
|
|
try:
|
|
with FileLock(self.store.workspace.runtime / "locks" / f"output-{key(case_id)}.lock"):
|
|
verified = await self.validate(case_id, package_id)
|
|
if verified.package != checked.package:
|
|
raise OutputError("OUTPUT_CHANGED")
|
|
with PackageReader(checked.package, self.store.limits) as reader:
|
|
m = reader.manifest
|
|
folder = self.store.workspace.review if checked.status == "REVIEW" else self.store.workspace.outbox
|
|
if folder.is_symlink():
|
|
raise OutputError("OUTPUT_UNSAFE_DESTINATION")
|
|
name = f"{key(case_id)}-{key(package_id)}-r{m['analysis_revision']:04d}-g{m['package_generation']:08d}.zip"
|
|
target = folder / name
|
|
if target.is_symlink():
|
|
raise OutputError("OUTPUT_UNSAFE_DESTINATION")
|
|
sha = hashlib.sha256()
|
|
reader._source.seek(0)
|
|
while chunk := reader._source.read(65536):
|
|
sha.update(chunk)
|
|
if target.exists():
|
|
previous = hashlib.sha256()
|
|
total = 0
|
|
with target.open("rb") as stream:
|
|
while chunk := stream.read(65536):
|
|
total += len(chunk)
|
|
if total > self.store.limits.max_archive_bytes:
|
|
raise OutputError("OUTPUT_EXPORT_CONFLICT")
|
|
previous.update(chunk)
|
|
if previous.digest() != sha.digest():
|
|
raise OutputError("OUTPUT_EXPORT_CONFLICT")
|
|
return OutputResult(checked.status, checked.package, target, True)
|
|
self._copy(reader, target)
|
|
return OutputResult(checked.status, checked.package, target)
|
|
except (OSError, ValueError, PackageReadError, LifecycleError) as error:
|
|
raise OutputError(getattr(error, "code", "OUTPUT_STORAGE")) from None
|
|
|
|
@staticmethod
|
|
def _copy(reader, target):
|
|
temporary = None
|
|
try:
|
|
with tempfile.NamedTemporaryFile(dir=target.parent, prefix=".vbcase-", delete=False) as stream:
|
|
temporary = Path(stream.name)
|
|
reader._source.seek(0)
|
|
while chunk := reader._source.read(65536):
|
|
stream.write(chunk)
|
|
stream.flush()
|
|
os.fsync(stream.fileno())
|
|
# Same-directory hard-link publication is atomic and refuses to
|
|
# overwrite a destination created since our existence check.
|
|
try:
|
|
os.link(temporary, target)
|
|
except FileExistsError:
|
|
raise OutputError("OUTPUT_EXPORT_CONFLICT") from None
|
|
finally:
|
|
if temporary is not None:
|
|
temporary.unlink(missing_ok=True)
|