165 lines
8.5 KiB
Python
165 lines
8.5 KiB
Python
import json
|
|
import re
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
from .config import TelemetryError
|
|
from .safety import safe_code, error_code
|
|
|
|
|
|
def scheduling(manifest, checkpoint, call, now):
|
|
from viano_bug_analysis.lifecycle.state import State, ACTIVE
|
|
state = manifest["state"]
|
|
if state == State.BLOCKED:
|
|
if manifest["routing"]["reason"] == "AWAITING_TEST_FEEDBACK":
|
|
return "human_wait"
|
|
if checkpoint.get("automatic_retry") is False:
|
|
return "manual_wait"
|
|
due = manifest["routing"]["retry_not_before"]
|
|
return "delayed" if due and datetime.fromisoformat(due.replace("Z", "+00:00")) > now else "ready"
|
|
if state in ACTIVE:
|
|
return "delayed" if call and call.get("status") == "RETRY_PENDING" and datetime.fromisoformat(call["retry_not_before"].replace("Z", "+00:00")) > now else "ready"
|
|
return {State.OUTPUT_READY: "output_ready", State.FAILED_TO_ANALYZE: "review", State.HANDED_OFF: "handed_off"}.get(state)
|
|
|
|
|
|
def count_cases(items):
|
|
from .safety import CATEGORIES
|
|
counts = {category: 0 for category in CATEGORIES}
|
|
for item in items:
|
|
if item["state"] == "BLOCKED":
|
|
counts["blocked"] += 1
|
|
if item.get("scheduling") in counts:
|
|
counts[item["scheduling"]] += 1
|
|
return counts
|
|
|
|
|
|
def utc_now():
|
|
return datetime.now(timezone.utc)
|
|
|
|
|
|
def age(value, now):
|
|
if value is None:
|
|
return None
|
|
return max(0.0, (now - datetime.fromisoformat(value.replace("Z", "+00:00"))).total_seconds())
|
|
|
|
|
|
def assess(snapshot, now=None):
|
|
now = now or utc_now()
|
|
result = dict(snapshot)
|
|
if not snapshot.get("heartbeat_at"):
|
|
result.update(observer="NOT_STARTED", work="QUEUE_UNKNOWN", health="UNKNOWN")
|
|
return result
|
|
heartbeat_age = age(snapshot["heartbeat_at"], now)
|
|
config = snapshot["config"]
|
|
result["heartbeat_age_seconds"] = heartbeat_age
|
|
result["progress_age_seconds"] = age(snapshot.get("last_progress_at") or snapshot["started_at"], now)
|
|
result["observer"] = "STOPPED" if snapshot.get("stopped_at") else "HEARTBEAT_STALE" if heartbeat_age > config["heartbeat_stale_seconds"] else "LIVE"
|
|
queue = snapshot.get("inventory")
|
|
queue_fresh = bool(queue and queue["complete"] and age(queue["observed_at"], now) <= config["inventory_stale_seconds"])
|
|
result["inventory_fresh"] = queue_fresh
|
|
active = any(item["operation"] != "inventory_scan" for item in snapshot["active_operations"])
|
|
if active:
|
|
work = "NO_RECENT_PROGRESS" if any(item.get("stalled") for item in snapshot["active_operations"] if item["operation"] != "inventory_scan") else "WORKING"
|
|
elif not queue_fresh:
|
|
work = "QUEUE_UNKNOWN"
|
|
elif queue["inbox_files"]:
|
|
work = "INPUT_PENDING"
|
|
elif queue["counts"].get("ready", 0):
|
|
work = "READY_NOT_PROCESSING"
|
|
elif queue["counts"].get("blocked", 0) or queue["counts"].get("delayed", 0):
|
|
work = "WAITING_EXTERNAL"
|
|
else:
|
|
work = "IDLE"
|
|
result["work"] = work
|
|
recent_error = snapshot["last_error"] and age(snapshot["last_error"]["timestamp"], now) <= config["progress_warning_seconds"]
|
|
result["health"] = "STOPPED" if result["observer"] == "STOPPED" else "HEARTBEAT_STALE" if result["observer"] != "LIVE" else "DEGRADED" if not snapshot["telemetry_healthy"] or recent_error or queue and not queue["complete"] else "NO_RECENT_PROGRESS" if work == "NO_RECENT_PROGRESS" else "OK"
|
|
return result
|
|
|
|
|
|
def read_status(root, *, now=None, limit=4 * 1024**2):
|
|
from viano_bug_analysis.package.manifest import parse_manifest
|
|
path = Path(root) / "runtime" / "telemetry" / "status.json"
|
|
if not path.exists():
|
|
return {"format": "viano-service-status", "format_version": 1, "observer": "NOT_STARTED", "work": "QUEUE_UNKNOWN", "health": "UNKNOWN"}
|
|
try:
|
|
if path.parent.is_symlink() or path.parent.parent.is_symlink() or path.is_symlink() or path.stat().st_size > limit:
|
|
raise TelemetryError("TELEMETRY_READ")
|
|
with path.open("rb") as stream:
|
|
data = parse_manifest(stream.read(limit + 1))
|
|
if len(json.dumps(data)) > limit * 2:
|
|
raise TelemetryError("TELEMETRY_READ")
|
|
from viano_bug_analysis.case.contracts import validate
|
|
validate(data, "service-status")
|
|
return assess(data, now)
|
|
except Exception as error:
|
|
if isinstance(error, TelemetryError):
|
|
raise
|
|
raise TelemetryError("TELEMETRY_READ") from None
|
|
|
|
|
|
def scan_workspace(root, *, limits=None, max_cases=1000, now=None, cancelled=None):
|
|
"""Passive bounded inventory. Filenames select candidates, never identity/state."""
|
|
from viano_bug_analysis.lifecycle.checkpoints import checkpoint_chain
|
|
from viano_bug_analysis.lifecycle.storage import key
|
|
from viano_bug_analysis.lifecycle.state import State, ACTIVE
|
|
from viano_bug_analysis.package import PackageReader
|
|
root = Path(root).resolve()
|
|
now = now or utc_now()
|
|
counts = {name: 0 for name in ("ready", "blocked", "delayed", "human_wait", "manual_wait", "output_ready", "review", "handed_off")}
|
|
result = {"observed_at": now.isoformat(), "complete": True, "inbox_files": 0, "counts": counts, "cases": [], "errors": []}
|
|
if type(max_cases) is not int or max_cases <= 0:
|
|
raise TelemetryError("TELEMETRY_CONFIG")
|
|
inbox, cases = root / "inbox", root / "cases"
|
|
if inbox.is_symlink() or cases.is_symlink():
|
|
raise TelemetryError("TELEMETRY_SCAN_PATH")
|
|
if inbox.exists():
|
|
result["inbox_files"] = sum(1 for p in inbox.iterdir() if p.is_file() or p.is_symlink())
|
|
if not cases.exists():
|
|
return result
|
|
seen = 0
|
|
for parent in cases.iterdir():
|
|
if parent.is_symlink() or not parent.is_dir() or not re.fullmatch("[0-9a-f]{64}", parent.name):
|
|
result["complete"] = False
|
|
continue
|
|
for directory in parent.iterdir():
|
|
if cancelled is not None and cancelled():
|
|
result["complete"] = False
|
|
result["counts"] = count_cases(result["cases"])
|
|
return result
|
|
seen += 1
|
|
if seen > max_cases:
|
|
result["complete"] = False
|
|
result["counts"] = count_cases(result["cases"])
|
|
return result
|
|
try:
|
|
if directory.is_symlink() or not directory.is_dir() or not re.fullmatch("[0-9a-f]{64}", directory.name):
|
|
raise TelemetryError("TELEMETRY_SCAN_PATH")
|
|
candidate = max(directory.glob("g*.zip"), default=None)
|
|
if candidate is None:
|
|
continue
|
|
if candidate.is_symlink():
|
|
raise TelemetryError("TELEMETRY_SCAN_PATH")
|
|
with PackageReader(candidate, limits) as reader:
|
|
m = reader.manifest
|
|
if (key(m["case_id"]) != parent.name or key(m["package_id"]) != directory.name
|
|
or candidate.name != f"g{m['package_generation']:08d}.zip"):
|
|
raise TelemetryError("TELEMETRY_SCAN_IDENTITY")
|
|
chain = checkpoint_chain(reader)
|
|
if not chain or chain[0][1]["package_generation"] != m["package_generation"]:
|
|
raise TelemetryError("TELEMETRY_SCAN_CHECKPOINT")
|
|
state = m["state"]
|
|
reason = safe_code(m["routing"]["reason"]) if m["routing"]["reason"] else None
|
|
item = {"case_key": parent.name, "package_key": directory.name, "state": state,
|
|
"analysis_revision": m["analysis_revision"], "package_generation": m["package_generation"],
|
|
"reason": reason, "retry_not_before": m["routing"]["retry_not_before"]}
|
|
previous = next((cp["details"] for _, cp in chain if cp["analysis_revision"] == m["analysis_revision"] and cp["details"].get("operation") == "llm-round" and cp["details"].get("phase") == state), None)
|
|
item["scheduling"] = scheduling(m, chain[0][1]["details"], previous, now)
|
|
if item["scheduling"] == "delayed" and previous and state in ACTIVE:
|
|
item["retry_not_before"] = previous["retry_not_before"]
|
|
result["cases"].append(item)
|
|
except Exception as error:
|
|
result["complete"] = False
|
|
result["errors"].append({"case_key": parent.name, "package_key": directory.name if re.fullmatch("[0-9a-f]{64}", directory.name) else None, "code": error_code(error)})
|
|
result["counts"] = count_cases(result["cases"])
|
|
return result
|