From c03f70e3e15cdf47b51f02561eb30d6cb86851c0 Mon Sep 17 00:00:00 2001 From: Nitsu45 Date: Wed, 14 Oct 2026 01:09:55 +0200 Subject: [PATCH] Add durable evidence storage and item-level novelty accounting --- README.md | 3 + docs/evidence-store-v1.md | 148 ++++++++++ src/viano_bug_analysis/budget/engine.py | 1 + src/viano_bug_analysis/budget/manager.py | 39 ++- src/viano_bug_analysis/case/assembler.py | 17 ++ src/viano_bug_analysis/evidence/__init__.py | 5 + src/viano_bug_analysis/evidence/scoring.py | 37 +++ src/viano_bug_analysis/evidence/store.py | 272 ++++++++++++++++++ .../schemas/budget-event-v1.schema.json | 5 + .../schemas/budget-request-v1.schema.json | 5 + .../schemas/evidence-content-v1.schema.json | 35 +++ .../schemas/evidence-index-v1.schema.json | 83 ++++++ .../schemas/evidence-record-v1.schema.json | 148 ++++++++++ tests/profiles/vcore/test_real_package.py | 14 + tests/test_evidence_store.py | 230 +++++++++++++++ 15 files changed, 1040 insertions(+), 2 deletions(-) create mode 100644 docs/evidence-store-v1.md create mode 100644 src/viano_bug_analysis/evidence/__init__.py create mode 100644 src/viano_bug_analysis/evidence/scoring.py create mode 100644 src/viano_bug_analysis/evidence/store.py create mode 100644 src/viano_bug_analysis/schemas/evidence-content-v1.schema.json create mode 100644 src/viano_bug_analysis/schemas/evidence-index-v1.schema.json create mode 100644 src/viano_bug_analysis/schemas/evidence-record-v1.schema.json create mode 100644 tests/test_evidence_store.py diff --git a/README.md b/README.md index 98995f5..281fec0 100644 --- a/README.md +++ b/README.md @@ -43,3 +43,6 @@ bot/client adapters are future integrations. Read-only packaged source/database queries and the strict tool allowlist: [investigation-tools-v1.md](docs/investigation-tools-v1.md). + +Portable evidence records, provenance, case-wide IDs and novelty-aware budget +integration: [evidence-store-v1.md](docs/evidence-store-v1.md). diff --git a/docs/evidence-store-v1.md b/docs/evidence-store-v1.md new file mode 100644 index 0000000..7c68db8 --- /dev/null +++ b/docs/evidence-store-v1.md @@ -0,0 +1,148 @@ +# EvidenceStore v1 + +This milestone persists validated tool replies and confirmed test feedback as +portable evidence. It adds no model calls, autonomous investigation, diagnosis, +patch generation, external research service, web UI or bot steering. The +authoritative architecture is unchanged. + +## Controller API + +```python +from viano_bug_analysis.evidence import EvidenceStore + +# Reserve each requested action through BudgetManager before the query. +reply = router.query(budget.snapshot(case_id, package_id).package, request) +evidence = EvidenceStore(store, router=router) +saved = evidence.store_tool_results( + case_id, package_id, "iteration-0001", request, reply, +) +budget.record_evidence(case_id, package_id, "H1", "query-0001", saved.record_refs[0]) +# A future validated controller assessment supplies the bounded semantic score. +budget.settle(case_id, package_id, "H1", "query-0001", + semantic_score=50, reason="Assessment of the stored evidence") +``` + +The caller's router must use the same trusted descriptor adapter and operational +limits as the original query. Before publication, EvidenceStore deterministically +recomputes the whole batch against the current package and compares the complete +reply. Invented snippets, rows, counts, provenance, changed identities or stale +package generations reject the entire batch. Each result must match a still +reserved budget action and operation. The full batch is published atomically; +no partial evidence records or ghost IDs survive rejection. + +`store_feedback(case_id, package_id, operation_id, feedback_ref)` imports an +already checkpoint-confirmed async test reply, bound to the original hypothesis, +action and pending reservation. Draft/unconfirmed feedback is not evidence. +Feedback and attachment hashes are verified; no new test request is submitted. + +`snapshot(case_id, package_id)` validates and returns all record references in +creation order without requiring an active current budget revision. History is +therefore readable while a reopened case has not yet rebuilt its hypotheses. +An empty store has `index_ref = None` and no records. Mutations require initialized +budget accounting and `GENERATING_HYPOTHESES` or `INVESTIGATING_HYPOTHESES`. +Blocked snapshots preserve their budget and evidence; new records wait for the +existing controlled resume path. + +## Records, content and immutable indices + +The controller assigns `E000001`, `E000002`, … across the entire case lifecycle. +IDs do not reset at an analysis revision or after service restart/transport. +The model cannot supply IDs, novelty scores, duplicate markers or budget values. + +```text +analysis/evidence/0001//request.json +analysis/evidence/0001//result.json +analysis/evidence/0001//E000001.json +analysis/evidence/0001//index.json +analysis/evidence/content/.json +``` + +`evidence-record-v1` binds case/package identity and revision, hypothesis/action, +operation, content ref/hash, first identical record (`duplicate_of`), novel atom +hashes and deterministic novelty score. Its origin links the unchanged request +and reply with hashes and original reply position, or the confirmed feedback ref +and hash. Source bytes, line numbers, row JSON pointers, coverage and truncation +markers remain in the stored normalized content and original reply. + +`evidence-content-v1` separates actual content from query IDs, provider metadata, +pagination bookkeeping and source package generation. Source occurrences retain +location/hash/text; database rows retain location/hash/selected row values; schema +descriptors retain columns and provenance. Full contents are reused by hash. +Identical content obtained again gets a new occurrence record with a reference +to the first record; neither original record nor provenance is overwritten. + +`evidence-index-v1` is an immutable cumulative index of record IDs/refs/hashes, +with a parent ref/hash and next-ID counter. `manifest.artifacts.evidence_index` +points to the latest index. Checkpoints bind that index hash and the new record +refs. Deleting a pointer or changing counts, contents, origin hashes, duplicate +markers or novelty metadata fails replay; it cannot reset IDs or restore novelty. +All original and unknown package payloads survive unchanged. New publications +increase `package_generation`, leaving `analysis_revision` unchanged. + +## Novelty and budget integration + +Each located content item is an atom. Its identity includes content kind, scope, +coverage and the original item's data/provenance. Atoms are tracked across all +stored evidence, hypotheses and analysis revisions. Sorting and deduplication +are deterministic; changing request/action IDs or grouping an existing excerpt +into a new result is not new evidence. + +```text +novelty_score = floor(100 * unseen_item_count / returned_item_count) +``` + +Empty findings and confirmed `unable_to_test` replies have score zero. A partly +overlapping excerpt can only claim its new items. For example, one new line in +a two-line excerpt caps novelty at 50; rediscovering both lines caps it at zero. +This detects exact located-content reuse, not semantic equivalence or correctness. +Different projections, genuinely different source snapshots and richer excerpts +can contain new atoms; judging their practical relevance remains the semantic +assessment's job. Coverage and truncation remain explicit, so an empty partial +snapshot is not a claim of server-wide absence. + +`BudgetManager.record_evidence` verifies record/index/checkpoint bindings, loads +the canonical content and passes only newly observed atoms to accounting. Its +optional `novelty_cap` result field is a backward-compatible addition to the +budget v1 request/event schemas. The deterministic score cannot exceed either +the evidence cap or the configured new-evidence score. Empty novel payloads earn +no novelty reward even when a controller supplies semantic score 100. Existing +direct `record_result` callers remain supported unchanged. + +Evidence publication alone does not debit, refund, settle, refill or terminate +any account. Result accounting and later semantic settlement remain separate +idempotent steps. A crash between those steps leaves the reservation recoverable, +not free to allocate again. Rewards still satisfy `reward < cost` and retain the +configured low-information stopping rules. No hypothesis confidence or root +cause is inferred by the store. + +## Robustness and limits + +The store uses a per-case evidence lock, then the existing short publication +locks and generation comparison. No lock waits on an external human or model. +Read-only recomputation verifies local snapshots while publication is protected. +Repeated identical `operation_id` payloads reuse the existing records; changed +payloads reject. IDs are scoped by revision/operation for operation replay but +the evidence ID sequence remains case-wide. Completed publication with a failed +local index update is repaired on replay without another record or budget debit. + +Replay validates schemas, index lineage, hashes, original request/reply bindings, +source/feedback attachment hashes and recomputes novelty/duplicate metadata. +Hashes detect inconsistency, not an adversary rewriting and rehashing an entire +case history; they are not signatures. + +`EvidenceLimits` defaults to 4 MiB per JSON artifact, 10,000 records and 64 MiB +across distinct retained evidence artifacts. Limits reject explicitly and are +configurable operational defaults, not permanent v1 format caps. Package, case, +router and budget limits remain independently applicable. Cumulative indices +and full history replay trade simplicity for overhead; optimization is later work. + +CaseAssembler excludes only checkpoint-declared evidence files and their content +refs from raw input identity. Thus storing investigation evidence does not +invalidate the accepted case; undeclared files still count as new raw input. + +Tests cover preservation, provenance, duplicate and overlapping excerpts, scoped +empty results, forged/stale replies, missing reservations, idempotence, limits, +pointer deletion, reopen/transport, publication crashes and confirmed test +feedback. The real BUG-20 regression additionally stores its source/schema/row +results and accounts them through the existing budget engine, using simulated +semantic assessments and no known fix. diff --git a/src/viano_bug_analysis/budget/engine.py b/src/viano_bug_analysis/budget/engine.py index 77871e4..431b139 100644 --- a/src/viano_bug_analysis/budget/engine.py +++ b/src/viano_bug_analysis/budget/engine.py @@ -60,6 +60,7 @@ def apply(state, request, policy): digest = evidence_digest(request["evidence"]) novelty = (policy["new_evidence_score"] if request["evidence"] and digest not in state["seen_evidence"] else 0) + novelty = min(novelty, request.get("novelty_cap",100)) if digest not in state["seen_evidence"]: state["seen_evidence"].append(digest) action.update(status="awaiting_assessment", evidence_sha256=digest, diff --git a/src/viano_bug_analysis/budget/manager.py b/src/viano_bug_analysis/budget/manager.py index 97f6912..9e8b546 100644 --- a/src/viano_bug_analysis/budget/manager.py +++ b/src/viano_bug_analysis/budget/manager.py @@ -66,9 +66,42 @@ class BudgetManager: return self._execute(case_id, package_id, {"operation": "reserve", "hypothesis_id": hypothesis_id, "action_id": action_id, "action": action}) - def record_result(self, case_id, package_id, hypothesis_id, action_id, evidence, *, source_refs): + def record_result(self, case_id, package_id, hypothesis_id, action_id, evidence, *, source_refs, novelty_cap=None): + extra = {} if novelty_cap is None else {"novelty_cap": novelty_cap} return self._execute(case_id, package_id, {"operation": "result", "hypothesis_id": hypothesis_id, - "action_id": action_id, "evidence": evidence, "source_refs": source_refs}) + "action_id": action_id, "evidence": evidence, "source_refs": source_refs, **extra}) + + def record_evidence(self, case_id, package_id, hypothesis_id, action_id, record_ref): + """Account canonical EvidenceStore content, never record/transport IDs.""" + from viano_bug_analysis.evidence.scoring import budget_payload + try: + path = self.store.latest(case_id, package_id) + if path is None: + raise BudgetError("BUDGET_CASE_NOT_FOUND") + with PackageReader(path, self.store.limits) as reader: + record = read_json(reader, record_ref, "evidence-record", self.limits.max_result_bytes) + content = read_json(reader, record["content_ref"], "evidence-content", self.limits.max_result_bytes) + index_ref = reader.manifest["artifacts"].get("evidence_index") + index = read_json(reader, index_ref, "evidence-index", self.limits.max_event_bytes) + if (any(record[field] != reader.manifest[field] for field in ("case_id", "package_id", "analysis_revision")) + or any(index[field] != reader.manifest[field] for field in ("case_id", "package_id", "analysis_revision")) + or record["hypothesis_id"] != hypothesis_id or record["action_id"] != action_id + or digest(content) != record["content_sha256"] + or not any(entry["record_ref"] == record_ref and entry["sha256"] == digest(record) for entry in index["entries"]) + or not any(cp["details"].get("operation") == "evidence-stored" + and cp["details"].get("index_ref") == index_ref + and cp["details"].get("index_sha256") == digest(index) for _,cp in checkpoint_chain(reader)) + or not any(cp["details"].get("operation") == "evidence-stored" + and record_ref in cp["details"].get("record_refs", []) for _,cp in checkpoint_chain(reader))): + raise BudgetError("BUDGET_EVIDENCE_BINDING") + evidence = budget_payload(content,record["novel_atoms"]) + novelty_cap = record["novelty_score"] + sources = [record_ref, record["content_ref"]] + return self.record_result(case_id, package_id, hypothesis_id, action_id, evidence, source_refs=sources, novelty_cap=novelty_cap) + except (CaseBuildError, LifecycleError, PackageReadError) as error: + raise BudgetError(getattr(error,"code","BUDGET_EVIDENCE_BINDING")) from None + except ARCHIVE_ERRORS + (ValueError,TypeError,KeyError,RecursionError): + raise BudgetError("BUDGET_EVIDENCE_BINDING") from None def settle(self, case_id, package_id, hypothesis_id, action_id, *, semantic_score, reason): return self._execute(case_id, package_id, {"operation": "settle", "hypothesis_id": hypothesis_id, @@ -132,6 +165,8 @@ class BudgetManager: def _sources(self, reader, request): if request["operation"] != "result": return + if "novelty_cap" in request and not integer(request["novelty_cap"],0,100): + raise BudgetError("BUDGET_NOVELTY_CAP") refs = request["source_refs"] if (not isinstance(refs, list) or not refs or len(refs) != len(set(refs))): raise BudgetError("BUDGET_SOURCE_REFERENCES") diff --git a/src/viano_bug_analysis/case/assembler.py b/src/viano_bug_analysis/case/assembler.py index 1e7e4a7..ae23893 100644 --- a/src/viano_bug_analysis/case/assembler.py +++ b/src/viano_bug_analysis/case/assembler.py @@ -84,6 +84,23 @@ class CaseAssembler: prior_refs = [] for _, checkpoint in reversed(chain): details = checkpoint["details"] + if details.get("operation") == "evidence-stored": + refs = [details.get("index_ref"), details.get("request_ref"), details.get("result_ref"), + *details.get("record_refs", [])] + for ref in refs: + if ref is None: + continue + if not isinstance(ref, str) or not ref.startswith("analysis/evidence/"): + raise CaseBuildError("PREVIOUS_CASE", "Invalid evidence checkpoint") + require_file(reader, ref) + excluded.add(ref) + for ref in details.get("record_refs", []): + record = read_json(reader, ref, "evidence-record", self.limits.max_json_bytes) + content_ref = record["content_ref"] + if not content_ref.startswith("analysis/evidence/content/"): + raise CaseBuildError("PREVIOUS_CASE", "Invalid evidence content reference") + require_file(reader, content_ref) + excluded.add(content_ref) if details.get("operation") in {"async-test-wait", "async-test-feedback-confirmed"}: for field in ("request_ref", "feedback_ref"): ref = details.get(field) diff --git a/src/viano_bug_analysis/evidence/__init__.py b/src/viano_bug_analysis/evidence/__init__.py new file mode 100644 index 0000000..c085b05 --- /dev/null +++ b/src/viano_bug_analysis/evidence/__init__.py @@ -0,0 +1,5 @@ +"""Durable evidence records with case-wide stable IDs and provenance.""" + +from .store import EvidenceStore, EvidenceResult, EvidenceError, EvidenceLimits + +__all__ = ["EvidenceStore", "EvidenceResult", "EvidenceError", "EvidenceLimits"] diff --git a/src/viano_bug_analysis/evidence/scoring.py b/src/viano_bug_analysis/evidence/scoring.py new file mode 100644 index 0000000..4671fe3 --- /dev/null +++ b/src/viano_bug_analysis/evidence/scoring.py @@ -0,0 +1,37 @@ +"""Normalize evidence content without request IDs or pagination bookkeeping.""" + +from viano_bug_analysis.lifecycle.storage import json_bytes +import hashlib + + +def tool_content(result): + operation, data = result["operation"], result["data"] + if operation == "database.describe": + items = [{"table": data["table"], "columns": data["columns"], "source": source} + for source in data["checked_sources"]] + kind = "database_schema" + else: + items = data["matches"] + kind = "source" if operation.startswith("source.") else "database_rows" + # Sort without changing or stripping original row/line pointers and hashes. + unique = {json_bytes(item): item for item in items} + return {"kind": kind, "scope": data["scope"], "coverage": data.get("coverage", "selected_files"), + "items": [unique[key] for key in sorted(unique)]} + + +def feedback_content(feedback): + items = [{name: feedback[name] for name in ("executor", "tester_id", "environment", "observations", "attachments")}] + return {"kind": "test_feedback", "scope": "confirmed_external_test", "coverage": feedback["outcome"], + "items": items if feedback["outcome"] == "completed" else []} + + +def atom_hashes(content): + return [hashlib.sha256(json_bytes({"kind":content["kind"],"scope":content["scope"], + "coverage":content["coverage"],"item":item})).hexdigest() + for item in content["items"]] + + +def budget_payload(content, novel_atoms=None): + # Empty findings remain auditable in the raw reply, but never earn novelty. + items = content["items"] if novel_atoms is None else [item for item,sha in zip(content["items"],atom_hashes(content)) if sha in novel_atoms] + return {**content,"items":items} if items else [] diff --git a/src/viano_bug_analysis/evidence/store.py b/src/viano_bug_analysis/evidence/store.py new file mode 100644 index 0000000..679d99e --- /dev/null +++ b/src/viano_bug_analysis/evidence/store.py @@ -0,0 +1,272 @@ +import copy +import hashlib +from dataclasses import asdict, dataclass +from pathlib import Path + +from viano_bug_analysis.budget import BudgetManager, BudgetError +from viano_bug_analysis.case.contracts import read_json, require_file, validate +from viano_bug_analysis.case.errors import CaseBuildError +from viano_bug_analysis.lifecycle import LifecycleError, State +from viano_bug_analysis.lifecycle.checkpoints import checkpoint_chain +from viano_bug_analysis.lifecycle.errors import LockBusy +from viano_bug_analysis.lifecycle.locking import FileLock +from viano_bug_analysis.lifecycle.storage import atomic_json, json_bytes, key +from viano_bug_analysis.package import PackageReader, PackageReadError +from viano_bug_analysis.package.validator import ARCHIVE_ERRORS +from viano_bug_analysis.tools import InvestigationToolRouter, ToolError + +from .scoring import tool_content, feedback_content, atom_hashes + + +class EvidenceError(Exception): + def __init__(self, code): + self.code = code + super().__init__(code) + + +@dataclass(frozen=True) +class EvidenceLimits: + max_json_bytes: int = 4 * 1024 * 1024 + max_records: int = 10000 + max_total_bytes: int = 64 * 1024 * 1024 + + def __post_init__(self): + if any(type(value) is not int or value <= 0 for value in asdict(self).values()): + raise EvidenceError("EVIDENCE_LIMIT_CONFIG") + + +@dataclass(frozen=True) +class EvidenceResult: + package: Path + index_ref: str | None + record_refs: tuple[str, ...] + reused: bool = False + + +def digest(value): + return hashlib.sha256(json_bytes(value)).hexdigest() + + +class EvidenceStore: + def __init__(self, store, limits=None, *, router=None): + self.store = store + self.limits = limits or EvidenceLimits() + self.router = router or InvestigationToolRouter() + + def store_tool_results(self, case_id, package_id, operation_id, request, result): + return self._execute(case_id, package_id, operation_id, {"kind":"tool", "request":request, "result":result}) + + def store_feedback(self, case_id, package_id, operation_id, feedback_ref): + return self._execute(case_id, package_id, operation_id, {"kind":"feedback", "feedback_ref":feedback_ref}) + + def snapshot(self, case_id, package_id): + return self._execute(case_id, package_id, None, None) + + def _read(self, reader, ref, contract): + return read_json(reader, ref, contract, self.limits.max_json_bytes) + + def _load(self, reader): + entries, records = [], [] + parent_ref = parent_hash = None + used = 0 + seen_atoms = set() + seen_refs = set() + def charge(ref): + nonlocal used + if ref is not None and ref not in seen_refs: + require_file(reader, ref) + seen_refs.add(ref) + used += reader._archive.getinfo(ref).file_size + if used > self.limits.max_total_bytes: + raise EvidenceError("EVIDENCE_HISTORY_LIMIT") + for _, cp in reversed(checkpoint_chain(reader)): + details = cp["details"] + if details.get("operation") != "evidence-stored": + continue + for field in ("index_ref", "request_ref", "result_ref"): + charge(details.get(field)) + index = self._read(reader, details["index_ref"], "evidence-index") + if (index["case_id"] != reader.manifest["case_id"] or index["package_id"] != reader.manifest["package_id"] + or index["analysis_revision"] != cp["analysis_revision"] or digest(index) != details["index_sha256"] + or index["parent_ref"] != parent_ref or index["parent_sha256"] != parent_hash): + raise EvidenceError("EVIDENCE_HISTORY_INVALID") + for ref in details["record_refs"]: + record = self._read(reader, ref, "evidence-record") + content = self._read(reader, record["content_ref"], "evidence-content") + charge(ref) + charge(record["content_ref"]) + if len(entries) >= self.limits.max_records: + raise EvidenceError("EVIDENCE_HISTORY_LIMIT") + expected_id = f"E{len(entries)+1:06d}" + if (record["id"] != expected_id or record["case_id"] != index["case_id"] + or record["package_id"] != index["package_id"] or record["analysis_revision"] != index["analysis_revision"] + or digest(content) != record["content_sha256"]): + raise EvidenceError("EVIDENCE_RECORD_INVALID") + if (not ref.startswith("analysis/evidence/") or not ref.endswith(f"/{expected_id}.json") + or record["content_ref"] != f"analysis/evidence/content/{record['content_sha256']}.json"): + raise EvidenceError("EVIDENCE_RECORD_INVALID") + duplicate = next((r["id"] for r in records if r["content_sha256"] == record["content_sha256"]), None) + if record["duplicate_of"] != duplicate: + raise EvidenceError("EVIDENCE_DUPLICATE_CHANGED") + atoms = atom_hashes(content) + novel = [sha for sha in atoms if sha not in seen_atoms] + if record["novel_atoms"] != novel or record["novelty_score"] != (100*len(novel)//len(atoms) if atoms else 0): + raise EvidenceError("EVIDENCE_NOVELTY_CHANGED") + seen_atoms.update(atoms) + self._verify_origin(reader, record, content) + entries.append({"id":record["id"], "record_ref":ref, "sha256":digest(record)}) + records.append(record) + if index["entries"] != entries or index["next_id"] != len(entries)+1: + raise EvidenceError("EVIDENCE_INDEX_CHANGED") + parent_ref, parent_hash = details["index_ref"], digest(index) + if reader.manifest["artifacts"].get("evidence_index") != parent_ref: + raise EvidenceError("EVIDENCE_POINTER_CHANGED") + return entries, records, parent_ref, parent_hash, used, seen_atoms + + def _verify_origin(self, reader, record, content): + origin = record["origin"] + if origin["kind"] == "tool": + request = self._read(reader, origin["request_ref"], "investigation-request") + result = self._read(reader, origin["result_ref"], "investigation-result") + position = origin["position"] + item, asked = result["results"][position], request["requests"][position] + if (digest(request) != origin["request_sha256"] or digest(result) != origin["result_sha256"] + or any(result[field] != record[field] for field in ("case_id","package_id","analysis_revision")) + or digest(request) != result["batch_sha256"] or item["request_sha256"] != digest(asked) + or any(item[field] != record[field] or asked[field] != record[field] + for field in ("hypothesis_id", "action_id", "operation")) + or tool_content(item) != content): + raise EvidenceError("EVIDENCE_ORIGIN_CHANGED") + for source in item["data"]["checked_sources"]: + self._source_hash(reader, source["source_ref"], source["sha256"]) + else: + feedback = self._read(reader, origin["feedback_ref"], "async-test-feedback") + if (digest(feedback) != origin["feedback_sha256"] or feedback_content(feedback) != content + or any(feedback[field] != record[field] for field in ("hypothesis_id", "action_id")) + or not any(cp["details"].get("operation") == "async-test-feedback-confirmed" + and cp["details"].get("feedback_ref") == origin["feedback_ref"] + and cp["details"].get("feedback_sha256") == digest(feedback) + for _, cp in checkpoint_chain(reader))): + raise EvidenceError("EVIDENCE_ORIGIN_CHANGED") + for attachment in feedback["attachments"]: + self._source_hash(reader,attachment["source_ref"],attachment["sha256"]) + if any((request if origin["kind"] == "tool" else feedback)[field] != record[field] + for field in ("case_id", "package_id", "analysis_revision")): + raise EvidenceError("EVIDENCE_IDENTITY") + + @staticmethod + def _source_hash(reader, ref, expected): + require_file(reader, ref) + h = hashlib.sha256() + with reader._archive.open(ref) as stream: + while chunk := stream.read(65536): + h.update(chunk) + if h.hexdigest() != expected: + raise EvidenceError("EVIDENCE_SOURCE_CHANGED") + + def _execute(self, case_id, package_id, operation_id, payload): + try: + payload = copy.deepcopy(payload) + if payload is not None and (not isinstance(operation_id,str) or not operation_id.strip()): + raise EvidenceError("EVIDENCE_OPERATION_ID") + with FileLock(self.store.workspace.runtime / "locks" / f"evidence-{key(case_id)}.lock"): + snapshot = BudgetManager(self.store).snapshot(case_id,package_id) if payload is not None else None + with self.store._locks(case_id): + path = self.store.latest(case_id,package_id) + if path is None: + raise EvidenceError("EVIDENCE_CASE_NOT_FOUND") + if snapshot is not None and path != snapshot.package: + raise EvidenceError("EVIDENCE_GENERATION_CHANGED") + with PackageReader(path,self.store.limits) as reader: + return self._locked(reader,operation_id,payload,snapshot) + except LockBusy: + raise + except (CaseBuildError,BudgetError,LifecycleError,PackageReadError,ToolError) as error: + raise EvidenceError(getattr(error,"code","EVIDENCE_INPUT")) from None + except ARCHIVE_ERRORS + (ValueError,TypeError,KeyError,IndexError,RecursionError): + raise EvidenceError("EVIDENCE_INPUT") from None + + def _locked(self, reader, operation_id, payload, budget): + entries, records, parent_ref, parent_hash, used, seen_atoms = self._load(reader) + if payload is None: + self.store._index(reader.path,reader.manifest) + atomic_json(self.store.workspace.runtime / "current_job.json",None) + return EvidenceResult(reader.path,parent_ref,tuple(e["record_ref"] for e in entries),True) + manifest = reader.manifest + revision = manifest["analysis_revision"] + base = f"analysis/evidence/{revision:04d}/{key(operation_id)}" + step = f"evidence:v1:{revision}:{key(operation_id)}" + existing = next((cp for _,cp in checkpoint_chain(reader) if cp["step_id"] == step),None) + request_hash = digest(payload) + if existing: + if existing["request_hash"] != request_hash: + raise EvidenceError("EVIDENCE_STEP_CONFLICT") + self.store._index(reader.path,manifest) + atomic_json(self.store.workspace.runtime / "current_job.json",None) + return EvidenceResult(reader.path,parent_ref,tuple(existing["details"]["record_refs"]),True) + if manifest["state"] not in {State.GENERATING_HYPOTHESES,State.INVESTIGATING_HYPOTHESES}: + raise EvidenceError("EVIDENCE_PHASE") + changes = {} + items = [] + if payload["kind"] == "tool": + request,result = payload["request"],payload["result"] + if self.router.query(reader.path,request) != result: + raise EvidenceError("EVIDENCE_RESULT_CHANGED") + request_ref,result_ref = base+"/request.json",base+"/result.json" + changes.update({request_ref:json_bytes(request),result_ref:json_bytes(result)}) + for position,item in enumerate(result["results"]): + items.append((item,tool_content(item),{"kind":"tool","request_ref":request_ref,"request_sha256":digest(request), + "result_ref":result_ref,"result_sha256":digest(result),"position":position})) + else: + feedback = self._read(reader,payload["feedback_ref"],"async-test-feedback") + if any(feedback[field] != manifest[field] for field in ("case_id","package_id","analysis_revision")): + raise EvidenceError("EVIDENCE_IDENTITY") + item = {"hypothesis_id":feedback["hypothesis_id"],"action_id":feedback["action_id"], + "operation":"testing.request_human" if feedback["executor"] == "human" else "testing.request_automated"} + items.append((item,feedback_content(feedback),{"kind":"feedback","feedback_ref":payload["feedback_ref"],"feedback_sha256":digest(feedback)})) + refs = [] + for item,content,origin in items: + account = budget.state["accounts"].get(item["hypothesis_id"],{}) + action = account.get("actions",{}).get(item["action_id"],{}) + if action.get("status") != "reserved" or action.get("action") != item["operation"]: + raise EvidenceError("EVIDENCE_RESERVATION") + sha = digest(content) + atoms = atom_hashes(content) + novel = [atom for atom in atoms if atom not in seen_atoms] + seen_atoms.update(atoms) + content_ref = f"analysis/evidence/content/{sha}.json" + ref = base+f"/E{len(entries)+1:06d}.json" + record = {"format":"viano-evidence-record","format_version":1, + **{f:manifest[f] for f in ("case_id","package_id","analysis_revision")}, + "id":f"E{len(entries)+1:06d}", "hypothesis_id":item["hypothesis_id"],"action_id":item["action_id"], + "operation":item["operation"],"content_ref":content_ref,"content_sha256":sha, + "novel_atoms":novel,"novelty_score":100*len(novel)//len(atoms) if atoms else 0, + "duplicate_of":next((r["id"] for r in records if r["content_sha256"] == sha),None),"origin":origin} + validate(record,"evidence-record") + validate(content,"evidence-content") + if origin["kind"] == "feedback": + self._verify_origin(reader,record,content) + if content_ref in reader.files: + if self._read(reader,content_ref,"evidence-content") != content: + raise EvidenceError("EVIDENCE_CONTENT_CHANGED") + else: + changes[content_ref] = json_bytes(content) + changes[ref] = json_bytes(record) + refs.append(ref) + entries.append({"id":record["id"],"record_ref":ref,"sha256":digest(record)}) + records.append(record) + index_ref = base+"/index.json" + index = {"format":"viano-evidence-index","format_version":1, + **{f:manifest[f] for f in ("case_id","package_id","analysis_revision")}, + "parent_ref":parent_ref,"parent_sha256":parent_hash,"next_id":len(entries)+1,"entries":entries} + validate(index,"evidence-index") + changes[index_ref] = json_bytes(index) + if (len(entries)>self.limits.max_records or any(len(data)>self.limits.max_json_bytes for data in changes.values()) + or used+sum(len(data) for data in changes.values())>self.limits.max_total_bytes): + raise EvidenceError("EVIDENCE_LIMIT") + path = self.store._commit(reader,step,request_hash,manifest["state"],additions=changes, + details={"operation":"evidence-stored","index_ref":index_ref,"index_sha256":digest(index), + "record_refs":refs,"request_ref":base+"/request.json" if payload["kind"] == "tool" else None, + "result_ref":base+"/result.json" if payload["kind"] == "tool" else None}, + updates={"artifacts":{"evidence_index":index_ref}}) + return EvidenceResult(path,index_ref,tuple(refs)) diff --git a/src/viano_bug_analysis/schemas/budget-event-v1.schema.json b/src/viano_bug_analysis/schemas/budget-event-v1.schema.json index d5848c1..efd8699 100644 --- a/src/viano_bug_analysis/schemas/budget-event-v1.schema.json +++ b/src/viano_bug_analysis/schemas/budget-event-v1.schema.json @@ -216,6 +216,11 @@ "type": "string", "minLength": 1, "pattern": "\\S" + }, + "novelty_cap": { + "type": "integer", + "minimum": 0, + "maximum": 100 } }, "required": [ diff --git a/src/viano_bug_analysis/schemas/budget-request-v1.schema.json b/src/viano_bug_analysis/schemas/budget-request-v1.schema.json index 20c1006..70a8b06 100644 --- a/src/viano_bug_analysis/schemas/budget-request-v1.schema.json +++ b/src/viano_bug_analysis/schemas/budget-request-v1.schema.json @@ -86,6 +86,11 @@ "type": "string", "minLength": 1, "pattern": "\\S" + }, + "novelty_cap": { + "type": "integer", + "minimum": 0, + "maximum": 100 } }, "required": [ diff --git a/src/viano_bug_analysis/schemas/evidence-content-v1.schema.json b/src/viano_bug_analysis/schemas/evidence-content-v1.schema.json new file mode 100644 index 0000000..f3b8dbb --- /dev/null +++ b/src/viano_bug_analysis/schemas/evidence-content-v1.schema.json @@ -0,0 +1,35 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "scope", + "coverage", + "items" + ], + "properties": { + "kind": { + "enum": [ + "source", + "database_rows", + "database_schema", + "test_feedback" + ] + }, + "scope": { + "type": "string", + "minLength": 1 + }, + "coverage": { + "type": "string", + "minLength": 1 + }, + "items": { + "type": "array", + "items": { + "type": "object" + } + } + } +} diff --git a/src/viano_bug_analysis/schemas/evidence-index-v1.schema.json b/src/viano_bug_analysis/schemas/evidence-index-v1.schema.json new file mode 100644 index 0000000..6e213e6 --- /dev/null +++ b/src/viano_bug_analysis/schemas/evidence-index-v1.schema.json @@ -0,0 +1,83 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "additionalProperties": false, + "required": [ + "format", + "format_version", + "case_id", + "package_id", + "analysis_revision", + "parent_ref", + "parent_sha256", + "next_id", + "entries" + ], + "properties": { + "format": { + "const": "viano-evidence-index" + }, + "format_version": { + "const": 1 + }, + "case_id": { + "type": "string", + "minLength": 1 + }, + "package_id": { + "type": "string", + "minLength": 1 + }, + "analysis_revision": { + "type": "integer", + "minimum": 1 + }, + "parent_ref": { + "type": [ + "string", + "null" + ] + }, + "parent_sha256": { + "anyOf": [ + { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + { + "type": "null" + } + ] + }, + "next_id": { + "type": "integer", + "minimum": 1 + }, + "entries": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "record_ref", + "sha256" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "record_ref": { + "type": "string", + "minLength": 1 + }, + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } + } + } + } +} diff --git a/src/viano_bug_analysis/schemas/evidence-record-v1.schema.json b/src/viano_bug_analysis/schemas/evidence-record-v1.schema.json new file mode 100644 index 0000000..b9d4f7e --- /dev/null +++ b/src/viano_bug_analysis/schemas/evidence-record-v1.schema.json @@ -0,0 +1,148 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "additionalProperties": false, + "required": [ + "format", + "format_version", + "case_id", + "package_id", + "analysis_revision", + "id", + "hypothesis_id", + "action_id", + "operation", + "content_ref", + "content_sha256", + "duplicate_of", + "origin", + "novel_atoms", + "novelty_score" + ], + "properties": { + "format": { + "const": "viano-evidence-record" + }, + "format_version": { + "const": 1 + }, + "case_id": { + "type": "string", + "minLength": 1 + }, + "package_id": { + "type": "string", + "minLength": 1 + }, + "analysis_revision": { + "type": "integer", + "minimum": 1 + }, + "id": { + "type": "string", + "pattern": "^E[0-9]{6,}$" + }, + "hypothesis_id": { + "type": "string", + "minLength": 1 + }, + "action_id": { + "type": "string", + "minLength": 1 + }, + "operation": { + "type": "string", + "minLength": 1 + }, + "content_ref": { + "type": "string", + "minLength": 1 + }, + "content_sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "duplicate_of": { + "type": [ + "string", + "null" + ] + }, + "origin": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "request_ref", + "result_ref", + "position", + "request_sha256", + "result_sha256" + ], + "properties": { + "kind": { + "const": "tool" + }, + "request_ref": { + "type": "string", + "minLength": 1 + }, + "result_ref": { + "type": "string", + "minLength": 1 + }, + "position": { + "type": "integer", + "minimum": 0 + }, + "request_sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "result_sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "feedback_ref", + "feedback_sha256" + ], + "properties": { + "kind": { + "const": "feedback" + }, + "feedback_ref": { + "type": "string", + "minLength": 1 + }, + "feedback_sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } + } + ] + }, + "novel_atoms": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + }, + "novelty_score": { + "type": "integer", + "minimum": 0, + "maximum": 100 + } + } +} diff --git a/tests/profiles/vcore/test_real_package.py b/tests/profiles/vcore/test_real_package.py index 8e24875..aa9d425 100644 --- a/tests/profiles/vcore/test_real_package.py +++ b/tests/profiles/vcore/test_real_package.py @@ -114,4 +114,18 @@ def test_full_bug20_package_pair(tmp_path): assert queried["results"][2]["data"]["matches"][0]["row"] == {"ID":11117} assert queried["results"][2]["data"]["coverage"] == "unknown" assert hashlib.sha256(accounted.package.read_bytes()).hexdigest() == before_query_hash + from viano_bug_analysis.evidence import EvidenceStore + for asked in request["requests"]: + budget.reserve(*identity,asked["hypothesis_id"],asked["action_id"],asked["operation"]) + current=budget.snapshot(*identity) + queried=InvestigationToolRouter(database_tool=DatabaseQueryTool(normalize_schema_descriptor)).query(current.package,request) + evidence=EvidenceStore(store,router=InvestigationToolRouter(database_tool=DatabaseQueryTool(normalize_schema_descriptor))) + saved=evidence.store_tool_results(*identity,"real-snapshot-query",request,queried) + assert len(saved.record_refs)==3 + assert budget.snapshot(*identity).state==current.state + assert PackageValidator().validate(saved.package).valid + for asked,ref in zip(request["requests"],saved.record_refs): + budget.record_evidence(*identity,asked["hypothesis_id"],asked["action_id"],ref) + budget.settle(*identity,asked["hypothesis_id"],asked["action_id"],semantic_score=50,reason="Offline provenance regression") + assert CaseBuilder(store).build(*identity).reused assert [hashlib.sha256(path.read_bytes()).hexdigest() for path in (diag, src)] == original_hashes diff --git a/tests/test_evidence_store.py b/tests/test_evidence_store.py new file mode 100644 index 0000000..d8f5eec --- /dev/null +++ b/tests/test_evidence_store.py @@ -0,0 +1,230 @@ +import copy +import json +from importlib.resources import files + +import pytest +from jsonschema import Draft202012Validator + +from viano_bug_analysis.budget import BudgetManager, BudgetError +from viano_bug_analysis.case import CaseBuilder +from viano_bug_analysis.evidence import EvidenceStore, EvidenceError, EvidenceLimits +from viano_bug_analysis.lifecycle import CheckpointStore, Workspace, State +from viano_bug_analysis.package import PackageReader, PackageValidator +from viano_bug_analysis.tools import InvestigationToolRouter +from test_investigation_tools import tool_case, batch +from test_case_builder import case_setup +from test_case_interpreter import built_case +from test_hypothesis_builder import interpreted_case +from test_budget_manager import hypotheses +from test_async_tests import setup as async_setup + +CASE=("case-A","stable-package") + + +def prepare(tool_case, *, action="A1", text="activate", budget=None): + store=tool_case[0] + budget=budget or BudgetManager(store) + budget.initialize(*CASE) + reserved=budget.reserve(*CASE,"H1",action,"source.find_text") + request=batch() + request["requests"][0].update(action_id=action,request_id=action) + request["requests"][0]["arguments"]["text"]=text + result=InvestigationToolRouter().query(reserved.package,request) + return budget,request,result + + +def test_roundtrip_stable_ids_provenance_preservation_and_budget(tool_case,tmp_path): + budget,request,result=prepare(tool_case) + before=budget.snapshot(*CASE) + evidence=EvidenceStore(tool_case[0]) + saved=evidence.store_tool_results(*CASE,"query1",request,result) + assert PackageValidator().validate(saved.package).valid + assert budget.snapshot(*CASE).state==before.state + with PackageReader(before.package) as old, PackageReader(saved.package) as reader: + record=json.loads(reader.read_bytes(saved.record_refs[0])) + content=json.loads(reader.read_bytes(record["content_ref"])) + assert record["id"]=="E000001" and record["duplicate_of"] is None + assert len(content["items"])==3 + assert record["origin"]["position"]==0 + assert reader.manifest["package_generation"]==old.manifest["package_generation"]+1 + for name in old.files: + if name!="manifest.json": assert old.read_bytes(name)==reader.read_bytes(name) + assert evidence.store_tool_results(*CASE,"query1",request,result).reused + assert CaseBuilder(tool_case[0]).build(*CASE).reused + budget.record_evidence(*CASE,"H1","A1",saved.record_refs[0]) + settled=budget.settle(*CASE,"H1","A1",semantic_score=100,reason="Relevant excerpt") + assert settled.state["accounts"]["H1"]["remaining"]==98 + portable=CheckpointStore(Workspace(tmp_path/"portable")) + portable.intake(settled.package) + restored=EvidenceStore(portable).snapshot(*CASE) + assert restored.record_refs==saved.record_refs + assert BudgetManager(portable).snapshot(*CASE).state==settled.state + + +def test_metadata_changes_do_not_make_duplicates_novel(tool_case): + budget,request,result=prepare(tool_case) + evidence=EvidenceStore(tool_case[0]) + first=evidence.store_tool_results(*CASE,"query1",request,result) + budget.record_evidence(*CASE,"H1","A1",first.record_refs[0]) + budget.settle(*CASE,"H1","A1",semantic_score=100,reason="Relevant") + budget,request,result=prepare(tool_case,action="A2",budget=budget) + second=evidence.store_tool_results(*CASE,"query2",request,result) + with PackageReader(second.package) as reader: + one=json.loads(reader.read_bytes(first.record_refs[0])) + two=json.loads(reader.read_bytes(second.record_refs[0])) + assert two["id"]=="E000002" and two["duplicate_of"]==one["id"] + assert two["content_ref"]==one["content_ref"] + budget.record_evidence(*CASE,"H1","A2",second.record_refs[0]) + settled=budget.settle(*CASE,"H1","A2",semantic_score=100,reason="Duplicate") + assert settled.state["accounts"]["H1"]["actions"]["A2"]["reward"]==0 + + +def test_empty_results_remain_auditable_without_novelty_reward(tool_case): + budget,request,result=prepare(tool_case,text="not-present") + saved=EvidenceStore(tool_case[0]).store_tool_results(*CASE,"empty",request,result) + budget.record_evidence(*CASE,"H1","A1",saved.record_refs[0]) + settled=budget.settle(*CASE,"H1","A1",semantic_score=100,reason="Scoped absence") + action=settled.state["accounts"]["H1"]["actions"]["A1"] + assert action["novelty_score"]==action["reward"]==0 + + +def test_overlapping_excerpts_only_reward_new_items(tool_case): + store=tool_case[0]; budget=BudgetManager(store); budget.initialize(*CASE) + first=budget.reserve(*CASE,"H1","A1","source.find_text") + request=batch();request["requests"][0]["arguments"]["limit"]=1 + result=InvestigationToolRouter().query(first.package,request) + evidence=EvidenceStore(store) + saved=evidence.store_tool_results(*CASE,"one-line",request,result) + budget.record_evidence(*CASE,"H1","A1",saved.record_refs[0]) + budget.settle(*CASE,"H1","A1",semantic_score=100,reason="First line") + second=budget.reserve(*CASE,"H1","A2","source.read_context") + request=batch("source.read_context",{"source_ref":"tool/code.cpp","start_line":1,"end_line":2}) + request["requests"][0].update(action_id="A2") + result=InvestigationToolRouter().query(second.package,request) + saved=evidence.store_tool_results(*CASE,"overlap",request,result) + with PackageReader(saved.package) as reader: + record=json.loads(reader.read_bytes(saved.record_refs[0])) + assert record["novelty_score"]==50 and len(record["novel_atoms"])==1 + budget.record_evidence(*CASE,"H1","A2",saved.record_refs[0]) + settled=budget.settle(*CASE,"H1","A2",semantic_score=100,reason="Only one new line") + assert settled.state["accounts"]["H1"]["actions"]["A2"]["reward"]==4 + + +def test_history_readable_after_reopen_and_ids_not_reset(tool_case): + import asyncio + from viano_bug_analysis.analysis import CaseInterpreter,HypothesisBuilder + from test_case_interpreter import gateway,FixtureAdapter + from test_hypothesis_builder import ProposalAdapter + budget,request,result=prepare(tool_case) + store=tool_case[0];evidence=EvidenceStore(store) + first=evidence.store_tool_results(*CASE,"query1",request,result) + budget.record_evidence(*CASE,"H1","A1",first.record_refs[0]) + budget.settle(*CASE,"H1","A1",semantic_score=100,reason="Known excerpt") + store.advance(*CASE,State.FAILED_TO_ANALYZE,step_id="review",reason="Review") + store.advance(*CASE,State.REOPENED,step_id="reopen") + assert evidence.snapshot(*CASE).record_refs==first.record_refs + CaseBuilder(store).build(*CASE) + asyncio.run(CaseInterpreter(store,gateway(FixtureAdapter())).interpret(*CASE)) + asyncio.run(HypothesisBuilder(store,gateway(ProposalAdapter())).build(*CASE)) + budget=BudgetManager(store);budget.initialize(*CASE) + reserved=budget.reserve(*CASE,"H1","A1","source.find_text") + request=batch();request["analysis_revision"]=2 + result=InvestigationToolRouter().query(reserved.package,request) + second=evidence.store_tool_results(*CASE,"query1",request,result) + with PackageReader(second.package) as reader: + record=json.loads(reader.read_bytes(second.record_refs[0])) + assert record["id"]=="E000002" and record["duplicate_of"]=="E000001" + budget.record_evidence(*CASE,"H1","A1",second.record_refs[0]) + settled=budget.settle(*CASE,"H1","A1",semantic_score=100,reason="Old evidence") + assert settled.state["accounts"]["H1"]["rewarded"]==0 + + +@pytest.mark.parametrize("change",[ + lambda r:r.update(package_generation=0), + lambda r:r["results"][0]["data"]["matches"][0].update(text="Invented code"), + lambda r:r["results"][0].update(hypothesis_id="H2"), + lambda r:r.update(batch_sha256="0"*64), +]) +def test_forged_results_rejected_atomically(tool_case,change): + budget,request,result=prepare(tool_case) + before=budget.snapshot(*CASE) + change(result) + with pytest.raises(EvidenceError): + EvidenceStore(tool_case[0]).store_tool_results(*CASE,"bad",request,result) + assert budget.snapshot(*CASE).package==before.package + + +def test_conflicting_operation_and_missing_reservation(tool_case): + store=tool_case[0] + budget,request,result=prepare(tool_case) + evidence=EvidenceStore(store) + saved=evidence.store_tool_results(*CASE,"query1",request,result) + changed=copy.deepcopy(result); changed["results"][0]["data"]["total_matches"]=999 + with pytest.raises(EvidenceError,match="EVIDENCE_STEP_CONFLICT"): + evidence.store_tool_results(*CASE,"query1",request,changed) + assert evidence.snapshot(*CASE).package==saved.package + fresh=copy.deepcopy(request); fresh["requests"][0]["action_id"]="unreserved" + result=InvestigationToolRouter().query(saved.package,fresh) + with pytest.raises(EvidenceError,match="EVIDENCE_RESERVATION"): + evidence.store_tool_results(*CASE,"unreserved",fresh,result) + + +def test_deleted_index_cannot_reset_ids(tool_case): + budget,request,result=prepare(tool_case) + evidence=EvidenceStore(tool_case[0]) + evidence.store_tool_results(*CASE,"query1",request,result) + tool_case[0].advance(*CASE,State.GENERATING_HYPOTHESES,step_id="deleted-index",artifacts={"evidence_index":None}) + with pytest.raises(EvidenceError,match="EVIDENCE_POINTER_CHANGED"): + evidence.snapshot(*CASE) + + +def test_budget_rejects_wrong_action_evidence_binding(tool_case): + budget,request,result=prepare(tool_case) + saved=EvidenceStore(tool_case[0]).store_tool_results(*CASE,"query1",request,result) + with pytest.raises(BudgetError): + budget.record_evidence(*CASE,"H2","A1",saved.record_refs[0]) + + +def test_limits_and_invalid_operation_ids_do_not_publish(tool_case): + budget,request,result=prepare(tool_case) + before=budget.snapshot(*CASE) + for limits in [EvidenceLimits(max_json_bytes=10),EvidenceLimits(max_total_bytes=10)]: + with pytest.raises(EvidenceError): + EvidenceStore(tool_case[0],limits).store_tool_results(*CASE,"query1",request,result) + for operation in [None,"",True]: + with pytest.raises(EvidenceError): + EvidenceStore(tool_case[0]).store_tool_results(*CASE,operation,request,result) + assert budget.snapshot(*CASE).package==before.package + + +def test_publication_index_crash_replay_does_not_create_new_ids(tool_case,monkeypatch): + budget,request,result=prepare(tool_case) + store=tool_case[0]; evidence=EvidenceStore(store) + original=store._index + def fail(path,manifest): + if manifest["artifacts"].get("evidence_index"): + raise OSError("Simulated crash after archive publication") + original(path,manifest) + monkeypatch.setattr(store,"_index",fail) + with pytest.raises(EvidenceError): evidence.store_tool_results(*CASE,"query1",request,result) + monkeypatch.setattr(store,"_index",original) + restored=evidence.store_tool_results(*CASE,"query1",request,result) + assert restored.reused and len(evidence.snapshot(*CASE).record_refs)==1 + + +def test_confirmed_test_feedback_evidence_does_not_refill_budget(hypotheses): + store,budget,coordinator,before,request,feedback=async_setup(hypotheses) + coordinator.submit(request) + resumed=coordinator.confirm_feedback(feedback) + evidence=EvidenceStore(store) + saved=evidence.store_feedback(*CASE,"test-result",resumed.feedback_ref) + assert budget.snapshot(*CASE).state==before.state + budget.record_evidence(*CASE,"H1","test-action",saved.record_refs[0]) + settled=budget.settle(*CASE,"H1","test-action",semantic_score=50,reason="Useful observation") + assert settled.state["accounts"]["H1"]["remaining"]==76 + assert evidence.snapshot(*CASE).record_refs==saved.record_refs + + +@pytest.mark.parametrize("name",["evidence-content","evidence-record","evidence-index"]) +def test_evidence_schemas_valid(name): + Draft202012Validator.check_schema(json.loads(files("viano_bug_analysis.schemas").joinpath(name+'-v1.schema.json').read_text()))