From 741c1ca4a5033bfc0cf92d7e7cd2b9f609c2fdc2 Mon Sep 17 00:00:00 2001 From: Nitsu45 Date: Tue, 13 Oct 2026 19:43:50 +0200 Subject: [PATCH] Add safe Vcore wowdiag and wowsrc imports with original-data provenance --- README.md | 2 + docs/package-format-v1.md | 6 + docs/vcore-adapters-v1.md | 129 +++++++++++ src/viano_bug_analysis/case/assembler.py | 16 +- .../case/report_collector.py | 7 +- .../case/reproduction_collector.py | 9 +- src/viano_bug_analysis/case/summary.py | 16 ++ src/viano_bug_analysis/package/__init__.py | 3 +- src/viano_bug_analysis/package/factory.py | 66 ++++++ src/viano_bug_analysis/profiles/__init__.py | 1 + .../profiles/vcore/__init__.py | 7 + .../profiles/vcore/archive_reader.py | 160 +++++++++++++ .../profiles/vcore/common.py | 105 +++++++++ .../profiles/vcore/errors.py | 5 + .../profiles/vcore/importer.py | 202 +++++++++++++++++ .../profiles/vcore/wowdiag_adapter.py | 112 +++++++++ .../profiles/vcore/wowsrc_adapter.py | 25 ++ .../schemas/case-v1.schema.json | 40 ++++ .../schemas/report-v1.schema.json | 63 ++++++ .../schemas/reproduction-v1.schema.json | 45 +++- tests/profiles/vcore/conftest.py | 52 +++++ tests/profiles/vcore/test_archive_reader.py | 110 +++++++++ tests/profiles/vcore/test_importer.py | 213 ++++++++++++++++++ tests/profiles/vcore/test_real_package.py | 40 ++++ tests/test_package_factory.py | 44 ++++ 25 files changed, 1469 insertions(+), 9 deletions(-) create mode 100644 docs/vcore-adapters-v1.md create mode 100644 src/viano_bug_analysis/package/factory.py create mode 100644 src/viano_bug_analysis/profiles/__init__.py create mode 100644 src/viano_bug_analysis/profiles/vcore/__init__.py create mode 100644 src/viano_bug_analysis/profiles/vcore/archive_reader.py create mode 100644 src/viano_bug_analysis/profiles/vcore/common.py create mode 100644 src/viano_bug_analysis/profiles/vcore/errors.py create mode 100644 src/viano_bug_analysis/profiles/vcore/importer.py create mode 100644 src/viano_bug_analysis/profiles/vcore/wowdiag_adapter.py create mode 100644 src/viano_bug_analysis/profiles/vcore/wowsrc_adapter.py create mode 100644 tests/profiles/vcore/conftest.py create mode 100644 tests/profiles/vcore/test_archive_reader.py create mode 100644 tests/profiles/vcore/test_importer.py create mode 100644 tests/profiles/vcore/test_real_package.py create mode 100644 tests/test_package_factory.py diff --git a/README.md b/README.md index f31c459..bbddf55 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,8 @@ Job state, checkpoints and inbox admission are documented in [docs/lifecycle-v1.md](docs/lifecycle-v1.md). Generic case construction and its input contracts are documented in [docs/case-builder-v1.md](docs/case-builder-v1.md). +Vcore imports for wowdiag/wowsrc are documented in +[docs/vcore-adapters-v1.md](docs/vcore-adapters-v1.md). Requires Python 3.11 or newer. diff --git a/docs/package-format-v1.md b/docs/package-format-v1.md index 8497157..e198733 100644 --- a/docs/package-format-v1.md +++ b/docs/package-format-v1.md @@ -126,6 +126,12 @@ validation itself streams. No artifact interpretation is performed. ## Writer API and publication contract +`PackageFactory.create(destination, manifest, artifacts)` creates a generation-zero +package from caller-supplied metadata and byte artifacts. It adds integrity records, +applies PackageLimits, validates the staged ZIP, flushes/fsyncs and publishes +atomically without replacing an existing target. It is project-independent; +subsequent enrichment still uses PackageWriter. + ```python from viano_bug_analysis.package import PackageWriter, PackageWriteError diff --git a/docs/vcore-adapters-v1.md b/docs/vcore-adapters-v1.md new file mode 100644 index 0000000..dac9e94 --- /dev/null +++ b/docs/vcore-adapters-v1.md @@ -0,0 +1,129 @@ +# Vcore adapters v1 + +The authoritative architecture is unchanged. All WoW/Vcore mapping is in +`profiles/vcore/`; supplied tool code and package text are data, never executed. +No LLM, diagnosis, interpretation, research, SQL or investigation runs here. + +## Source contracts + +Supports wowdiag bug format 2 (tested 0.10.6) and optional wowsrc bug format 7 +(tested 0.7). Manifests supply identity/version; filenames and wrapper names do +not. A pair must agree on bug ID, core revision, input wowdiag version/format and +the entire embedded wowdiag manifest. This checks metadata consistency, not +authenticity. Missing core revision is reported. Other commands/formats and +malformed known records are rejected; unknown fields/files remain preserved. + +## Archive safety + +ArchiveReader takes an immutable bounded compressed snapshot and checks the +complete gzip stream including CRC/trailer, then reads TAR through a temporary +spool without member extraction. Source manifests are at the root or one wrapper +level, exactly one per archive. Ordinary members outside the wrapper are retained. +Absolute/parent/dot/empty paths, backslashes, colons, controls, duplicate or +case-colliding names, file/directory conflicts, links, sparse files, devices and +special entries are rejected. Member reads verify length. PAX/GNU metadata is +bounded before allocation through a TarInfo subclass tested on Python 3.11. + +ArchiveLimits are configurable operational defaults, not format restrictions: + +| Limit | Default | +|---|---:| +| Compressed archive | 128 MiB | +| Expanded TAR, including padding/headers | 256 MiB | +| Ordinary member | 64 MiB | +| Visible entries | 10,000 | +| Parsed source JSON | 4 MiB | +| Extended metadata header | 64 KiB | +| Expansion ratio | 200:1 | +| Combined import payload | 128 MiB | + +Normalized/materialized bytes are held in memory within the import budget; +bounded snapshots and spool buffers add overhead. PackageLimits and +CaseBuilderLimits apply independently. Inputs must be complete trusted local +files. Original TAR bytes preserve empty directories and unknown metadata. + +## Mapping and provenance + +wowdiag descriptions become supplied observations, expected_behavior stays +supplied expected text, reproduction_notes become comments, and aggregate bug +title/description stay a separate record. Reproductions preserve observations, +the whole steps string as one verbatim element, result and explicit location/focus +fields. Tester_note data_json text retains the original field pointer and decoded +JSON pointer. Server events, snapshots and nested contexts stay raw references. +Explicit typed target/quest/focus values become entities; prose and numeric hits +never generate entities or hypotheses. Duplicate IDs/unresolved links are rejected; +a reproduction can belong through its report when direct bug_id is null. + +Missing expected behavior/steps remain missing. Completeness is not invented. +Timezone-less times remain unchanged, with SOURCE_TIME_UNQUALIFIED; no offset is +borrowed from generated_at. Source authors are preserved. Database/source/runtime +and copied attachment files receive generic roles; unknown files are unclassified. + +wowsrc context remains opaque source/build/custom/change/environment and schema +references. A copied development file requires a safe existing package_path and +matching size; copied=false retains its reason as SOURCE_NOT_COPIED. Absolute +server file paths are metadata, never local I/O targets. Findings, rankings and +location-script status remain source statements without causal interpretation. + +Generic v1 contracts have additive optional origin metadata: raw source_ref, +JSON Pointer, archive_ref/member_name, decoded pointer and unchanged timestamp. +CaseBuilder propagates these into human/fact/entity/reproduction provenance and +checks referenced files exist. Summaries show original source/time. Generic report +data_quality_issues preserve source gaps. These extensions are project-independent; +older v1 documents remain valid. Normalized report-document counts include aggregate +and quality records, so differ from original database report counts. + +## Import API and layout + +```python +from viano_bug_analysis.profiles.vcore import VcoreImporter, AdapterError +try: + VcoreImporter().create( + 'diagnostic.tar.gz', 'incoming.vbcase.zip', wowsrc='context.tar.gz', + package_id='caller-owned-stable-id', case_id='caller-owned-case-id', + ) +except AdapterError as error: + print(error.code, error.path, str(error)) +``` + +Initial identity is supplied explicitly, never derived from filenames or silently +equated with source bug ID. Initial state is NEW, generation/revision zero and +project wow/vcore. Neutral PackageFactory creates a validated ZIP atomically with +no clobber from a caller-supplied manifest/bytes; it has no project semantics. + +Each source snapshot is stored at original/vcore/imports/// with +archive.tar.gz and members/. Both compressed originals and +every ordinary member survive byte for byte. Generic views, input index and +versioned import.json receipt are at case/inputs/vcore//. Receipt records +source versions/hashes, bug ID and issues. Manifest pointers select case_inputs +and vcore_import. The pair hash uses archive SHA-256 values; renaming does not +change it, while repacking is a distinct source snapshot. + +prepare returns an ImportPlan without writing. augment(source, destination, +wowdiag=..., wowsrc=...) uses PackageWriter, preserves old contents and identity, +increments generation and updates manifest indices. Other profiles/bugs, content +collisions and identical repeated imports are rejected (IMPORT_EXISTS: use the +existing package). No automatic reopening occurs; new evidence after completion +requires the explicit lifecycle transition to a new analysis revision. +Existing explicit input roles and index extensions are merged rather than lost. +When the same diagnostic archive is supplemented with wowsrc, unchanged normalized +source rows are reused by archive hash, original member/pointer and equivalent +content. This avoids duplicating human observations; all original artifacts remain. + +## Regression tests + +Portable sanitized synthetic fixtures cover mappings, preservation, malformed +archives, links/path/header/size limits, pair consistency, missing copied files, +times and CaseBuilder roundtrips. No original binaries or credentials are committed. +The complete supplied BUG-20 pair has an opt-in test: + +```powershell +$env:VBA_WOWDIAG_SAMPLE = 'path/to/diagnostic.tar.gz' +$env:VBA_WOWSRC_SAMPLE = 'path/to/source-context.tar.gz' +.\.venv\Scripts\python.exe -m pytest tests/profiles/vcore/test_real_package.py -q +``` + +It checks every raw member byte and unchanged original hashes, three observations, +missing expected behavior, 103 database/93 source references, valid output and +idempotent repeated construction. Without paths it is skipped; portable tests +remain runnable offline. diff --git a/src/viano_bug_analysis/case/assembler.py b/src/viano_bug_analysis/case/assembler.py index 4997749..f8a1f72 100644 --- a/src/viano_bug_analysis/case/assembler.py +++ b/src/viano_bug_analysis/case/assembler.py @@ -150,6 +150,13 @@ class CaseAssembler: add(path, role) reports = [(path, load(path, "report")) for path, role in roles if role == "report"] reproductions = [(path, load(path, "reproduction")) for path, role in roles if role == "reproduction"] + for _, document in reports + reproductions: + nodes = [document] + [node for field in ("observations", "expected_behavior", "comments", "tester_hypotheses", "facts", "entities", "data_quality_issues") for node in document.get(field, [])] + for node in nodes: + if "origin" in node: + for field in ("source_ref", "archive_ref"): + if node["origin"].get(field) is not None: + require_file(reader, node["origin"][field]) for _, document in reproductions: for path in document.get("attachments", []): add(path, "attachment") @@ -174,11 +181,15 @@ class CaseAssembler: identity = (entity["type"], entity_id) record = entities.setdefault(identity, {"type": identity[0], "id": identity[1], "roles": [], "provenance": []}) record["roles"] = sorted(set(record["roles"]) | set(entity.get("roles", []))) - record["provenance"].append(provenance(path, f"/entities/{index}", "report" if document["format"] == "viano-bug-report" else "reproduction_record")) + record["provenance"].append(provenance(path, f"/entities/{index}", "report" if document["format"] == "viano-bug-report" else "reproduction_record", entity)) issues = [] def issue(code, message, refs=None): issues.append({"code": code, "message": message, "refs": refs or []}) + for path, report in reports: + for index, item in enumerate(report.get("data_quality_issues", [])): + source = item.get("origin", {"source_ref": path, "json_pointer": f"/data_quality_issues/{index}"}) + issue(item["code"], item["message"], [{"source_ref": source["source_ref"], "json_pointer": source["json_pointer"]}]) if not index_ref: issue("MISSING_INPUT_INDEX", "No explicit case input index was supplied") @@ -215,7 +226,8 @@ class CaseAssembler: "format": "viano-case", "format_version": 1, "builder_version": "1", "case": {"case_id": manifest["case_id"], "package_id": manifest["package_id"], "analysis_revision": int(manifest["analysis_revision"]), "source_generation": int(manifest["package_generation"]), "project": copy.deepcopy(manifest["project"])}, - "bug": {"reports": [{"source_ref": path, "title": document.get("title"), "summary": document.get("summary")} for path, document in reports]}, + "bug": {"reports": [{"source_ref": path, "title": document.get("title"), "summary": document.get("summary"), + **({"origin": copy.deepcopy(document["origin"])} if "origin" in document else {})} for path, document in reports]}, "human_observations": {"observations": observations, "comments": comments, "tester_hypotheses": hypotheses}, "expected_behavior": expected, "reproduction": {"records": reproduction_records}, diff --git a/src/viano_bug_analysis/case/report_collector.py b/src/viano_bug_analysis/case/report_collector.py index 6585259..d359446 100644 --- a/src/viano_bug_analysis/case/report_collector.py +++ b/src/viano_bug_analysis/case/report_collector.py @@ -3,8 +3,11 @@ import copy def provenance(path: str, pointer: str, kind: str, statement: dict | None = None) -> dict: value = statement or {} - return {"kind": kind, "source_ref": path, "json_pointer": pointer, - "author": value.get("author"), "observed_at": value.get("observed_at")} + result = {"kind": kind, "source_ref": path, "json_pointer": pointer, + "author": value.get("author"), "observed_at": value.get("observed_at")} + if "origin" in value: + result["origin"] = copy.deepcopy(value["origin"]) + return result def collect_statements(documents: list[tuple[str, dict]], field: str, prefix: str) -> list[dict]: diff --git a/src/viano_bug_analysis/case/reproduction_collector.py b/src/viano_bug_analysis/case/reproduction_collector.py index 62e1a52..d55a131 100644 --- a/src/viano_bug_analysis/case/reproduction_collector.py +++ b/src/viano_bug_analysis/case/reproduction_collector.py @@ -8,12 +8,15 @@ from .report_collector import provenance def collect_reproductions(documents: list[tuple[str, dict]], inline_limit: int) -> tuple[list[dict], list[dict]]: records, facts = [], [] for path, document in documents: - records.append({"id": f"R{len(records) + 1:04d}", "source_ref": path, + record = {"id": f"R{len(records) + 1:04d}", "source_ref": path, "steps": list(document.get("steps", [])), "events_ref": document.get("events_ref"), - "attachments": list(document.get("attachments", []))}) + "attachments": list(document.get("attachments", []))} + if "origin" in document: + record["origin"] = copy.deepcopy(document["origin"]) + records.append(record) for index, fact in enumerate(document.get("facts", [])): record = {"id": f"F{len(facts) + 1:04d}", "name": fact["name"], - "provenance": provenance(path, f"/facts/{index}", "reproduction_record")} + "provenance": provenance(path, f"/facts/{index}", "reproduction_record", fact)} if len(json_bytes(fact["value"])) > inline_limit: record["value_ref"] = {"source_ref": path, "json_pointer": f"/facts/{index}/value"} else: diff --git a/src/viano_bug_analysis/case/summary.py b/src/viano_bug_analysis/case/summary.py index fc89676..0a30696 100644 --- a/src/viano_bug_analysis/case/summary.py +++ b/src/viano_bug_analysis/case/summary.py @@ -11,6 +11,17 @@ def _literal(value) -> str: return text.replace("\r", "\\r").replace("\n", "
") +def _origin_lines(origin: dict | None) -> list[str]: + if origin is None: + return [] + result = [f" Original source: {_literal(origin['source_ref'])} {_literal(origin['json_pointer'])}"] + if "decoded_json_pointer" in origin: + result.append(f" Decoded JSON pointer: {_literal(origin['decoded_json_pointer'])}") + if "source_timestamp" in origin: + result.append(f" Source timestamp (unchanged): {_literal(origin['source_timestamp'])}") + return result + + def render_summary(case: dict) -> str: """Pure presentation of the canonical case, with no additional interpretation.""" validate(case, "case") @@ -22,6 +33,7 @@ def render_summary(case: dict) -> str: lines += [f"- Source: {_literal(report['source_ref'])}", f" Title: {_literal(report['title']) if report['title'] is not None else '(not supplied)'}", f" Summary: {_literal(report['summary']) if report['summary'] is not None else '(not supplied)'}"] + lines.extend(_origin_lines(report.get("origin"))) for label, records in (("Direct human observations", case["human_observations"]["observations"]), ("Tester comments", case["human_observations"]["comments"]), ("Tester hypotheses (supplied)", case["human_observations"]["tester_hypotheses"]), @@ -33,11 +45,13 @@ def render_summary(case: dict) -> str: f" Source: {_literal(source['source_ref'])} {_literal(source['json_pointer'])}", f" Author: {_literal(source['author']) if source['author'] is not None else '(not supplied)'}", f" Observed at: {_literal(source['observed_at']) if source['observed_at'] is not None else '(not supplied)'}"] + lines.extend(_origin_lines(source.get("origin"))) if "claim" in record: lines.append(f" Claim: {_literal(record['claim']['key'])} = {_literal(json.dumps(record['claim']['value'], sort_keys=True, ensure_ascii=False))}") lines += ["", f"Expected-behavior completeness: {case['expected_behavior']['completeness']}", "", "## Reproductions", ""] for record in case["reproduction"]["records"]: lines.append(f"- {_literal(record['id'])}: {_literal(record['source_ref'])}") + lines.extend(_origin_lines(record.get("origin"))) for step in record["steps"]: lines.append(f" Step: {_literal(step)}") if record["events_ref"]: @@ -49,6 +63,7 @@ def render_summary(case: dict) -> str: lines.append(f"- {_literal(entity['type'])} / {_literal(entity['id'])}; roles: {_literal(', '.join(entity['roles']))}") for source in entity["provenance"]: lines.append(f" Source: {_literal(source['source_ref'])} {_literal(source['json_pointer'])}") + lines.extend(_origin_lines(source.get("origin"))) lines += ["", "## Facts supplied by reproduction records", ""] for fact in case["runtime_facts"]["facts"]: source = fact["provenance"] @@ -59,6 +74,7 @@ def render_summary(case: dict) -> str: value = _literal(json.dumps(fact['value'], sort_keys=True, ensure_ascii=False)) lines.append(f"- {_literal(fact['id'])}: {_literal(fact['name'])} = {value}") lines.append(f" Source: {_literal(source['source_ref'])} {_literal(source['json_pointer'])}") + lines.extend(_origin_lines(source.get("origin"))) for label, refs in (("Runtime data", case["runtime_facts"]["raw_refs"]), ("Database context", case["database_context"]["refs"]), ("Source context", case["source_context"]["refs"]), ("Supplied research", case["research"]["refs"]), ("Prior cases", case["inputs"]["prior_cases"]), ("Unclassified inputs", case["inputs"]["unclassified"])): diff --git a/src/viano_bug_analysis/package/__init__.py b/src/viano_bug_analysis/package/__init__.py index 183be2a..e7e622c 100644 --- a/src/viano_bug_analysis/package/__init__.py +++ b/src/viano_bug_analysis/package/__init__.py @@ -3,6 +3,7 @@ from .limits import PackageLimits from .reader import PackageReader from .validator import PackageValidator from .writer import PackageWriter +from .factory import PackageFactory __all__ = ["PackageReader", "PackageValidator", "PackageLimits", "PackageReadError", - "ValidationIssue", "ValidationResult", "PackageWriter", "PackageWriteError"] + "ValidationIssue", "ValidationResult", "PackageWriter", "PackageWriteError", "PackageFactory"] diff --git a/src/viano_bug_analysis/package/factory.py b/src/viano_bug_analysis/package/factory.py new file mode 100644 index 0000000..064f1c9 --- /dev/null +++ b/src/viano_bug_analysis/package/factory.py @@ -0,0 +1,66 @@ +import copy +import hashlib +import json +import os +import tempfile +import warnings +import zipfile +from pathlib import Path + +from .errors import PackageWriteError +from .manifest import safe_path +from .validator import ARCHIVE_ERRORS, PackageValidator + + +class PackageFactory: + """Create an initial v1 package from caller-owned identity and supplied bytes.""" + + def __init__(self, limits=None): + self.validator = PackageValidator(limits) + + def create(self, destination, manifest: dict, artifacts: dict[str, bytes]): + temporary = None + published = False + target = Path(destination) + try: + if target.exists() or target.is_symlink(): + raise PackageWriteError(self.validator._failure("TARGET_EXISTS", "Initial package target already exists")) + document = copy.deepcopy(manifest) + if document.get("package_generation") != 0: + raise PackageWriteError(self.validator._failure("INITIAL_GENERATION", "Initial generation must be zero")) + for path, data in artifacts.items(): + if not safe_path(path) or path == "manifest.json" or not isinstance(data, bytes): + raise PackageWriteError(self.validator._failure("ARTIFACT_INVALID", "Unsafe path or non-byte artifact", path)) + if len(data) > self.validator.limits.max_file_bytes: + raise PackageWriteError(self.validator._failure("FILE_LIMIT", "Initial artifact exceeds configured limit", path)) + document.setdefault("integrity", {}).setdefault(path, {}).update(size=len(data), sha256=hashlib.sha256(data).hexdigest()) + data = json.dumps(document, sort_keys=True, ensure_ascii=False, allow_nan=False).encode("utf-8") + if len(data) > min(self.validator.limits.max_manifest_bytes, self.validator.limits.max_file_bytes): + raise PackageWriteError(self.validator._failure("FILE_LIMIT", "Initial manifest exceeds configured limit")) + if len(artifacts) + 1 > self.validator.limits.max_entries or sum(map(len, artifacts.values())) + len(data) > min(self.validator.limits.max_total_bytes, self.validator.limits.max_archive_bytes): + raise PackageWriteError(self.validator._failure("PACKAGE_LIMIT", "Initial package exceeds configured limits")) + with tempfile.NamedTemporaryFile(dir=target.parent, prefix=".vbcase-", suffix=".tmp", delete=False) as stream: + temporary = Path(stream.name) + with zipfile.ZipFile(stream, "w", compression=zipfile.ZIP_STORED) as archive: + for path, payload in artifacts.items(): + archive.writestr(path, payload) + archive.writestr("manifest.json", data) + stream.flush() + os.fsync(stream.fileno()) + result = self.validator.validate(temporary) + if not result.valid: + raise PackageWriteError(result) + os.link(temporary, target) + published = True + return result + except ARCHIVE_ERRORS + (TypeError, KeyError, RecursionError, AttributeError) as error: + raise PackageWriteError(self.validator._failure("CREATE_ERROR", str(error))) from error + finally: + if temporary is not None: + try: + temporary.unlink(missing_ok=True) + except OSError as error: + if published: + warnings.warn(f"Package created but staging cleanup failed: {error}", ResourceWarning) + else: + raise PackageWriteError(self.validator._failure("CLEANUP_ERROR", str(error))) from error diff --git a/src/viano_bug_analysis/profiles/__init__.py b/src/viano_bug_analysis/profiles/__init__.py new file mode 100644 index 0000000..f821c5e --- /dev/null +++ b/src/viano_bug_analysis/profiles/__init__.py @@ -0,0 +1 @@ +"""Project-specific adapters, separate from deterministic package/case foundations.""" diff --git a/src/viano_bug_analysis/profiles/vcore/__init__.py b/src/viano_bug_analysis/profiles/vcore/__init__.py new file mode 100644 index 0000000..39651ff --- /dev/null +++ b/src/viano_bug_analysis/profiles/vcore/__init__.py @@ -0,0 +1,7 @@ +from .archive_reader import ArchiveLimits, ArchiveReader +from .errors import AdapterError +from .importer import VcoreImporter, ImportPlan +from .wowdiag_adapter import WowdiagAdapter +from .wowsrc_adapter import WowsrcAdapter + +__all__ = ["ArchiveLimits", "ArchiveReader", "AdapterError", "VcoreImporter", "ImportPlan", "WowdiagAdapter", "WowsrcAdapter"] diff --git a/src/viano_bug_analysis/profiles/vcore/archive_reader.py b/src/viano_bug_analysis/profiles/vcore/archive_reader.py new file mode 100644 index 0000000..2bf2251 --- /dev/null +++ b/src/viano_bug_analysis/profiles/vcore/archive_reader.py @@ -0,0 +1,160 @@ +import gzip +import io +import os +import tarfile +import tempfile +import zlib +from dataclasses import dataclass + +from viano_bug_analysis.package.manifest import parse_manifest, safe_path +from viano_bug_analysis.package.validator import CHUNK_SIZE + +from .errors import AdapterError + + +@dataclass(frozen=True) +class ArchiveLimits: + max_archive_bytes: int = 128 * 1024**2 + max_tar_bytes: int = 256 * 1024**2 + max_file_bytes: int = 64 * 1024**2 + max_entries: int = 10_000 + max_json_bytes: int = 4 * 1024**2 + max_header_bytes: int = 64 * 1024 + max_ratio: int = 200 + max_import_bytes: int = 128 * 1024**2 + + def __post_init__(self): + if any(isinstance(value, bool) or not isinstance(value, int) or value <= 0 for value in vars(self).values()): + raise ValueError("Archive limits must be positive integers") + + +class ArchiveReader: + """Validate gzip fully and read TAR members without filesystem extraction.""" + + def __init__(self, path, limits: ArchiveLimits | None = None): + self.path = path + self.limits = limits or ArchiveLimits() + self._original = self._spool = self._tar = None + self._raw = None + self.members = {} + + def __enter__(self): + try: + self._original = open(self.path, "rb") + archive_size = os.fstat(self._original.fileno()).st_size + if archive_size > self.limits.max_archive_bytes: + raise AdapterError("ARCHIVE_LIMIT", "Compressed source exceeds configured limit") + self._raw = self._original.read(self.limits.max_archive_bytes + 1) + if len(self._raw) > self.limits.max_archive_bytes: + raise AdapterError("ARCHIVE_LIMIT", "Source archive changed or exceeds configured limit") + archive_size = len(self._raw) + self._spool = tempfile.SpooledTemporaryFile(max_size=4 * 1024**2) + size = 0 + with gzip.GzipFile(fileobj=io.BytesIO(self._raw)) as compressed: + while chunk := compressed.read(CHUNK_SIZE): + size += len(chunk) + if size > self.limits.max_tar_bytes or size > max(archive_size, 1) * self.limits.max_ratio: + raise AdapterError("EXPANSION_LIMIT", "TAR expansion exceeds configured policy") + self._spool.write(chunk) + self._spool.seek(0) + header_limit = self.limits.max_header_bytes + + class BoundedTarInfo(tarfile.TarInfo): + def _proc_member(self, archive): + metadata_types = (tarfile.XHDTYPE, tarfile.XGLTYPE, tarfile.SOLARIS_XHDTYPE, tarfile.GNUTYPE_LONGNAME, tarfile.GNUTYPE_LONGLINK) + if self.type not in metadata_types and (self.type == tarfile.GNUTYPE_SPARSE or not (self.isfile() or self.isdir())): + raise AdapterError("SPECIAL_MEMBER", "Special TAR member rejected before extended processing", self.name) + return super()._proc_member(archive) + + def _proc_pax(self, *args, **kwargs): + if self.size < 0 or self.size > header_limit: + raise AdapterError("HEADER_LIMIT", "PAX metadata exceeds configured limit") + return super()._proc_pax(*args, **kwargs) + + def _proc_gnulong(self, *args, **kwargs): + if self.size < 0 or self.size > header_limit: + raise AdapterError("HEADER_LIMIT", "GNU metadata exceeds configured limit") + return super()._proc_gnulong(*args, **kwargs) + + self._tar = tarfile.open(fileobj=self._spool, mode="r:", tarinfo=BoundedTarInfo) + names = {} + for member in self._tar: + if len(names) >= self.limits.max_entries: + raise AdapterError("ENTRY_LIMIT", "Too many TAR entries") + name = member.name + if not safe_path(name, directory=member.isdir()): + raise AdapterError("UNSAFE_PATH", "Unsafe TAR member path", name) + if not (member.isfile() or member.isdir()) or member.issparse(): + raise AdapterError("SPECIAL_MEMBER", "Links, sparse files and special TAR entries are unsupported", name) + if member.size < 0 or member.size > self.limits.max_file_bytes or (member.isdir() and member.size): + raise AdapterError("FILE_LIMIT", "TAR member exceeds configured limit", name) + normalized = name.rstrip("/").casefold() + if normalized in names: + raise AdapterError("MEMBER_COLLISION", "Duplicate or case-colliding TAR entry", name) + names[normalized] = member.isdir() + if member.isfile(): + self.members[name] = member + for name in names: + parts = name.split("/") + if any(names.get("/".join(parts[:index])) is False for index in range(1, len(parts))): + raise AdapterError("MEMBER_COLLISION", "A TAR file is also used as a directory", name) + candidates = [name for name in self.members if name.split("/")[-1] == "manifest.json" and len(name.split("/")) <= 2] + if len(candidates) != 1: + raise AdapterError("MANIFEST_AMBIGUOUS", "Expected exactly one package-level source manifest") + self.manifest_member = candidates[0] + self.root = self.manifest_member.removesuffix("manifest.json") + self.manifest = self.read_json("manifest.json") + if not isinstance(self.manifest, dict): + raise AdapterError("SOURCE_MANIFEST", "Source manifest must be an object") + return self + except AdapterError: + self.close() + raise + except (OSError, EOFError, tarfile.TarError, zlib.error, ValueError, RecursionError) as error: + self.close() + raise AdapterError("ARCHIVE_INVALID", str(error)) from error + + def member_name(self, relative: str) -> str: + if not safe_path(relative) or self.root + relative not in self.members: + raise AdapterError("SOURCE_REFERENCE", "Referenced source member is absent or unsafe", relative) + return self.root + relative + + def read_member(self, name: str, limit: int | None = None) -> bytes: + if self._tar is None: + raise RuntimeError("Use ArchiveReader in a with block") + if name not in self.members: + raise AdapterError("SOURCE_REFERENCE", "TAR member not found", name) + limit = min(limit or self.limits.max_file_bytes, self.limits.max_file_bytes) + if self.members[name].size > limit: + raise AdapterError("FILE_LIMIT", "Source member exceeds configured read limit", name) + try: + with self._tar.extractfile(self.members[name]) as stream: + data = stream.read(limit + 1) + if len(data) != self.members[name].size or len(data) > limit: + raise AdapterError("MEMBER_TRUNCATED", "TAR member length differs from header", name) + return data + except (OSError, EOFError, tarfile.TarError, ValueError) as error: + raise AdapterError("ARCHIVE_INVALID", str(error), name) from error + + def read_json(self, relative: str): + name = self.member_name(relative) + try: + return parse_manifest(self.read_member(name, self.limits.max_json_bytes)) + except (ValueError, RecursionError) as error: + raise AdapterError("SOURCE_JSON", str(error), name) from error + + def original_bytes(self) -> bytes: + if self._raw is None: + raise RuntimeError("ArchiveReader is closed") + return self._raw + + def close(self): + for stream in (self._tar, self._spool, self._original): + if stream is not None: + stream.close() + self._tar = self._spool = self._original = None + self._raw = None + self.members = {} + + def __exit__(self, *args): + self.close() diff --git a/src/viano_bug_analysis/profiles/vcore/common.py b/src/viano_bug_analysis/profiles/vcore/common.py new file mode 100644 index 0000000..6e76958 --- /dev/null +++ b/src/viano_bug_analysis/profiles/vcore/common.py @@ -0,0 +1,105 @@ +from dataclasses import dataclass, field + +from viano_bug_analysis.lifecycle.storage import json_bytes +from viano_bug_analysis.package.manifest import FORMAT_CHECKER + +from .errors import AdapterError + + +@dataclass +class AdaptedInputs: + additions: dict[str, bytes] = field(default_factory=dict) + reports: list[str] = field(default_factory=list) + reproductions: list[str] = field(default_factory=list) + attachments: list[str] = field(default_factory=list) + context: dict = field(default_factory=lambda: {key: [] for key in ("runtime", "database", "source", "research")}) + issues: list[dict] = field(default_factory=list) + + +def check_manifest(reader, tool: str, version: int): + manifest = reader.manifest + if manifest.get("tool") != tool or type(manifest.get("format_version")) is not int or manifest["format_version"] != version or manifest.get("command") != "bug": + raise AdapterError("SOURCE_FORMAT", f"Expected {tool} bug package format {version}") + if type(manifest.get("bug_id")) is not int or manifest["bug_id"] < 0 or not isinstance(manifest.get("tool_version"), str) or not manifest["tool_version"].strip(): + raise AdapterError("SOURCE_MANIFEST", "Source requires bug_id and tool_version") + if not isinstance(manifest.get("warnings", []), list) or any(not isinstance(value, str) for value in manifest.get("warnings", [])): + raise AdapterError("SOURCE_MANIFEST", "Source warnings must be a list of strings") + if manifest.get("core_revision") is not None and not isinstance(manifest["core_revision"], str): + raise AdapterError("SOURCE_MANIFEST", "core_revision must be text or null") + + +def payload_ref(reader, prefix: str, relative: str) -> str: + return prefix + "/members/" + reader.member_name(relative) + + +def origin(reader, prefix, relative, pointer, timestamp=None, decoded=None): + result = {"source_ref": payload_ref(reader, prefix, relative), "json_pointer": pointer, + "archive_ref": prefix + "/archive.tar.gz", "member_name": reader.member_name(relative)} + if timestamp is not None: + result["source_timestamp"] = timestamp + if decoded is not None: + result["decoded_json_pointer"] = decoded + return result + + +def materialize(reader, prefix, role_for) -> AdaptedInputs: + result = AdaptedInputs() + original = reader.original_bytes() + result.additions[prefix + "/archive.tar.gz"] = original + total = len(original) + for name in sorted(reader.members): + if total + reader.members[name].size > reader.limits.max_import_bytes: + raise AdapterError("IMPORT_LIMIT", "Materialized source exceeds configured memory/import budget") + data = reader.read_member(name) + total += len(data) + if total > reader.limits.max_import_bytes: + raise AdapterError("IMPORT_LIMIT", "Materialized source exceeds configured memory/import budget") + path = prefix + "/members/" + name + result.additions[path] = data + logical = name.removeprefix(reader.root) if name.startswith(reader.root) else None + role = role_for(logical) if logical is not None else None + if role == "attachment": + result.attachments.append(path) + elif role is not None: + result.context[role].append(path) + return result + + +def rows(reader, relative): + value = reader.read_json(relative) + if not isinstance(value, list) or any(not isinstance(row, dict) for row in value): + raise AdapterError("SOURCE_ROWS", "Expected a JSON array of objects", relative) + return value + + +def statement(text, reader, prefix, relative, pointer, author=None, timestamp=None, decoded=None, issues=None): + if text is None or text == "": + return None + if not isinstance(text, str): + raise AdapterError("SOURCE_FIELD", "Expected textual observation", relative + pointer) + if not text.strip(): + return None + result = {"text": text, "origin": origin(reader, prefix, relative, pointer, timestamp, decoded)} + if author is not None: + if not isinstance(author, str): + raise AdapterError("SOURCE_FIELD", "Expected textual author") + if author.strip(): + result["author"] = author + if timestamp is not None: + if not isinstance(timestamp, str): + raise AdapterError("SOURCE_FIELD", "Expected timestamp text") + if FORMAT_CHECKER.conforms(timestamp, "date-time"): + result["observed_at"] = timestamp + elif issues is not None: + issues.append({"code": "SOURCE_TIME_UNQUALIFIED", "message": "Source time has no usable timezone; retained without conversion", "origin": result["origin"]}) + return result + + +def write_document(result, path, document, contract): + from viano_bug_analysis.case.contracts import validate + from viano_bug_analysis.case.errors import CaseBuildError + try: + validate(document, contract, path) + result.additions[path] = json_bytes(document) + except (CaseBuildError, ValueError, TypeError) as error: + raise AdapterError("NORMALIZATION_ERROR", str(error), path) from error diff --git a/src/viano_bug_analysis/profiles/vcore/errors.py b/src/viano_bug_analysis/profiles/vcore/errors.py new file mode 100644 index 0000000..a86e030 --- /dev/null +++ b/src/viano_bug_analysis/profiles/vcore/errors.py @@ -0,0 +1,5 @@ +class AdapterError(Exception): + def __init__(self, code: str, message: str, path: str | None = None): + self.code = code + self.path = path + super().__init__(message) diff --git a/src/viano_bug_analysis/profiles/vcore/importer.py b/src/viano_bug_analysis/profiles/vcore/importer.py new file mode 100644 index 0000000..48f1020 --- /dev/null +++ b/src/viano_bug_analysis/profiles/vcore/importer.py @@ -0,0 +1,202 @@ +import hashlib +from contextlib import ExitStack +from dataclasses import dataclass, replace +from pathlib import Path + +from viano_bug_analysis.case.contracts import validate +from viano_bug_analysis.case.errors import CaseBuildError +from viano_bug_analysis.lifecycle.job import timestamp +from viano_bug_analysis.lifecycle.storage import json_bytes +from viano_bug_analysis.package import PackageReader, PackageReadError, PackageWriter, PackageWriteError +from viano_bug_analysis.package.factory import PackageFactory +from viano_bug_analysis.package.manifest import parse_manifest + +from .archive_reader import ArchiveLimits, ArchiveReader +from .common import check_manifest, write_document +from .errors import AdapterError +from .wowdiag_adapter import WowdiagAdapter +from .wowsrc_adapter import WowsrcAdapter + + +@dataclass(frozen=True) +class ImportPlan: + additions: dict[str, bytes] + index_ref: str + receipt_ref: str + bug_id: int + reproductions_count: int + + +class VcoreImporter: + def __init__(self, archive_limits: ArchiveLimits | None = None, package_limits=None): + self.archive_limits = archive_limits or ArchiveLimits() + self.package_limits = package_limits + + def _package_json(self, reader, path): + with reader._archive.open(path) as stream: + data = reader.validator._read_bounded(stream, min(self.archive_limits.max_json_bytes, reader.validator.limits.max_file_bytes)) + return parse_manifest(data) + + def _merge_inputs(self, reader, plan): + old_ref = reader.manifest["artifacts"].get("case_inputs") + if not old_ref: + return dict(plan.additions) + old = self._package_json(reader, old_ref) + new = parse_manifest(plan.additions[plan.index_ref]) + validate(old, "case-inputs") + if any(len(old.get(field, [])) != len(set(old.get(field, []))) for field in ("reports", "reproductions", "attachments")): + raise AdapterError("SOURCE_INDEX", "Existing input index repeats source documents") + old_documents = set(old.get("reports", []) + old.get("reproductions", []) + [old_ref]) + if any(path not in reader.files for path in old_documents): + raise AdapterError("SOURCE_REFERENCE", "Existing input index has unresolved documents") + if sum(reader._archive.getinfo(path).file_size for path in old_documents) + sum(map(len, plan.additions.values())) > self.archive_limits.max_import_bytes: + raise AdapterError("IMPORT_LIMIT", "Existing structured views plus new import exceed configured budget") + # Reuse the same normalized source row when supplementing a diag-only + # import with wowsrc. No unrelated existing input roles are dropped. + def canonical(value): + if isinstance(value, dict): + result = {} + for key, child in value.items(): + if key == "value": + result[key] = child # arbitrary fact/claim JSON is not metadata + elif key == "origin" and isinstance(child, dict): + result[key] = {k: v for k, v in child.items() if k not in {"source_ref", "archive_ref"}} + elif key == "events_ref" and isinstance(child, str): + result[key] = child.split("/members/", 1)[-1] + else: + result[key] = canonical(child) + return result + if isinstance(value, list): + return [canonical(child) for child in value] + return value + + def signature(document, raw_hash): + source = document.get("origin") + if not isinstance(source, dict) or not raw_hash: + return None + return raw_hash, source.get("member_name"), source.get("json_pointer"), json_bytes(canonical(document)) + + for field, contract in (("reports", "report"), ("reproductions", "reproduction")): + known = {} + for path in old.get(field, []): + document = self._package_json(reader, path) + validate(document, contract) + archive = document.get("origin", {}).get("archive_ref") + hash_ = reader.manifest["integrity"].get(archive, {}).get("sha256") + token = signature(document, hash_) + if token is not None: + known[token] = path + selected = [] + for path in new.get(field, []): + document = parse_manifest(plan.additions[path]) + archive = document.get("origin", {}).get("archive_ref") + hash_ = hashlib.sha256(plan.additions[archive]).hexdigest() if archive in plan.additions else None + selected.append(known.get(signature(document, hash_), path)) + new[field] = list(dict.fromkeys(old.get(field, []) + selected)) + new["attachments"] = list(dict.fromkeys(old.get("attachments", []) + new.get("attachments", []))) + context = dict(old.get("context", {})) + for role, paths in new.get("context", {}).items(): + context[role] = list(dict.fromkeys(context.get(role, []) + paths)) + new["context"] = context + merged = old | new + validate(merged, "case-inputs") + return dict(plan.additions) | {plan.index_ref: json_bytes(merged)} + + def prepare(self, wowdiag, wowsrc=None) -> ImportPlan: + try: + if sum(Path(path).stat().st_size for path in (wowdiag, wowsrc) if path is not None) > self.archive_limits.max_import_bytes: + raise AdapterError("IMPORT_LIMIT", "Combined original archives exceed configured budget") + with ExitStack() as stack: + diag = stack.enter_context(ArchiveReader(wowdiag, self.archive_limits)) + check_manifest(diag, "wowdiag", 2) + src = stack.enter_context(ArchiveReader(wowsrc, self.archive_limits)) if wowsrc is not None else None + if src: + check_manifest(src, "wowsrc", 7) + embedded = src.read_json("input/wowdiag_manifest.json") + if src.manifest["bug_id"] != diag.manifest["bug_id"] or src.manifest.get("core_revision") != diag.manifest.get("core_revision") or json_bytes(embedded) != json_bytes(diag.manifest) or src.manifest.get("input_wowdiag_format") != 2 or src.manifest.get("input_wowdiag_version") != diag.manifest["tool_version"]: + raise AdapterError("SOURCE_PAIR_MISMATCH", "wowsrc does not describe the supplied wowdiag package") + hashes = [hashlib.sha256(reader.original_bytes()).hexdigest() for reader in (diag, src) if reader is not None] + import_id = hashlib.sha256(json_bytes(hashes)).hexdigest() + raw_prefix = f"original/vcore/imports/{import_id}" + normalized_prefix = f"case/inputs/vcore/{import_id}" + inputs = WowdiagAdapter().adapt(diag, raw_prefix + "/wowdiag", normalized_prefix + "/wowdiag") + if not diag.manifest.get("core_revision"): + inputs.issues.append({"code": "CORE_REVISION_MISSING", "message": "No core revision was supplied; matching revisions cannot be verified"}) + if src: + remaining = self.archive_limits.max_import_bytes - sum(map(len, inputs.additions.values())) + if remaining <= 0: + raise AdapterError("IMPORT_LIMIT", "No remaining source import budget") + src.limits = replace(src.limits, max_import_bytes=remaining) + sources = WowsrcAdapter().adapt(src, raw_prefix + "/wowsrc") + inputs.additions.update(sources.additions) + for role, refs in sources.context.items(): + inputs.context[role].extend(refs) + inputs.issues.extend(sources.issues) + if sum(map(len, inputs.additions.values())) > self.archive_limits.max_import_bytes: + raise AdapterError("IMPORT_LIMIT", "Combined imported sources exceed configured budget") + quality = {"format": "viano-bug-report", "format_version": 1, + "data_quality_issues": [{"code": issue["code"], "message": issue["message"], **({"origin": issue["origin"]} if "origin" in issue else {})} for issue in inputs.issues]} + quality_ref = normalized_prefix + "/source-quality.json" + write_document(inputs, quality_ref, quality, "report") + inputs.reports.append(quality_ref) + receipt_ref = normalized_prefix + "/import.json" + receipt = {"format": "viano-vcore-import", "format_version": 1, "adapter_version": "1", "bug_id": diag.manifest["bug_id"], + "sources": [{"tool": reader.manifest["tool"], "tool_version": reader.manifest["tool_version"], + "format_version": reader.manifest["format_version"], "sha256": hash_, + "archive_ref": raw_prefix + "/" + reader.manifest["tool"] + "/archive.tar.gz"} + for reader, hash_ in zip((diag, src), hashes)], "issues": inputs.issues} + inputs.additions[receipt_ref] = json_bytes(receipt) + inputs.context["runtime"].append(receipt_ref) + index_ref = normalized_prefix + "/case-inputs.json" + index = {"format": "viano-case-inputs", "format_version": 1, "reports": inputs.reports, + "reproductions": inputs.reproductions, "context": inputs.context, "attachments": inputs.attachments} + validate(index, "case-inputs") + inputs.additions[index_ref] = json_bytes(index) + if sum(map(len, inputs.additions.values())) > self.archive_limits.max_import_bytes: + raise AdapterError("IMPORT_LIMIT", "Normalized import exceeds configured budget") + return ImportPlan(inputs.additions, index_ref, receipt_ref, diag.manifest["bug_id"], len(inputs.reproductions)) + except (ValueError, TypeError, OSError, CaseBuildError) as error: + raise AdapterError("IMPORT_ERROR", str(error)) from error + + def create(self, wowdiag, destination, *, package_id: str, case_id: str, wowsrc=None): + plan = self.prepare(wowdiag, wowsrc) + created = timestamp() + manifest = {"format": "viano-bug-case-package", "format_version": 1, "package_id": package_id, "package_generation": 0, + "case_id": case_id, "project": {"type": "wow", "profile": "vcore"}, "created_at": created, "updated_at": created, + "state": "NEW", "routing": {"destination": "inbox", "reason": None, "retry_count": 0, "retry_not_before": None}, + "analysis_revision": 0, "artifacts": {"case": None, "diagnosis": None, "change_request": None, + "case_inputs": plan.index_ref, "vcore_import": plan.receipt_ref}, + "history": {"analysis_revisions": 0, "reproductions": plan.reproductions_count, "evidence": 0, "handoffs": 0, "feedback": 0, "events_ref": None}, "integrity": {}} + try: + return PackageFactory(self.package_limits).create(destination, manifest, plan.additions) + except PackageWriteError as error: + raise AdapterError("PACKAGE_ERROR", str(error)) from error + + def augment(self, source, destination, *, wowdiag, wowsrc=None): + plan = self.prepare(wowdiag, wowsrc) + try: + with PackageReader(source, self.package_limits) as reader: + manifest = reader.manifest + if manifest["project"]["type"] != "wow" or manifest["project"]["profile"] != "vcore": + raise AdapterError("PROJECT_MISMATCH", "Vcore inputs require a wow/vcore case") + old_receipt = manifest["artifacts"].get("vcore_import") + if old_receipt: + previous = self._package_json(reader, old_receipt) + if not isinstance(previous, dict) or previous.get("format") != "viano-vcore-import" or type(previous.get("format_version")) is not int or previous["format_version"] != 1: + raise AdapterError("IMPORT_RECEIPT", "Unsupported or malformed previous import receipt", old_receipt) + if previous.get("bug_id") != plan.bug_id: + raise AdapterError("BUG_ID_MISMATCH", "Cannot attach a different source bug to this case") + additions = self._merge_inputs(reader, plan) + for name, data in additions.items(): + if name in reader.files and reader.read_bytes(name) != data: + raise AdapterError("IMPORT_COLLISION", "An existing import artifact has different content", name) + if manifest["artifacts"].get("vcore_import") == plan.receipt_ref and manifest["artifacts"].get("case_inputs") == plan.index_ref: + raise AdapterError("IMPORT_EXISTS", "Identical source archives are already selected; use the existing package") + changes = {name: data for name, data in additions.items() if name not in reader.files} + return PackageWriter(self.package_limits).write(source, destination, additions=changes, + manifest_updates={"artifacts": {"case_inputs": plan.index_ref, "vcore_import": plan.receipt_ref}, + "history": {"reproductions": max(manifest["history"]["reproductions"], plan.reproductions_count)}}) + except (PackageReadError, PackageWriteError) as error: + raise AdapterError("PACKAGE_ERROR", str(error)) from error + except (OSError, ValueError, TypeError, KeyError, RecursionError, CaseBuildError) as error: + raise AdapterError("IMPORT_ERROR", str(error)) from error diff --git a/src/viano_bug_analysis/profiles/vcore/wowdiag_adapter.py b/src/viano_bug_analysis/profiles/vcore/wowdiag_adapter.py new file mode 100644 index 0000000..e968186 --- /dev/null +++ b/src/viano_bug_analysis/profiles/vcore/wowdiag_adapter.py @@ -0,0 +1,112 @@ +from viano_bug_analysis.package.manifest import parse_manifest + +from .common import check_manifest, materialize, origin, payload_ref, rows, statement, write_document +from .errors import AdapterError + + +class WowdiagAdapter: + def adapt(self, reader, prefix, normalized_prefix=None): + check_manifest(reader, "wowdiag", 2) + role_for = lambda path: "attachment" if path.startswith("attachments/") else "database" if path.startswith("database/") else "source" if path.startswith("source/") else "runtime" if path.startswith(("bug/", "environment/", "findings/")) or path in {"manifest.json", "findings.json"} else None + result = materialize(reader, prefix, role_for) + normalized_prefix = normalized_prefix or prefix + "/normalized" + bug = reader.read_json("bug/bug.json") + if not isinstance(bug, dict) or type(bug.get("bug_id")) is not int or bug["bug_id"] != reader.manifest["bug_id"]: + raise AdapterError("BUG_ID_MISMATCH", "Bug record disagrees with source manifest") + reports = rows(reader, "bug/reports.json") + reproductions = rows(reader, "bug/reproductions.json") + events = rows(reader, "bug/reproduction_events.json") + if not reports: + result.issues.append({"code": "SOURCE_REPORTS_MISSING", "message": "wowdiag supplied no individual bug reports"}) + for relative, records, id_field in (("bug/reports.json", reports, "report_id"), ("bug/reproductions.json", reproductions, "reproduction_id"), ("bug/reproduction_events.json", events, "event_id")): + identities = set() + for row in records: + identity = row.get(id_field) + if type(identity) is not int or identity < 0 or identity in identities: + raise AdapterError("SOURCE_ID", "Missing, invalid or duplicate source row ID", relative) + identities.add(identity) + if row.get("bug_id") is not None and (type(row["bug_id"]) is not int or row["bug_id"] != reader.manifest["bug_id"]): + raise AdapterError("BUG_ID_MISMATCH", "Source row belongs to another bug", relative) + reproduction_ids = {row["reproduction_id"] for row in reproductions} + report_ids = {row["report_id"] for row in reports} + if any(event.get("reproduction_id") not in reproduction_ids for event in events) or any(row.get("report_id") is not None and row["report_id"] not in report_ids for row in reproductions): + raise AdapterError("SOURCE_RELATION", "Source rows have unresolved reproduction/report links") + if any(row.get("bug_id") is None and row.get("report_id") not in report_ids for row in reproductions): + raise AdapterError("SOURCE_RELATION", "Reproduction needs a direct bug or linked report association") + for index, row in enumerate(reports): + normalized = {"format": "viano-bug-report", "format_version": 1, + "observations": [], "expected_behavior": [], "comments": [], "tester_hypotheses": [], + "origin": origin(reader, prefix, "bug/reports.json", f"/{index}"), "entities": []} + title = row.get("title") + if isinstance(title, str) and title.strip(): + normalized["title"] = title + for field, destination in (("description", "observations"), ("expected_behavior", "expected_behavior")): + value = statement(row.get(field), reader, prefix, "bug/reports.json", f"/{index}/{field}", row.get("character_name"), row.get("created_at"), issues=result.issues) + if value: + normalized[destination].append(value) + # Free-form reproduction notes are supplied prose, not inferred steps. + note = statement(row.get("reproduction_notes"), reader, prefix, "bug/reports.json", f"/{index}/reproduction_notes", row.get("character_name"), issues=result.issues) + if note: + normalized["comments"].append(note) + self._entity(normalized, row.get("target_type"), row.get("target_entry"), "reported_target", origin(reader, prefix, "bug/reports.json", f"/{index}/target_entry")) + self._entity(normalized, "quest", row.get("quest_id"), "explicit_quest", origin(reader, prefix, "bug/reports.json", f"/{index}/quest_id")) + self._entity(normalized, reader.manifest.get("diagnostic_focus_type"), reader.manifest.get("diagnostic_focus_id"), "diagnostic_focus", origin(reader, prefix, "manifest.json", "/diagnostic_focus_id")) + path = normalized_prefix + f"/report-{row['report_id']}.json" + write_document(result, path, normalized, "report") + result.reports.append(path) + # Preserve aggregate title/description as an explicitly separate supplied record. + aggregate = {"format": "viano-bug-report", "format_version": 1, "origin": origin(reader, prefix, "bug/bug.json", ""), "entities": []} + for field, destination in (("title", "title"), ("description", "summary")): + value = bug.get(field) + if isinstance(value, str) and value.strip(): + aggregate[destination] = value + self._entity(aggregate, bug.get("primary_entity_type"), bug.get("primary_entity_id"), "reported_primary_entity", origin(reader, prefix, "bug/bug.json", "/primary_entity_id")) + self._entity(aggregate, reader.manifest.get("diagnostic_focus_type"), reader.manifest.get("diagnostic_focus_id"), "diagnostic_focus", origin(reader, prefix, "manifest.json", "/diagnostic_focus_id")) + if reader.root + "bug/runtime_entities.json" in reader.members: + for index, entity in enumerate(rows(reader, "bug/runtime_entities.json")): + self._entity(aggregate, entity.get("type"), entity.get("id"), "runtime_observed", origin(reader, prefix, "bug/runtime_entities.json", f"/{index}")) + path = normalized_prefix + "/bug-record.json" + write_document(result, path, aggregate, "report") + result.reports.append(path) + for index, row in enumerate(reproductions): + normalized = {"format": "viano-reproduction", "format_version": 1, + "origin": origin(reader, prefix, "bug/reproductions.json", f"/{index}"), + "observations": [], "facts": [], "entities": [], + "events_ref": payload_ref(reader, prefix, "bug/reproduction_events.json")} + steps = row.get("steps") + if steps is not None and not isinstance(steps, str): + raise AdapterError("SOURCE_FIELD", "Source steps must be text or null") + normalized["steps"] = [steps] if steps and steps.strip() else [] + observed = statement(row.get("observation"), reader, prefix, "bug/reproductions.json", f"/{index}/observation", row.get("tester_character_name"), row.get("finished_at"), issues=result.issues) + if observed: + normalized["observations"].append(observed) + for event_index, event in enumerate(events): + if event["reproduction_id"] == row["reproduction_id"] and event.get("source") == "tester" and event.get("event_type") == "tester_note": + raw = event.get("data_json") + try: + decoded = parse_manifest(raw.encode("utf-8")) if isinstance(raw, str) else raw + except (ValueError, RecursionError) as error: + raise AdapterError("SOURCE_JSON", str(error), "bug/reproduction_events.json") from error + if not isinstance(decoded, dict): + raise AdapterError("SOURCE_FIELD", "Tester note data must be a JSON object") + note = statement(decoded.get("text"), reader, prefix, "bug/reproduction_events.json", f"/{event_index}/data_json", row.get("tester_character_name"), event.get("event_time"), decoded="/text", issues=result.issues) + if note: + normalized["observations"].append(note) + for field in ("result", "map_id", "zone_id", "area_id", "position_x", "position_y", "position_z", "orientation", "quest_id", "diagnostic_focus_type", "diagnostic_focus_id"): + if row.get(field) is not None: + normalized["facts"].append({"name": field, "value": row[field], "origin": origin(reader, prefix, "bug/reproductions.json", f"/{index}/{field}")}) + self._entity(normalized, row.get("diagnostic_focus_type"), row.get("diagnostic_focus_id"), "diagnostic_focus", origin(reader, prefix, "bug/reproductions.json", f"/{index}/diagnostic_focus_id")) + path = normalized_prefix + f"/reproduction-{row['reproduction_id']}.json" + write_document(result, path, normalized, "reproduction") + result.reproductions.append(path) + for warning in reader.manifest.get("warnings", []): + result.issues.append({"code": "SOURCE_WARNING", "message": str(warning), "tool": "wowdiag"}) + return result + + @staticmethod + def _entity(document, type_, id_, role, origin_): + if id_ is None or type_ is None: + return + if not isinstance(type_, str) or not type_.strip() or type(id_) is not int or id_ < 0: + raise AdapterError("SOURCE_ENTITY", "Invalid typed source entity") + document["entities"].append({"type": type_, "id": id_, "roles": [role], "origin": origin_}) diff --git a/src/viano_bug_analysis/profiles/vcore/wowsrc_adapter.py b/src/viano_bug_analysis/profiles/vcore/wowsrc_adapter.py new file mode 100644 index 0000000..5aafe0b --- /dev/null +++ b/src/viano_bug_analysis/profiles/vcore/wowsrc_adapter.py @@ -0,0 +1,25 @@ +from .common import check_manifest, materialize, payload_ref, rows +from .errors import AdapterError + + +class WowsrcAdapter: + def adapt(self, reader, prefix): + check_manifest(reader, "wowsrc", 7) + role_for = lambda path: "database" if path.startswith("database/") else "source" if path.startswith(("source/", "build/", "custom/", "changes/", "environment/", "input/", "handoff/")) or path == "manifest.json" else None + result = materialize(reader, prefix, role_for) + for field in ("primary_development_context", "supporting_development_context", "context_priority_policy", "source_ranking_policy", "database_schema_index", "database_relationship_map"): + if reader.manifest.get(field) is not None: + payload_ref(reader, prefix, reader.manifest[field]) + for record in rows(reader, "source/development_files_index.json"): + if record.get("copied") is True: + path = record.get("package_path") + name = reader.member_name(path) + if type(record.get("size")) is not int or record["size"] != reader.members[name].size: + raise AdapterError("SOURCE_SIZE", "Copied source size differs from its index", path) + elif record.get("copied") is False: + result.issues.append({"code": "SOURCE_NOT_COPIED", "message": str(record.get("reason_not_copied", "not copied")), "source_file": record.get("file")}) + else: + raise AdapterError("SOURCE_INDEX", "Development file index requires copied boolean") + for warning in reader.manifest.get("warnings", []): + result.issues.append({"code": "SOURCE_WARNING", "message": str(warning), "tool": "wowsrc"}) + return result diff --git a/src/viano_bug_analysis/schemas/case-v1.schema.json b/src/viano_bug_analysis/schemas/case-v1.schema.json index 8c2cb98..597bbb4 100644 --- a/src/viano_bug_analysis/schemas/case-v1.schema.json +++ b/src/viano_bug_analysis/schemas/case-v1.schema.json @@ -104,6 +104,9 @@ "type": "null" } ] + }, + "origin": { + "$ref": "#/$defs/origin" } } } @@ -221,6 +224,9 @@ "type": "string", "minLength": 1 } + }, + "origin": { + "$ref": "#/$defs/origin" } } } @@ -587,6 +593,9 @@ "null" ], "format": "date-time" + }, + "origin": { + "$ref": "#/$defs/origin" } } }, @@ -652,6 +661,37 @@ "$ref": "#/$defs/claim" } } + }, + "origin": { + "type": "object", + "required": [ + "source_ref", + "json_pointer" + ], + "properties": { + "source_ref": { + "type": "string", + "minLength": 1 + }, + "json_pointer": { + "type": "string", + "pattern": "^(|/.*)$" + }, + "archive_ref": { + "type": "string", + "minLength": 1 + }, + "member_name": { + "type": "string" + }, + "decoded_json_pointer": { + "type": "string", + "pattern": "^(|/.*)$" + }, + "source_timestamp": { + "type": "string" + } + } } } } diff --git a/src/viano_bug_analysis/schemas/report-v1.schema.json b/src/viano_bug_analysis/schemas/report-v1.schema.json index f775c78..7e98480 100644 --- a/src/viano_bug_analysis/schemas/report-v1.schema.json +++ b/src/viano_bug_analysis/schemas/report-v1.schema.json @@ -57,6 +57,32 @@ "items": { "$ref": "#/$defs/entity" } + }, + "origin": { + "$ref": "#/$defs/origin" + }, + "data_quality_issues": { + "type": "array", + "items": { + "type": "object", + "required": [ + "code", + "message" + ], + "properties": { + "code": { + "type": "string", + "minLength": 1 + }, + "message": { + "type": "string", + "minLength": 1 + }, + "origin": { + "$ref": "#/$defs/origin" + } + } + } } }, "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -108,6 +134,9 @@ }, "claim": { "$ref": "#/$defs/claim" + }, + "origin": { + "$ref": "#/$defs/origin" } } }, @@ -142,6 +171,40 @@ "minLength": 1, "pattern": "\\S" } + }, + "origin": { + "$ref": "#/$defs/origin" + } + } + }, + "origin": { + "type": "object", + "required": [ + "source_ref", + "json_pointer" + ], + "properties": { + "source_ref": { + "type": "string", + "minLength": 1 + }, + "json_pointer": { + "type": "string", + "pattern": "^(|/.*)$" + }, + "archive_ref": { + "type": "string", + "minLength": 1 + }, + "member_name": { + "type": "string" + }, + "decoded_json_pointer": { + "type": "string", + "pattern": "^(|/.*)$" + }, + "source_timestamp": { + "type": "string" } } } diff --git a/src/viano_bug_analysis/schemas/reproduction-v1.schema.json b/src/viano_bug_analysis/schemas/reproduction-v1.schema.json index f131555..965a502 100644 --- a/src/viano_bug_analysis/schemas/reproduction-v1.schema.json +++ b/src/viano_bug_analysis/schemas/reproduction-v1.schema.json @@ -58,7 +58,10 @@ "minLength": 1, "pattern": "\\S" }, - "value": {} + "value": {}, + "origin": { + "$ref": "#/$defs/origin" + } } } }, @@ -67,6 +70,9 @@ "items": { "$ref": "#/$defs/entity" } + }, + "origin": { + "$ref": "#/$defs/origin" } }, "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -118,6 +124,9 @@ }, "claim": { "$ref": "#/$defs/claim" + }, + "origin": { + "$ref": "#/$defs/origin" } } }, @@ -152,6 +161,40 @@ "minLength": 1, "pattern": "\\S" } + }, + "origin": { + "$ref": "#/$defs/origin" + } + } + }, + "origin": { + "type": "object", + "required": [ + "source_ref", + "json_pointer" + ], + "properties": { + "source_ref": { + "type": "string", + "minLength": 1 + }, + "json_pointer": { + "type": "string", + "pattern": "^(|/.*)$" + }, + "archive_ref": { + "type": "string", + "minLength": 1 + }, + "member_name": { + "type": "string" + }, + "decoded_json_pointer": { + "type": "string", + "pattern": "^(|/.*)$" + }, + "source_timestamp": { + "type": "string" } } } diff --git a/tests/profiles/vcore/conftest.py b/tests/profiles/vcore/conftest.py new file mode 100644 index 0000000..4e6c1e8 --- /dev/null +++ b/tests/profiles/vcore/conftest.py @@ -0,0 +1,52 @@ +import io +import json +import tarfile + +import pytest + + +@pytest.fixture +def make_tar(tmp_path): + counter = 0 + def create(entries, *, root="arbitrary-wrapper", extra=None): + nonlocal counter + counter += 1 + path = tmp_path / f"source-{counter}.data" + with tarfile.open(path, "w:gz") as archive: + for name, payload in entries.items(): + data = payload if isinstance(payload, bytes) else json.dumps(payload).encode() + info = tarfile.TarInfo(root + "/" + name if root else name) + info.size = len(data) + archive.addfile(info, io.BytesIO(data)) + for info, data in extra or []: + archive.addfile(info, io.BytesIO(data) if data is not None else None) + return path + return create + + +@pytest.fixture +def source_pair(make_tar): + diag_manifest = {"tool": "wowdiag", "format_version": 2, "tool_version": "0.10.6", "command": "bug", "bug_id": 20, + "core_revision": "verified-source-revision", "diagnostic_focus_type": "quest", "diagnostic_focus_id": 11117, "warnings": []} + diag_entries = { + "manifest.json": diag_manifest, + "bug/bug.json": {"bug_id": 20, "title": "Wolpertinger spawnen nicht", "description": "Supplied aggregate description"}, + "bug/reports.json": [{"report_id": 32, "bug_id": 20, "description": "Wolpertinger spawnen nicht", "expected_behavior": None, + "character_name": "Tester", "created_at": "2026-10-08T13:02:21", "target_type": "creature", "target_entry": 23486, "quest_id": 11117}], + "bug/reproductions.json": [{"reproduction_id": 41, "report_id": 32, "bug_id": 20, "result": "reproduced", "steps": None, + "observation": "No spawns here; the other location worked", "tester_character_name": "Tester", + "finished_at": "2026-10-08T13:04:02", "diagnostic_focus_type": "quest", "diagnostic_focus_id": 11117, + "map_id": 0, "zone_id": 1, "area_id": 1}], + "bug/reproduction_events.json": [{"event_id": 527, "reproduction_id": 41, "source": "tester", "event_type": "tester_note", + "event_time": "2026-10-08T13:02:55", "data_json": json.dumps({"text": "Still no spawns"})}], + "database/quest_template.json": [{"ID": 11117}], "source/secondary.json": [{"numeric_hit": 11117}], + "unknown.bin": b"\x00\xff", "environment/core.json": {"commit": "verified-source-revision"}, + } + src_manifest = {"tool": "wowsrc", "format_version": 7, "tool_version": "0.7", "command": "bug", "bug_id": 20, + "core_revision": "verified-source-revision", "input_wowdiag_format": 2, "input_wowdiag_version": "0.10.6", "warnings": []} + src_entries = {"manifest.json": src_manifest, "input/wowdiag_manifest.json": diag_manifest, + "source/development_files_index.json": [{"file": "/unavailable/server/source.cpp", "package_path": "source/development_files/source.cpp", "copied": True, "size": 4}], + "source/development_files/source.cpp": b"code", "database/schema.json": {"columns": []}, + "source/primary_implementation_units.json": [], "source/source_ranking_policy.json": {"numeric_only_fallback_promotion": False}, + "unknown.extension": b"retained"} + return make_tar(diag_entries), make_tar(src_entries), diag_entries, src_entries diff --git a/tests/profiles/vcore/test_archive_reader.py b/tests/profiles/vcore/test_archive_reader.py new file mode 100644 index 0000000..40d337f --- /dev/null +++ b/tests/profiles/vcore/test_archive_reader.py @@ -0,0 +1,110 @@ +import gzip +import tarfile +from dataclasses import replace + +import pytest + +from viano_bug_analysis.profiles.vcore import AdapterError, ArchiveLimits, ArchiveReader + + +@pytest.mark.parametrize("name", ["../escape", "/absolute", "C:/escape", "a\\b", "a//b", "a/./b", "a:stream"]) +def test_unsafe_paths(make_tar, name): + info = tarfile.TarInfo(name) + info.size = 1 + path = make_tar({"manifest.json": {}}, extra=[(info, b"x")]) + with pytest.raises(AdapterError) as error: + with ArchiveReader(path): + pass + assert error.value.code == "UNSAFE_PATH" + + +@pytest.mark.parametrize("type_", [tarfile.SYMTYPE, tarfile.LNKTYPE, tarfile.FIFOTYPE, tarfile.CHRTYPE, tarfile.BLKTYPE, tarfile.GNUTYPE_SPARSE]) +def test_special_members_rejected(make_tar, type_): + info = tarfile.TarInfo("wrapper/link") + info.type = type_ + info.linkname = "../escape" + path = make_tar({"manifest.json": {}}, extra=[(info, None)]) + with pytest.raises(AdapterError): + with ArchiveReader(path): + pass + + +@pytest.mark.parametrize("entries", [{"a": b"x", "A": b"y"}, {"a": b"x", "a/b": b"y"}]) +def test_member_collisions(make_tar, entries): + path = make_tar({"manifest.json": {}} | entries) + with pytest.raises(AdapterError) as error: + with ArchiveReader(path): + pass + assert error.value.code == "MEMBER_COLLISION" + + +@pytest.mark.parametrize("damage", ["gzip", "truncated", "crc", "tar"]) +def test_corrupt_source_rejected(make_tar, damage): + path = make_tar({"manifest.json": {}}) + raw = bytearray(path.read_bytes()) + if damage == "gzip": + raw = b"not gzip" + elif damage == "truncated": + raw = raw[:-4] + elif damage == "crc": + raw[-8] ^= 1 + else: + unpacked = bytearray(gzip.decompress(raw)) + unpacked[0] ^= 1 + raw = gzip.compress(unpacked) + path.write_bytes(raw) + with pytest.raises(AdapterError): + with ArchiveReader(path): + pass + + +@pytest.mark.parametrize("field,value", [("max_archive_bytes", 1), ("max_tar_bytes", 512), ("max_file_bytes", 1), ("max_entries", 1), ("max_json_bytes", 1), ("max_ratio", 1)]) +def test_configurable_limits(make_tar, field, value): + path = make_tar({"manifest.json": {}, "data": b"abc"}) + with pytest.raises(AdapterError): + with ArchiveReader(path, replace(ArchiveLimits(), **{field: value})): + pass + + +def test_pax_metadata_bound(make_tar): + info = tarfile.TarInfo("extra") + info.pax_headers = {"comment": "x" * 2000} + info.size = 1 + path = make_tar({"manifest.json": {}}, extra=[(info, b"x")]) + with pytest.raises(AdapterError) as error: + with ArchiveReader(path, ArchiveLimits(max_header_bytes=100)): + pass + assert error.value.code == "HEADER_LIMIT" + + +def test_ambiguous_manifest(make_tar): + path = make_tar({"one/manifest.json": {}, "two/manifest.json": {}}, root="") + with pytest.raises(AdapterError) as error: + with ArchiveReader(path): + pass + assert error.value.code == "MANIFEST_AMBIGUOUS" + + +def test_snapshot_is_immutable_and_nothing_is_extracted(make_tar, tmp_path): + path = make_tar({"manifest.json": {}, "unknown": b"data"}) + raw = path.read_bytes() + with ArchiveReader(path) as reader: + path.write_bytes(b"changed external file") + assert reader.original_bytes() == raw + assert reader.read_member(reader.member_name("unknown")) == b"data" + with pytest.raises(AdapterError): + reader.member_name("../escape") + assert not (tmp_path / "arbitrary-wrapper").exists() + + +def test_sparse_rejected_before_sparse_map_processing(make_tar, monkeypatch): + info = tarfile.TarInfo("sparse") + info.type = tarfile.GNUTYPE_SPARSE + path = make_tar({"manifest.json": {}}, extra=[(info, None)]) + def never_parse(*args): + pytest.fail("Sparse map processing should not execute") + monkeypatch.setattr(tarfile.TarInfo, "_proc_sparse", never_parse) + with pytest.raises(AdapterError) as error: + with ArchiveReader(path): + pass + assert error.value.code == "SPECIAL_MEMBER" diff --git a/tests/profiles/vcore/test_importer.py b/tests/profiles/vcore/test_importer.py new file mode 100644 index 0000000..46e19f8 --- /dev/null +++ b/tests/profiles/vcore/test_importer.py @@ -0,0 +1,213 @@ +import json +import shutil + +import pytest + +from viano_bug_analysis.case import CaseBuilder +from viano_bug_analysis.lifecycle import CheckpointStore, Workspace +from viano_bug_analysis.package import PackageReader, PackageValidator +from viano_bug_analysis.profiles.vcore import AdapterError, VcoreImporter +from viano_bug_analysis.profiles.vcore import ArchiveLimits + + +def test_pair_roundtrip_and_provenance(source_pair, tmp_path): + diag, src, diag_entries, src_entries = source_pair + destination = tmp_path / "renamed.package" + importer = VcoreImporter() + result = importer.create(diag, destination, wowsrc=src, package_id="stable", case_id="opaque-case") + assert result.valid and result.manifest["package_generation"] == 0 + assert result.manifest["history"]["reproductions"] == 1 + store = CheckpointStore(Workspace(tmp_path / "service")) + store.intake(destination) + built = CaseBuilder(store).build("opaque-case", "stable") + assert PackageValidator().validate(built.package).valid + with PackageReader(built.package) as reader: + case = json.loads(reader.read_bytes(built.case_ref)) + observations = case["human_observations"]["observations"] + assert [row["text"] for row in observations] == ["Wolpertinger spawnen nicht", "No spawns here; the other location worked", "Still no spawns"] + assert case["expected_behavior"]["completeness"] == "missing" + assert case["human_observations"]["tester_hypotheses"] == [] + note = observations[-1]["provenance"]["origin"] + assert note["json_pointer"] == "/0/data_json" and note["decoded_json_pointer"] == "/text" + assert note["source_timestamp"] == "2026-10-08T13:02:55" + assert observations[-1]["provenance"]["observed_at"] is None + raw_note = json.loads(reader.read_bytes(note["source_ref"]))[0]["data_json"] + assert json.loads(raw_note)["text"] == observations[-1]["text"] + assert reader.read_bytes(note["archive_ref"]) == diag.read_bytes() + assert any(row["type"] == "quest" and row["id"] == "11117" and "diagnostic_focus" in row["roles"] for row in case["entities"]) + assert any(issue["code"] == "SOURCE_TIME_UNQUALIFIED" for issue in case["data_quality"]["issues"]) + assert any(ref["path"].endswith("source.cpp") for ref in case["source_context"]["refs"]) + assert any(ref["path"].endswith("unknown.extension") for ref in case["inputs"]["unclassified"]) + assert "Source timestamp (unchanged):" in reader.read_bytes(built.summary_ref).decode() + # All materialized raw files and both compressed originals are preserved. + for entries in (diag_entries, src_entries): + for name, data in entries.items(): + paths = [path for path in reader.files if "/members/" in path and path.endswith("/" + name)] + expected = data if isinstance(data, bytes) else json.dumps(data).encode() + assert any(reader.read_bytes(path) == expected for path in paths) + assert CaseBuilder(store).build("opaque-case", "stable").reused + + +def test_filename_has_no_meaning(source_pair, tmp_path): + diag, src, _, _ = source_pair + renamed_diag, renamed_src = tmp_path / "unrelated.one", tmp_path / "unrelated.two" + shutil.copyfile(diag, renamed_diag) + shutil.copyfile(src, renamed_src) + first = VcoreImporter().prepare(diag, src) + again = VcoreImporter().prepare(renamed_diag, renamed_src) + assert first == again + + +@pytest.mark.parametrize("field,value", [("tool", "other"), ("format_version", 99), ("command", "quest"), ("bug_id", 21), ("core_revision", "different"), ("input_wowdiag_format", 3)]) +def test_mismatched_pair_rejected(source_pair, make_tar, tmp_path, field, value): + diag, _, _, src_entries = source_pair + src_entries["manifest.json"] = dict(src_entries["manifest.json"], **{field: value}) + src = make_tar(src_entries) + destination = tmp_path / "out" + with pytest.raises(AdapterError): + VcoreImporter().create(diag, destination, wowsrc=src, package_id="p", case_id="c") + assert not destination.exists() + + +@pytest.mark.parametrize("path", ["../outside", "/external/file", "missing"]) +def test_copied_source_references_not_followed(source_pair, make_tar, path): + diag, _, _, entries = source_pair + entries["source/development_files_index.json"][0]["package_path"] = path + with pytest.raises(AdapterError) as error: + VcoreImporter().prepare(diag, make_tar(entries)) + assert error.value.code == "SOURCE_REFERENCE" + + +def test_missing_copied_source_reported(source_pair, make_tar): + diag, _, _, entries = source_pair + entries["source/development_files_index.json"][0] = {"file": "/external/large.cpp", "copied": False, "reason_not_copied": "size_limit"} + plan = VcoreImporter().prepare(diag, make_tar(entries)) + receipt = json.loads(plan.additions[plan.receipt_ref]) + assert any(issue["code"] == "SOURCE_NOT_COPIED" for issue in receipt["issues"]) + + +def test_wrong_copied_size_rejected(source_pair, make_tar): + diag, _, _, entries = source_pair + entries["source/development_files_index.json"][0]["size"] = 123 + with pytest.raises(AdapterError) as error: + VcoreImporter().prepare(diag, make_tar(entries)) + assert error.value.code == "SOURCE_SIZE" + + +def test_augment_preserves_old_content_and_rejects_duplicate(source_pair, tmp_path): + diag, src, _, _ = source_pair + first, second = tmp_path / "first", tmp_path / "second" + importer = VcoreImporter() + importer.create(diag, first, package_id="stable", case_id="opaque") + result = importer.augment(first, second, wowdiag=diag, wowsrc=src) + assert result.manifest["package_generation"] == 1 + with PackageReader(first) as before, PackageReader(second) as after: + for name in before.files: + if name != "manifest.json": + assert before.read_bytes(name) == after.read_bytes(name) + with pytest.raises(AdapterError) as error: + importer.augment(second, tmp_path / "third", wowdiag=diag, wowsrc=src) + assert error.value.code == "IMPORT_EXISTS" + + +def test_case_identity_is_caller_owned(source_pair, tmp_path): + diag, _, _, _ = source_pair + result = VcoreImporter().create(diag, tmp_path / "BUG-999.zip", package_id="caller-package", case_id="caller-case") + assert result.manifest["package_id"] == "caller-package" + assert result.manifest["case_id"] == "caller-case" + + +def test_combined_import_budget(source_pair): + diag, src, _, _ = source_pair + with pytest.raises(AdapterError) as error: + VcoreImporter(ArchiveLimits(max_import_bytes=100)).prepare(diag, src) + assert error.value.code == "IMPORT_LIMIT" + + +def test_unresolved_row_relations_rejected(source_pair, make_tar): + _, src, entries, _ = source_pair + entries["bug/reproduction_events.json"][0]["reproduction_id"] = 999 + with pytest.raises(AdapterError) as error: + VcoreImporter().prepare(make_tar(entries)) + assert error.value.code == "SOURCE_RELATION" + + +def test_duplicate_source_ids_rejected(source_pair, make_tar): + _, _, entries, _ = source_pair + entries["bug/reports.json"] *= 2 + with pytest.raises(AdapterError) as error: + VcoreImporter().prepare(make_tar(entries)) + assert error.value.code == "SOURCE_ID" + + +def test_explicit_expected_behavior_is_copied_not_completed(source_pair, make_tar): + _, _, entries, _ = source_pair + entries["bug/reports.json"][0]["expected_behavior"] = "Supplied expected behavior" + plan = VcoreImporter().prepare(make_tar(entries)) + index = json.loads(plan.additions[plan.index_ref]) + report = json.loads(plan.additions[index["reports"][0]]) + assert report["expected_behavior"][0]["text"] == "Supplied expected behavior" + assert "expected_behavior_complete" not in report + + +def test_aware_source_time_preserved(source_pair, make_tar): + _, _, entries, _ = source_pair + entries["bug/reports.json"][0]["created_at"] = "2026-10-08T13:02:21+02:00" + plan = VcoreImporter().prepare(make_tar(entries)) + report = json.loads(plan.additions[json.loads(plan.additions[plan.index_ref])["reports"][0]]) + assert report["observations"][0]["observed_at"] == "2026-10-08T13:02:21+02:00" + + +def test_bad_existing_receipt_is_controlled(source_pair, manifest, tmp_path): + from viano_bug_analysis.package import PackageFactory + diag, _, _, _ = source_pair + manifest["project"] = {"type": "wow", "profile": "vcore"} + manifest["artifacts"]["vcore_import"] = "bad-receipt" + original = tmp_path / "old" + PackageFactory().create(original, manifest, {"bad-receipt": b"[]"}) + with pytest.raises(AdapterError) as error: + VcoreImporter().augment(original, tmp_path / "out", wowdiag=diag) + assert error.value.code == "IMPORT_RECEIPT" + assert not (tmp_path / "out").exists() + + +def test_copied_attachments_are_classified(source_pair, make_tar): + _, _, entries, _ = source_pair + entries["attachments/32/screenshot.bin"] = b"copied attachment" + plan = VcoreImporter().prepare(make_tar(entries)) + index = json.loads(plan.additions[plan.index_ref]) + assert len(index["attachments"]) == 1 + assert plan.additions[index["attachments"][0]] == b"copied attachment" + + +def test_report_link_allows_null_direct_bug_id(source_pair, make_tar): + _, _, entries, _ = source_pair + entries["bug/reproductions.json"][0]["bug_id"] = None + assert VcoreImporter().prepare(make_tar(entries)).reproductions_count == 1 + + +def test_explicit_runtime_entities_are_normalized(source_pair, make_tar): + _, _, entries, _ = source_pair + entries["bug/runtime_entities.json"] = [{"type": "creature", "id": 123, "reasons": ["recorded_event"]}] + plan = VcoreImporter().prepare(make_tar(entries)) + index = json.loads(plan.additions[plan.index_ref]) + records = [json.loads(plan.additions[path]) for path in index["reports"]] + entities = [entity for record in records for entity in record.get("entities", [])] + assert any(entity["id"] == 123 and entity["roles"] == ["runtime_observed"] and entity["origin"]["json_pointer"] == "/0" for entity in entities) + + +def test_adding_wowsrc_does_not_duplicate_original_observations(source_pair, tmp_path): + diag, src, _, _ = source_pair + first, augmented = tmp_path / "first", tmp_path / "augmented" + importer = VcoreImporter() + importer.create(diag, first, package_id="p", case_id="c") + importer.augment(first, augmented, wowdiag=diag, wowsrc=src) + store = CheckpointStore(Workspace(tmp_path / "service")) + store.intake(augmented) + result = CaseBuilder(store).build("c", "p") + with PackageReader(result.package) as reader: + case = json.loads(reader.read_bytes(result.case_ref)) + assert len(case["human_observations"]["observations"]) == 3 + with pytest.raises(AdapterError) as error: + importer.augment(augmented, tmp_path / "repeated", wowdiag=diag, wowsrc=src) + assert error.value.code == "IMPORT_EXISTS" diff --git a/tests/profiles/vcore/test_real_package.py b/tests/profiles/vcore/test_real_package.py new file mode 100644 index 0000000..45a572f --- /dev/null +++ b/tests/profiles/vcore/test_real_package.py @@ -0,0 +1,40 @@ +"""Opt-in full-archive regression, without machine-specific paths or binary fixtures.""" +import hashlib +import json +import os +from pathlib import Path + +import pytest + +from viano_bug_analysis.case import CaseBuilder +from viano_bug_analysis.lifecycle import CheckpointStore, Workspace +from viano_bug_analysis.package import PackageReader, PackageValidator +from viano_bug_analysis.profiles.vcore import ArchiveReader, VcoreImporter + + +@pytest.mark.skipif(not os.environ.get("VBA_WOWDIAG_SAMPLE") or not os.environ.get("VBA_WOWSRC_SAMPLE"), + reason="Set VBA_WOWDIAG_SAMPLE and VBA_WOWSRC_SAMPLE for the BUG-20 full archive regression") +def test_full_bug20_package_pair(tmp_path): + diag, src = Path(os.environ["VBA_WOWDIAG_SAMPLE"]), Path(os.environ["VBA_WOWSRC_SAMPLE"]) + original_hashes = [hashlib.sha256(path.read_bytes()).hexdigest() for path in (diag, src)] + package = tmp_path / "anything.bin" + VcoreImporter().create(diag, package, package_id="regression-package", case_id="regression-case", wowsrc=src) + store = CheckpointStore(Workspace(tmp_path / "service")) + store.intake(package) + result = CaseBuilder(store).build("regression-case", "regression-package") + assert PackageValidator().validate(result.package).valid + with PackageReader(result.package) as reader: + case = json.loads(reader.read_bytes(result.case_ref)) + assert len(case["human_observations"]["observations"]) == 3 + assert case["expected_behavior"]["completeness"] == "missing" + assert case["human_observations"]["tester_hypotheses"] == [] + assert len(case["database_context"]["refs"]) == 103 + assert len(case["source_context"]["refs"]) == 93 + for input_path in (diag, src): + with ArchiveReader(input_path) as source: + for member in source.members: + matching = [path for path in reader.files if path.endswith("/members/" + member)] + assert len(matching) == 1 + assert reader.read_bytes(matching[0]) == source.read_member(member) + assert CaseBuilder(store).build("regression-case", "regression-package").reused + assert [hashlib.sha256(path.read_bytes()).hexdigest() for path in (diag, src)] == original_hashes diff --git a/tests/test_package_factory.py b/tests/test_package_factory.py new file mode 100644 index 0000000..aa4745b --- /dev/null +++ b/tests/test_package_factory.py @@ -0,0 +1,44 @@ +import copy +import os + +import pytest + +from viano_bug_analysis.package import PackageFactory, PackageReader, PackageWriteError + + +def test_generic_factory(manifest, tmp_path): + original = copy.deepcopy(manifest) + path = tmp_path / "first.package" + assert PackageFactory().create(path, manifest, {"unknown": b"opaque"}).valid + assert manifest == original + with PackageReader(path) as reader: + assert reader.manifest["package_generation"] == 0 + assert reader.read_bytes("unknown") == b"opaque" + + +@pytest.mark.parametrize("artifacts", [{"../escape": b"x"}, {"manifest.json": b"x"}, {"a": b"x", "A": b"y"}, {"a": b"x", "a/b": b"y"}, {"a": "not bytes"}]) +def test_invalid_creation_keeps_target_absent(manifest, tmp_path, artifacts): + target = tmp_path / "out" + with pytest.raises(PackageWriteError): + PackageFactory().create(target, manifest, artifacts) + assert not target.exists() + assert not list(tmp_path.glob(".vbcase-*.tmp")) + + +def test_factory_failure_preserves_existing_target(manifest, tmp_path): + target = tmp_path / "out" + target.write_bytes(b"previous") + with pytest.raises(PackageWriteError): + PackageFactory().create(target, manifest, {}) + assert target.read_bytes() == b"previous" + + +def test_factory_fsync_failure(manifest, tmp_path, monkeypatch): + target = tmp_path / "out" + def fail(*args): + raise OSError("disk error") + monkeypatch.setattr(os, "fsync", fail) + with pytest.raises(PackageWriteError): + PackageFactory().create(target, manifest, {}) + assert not target.exists() + assert not list(tmp_path.glob(".vbcase-*.tmp"))