import asyncio import copy import hashlib import json from dataclasses import replace from datetime import datetime, timedelta, timezone from importlib.resources import files import pytest from jsonschema import Draft202012Validator from viano_bug_analysis.analysis import (CaseInterpreter, HypothesisBuilder, HypothesisConfig, HypothesisError, HypothesisLimits, render_hypotheses) from viano_bug_analysis.analysis.hypothesis_contracts import validate_generation from viano_bug_analysis.case import CaseBuilder from viano_bug_analysis.lifecycle import CheckpointStore, State, Workspace from viano_bug_analysis.lifecycle.errors import LockBusy from viano_bug_analysis.llm import GatewayConfig, LLMGateway, ProviderConfig, ProviderResponse, ProviderUnavailable from viano_bug_analysis.package import PackageReader, PackageValidator, PackageWriter from test_case_interpreter import built_case, FixtureAdapter, gateway, http_endpoint from test_case_builder import case_setup def proposals(context, count=3): sources = context["interpretation_sources"] primary = next((key for kind in ("first_known_divergence", "failure_stage", "observed_behavior") for key, entry in sources.items() if entry["kind"] == kind), next(iter(sources))) any_ref = next(iter(sources)) mechanisms = ["The interaction event may not reach the state handler", "The stored state may be overwritten after interaction", "The client display may not reflect the updated state", "The interaction may target another instance", "The operation may finish after the observation window"] return {"hypotheses": [{"title":f"Alternative {index + 1}", "mechanism":mechanisms[index % len(mechanisms)] + (f" variant {index}" if index >= 5 else ""), "rationale":"The reported observation is consistent with this untested possibility", "distinction":f"Candidate mechanism {index + 1} concerns a different stage", "source_ids":["O0001"], "interpretation_ids":[primary if index == 0 else any_ref], "expected_evidence":[f"A trace would expose candidate mechanism {index + 1}"], "falsifying_evidence":[f"An intact stage {index + 1} would disconfirm this candidate"], "open_questions":[f"Does candidate stage {index + 1} operate as expected?"]} for index in range(count)]} class ProposalAdapter: def __init__(self, change=None, count=3, response=None): self.change, self.count, self.response = change, count, response self.requests = [] async def generate(self, config, request, *, max_response_bytes): self.requests.append(copy.deepcopy(request)) if self.response: raise self.response context = json.loads(request["messages"][1]["content"])["input"] result = proposals(context,self.count) if self.change: self.change(result,context) return ProviderResponse(result,"fixture-hypothesis-model",12345,678) @pytest.fixture def interpreted_case(built_case): store, builder, built, original, entries = built_case interpreted = asyncio.run(CaseInterpreter(store,gateway(FixtureAdapter())).interpret("case-A","stable-package")) return store, builder, interpreted, original, entries def run(builder, **kw): return asyncio.run(builder.build("case-A","stable-package",**kw)) def test_generation_budget_registry_and_preservation(interpreted_case): store, case_builder, interpreted, original, entries = interpreted_case original_bytes = original.read_bytes() adapter = ProposalAdapter() result = run(HypothesisBuilder(store,gateway(adapter))) assert result.status == "COMPLETED" and not result.reused assert PackageValidator().validate(result.package).valid with PackageReader(interpreted.package) as before, PackageReader(result.package) as after: document = json.loads(after.read_bytes(result.hypotheses_ref)) registry = json.loads(after.read_bytes(result.registry_ref)) assert [h["id"] for h in document["hypotheses"]] == ["H1","H2","H3"] assert {h["status"] for h in document["hypotheses"]} == {"new"} assert {h["confidence"] for h in document["hypotheses"]} == {"unassessed"} assert {h["budget_allocation"]["initial"] for h in document["hypotheses"]} == {100} assert document["hypotheses"][0]["focus"] == "tester_primary" assert document["budget_accounting"] == "not_started" assert registry["created_count"] == 3 and registry["remaining_slots"] == 2 and registry["next_id"] == 4 assert [c["allocated_budget"] for c in registry["creations"]] == [100,100,100] assert registry["hypotheses_sha256"] == hashlib.sha256(after.read_bytes(result.hypotheses_ref)).hexdigest() assert document["llm_metadata"]["input_tokens"] == 12345 assert after.manifest["state"] == State.GENERATING_HYPOTHESES assert after.manifest["package_generation"] == 9 and after.manifest["analysis_revision"] == 1 assert after.manifest["package_id"] == "stable-package" assert after.manifest["artifacts"]["diagnosis"] is None assert after.manifest["artifacts"]["change_request"] is None assert after.read_bytes(result.summary_ref).decode() == render_hypotheses(document) for name in before.files: if name != "manifest.json": assert before.read_bytes(name) == after.read_bytes(name) assert original.read_bytes() == original_bytes assert len(adapter.requests) == 1 assert adapter.requests[0]["prompt_version"] == "hypotheses_v2" assert case_builder.build("case-A","stable-package").reused assert asyncio.run(CaseInterpreter(store,gateway(adapter)).interpret("case-A","stable-package")).reused def test_configuration_defaults_and_invalid_values(tmp_path): assert HypothesisConfig.load("config/analysis.example.toml") == HypothesisConfig() assert HypothesisConfig(max_hypotheses=7,initial_budget=23).initial_budget == 23 for values in ({"min_hypotheses":0},{"max_hypotheses":True},{"initial_budget":1.5},{"initial_budget":-1},{"min_hypotheses":6}): with pytest.raises(HypothesisError): HypothesisConfig(**values) invalid = tmp_path / "invalid.toml" invalid.write_text('[hypotheses]\nunknown=1') with pytest.raises(HypothesisError): HypothesisConfig.load(invalid) def test_custom_equal_budget_and_count(interpreted_case): cfg = HypothesisConfig(min_hypotheses=4,max_hypotheses=4,initial_budget=23) result = run(HypothesisBuilder(interpreted_case[0],gateway(ProposalAdapter(count=4)),cfg)) assert result.status == "COMPLETED" with PackageReader(result.package) as reader: document = json.loads(reader.read_bytes(result.hypotheses_ref)) assert len(document["hypotheses"]) == 4 assert [h["budget_allocation"]["initial"] for h in document["hypotheses"]] == [23]*4 assert json.loads(reader.read_bytes(result.registry_ref))["remaining_slots"] == 0 @pytest.mark.parametrize("field", ["budget","id","status","confidence","priority","tool_requests","patch","root_cause"]) def test_model_cannot_assign_budget_ids_state_or_actions(interpreted_case,field): def change(payload,context): payload["hypotheses"][0][field] = 1000000 result = run(HypothesisBuilder(interpreted_case[0],gateway(ProposalAdapter(change)))) assert result.status == "ERROR" and result.code == "LLM_OUTPUT_SCHEMA" with PackageReader(result.package) as reader: assert reader.manifest["artifacts"]["hypothesis_registry"] is None assert reader.manifest["artifacts"]["hypotheses"] is None assert not any(name.endswith("/hypotheses.json") for name in reader.files) @pytest.mark.parametrize("count", [0,1,2,6,7]) def test_out_of_policy_count_is_rejected_before_creation(interpreted_case,count): result = run(HypothesisBuilder(interpreted_case[0],gateway(ProposalAdapter(count=count)))) assert result.status == "ERROR" and result.code == "LLM_OUTPUT_SCHEMA" with PackageReader(result.package) as reader: assert reader.manifest["artifacts"]["hypothesis_registry"] is None @pytest.mark.parametrize("change,code", [("duplicate_mechanism","HYPOTHESIS_DUPLICATE"),("duplicate_title","HYPOTHESIS_DUPLICATE"), ("unknown_source","HYPOTHESIS_SOURCE_UNKNOWN"),("unknown_interpretation","HYPOTHESIS_SOURCE_UNKNOWN"), ("ungrounded","HYPOTHESIS_GROUNDING"),("primary_not_human","HYPOTHESIS_PRIMARY_FOCUS"),("primary_wrong_focus","HYPOTHESIS_PRIMARY_FOCUS")]) def test_invalid_set_rejected_atomically_and_replayed(interpreted_case,change,code): def mutate(payload,context): first, second = payload["hypotheses"][:2] if change == "duplicate_mechanism": second["mechanism"] = " " + first["mechanism"].upper() + "!!!" if change == "duplicate_title": second["title"] = first["title"].upper() if change == "unknown_source": second["source_ids"] = ["O9999"] if change == "unknown_interpretation": second["interpretation_ids"] = ["I9999"] if change == "ungrounded": second["source_ids"] = ["T0001","E0001"] if change == "primary_not_human": first["source_ids"] = ["F0001"] if change == "primary_wrong_focus": first["interpretation_ids"] = [next(key for key,row in context["interpretation_sources"].items() if row["kind"] == "unknowns")] adapter = ProposalAdapter(mutate) builder = HypothesisBuilder(interpreted_case[0],gateway(adapter)) result = run(builder) assert result.status == "ERROR" and result.code == code with PackageReader(result.package) as reader: assert reader.manifest["artifacts"]["hypothesis_registry"] is None assert reader.manifest["artifacts"]["hypotheses"] is None again = run(builder) assert again.reused and again.package == result.package assert len(adapter.requests) == 1 def test_completed_replay_never_refills_or_regenerates_on_config_change(interpreted_case): adapter = ProposalAdapter() store = interpreted_case[0] first = run(HypothesisBuilder(store,gateway(adapter))) changed = HypothesisConfig(min_hypotheses=1,max_hypotheses=99,initial_budget=1000000) again = run(HypothesisBuilder(CheckpointStore(store.workspace),gateway(adapter),changed),attempt_id="another") assert again.reused and again.package == first.package assert len(adapter.requests) == 1 with PackageReader(again.package) as reader: registry = json.loads(reader.read_bytes(again.registry_ref)) assert registry["created_count"] == 3 and registry["max_hypotheses"] == 5 assert registry["initial_budget"] == 100 def test_policy_is_pinned_across_pending_retries(interpreted_case): store = interpreted_case[0] adapter = ProposalAdapter(response=ProviderUnavailable("LLM_CONNECTION")) first = run(HypothesisBuilder(store,gateway(adapter))) assert first.status == "RETRY_PENDING" with pytest.raises(HypothesisError,match="HYPOTHESIS_POLICY_CHANGED"): run(HypothesisBuilder(store,gateway(adapter),HypothesisConfig(initial_budget=200)),attempt_id="new") assert len(adapter.requests) == 1 def test_requires_confirmed_interpretation_without_calling_earlier_phase(built_case): adapter = ProposalAdapter() with pytest.raises(HypothesisError,match="HYPOTHESIS_INTERPRETATION_NOT_READY"): run(HypothesisBuilder(built_case[0],gateway(adapter))) assert not adapter.requests def test_portable_completed_generation(interpreted_case,tmp_path): adapter = ProposalAdapter() first = run(HypothesisBuilder(interpreted_case[0],gateway(adapter))) receiving = CheckpointStore(Workspace(tmp_path / "other")) receiving.intake(first.package) result = run(HypothesisBuilder(receiving,gateway(adapter))) assert result.reused and len(adapter.requests) == 1 assert result.package == receiving.latest("case-A","stable-package") def test_explicit_recovery_after_rejection_does_not_create_ghost_ids(interpreted_case): adapter = ProposalAdapter(count=6) builder = HypothesisBuilder(interpreted_case[0],gateway(adapter)) assert run(builder).status == "ERROR" adapter.count = 3 recovered = run(builder,attempt_id="corrected") assert recovered.status == "COMPLETED" with PackageReader(recovered.package) as reader: registry = json.loads(reader.read_bytes(recovered.registry_ref)) assert registry["created_count"] == 3 and registry["next_id"] == 4 assert [r["id"] for r in registry["creations"]] == ["H1","H2","H3"] assert len([name for name in reader.files if "/hypotheses/0001/errors/" in name]) == 1 def test_registry_and_allocation_tampering_is_rejected(interpreted_case): result = run(HypothesisBuilder(interpreted_case[0],gateway(ProposalAdapter()))) with PackageReader(result.package) as reader: document = json.loads(reader.read_bytes(result.hypotheses_ref)) registry = json.loads(reader.read_bytes(result.registry_ref)) context = json.loads(reader.read_bytes(document["context_ref"])) changed = copy.deepcopy(registry) changed["created_count"] = 0 with pytest.raises(HypothesisError,match="HYPOTHESIS_REGISTRY_CHANGED"): validate_generation(document,changed,context,result.hypotheses_ref) changed = copy.deepcopy(document) changed["hypotheses"][1]["budget_allocation"]["initial"] = 1000 with pytest.raises(HypothesisError,match="HYPOTHESIS_ALLOCATION_CHANGED"): validate_generation(changed,registry,context,result.hypotheses_ref) def test_retry_deferral_exhaustion_and_phase_logging(interpreted_case,caplog): store = interpreted_case[0] now = [datetime(2026,1,1,tzinfo=timezone.utc)] adapter = ProposalAdapter(response=ProviderUnavailable("LLM_CONNECTION")) cfg = gateway(adapter,max_attempts=2,retry_delay_seconds=1) builder = HypothesisBuilder(store,cfg,clock=lambda:now[0]) first = run(builder) assert first.status == "RETRY_PENDING" deferred = run(HypothesisBuilder(CheckpointStore(store.workspace),cfg,clock=lambda:now[0])) assert deferred.status == "DEFERRED" and deferred.package == first.package assert len(adapter.requests) == 1 now[0] += timedelta(seconds=1) blocked = run(builder) assert blocked.status == "BLOCKED" and blocked.code == "LLM_UNAVAILABLE" with PackageReader(blocked.package) as reader: assert reader.manifest["state"] == State.BLOCKED assert reader.manifest["artifacts"]["hypothesis_registry"] is None assert "phase GENERATING_HYPOTHESES failed" in caplog.text @pytest.mark.parametrize("field,value,code", [("max_context_bytes",100,"HYPOTHESIS_CONTEXT_LIMIT"), ("max_output_bytes",100,"HYPOTHESIS_OUTPUT_LIMIT"),("max_summary_bytes",100,"HYPOTHESIS_SUMMARY_LIMIT")]) def test_operational_limits(interpreted_case,field,value,code): adapter = ProposalAdapter() builder = HypothesisBuilder(interpreted_case[0],gateway(adapter),limits=HypothesisLimits(**{field:value})) if field == "max_context_bytes": with pytest.raises(HypothesisError,match=code): run(builder) assert not adapter.requests else: result = run(builder) assert result.status == "ERROR" and result.code == code @pytest.mark.parametrize("contract", ["hypotheses-output","hypotheses","hypothesis-policy","hypothesis-context","hypothesis-registry","hypothesis-error"]) def test_hypothesis_schemas(contract): Draft202012Validator.check_schema(json.loads(files("viano_bug_analysis.schemas").joinpath(contract+'-v1.schema.json').read_text())) def test_real_http_adapter(interpreted_case,http_endpoint): url, seen, reply = http_endpoint original_body = reply["body"] # Obtain the exact I catalogue from the persisted phase context through an offline preparation. adapter = ProposalAdapter(response=ProviderUnavailable("LLM_CONNECTION")) store = interpreted_case[0] pending = run(HypothesisBuilder(store,gateway(adapter))) with PackageReader(pending.package) as reader: context = json.loads(reader.read_bytes(reader.manifest["artifacts"]["hypothesis_context"])) original_body["choices"][0]["message"]["content"] = json.dumps(proposals(context)) cfg = GatewayConfig((ProviderConfig(id="http",model="model",base_url=url,response_format="json_schema"),)) # New LLM operation, same pinned generation policy, after the existing retry's due time. result = run(HypothesisBuilder(store,LLMGateway(cfg)),attempt_id="http") assert result.status == "COMPLETED" assert seen[0]["body"]["response_format"]["json_schema"]["schema"]["properties"]["hypotheses"]["maxItems"] == 5 def test_summary_escapes_untrusted_text(interpreted_case): def change(payload,context): payload["hypotheses"][0]["title"] = ' [link](javascript:x)\n# injected' result = run(HypothesisBuilder(interpreted_case[0],gateway(ProposalAdapter(change)))) with PackageReader(result.package) as reader: summary = reader.read_bytes(result.summary_ref).decode() assert '