Recover joint planning dependencies and preserve valid hypothesis siblings
This commit is contained in:
1 parent
83ef9e6d6c
commit
c3a4690a55
15 files changed
+237
-10
No files matched your search
@@ -97,3 +97,28 @@ Sperren erhaltener Vorschläge, falsch geformte Modellfelder, Abbruch/Wiederaufn
|
||||
Import in einen neuen Workspace ab. Ein Fall des unveränderten kuratierten Fehlerkorpus
|
||||
prüft die Lokalisierung der tatsächlich beobachteten Fehler. Kein bezahlter API-Test
|
||||
wurde ausgeführt; Kosten und semantische Qualität sind im nächsten Realtest zu messen.
|
||||
|
||||
|
||||
## Dependency recovery revision 1
|
||||
|
||||
New joint contexts freeze `joint_generation.recovery_version = 1`. Contexts without
|
||||
this marker retain the earlier projection and request behavior for replay.
|
||||
Valid explicit premise references are unioned into their proposal dependency lists
|
||||
on a copied projection, never by editing the saved LLM answer. Only existing readable
|
||||
source IDs and exact existing interpretation statement pointers are eligible. Unknown
|
||||
references and grounded unread references still fail validation. No premise is promoted
|
||||
from unverified to grounded, and no claim is deleted or rewritten.
|
||||
|
||||
Each candidate is checked independently before freezing `joint_retained` in the
|
||||
hypothesis context. Initial candidates based on an already accepted interpretation
|
||||
can be retained directly; after interpretation repair, explicit dependency review is
|
||||
still required. Invalid candidates remain unlocked. Duplicate candidates are not locked
|
||||
against a necessary diversity correction. A followup corrects only unlocked positions;
|
||||
locked proposals are enforced with schema constants and local equality checks. Final
|
||||
publication still validates the full set for count, grounding, primary focus and diversity.
|
||||
|
||||
The joint prompt repeats the required `described_behavior.verification` value. Missing
|
||||
verification is not silently filled: the scoped statement correction must provide it.
|
||||
The original 2026-10-20 replies and a bounded validation context are preserved in the
|
||||
curated rejection corpus. Offline regression checks hashes, reference union, sibling
|
||||
preservation and rejection of invented references. No paid API request was needed.
|
||||
@@ -41,6 +41,28 @@ def _normalized(value):
|
||||
return re.sub(r"[\W_]+", " ", unicodedata.normalize("NFKC", value).casefold()).strip()
|
||||
|
||||
|
||||
def validate_individual(proposal, context, position):
|
||||
"""Validate a candidate without accepting the set's count/diversity prematurely."""
|
||||
from jsonschema import Draft202012Validator
|
||||
schema = output_schema(context['policy'])
|
||||
if not Draft202012Validator(schema).evolve(schema=schema['$defs']['proposal']).is_valid(proposal):
|
||||
raise HypothesisError('HYPOTHESIS_SCHEMA')
|
||||
sources=context['case_context']['sources']; interpreted=context['interpretation_sources']
|
||||
if any(r not in sources for r in proposal['source_ids']) or any(r not in interpreted for r in proposal['interpretation_ids']):
|
||||
raise HypothesisError('HYPOTHESIS_SOURCE_UNKNOWN')
|
||||
kinds={sources[r]['kind'] for r in proposal['source_ids']}
|
||||
if not kinds & (OBSERVATIONS | {'fact'}):raise HypothesisError('HYPOTHESIS_GROUNDING')
|
||||
if not _normalized(proposal['title']) or not _normalized(proposal['mechanism']):
|
||||
raise HypothesisError('HYPOTHESIS_DUPLICATE')
|
||||
if position==0:
|
||||
focus=set()
|
||||
for kind in ('first_known_divergence','failure_stage','observed_behavior'):
|
||||
focus={k for k,v in interpreted.items() if v['kind']==kind}
|
||||
if focus:break
|
||||
if not kinds & OBSERVATIONS or not set(proposal['interpretation_ids']) & focus:
|
||||
raise HypothesisError('HYPOTHESIS_PRIMARY_FOCUS')
|
||||
|
||||
|
||||
def validate_proposals(payload, context):
|
||||
try:
|
||||
validate(payload, "hypotheses-output")
|
||||
|
||||
@@ -32,5 +32,5 @@ class HypothesisPromptBuilder(PromptBuilder):
|
||||
from .json_consolidation import consolidate
|
||||
model_context = consolidate(model_context)
|
||||
return {"prompt_version": version, "privacy": "configured", "output_schema": schema,
|
||||
"messages": [{"role": "system", "content": files("viano_bug_analysis.llm.prompts").joinpath(version + ".txt").read_text(encoding="utf-8") + ("\nPreserve joint_retained.proposals exactly at their zero-based positions. Generate ONLY the other positions; return the full array for deterministic merging/validation. Retained proposals were explicitly reviewed; do not rewrite or reorder them." if context.get("joint_retained") else "")},
|
||||
"messages": [{"role": "system", "content": files("viano_bug_analysis.llm.prompts").joinpath(version + ".txt").read_text(encoding="utf-8") + ("\nPreserve joint_retained.proposals exactly at their zero-based positions. Generate ONLY the other positions; return the full array for deterministic merging/validation. Retained proposals were explicitly reviewed; do not rewrite or reorder them." if context.get("joint_retained") else "") + ("\nRepair only invalid hypotheses at unlocked positions. Preserve their valid claims and mechanisms; correct the reported dependency/source defect. Do not regenerate accepted interpretation or locked candidates." if context.get("joint_retained") and context.get("case_context",{}).get("joint_generation",{}).get("recovery_version")==1 else "")},
|
||||
{"role": "user", "content": encode({"task": "generate_hypotheses", "input": model_context, "output_schema": schema})}]}
|
||||
@@ -134,7 +134,7 @@ class CaseInterpreter:
|
||||
from .hypothesis_builder import HypothesisBuilder
|
||||
prior = HypothesisBuilder(self.store, self.gateway, self.joint_config)._history(reader, checkpoint_chain(reader), revision)
|
||||
context["joint_generation"] = {"policy": self.joint_config.policy(), "prior_generations": prior}
|
||||
if self.joint_version==2:context["joint_generation"]["version"]=2
|
||||
if self.joint_version==2:context["joint_generation"].update(version=2, recovery_version=1)
|
||||
prompt_version = self.prompt.version(context)
|
||||
chain = checkpoint_chain(reader)
|
||||
complete_id = f"case-interpreter:v1:revision:{revision}:complete"
|
||||
|
||||
@@ -51,7 +51,7 @@ def build_request(context):
|
||||
return {"prompt_version": version, "privacy": "configured",
|
||||
"output_schema": {"type": "object", "properties": {"interpretation": {}, "hypotheses": {}},
|
||||
"additionalProperties": False},
|
||||
"messages": [{"role": "system", "content": files("viano_bug_analysis.llm.prompts").joinpath(version + ".txt").read_text(encoding="utf-8")},
|
||||
"messages": [{"role": "system", "content": files("viano_bug_analysis.llm.prompts").joinpath(version + ".txt").read_text(encoding="utf-8") + ("\nFor every described_behavior statement include verification=not_independently_verified. Each premise reference must also occur in its proposal dependency lists; declare the union.\n" if settings.get("recovery_version")==1 else "")},
|
||||
{"role": "user", "content": model_json({"task": "interpret_and_propose", "input": consolidate(compact_context(view)),
|
||||
"policy": settings["policy"], "prior_generations": settings["prior_generations"],
|
||||
"block_contracts": {"interpretation": interpretation, "hypotheses": hypotheses}})}]}
|
||||
|
||||
@@ -59,6 +59,46 @@ def get(value, path):
|
||||
return value
|
||||
|
||||
|
||||
def recovery_enabled(context):
|
||||
return enabled(context) and context['joint_generation'].get('recovery_version') == 1
|
||||
|
||||
|
||||
def complete_dependencies(raw, interpretation, context):
|
||||
"""Union explicit valid premise dependencies; never change claims or invent refs."""
|
||||
result = copy.deepcopy(raw)
|
||||
if not recovery_enabled(context):
|
||||
return result
|
||||
from .joint_planning import schemas
|
||||
schema = schemas(context)[1]
|
||||
item = Draft202012Validator(schema).evolve(schema=schema['$defs']['proposal'])
|
||||
for proposal in result.get('hypotheses', []):
|
||||
if not item.is_valid(proposal):
|
||||
continue
|
||||
sources = list(proposal['source_ids'])
|
||||
paths = list(proposal['interpretation_paths'])
|
||||
for premise in proposal['premises']:
|
||||
sr = premise['source_ids']; ip = premise['interpretation_paths']
|
||||
# Validate every supplied reference before adding any dependency.
|
||||
if any(ref not in context['sources'] for ref in sr):
|
||||
continue
|
||||
try:
|
||||
for path in ip:
|
||||
parts = path.split('/')
|
||||
if len(parts) not in (3,4) or parts[1] != 'interpretation' or (len(parts)==4 and not parts[3].isdigit()):
|
||||
raise ValueError('not a statement pointer')
|
||||
value = get({'interpretation': interpretation}, path)
|
||||
if not isinstance(value, dict):raise ValueError('not a statement')
|
||||
except (KeyError, IndexError, ValueError, TypeError):
|
||||
continue
|
||||
for ref in sr:
|
||||
if ref not in sources:sources.append(ref)
|
||||
for path in ip:
|
||||
if path not in paths:paths.append(path)
|
||||
proposal['source_ids'] = sources
|
||||
proposal['interpretation_paths'] = paths
|
||||
return result
|
||||
|
||||
|
||||
def validate_premises(raw, interpretation, context):
|
||||
if not isinstance(raw,dict):raise HypothesisError("HYPOTHESIS_SCHEMA")
|
||||
for proposal in raw.get("hypotheses",[]):
|
||||
@@ -246,8 +286,13 @@ def candidate(reader,call,context,strict=True,depth=0):
|
||||
raw=normalize_block(raw,schemas(context)[1])
|
||||
if not Draft202012Validator(schemas(context)[1]).is_valid(raw):
|
||||
if strict:raise HypothesisError("HYPOTHESIS_SCHEMA")
|
||||
return None
|
||||
return raw
|
||||
if not recovery_enabled(context) or not isinstance(raw,dict) or not isinstance(raw.get('hypotheses'),list):return None
|
||||
count=len(raw['hypotheses']);policy=context['joint_generation']['policy']
|
||||
if not policy['min_hypotheses']<=count<=policy['max_hypotheses']:return None
|
||||
schema=schemas(context)[1]
|
||||
validator=Draft202012Validator(schema).evolve(schema=schema['$defs']['proposal'])
|
||||
raw={'hypotheses':[p if validator.is_valid(p) else None for p in raw['hypotheses']]}
|
||||
return complete_dependencies(raw, interpretation_payload(call,context,reader,depth), context)
|
||||
if version!=REPAIR:
|
||||
if strict:raise HypothesisError("HYPOTHESIS_DEPENDENCY_UNCERTAIN")
|
||||
return None
|
||||
@@ -279,22 +324,28 @@ def candidate(reader,call,context,strict=True,depth=0):
|
||||
result['hypotheses'].append(copy.deepcopy(proposal))
|
||||
elif review['decision']=='replace' and review['replacement'] is not None:
|
||||
replacements=review['replacement'].get('hypotheses',[])
|
||||
if not isinstance(replacements,list) or len(replacements)!=1:raise HypothesisError("HYPOTHESIS_SCHEMA")
|
||||
if not isinstance(replacements,list) or len(replacements)!=1:
|
||||
if not strict and recovery_enabled(context):
|
||||
result['hypotheses'].append(None)
|
||||
continue
|
||||
raise HypothesisError("HYPOTHESIS_SCHEMA")
|
||||
result['hypotheses'].append(copy.deepcopy(replacements[0]))
|
||||
else:
|
||||
if strict:raise HypothesisError("HYPOTHESIS_DEPENDENCY_UNCERTAIN")
|
||||
result['hypotheses'].append(None)
|
||||
return result
|
||||
return complete_dependencies(result, new, context)
|
||||
|
||||
|
||||
def retained_context(reader,context):
|
||||
"""Lock only explicitly reviewed independent proposals; never infer from missing refs."""
|
||||
"""Lock validated candidates; changed interpretations still require explicit review."""
|
||||
from .joint_planning import joint_call,schemas
|
||||
source=joint_call(reader,context)
|
||||
if source is None or source['outcome']['metadata']['prompt_version']!=REPAIR:return {}
|
||||
if source is None:return {}
|
||||
if not recovery_enabled(context['case_context']) and source['outcome']['metadata']['prompt_version']!=REPAIR:return {}
|
||||
try:raw=candidate(reader,source,context['case_context'],strict=False)
|
||||
except (HypothesisError,InterpretationError):return {}
|
||||
if not raw or all(p is not None for p in raw['hypotheses']):return {}
|
||||
if not raw:return {}
|
||||
if not recovery_enabled(context['case_context']) and all(p is not None for p in raw['hypotheses']):return {}
|
||||
count=len(raw['hypotheses'])
|
||||
if not context['policy']['min_hypotheses']<=count<=context['policy']['max_hypotheses']:return {}
|
||||
mapping={v['interpretation_pointer']:k for k,v in context['interpretation_sources'].items()}
|
||||
@@ -307,5 +358,19 @@ def retained_context(reader,context):
|
||||
validate_premises({'hypotheses':[proposal]}, accepted['interpretation'], context['case_context'])
|
||||
p=copy.deepcopy(proposal);p.pop('premises');p['interpretation_ids']=[mapping[path] for path in p.pop('interpretation_paths')]
|
||||
except (HypothesisError,KeyError):continue
|
||||
if recovery_enabled(context['case_context']):
|
||||
try:
|
||||
from .hypothesis_contracts import validate_individual
|
||||
validate_individual(p,context,i)
|
||||
except HypothesisError:continue
|
||||
locked[str(i)]=p
|
||||
# Do not lock members of a duplicate set: the followup must be able to restore diversity.
|
||||
if recovery_enabled(context['case_context']):
|
||||
from .hypothesis_contracts import _normalized
|
||||
for field in ('title','mechanism'):
|
||||
groups={}
|
||||
for position,proposal in locked.items():groups.setdefault(_normalized(proposal[field]),[]).append(position)
|
||||
for positions in groups.values():
|
||||
if len(positions)>1:
|
||||
for position in positions:locked.pop(position,None)
|
||||
return {'count':count,'proposals':locked,'source_ref':source['result_ref']} if locked else {}
|
||||
@@ -120,6 +120,9 @@
|
||||
"enum": [
|
||||
2
|
||||
]
|
||||
},
|
||||
"recovery_version": {
|
||||
"const": 1
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
|
||||
@@ -129,6 +129,9 @@
|
||||
"enum": [
|
||||
2
|
||||
]
|
||||
},
|
||||
"recovery_version": {
|
||||
"const": 1
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
# BUG-000020 joint v2, 2026-10-20
|
||||
|
||||
Original saved call documents, unchanged bytes. No credentials or full package archive.
|
||||
Initial interpretation omitted described_behavior[0].verification. The scoped patch added
|
||||
not_independently_verified without changing the statement text. Two candidates were reviewed
|
||||
as independent; one was explicitly replaced. Hypothesis acceptance then failed because
|
||||
premise interpretation_paths were missing from their parent dependency lists (all three
|
||||
candidates reference investigation_direction from a premise without declaring it above).
|
||||
These are existing, readable statement references, not invented evidence. Unioning validated
|
||||
explicit references is safe dependency bookkeeping, not semantic endorsement.
|
||||
|
||||
Unknown paths/sources, unread grounded evidence, unreviewed semantic dependencies and changed
|
||||
retained inputs remain rejected. Regression replays this corpus offline. Original replies stay
|
||||
immutable; normalized projections are derived. Retaining a candidate never accepts the whole set
|
||||
without the existing diversity, grounding and count validation.
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"analysis_revision": 1, "case_id": "BUG-000020", "case_ref": "case/revisions/0001/case.json", "case_sha256": "806e276fa3a2c81766fc8ceba13080aca82347f90240d5466790b8a6104bbdf1", "context_ref": "analysis/interpretations/0001/context.json", "context_sha256": "7b026a4d0166bf5b18ee88691721077a710090a752cc1fad1d11dfdda7b4efba", "expected_behavior_completeness": "missing", "expected_behavior_provenance": "none", "format": "viano-case-interpretation", "format_version": 1, "interpretation": {"behavior_gap": null, "confirmed_technical_facts": [{"source_ids": ["F0001", "F0002", "F0003", "F0004", "F0005", "F0006", "F0007", "F0008", "F0009", "F0010", "F0011"], "text": "Reproduction test reproduced the issue at Ironforge/Dun Morogh coordinates (-5178.33, -577.744, 397.622, orientation 5.45) on map 0, zone 1, area 1 under diagnostic focus quest 11117."}, {"source_ids": ["F0014", "F0015", "F0019"], "text": "Creature template 23486 is named Goldark Snipehunter using SmartAI; its smart_scripts handle gossip option 0 by casting spell 66592 (Create Item - Synthebrew Goggles) on target and closing gossip."}, {"source_ids": ["F0016", "F0018", "F0024", "F0025", "F0026", "F0027"], "text": "Quest template 11117 ('Catch the Wild Wolpertinger!') specifies StartItem 32907, RequiredItemId1 32906, RequiredItemId2 32907, and 0 for RequiredNpcOrGo1-4 and RequiredSpell."}, {"source_ids": ["F0020", "F0021"], "text": "Item script name lookup in item_script_names returned no rows for item 32906 and item 32907."}, {"source_ids": ["F0022"], "text": "Querying creature_queststarter for quest 11117 returned no rows in the queried snapshot."}, {"source_ids": ["F0029", "F0030", "F0031"], "text": "Creature template entry 23487 exists with name 'Wild Wolpertinger' (German locale 'Wilder Wolpertinger'), min/max level 1, faction 188, unit_flags 512, and AIName SmartAI, distinct from pet/base creature entry 22943 ('Wolpertinger')."}], "contradictions": [], "described_behavior": [{"basis": "local_description", "source_ids": ["F0017"], "text": "Quest 11117 text indicates players must capture 5 Stunned Wolpertingers using a net while intoxicated or wearing Synthebrew Goggles to see them in the hills around Brewfest.", "verification": "not_independently_verified"}], "expected_behavior": [], "failure_stage": null, "first_known_divergence": null, "interpretation_summary": {"source_ids": ["B0002", "O0002", "F0001", "F0017"], "text": "During Brewfest outside Ironforge, tester reported that Wild Wolpertingers do not spawn even when wearing Synthebrew Goggles, contrasting with the Horde counterpart in Orgrimmar which functioned properly."}, "investigation_direction": [{"source_ids": ["O0002", "F0029"], "text": "Investigate creature spawn records, phase definitions, game event creature spawns, and visibility/auras for creature template 23487 around Dun Morogh."}, {"source_ids": ["F0019", "F0016"], "text": "Investigate item 32906, item 32907, spell 66592, and the invisibility/stealth detection aura required to perceive Wild Wolpertingers."}], "observed_behavior": [{"source_ids": ["B0002", "O0001", "O0003"], "text": "Tester reports that Wolpertingers do not spawn at the Brewfest location."}, {"source_ids": ["O0002"], "text": "Tester observed that in Ironforge Brewfest no Wolpertingers spawn even while wearing the necessary goggles, whereas in Orgrimmar the Horde equivalent quest worked and Wolpertingers spawned correctly."}], "provisional_behavior_gap": {"basis": "local_description", "source_ids": ["F0017", "O0002"], "status": "provisional", "text": "Quest description describes spotting Wolpertingers when intoxicated or wearing Synthebrew Goggles, but tester observed no Wolpertingers spawning or visible at the Dun Morogh/Ironforge Brewfest grounds despite wearing goggles."}, "relevant_entities": [{"entity_id": "N0001", "reason": "NPC quest contact Goldark Snipehunter providing the quest dialogue and Synthebrew Goggles.", "source_ids": ["B0001", "F0014", "F0015", "F0019"]}, {"entity_id": "N0002", "reason": "Recorded entity: item 32906. Display name and functional identity are not verified.", "source_ids": ["F0016", "F0025"]}, {"entity_id": "N0003", "reason": "Recorded entity: item 32907. Display name and functional identity are not verified.", "source_ids": ["F0016", "F0024", "F0026"]}, {"entity_id": "N0004", "reason": "Brewfest quest 11117 'Catch the Wild Wolpertinger!' under diagnostic focus.", "source_ids": ["B0001", "F0009", "F0011", "F0016", "F0018"]}, {"entity_id": "N0005", "reason": "Target wild creature entry 23487 ('Wild Wolpertinger') sought during quest 11117.", "source_ids": ["B0002", "O0001", "O0002", "O0003", "F0017", "F0029", "F0030", "F0031"]}], "tester_assumptions": [], "unknowns": [{"source_ids": ["Q0001"], "text": "Expected behavior completeness is missing; full authoritative specification of Wolpertinger spawning mechanics and visibility conditions is not yet verified."}, {"source_ids": ["F0022"], "text": "Why creature_queststarter returned emptyLine truncated
|
||||
Vendored
+1
@@ -0,0 +1 @@
|
||||
{"analysis_revision": 1, "case_id": "BUG-000020", "code": null, "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", "format": "viano-llm-call", "format_version": 1, "input_request_hash": "b859b2868433a9840fb5d530494d43e7d3baaf93af41cc5e26f1463451074a57", "operation": "llm-round", "operation_key": "cdf9fddd5dccf7c6859f2f4c5aa65e46778d6d44e446cf579fa6b6f462b9b85c", "outcome": {"document": {"patches": [{"path": "/described_behavior/0", "replacement": {"basis": "local_description", "source_ids": ["F0017"], "text": "Quest 11117 text indicates players must capture 5 Stunned Wolpertingers using a net while intoxicated or wearing Synthebrew Goggles to see them in the hills around Brewfest.", "verification": "not_independently_verified"}}], "reviews": [{"decision": "retain", "position": 0, "reason": "Hypothesis 0 relies exclusively on /interpretation/observed_behavior/0 and /interpretation/confirmed_technical_facts/5, both of which remain completely unchanged. The repaired statement at /described_behavior/0 is not a prerequisite.", "replacement": null}, {"decision": "replace", "position": 1, "reason": "Hypothesis 1 directly declares /interpretation/described_behavior/0 as an input path and premise. Because that statement was modified during repair, the hypothesis is resubmitted to revalidate against the repaired interpretation while preserving its mechanism and scope.", "replacement": {"hypotheses": [{"distinction": "Focuses on stealth, invisibility, or phase mask mismatches preventing spawned creatures from being rendered to players wearing goggles.", "expected_evidence": ["Creature 23487 has an invisibility flag, stealth level, or phase setting that is not counteracted by the Synthebrew Goggles buff or drunkenness in Dun Morogh."], "falsifying_evidence": ["Creature 23487 has no invisibility flags or phases, or the goggles aura matches the creature's detection requirement exactly."], "interpretation_paths": ["/interpretation/observed_behavior/1", "/interpretation/described_behavior/0"], "mechanism": "Wild Wolpertingers are spawned in Dun Morogh but possess an invisibility type, phase mask, or stealth value that the Alliance Synthebrew Goggles buff fails to detect, making them appear completely absent to the tester.", "open_questions": ["What aura or spell does Synthebrew Goggles apply to the player, and what invisibility aura/type does creature 23487 use?", "Does creature 23487 use phasing or conditions in Dun Morogh?"], "premises": [{"interpretation_paths": ["/interpretation/observed_behavior/1"], "kind": "grounded", "source_ids": ["O0002"], "text": "Tester wore the goggles at Ironforge Brewfest but could not see Wolpertingers."}, {"interpretation_paths": ["/interpretation/described_behavior/0"], "kind": "grounded", "source_ids": ["F0017"], "text": "Quest description states players must be intoxicated or wearing Synthebrew Goggles to see Wolpertingers."}, {"interpretation_paths": ["/interpretation/investigation_direction/1"], "kind": "unverified", "source_ids": ["F0019"], "text": "The detection spell or buff granted by the goggles does not match the stealth/invisibility layer applied to creature 23487."}], "rationale": "Because the quest lore states Wolpertingers can only be seen with goggles or drunkenness, they may be physically present in the database but invisible due to an aura or phase mismatch.", "source_ids": ["O0002", "F0017", "F0019"], "title": "Invisibility Detection or Phasing Mismatch for Wild Wolpertinger"}]}}, {"decision": "retain", "position": 2, "reason": "Hypothesis 2 relies exclusively on /interpretation/observed_behavior/1 and /interpretation/confirmed_technical_facts/0, which remain unchanged. The repaired statement at /described_behavior/0 is not a prerequisite.", "replacement": null}]}, "metadata": {"attempts": [], "cache_created_tokens": 0, "cache_status": "below_minimum", "cached_input_tokens": null, "cost_currency": "USD", "elapsed_seconds": 39.812, "estimated_cache_storage_cost": 0.0, "estimated_cached_input_cost": null, "estimated_cost": null, "estimated_input_cost": null, "estimated_output_cost": 0.02258625, "input_tokens": 7116, "model": "gemini-3.8-flash", "output_tokens": 6023, "prompt_version": "case_plan_statement_repair_v2", "provider": "gemini-native", "reasoning_tokens": 5269, "uncached_input_tokens": null}, "response_assessment": {"decision": "protocol_valid", "document_retained": true, "raw_retained": true, "version": 1}, "response_capture": {"bytes": 3297, "complete": true, "retained": true, "sha256": "3fb4618572575f4e15e112c3d5a328a022ce33a670e8aeee56ea3748fafa6740", "text": "{\"patches\":[{\"path\":\"/described_behavior/0\",\"replacement\":{\"basis\":\"local_description\",\"source_ids\":[\"F0017\"],\"text\":\"Quest 11117 text indicates players must capture 5 Stunned Wolpertingers using a net while intoxicated or wearing Synthebrew Goggles to see them in the hills around Brewfest.\",\"verification\":\"not_independently_verified\"}}],\"reviewsLine truncated
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"analysis_revision": 1, "case_id": "BUG-000020", "code": null, "config_hash": "249a08e844d8a62c6b202c0bf856d2fa67f9c77f7a1f03174e79f874bf24ac69", "format": "viano-llm-call", "format_version": 1, "input_request_hash": "1a806d62256d76c185b7b4a4e63cc218cf5866e3f1adc6d496628fab12d619db", "operation": "llm-round", "operation_key": "572afccbd4034d9c5485045cb4bb28b61fdd40e48dc8a3fab67cfc17e997cd65", "outcome": {"document": {"hypotheses": {"hypotheses": [{"distinction": "Focuses on missing world spawn points or game_event_creature associations for creature 23487 in Dun Morogh rather than visibility aura issues.", "expected_evidence": ["Querying table creature or game_event_creature for id 23487 in map 0 Dun Morogh returns zero rows or disabled game event IDs."], "falsifying_evidence": ["Table creature or game_event_creature contains active spawn records for entry 23487 near the Dun Morogh Brewfest site matching the active Brewfest event."], "interpretation_paths": ["/interpretation/observed_behavior/0", "/interpretation/confirmed_technical_facts/5"], "mechanism": "Wild Wolpertingers (entry 23487) lack active creature spawn entries or game_event_creature links for the Alliance Brewfest location in Dun Morogh, meaning no entities exist in the world to be seen.", "open_questions": ["Are creature spawns for entry 23487 present in table creature or game_event_creature for map 0?", "How are the Horde Wolpertingers spawned in Durotar compared to Dun Morogh?"], "premises": [{"interpretation_paths": ["/interpretation/observed_behavior/0"], "kind": "grounded", "source_ids": ["B0002", "O0001"], "text": "Tester observed that Wolpertingers do not spawn at the Brewfest location."}, {"interpretation_paths": ["/interpretation/confirmed_technical_facts/5"], "kind": "grounded", "source_ids": ["F0029"], "text": "Entry 23487 exists in creature_template as 'Wild Wolpertinger'."}, {"interpretation_paths": ["/interpretation/investigation_direction/0"], "kind": "unverified", "source_ids": ["O0002"], "text": "Spawn records in table creature or game_event_creature for creature 23487 in Dun Morogh are missing or misconfigured."}], "rationale": "The tester reported that Wolpertingers do not spawn in Ironforge, whereas the Orgrimmar equivalent worked. If Dun Morogh simply lacks spawn table entries for creature 23487, no creatures will appear regardless of player items or buffs.", "source_ids": ["B0002", "O0001", "O0002", "F0029"], "title": "Missing Dun Morogh Creature Spawns for Wild Wolpertinger"}, {"distinction": "Focuses on stealth, invisibility, or phase mask mismatches preventing spawned creatures from being rendered to players wearing goggles.", "expected_evidence": ["Creature 23487 has an invisibility flag, stealth level, or phase setting that is not counteracted by the Synthebrew Goggles buff or drunkenness in Dun Morogh."], "falsifying_evidence": ["Creature 23487 has no invisibility flags or phases, or the goggles aura matches the creature's detection requirement exactly."], "interpretation_paths": ["/interpretation/observed_behavior/1", "/interpretation/described_behavior/0"], "mechanism": "Wild Wolpertingers are spawned in Dun Morogh but possess an invisibility type, phase mask, or stealth value that the Alliance Synthebrew Goggles buff fails to detect, making them appear completely absent to the tester.", "open_questions": ["What aura or spell does Synthebrew Goggles apply to the player, and what invisibility aura/type does creature 23487 use?", "Does creature 23487 use phasing or conditions in Dun Morogh?"], "premises": [{"interpretation_paths": ["/interpretation/observed_behavior/1"], "kind": "grounded", "source_ids": ["O0002"], "text": "Tester wore the goggles at Ironforge Brewfest but could not see Wolpertingers."}, {"interpretation_paths": ["/interpretation/described_behavior/0"], "kind": "grounded", "source_ids": ["F0017"], "text": "Quest description states players must be intoxicated or wearing Synthebrew Goggles to see Wolpertingers."}, {"interpretation_paths": ["/interpretation/investigation_direction/1"], "kind": "unverified", "source_ids": ["F0019"], "text": "The detection spell or buff granted by the goggles does not match the stealth/invisibility layer applied to creature 23487."}], "rationale": "Because the quest lore states Wolpertingers can only be seen with goggles or drunkenness, they may be physically present in the database but invisible due to an aura or phase mismatch.", "source_ids": ["O0002", "F0017", "F0019"], "title": "Invisibility Detection or Phasing Mismatch for Wild Wolpertinger"}, {"distinction": "Focuses on event-activation conditions in game_event or conditions table tying spawn availability to the wrong seasonal event or holiday flag.", "expected_evidence": ["Database records linking creature 23487 to game_event have a holiday or event ID that is inactive or differs between Horde and Alliance Brewfest areas."], "falsifying_evidence": ["The event ID tied to Dun Morogh Wolpertingers is active and mLine truncated
|
||||
@@ -0,0 +1 @@
|
||||
{"case": {"analysis_revision": 1, "case_id": "BUG-000020", "package_id": "trial-joint-v2-BUG-000020-20261020", "project": {"profile": "vcore", "type": "wow"}, "source_generation": 3}, "case_ref": "case/revisions/0001/case.json", "case_sha256": "806e276fa3a2c81766fc8ceba13080aca82347f90240d5466790b8a6104bbdf1", "context_selection": "phase_selection_v1", "expected_behavior_completeness": "missing", "expected_behavior_provenance": "none", "format": "viano-case-interpretation-context", "format_version": 1, "joint_generation": {"policy": {"budget_unit": "investigation_units", "format": "viano-hypothesis-policy", "format_version": 1, "initial_budget": 100, "max_hypotheses": 5, "min_hypotheses": 3}, "prior_generations": [], "version": 2}, "model_encoding": "shared_json_v1", "sources": {"B0001": {"case_pointer": "/bug/reports/0", "data": {"origin": {"archive_ref": "original/vcore/imports/768ba7813ccc3ac656328e895d4abcc8d56822c2976a149a95e99cf91f8bb096/wowdiag/archive.tar.gz", "json_pointer": "/0", "member_name": "BUG-000020-20261008-130429/bug/reports.json", "source_ref": "original/vcore/imports/768ba7813ccc3ac656328e895d4abcc8d56822c2976a149a95e99cf91f8bb096/wowdiag/members/BUG-000020-20261008-130429/bug/reports.json"}, "source_ref": "case/inputs/vcore/768ba7813ccc3ac656328e895d4abcc8d56822c2976a149a95e99cf91f8bb096/wowdiag/report-32.json", "summary": null, "title": null}, "kind": "report"}, "B0002": {"case_pointer": "/bug/reports/1", "data": {"origin": {"archive_ref": "original/vcore/imports/768ba7813ccc3ac656328e895d4abcc8d56822c2976a149a95e99cf91f8bb096/wowdiag/archive.tar.gz", "json_pointer": "", "member_name": "BUG-000020-20261008-130429/bug/bug.json", "source_ref": "original/vcore/imports/768ba7813ccc3ac656328e895d4abcc8d56822c2976a149a95e99cf91f8bb096/wowdiag/members/BUG-000020-20261008-130429/bug/bug.json"}, "source_ref": "case/inputs/vcore/768ba7813ccc3ac656328e895d4abcc8d56822c2976a149a95e99cf91f8bb096/wowdiag/bug-record.json", "summary": "Wolpertinger spawnen nicht", "title": "Wolpertinger spawnen nicht"}, "kind": "report"}, "B0003": {"case_pointer": "/bug/reports/2", "data": {"source_ref": "case/inputs/vcore/768ba7813ccc3ac656328e895d4abcc8d56822c2976a149a95e99cf91f8bb096/source-quality.json", "summary": null, "title": null}, "kind": "report"}, "D0001": {"case_pointer": "/data_quality/issues/0", "data": {"code": "SOURCE_TIME_UNQUALIFIED", "message": "Source time has no usable timezone; retained without conversion", "refs": [{"json_pointer": "/0/description", "source_ref": "original/vcore/imports/768ba7813ccc3ac656328e895d4abcc8d56822c2976a149a95e99cf91f8bb096/wowdiag/members/BUG-000020-20261008-130429/bug/reports.json"}]}, "kind": "data_quality"}, "D0002": {"case_pointer": "/data_quality/issues/1", "data": {"code": "SOURCE_TIME_UNQUALIFIED", "message": "Source time has no usable timezone; retained without conversion", "refs": [{"json_pointer": "/0/observation", "source_ref": "original/vcore/imports/768ba7813ccc3ac656328e895d4abcc8d56822c2976a149a95e99cf91f8bb096/wowdiag/members/BUG-000020-20261008-130429/bug/reproductions.json"}]}, "kind": "data_quality"}, "D0003": {"case_pointer": "/data_quality/issues/2", "data": {"code": "SOURCE_TIME_UNQUALIFIED", "message": "Source time has no usable timezone; retained without conversion", "refs": [{"json_pointer": "/1/data_json", "source_ref": "original/vcore/imports/768ba7813ccc3ac656328e895d4abcc8d56822c2976a149a95e99cf91f8bb096/wowdiag/members/BUG-000020-20261008-130429/bug/reproduction_events.json"}]}, "kind": "data_quality"}, "D0004": {"case_pointer": "/data_quality/issues/3", "data": {"code": "EXPECTED_BEHAVIOR_MISSING", "message": "Expected-behavior completeness is missing", "refs": []}, "kind": "data_quality"}, "D0005": {"case_pointer": "/interpretation_quality/policy", "data": {"citation_policy": "grouped_capture_v3", "code": "DESCRIPTION_PROVENANCE_POLICY", "meaning": "local_descriptions_are_not_independently_verified_expectations"}, "kind": "data_quality"}, "D0006": {"case_pointer": "/interpretation_quality/model_context", "data": {"model_context_policy": "compact_storage_v1"}, "kind": "data_quality"}, "D0007": {"case_pointer": "/interpretation_quality/name_bindings", "data": {"name_binding_policy": "typed_name_bindings_v1"}, "kind": "data_quality"}, "D0008": {"case_pointer": "/interpretation_quality/identity", "data": {"identity_policy": "explicit_names_v1", "meaning": "numeric_references_do_not_bind_descriptive_names"}, "kind": "data_quality"}, "F0001": {"case_pointer": "/runtime_facts/facts/0", "data": {"id": "F0001", "name": "result", "provenance": {"author": null, "json_pointer": "/facts/0", "kind": "reproduction_record", "observed_at": null, "origin": {"archive_ref": "original/vcore/imports/768ba7813ccc3ac656328e895d4abcc8d56822c2976a149a95e99cf91f8bb096/wowdiag/archive.tar.gz", "json_pointer": "/0/result", "member_name": "BUG-000020-20261008-130429/bug/reproductions.json", "source_ref": "original/vcore/imports/768ba7813ccc3ac656328e895d4abcc8d56Line truncated
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"case_interpretation.json": "d884dd21ee2dca9d3e6fc464c9dfac7dd27750c39682a2e4f89d034c5aafc8a7",
|
||||
"case_plan_statement_repair_v2.json": "c2ea0441d010892218604a27cb1246bdcf1691f31c228f811e272db8e76fc7c9",
|
||||
"case_plan_v2.json": "07eb2df391d4abca1c85733f8fc4f2559dcc3c71033c92022cfc70b3812e4f94",
|
||||
"context.json": "7b026a4d0166bf5b18ee88691721077a710090a752cc1fad1d11dfdda7b4efba"
|
||||
}
|
||||
@@ -208,3 +208,87 @@ def test_bad_review_does_not_discard_valid_interpretation_patch(built_case,malfo
|
||||
assert interpret(store,gw).status=='COMPLETED'
|
||||
assert hypotheses(store,gw).status=='REJECTED'
|
||||
with PackageReader(store.latest('case-A','stable-package')) as r:assert r.manifest['artifacts']['case_interpretation']
|
||||
|
||||
|
||||
def test_valid_premise_dependencies_are_unioned_without_mutating_answer(built_case):
|
||||
def missing(d):
|
||||
h=d['hypotheses']['hypotheses'][0]
|
||||
h['premises'][0]['interpretation_paths'].append('/interpretation/observed_behavior/0')
|
||||
h['interpretation_paths']=['/interpretation/unknowns/0']
|
||||
store=built_case[0];adapter=Adapter(missing);gw=gateway(adapter)
|
||||
assert interpret(store,gw).status=='COMPLETED'
|
||||
assert hypotheses(store,gw).status=='COMPLETED'
|
||||
assert len(adapter.requests)==1
|
||||
|
||||
|
||||
def test_single_invalid_candidate_locks_valid_siblings(built_case):
|
||||
def bad(d):d['hypotheses']['hypotheses'][1]['premises'][0]['source_ids']=['F9999']
|
||||
store=built_case[0];adapter=Adapter(bad);gw=gateway(adapter)
|
||||
assert interpret(store,gw).status=='COMPLETED'
|
||||
result=hypotheses(store,gw);assert result.code=='HYPOTHESIS_PREMISE_SOURCE'
|
||||
with PackageReader(result.package) as r:
|
||||
c=json.loads(r.read_bytes(r.manifest['artifacts']['hypothesis_context']))
|
||||
assert set(c['joint_retained']['proposals'])=={'0','2'}
|
||||
assert hypotheses(store,gw).status=='COMPLETED'
|
||||
assert len(adapter.requests)==2
|
||||
|
||||
|
||||
def test_real_dependency_corpus_normalization_is_lossless_and_bounded():
|
||||
from pathlib import Path
|
||||
import hashlib
|
||||
from viano_bug_analysis.analysis.joint_v2 import complete_dependencies,validate_premises
|
||||
root=Path('tests/fixtures/llm_rejections/20261020/joint-v2-dependencies')
|
||||
for name,digest in json.loads((root/'hashes.json').read_text(encoding='utf-8')).items():
|
||||
assert hashlib.sha256((root/name).read_bytes()).hexdigest()==digest
|
||||
context=json.loads((root/'context.json').read_text(encoding='utf-8'))
|
||||
accepted=json.loads((root/'case_interpretation.json').read_text(encoding='utf-8'))['interpretation']
|
||||
doc=json.loads((root/'case_plan_v2.json').read_text(encoding='utf-8'))['outcome']['document']
|
||||
raw=doc['hypotheses'];before=copy.deepcopy(raw)
|
||||
context['joint_generation']['recovery_version']=1
|
||||
fixed=complete_dependencies(raw,accepted,context)
|
||||
validate_premises(fixed,accepted,context)
|
||||
from viano_bug_analysis.analysis.hypothesis_contracts import build_context,validate_proposals
|
||||
from viano_bug_analysis.analysis.hypothesis_config import HypothesisLimits
|
||||
accepted_doc=json.loads((root/'case_interpretation.json').read_text(encoding='utf-8'))
|
||||
hc=build_context(context,accepted_doc,'accepted.json',context['joint_generation']['policy'],[],HypothesisLimits())
|
||||
mapping={v['interpretation_pointer']:k for k,v in hc['interpretation_sources'].items()}
|
||||
canonical=copy.deepcopy(fixed)
|
||||
for proposal in canonical['hypotheses']:
|
||||
proposal.pop('premises')
|
||||
proposal['interpretation_ids']=[mapping[p] for p in proposal.pop('interpretation_paths')]
|
||||
validate_proposals(canonical,hc)
|
||||
assert raw==before
|
||||
for old,new in zip(raw['hypotheses'],fixed['hypotheses']):
|
||||
assert {k:v for k,v in old.items() if k not in {'source_ids','interpretation_paths'}}=={k:v for k,v in new.items() if k not in {'source_ids','interpretation_paths'}}
|
||||
assert set(old['interpretation_paths'])<=set(new['interpretation_paths'])
|
||||
del context['joint_generation']['recovery_version']
|
||||
assert complete_dependencies(raw,accepted,context)==raw
|
||||
|
||||
|
||||
def test_invalid_premise_path_is_not_promoted(built_case):
|
||||
def bad(d):d['hypotheses']['hypotheses'][0]['premises'][0]['interpretation_paths']=['/interpretation/unknowns/9999']
|
||||
store=built_case[0];adapter=Adapter(bad);gw=gateway(adapter)
|
||||
assert interpret(store,gw).status=='COMPLETED'
|
||||
assert hypotheses(store,gw).code=='HYPOTHESIS_PREMISE_SOURCE'
|
||||
|
||||
|
||||
def test_malformed_candidate_does_not_discard_siblings(built_case):
|
||||
def bad(d):d['hypotheses']['hypotheses'][1].pop('mechanism')
|
||||
store=built_case[0];adapter=Adapter(bad);gw=gateway(adapter)
|
||||
assert interpret(store,gw).status=='COMPLETED'
|
||||
result=hypotheses(store,gw);assert result.code=='HYPOTHESIS_SCHEMA'
|
||||
with PackageReader(result.package) as r:
|
||||
context=json.loads(r.read_bytes(r.manifest['artifacts']['hypothesis_context']))
|
||||
assert set(context['joint_retained']['proposals'])=={'0','2'}
|
||||
assert hypotheses(store,gw).status=='COMPLETED'
|
||||
|
||||
|
||||
def test_added_dependency_prevents_false_independence_after_patch(built_case):
|
||||
def bad(d):
|
||||
d['interpretation']['observed_behavior'][0]['source_ids']=['F0001']
|
||||
# Explicit premise points at changed statement, missing only from parent list.
|
||||
d['hypotheses']['hypotheses'][1]['premises'][0]['interpretation_paths']=['/interpretation/observed_behavior/0']
|
||||
store=built_case[0];adapter=Adapter(bad);gw=gateway(adapter)
|
||||
assert interpret(store,gw).status=='REJECTED'
|
||||
assert interpret(store,gw).status=='COMPLETED'
|
||||
assert hypotheses(store,gw).code=='HYPOTHESIS_DEPENDENCY_CHANGED'
|
||||
Reference in new issue
Block a user