Nico/VianoBugAnalysis
1
1 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Repair only invalid entity assessments with precise scoped feedback

Browse Source
This commit is contained in:
Nico committed 2026-10-24 21:18:51 +02:00
1 parent 33603db9bd
commit 4cbba4c1e4
7 files changed
+920 -33
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

docs/test-results/bug21-resolution-partial-repair-20261024.md
+13
View File
@@ -0,0 +1,13 @@
# BUG-000021 revision 2: scoped entity-assessment repair
Input: operator-provided `BUG-000021-Fallakte (1).zip`, revision 2, generation 43. No paid calls were executed for diagnosis or testing. The original package is unchanged.
All three assessments correctly selected Furien, entry 35286, from the complete base-name lookup. Each incorrectly used `no_match` for a nonempty, truncated German substring lookup containing unrelated harpy names. The validator correctly requires zero rows for `no_match`; previous feedback only supplied the generic `ENTITY_RESOLUTION_SELECTION` code.
The repair now freezes durable, index-specific diagnostics containing the actual status, row count, truncation, allowed candidates and citation constraints. Each structurally and semantically valid sibling entry is retained exactly. Only invalid entries are requested in the model response, with original search indices constrained by the local schema. The program merges them with frozen entries and runs the unchanged full assessment validation. Malformed patches retain the previously frozen entries and receive another bounded repair attempt; they cannot unlock an accepted sibling. No identity is selected automatically and no status is silently changed by string matching on the reason.
Feedback is saved before the call. Existing calls without version-2 feedback retain their exact original request. The projection is also applied when checking completed calls and replaying rejections, so persisted full assessments remain bound to the original partial response and its saved request. Limits and attempt budgets are unchanged. This patch does not silently reopen the already exhausted revision 2 or grant it another attempt.
Curated corpus: `tests/fixtures/llm_rejections/20261024/bug21-localized-no-match/`, approximately 18 KB. Usage snapshot: `docs/test-results/token-usage/bug21-revision2-resolution-blocked-20261024.json`; first-field-run snapshot remains separate. Missing cache/think-token values remain unknown; output totals do not add reasoning twice.
Validation: 60 tests passed across partial entity repair, legacy entity resolution and multidatabase research. Coverage includes the real rejected answer, strict rejection of invalid replacement sources/candidates/status, two-search completion, malformed patch recovery, restart without re-querying research, and portable replay without further model calls. Linux field verification remains pending deployment.
docs/test-results/token-usage/bug21-revision2-resolution-blocked-20261024.json
+150
View File
@@ -0,0 +1,150 @@
{
"case_id": "BUG-000021",
"analysis_revision": 2,
"package_generation": 43,
"result": "BLOCKED",
"reason": "ENTITY_RESOLUTION_INVALID_RESPONSES",
"recorded_at": "2026-10-24",
"scope": "revision 2 only; prior snapshot retained separately",
"calls": [
{
"ref": "llm/operations/2bb2e79d81a427683503f00f994096c132e4f3996cdd1250c22e65b5345c47b6/round-0001.json",
"status": "SUCCESS",
"code": null,
"config_hash": "692c8bac07efc915a44a89ac8aab1b4caa3e125d8619745f13c2ee570cb9e054",
"request_hash": "498c1d4c6fb3b32da6ad143ba78b533fbc7ce88d84ed656a26c659ede7a60e40",
"classification": "regular",
"metadata": {
"attempts": [],
"cache_created_tokens": 0,
"cache_status": "below_minimum",
"cached_input_tokens": null,
"cost_currency": "USD",
"elapsed_seconds": 2.865948,
"estimated_cache_storage_cost": 0.0,
"estimated_cached_input_cost": null,
"estimated_cost": null,
"estimated_input_cost": null,
"estimated_output_cost": 0.00043875,
"input_tokens": 20304,
"model": "gemini-3.8-flash",
"output_tokens": 117,
"prompt_version": "entity_resolution_plan_v4",
"provider": "gemini-native",
"reasoning_tokens": null,
"uncached_input_tokens": null
}
},
{
"ref": "llm/operations/f3b089e666430152253a7ad993e97035bd9b964286b1fad8fdc3df2f1815ee5c/round-0001.json",
"status": "SUCCESS",
"code": null,
"config_hash": "692c8bac07efc915a44a89ac8aab1b4caa3e125d8619745f13c2ee570cb9e054",
"request_hash": "a88dd9acdf5bdbfac2eb2f6962d20e77dcaa6c6e52d23ba2cc180bc11e1cc950",
"classification": "regular",
"metadata": {
"attempts": [],
"cache_created_tokens": 0,
"cache_status": "below_minimum",
"cached_input_tokens": null,
"cost_currency": "USD",
"elapsed_seconds": 3.737966,
"estimated_cache_storage_cost": 0.0,
"estimated_cached_input_cost": null,
"estimated_cost": null,
"estimated_input_cost": null,
"estimated_output_cost": 0.0010275,
"input_tokens": 23234,
"model": "gemini-3.8-flash",
"output_tokens": 274,
"prompt_version": "entity_resolution_assessment_v4",
"provider": "gemini-native",
"reasoning_tokens": null,
"uncached_input_tokens": null
}
},
{
"ref": "llm/operations/3237e2db6c942e4528382afeac6bbed66cce7c9b54a7c2bc8e1601ca9943b1fb/round-0001.json",
"status": "SUCCESS",
"code": null,
"config_hash": "692c8bac07efc915a44a89ac8aab1b4caa3e125d8619745f13c2ee570cb9e054",
"request_hash": "aa49d9159ae7894e5870e2c0abebded94877f7f23201741c79066ab37264d623",
"classification": "correction",
"metadata": {
"attempts": [],
"cache_created_tokens": 0,
"cache_status": "below_minimum",
"cached_input_tokens": 21188,
"cost_currency": "USD",
"elapsed_seconds": 5.118418,
"estimated_cache_storage_cost": 0.0,
"estimated_cached_input_cost": 0.0015891,
"estimated_cost": 0.0042336000000000006,
"estimated_input_cost": 0.0016095,
"estimated_output_cost": 0.001035,
"input_tokens": 23334,
"model": "gemini-3.8-flash",
"output_tokens": 276,
"prompt_version": "entity_resolution_assessment_v4",
"provider": "gemini-native",
"reasoning_tokens": null,
"uncached_input_tokens": 2146
}
},
{
"ref": "llm/operations/4a0ee3b438d0e207215ca4a5dddc522f11ac8756afc4a85b3c768815a69ce1e8/round-0001.json",
"status": "SUCCESS",
"code": null,
"config_hash": "692c8bac07efc915a44a89ac8aab1b4caa3e125d8619745f13c2ee570cb9e054",
"request_hash": "61283f3a158d361b9842431609b826d5d353b3de4c7b387a8d97fb413791d234",
"classification": "correction",
"metadata": {
"attempts": [],
"cache_created_tokens": 0,
"cache_status": "below_minimum",
"cached_input_tokens": 21288,
"cost_currency": "USD",
"elapsed_seconds": 5.437512,
"estimated_cache_storage_cost": 0.0,
"estimated_cached_input_cost": 0.0015965999999999999,
"estimated_cost": 0.0043251,
"estimated_input_cost": 0.00153225,
"estimated_output_cost": 0.00119625,
"input_tokens": 23331,
"model": "gemini-3.8-flash",
"output_tokens": 319,
"prompt_version": "entity_resolution_assessment_v4",
"provider": "gemini-native",
"reasoning_tokens": null,
"uncached_input_tokens": 2043
}
}
],
"model": "gemini-3.8-flash",
"pricing": {
"currency": "USD",
"input_per_million": 0.75,
"cached_input_per_million": 0.075,
"output_including_reasoning_per_million": 3.75,
"source": "saved API metadata and configured pricing",
"captured_at": "2026-10-24"
},
"account_balance_eur": "unknown",
"limits": "Missing cache counts are unknown, not zero; reasoning tokens unknown and not added to output. No new cloud calls.",
"total": {
"input_tokens": 90203,
"output_tokens": 986,
"model_seconds": 17.159844,
"estimated_cost_usd": "unknown (missing cache counts)",
"output_cost_usd": 0.0036975000000000003,
"cache_storage_cost_usd": 0.0
},
"clean": {
"input_tokens": 43538,
"output_tokens": 391,
"model_seconds": 6.603914,
"estimated_cost_usd": "unknown (missing cache counts)",
"output_cost_usd": 0.00146625,
"cache_storage_cost_usd": 0.0
}
}
src/viano_bug_analysis/analysis/entity_resolution.py
+99 -30
View File
@@ -127,19 +127,67 @@ def validate_assessment(payload, plan, captures, candidates, context):
if len(payload["resolutions"]) != len(plan["searches"]):
raise InterpretationError("ENTITY_RESOLUTION_ASSESSMENT_COUNT")
for index, item in enumerate(payload["resolutions"]):
if type(item["search_index"]) is not int or item["search_index"] != index:
raise InterpretationError("ENTITY_RESOLUTION_ASSESSMENT_INDEX")
allowed = set(context["sources"]) | {f"V{index+1:04d}"}
if not set(item["source_ids"]) <= allowed or not set(item["source_ids"]) & set(plan["searches"][index]["source_ids"]):
raise InterpretationError("ENTITY_RESOLUTION_SOURCE")
if f"V{index+1:04d}" not in item["source_ids"]:
raise InterpretationError("ENTITY_RESOLUTION_CAPTURE_SOURCE")
if item["status"] == "selected":
candidate = next((c for c in candidates if c["candidate_id"] == item["candidate_id"] and c["search_index"] == index), None)
if candidate is None or captures[index]["result"]["truncated"]:
raise InterpretationError("ENTITY_RESOLUTION_SELECTION")
elif item["candidate_id"] is not None or (item["status"] == "no_match" and captures[index]["result"]["items"]):
raise InterpretationError("ENTITY_RESOLUTION_SELECTION")
validate_resolution(item,index,plan,captures,candidates,context)
def validate_resolution(item,index,plan,captures,candidates,context):
if type(item['search_index']) is not int or item['search_index'] != index:
raise InterpretationError('ENTITY_RESOLUTION_ASSESSMENT_INDEX')
allowed=set(context['sources']) | {f'V{index+1:04d}'}
if not set(item['source_ids']) <= allowed or not set(item['source_ids']) & set(plan['searches'][index]['source_ids']):
raise InterpretationError('ENTITY_RESOLUTION_SOURCE')
if f'V{index+1:04d}' not in item['source_ids']:
raise InterpretationError('ENTITY_RESOLUTION_CAPTURE_SOURCE')
if item['status'] == 'selected':
candidate=next((c for c in candidates if c['candidate_id']==item['candidate_id'] and c['search_index']==index),None)
if candidate is None or captures[index]['result']['truncated']:
raise InterpretationError('ENTITY_RESOLUTION_SELECTION')
elif item['candidate_id'] is not None or (item['status']=='no_match' and captures[index]['result']['items']):
raise InterpretationError('ENTITY_RESOLUTION_SELECTION')
def assessment_feedback(document,plan,captures,candidates,context):
"""Freeze structurally valid, independently checked entries; never infer identity."""
try:validate(document,'entity-resolution-assessment')
except CaseBuildError:return None
if len(document['resolutions']) != len(plan['searches']):return None
issues=[]; locked=[]
for index,item in enumerate(document['resolutions']):
try:validate_resolution(item,index,plan,captures,candidates,context)
except InterpretationError as error:
result=captures[index]['result']
issue={'search_index':index,'path':f'/resolutions/{index}', 'code':error.code,
'capture_id':f'V{index+1:04d}', 'truncated':result['truncated'],
'returned_rows':len(result['items']), 'submitted_status':item['status'],
'required_original_source_ids':plan['searches'][index]['source_ids']}
if error.code=='ENTITY_RESOLUTION_SELECTION':
issue['allowed_candidate_ids']=[c['candidate_id'] for c in candidates if c['search_index']==index]
if item['status']=='no_match' and result['items']:
issue['instruction']='no_match requires zero returned rows. Use ambiguous when no candidate can be validly selected. Truncated results cannot support selected.'
elif item['status']=='selected' and result['truncated']:
issue['instruction']='Selection from a truncated search is forbidden. Return ambiguous with candidate_id null.'
else:issue['instruction']='Select only a candidate from this search and only from complete results; otherwise use ambiguous with candidate_id null.'
elif error.code=='ENTITY_RESOLUTION_CAPTURE_SOURCE':
issue['instruction']='Include capture_id in source_ids; a mention in reason is insufficient.'
elif error.code=='ENTITY_RESOLUTION_SOURCE':
issue['allowed_source_ids']=sorted(set(context['sources']) | {f'V{index+1:04d}'})
issue['instruction']='Cite only allowed sources and at least one required_original_source_id.'
else:issue['instruction']='Use the zero-based search_index matching this array position.'
issues.append(issue)
else:locked.append(copy.deepcopy(item))
return {'version':2,'issues':issues,'locked_resolutions':locked,
'instruction':'Return only corrected entries listed in issues, in their original order. Do not return or reconsider locked_resolutions; the program retains them unchanged.'}
def project_assessment(document, feedback):
if not feedback or feedback.get('version') != 2:return document
try:validate(document,'entity-resolution-assessment')
except CaseBuildError:raise InterpretationError('ENTITY_RESOLUTION_ASSESSMENT_SCHEMA') from None
expected=[i['search_index'] for i in feedback['issues']]
if [r['search_index'] for r in document['resolutions']] != expected:
raise InterpretationError('ENTITY_RESOLUTION_ASSESSMENT_INDEX')
combined=copy.deepcopy(feedback['locked_resolutions'])+copy.deepcopy(document['resolutions'])
return {'resolutions':sorted(combined,key=lambda r:r['search_index'])}
def resolution_guidance(context, candidates=()):
@@ -178,6 +226,7 @@ def resolution_guidance(context, candidates=()):
def request_for(stage, context, settings, plan=None, captures=None, candidates=None, errors=(), protocol_version=1):
partial_repair=False
schema = json.loads(files("viano_bug_analysis.schemas").joinpath(f"entity-resolution-{stage}-v1.schema.json").read_text())
if stage == "plan":
schema["properties"]["searches"]["maxItems"] = settings["max_searches"]
@@ -211,11 +260,24 @@ def request_for(stage, context, settings, plan=None, captures=None, candidates=N
data['identity_guidance'] = resolution_guidance(context, candidates or ())
if errors:
data["validation_feedback"] = errors[-1]
feedback=errors[-1].get('citation_corrections',{})
if stage=='assessment' and feedback.get('version')==2:
partial_repair=True
# Only invalid entries are generated. Valid entries remain bound to
# the preceding saved call and are merged locally before validation.
indexes=[issue['search_index'] for issue in feedback['issues']]
array=schema['properties']['resolutions']
all_items=array.get('prefixItems') or [copy.deepcopy(array['items']) for _ in plan['searches']]
array['prefixItems']=[all_items[i] for i in indexes]
array.update(minItems=len(indexes),maxItems=len(indexes),items=False)
if protocol_version >= 2:
# json_object providers receive messages only, not the internal schema.
data["output_schema"] = schema
system=files('viano_bug_analysis.llm.prompts').joinpath(f'entity_resolution_{stage}_v{protocol_version}.txt').read_text(encoding='utf-8')
if partial_repair:
system+='\nREPAIR SCOPE OVERRIDE: Return only the resolutions whose original search_index appears in validation_feedback.citation_corrections.issues, in that order. Do not return locked_resolutions. This overrides the earlier requirement to return every plan search. The program preserves and merges the locked entries and validates the complete assessment.'
return {"prompt_version": f"entity_resolution_{stage}_v{protocol_version}", "privacy": "configured", "output_schema": schema,
"messages": [{"role":"system", "content": files("viano_bug_analysis.llm.prompts").joinpath(f"entity_resolution_{stage}_v{protocol_version}.txt").read_text(encoding="utf-8")},
"messages": [{"role":"system", "content": system},
{"role":"user", "content": json_bytes(data).decode()}]}
@@ -301,7 +363,8 @@ class EntityResolver:
call_request=json.loads(r.read_bytes(saved['call_ref'].rsplit('/',1)[0]+'/request.json'))
request['privacy']=call_request['privacy']
call=self.checked_call(r,context,stage,saved['attempt'],request,saved['call_ref'])
if call['status'] != 'SUCCESS' or saved['document'] != call['outcome']['document']:
document=project_assessment(call['outcome'].get('document'),feedback) if stage=='assessment' and call['status']=='SUCCESS' else call['outcome'].get('document')
if call['status'] != 'SUCCESS' or saved['document'] != document:
raise InterpretationError('ENTITY_RESOLUTION_LLM_BINDING')
validator(saved['document'])
return saved
@@ -312,7 +375,7 @@ class EntityResolver:
rejected=self.checked_call(r,context,stage,attempt,request,previous['call_ref'])
if rejected['status'] == 'SUCCESS':
try:
validator(rejected['outcome']['document'])
validator(project_assessment(rejected['outcome']['document'],feedback) if stage=='assessment' else rejected['outcome']['document'])
except InterpretationError as e:
if e.code != previous['code']:
raise InterpretationError('ENTITY_RESOLUTION_HISTORY') from None
@@ -322,23 +385,28 @@ class EntityResolver:
raise InterpretationError('ENTITY_RESOLUTION_HISTORY')
errors.append(previous)
continue
if (feedback is None and not already_started and errors and stage == 'assessment'
and errors[-1]['code'] == 'ENTITY_RESOLUTION_CAPTURE_SOURCE'):
if (feedback is None and not already_started and errors and stage == 'assessment'):
with PackageReader(self.store.latest(context['case']['case_id'],context['case']['package_id']),self.store.limits) as r:
rejected=read_json(r,errors[-1]['call_ref'],'llm-call',
self.owner.gateway.config.max_response_bytes+self.owner.gateway.config.max_request_bytes+16384)
document=rejected['outcome']['document']
feedback={"instruction":"Add each missing capture ID to the structured source_ids array; mentioning it in reason is insufficient. Return the complete corrected assessment.",
"missing_capture_sources":[{"search_index":index,"required_source_id":f"V{index+1:04d}",
"field":f"/resolutions/{index}/source_ids",
"original_mention_source_ids":search['source_ids']}
for index,search in enumerate(args['plan']['searches'])
if f"V{index+1:04d}" not in document['resolutions'][index]['source_ids']]}
previous_feedback=self.read(r,context,f"feedback-{stage}-{errors[-1]['attempt']:04d}")
document=rejected['outcome'].get('document')
if rejected['status']=='SUCCESS':
try:document=project_assessment(document,previous_feedback)
except InterpretationError:document=None
feedback=assessment_feedback(document,args['plan'],args['captures'],args['candidates'],context)
if feedback is None and previous_feedback and previous_feedback.get('version')==2:
# A malformed patch must not unlock the already accepted entries.
feedback=copy.deepcopy(previous_feedback)
feedback['repair_error']=errors[-1]['code']
# Freeze feedback separately before the new call. Older attempts keep
# their original request hashes; resumed calls use this exact payload.
self.save(context,feedback_stage,feedback)
request_errors=[*errors[:-1],{**errors[-1],"citation_corrections":feedback}]
request=self.owner.gateway.validate_request(request_for(stage,context,settings,errors=request_errors,**args))
if feedback and feedback['issues']:
self.save(context,feedback_stage,feedback)
request_errors=[*errors[:-1],{**errors[-1],"citation_corrections":feedback}]
request=self.owner.gateway.validate_request(request_for(stage,context,settings,errors=request_errors,**args))
else:
feedback=None
i=context['case']
result=await self.owner.runner.run(i['case_id'],i['package_id'],operation_id=f"entity-resolution:v1:{stage}:{attempt}",request=request)
if result.status not in {'SUCCESS','ERROR'}:
@@ -348,11 +416,12 @@ class EntityResolver:
code=result.code
if result.status == 'SUCCESS':
try:
validator(call['outcome']['document'])
document=project_assessment(call['outcome']['document'],feedback) if stage=='assessment' else call['outcome']['document']
validator(document)
except InterpretationError as e:
code=e.code
else:
saved={'attempt':attempt,'document':call['outcome']['document'],'call_ref':result.result_ref}
saved={'attempt':attempt,'document':document,'call_ref':result.result_ref}
self.save(context,stage,saved)
return saved
if code not in {'LLM_OUTPUT_SCHEMA','LLM_RESPONSE','LLM_INCOMPLETE_OUTPUT'} and not (code or '').startswith('ENTITY_RESOLUTION_'):
tests/fixtures/llm_rejections/20261024/bug21-localized-no-match/analysis.md
Vendored
+5
View File
@@ -0,0 +1,5 @@
# BUG-000021 revision 2: no_match on a nonempty localized search
The base creature search correctly selected Furien (35286). The localized substring query returned unrelated harpy rows and was truncated. All three assessments marked it no_match, although the contract requires zero rows for no_match. The generic ENTITY_RESOLUTION_SELECTION feedback did not identify the index or rule.
Keep the independently valid selection unchanged. Explain nonempty/truncated coverage at search index 1; request only its correction. The expected repair is ambiguous, not a deterministic assertion that the other candidates match or do not match. The fixture contains one answer and the required scoped validation inputs, not a full run archive.
tests/fixtures/llm_rejections/20261024/bug21-localized-no-match/validation-case.json
Vendored
+561
View File
@@ -0,0 +1,561 @@
{
"context": {
"sources": {
"B0002": {
"case_pointer": "/bug/reports/1",
"data": {
"origin": {
"archive_ref": "original/vcore/imports/c9a524054fee52c755ce3e247237f9e2fd8640dc1703e7700a11ab689cfa28a3/wowdiag/archive.tar.gz",
"json_pointer": "",
"member_name": "BUG-000021-20261024-174219/bug/bug.json",
"source_ref": "original/vcore/imports/c9a524054fee52c755ce3e247237f9e2fd8640dc1703e7700a11ab689cfa28a3/wowdiag/members/BUG-000021-20261024-174219/bug/bug.json"
},
"source_ref": "case/inputs/vcore/c9a524054fee52c755ce3e247237f9e2fd8640dc1703e7700a11ab689cfa28a3/wowdiag/bug-record.json",
"summary": "Furien ist nicht hier.",
"title": "NPC Furien ist nicht existent"
},
"kind": "report"
},
"F0015": {
"case_pointer": "/runtime_facts/facts/14",
"data": {
"id": "F0015",
"name": "project_description",
"provenance": {
"author": null,
"json_pointer": "/facts/3",
"kind": "reproduction_record",
"observed_at": null,
"source_ref": "research/project/revisions/0001/facts.json"
},
"value": {
"context_role": "quest_definition",
"entity_id": 28548,
"meaning": "source_description_not_verified_runtime_behavior",
"table": "quest_template",
"texts": [
{
"column": "LogTitle",
"origin": {
"json_pointer": "/result/items/0/row/LogTitle",
"source_ref": "research/project/2997bf767d0e0ea58265f2606a8c79aec557c6d0905f85e32b417c3a0bfa2847/capture.json"
},
"text": "Warchief's Command: Desolace!"
},
{
"column": "LogDescription",
"origin": {
"json_pointer": "/result/items/0/row/LogDescription",
"source_ref": "research/project/2997bf767d0e0ea58265f2606a8c79aec557c6d0905f85e32b417c3a0bfa2847/capture.json"
},
"text": "Report to Furien at Tethris Aran in Desolace."
},
{
"column": "QuestDescription",
"origin": {
"json_pointer": "/result/items/0/row/QuestDescription",
"source_ref": "research/project/2997bf767d0e0ea58265f2606a8c79aec557c6d0905f85e32b417c3a0bfa2847/capture.json"
},
"text": "All able bodied members of the Horde are hereby ordered to report to Furien at Tethris Aran in Desolace. The brave Horde warriors of Desolace need your help to beat back the armies of the Alliance and ancient demons.$B$BTo reach Tethris Aran, travel south from the Charred Vale in the Stonetalon Mountains.$B$BDestiny calls!"
}
]
}
},
"kind": "fact"
},
"F0023": {
"case_pointer": "/runtime_facts/facts/22",
"data": {
"id": "F0023",
"name": "project_description",
"provenance": {
"author": null,
"json_pointer": "/facts/3",
"kind": "reproduction_record",
"observed_at": null,
"source_ref": "research/project/revisions/0002/facts.json"
},
"value": {
"context_role": "quest_definition",
"entity_id": 28548,
"meaning": "source_description_not_verified_runtime_behavior",
"table": "quest_template",
"texts": [
{
"column": "LogTitle",
"origin": {
"json_pointer": "/result/items/0/row/LogTitle",
"source_ref": "research/project/2997bf767d0e0ea58265f2606a8c79aec557c6d0905f85e32b417c3a0bfa2847/capture.json"
},
"text": "Warchief's Command: Desolace!"
},
{
"column": "LogDescription",
"origin": {
"json_pointer": "/result/items/0/row/LogDescription",
"source_ref": "research/project/2997bf767d0e0ea58265f2606a8c79aec557c6d0905f85e32b417c3a0bfa2847/capture.json"
},
"text": "Report to Furien at Tethris Aran in Desolace."
},
{
"column": "QuestDescription",
"origin": {
"json_pointer": "/result/items/0/row/QuestDescription",
"source_ref": "research/project/2997bf767d0e0ea58265f2606a8c79aec557c6d0905f85e32b417c3a0bfa2847/capture.json"
},
"text": "All able bodied members of the Horde are hereby ordered to report to Furien at Tethris Aran in Desolace. The brave Horde warriors of Desolace need your help to beat back the armies of the Alliance and ancient demons.$B$BTo reach Tethris Aran, travel south from the Charred Vale in the Stonetalon Mountains.$B$BDestiny calls!"
}
]
}
},
"kind": "fact"
},
"O0001": {
"case_pointer": "/human_observations/observations/0",
"data": {
"id": "O0001",
"provenance": {
"author": "Elena",
"json_pointer": "/observations/0",
"kind": "tester",
"observed_at": null,
"origin": {
"archive_ref": "original/vcore/imports/c9a524054fee52c755ce3e247237f9e2fd8640dc1703e7700a11ab689cfa28a3/wowdiag/archive.tar.gz",
"json_pointer": "/0/description",
"member_name": "BUG-000021-20261024-174219/bug/reports.json",
"source_ref": "original/vcore/imports/c9a524054fee52c755ce3e247237f9e2fd8640dc1703e7700a11ab689cfa28a3/wowdiag/members/BUG-000021-20261024-174219/bug/reports.json",
"source_timestamp": "2026-10-24T17:39:03"
},
"source_ref": "case/inputs/vcore/c9a524054fee52c755ce3e247237f9e2fd8640dc1703e7700a11ab689cfa28a3/wowdiag/report-34.json"
},
"text": "Furien ist nicht hier."
},
"kind": "observation"
}
}
},
"plan": {
"searches": [
{
"catalog": "creatures",
"role": "affected_subject",
"source_ids": [
"B0002",
"F0015",
"F0023",
"O0001"
],
"term": "Furien"
},
{
"catalog": "localized_creatures",
"role": "affected_subject",
"source_ids": [
"B0002",
"F0015",
"F0023",
"O0001"
],
"term": "Furien"
}
]
},
"captures": [
{
"format": "viano-project-capture",
"format_version": 1,
"origin": {
"additional_databases": {
"cata_characters": [
"*"
],
"cata_hotfixes": [
"*"
]
},
"database": "cata_world",
"kind": "mariadb",
"selected_database": "cata_world",
"selected_table": "creature_template",
"snapshot_id": "vps-live",
"table_reference": "database.table; unqualified tables use the default database only",
"tables": [
"*"
],
"time_scope": "configured_snapshot_not_current_production"
},
"query": {
"columns": [
"entry",
"name",
"subname",
"AIName",
"ScriptName",
"minlevel",
"maxlevel",
"faction",
"unit_flags"
],
"filters": [
{
"column": "name",
"operator": "contains",
"value": "Furien"
}
],
"kind": "database.rows",
"limit": 8,
"offset": 0,
"table": "creature_template"
},
"result": {
"coverage": "complete_filtered_rows",
"items": [
{
"database": "cata_world",
"row": {
"AIName": "",
"ScriptName": "",
"entry": 35286,
"faction": 1604,
"maxlevel": 35,
"minlevel": 35,
"name": "Furien",
"subname": "",
"unit_flags": 0
},
"table": "creature_template"
},
{
"database": "cata_world",
"row": {
"AIName": "",
"ScriptName": "",
"entry": 35287,
"faction": 35,
"maxlevel": 30,
"minlevel": 30,
"name": "Furien's Mount",
"subname": "",
"unit_flags": 0
},
"table": "creature_template"
},
{
"database": "cata_world",
"row": {
"AIName": "",
"ScriptName": "",
"entry": 35288,
"faction": 29,
"maxlevel": 25,
"minlevel": 25,
"name": "Furien's Dragonhawk",
"subname": "",
"unit_flags": 0
},
"table": "creature_template"
},
{
"database": "cata_world",
"row": {
"AIName": "",
"ScriptName": "",
"entry": 35316,
"faction": 29,
"maxlevel": 25,
"minlevel": 25,
"name": "Furien's Dragonhawk",
"subname": "",
"unit_flags": 0
},
"table": "creature_template"
},
{
"database": "cata_world",
"row": {
"AIName": "",
"ScriptName": "",
"entry": 51554,
"faction": 35,
"maxlevel": 1,
"minlevel": 1,
"name": "Furien's Research Assistant",
"subname": "",
"unit_flags": 0
},
"table": "creature_template"
}
],
"next_offset": null,
"truncated": false
}
},
{
"format": "viano-project-capture",
"format_version": 1,
"origin": {
"additional_databases": {
"cata_characters": [
"*"
],
"cata_hotfixes": [
"*"
]
},
"database": "cata_world",
"kind": "mariadb",
"selected_database": "cata_world",
"selected_table": "creature_template_locale",
"snapshot_id": "vps-live",
"table_reference": "database.table; unqualified tables use the default database only",
"tables": [
"*"
],
"time_scope": "configured_snapshot_not_current_production"
},
"query": {
"columns": [
"entry",
"locale",
"Name",
"Title"
],
"filters": [
{
"column": "Name",
"operator": "contains",
"value": "Furien"
}
],
"kind": "database.rows",
"limit": 8,
"offset": 0,
"table": "creature_template_locale"
},
"result": {
"coverage": "partial_filtered_rows",
"items": [
{
"database": "cata_world",
"row": {
"Name": "Harpyie der Windfurien",
"Title": "",
"entry": 2962,
"locale": "deDE"
},
"table": "creature_template_locale"
},
{
"database": "cata_world",
"row": {
"Name": "Windhexe der Windfurien",
"Title": "",
"entry": 2963,
"locale": "deDE"
},
"table": "creature_template_locale"
},
{
"database": "cata_world",
"row": {
"Name": "Zauberin der Windfurien",
"Title": "",
"entry": 2964,
"locale": "deDE"
},
"table": "creature_template_locale"
},
{
"database": "cata_world",
"row": {
"Name": "Matriarchin der Windfurien",
"Title": "",
"entry": 2965,
"locale": "deDE"
},
"table": "creature_template_locale"
},
{
"database": "cata_world",
"row": {
"Name": "Harpyie der Blutfurien",
"Title": "",
"entry": 4022,
"locale": "deDE"
},
"table": "creature_template_locale"
},
{
"database": "cata_world",
"row": {
"Name": "Wildfeder der Blutfurien",
"Title": "",
"entry": 4023,
"locale": "deDE"
},
"table": "creature_template_locale"
},
{
"database": "cata_world",
"row": {
"Name": "Töterin der Blutfurien",
"Title": "",
"entry": 4024,
"locale": "deDE"
},
"table": "creature_template_locale"
},
{
"database": "cata_world",
"row": {
"Name": "Wegelagerin der Blutfurien",
"Title": "",
"entry": 4025,
"locale": "deDE"
},
"table": "creature_template_locale"
}
],
"next_offset": 8,
"truncated": true
}
}
],
"candidates": [
{
"candidate_id": "K0001",
"id": "35286",
"name": "Furien",
"row_index": 0,
"search_index": 0,
"type": "creature"
},
{
"candidate_id": "K0002",
"id": "35287",
"name": "Furien's Mount",
"row_index": 1,
"search_index": 0,
"type": "creature"
},
{
"candidate_id": "K0003",
"id": "35288",
"name": "Furien's Dragonhawk",
"row_index": 2,
"search_index": 0,
"type": "creature"
},
{
"candidate_id": "K0004",
"id": "35316",
"name": "Furien's Dragonhawk",
"row_index": 3,
"search_index": 0,
"type": "creature"
},
{
"candidate_id": "K0005",
"id": "51554",
"name": "Furien's Research Assistant",
"row_index": 4,
"search_index": 0,
"type": "creature"
},
{
"candidate_id": "K0006",
"id": "2962",
"name": "Harpyie der Windfurien",
"row_index": 0,
"search_index": 1,
"type": "creature"
},
{
"candidate_id": "K0007",
"id": "2963",
"name": "Windhexe der Windfurien",
"row_index": 1,
"search_index": 1,
"type": "creature"
},
{
"candidate_id": "K0008",
"id": "2964",
"name": "Zauberin der Windfurien",
"row_index": 2,
"search_index": 1,
"type": "creature"
},
{
"candidate_id": "K0009",
"id": "2965",
"name": "Matriarchin der Windfurien",
"row_index": 3,
"search_index": 1,
"type": "creature"
},
{
"candidate_id": "K0010",
"id": "4022",
"name": "Harpyie der Blutfurien",
"row_index": 4,
"search_index": 1,
"type": "creature"
},
{
"candidate_id": "K0011",
"id": "4023",
"name": "Wildfeder der Blutfurien",
"row_index": 5,
"search_index": 1,
"type": "creature"
},
{
"candidate_id": "K0012",
"id": "4024",
"name": "Töterin der Blutfurien",
"row_index": 6,
"search_index": 1,
"type": "creature"
},
{
"candidate_id": "K0013",
"id": "4025",
"name": "Wegelagerin der Blutfurien",
"row_index": 7,
"search_index": 1,
"type": "creature"
}
],
"answer": {
"resolutions": [
{
"candidate_id": "K0001",
"reason": "Candidate K0001 (entry 35286, 'Furien') precisely matches the reported missing NPC name 'Furien' and corresponds to the quest target described in quest 28548 ('Report to Furien at Tethris Aran in Desolace'). The other candidates represent associated mounts or assistants rather than the NPC itself.",
"search_index": 0,
"source_ids": [
"B0002",
"F0015",
"F0023",
"O0001",
"V0001"
],
"status": "selected"
},
{
"candidate_id": null,
"reason": "Capture V0002 was truncated, and the returned candidates are German harpies ('der Windfurien', 'der Blutfurien') rather than the NPC 'Furien'. Because the search was truncated and does not match the affected NPC entity, a candidate cannot be selected.",
"search_index": 1,
"source_ids": [
"B0002",
"F0015",
"F0023",
"O0001",
"V0002"
],
"status": "no_match"
}
]
}
}
tests/test_entity_resolution.py
+4 -3
View File
@@ -386,9 +386,10 @@ def test_missing_capture_feedback_resumes_and_replays_without_repeating_research
resolution['reason'] = 'The right capture is V0001 but only mentioned here.'
else:
fixes = data['validation_feedback']['citation_corrections']
assert fixes['missing_capture_sources'] == [{
'search_index':0, 'required_source_id':'V0001',
'field':'/resolutions/0/source_ids', 'original_mention_source_ids':['O0001']}]
assert fixes['version']==2
assert fixes['issues'][0]['search_index']==0
assert fixes['issues'][0]['capture_id']=='V0001'
assert fixes['issues'][0]['code']=='ENTITY_RESOLUTION_CAPTURE_SOURCE'
return response
store = built_case[0]
model = MissingCaptureModel()
tests/test_entity_resolution_partial_repair.py
+88
View File
@@ -0,0 +1,88 @@
import asyncio
import copy
import json
from pathlib import Path
import pytest
from jsonschema import Draft202012Validator
from test_case_interpreter import built_case
from test_case_builder import case_setup
from test_entity_resolution import Model,Database,PLAN,service_for,run,gateway
from viano_bug_analysis.analysis.entity_resolution import (
assessment_feedback,project_assessment,validate_assessment,request_for,policy)
from viano_bug_analysis.analysis.errors import InterpretationError
from viano_bug_analysis.llm import ProviderResponse
from viano_bug_analysis.lifecycle import CheckpointStore
from viano_bug_analysis.package import PackageReader
def corpus():
return json.loads(Path('tests/fixtures/llm_rejections/20261024/bug21-localized-no-match/validation-case.json').read_text(encoding='utf-8'))
def test_real_rejection_yields_scoped_repair_and_retains_furien():
d=corpus();before=copy.deepcopy(d)
feedback=assessment_feedback(d['answer'],d['plan'],d['captures'],d['candidates'],d['context'])
assert feedback['locked_resolutions']==[d['answer']['resolutions'][0]]
assert len(feedback['issues'])==1
issue=feedback['issues'][0]
assert issue['search_index']==1 and issue['returned_rows']==8 and issue['truncated']
assert 'zero returned rows' in issue['instruction']
replacement=copy.deepcopy(d['answer']['resolutions'][1]);replacement['status']='ambiguous'
result=project_assessment({'resolutions':[replacement]},feedback)
validate_assessment(result,d['plan'],d['captures'],d['candidates'],d['context'])
assert result['resolutions'][0]==d['answer']['resolutions'][0]
with pytest.raises(InterpretationError):project_assessment(d['answer'],feedback)
assert d==before
@pytest.mark.parametrize('bad', ['no_match','selected','source'])
def test_invalid_repaired_entry_is_still_rejected(bad):
d=corpus();feedback=assessment_feedback(d['answer'],d['plan'],d['captures'],d['candidates'],d['context'])
replacement=copy.deepcopy(d['answer']['resolutions'][1])
if bad=='selected':replacement.update(status='selected',candidate_id='K9999')
if bad=='source':replacement.update(status='ambiguous',source_ids=['F9999'])
combined=project_assessment({'resolutions':[replacement]},feedback)
with pytest.raises(InterpretationError):validate_assessment(combined,d['plan'],d['captures'],d['candidates'],d['context'])
@pytest.mark.parametrize('malformed_patch',[False,True])
def test_two_search_repair_survives_restart_and_portable_replay(built_case,tmp_path,malformed_patch):
plan=copy.deepcopy(PLAN)
plan['searches'].append({**plan['searches'][0],'catalog':'localized'})
class PartialModel(Model):
def __init__(self):super().__init__(plan=plan);self.repairs=0
async def generate(self,config,request,**kwargs):
if request['prompt_version']=='entity_resolution_assessment_v4':
wire=json.loads(request['messages'][1]['content'])
if 'validation_feedback' in wire:
self.requests.append(copy.deepcopy(request));self.repairs+=1
feedback=wire['validation_feedback']['citation_corrections']
assert feedback['locked_resolutions'][0]['candidate_id']=='K0001'
assert [i['search_index'] for i in feedback['issues']]==[1]
out={'resolutions':[{'search_index':1,'status':'ambiguous','candidate_id':None,
'reason':'Unresolved localized identity','source_ids':['O0001','V0002']}]}
assert Draft202012Validator(request['output_schema']).is_valid(out)
assert 'REPAIR SCOPE OVERRIDE' in request['messages'][0]['content']
if malformed_patch and self.repairs==1:out['resolutions'][0]['search_index']=0
return ProviderResponse(out,'fixture',100,50)
response=await super().generate(config,request,**kwargs)
response.document['resolutions'][1].update(status='no_match',candidate_id=None)
return response
return await super().generate(config,request,**kwargs)
store=built_case[0];service=service_for(store,Database());model=PartialModel()
service.entity_resolution['catalogs'].append({**service.entity_resolution['catalogs'][0],'id':'localized'})
assert run(store,model,service).status=='REJECTED'
if malformed_patch:assert run(store,model,service).status=='REJECTED'
result=run(store,model,service)
# Identical fixture queries share the existing research cache.
assert result.status=='COMPLETED' and len(service.database.calls)==1
with PackageReader(result.package) as reader:
assessment=json.loads(reader.read_bytes('analysis/interpretations/0001/entity-resolution/assessment.json'))['payload']['document']
assert assessment['resolutions'][0]['candidate_id']=='K0001'
assert assessment['resolutions'][1]['status']=='ambiguous'
recipient=CheckpointStore(type(store.workspace)(tmp_path/'recipient'))
recipient.intake(result.package)
count=len(model.requests)
assert run(recipient,model).reused
assert len(model.requests)==count