Nico/VianoBugAnalysis
1
1 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Repair only invalid entity assessments with precise scoped feedback

Browse Source
This commit is contained in:
Nico committed 2026-10-24 21:18:51 +02:00
1 parent 33603db9bd
commit 4cbba4c1e4
7 files changed
+920 -33
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/fixtures/llm_rejections/20261024/bug21-localized-no-match/analysis.md
Vendored
+5
View File
@@ -0,0 +1,5 @@
# BUG-000021 revision 2: no_match on a nonempty localized search
The base creature search correctly selected Furien (35286). The localized substring query returned unrelated harpy rows and was truncated. All three assessments marked it no_match, although the contract requires zero rows for no_match. The generic ENTITY_RESOLUTION_SELECTION feedback did not identify the index or rule.
Keep the independently valid selection unchanged. Explain nonempty/truncated coverage at search index 1; request only its correction. The expected repair is ambiguous, not a deterministic assertion that the other candidates match or do not match. The fixture contains one answer and the required scoped validation inputs, not a full run archive.
tests/fixtures/llm_rejections/20261024/bug21-localized-no-match/validation-case.json
Vendored
+561
View File
@@ -0,0 +1,561 @@
{
"context": {
"sources": {
"B0002": {
"case_pointer": "/bug/reports/1",
"data": {
"origin": {
"archive_ref": "original/vcore/imports/c9a524054fee52c755ce3e247237f9e2fd8640dc1703e7700a11ab689cfa28a3/wowdiag/archive.tar.gz",
"json_pointer": "",
"member_name": "BUG-000021-20261024-174219/bug/bug.json",
"source_ref": "original/vcore/imports/c9a524054fee52c755ce3e247237f9e2fd8640dc1703e7700a11ab689cfa28a3/wowdiag/members/BUG-000021-20261024-174219/bug/bug.json"
},
"source_ref": "case/inputs/vcore/c9a524054fee52c755ce3e247237f9e2fd8640dc1703e7700a11ab689cfa28a3/wowdiag/bug-record.json",
"summary": "Furien ist nicht hier.",
"title": "NPC Furien ist nicht existent"
},
"kind": "report"
},
"F0015": {
"case_pointer": "/runtime_facts/facts/14",
"data": {
"id": "F0015",
"name": "project_description",
"provenance": {
"author": null,
"json_pointer": "/facts/3",
"kind": "reproduction_record",
"observed_at": null,
"source_ref": "research/project/revisions/0001/facts.json"
},
"value": {
"context_role": "quest_definition",
"entity_id": 28548,
"meaning": "source_description_not_verified_runtime_behavior",
"table": "quest_template",
"texts": [
{
"column": "LogTitle",
"origin": {
"json_pointer": "/result/items/0/row/LogTitle",
"source_ref": "research/project/2997bf767d0e0ea58265f2606a8c79aec557c6d0905f85e32b417c3a0bfa2847/capture.json"
},
"text": "Warchief's Command: Desolace!"
},
{
"column": "LogDescription",
"origin": {
"json_pointer": "/result/items/0/row/LogDescription",
"source_ref": "research/project/2997bf767d0e0ea58265f2606a8c79aec557c6d0905f85e32b417c3a0bfa2847/capture.json"
},
"text": "Report to Furien at Tethris Aran in Desolace."
},
{
"column": "QuestDescription",
"origin": {
"json_pointer": "/result/items/0/row/QuestDescription",
"source_ref": "research/project/2997bf767d0e0ea58265f2606a8c79aec557c6d0905f85e32b417c3a0bfa2847/capture.json"
},
"text": "All able bodied members of the Horde are hereby ordered to report to Furien at Tethris Aran in Desolace. The brave Horde warriors of Desolace need your help to beat back the armies of the Alliance and ancient demons.$B$BTo reach Tethris Aran, travel south from the Charred Vale in the Stonetalon Mountains.$B$BDestiny calls!"
}
]
}
},
"kind": "fact"
},
"F0023": {
"case_pointer": "/runtime_facts/facts/22",
"data": {
"id": "F0023",
"name": "project_description",
"provenance": {
"author": null,
"json_pointer": "/facts/3",
"kind": "reproduction_record",
"observed_at": null,
"source_ref": "research/project/revisions/0002/facts.json"
},
"value": {
"context_role": "quest_definition",
"entity_id": 28548,
"meaning": "source_description_not_verified_runtime_behavior",
"table": "quest_template",
"texts": [
{
"column": "LogTitle",
"origin": {
"json_pointer": "/result/items/0/row/LogTitle",
"source_ref": "research/project/2997bf767d0e0ea58265f2606a8c79aec557c6d0905f85e32b417c3a0bfa2847/capture.json"
},
"text": "Warchief's Command: Desolace!"
},
{
"column": "LogDescription",
"origin": {
"json_pointer": "/result/items/0/row/LogDescription",
"source_ref": "research/project/2997bf767d0e0ea58265f2606a8c79aec557c6d0905f85e32b417c3a0bfa2847/capture.json"
},
"text": "Report to Furien at Tethris Aran in Desolace."
},
{
"column": "QuestDescription",
"origin": {
"json_pointer": "/result/items/0/row/QuestDescription",
"source_ref": "research/project/2997bf767d0e0ea58265f2606a8c79aec557c6d0905f85e32b417c3a0bfa2847/capture.json"
},
"text": "All able bodied members of the Horde are hereby ordered to report to Furien at Tethris Aran in Desolace. The brave Horde warriors of Desolace need your help to beat back the armies of the Alliance and ancient demons.$B$BTo reach Tethris Aran, travel south from the Charred Vale in the Stonetalon Mountains.$B$BDestiny calls!"
}
]
}
},
"kind": "fact"
},
"O0001": {
"case_pointer": "/human_observations/observations/0",
"data": {
"id": "O0001",
"provenance": {
"author": "Elena",
"json_pointer": "/observations/0",
"kind": "tester",
"observed_at": null,
"origin": {
"archive_ref": "original/vcore/imports/c9a524054fee52c755ce3e247237f9e2fd8640dc1703e7700a11ab689cfa28a3/wowdiag/archive.tar.gz",
"json_pointer": "/0/description",
"member_name": "BUG-000021-20261024-174219/bug/reports.json",
"source_ref": "original/vcore/imports/c9a524054fee52c755ce3e247237f9e2fd8640dc1703e7700a11ab689cfa28a3/wowdiag/members/BUG-000021-20261024-174219/bug/reports.json",
"source_timestamp": "2026-10-24T17:39:03"
},
"source_ref": "case/inputs/vcore/c9a524054fee52c755ce3e247237f9e2fd8640dc1703e7700a11ab689cfa28a3/wowdiag/report-34.json"
},
"text": "Furien ist nicht hier."
},
"kind": "observation"
}
}
},
"plan": {
"searches": [
{
"catalog": "creatures",
"role": "affected_subject",
"source_ids": [
"B0002",
"F0015",
"F0023",
"O0001"
],
"term": "Furien"
},
{
"catalog": "localized_creatures",
"role": "affected_subject",
"source_ids": [
"B0002",
"F0015",
"F0023",
"O0001"
],
"term": "Furien"
}
]
},
"captures": [
{
"format": "viano-project-capture",
"format_version": 1,
"origin": {
"additional_databases": {
"cata_characters": [
"*"
],
"cata_hotfixes": [
"*"
]
},
"database": "cata_world",
"kind": "mariadb",
"selected_database": "cata_world",
"selected_table": "creature_template",
"snapshot_id": "vps-live",
"table_reference": "database.table; unqualified tables use the default database only",
"tables": [
"*"
],
"time_scope": "configured_snapshot_not_current_production"
},
"query": {
"columns": [
"entry",
"name",
"subname",
"AIName",
"ScriptName",
"minlevel",
"maxlevel",
"faction",
"unit_flags"
],
"filters": [
{
"column": "name",
"operator": "contains",
"value": "Furien"
}
],
"kind": "database.rows",
"limit": 8,
"offset": 0,
"table": "creature_template"
},
"result": {
"coverage": "complete_filtered_rows",
"items": [
{
"database": "cata_world",
"row": {
"AIName": "",
"ScriptName": "",
"entry": 35286,
"faction": 1604,
"maxlevel": 35,
"minlevel": 35,
"name": "Furien",
"subname": "",
"unit_flags": 0
},
"table": "creature_template"
},
{
"database": "cata_world",
"row": {
"AIName": "",
"ScriptName": "",
"entry": 35287,
"faction": 35,
"maxlevel": 30,
"minlevel": 30,
"name": "Furien's Mount",
"subname": "",
"unit_flags": 0
},
"table": "creature_template"
},
{
"database": "cata_world",
"row": {
"AIName": "",
"ScriptName": "",
"entry": 35288,
"faction": 29,
"maxlevel": 25,
"minlevel": 25,
"name": "Furien's Dragonhawk",
"subname": "",
"unit_flags": 0
},
"table": "creature_template"
},
{
"database": "cata_world",
"row": {
"AIName": "",
"ScriptName": "",
"entry": 35316,
"faction": 29,
"maxlevel": 25,
"minlevel": 25,
"name": "Furien's Dragonhawk",
"subname": "",
"unit_flags": 0
},
"table": "creature_template"
},
{
"database": "cata_world",
"row": {
"AIName": "",
"ScriptName": "",
"entry": 51554,
"faction": 35,
"maxlevel": 1,
"minlevel": 1,
"name": "Furien's Research Assistant",
"subname": "",
"unit_flags": 0
},
"table": "creature_template"
}
],
"next_offset": null,
"truncated": false
}
},
{
"format": "viano-project-capture",
"format_version": 1,
"origin": {
"additional_databases": {
"cata_characters": [
"*"
],
"cata_hotfixes": [
"*"
]
},
"database": "cata_world",
"kind": "mariadb",
"selected_database": "cata_world",
"selected_table": "creature_template_locale",
"snapshot_id": "vps-live",
"table_reference": "database.table; unqualified tables use the default database only",
"tables": [
"*"
],
"time_scope": "configured_snapshot_not_current_production"
},
"query": {
"columns": [
"entry",
"locale",
"Name",
"Title"
],
"filters": [
{
"column": "Name",
"operator": "contains",
"value": "Furien"
}
],
"kind": "database.rows",
"limit": 8,
"offset": 0,
"table": "creature_template_locale"
},
"result": {
"coverage": "partial_filtered_rows",
"items": [
{
"database": "cata_world",
"row": {
"Name": "Harpyie der Windfurien",
"Title": "",
"entry": 2962,
"locale": "deDE"
},
"table": "creature_template_locale"
},
{
"database": "cata_world",
"row": {
"Name": "Windhexe der Windfurien",
"Title": "",
"entry": 2963,
"locale": "deDE"
},
"table": "creature_template_locale"
},
{
"database": "cata_world",
"row": {
"Name": "Zauberin der Windfurien",
"Title": "",
"entry": 2964,
"locale": "deDE"
},
"table": "creature_template_locale"
},
{
"database": "cata_world",
"row": {
"Name": "Matriarchin der Windfurien",
"Title": "",
"entry": 2965,
"locale": "deDE"
},
"table": "creature_template_locale"
},
{
"database": "cata_world",
"row": {
"Name": "Harpyie der Blutfurien",
"Title": "",
"entry": 4022,
"locale": "deDE"
},
"table": "creature_template_locale"
},
{
"database": "cata_world",
"row": {
"Name": "Wildfeder der Blutfurien",
"Title": "",
"entry": 4023,
"locale": "deDE"
},
"table": "creature_template_locale"
},
{
"database": "cata_world",
"row": {
"Name": "Töterin der Blutfurien",
"Title": "",
"entry": 4024,
"locale": "deDE"
},
"table": "creature_template_locale"
},
{
"database": "cata_world",
"row": {
"Name": "Wegelagerin der Blutfurien",
"Title": "",
"entry": 4025,
"locale": "deDE"
},
"table": "creature_template_locale"
}
],
"next_offset": 8,
"truncated": true
}
}
],
"candidates": [
{
"candidate_id": "K0001",
"id": "35286",
"name": "Furien",
"row_index": 0,
"search_index": 0,
"type": "creature"
},
{
"candidate_id": "K0002",
"id": "35287",
"name": "Furien's Mount",
"row_index": 1,
"search_index": 0,
"type": "creature"
},
{
"candidate_id": "K0003",
"id": "35288",
"name": "Furien's Dragonhawk",
"row_index": 2,
"search_index": 0,
"type": "creature"
},
{
"candidate_id": "K0004",
"id": "35316",
"name": "Furien's Dragonhawk",
"row_index": 3,
"search_index": 0,
"type": "creature"
},
{
"candidate_id": "K0005",
"id": "51554",
"name": "Furien's Research Assistant",
"row_index": 4,
"search_index": 0,
"type": "creature"
},
{
"candidate_id": "K0006",
"id": "2962",
"name": "Harpyie der Windfurien",
"row_index": 0,
"search_index": 1,
"type": "creature"
},
{
"candidate_id": "K0007",
"id": "2963",
"name": "Windhexe der Windfurien",
"row_index": 1,
"search_index": 1,
"type": "creature"
},
{
"candidate_id": "K0008",
"id": "2964",
"name": "Zauberin der Windfurien",
"row_index": 2,
"search_index": 1,
"type": "creature"
},
{
"candidate_id": "K0009",
"id": "2965",
"name": "Matriarchin der Windfurien",
"row_index": 3,
"search_index": 1,
"type": "creature"
},
{
"candidate_id": "K0010",
"id": "4022",
"name": "Harpyie der Blutfurien",
"row_index": 4,
"search_index": 1,
"type": "creature"
},
{
"candidate_id": "K0011",
"id": "4023",
"name": "Wildfeder der Blutfurien",
"row_index": 5,
"search_index": 1,
"type": "creature"
},
{
"candidate_id": "K0012",
"id": "4024",
"name": "Töterin der Blutfurien",
"row_index": 6,
"search_index": 1,
"type": "creature"
},
{
"candidate_id": "K0013",
"id": "4025",
"name": "Wegelagerin der Blutfurien",
"row_index": 7,
"search_index": 1,
"type": "creature"
}
],
"answer": {
"resolutions": [
{
"candidate_id": "K0001",
"reason": "Candidate K0001 (entry 35286, 'Furien') precisely matches the reported missing NPC name 'Furien' and corresponds to the quest target described in quest 28548 ('Report to Furien at Tethris Aran in Desolace'). The other candidates represent associated mounts or assistants rather than the NPC itself.",
"search_index": 0,
"source_ids": [
"B0002",
"F0015",
"F0023",
"O0001",
"V0001"
],
"status": "selected"
},
{
"candidate_id": null,
"reason": "Capture V0002 was truncated, and the returned candidates are German harpies ('der Windfurien', 'der Blutfurien') rather than the NPC 'Furien'. Because the search was truncated and does not match the affected NPC entity, a candidate cannot be selected.",
"search_index": 1,
"source_ids": [
"B0002",
"F0015",
"F0023",
"O0001",
"V0002"
],
"status": "no_match"
}
]
}
}
tests/test_entity_resolution.py
+4 -3
View File
@@ -386,9 +386,10 @@ def test_missing_capture_feedback_resumes_and_replays_without_repeating_research
resolution['reason'] = 'The right capture is V0001 but only mentioned here.'
else:
fixes = data['validation_feedback']['citation_corrections']
assert fixes['missing_capture_sources'] == [{
'search_index':0, 'required_source_id':'V0001',
'field':'/resolutions/0/source_ids', 'original_mention_source_ids':['O0001']}]
assert fixes['version']==2
assert fixes['issues'][0]['search_index']==0
assert fixes['issues'][0]['capture_id']=='V0001'
assert fixes['issues'][0]['code']=='ENTITY_RESOLUTION_CAPTURE_SOURCE'
return response
store = built_case[0]
model = MissingCaptureModel()
tests/test_entity_resolution_partial_repair.py
+88
View File
@@ -0,0 +1,88 @@
import asyncio
import copy
import json
from pathlib import Path
import pytest
from jsonschema import Draft202012Validator
from test_case_interpreter import built_case
from test_case_builder import case_setup
from test_entity_resolution import Model,Database,PLAN,service_for,run,gateway
from viano_bug_analysis.analysis.entity_resolution import (
assessment_feedback,project_assessment,validate_assessment,request_for,policy)
from viano_bug_analysis.analysis.errors import InterpretationError
from viano_bug_analysis.llm import ProviderResponse
from viano_bug_analysis.lifecycle import CheckpointStore
from viano_bug_analysis.package import PackageReader
def corpus():
return json.loads(Path('tests/fixtures/llm_rejections/20261024/bug21-localized-no-match/validation-case.json').read_text(encoding='utf-8'))
def test_real_rejection_yields_scoped_repair_and_retains_furien():
d=corpus();before=copy.deepcopy(d)
feedback=assessment_feedback(d['answer'],d['plan'],d['captures'],d['candidates'],d['context'])
assert feedback['locked_resolutions']==[d['answer']['resolutions'][0]]
assert len(feedback['issues'])==1
issue=feedback['issues'][0]
assert issue['search_index']==1 and issue['returned_rows']==8 and issue['truncated']
assert 'zero returned rows' in issue['instruction']
replacement=copy.deepcopy(d['answer']['resolutions'][1]);replacement['status']='ambiguous'
result=project_assessment({'resolutions':[replacement]},feedback)
validate_assessment(result,d['plan'],d['captures'],d['candidates'],d['context'])
assert result['resolutions'][0]==d['answer']['resolutions'][0]
with pytest.raises(InterpretationError):project_assessment(d['answer'],feedback)
assert d==before
@pytest.mark.parametrize('bad', ['no_match','selected','source'])
def test_invalid_repaired_entry_is_still_rejected(bad):
d=corpus();feedback=assessment_feedback(d['answer'],d['plan'],d['captures'],d['candidates'],d['context'])
replacement=copy.deepcopy(d['answer']['resolutions'][1])
if bad=='selected':replacement.update(status='selected',candidate_id='K9999')
if bad=='source':replacement.update(status='ambiguous',source_ids=['F9999'])
combined=project_assessment({'resolutions':[replacement]},feedback)
with pytest.raises(InterpretationError):validate_assessment(combined,d['plan'],d['captures'],d['candidates'],d['context'])
@pytest.mark.parametrize('malformed_patch',[False,True])
def test_two_search_repair_survives_restart_and_portable_replay(built_case,tmp_path,malformed_patch):
plan=copy.deepcopy(PLAN)
plan['searches'].append({**plan['searches'][0],'catalog':'localized'})
class PartialModel(Model):
def __init__(self):super().__init__(plan=plan);self.repairs=0
async def generate(self,config,request,**kwargs):
if request['prompt_version']=='entity_resolution_assessment_v4':
wire=json.loads(request['messages'][1]['content'])
if 'validation_feedback' in wire:
self.requests.append(copy.deepcopy(request));self.repairs+=1
feedback=wire['validation_feedback']['citation_corrections']
assert feedback['locked_resolutions'][0]['candidate_id']=='K0001'
assert [i['search_index'] for i in feedback['issues']]==[1]
out={'resolutions':[{'search_index':1,'status':'ambiguous','candidate_id':None,
'reason':'Unresolved localized identity','source_ids':['O0001','V0002']}]}
assert Draft202012Validator(request['output_schema']).is_valid(out)
assert 'REPAIR SCOPE OVERRIDE' in request['messages'][0]['content']
if malformed_patch and self.repairs==1:out['resolutions'][0]['search_index']=0
return ProviderResponse(out,'fixture',100,50)
response=await super().generate(config,request,**kwargs)
response.document['resolutions'][1].update(status='no_match',candidate_id=None)
return response
return await super().generate(config,request,**kwargs)
store=built_case[0];service=service_for(store,Database());model=PartialModel()
service.entity_resolution['catalogs'].append({**service.entity_resolution['catalogs'][0],'id':'localized'})
assert run(store,model,service).status=='REJECTED'
if malformed_patch:assert run(store,model,service).status=='REJECTED'
result=run(store,model,service)
# Identical fixture queries share the existing research cache.
assert result.status=='COMPLETED' and len(service.database.calls)==1
with PackageReader(result.package) as reader:
assessment=json.loads(reader.read_bytes('analysis/interpretations/0001/entity-resolution/assessment.json'))['payload']['document']
assert assessment['resolutions'][0]['candidate_id']=='K0001'
assert assessment['resolutions'][1]['status']=='ambiguous'
recipient=CheckpointStore(type(store.workspace)(tmp_path/'recipient'))
recipient.intake(result.package)
count=len(model.requests)
assert run(recipient,model).reused
assert len(model.requests)==count