From 6c3415b13aa5de22e5c2c27f34c6879a34e7cd86 Mon Sep 17 00:00:00 2001 From: Nitsu45 Date: Thu, 15 Oct 2026 09:04:40 +0200 Subject: [PATCH] Explain missing capture citations in entity assessment corrections --- docs/entity-resolution-v1.md | 14 ++++++++ .../analysis/entity_resolution.py | 25 ++++++++++++- .../schemas/entity-resolution-v1.schema.json | 2 +- tests/test_entity_resolution.py | 35 +++++++++++++++++++ 4 files changed, 74 insertions(+), 2 deletions(-) diff --git a/docs/entity-resolution-v1.md b/docs/entity-resolution-v1.md index d553d40..343de65 100644 --- a/docs/entity-resolution-v1.md +++ b/docs/entity-resolution-v1.md @@ -216,3 +216,17 @@ IDs/types, valid alternative capture citations, a complete multi-query pipeline, formal/local source separation, two-sided provisional gaps, downstream context propagation, summaries, actual DB integration with fixture model replies, and historical protocol replay. No paid API call is required for these checks. + + +### Capture citation correction feedback + +For `ENTITY_RESOLUTION_CAPTURE_SOURCE`, the next assessment attempt receives +an explicit list of missing V capture IDs, zero-based search indices, exact +`/resolutions/N/source_ids` paths and original mention source IDs. A mention in +`reason` is not a structured citation. The model must return a complete corrected +assessment; validation remains unchanged and citations are never auto-inserted. + +Feedback is frozen in an append-only `feedback-assessment-NNNN.json` checkpoint +before the new call. Existing requests and error artifacts retain their original +hashes. This also supports resuming older failed assessments without repeating +accepted plans or database queries. Retry limits and budgets are unchanged. diff --git a/src/viano_bug_analysis/analysis/entity_resolution.py b/src/viano_bug_analysis/analysis/entity_resolution.py index 21421e1..eafe40a 100644 --- a/src/viano_bug_analysis/analysis/entity_resolution.py +++ b/src/viano_bug_analysis/analysis/entity_resolution.py @@ -237,7 +237,13 @@ class EntityResolver: errors=[] for attempt in range(1,settings["max_invalid_responses"]+1): error_stage=f"error-{stage}-{attempt:04d}" - request=self.owner.gateway.validate_request(request_for(stage,context,settings,errors=errors,**args)) + feedback_stage=f"feedback-{stage}-{attempt:04d}" + with PackageReader(self.store.latest(context['case']['case_id'],context['case']['package_id']),self.store.limits) as r: + feedback=self.read(r,context,feedback_stage) + operation=key(f"{context['case']['analysis_revision']}:entity-resolution:v1:{stage}:{attempt}") + already_started=f"llm/operations/{operation}/request.json" in r.files + request_errors=[*errors[:-1],{**errors[-1],"citation_corrections":feedback}] if feedback and errors else errors + request=self.owner.gateway.validate_request(request_for(stage,context,settings,errors=request_errors,**args)) with PackageReader(self.store.latest(context['case']['case_id'],context['case']['package_id']),self.store.limits) as r: saved=self.read(r,context,stage) if saved and saved['attempt'] == attempt: @@ -266,6 +272,23 @@ class EntityResolver: raise InterpretationError('ENTITY_RESOLUTION_HISTORY') errors.append(previous) continue + if (feedback is None and not already_started and errors and stage == 'assessment' + and errors[-1]['code'] == 'ENTITY_RESOLUTION_CAPTURE_SOURCE'): + with PackageReader(self.store.latest(context['case']['case_id'],context['case']['package_id']),self.store.limits) as r: + rejected=read_json(r,errors[-1]['call_ref'],'llm-call', + self.owner.gateway.config.max_response_bytes+self.owner.gateway.config.max_request_bytes+16384) + document=rejected['outcome']['document'] + feedback={"instruction":"Add each missing capture ID to the structured source_ids array; mentioning it in reason is insufficient. Return the complete corrected assessment.", + "missing_capture_sources":[{"search_index":index,"required_source_id":f"V{index+1:04d}", + "field":f"/resolutions/{index}/source_ids", + "original_mention_source_ids":search['source_ids']} + for index,search in enumerate(args['plan']['searches']) + if f"V{index+1:04d}" not in document['resolutions'][index]['source_ids']]} + # Freeze feedback separately before the new call. Older attempts keep + # their original request hashes; resumed calls use this exact payload. + self.save(context,feedback_stage,feedback) + request_errors=[*errors[:-1],{**errors[-1],"citation_corrections":feedback}] + request=self.owner.gateway.validate_request(request_for(stage,context,settings,errors=request_errors,**args)) i=context['case'] result=await self.owner.runner.run(i['case_id'],i['package_id'],operation_id=f"entity-resolution:v1:{stage}:{attempt}",request=request) if result.status not in {'SUCCESS','ERROR'}: diff --git a/src/viano_bug_analysis/schemas/entity-resolution-v1.schema.json b/src/viano_bug_analysis/schemas/entity-resolution-v1.schema.json index 06326c8..a6ee91e 100644 --- a/src/viano_bug_analysis/schemas/entity-resolution-v1.schema.json +++ b/src/viano_bug_analysis/schemas/entity-resolution-v1.schema.json @@ -24,7 +24,7 @@ }, "stage": { "type": "string", - "pattern": "^(input|plan|search|assessment|error-(plan|assessment)-[0-9]{4})$" + "pattern": "^(input|plan|search|assessment|(error|feedback)-(plan|assessment)-[0-9]{4})$" }, "payload": { "type": "object" diff --git a/tests/test_entity_resolution.py b/tests/test_entity_resolution.py index 5090a63..16e0466 100644 --- a/tests/test_entity_resolution.py +++ b/tests/test_entity_resolution.py @@ -332,3 +332,38 @@ def test_legacy_resolution_requests_and_completed_packages_remain_replayable(bui assert result.status=='COMPLETED' assert model.requests[0]['prompt_version']==f'entity_resolution_plan_v{legacy_version}' assert run(store,model).reused and len(model.requests)==3 + + +def test_missing_capture_feedback_resumes_and_replays_without_repeating_research(built_case): + class MissingCaptureModel(Model): + async def generate(self, config, request, *, max_response_bytes): + response = await super().generate(config, request, max_response_bytes=max_response_bytes) + if request['prompt_version'] == 'entity_resolution_assessment_v3': + data = json.loads(request['messages'][1]['content']) + if 'validation_feedback' not in data: + for resolution in response.document['resolutions']: + resolution['source_ids'] = ['O0001'] + resolution['reason'] = 'The right capture is V0001 but only mentioned here.' + else: + fixes = data['validation_feedback']['citation_corrections'] + assert fixes['missing_capture_sources'] == [{ + 'search_index':0, 'required_source_id':'V0001', + 'field':'/resolutions/0/source_ids', 'original_mention_source_ids':['O0001']}] + return response + store = built_case[0] + model = MissingCaptureModel() + service = service_for(store) + rejected = run(store, model, service) + assert rejected.status == 'REJECTED' + assert rejected.code == 'ENTITY_RESOLUTION_CAPTURE_SOURCE' + assert len(service.database.calls) == 1 + result = run(store, model, service) + assert result.status == 'COMPLETED' + assert len(service.database.calls) == 1 + assert len(model.requests) == 4 + with PackageReader(result.package) as reader: + feedback_ref = 'analysis/interpretations/0001/entity-resolution/feedback-assessment-0002.json' + assert feedback_ref in reader.files + assert PackageValidator().validate(result.package).valid + assert run(store, model).reused + assert len(model.requests) == 4