Explain missing capture citations in entity assessment corrections

This commit is contained in:
Nico committed 2026-10-15 09:04:40 +02:00
1 parent 79a6934b9e
commit 6c3415b13a
4 files changed
+74 -2

No files matched your search

+14
View File
@@ -216,3 +216,17 @@ IDs/types, valid alternative capture citations, a complete multi-query pipeline,
formal/local source separation, two-sided provisional gaps, downstream context
propagation, summaries, actual DB integration with fixture model replies, and
historical protocol replay. No paid API call is required for these checks.
### Capture citation correction feedback
For `ENTITY_RESOLUTION_CAPTURE_SOURCE`, the next assessment attempt receives
an explicit list of missing V capture IDs, zero-based search indices, exact
`/resolutions/N/source_ids` paths and original mention source IDs. A mention in
`reason` is not a structured citation. The model must return a complete corrected
assessment; validation remains unchanged and citations are never auto-inserted.
Feedback is frozen in an append-only `feedback-assessment-NNNN.json` checkpoint
before the new call. Existing requests and error artifacts retain their original
hashes. This also supports resuming older failed assessments without repeating
accepted plans or database queries. Retry limits and budgets are unchanged.
@@ -237,7 +237,13 @@ class EntityResolver:
errors=[]
for attempt in range(1,settings["max_invalid_responses"]+1):
error_stage=f"error-{stage}-{attempt:04d}"
request=self.owner.gateway.validate_request(request_for(stage,context,settings,errors=errors,**args))
feedback_stage=f"feedback-{stage}-{attempt:04d}"
with PackageReader(self.store.latest(context['case']['case_id'],context['case']['package_id']),self.store.limits) as r:
feedback=self.read(r,context,feedback_stage)
operation=key(f"{context['case']['analysis_revision']}:entity-resolution:v1:{stage}:{attempt}")
already_started=f"llm/operations/{operation}/request.json" in r.files
request_errors=[*errors[:-1],{**errors[-1],"citation_corrections":feedback}] if feedback and errors else errors
request=self.owner.gateway.validate_request(request_for(stage,context,settings,errors=request_errors,**args))
with PackageReader(self.store.latest(context['case']['case_id'],context['case']['package_id']),self.store.limits) as r:
saved=self.read(r,context,stage)
if saved and saved['attempt'] == attempt:
@@ -266,6 +272,23 @@ class EntityResolver:
raise InterpretationError('ENTITY_RESOLUTION_HISTORY')
errors.append(previous)
continue
if (feedback is None and not already_started and errors and stage == 'assessment'
and errors[-1]['code'] == 'ENTITY_RESOLUTION_CAPTURE_SOURCE'):
with PackageReader(self.store.latest(context['case']['case_id'],context['case']['package_id']),self.store.limits) as r:
rejected=read_json(r,errors[-1]['call_ref'],'llm-call',
self.owner.gateway.config.max_response_bytes+self.owner.gateway.config.max_request_bytes+16384)
document=rejected['outcome']['document']
feedback={"instruction":"Add each missing capture ID to the structured source_ids array; mentioning it in reason is insufficient. Return the complete corrected assessment.",
"missing_capture_sources":[{"search_index":index,"required_source_id":f"V{index+1:04d}",
"field":f"/resolutions/{index}/source_ids",
"original_mention_source_ids":search['source_ids']}
for index,search in enumerate(args['plan']['searches'])
if f"V{index+1:04d}" not in document['resolutions'][index]['source_ids']]}
# Freeze feedback separately before the new call. Older attempts keep
# their original request hashes; resumed calls use this exact payload.
self.save(context,feedback_stage,feedback)
request_errors=[*errors[:-1],{**errors[-1],"citation_corrections":feedback}]
request=self.owner.gateway.validate_request(request_for(stage,context,settings,errors=request_errors,**args))
i=context['case']
result=await self.owner.runner.run(i['case_id'],i['package_id'],operation_id=f"entity-resolution:v1:{stage}:{attempt}",request=request)
if result.status not in {'SUCCESS','ERROR'}:
@@ -24,7 +24,7 @@
},
"stage": {
"type": "string",
"pattern": "^(input|plan|search|assessment|error-(plan|assessment)-[0-9]{4})$"
"pattern": "^(input|plan|search|assessment|(error|feedback)-(plan|assessment)-[0-9]{4})$"
},
"payload": {
"type": "object"
+35
View File
@@ -332,3 +332,38 @@ def test_legacy_resolution_requests_and_completed_packages_remain_replayable(bui
assert result.status=='COMPLETED'
assert model.requests[0]['prompt_version']==f'entity_resolution_plan_v{legacy_version}'
assert run(store,model).reused and len(model.requests)==3
def test_missing_capture_feedback_resumes_and_replays_without_repeating_research(built_case):
class MissingCaptureModel(Model):
async def generate(self, config, request, *, max_response_bytes):
response = await super().generate(config, request, max_response_bytes=max_response_bytes)
if request['prompt_version'] == 'entity_resolution_assessment_v3':
data = json.loads(request['messages'][1]['content'])
if 'validation_feedback' not in data:
for resolution in response.document['resolutions']:
resolution['source_ids'] = ['O0001']
resolution['reason'] = 'The right capture is V0001 but only mentioned here.'
else:
fixes = data['validation_feedback']['citation_corrections']
assert fixes['missing_capture_sources'] == [{
'search_index':0, 'required_source_id':'V0001',
'field':'/resolutions/0/source_ids', 'original_mention_source_ids':['O0001']}]
return response
store = built_case[0]
model = MissingCaptureModel()
service = service_for(store)
rejected = run(store, model, service)
assert rejected.status == 'REJECTED'
assert rejected.code == 'ENTITY_RESOLUTION_CAPTURE_SOURCE'
assert len(service.database.calls) == 1
result = run(store, model, service)
assert result.status == 'COMPLETED'
assert len(service.database.calls) == 1
assert len(model.requests) == 4
with PackageReader(result.package) as reader:
feedback_ref = 'analysis/interpretations/0001/entity-resolution/feedback-assessment-0002.json'
assert feedback_ref in reader.files
assert PackageValidator().validate(result.package).valid
assert run(store, model).reused
assert len(model.requests) == 4