Explain missing capture citations in entity assessment corrections
This commit is contained in:
1 parent
79a6934b9e
commit
6c3415b13a
4 files changed
+74
-2
No files matched your search
@@ -216,3 +216,17 @@ IDs/types, valid alternative capture citations, a complete multi-query pipeline,
|
||||
formal/local source separation, two-sided provisional gaps, downstream context
|
||||
propagation, summaries, actual DB integration with fixture model replies, and
|
||||
historical protocol replay. No paid API call is required for these checks.
|
||||
|
||||
|
||||
### Capture citation correction feedback
|
||||
|
||||
For `ENTITY_RESOLUTION_CAPTURE_SOURCE`, the next assessment attempt receives
|
||||
an explicit list of missing V capture IDs, zero-based search indices, exact
|
||||
`/resolutions/N/source_ids` paths and original mention source IDs. A mention in
|
||||
`reason` is not a structured citation. The model must return a complete corrected
|
||||
assessment; validation remains unchanged and citations are never auto-inserted.
|
||||
|
||||
Feedback is frozen in an append-only `feedback-assessment-NNNN.json` checkpoint
|
||||
before the new call. Existing requests and error artifacts retain their original
|
||||
hashes. This also supports resuming older failed assessments without repeating
|
||||
accepted plans or database queries. Retry limits and budgets are unchanged.
|
||||
@@ -237,7 +237,13 @@ class EntityResolver:
|
||||
errors=[]
|
||||
for attempt in range(1,settings["max_invalid_responses"]+1):
|
||||
error_stage=f"error-{stage}-{attempt:04d}"
|
||||
request=self.owner.gateway.validate_request(request_for(stage,context,settings,errors=errors,**args))
|
||||
feedback_stage=f"feedback-{stage}-{attempt:04d}"
|
||||
with PackageReader(self.store.latest(context['case']['case_id'],context['case']['package_id']),self.store.limits) as r:
|
||||
feedback=self.read(r,context,feedback_stage)
|
||||
operation=key(f"{context['case']['analysis_revision']}:entity-resolution:v1:{stage}:{attempt}")
|
||||
already_started=f"llm/operations/{operation}/request.json" in r.files
|
||||
request_errors=[*errors[:-1],{**errors[-1],"citation_corrections":feedback}] if feedback and errors else errors
|
||||
request=self.owner.gateway.validate_request(request_for(stage,context,settings,errors=request_errors,**args))
|
||||
with PackageReader(self.store.latest(context['case']['case_id'],context['case']['package_id']),self.store.limits) as r:
|
||||
saved=self.read(r,context,stage)
|
||||
if saved and saved['attempt'] == attempt:
|
||||
@@ -266,6 +272,23 @@ class EntityResolver:
|
||||
raise InterpretationError('ENTITY_RESOLUTION_HISTORY')
|
||||
errors.append(previous)
|
||||
continue
|
||||
if (feedback is None and not already_started and errors and stage == 'assessment'
|
||||
and errors[-1]['code'] == 'ENTITY_RESOLUTION_CAPTURE_SOURCE'):
|
||||
with PackageReader(self.store.latest(context['case']['case_id'],context['case']['package_id']),self.store.limits) as r:
|
||||
rejected=read_json(r,errors[-1]['call_ref'],'llm-call',
|
||||
self.owner.gateway.config.max_response_bytes+self.owner.gateway.config.max_request_bytes+16384)
|
||||
document=rejected['outcome']['document']
|
||||
feedback={"instruction":"Add each missing capture ID to the structured source_ids array; mentioning it in reason is insufficient. Return the complete corrected assessment.",
|
||||
"missing_capture_sources":[{"search_index":index,"required_source_id":f"V{index+1:04d}",
|
||||
"field":f"/resolutions/{index}/source_ids",
|
||||
"original_mention_source_ids":search['source_ids']}
|
||||
for index,search in enumerate(args['plan']['searches'])
|
||||
if f"V{index+1:04d}" not in document['resolutions'][index]['source_ids']]}
|
||||
# Freeze feedback separately before the new call. Older attempts keep
|
||||
# their original request hashes; resumed calls use this exact payload.
|
||||
self.save(context,feedback_stage,feedback)
|
||||
request_errors=[*errors[:-1],{**errors[-1],"citation_corrections":feedback}]
|
||||
request=self.owner.gateway.validate_request(request_for(stage,context,settings,errors=request_errors,**args))
|
||||
i=context['case']
|
||||
result=await self.owner.runner.run(i['case_id'],i['package_id'],operation_id=f"entity-resolution:v1:{stage}:{attempt}",request=request)
|
||||
if result.status not in {'SUCCESS','ERROR'}:
|
||||
|
||||
@@ -24,7 +24,7 @@
|
||||
},
|
||||
"stage": {
|
||||
"type": "string",
|
||||
"pattern": "^(input|plan|search|assessment|error-(plan|assessment)-[0-9]{4})$"
|
||||
"pattern": "^(input|plan|search|assessment|(error|feedback)-(plan|assessment)-[0-9]{4})$"
|
||||
},
|
||||
"payload": {
|
||||
"type": "object"
|
||||
|
||||
@@ -332,3 +332,38 @@ def test_legacy_resolution_requests_and_completed_packages_remain_replayable(bui
|
||||
assert result.status=='COMPLETED'
|
||||
assert model.requests[0]['prompt_version']==f'entity_resolution_plan_v{legacy_version}'
|
||||
assert run(store,model).reused and len(model.requests)==3
|
||||
|
||||
|
||||
def test_missing_capture_feedback_resumes_and_replays_without_repeating_research(built_case):
|
||||
class MissingCaptureModel(Model):
|
||||
async def generate(self, config, request, *, max_response_bytes):
|
||||
response = await super().generate(config, request, max_response_bytes=max_response_bytes)
|
||||
if request['prompt_version'] == 'entity_resolution_assessment_v3':
|
||||
data = json.loads(request['messages'][1]['content'])
|
||||
if 'validation_feedback' not in data:
|
||||
for resolution in response.document['resolutions']:
|
||||
resolution['source_ids'] = ['O0001']
|
||||
resolution['reason'] = 'The right capture is V0001 but only mentioned here.'
|
||||
else:
|
||||
fixes = data['validation_feedback']['citation_corrections']
|
||||
assert fixes['missing_capture_sources'] == [{
|
||||
'search_index':0, 'required_source_id':'V0001',
|
||||
'field':'/resolutions/0/source_ids', 'original_mention_source_ids':['O0001']}]
|
||||
return response
|
||||
store = built_case[0]
|
||||
model = MissingCaptureModel()
|
||||
service = service_for(store)
|
||||
rejected = run(store, model, service)
|
||||
assert rejected.status == 'REJECTED'
|
||||
assert rejected.code == 'ENTITY_RESOLUTION_CAPTURE_SOURCE'
|
||||
assert len(service.database.calls) == 1
|
||||
result = run(store, model, service)
|
||||
assert result.status == 'COMPLETED'
|
||||
assert len(service.database.calls) == 1
|
||||
assert len(model.requests) == 4
|
||||
with PackageReader(result.package) as reader:
|
||||
feedback_ref = 'analysis/interpretations/0001/entity-resolution/feedback-assessment-0002.json'
|
||||
assert feedback_ref in reader.files
|
||||
assert PackageValidator().validate(result.package).valid
|
||||
assert run(store, model).reused
|
||||
assert len(model.requests) == 4
|
||||
Reference in new issue
Block a user