Scope entity resolution reasoning effort independently

This commit is contained in:
Nico committed 2026-10-15 09:31:30 +02:00
1 parent 6c3415b13a
commit 69e4f3b349
6 files changed
+105 -4

No files matched your search

+2
View File
@@ -19,6 +19,8 @@ token_parameter = "max_tokens"
max_output_tokens = 4096
# Optional; omit for provider defaults. Supported values depend on the model.
# reasoning_effort = "medium"
# Optional override for entity resolution planning and assessment only:
# entity_resolution_reasoning_effort = "low"
[[llm.providers]]
id = "own-server"
+21
View File
@@ -40,3 +40,24 @@ The output limit is a ceiling, not a request to generate that many tokens. Think
and output accounting are provider-dependent. More output capacity can increase
cost; monitor actual usage and finish diagnostics. No raw reasoning is requested
or persisted by these settings.
## Identity-only reasoning override
Each provider optionally accepts `entity_resolution_reasoning_effort` with the
same validated values as `reasoning_effort`. It applies only to versioned
`entity_resolution_plan_vN` and `entity_resolution_assessment_vN` requests.
All interpretation, hypothesis and investigation requests retain the provider's
normal `reasoning_effort`. Omission preserves previous configuration snapshots.
```toml
reasoning_effort = "medium"
entity_resolution_reasoning_effort = "low"
max_output_tokens = 16384
```
The override is recorded in configuration snapshots and their hashes. Existing
pending calls cannot silently change configuration mid-retry. A new entity
assessment attempt uses the new settings; accepted plan and previous assessment
artifacts remain unchanged and are checked without repeating their API calls.
No attempt counters or investigation budgets are reset.
+7 -4
View File
@@ -33,6 +33,7 @@ class ProviderConfig:
output_cost_per_million: float | None = None
cost_currency: str = "USD"
reasoning_effort: str | None = None
entity_resolution_reasoning_effort: str | None = None
def __post_init__(self):
if not isinstance(self.id, str) or not re.fullmatch(r"[A-Za-z0-9_-]{1,64}", self.id):
@@ -65,8 +66,9 @@ class ProviderConfig:
for value in (self.enabled, self.allow_insecure_http):
if not isinstance(value, bool):
raise LLMError("LLM_CONFIG")
if self.reasoning_effort is not None and (not isinstance(self.reasoning_effort, str) or self.reasoning_effort not in {"none", "minimal", "low", "medium", "high"}):
raise LLMError("LLM_CONFIG")
for effort in (self.reasoning_effort, self.entity_resolution_reasoning_effort):
if effort is not None and (not isinstance(effort, str) or effort not in {"none", "minimal", "low", "medium", "high"}):
raise LLMError("LLM_CONFIG")
positive(self.timeout_seconds)
positive(self.max_output_tokens, integer=True)
if not isinstance(self.cost_currency, str) or not re.fullmatch(r"[A-Z]{3}", self.cost_currency):
@@ -115,6 +117,7 @@ class GatewayConfig:
# Only configuration references; never resolve credentials for persistence.
result = asdict(self)
for provider in result["providers"]:
if provider["reasoning_effort"] is None:
del provider["reasoning_effort"]
for name in ("reasoning_effort", "entity_resolution_reasoning_effort"):
if provider[name] is None:
del provider[name]
return result
+5
View File
@@ -3,6 +3,8 @@ import copy
import logging
import time
import math
import re
from dataclasses import replace
from jsonschema import Draft202012Validator
from jsonschema.exceptions import SchemaError, ValidationError
@@ -73,6 +75,9 @@ class LLMGateway:
for provider in self.config.providers:
if not provider.enabled or provider.privacy not in (PRIVACY[request["privacy"]] & PRIVACY[self.config.privacy_profile]):
continue
if (provider.entity_resolution_reasoning_effort is not None
and re.fullmatch(r"entity_resolution_(plan|assessment)_v[1-9][0-9]*", request["prompt_version"])):
provider = replace(provider, reasoning_effort=provider.entity_resolution_reasoning_effort)
started = time.monotonic()
response = None
try:
+36
View File
@@ -367,3 +367,39 @@ def test_missing_capture_feedback_resumes_and_replays_without_repeating_research
assert PackageValidator().validate(result.package).valid
assert run(store, model).reused
assert len(model.requests) == 4
def test_third_assessment_can_use_lower_reasoning_without_resetting_history(built_case):
from dataclasses import replace
from viano_bug_analysis.llm import LLMError, LLMGateway
class TruncatedModel(Model):
failures = 2
efforts = []
async def generate(self, config, request, *, max_response_bytes):
self.efforts.append((request['prompt_version'], config.reasoning_effort))
if request['prompt_version']=='entity_resolution_assessment_v3' and self.failures:
self.failures -= 1
raise LLMError('LLM_INCOMPLETE_OUTPUT')
return await super().generate(config, request, max_response_bytes=max_response_bytes)
store = built_case[0]
model = TruncatedModel()
service = service_for(store)
old = gateway(model)
old.config = replace(old.config, providers=tuple(replace(p, reasoning_effort='medium') for p in old.config.providers))
owner = CaseInterpreter(store, old, project_research=service)
for _ in range(2):
result = asyncio.run(owner.interpret('case-A','stable-package'))
assert result.status == 'REJECTED'
adjusted = replace(old.config, providers=tuple(replace(p, entity_resolution_reasoning_effort='low') for p in old.config.providers))
owner = CaseInterpreter(store, LLMGateway(adjusted, {'fixture': model}), project_research=service)
result = asyncio.run(owner.interpret('case-A','stable-package'))
assert result.status == 'COMPLETED'
assert len(service.database.calls) == 1
assert model.efforts == [('entity_resolution_plan_v3','medium'),
('entity_resolution_assessment_v3','medium'),('entity_resolution_assessment_v3','medium'),
('entity_resolution_assessment_v3','low'),('case_interpretation_resolved_v3','medium')]
with PackageReader(result.package) as reader:
saved = json.loads(reader.read_bytes('analysis/interpretations/0001/entity-resolution/assessment.json'))
assert saved['payload']['attempt'] == 3
assert asyncio.run(owner.interpret('case-A','stable-package')).reused
assert len(model.efforts) == 5
+34
View File
@@ -50,3 +50,37 @@ def test_identity_projection_keeps_readable_evidence_and_legacy_requests(stage):
assert value == context["sources"][key]
assert len(new["messages"][1]["content"]) < len(old["messages"][1]["content"]) / 2
assert data["output_schema"] == new["output_schema"]
@pytest.mark.parametrize("version,expected", [
("entity_resolution_plan_v1", "low"),
("entity_resolution_assessment_v3", "low"),
("case_interpretation_resolved_v3", "medium"),
("hypothesis_generation_v1", "medium"),
("investigation_v1", "medium"),
("entity_resolution_assessment_v3_suffix", "medium"),
])
def test_identity_override_is_scoped_at_http_transport(http_endpoint, version, expected):
url, seen, _ = http_endpoint
provider = ProviderConfig(id="test", model="fixture", base_url=url,
reasoning_effort="medium", entity_resolution_reasoning_effort="low", max_output_tokens=16384)
config = GatewayConfig((provider,))
query = request()
query['prompt_version'] = version
asyncio.run(LLMGateway(config).complete_round(query))
assert seen[0]['body']['reasoning_effort'] == expected
assert seen[0]['body'][provider.token_parameter] == 16384
assert provider.reasoning_effort == "medium"
assert config.snapshot()['providers'][0]['entity_resolution_reasoning_effort'] == "low"
@pytest.mark.parametrize("effort", [True, 8192, [], "unlimited"])
def test_invalid_identity_reasoning_override(effort):
with pytest.raises(LLMError, match="LLM_CONFIG"):
ProviderConfig(id="test", model="fixture", base_url="http://localhost",
entity_resolution_reasoning_effort=effort)
def test_omitted_identity_override_preserves_previous_snapshot():
provider = ProviderConfig(id="test", model="fixture", base_url="http://localhost", reasoning_effort="medium")
assert 'entity_resolution_reasoning_effort' not in GatewayConfig((provider,)).snapshot()['providers'][0]