Scope entity resolution reasoning effort independently

This commit is contained in:
Nico committed 2026-10-15 09:31:30 +02:00
1 parent 6c3415b13a
commit 69e4f3b349
6 files changed
+104 -3

No files matched your search

+2
View File
@@ -19,6 +19,8 @@ token_parameter = "max_tokens"
max_output_tokens = 4096 max_output_tokens = 4096
# Optional; omit for provider defaults. Supported values depend on the model. # Optional; omit for provider defaults. Supported values depend on the model.
# reasoning_effort = "medium" # reasoning_effort = "medium"
# Optional override for entity resolution planning and assessment only:
# entity_resolution_reasoning_effort = "low"
[[llm.providers]] [[llm.providers]]
id = "own-server" id = "own-server"
+21
View File
@@ -40,3 +40,24 @@ The output limit is a ceiling, not a request to generate that many tokens. Think
and output accounting are provider-dependent. More output capacity can increase and output accounting are provider-dependent. More output capacity can increase
cost; monitor actual usage and finish diagnostics. No raw reasoning is requested cost; monitor actual usage and finish diagnostics. No raw reasoning is requested
or persisted by these settings. or persisted by these settings.
## Identity-only reasoning override
Each provider optionally accepts `entity_resolution_reasoning_effort` with the
same validated values as `reasoning_effort`. It applies only to versioned
`entity_resolution_plan_vN` and `entity_resolution_assessment_vN` requests.
All interpretation, hypothesis and investigation requests retain the provider's
normal `reasoning_effort`. Omission preserves previous configuration snapshots.
```toml
reasoning_effort = "medium"
entity_resolution_reasoning_effort = "low"
max_output_tokens = 16384
```
The override is recorded in configuration snapshots and their hashes. Existing
pending calls cannot silently change configuration mid-retry. A new entity
assessment attempt uses the new settings; accepted plan and previous assessment
artifacts remain unchanged and are checked without repeating their API calls.
No attempt counters or investigation budgets are reset.
+6 -3
View File
@@ -33,6 +33,7 @@ class ProviderConfig:
output_cost_per_million: float | None = None output_cost_per_million: float | None = None
cost_currency: str = "USD" cost_currency: str = "USD"
reasoning_effort: str | None = None reasoning_effort: str | None = None
entity_resolution_reasoning_effort: str | None = None
def __post_init__(self): def __post_init__(self):
if not isinstance(self.id, str) or not re.fullmatch(r"[A-Za-z0-9_-]{1,64}", self.id): if not isinstance(self.id, str) or not re.fullmatch(r"[A-Za-z0-9_-]{1,64}", self.id):
@@ -65,7 +66,8 @@ class ProviderConfig:
for value in (self.enabled, self.allow_insecure_http): for value in (self.enabled, self.allow_insecure_http):
if not isinstance(value, bool): if not isinstance(value, bool):
raise LLMError("LLM_CONFIG") raise LLMError("LLM_CONFIG")
if self.reasoning_effort is not None and (not isinstance(self.reasoning_effort, str) or self.reasoning_effort not in {"none", "minimal", "low", "medium", "high"}): for effort in (self.reasoning_effort, self.entity_resolution_reasoning_effort):
if effort is not None and (not isinstance(effort, str) or effort not in {"none", "minimal", "low", "medium", "high"}):
raise LLMError("LLM_CONFIG") raise LLMError("LLM_CONFIG")
positive(self.timeout_seconds) positive(self.timeout_seconds)
positive(self.max_output_tokens, integer=True) positive(self.max_output_tokens, integer=True)
@@ -115,6 +117,7 @@ class GatewayConfig:
# Only configuration references; never resolve credentials for persistence. # Only configuration references; never resolve credentials for persistence.
result = asdict(self) result = asdict(self)
for provider in result["providers"]: for provider in result["providers"]:
if provider["reasoning_effort"] is None: for name in ("reasoning_effort", "entity_resolution_reasoning_effort"):
del provider["reasoning_effort"] if provider[name] is None:
del provider[name]
return result return result
+5
View File
@@ -3,6 +3,8 @@ import copy
import logging import logging
import time import time
import math import math
import re
from dataclasses import replace
from jsonschema import Draft202012Validator from jsonschema import Draft202012Validator
from jsonschema.exceptions import SchemaError, ValidationError from jsonschema.exceptions import SchemaError, ValidationError
@@ -73,6 +75,9 @@ class LLMGateway:
for provider in self.config.providers: for provider in self.config.providers:
if not provider.enabled or provider.privacy not in (PRIVACY[request["privacy"]] & PRIVACY[self.config.privacy_profile]): if not provider.enabled or provider.privacy not in (PRIVACY[request["privacy"]] & PRIVACY[self.config.privacy_profile]):
continue continue
if (provider.entity_resolution_reasoning_effort is not None
and re.fullmatch(r"entity_resolution_(plan|assessment)_v[1-9][0-9]*", request["prompt_version"])):
provider = replace(provider, reasoning_effort=provider.entity_resolution_reasoning_effort)
started = time.monotonic() started = time.monotonic()
response = None response = None
try: try:
+36
View File
@@ -367,3 +367,39 @@ def test_missing_capture_feedback_resumes_and_replays_without_repeating_research
assert PackageValidator().validate(result.package).valid assert PackageValidator().validate(result.package).valid
assert run(store, model).reused assert run(store, model).reused
assert len(model.requests) == 4 assert len(model.requests) == 4
def test_third_assessment_can_use_lower_reasoning_without_resetting_history(built_case):
from dataclasses import replace
from viano_bug_analysis.llm import LLMError, LLMGateway
class TruncatedModel(Model):
failures = 2
efforts = []
async def generate(self, config, request, *, max_response_bytes):
self.efforts.append((request['prompt_version'], config.reasoning_effort))
if request['prompt_version']=='entity_resolution_assessment_v3' and self.failures:
self.failures -= 1
raise LLMError('LLM_INCOMPLETE_OUTPUT')
return await super().generate(config, request, max_response_bytes=max_response_bytes)
store = built_case[0]
model = TruncatedModel()
service = service_for(store)
old = gateway(model)
old.config = replace(old.config, providers=tuple(replace(p, reasoning_effort='medium') for p in old.config.providers))
owner = CaseInterpreter(store, old, project_research=service)
for _ in range(2):
result = asyncio.run(owner.interpret('case-A','stable-package'))
assert result.status == 'REJECTED'
adjusted = replace(old.config, providers=tuple(replace(p, entity_resolution_reasoning_effort='low') for p in old.config.providers))
owner = CaseInterpreter(store, LLMGateway(adjusted, {'fixture': model}), project_research=service)
result = asyncio.run(owner.interpret('case-A','stable-package'))
assert result.status == 'COMPLETED'
assert len(service.database.calls) == 1
assert model.efforts == [('entity_resolution_plan_v3','medium'),
('entity_resolution_assessment_v3','medium'),('entity_resolution_assessment_v3','medium'),
('entity_resolution_assessment_v3','low'),('case_interpretation_resolved_v3','medium')]
with PackageReader(result.package) as reader:
saved = json.loads(reader.read_bytes('analysis/interpretations/0001/entity-resolution/assessment.json'))
assert saved['payload']['attempt'] == 3
assert asyncio.run(owner.interpret('case-A','stable-package')).reused
assert len(model.efforts) == 5
+34
View File
@@ -50,3 +50,37 @@ def test_identity_projection_keeps_readable_evidence_and_legacy_requests(stage):
assert value == context["sources"][key] assert value == context["sources"][key]
assert len(new["messages"][1]["content"]) < len(old["messages"][1]["content"]) / 2 assert len(new["messages"][1]["content"]) < len(old["messages"][1]["content"]) / 2
assert data["output_schema"] == new["output_schema"] assert data["output_schema"] == new["output_schema"]
@pytest.mark.parametrize("version,expected", [
("entity_resolution_plan_v1", "low"),
("entity_resolution_assessment_v3", "low"),
("case_interpretation_resolved_v3", "medium"),
("hypothesis_generation_v1", "medium"),
("investigation_v1", "medium"),
("entity_resolution_assessment_v3_suffix", "medium"),
])
def test_identity_override_is_scoped_at_http_transport(http_endpoint, version, expected):
url, seen, _ = http_endpoint
provider = ProviderConfig(id="test", model="fixture", base_url=url,
reasoning_effort="medium", entity_resolution_reasoning_effort="low", max_output_tokens=16384)
config = GatewayConfig((provider,))
query = request()
query['prompt_version'] = version
asyncio.run(LLMGateway(config).complete_round(query))
assert seen[0]['body']['reasoning_effort'] == expected
assert seen[0]['body'][provider.token_parameter] == 16384
assert provider.reasoning_effort == "medium"
assert config.snapshot()['providers'][0]['entity_resolution_reasoning_effort'] == "low"
@pytest.mark.parametrize("effort", [True, 8192, [], "unlimited"])
def test_invalid_identity_reasoning_override(effort):
with pytest.raises(LLMError, match="LLM_CONFIG"):
ProviderConfig(id="test", model="fixture", base_url="http://localhost",
entity_resolution_reasoning_effort=effort)
def test_omitted_identity_override_preserves_previous_snapshot():
provider = ProviderConfig(id="test", model="fixture", base_url="http://localhost", reasoning_effort="medium")
assert 'entity_resolution_reasoning_effort' not in GatewayConfig((provider,)).snapshot()['providers'][0]