Scope entity resolution reasoning effort independently
This commit is contained in:
1 parent
6c3415b13a
commit
69e4f3b349
6 files changed
+105
-4
No files matched your search
@@ -19,6 +19,8 @@ token_parameter = "max_tokens"
|
||||
max_output_tokens = 4096
|
||||
# Optional; omit for provider defaults. Supported values depend on the model.
|
||||
# reasoning_effort = "medium"
|
||||
# Optional override for entity resolution planning and assessment only:
|
||||
# entity_resolution_reasoning_effort = "low"
|
||||
|
||||
[[llm.providers]]
|
||||
id = "own-server"
|
||||
|
||||
@@ -40,3 +40,24 @@ The output limit is a ceiling, not a request to generate that many tokens. Think
|
||||
and output accounting are provider-dependent. More output capacity can increase
|
||||
cost; monitor actual usage and finish diagnostics. No raw reasoning is requested
|
||||
or persisted by these settings.
|
||||
|
||||
|
||||
## Identity-only reasoning override
|
||||
|
||||
Each provider optionally accepts `entity_resolution_reasoning_effort` with the
|
||||
same validated values as `reasoning_effort`. It applies only to versioned
|
||||
`entity_resolution_plan_vN` and `entity_resolution_assessment_vN` requests.
|
||||
All interpretation, hypothesis and investigation requests retain the provider's
|
||||
normal `reasoning_effort`. Omission preserves previous configuration snapshots.
|
||||
|
||||
```toml
|
||||
reasoning_effort = "medium"
|
||||
entity_resolution_reasoning_effort = "low"
|
||||
max_output_tokens = 16384
|
||||
```
|
||||
|
||||
The override is recorded in configuration snapshots and their hashes. Existing
|
||||
pending calls cannot silently change configuration mid-retry. A new entity
|
||||
assessment attempt uses the new settings; accepted plan and previous assessment
|
||||
artifacts remain unchanged and are checked without repeating their API calls.
|
||||
No attempt counters or investigation budgets are reset.
|
||||
@@ -33,6 +33,7 @@ class ProviderConfig:
|
||||
output_cost_per_million: float | None = None
|
||||
cost_currency: str = "USD"
|
||||
reasoning_effort: str | None = None
|
||||
entity_resolution_reasoning_effort: str | None = None
|
||||
|
||||
def __post_init__(self):
|
||||
if not isinstance(self.id, str) or not re.fullmatch(r"[A-Za-z0-9_-]{1,64}", self.id):
|
||||
@@ -65,8 +66,9 @@ class ProviderConfig:
|
||||
for value in (self.enabled, self.allow_insecure_http):
|
||||
if not isinstance(value, bool):
|
||||
raise LLMError("LLM_CONFIG")
|
||||
if self.reasoning_effort is not None and (not isinstance(self.reasoning_effort, str) or self.reasoning_effort not in {"none", "minimal", "low", "medium", "high"}):
|
||||
raise LLMError("LLM_CONFIG")
|
||||
for effort in (self.reasoning_effort, self.entity_resolution_reasoning_effort):
|
||||
if effort is not None and (not isinstance(effort, str) or effort not in {"none", "minimal", "low", "medium", "high"}):
|
||||
raise LLMError("LLM_CONFIG")
|
||||
positive(self.timeout_seconds)
|
||||
positive(self.max_output_tokens, integer=True)
|
||||
if not isinstance(self.cost_currency, str) or not re.fullmatch(r"[A-Z]{3}", self.cost_currency):
|
||||
@@ -115,6 +117,7 @@ class GatewayConfig:
|
||||
# Only configuration references; never resolve credentials for persistence.
|
||||
result = asdict(self)
|
||||
for provider in result["providers"]:
|
||||
if provider["reasoning_effort"] is None:
|
||||
del provider["reasoning_effort"]
|
||||
for name in ("reasoning_effort", "entity_resolution_reasoning_effort"):
|
||||
if provider[name] is None:
|
||||
del provider[name]
|
||||
return result
|
||||
@@ -3,6 +3,8 @@ import copy
|
||||
import logging
|
||||
import time
|
||||
import math
|
||||
import re
|
||||
from dataclasses import replace
|
||||
|
||||
from jsonschema import Draft202012Validator
|
||||
from jsonschema.exceptions import SchemaError, ValidationError
|
||||
@@ -73,6 +75,9 @@ class LLMGateway:
|
||||
for provider in self.config.providers:
|
||||
if not provider.enabled or provider.privacy not in (PRIVACY[request["privacy"]] & PRIVACY[self.config.privacy_profile]):
|
||||
continue
|
||||
if (provider.entity_resolution_reasoning_effort is not None
|
||||
and re.fullmatch(r"entity_resolution_(plan|assessment)_v[1-9][0-9]*", request["prompt_version"])):
|
||||
provider = replace(provider, reasoning_effort=provider.entity_resolution_reasoning_effort)
|
||||
started = time.monotonic()
|
||||
response = None
|
||||
try:
|
||||
|
||||
@@ -367,3 +367,39 @@ def test_missing_capture_feedback_resumes_and_replays_without_repeating_research
|
||||
assert PackageValidator().validate(result.package).valid
|
||||
assert run(store, model).reused
|
||||
assert len(model.requests) == 4
|
||||
|
||||
|
||||
def test_third_assessment_can_use_lower_reasoning_without_resetting_history(built_case):
|
||||
from dataclasses import replace
|
||||
from viano_bug_analysis.llm import LLMError, LLMGateway
|
||||
class TruncatedModel(Model):
|
||||
failures = 2
|
||||
efforts = []
|
||||
async def generate(self, config, request, *, max_response_bytes):
|
||||
self.efforts.append((request['prompt_version'], config.reasoning_effort))
|
||||
if request['prompt_version']=='entity_resolution_assessment_v3' and self.failures:
|
||||
self.failures -= 1
|
||||
raise LLMError('LLM_INCOMPLETE_OUTPUT')
|
||||
return await super().generate(config, request, max_response_bytes=max_response_bytes)
|
||||
store = built_case[0]
|
||||
model = TruncatedModel()
|
||||
service = service_for(store)
|
||||
old = gateway(model)
|
||||
old.config = replace(old.config, providers=tuple(replace(p, reasoning_effort='medium') for p in old.config.providers))
|
||||
owner = CaseInterpreter(store, old, project_research=service)
|
||||
for _ in range(2):
|
||||
result = asyncio.run(owner.interpret('case-A','stable-package'))
|
||||
assert result.status == 'REJECTED'
|
||||
adjusted = replace(old.config, providers=tuple(replace(p, entity_resolution_reasoning_effort='low') for p in old.config.providers))
|
||||
owner = CaseInterpreter(store, LLMGateway(adjusted, {'fixture': model}), project_research=service)
|
||||
result = asyncio.run(owner.interpret('case-A','stable-package'))
|
||||
assert result.status == 'COMPLETED'
|
||||
assert len(service.database.calls) == 1
|
||||
assert model.efforts == [('entity_resolution_plan_v3','medium'),
|
||||
('entity_resolution_assessment_v3','medium'),('entity_resolution_assessment_v3','medium'),
|
||||
('entity_resolution_assessment_v3','low'),('case_interpretation_resolved_v3','medium')]
|
||||
with PackageReader(result.package) as reader:
|
||||
saved = json.loads(reader.read_bytes('analysis/interpretations/0001/entity-resolution/assessment.json'))
|
||||
assert saved['payload']['attempt'] == 3
|
||||
assert asyncio.run(owner.interpret('case-A','stable-package')).reused
|
||||
assert len(model.efforts) == 5
|
||||
@@ -50,3 +50,37 @@ def test_identity_projection_keeps_readable_evidence_and_legacy_requests(stage):
|
||||
assert value == context["sources"][key]
|
||||
assert len(new["messages"][1]["content"]) < len(old["messages"][1]["content"]) / 2
|
||||
assert data["output_schema"] == new["output_schema"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("version,expected", [
|
||||
("entity_resolution_plan_v1", "low"),
|
||||
("entity_resolution_assessment_v3", "low"),
|
||||
("case_interpretation_resolved_v3", "medium"),
|
||||
("hypothesis_generation_v1", "medium"),
|
||||
("investigation_v1", "medium"),
|
||||
("entity_resolution_assessment_v3_suffix", "medium"),
|
||||
])
|
||||
def test_identity_override_is_scoped_at_http_transport(http_endpoint, version, expected):
|
||||
url, seen, _ = http_endpoint
|
||||
provider = ProviderConfig(id="test", model="fixture", base_url=url,
|
||||
reasoning_effort="medium", entity_resolution_reasoning_effort="low", max_output_tokens=16384)
|
||||
config = GatewayConfig((provider,))
|
||||
query = request()
|
||||
query['prompt_version'] = version
|
||||
asyncio.run(LLMGateway(config).complete_round(query))
|
||||
assert seen[0]['body']['reasoning_effort'] == expected
|
||||
assert seen[0]['body'][provider.token_parameter] == 16384
|
||||
assert provider.reasoning_effort == "medium"
|
||||
assert config.snapshot()['providers'][0]['entity_resolution_reasoning_effort'] == "low"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("effort", [True, 8192, [], "unlimited"])
|
||||
def test_invalid_identity_reasoning_override(effort):
|
||||
with pytest.raises(LLMError, match="LLM_CONFIG"):
|
||||
ProviderConfig(id="test", model="fixture", base_url="http://localhost",
|
||||
entity_resolution_reasoning_effort=effort)
|
||||
|
||||
|
||||
def test_omitted_identity_override_preserves_previous_snapshot():
|
||||
provider = ProviderConfig(id="test", model="fixture", base_url="http://localhost", reasoning_effort="medium")
|
||||
assert 'entity_resolution_reasoning_effort' not in GatewayConfig((provider,)).snapshot()['providers'][0]
|
||||
Reference in new issue
Block a user