Scope entity resolution reasoning effort independently
This commit is contained in:
1 parent
6c3415b13a
commit
69e4f3b349
6 files changed
+104
-3
No files matched your search
@@ -19,6 +19,8 @@ token_parameter = "max_tokens"
|
|||||||
max_output_tokens = 4096
|
max_output_tokens = 4096
|
||||||
# Optional; omit for provider defaults. Supported values depend on the model.
|
# Optional; omit for provider defaults. Supported values depend on the model.
|
||||||
# reasoning_effort = "medium"
|
# reasoning_effort = "medium"
|
||||||
|
# Optional override for entity resolution planning and assessment only:
|
||||||
|
# entity_resolution_reasoning_effort = "low"
|
||||||
|
|
||||||
[[llm.providers]]
|
[[llm.providers]]
|
||||||
id = "own-server"
|
id = "own-server"
|
||||||
|
|||||||
@@ -40,3 +40,24 @@ The output limit is a ceiling, not a request to generate that many tokens. Think
|
|||||||
and output accounting are provider-dependent. More output capacity can increase
|
and output accounting are provider-dependent. More output capacity can increase
|
||||||
cost; monitor actual usage and finish diagnostics. No raw reasoning is requested
|
cost; monitor actual usage and finish diagnostics. No raw reasoning is requested
|
||||||
or persisted by these settings.
|
or persisted by these settings.
|
||||||
|
|
||||||
|
|
||||||
|
## Identity-only reasoning override
|
||||||
|
|
||||||
|
Each provider optionally accepts `entity_resolution_reasoning_effort` with the
|
||||||
|
same validated values as `reasoning_effort`. It applies only to versioned
|
||||||
|
`entity_resolution_plan_vN` and `entity_resolution_assessment_vN` requests.
|
||||||
|
All interpretation, hypothesis and investigation requests retain the provider's
|
||||||
|
normal `reasoning_effort`. Omission preserves previous configuration snapshots.
|
||||||
|
|
||||||
|
```toml
|
||||||
|
reasoning_effort = "medium"
|
||||||
|
entity_resolution_reasoning_effort = "low"
|
||||||
|
max_output_tokens = 16384
|
||||||
|
```
|
||||||
|
|
||||||
|
The override is recorded in configuration snapshots and their hashes. Existing
|
||||||
|
pending calls cannot silently change configuration mid-retry. A new entity
|
||||||
|
assessment attempt uses the new settings; accepted plan and previous assessment
|
||||||
|
artifacts remain unchanged and are checked without repeating their API calls.
|
||||||
|
No attempt counters or investigation budgets are reset.
|
||||||
@@ -33,6 +33,7 @@ class ProviderConfig:
|
|||||||
output_cost_per_million: float | None = None
|
output_cost_per_million: float | None = None
|
||||||
cost_currency: str = "USD"
|
cost_currency: str = "USD"
|
||||||
reasoning_effort: str | None = None
|
reasoning_effort: str | None = None
|
||||||
|
entity_resolution_reasoning_effort: str | None = None
|
||||||
|
|
||||||
def __post_init__(self):
|
def __post_init__(self):
|
||||||
if not isinstance(self.id, str) or not re.fullmatch(r"[A-Za-z0-9_-]{1,64}", self.id):
|
if not isinstance(self.id, str) or not re.fullmatch(r"[A-Za-z0-9_-]{1,64}", self.id):
|
||||||
@@ -65,7 +66,8 @@ class ProviderConfig:
|
|||||||
for value in (self.enabled, self.allow_insecure_http):
|
for value in (self.enabled, self.allow_insecure_http):
|
||||||
if not isinstance(value, bool):
|
if not isinstance(value, bool):
|
||||||
raise LLMError("LLM_CONFIG")
|
raise LLMError("LLM_CONFIG")
|
||||||
if self.reasoning_effort is not None and (not isinstance(self.reasoning_effort, str) or self.reasoning_effort not in {"none", "minimal", "low", "medium", "high"}):
|
for effort in (self.reasoning_effort, self.entity_resolution_reasoning_effort):
|
||||||
|
if effort is not None and (not isinstance(effort, str) or effort not in {"none", "minimal", "low", "medium", "high"}):
|
||||||
raise LLMError("LLM_CONFIG")
|
raise LLMError("LLM_CONFIG")
|
||||||
positive(self.timeout_seconds)
|
positive(self.timeout_seconds)
|
||||||
positive(self.max_output_tokens, integer=True)
|
positive(self.max_output_tokens, integer=True)
|
||||||
@@ -115,6 +117,7 @@ class GatewayConfig:
|
|||||||
# Only configuration references; never resolve credentials for persistence.
|
# Only configuration references; never resolve credentials for persistence.
|
||||||
result = asdict(self)
|
result = asdict(self)
|
||||||
for provider in result["providers"]:
|
for provider in result["providers"]:
|
||||||
if provider["reasoning_effort"] is None:
|
for name in ("reasoning_effort", "entity_resolution_reasoning_effort"):
|
||||||
del provider["reasoning_effort"]
|
if provider[name] is None:
|
||||||
|
del provider[name]
|
||||||
return result
|
return result
|
||||||
@@ -3,6 +3,8 @@ import copy
|
|||||||
import logging
|
import logging
|
||||||
import time
|
import time
|
||||||
import math
|
import math
|
||||||
|
import re
|
||||||
|
from dataclasses import replace
|
||||||
|
|
||||||
from jsonschema import Draft202012Validator
|
from jsonschema import Draft202012Validator
|
||||||
from jsonschema.exceptions import SchemaError, ValidationError
|
from jsonschema.exceptions import SchemaError, ValidationError
|
||||||
@@ -73,6 +75,9 @@ class LLMGateway:
|
|||||||
for provider in self.config.providers:
|
for provider in self.config.providers:
|
||||||
if not provider.enabled or provider.privacy not in (PRIVACY[request["privacy"]] & PRIVACY[self.config.privacy_profile]):
|
if not provider.enabled or provider.privacy not in (PRIVACY[request["privacy"]] & PRIVACY[self.config.privacy_profile]):
|
||||||
continue
|
continue
|
||||||
|
if (provider.entity_resolution_reasoning_effort is not None
|
||||||
|
and re.fullmatch(r"entity_resolution_(plan|assessment)_v[1-9][0-9]*", request["prompt_version"])):
|
||||||
|
provider = replace(provider, reasoning_effort=provider.entity_resolution_reasoning_effort)
|
||||||
started = time.monotonic()
|
started = time.monotonic()
|
||||||
response = None
|
response = None
|
||||||
try:
|
try:
|
||||||
|
|||||||
@@ -367,3 +367,39 @@ def test_missing_capture_feedback_resumes_and_replays_without_repeating_research
|
|||||||
assert PackageValidator().validate(result.package).valid
|
assert PackageValidator().validate(result.package).valid
|
||||||
assert run(store, model).reused
|
assert run(store, model).reused
|
||||||
assert len(model.requests) == 4
|
assert len(model.requests) == 4
|
||||||
|
|
||||||
|
|
||||||
|
def test_third_assessment_can_use_lower_reasoning_without_resetting_history(built_case):
|
||||||
|
from dataclasses import replace
|
||||||
|
from viano_bug_analysis.llm import LLMError, LLMGateway
|
||||||
|
class TruncatedModel(Model):
|
||||||
|
failures = 2
|
||||||
|
efforts = []
|
||||||
|
async def generate(self, config, request, *, max_response_bytes):
|
||||||
|
self.efforts.append((request['prompt_version'], config.reasoning_effort))
|
||||||
|
if request['prompt_version']=='entity_resolution_assessment_v3' and self.failures:
|
||||||
|
self.failures -= 1
|
||||||
|
raise LLMError('LLM_INCOMPLETE_OUTPUT')
|
||||||
|
return await super().generate(config, request, max_response_bytes=max_response_bytes)
|
||||||
|
store = built_case[0]
|
||||||
|
model = TruncatedModel()
|
||||||
|
service = service_for(store)
|
||||||
|
old = gateway(model)
|
||||||
|
old.config = replace(old.config, providers=tuple(replace(p, reasoning_effort='medium') for p in old.config.providers))
|
||||||
|
owner = CaseInterpreter(store, old, project_research=service)
|
||||||
|
for _ in range(2):
|
||||||
|
result = asyncio.run(owner.interpret('case-A','stable-package'))
|
||||||
|
assert result.status == 'REJECTED'
|
||||||
|
adjusted = replace(old.config, providers=tuple(replace(p, entity_resolution_reasoning_effort='low') for p in old.config.providers))
|
||||||
|
owner = CaseInterpreter(store, LLMGateway(adjusted, {'fixture': model}), project_research=service)
|
||||||
|
result = asyncio.run(owner.interpret('case-A','stable-package'))
|
||||||
|
assert result.status == 'COMPLETED'
|
||||||
|
assert len(service.database.calls) == 1
|
||||||
|
assert model.efforts == [('entity_resolution_plan_v3','medium'),
|
||||||
|
('entity_resolution_assessment_v3','medium'),('entity_resolution_assessment_v3','medium'),
|
||||||
|
('entity_resolution_assessment_v3','low'),('case_interpretation_resolved_v3','medium')]
|
||||||
|
with PackageReader(result.package) as reader:
|
||||||
|
saved = json.loads(reader.read_bytes('analysis/interpretations/0001/entity-resolution/assessment.json'))
|
||||||
|
assert saved['payload']['attempt'] == 3
|
||||||
|
assert asyncio.run(owner.interpret('case-A','stable-package')).reused
|
||||||
|
assert len(model.efforts) == 5
|
||||||
@@ -50,3 +50,37 @@ def test_identity_projection_keeps_readable_evidence_and_legacy_requests(stage):
|
|||||||
assert value == context["sources"][key]
|
assert value == context["sources"][key]
|
||||||
assert len(new["messages"][1]["content"]) < len(old["messages"][1]["content"]) / 2
|
assert len(new["messages"][1]["content"]) < len(old["messages"][1]["content"]) / 2
|
||||||
assert data["output_schema"] == new["output_schema"]
|
assert data["output_schema"] == new["output_schema"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("version,expected", [
|
||||||
|
("entity_resolution_plan_v1", "low"),
|
||||||
|
("entity_resolution_assessment_v3", "low"),
|
||||||
|
("case_interpretation_resolved_v3", "medium"),
|
||||||
|
("hypothesis_generation_v1", "medium"),
|
||||||
|
("investigation_v1", "medium"),
|
||||||
|
("entity_resolution_assessment_v3_suffix", "medium"),
|
||||||
|
])
|
||||||
|
def test_identity_override_is_scoped_at_http_transport(http_endpoint, version, expected):
|
||||||
|
url, seen, _ = http_endpoint
|
||||||
|
provider = ProviderConfig(id="test", model="fixture", base_url=url,
|
||||||
|
reasoning_effort="medium", entity_resolution_reasoning_effort="low", max_output_tokens=16384)
|
||||||
|
config = GatewayConfig((provider,))
|
||||||
|
query = request()
|
||||||
|
query['prompt_version'] = version
|
||||||
|
asyncio.run(LLMGateway(config).complete_round(query))
|
||||||
|
assert seen[0]['body']['reasoning_effort'] == expected
|
||||||
|
assert seen[0]['body'][provider.token_parameter] == 16384
|
||||||
|
assert provider.reasoning_effort == "medium"
|
||||||
|
assert config.snapshot()['providers'][0]['entity_resolution_reasoning_effort'] == "low"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("effort", [True, 8192, [], "unlimited"])
|
||||||
|
def test_invalid_identity_reasoning_override(effort):
|
||||||
|
with pytest.raises(LLMError, match="LLM_CONFIG"):
|
||||||
|
ProviderConfig(id="test", model="fixture", base_url="http://localhost",
|
||||||
|
entity_resolution_reasoning_effort=effort)
|
||||||
|
|
||||||
|
|
||||||
|
def test_omitted_identity_override_preserves_previous_snapshot():
|
||||||
|
provider = ProviderConfig(id="test", model="fixture", base_url="http://localhost", reasoning_effort="medium")
|
||||||
|
assert 'entity_resolution_reasoning_effort' not in GatewayConfig((provider,)).snapshot()['providers'][0]
|
||||||
Reference in new issue
Block a user