diff --git a/config/llm.example.toml b/config/llm.example.toml index 94380a4..1557222 100644 --- a/config/llm.example.toml +++ b/config/llm.example.toml @@ -19,6 +19,8 @@ token_parameter = "max_tokens" max_output_tokens = 4096 # Optional; omit for provider defaults. Supported values depend on the model. # reasoning_effort = "medium" +# Optional override for entity resolution planning and assessment only: +# entity_resolution_reasoning_effort = "low" [[llm.providers]] id = "own-server" diff --git a/docs/llm-context-efficiency-v1.md b/docs/llm-context-efficiency-v1.md index 5d9fef5..099f6af 100644 --- a/docs/llm-context-efficiency-v1.md +++ b/docs/llm-context-efficiency-v1.md @@ -40,3 +40,24 @@ The output limit is a ceiling, not a request to generate that many tokens. Think and output accounting are provider-dependent. More output capacity can increase cost; monitor actual usage and finish diagnostics. No raw reasoning is requested or persisted by these settings. + + +## Identity-only reasoning override + +Each provider optionally accepts `entity_resolution_reasoning_effort` with the +same validated values as `reasoning_effort`. It applies only to versioned +`entity_resolution_plan_vN` and `entity_resolution_assessment_vN` requests. +All interpretation, hypothesis and investigation requests retain the provider's +normal `reasoning_effort`. Omission preserves previous configuration snapshots. + +```toml +reasoning_effort = "medium" +entity_resolution_reasoning_effort = "low" +max_output_tokens = 16384 +``` + +The override is recorded in configuration snapshots and their hashes. Existing +pending calls cannot silently change configuration mid-retry. A new entity +assessment attempt uses the new settings; accepted plan and previous assessment +artifacts remain unchanged and are checked without repeating their API calls. +No attempt counters or investigation budgets are reset. diff --git a/src/viano_bug_analysis/llm/config.py b/src/viano_bug_analysis/llm/config.py index a3084c5..6eb8c73 100644 --- a/src/viano_bug_analysis/llm/config.py +++ b/src/viano_bug_analysis/llm/config.py @@ -33,6 +33,7 @@ class ProviderConfig: output_cost_per_million: float | None = None cost_currency: str = "USD" reasoning_effort: str | None = None + entity_resolution_reasoning_effort: str | None = None def __post_init__(self): if not isinstance(self.id, str) or not re.fullmatch(r"[A-Za-z0-9_-]{1,64}", self.id): @@ -65,8 +66,9 @@ class ProviderConfig: for value in (self.enabled, self.allow_insecure_http): if not isinstance(value, bool): raise LLMError("LLM_CONFIG") - if self.reasoning_effort is not None and (not isinstance(self.reasoning_effort, str) or self.reasoning_effort not in {"none", "minimal", "low", "medium", "high"}): - raise LLMError("LLM_CONFIG") + for effort in (self.reasoning_effort, self.entity_resolution_reasoning_effort): + if effort is not None and (not isinstance(effort, str) or effort not in {"none", "minimal", "low", "medium", "high"}): + raise LLMError("LLM_CONFIG") positive(self.timeout_seconds) positive(self.max_output_tokens, integer=True) if not isinstance(self.cost_currency, str) or not re.fullmatch(r"[A-Z]{3}", self.cost_currency): @@ -115,6 +117,7 @@ class GatewayConfig: # Only configuration references; never resolve credentials for persistence. result = asdict(self) for provider in result["providers"]: - if provider["reasoning_effort"] is None: - del provider["reasoning_effort"] + for name in ("reasoning_effort", "entity_resolution_reasoning_effort"): + if provider[name] is None: + del provider[name] return result diff --git a/src/viano_bug_analysis/llm/gateway.py b/src/viano_bug_analysis/llm/gateway.py index 9f388e8..bb47fa1 100644 --- a/src/viano_bug_analysis/llm/gateway.py +++ b/src/viano_bug_analysis/llm/gateway.py @@ -3,6 +3,8 @@ import copy import logging import time import math +import re +from dataclasses import replace from jsonschema import Draft202012Validator from jsonschema.exceptions import SchemaError, ValidationError @@ -73,6 +75,9 @@ class LLMGateway: for provider in self.config.providers: if not provider.enabled or provider.privacy not in (PRIVACY[request["privacy"]] & PRIVACY[self.config.privacy_profile]): continue + if (provider.entity_resolution_reasoning_effort is not None + and re.fullmatch(r"entity_resolution_(plan|assessment)_v[1-9][0-9]*", request["prompt_version"])): + provider = replace(provider, reasoning_effort=provider.entity_resolution_reasoning_effort) started = time.monotonic() response = None try: diff --git a/tests/test_entity_resolution.py b/tests/test_entity_resolution.py index 16e0466..6579736 100644 --- a/tests/test_entity_resolution.py +++ b/tests/test_entity_resolution.py @@ -367,3 +367,39 @@ def test_missing_capture_feedback_resumes_and_replays_without_repeating_research assert PackageValidator().validate(result.package).valid assert run(store, model).reused assert len(model.requests) == 4 + + +def test_third_assessment_can_use_lower_reasoning_without_resetting_history(built_case): + from dataclasses import replace + from viano_bug_analysis.llm import LLMError, LLMGateway + class TruncatedModel(Model): + failures = 2 + efforts = [] + async def generate(self, config, request, *, max_response_bytes): + self.efforts.append((request['prompt_version'], config.reasoning_effort)) + if request['prompt_version']=='entity_resolution_assessment_v3' and self.failures: + self.failures -= 1 + raise LLMError('LLM_INCOMPLETE_OUTPUT') + return await super().generate(config, request, max_response_bytes=max_response_bytes) + store = built_case[0] + model = TruncatedModel() + service = service_for(store) + old = gateway(model) + old.config = replace(old.config, providers=tuple(replace(p, reasoning_effort='medium') for p in old.config.providers)) + owner = CaseInterpreter(store, old, project_research=service) + for _ in range(2): + result = asyncio.run(owner.interpret('case-A','stable-package')) + assert result.status == 'REJECTED' + adjusted = replace(old.config, providers=tuple(replace(p, entity_resolution_reasoning_effort='low') for p in old.config.providers)) + owner = CaseInterpreter(store, LLMGateway(adjusted, {'fixture': model}), project_research=service) + result = asyncio.run(owner.interpret('case-A','stable-package')) + assert result.status == 'COMPLETED' + assert len(service.database.calls) == 1 + assert model.efforts == [('entity_resolution_plan_v3','medium'), + ('entity_resolution_assessment_v3','medium'),('entity_resolution_assessment_v3','medium'), + ('entity_resolution_assessment_v3','low'),('case_interpretation_resolved_v3','medium')] + with PackageReader(result.package) as reader: + saved = json.loads(reader.read_bytes('analysis/interpretations/0001/entity-resolution/assessment.json')) + assert saved['payload']['attempt'] == 3 + assert asyncio.run(owner.interpret('case-A','stable-package')).reused + assert len(model.efforts) == 5 diff --git a/tests/test_llm_context_efficiency.py b/tests/test_llm_context_efficiency.py index 1baf795..612bed5 100644 --- a/tests/test_llm_context_efficiency.py +++ b/tests/test_llm_context_efficiency.py @@ -50,3 +50,37 @@ def test_identity_projection_keeps_readable_evidence_and_legacy_requests(stage): assert value == context["sources"][key] assert len(new["messages"][1]["content"]) < len(old["messages"][1]["content"]) / 2 assert data["output_schema"] == new["output_schema"] + + +@pytest.mark.parametrize("version,expected", [ + ("entity_resolution_plan_v1", "low"), + ("entity_resolution_assessment_v3", "low"), + ("case_interpretation_resolved_v3", "medium"), + ("hypothesis_generation_v1", "medium"), + ("investigation_v1", "medium"), + ("entity_resolution_assessment_v3_suffix", "medium"), +]) +def test_identity_override_is_scoped_at_http_transport(http_endpoint, version, expected): + url, seen, _ = http_endpoint + provider = ProviderConfig(id="test", model="fixture", base_url=url, + reasoning_effort="medium", entity_resolution_reasoning_effort="low", max_output_tokens=16384) + config = GatewayConfig((provider,)) + query = request() + query['prompt_version'] = version + asyncio.run(LLMGateway(config).complete_round(query)) + assert seen[0]['body']['reasoning_effort'] == expected + assert seen[0]['body'][provider.token_parameter] == 16384 + assert provider.reasoning_effort == "medium" + assert config.snapshot()['providers'][0]['entity_resolution_reasoning_effort'] == "low" + + +@pytest.mark.parametrize("effort", [True, 8192, [], "unlimited"]) +def test_invalid_identity_reasoning_override(effort): + with pytest.raises(LLMError, match="LLM_CONFIG"): + ProviderConfig(id="test", model="fixture", base_url="http://localhost", + entity_resolution_reasoning_effort=effort) + + +def test_omitted_identity_override_preserves_previous_snapshot(): + provider = ProviderConfig(id="test", model="fixture", base_url="http://localhost", reasoning_effort="medium") + assert 'entity_resolution_reasoning_effort' not in GatewayConfig((provider,)).snapshot()['providers'][0]