diff --git a/config/llm.gemini-cache.example.toml b/config/llm.gemini-cache.example.toml index 6ad358c..43f2016 100644 --- a/config/llm.gemini-cache.example.toml +++ b/config/llm.gemini-cache.example.toml @@ -1,4 +1,4 @@ -# Optional separate configuration. Copy to an ignored *.local.toml file. +# Native Gemini configuration. Copy to an ignored *.local.toml file. # This does not replace the existing OpenAI-compatible configuration. [llm] privacy_profile = "cloud_allowed" @@ -19,7 +19,8 @@ max_output_tokens = 16384 reasoning_effort = "medium" entity_resolution_reasoning_effort = "low" -# Opt-in. Instruction/schema prefix only; live evidence is always complete. +# Enabled by default on this adapter. Instruction/schema prefix only; +# live evidence is always complete. Explicit false is an operational override. context_cache = true cache_ttl_seconds = 300 # A character heuristic, not a tokenizer or a format limit. Provider minimums diff --git a/docs/llm-context-caching.md b/docs/llm-context-caching.md index df94064..41892de 100644 --- a/docs/llm-context-caching.md +++ b/docs/llm-context-caching.md @@ -1,4 +1,4 @@ -# Optional context caching and usage accounting +# Context caching and usage accounting The package remains the complete, authoritative record. Saved requests, evidence, responses, checkpoints and provider configuration references remain portable. @@ -14,14 +14,14 @@ The existing `openai_compatible` adapter now reads hits when the provider actually reports them, without changing existing requests or enabling explicit caching. An absent count means unknown, not zero. -The optional `gemini` adapter uses the native GenerateContent API. It moves the +The `gemini` adapter uses the native GenerateContent API. It moves the unchanged system instructions and authoritative output schema to the beginning of the wire request. Only an exact top-level schema duplicate is removed from the live user JSON; no case facts, identifiers, evidence or assessments are removed. The archived request is not changed. This is deterministic transport preparation, not another LLM summary. -With `context_cache = true`, this instruction/schema prefix can be stored as a +By default on the native Gemini adapter, this instruction/schema prefix is stored as a Gemini `cachedContents` resource. Case-specific input and all research history are still sent in full. The prefix is independent of the currently selected hypothesis. Reuse requires an exact match of endpoint, provider ID, model, @@ -41,12 +41,16 @@ improvement. ## Configuration and fallback See `config/llm.gemini-cache.example.toml`. Existing local configuration files -are not edited. The native base URL ends in `/v1beta`, **not** `/v1beta/openai`. -`context_cache` defaults to false and is allowed only for the native adapter. +use the adapter selected in each provider entry. The native base URL ends in `/v1beta`, **not** `/v1beta/openai`. +Omitting `context_cache` enables explicit caching on the native Gemini adapter. +Adapters without explicit-cache support remain usable without it. Explicit true +is allowed only for the native adapter; false remains an operational override. `cached_input_cost_per_million` can also be set for OpenAI-compatible providers that report cache usage. Prices remain optional, model-specific configuration. -New default fields are omitted from configuration snapshots to preserve frozen -operation hashes and replay of old cases. +The effective enabled choice is persisted for native Gemini, so a change from +uncached processing is visible to frozen-operation configuration checks. Other +adapters retain their old default snapshots. Existing pending operations must +keep their original configuration or be tested in a fresh workspace. `cache_min_prefix_chars` is an eligibility heuristic, not a token count or API minimum. A short prefix may therefore obtain no explicit cache saving. The @@ -80,7 +84,8 @@ positive, consistent provider-reported cached token count (`LLM_CACHE_USAGE_UNCONFIRMED` otherwise). This is a test assertion, not a promise that Gemini will accept a prefix. A rejected/too-small cache ends the test explicitly. A response without confirmed usage may already have incurred -API costs. Default behaviour remains optional; old operation hashes stay valid. +API costs. Default behaviour enables caching where supported with graceful fallback; +strict enforcement is a separate test mode. Strict cache creation may use half of the configured request timeout, instead of the five-second optional-optimization allowance. The overall deadline still applies; a creation HTTP status is retained for diagnosing a rejected cache. @@ -145,3 +150,8 @@ for this inspection; it is not a new quality or cost benchmark. - [Thinking configuration](https://ai.google.dev/gemini-api/docs/generate-content/thinking) - [Model-specific pricing](https://ai.google.dev/gemini-api/docs/pricing) - [Interactions session API, for a separate future adapter](https://ai.google.dev/gemini-api/docs/interactions-overview) + +The regular local Gemini configurations were switched to the native adapter for +this policy. Small prefixes still respect the configured eligibility heuristic; +API minimums are not bypassed by padding. Transient cache failures retain the +full-input fallback so an optimization outage does not stop normal analysis. diff --git a/src/viano_bug_analysis/llm/config.py b/src/viano_bug_analysis/llm/config.py index 302ec8a..dc50186 100644 --- a/src/viano_bug_analysis/llm/config.py +++ b/src/viano_bug_analysis/llm/config.py @@ -36,7 +36,7 @@ class ProviderConfig: entity_resolution_reasoning_effort: str | None = None cached_input_cost_per_million: float | None = None cache_storage_cost_per_million_hour: float | None = None - context_cache: bool = False + context_cache: bool | None = None cache_ttl_seconds: int = 300 cache_min_prefix_chars: int = 16000 cache_required: bool = False @@ -70,9 +70,11 @@ class ProviderConfig: raise LLMError("LLM_CONFIG") if not isinstance(self.adapter, str) or not re.fullmatch(r"[A-Za-z0-9_-]{1,64}", self.adapter): raise LLMError("LLM_CONFIG") - for value in (self.enabled, self.allow_insecure_http, self.context_cache, self.cache_required): + for value in (self.enabled, self.allow_insecure_http, self.cache_required): if not isinstance(value, bool): raise LLMError("LLM_CONFIG") + if self.context_cache is not None and type(self.context_cache) is not bool: + raise LLMError('LLM_CONFIG') for effort in (self.reasoning_effort, self.entity_resolution_reasoning_effort): if effort is not None and (not isinstance(effort, str) or effort not in {"none", "minimal", "low", "medium", "high"}): raise LLMError("LLM_CONFIG") @@ -82,7 +84,7 @@ class ProviderConfig: positive(self.cache_min_prefix_chars, integer=True) if self.context_cache and self.adapter != 'gemini': raise LLMError('LLM_CONFIG') - if self.cache_required and not self.context_cache: + if self.cache_required and not self.use_context_cache: raise LLMError('LLM_CONFIG') if self.cache_required_prompt_prefix is not None and (not self.cache_required or not isinstance(self.cache_required_prompt_prefix, str) @@ -98,6 +100,11 @@ class ProviderConfig: if cost is not None and (isinstance(cost, bool) or not isinstance(cost, (int, float)) or not math.isfinite(cost) or cost < 0): raise LLMError("LLM_CONFIG") + @property + def use_context_cache(self): + """Default to explicit caching on adapters that implement the protocol.""" + return self.adapter == 'gemini' and self.context_cache is not False + @dataclass(frozen=True) class GatewayConfig: @@ -138,6 +145,10 @@ class GatewayConfig: # Only configuration references; never resolve credentials for persistence. result = asdict(self) for provider in result["providers"]: + if provider['context_cache'] is None: + # Persist the effective choice for capable adapters. This makes + # the policy change visible to frozen-operation hash checks. + provider['context_cache'] = provider['adapter'] == 'gemini' # Omit new defaults so frozen operations keep their configuration hash. for name, default in {'cached_input_cost_per_million': None, 'cache_storage_cost_per_million_hour': None, diff --git a/src/viano_bug_analysis/llm/gemini.py b/src/viano_bug_analysis/llm/gemini.py index cca96ea..7c89979 100644 --- a/src/viano_bug_analysis/llm/gemini.py +++ b/src/viano_bug_analysis/llm/gemini.py @@ -109,7 +109,7 @@ class GeminiProvider: cache_http_status = None fingerprint = hashlib.sha256(json_bytes([base, config.id, config.model, secret, system, prefix, config.cache_ttl_seconds])).hexdigest() - if config.context_cache: + if config.use_context_cache: status = 'below_minimum' if len(json_bytes([system, prefix]).decode('utf-8')) >= config.cache_min_prefix_chars: with self._lock: diff --git a/tests/test_llm_caching.py b/tests/test_llm_caching.py index cedb719..bca4458 100644 --- a/tests/test_llm_caching.py +++ b/tests/test_llm_caching.py @@ -156,6 +156,23 @@ def test_config_defaults_keep_frozen_snapshot_compatible(): assert GatewayConfig((config(),)).snapshot()['providers'][0]['context_cache'] is True +def test_capable_adapter_defaults_to_explicit_caching_and_pins_choice(): + cfg=ProviderConfig('native','gemini-test','http://localhost/v1beta',adapter='gemini') + assert cfg.use_context_cache + assert GatewayConfig((cfg,)).snapshot()['providers'][0]['context_cache'] is True + assert not replace(cfg,context_cache=False).use_context_cache + assert not ProviderConfig('other','m','http://localhost').use_context_cache + assert replace(cfg,cache_required=True).use_context_cache + + +def test_default_native_configuration_creates_cache(monkeypatch): + adapter=GeminiProvider() + cfg=replace(config(),context_cache=None) + seen=scripted(adapter,monkeypatch,[CACHE,native()]) + result=asyncio.run(adapter.generate(cfg,request(),max_response_bytes=10000)) + assert result.diagnostics['cache_status']=='created' and len(seen)==2 + + @pytest.mark.parametrize('changes', [{'adapter':'openai_compatible'}, {'cache_ttl_seconds':True}, {'cache_min_prefix_chars':0}, {'cached_input_cost_per_million':-1}, {'model':'../other'}, {'reasoning_effort':'none'}]) def test_bad_cache_configuration_rejected(changes):