Enable explicit caching by default on capable Gemini adapter

This commit is contained in:
Nico committed 2026-10-16 16:02:40 +02:00
1 parent 02a7364c58
commit d5dac27f56
5 files changed
+53 -14

No files matched your search

+3 -2
View File
@@ -1,4 +1,4 @@
# Optional separate configuration. Copy to an ignored *.local.toml file.
# Native Gemini configuration. Copy to an ignored *.local.toml file.
# This does not replace the existing OpenAI-compatible configuration.
[llm]
privacy_profile = "cloud_allowed"
@@ -19,7 +19,8 @@ max_output_tokens = 16384
reasoning_effort = "medium"
entity_resolution_reasoning_effort = "low"
# Opt-in. Instruction/schema prefix only; live evidence is always complete.
# Enabled by default on this adapter. Instruction/schema prefix only;
# live evidence is always complete. Explicit false is an operational override.
context_cache = true
cache_ttl_seconds = 300
# A character heuristic, not a tokenizer or a format limit. Provider minimums
+18 -8
View File
@@ -1,4 +1,4 @@
# Optional context caching and usage accounting
# Context caching and usage accounting
The package remains the complete, authoritative record. Saved requests, evidence,
responses, checkpoints and provider configuration references remain portable.
@@ -14,14 +14,14 @@ The existing `openai_compatible` adapter now reads
hits when the provider actually reports them, without changing existing requests
or enabling explicit caching. An absent count means unknown, not zero.
The optional `gemini` adapter uses the native GenerateContent API. It moves the
The `gemini` adapter uses the native GenerateContent API. It moves the
unchanged system instructions and authoritative output schema to the beginning
of the wire request. Only an exact top-level schema duplicate is removed from
the live user JSON; no case facts, identifiers, evidence or assessments are
removed. The archived request is not changed. This is deterministic transport
preparation, not another LLM summary.
With `context_cache = true`, this instruction/schema prefix can be stored as a
By default on the native Gemini adapter, this instruction/schema prefix is stored as a
Gemini `cachedContents` resource. Case-specific input and all research history
are still sent in full. The prefix is independent of the currently selected
hypothesis. Reuse requires an exact match of endpoint, provider ID, model,
@@ -41,12 +41,16 @@ improvement.
## Configuration and fallback
See `config/llm.gemini-cache.example.toml`. Existing local configuration files
are not edited. The native base URL ends in `/v1beta`, **not** `/v1beta/openai`.
`context_cache` defaults to false and is allowed only for the native adapter.
use the adapter selected in each provider entry. The native base URL ends in `/v1beta`, **not** `/v1beta/openai`.
Omitting `context_cache` enables explicit caching on the native Gemini adapter.
Adapters without explicit-cache support remain usable without it. Explicit true
is allowed only for the native adapter; false remains an operational override.
`cached_input_cost_per_million` can also be set for OpenAI-compatible providers
that report cache usage. Prices remain optional, model-specific configuration.
New default fields are omitted from configuration snapshots to preserve frozen
operation hashes and replay of old cases.
The effective enabled choice is persisted for native Gemini, so a change from
uncached processing is visible to frozen-operation configuration checks. Other
adapters retain their old default snapshots. Existing pending operations must
keep their original configuration or be tested in a fresh workspace.
`cache_min_prefix_chars` is an eligibility heuristic, not a token count or API
minimum. A short prefix may therefore obtain no explicit cache saving. The
@@ -80,7 +84,8 @@ positive, consistent provider-reported cached token count
(`LLM_CACHE_USAGE_UNCONFIRMED` otherwise). This is a test assertion, not a
promise that Gemini will accept a prefix. A rejected/too-small cache ends the
test explicitly. A response without confirmed usage may already have incurred
API costs. Default behaviour remains optional; old operation hashes stay valid.
API costs. Default behaviour enables caching where supported with graceful fallback;
strict enforcement is a separate test mode.
Strict cache creation may use half of the configured request timeout, instead
of the five-second optional-optimization allowance. The overall deadline still
applies; a creation HTTP status is retained for diagnosing a rejected cache.
@@ -145,3 +150,8 @@ for this inspection; it is not a new quality or cost benchmark.
- [Thinking configuration](https://ai.google.dev/gemini-api/docs/generate-content/thinking)
- [Model-specific pricing](https://ai.google.dev/gemini-api/docs/pricing)
- [Interactions session API, for a separate future adapter](https://ai.google.dev/gemini-api/docs/interactions-overview)
The regular local Gemini configurations were switched to the native adapter for
this policy. Small prefixes still respect the configured eligibility heuristic;
API minimums are not bypassed by padding. Transient cache failures retain the
full-input fallback so an optimization outage does not stop normal analysis.
+14 -3
View File
@@ -36,7 +36,7 @@ class ProviderConfig:
entity_resolution_reasoning_effort: str | None = None
cached_input_cost_per_million: float | None = None
cache_storage_cost_per_million_hour: float | None = None
context_cache: bool = False
context_cache: bool | None = None
cache_ttl_seconds: int = 300
cache_min_prefix_chars: int = 16000
cache_required: bool = False
@@ -70,9 +70,11 @@ class ProviderConfig:
raise LLMError("LLM_CONFIG")
if not isinstance(self.adapter, str) or not re.fullmatch(r"[A-Za-z0-9_-]{1,64}", self.adapter):
raise LLMError("LLM_CONFIG")
for value in (self.enabled, self.allow_insecure_http, self.context_cache, self.cache_required):
for value in (self.enabled, self.allow_insecure_http, self.cache_required):
if not isinstance(value, bool):
raise LLMError("LLM_CONFIG")
if self.context_cache is not None and type(self.context_cache) is not bool:
raise LLMError('LLM_CONFIG')
for effort in (self.reasoning_effort, self.entity_resolution_reasoning_effort):
if effort is not None and (not isinstance(effort, str) or effort not in {"none", "minimal", "low", "medium", "high"}):
raise LLMError("LLM_CONFIG")
@@ -82,7 +84,7 @@ class ProviderConfig:
positive(self.cache_min_prefix_chars, integer=True)
if self.context_cache and self.adapter != 'gemini':
raise LLMError('LLM_CONFIG')
if self.cache_required and not self.context_cache:
if self.cache_required and not self.use_context_cache:
raise LLMError('LLM_CONFIG')
if self.cache_required_prompt_prefix is not None and (not self.cache_required
or not isinstance(self.cache_required_prompt_prefix, str)
@@ -98,6 +100,11 @@ class ProviderConfig:
if cost is not None and (isinstance(cost, bool) or not isinstance(cost, (int, float)) or not math.isfinite(cost) or cost < 0):
raise LLMError("LLM_CONFIG")
@property
def use_context_cache(self):
"""Default to explicit caching on adapters that implement the protocol."""
return self.adapter == 'gemini' and self.context_cache is not False
@dataclass(frozen=True)
class GatewayConfig:
@@ -138,6 +145,10 @@ class GatewayConfig:
# Only configuration references; never resolve credentials for persistence.
result = asdict(self)
for provider in result["providers"]:
if provider['context_cache'] is None:
# Persist the effective choice for capable adapters. This makes
# the policy change visible to frozen-operation hash checks.
provider['context_cache'] = provider['adapter'] == 'gemini'
# Omit new defaults so frozen operations keep their configuration hash.
for name, default in {'cached_input_cost_per_million': None,
'cache_storage_cost_per_million_hour': None,
+1 -1
View File
@@ -109,7 +109,7 @@ class GeminiProvider:
cache_http_status = None
fingerprint = hashlib.sha256(json_bytes([base, config.id, config.model, secret,
system, prefix, config.cache_ttl_seconds])).hexdigest()
if config.context_cache:
if config.use_context_cache:
status = 'below_minimum'
if len(json_bytes([system, prefix]).decode('utf-8')) >= config.cache_min_prefix_chars:
with self._lock:
+17
View File
@@ -156,6 +156,23 @@ def test_config_defaults_keep_frozen_snapshot_compatible():
assert GatewayConfig((config(),)).snapshot()['providers'][0]['context_cache'] is True
def test_capable_adapter_defaults_to_explicit_caching_and_pins_choice():
cfg=ProviderConfig('native','gemini-test','http://localhost/v1beta',adapter='gemini')
assert cfg.use_context_cache
assert GatewayConfig((cfg,)).snapshot()['providers'][0]['context_cache'] is True
assert not replace(cfg,context_cache=False).use_context_cache
assert not ProviderConfig('other','m','http://localhost').use_context_cache
assert replace(cfg,cache_required=True).use_context_cache
def test_default_native_configuration_creates_cache(monkeypatch):
adapter=GeminiProvider()
cfg=replace(config(),context_cache=None)
seen=scripted(adapter,monkeypatch,[CACHE,native()])
result=asyncio.run(adapter.generate(cfg,request(),max_response_bytes=10000))
assert result.diagnostics['cache_status']=='created' and len(seen)==2
@pytest.mark.parametrize('changes', [{'adapter':'openai_compatible'}, {'cache_ttl_seconds':True},
{'cache_min_prefix_chars':0}, {'cached_input_cost_per_million':-1}, {'model':'../other'}, {'reasoning_effort':'none'}])
def test_bad_cache_configuration_rejected(changes):