Enable explicit caching by default on capable Gemini adapter

This commit is contained in:
Nico committed 2026-10-16 16:02:40 +02:00
1 parent 02a7364c58
commit d5dac27f56
5 files changed
+53 -14

No files matched your search

+3 -2
View File
@@ -1,4 +1,4 @@
# Optional separate configuration. Copy to an ignored *.local.toml file. # Native Gemini configuration. Copy to an ignored *.local.toml file.
# This does not replace the existing OpenAI-compatible configuration. # This does not replace the existing OpenAI-compatible configuration.
[llm] [llm]
privacy_profile = "cloud_allowed" privacy_profile = "cloud_allowed"
@@ -19,7 +19,8 @@ max_output_tokens = 16384
reasoning_effort = "medium" reasoning_effort = "medium"
entity_resolution_reasoning_effort = "low" entity_resolution_reasoning_effort = "low"
# Opt-in. Instruction/schema prefix only; live evidence is always complete. # Enabled by default on this adapter. Instruction/schema prefix only;
# live evidence is always complete. Explicit false is an operational override.
context_cache = true context_cache = true
cache_ttl_seconds = 300 cache_ttl_seconds = 300
# A character heuristic, not a tokenizer or a format limit. Provider minimums # A character heuristic, not a tokenizer or a format limit. Provider minimums
+18 -8
View File
@@ -1,4 +1,4 @@
# Optional context caching and usage accounting # Context caching and usage accounting
The package remains the complete, authoritative record. Saved requests, evidence, The package remains the complete, authoritative record. Saved requests, evidence,
responses, checkpoints and provider configuration references remain portable. responses, checkpoints and provider configuration references remain portable.
@@ -14,14 +14,14 @@ The existing `openai_compatible` adapter now reads
hits when the provider actually reports them, without changing existing requests hits when the provider actually reports them, without changing existing requests
or enabling explicit caching. An absent count means unknown, not zero. or enabling explicit caching. An absent count means unknown, not zero.
The optional `gemini` adapter uses the native GenerateContent API. It moves the The `gemini` adapter uses the native GenerateContent API. It moves the
unchanged system instructions and authoritative output schema to the beginning unchanged system instructions and authoritative output schema to the beginning
of the wire request. Only an exact top-level schema duplicate is removed from of the wire request. Only an exact top-level schema duplicate is removed from
the live user JSON; no case facts, identifiers, evidence or assessments are the live user JSON; no case facts, identifiers, evidence or assessments are
removed. The archived request is not changed. This is deterministic transport removed. The archived request is not changed. This is deterministic transport
preparation, not another LLM summary. preparation, not another LLM summary.
With `context_cache = true`, this instruction/schema prefix can be stored as a By default on the native Gemini adapter, this instruction/schema prefix is stored as a
Gemini `cachedContents` resource. Case-specific input and all research history Gemini `cachedContents` resource. Case-specific input and all research history
are still sent in full. The prefix is independent of the currently selected are still sent in full. The prefix is independent of the currently selected
hypothesis. Reuse requires an exact match of endpoint, provider ID, model, hypothesis. Reuse requires an exact match of endpoint, provider ID, model,
@@ -41,12 +41,16 @@ improvement.
## Configuration and fallback ## Configuration and fallback
See `config/llm.gemini-cache.example.toml`. Existing local configuration files See `config/llm.gemini-cache.example.toml`. Existing local configuration files
are not edited. The native base URL ends in `/v1beta`, **not** `/v1beta/openai`. use the adapter selected in each provider entry. The native base URL ends in `/v1beta`, **not** `/v1beta/openai`.
`context_cache` defaults to false and is allowed only for the native adapter. Omitting `context_cache` enables explicit caching on the native Gemini adapter.
Adapters without explicit-cache support remain usable without it. Explicit true
is allowed only for the native adapter; false remains an operational override.
`cached_input_cost_per_million` can also be set for OpenAI-compatible providers `cached_input_cost_per_million` can also be set for OpenAI-compatible providers
that report cache usage. Prices remain optional, model-specific configuration. that report cache usage. Prices remain optional, model-specific configuration.
New default fields are omitted from configuration snapshots to preserve frozen The effective enabled choice is persisted for native Gemini, so a change from
operation hashes and replay of old cases. uncached processing is visible to frozen-operation configuration checks. Other
adapters retain their old default snapshots. Existing pending operations must
keep their original configuration or be tested in a fresh workspace.
`cache_min_prefix_chars` is an eligibility heuristic, not a token count or API `cache_min_prefix_chars` is an eligibility heuristic, not a token count or API
minimum. A short prefix may therefore obtain no explicit cache saving. The minimum. A short prefix may therefore obtain no explicit cache saving. The
@@ -80,7 +84,8 @@ positive, consistent provider-reported cached token count
(`LLM_CACHE_USAGE_UNCONFIRMED` otherwise). This is a test assertion, not a (`LLM_CACHE_USAGE_UNCONFIRMED` otherwise). This is a test assertion, not a
promise that Gemini will accept a prefix. A rejected/too-small cache ends the promise that Gemini will accept a prefix. A rejected/too-small cache ends the
test explicitly. A response without confirmed usage may already have incurred test explicitly. A response without confirmed usage may already have incurred
API costs. Default behaviour remains optional; old operation hashes stay valid. API costs. Default behaviour enables caching where supported with graceful fallback;
strict enforcement is a separate test mode.
Strict cache creation may use half of the configured request timeout, instead Strict cache creation may use half of the configured request timeout, instead
of the five-second optional-optimization allowance. The overall deadline still of the five-second optional-optimization allowance. The overall deadline still
applies; a creation HTTP status is retained for diagnosing a rejected cache. applies; a creation HTTP status is retained for diagnosing a rejected cache.
@@ -145,3 +150,8 @@ for this inspection; it is not a new quality or cost benchmark.
- [Thinking configuration](https://ai.google.dev/gemini-api/docs/generate-content/thinking) - [Thinking configuration](https://ai.google.dev/gemini-api/docs/generate-content/thinking)
- [Model-specific pricing](https://ai.google.dev/gemini-api/docs/pricing) - [Model-specific pricing](https://ai.google.dev/gemini-api/docs/pricing)
- [Interactions session API, for a separate future adapter](https://ai.google.dev/gemini-api/docs/interactions-overview) - [Interactions session API, for a separate future adapter](https://ai.google.dev/gemini-api/docs/interactions-overview)
The regular local Gemini configurations were switched to the native adapter for
this policy. Small prefixes still respect the configured eligibility heuristic;
API minimums are not bypassed by padding. Transient cache failures retain the
full-input fallback so an optimization outage does not stop normal analysis.
+14 -3
View File
@@ -36,7 +36,7 @@ class ProviderConfig:
entity_resolution_reasoning_effort: str | None = None entity_resolution_reasoning_effort: str | None = None
cached_input_cost_per_million: float | None = None cached_input_cost_per_million: float | None = None
cache_storage_cost_per_million_hour: float | None = None cache_storage_cost_per_million_hour: float | None = None
context_cache: bool = False context_cache: bool | None = None
cache_ttl_seconds: int = 300 cache_ttl_seconds: int = 300
cache_min_prefix_chars: int = 16000 cache_min_prefix_chars: int = 16000
cache_required: bool = False cache_required: bool = False
@@ -70,9 +70,11 @@ class ProviderConfig:
raise LLMError("LLM_CONFIG") raise LLMError("LLM_CONFIG")
if not isinstance(self.adapter, str) or not re.fullmatch(r"[A-Za-z0-9_-]{1,64}", self.adapter): if not isinstance(self.adapter, str) or not re.fullmatch(r"[A-Za-z0-9_-]{1,64}", self.adapter):
raise LLMError("LLM_CONFIG") raise LLMError("LLM_CONFIG")
for value in (self.enabled, self.allow_insecure_http, self.context_cache, self.cache_required): for value in (self.enabled, self.allow_insecure_http, self.cache_required):
if not isinstance(value, bool): if not isinstance(value, bool):
raise LLMError("LLM_CONFIG") raise LLMError("LLM_CONFIG")
if self.context_cache is not None and type(self.context_cache) is not bool:
raise LLMError('LLM_CONFIG')
for effort in (self.reasoning_effort, self.entity_resolution_reasoning_effort): for effort in (self.reasoning_effort, self.entity_resolution_reasoning_effort):
if effort is not None and (not isinstance(effort, str) or effort not in {"none", "minimal", "low", "medium", "high"}): if effort is not None and (not isinstance(effort, str) or effort not in {"none", "minimal", "low", "medium", "high"}):
raise LLMError("LLM_CONFIG") raise LLMError("LLM_CONFIG")
@@ -82,7 +84,7 @@ class ProviderConfig:
positive(self.cache_min_prefix_chars, integer=True) positive(self.cache_min_prefix_chars, integer=True)
if self.context_cache and self.adapter != 'gemini': if self.context_cache and self.adapter != 'gemini':
raise LLMError('LLM_CONFIG') raise LLMError('LLM_CONFIG')
if self.cache_required and not self.context_cache: if self.cache_required and not self.use_context_cache:
raise LLMError('LLM_CONFIG') raise LLMError('LLM_CONFIG')
if self.cache_required_prompt_prefix is not None and (not self.cache_required if self.cache_required_prompt_prefix is not None and (not self.cache_required
or not isinstance(self.cache_required_prompt_prefix, str) or not isinstance(self.cache_required_prompt_prefix, str)
@@ -98,6 +100,11 @@ class ProviderConfig:
if cost is not None and (isinstance(cost, bool) or not isinstance(cost, (int, float)) or not math.isfinite(cost) or cost < 0): if cost is not None and (isinstance(cost, bool) or not isinstance(cost, (int, float)) or not math.isfinite(cost) or cost < 0):
raise LLMError("LLM_CONFIG") raise LLMError("LLM_CONFIG")
@property
def use_context_cache(self):
"""Default to explicit caching on adapters that implement the protocol."""
return self.adapter == 'gemini' and self.context_cache is not False
@dataclass(frozen=True) @dataclass(frozen=True)
class GatewayConfig: class GatewayConfig:
@@ -138,6 +145,10 @@ class GatewayConfig:
# Only configuration references; never resolve credentials for persistence. # Only configuration references; never resolve credentials for persistence.
result = asdict(self) result = asdict(self)
for provider in result["providers"]: for provider in result["providers"]:
if provider['context_cache'] is None:
# Persist the effective choice for capable adapters. This makes
# the policy change visible to frozen-operation hash checks.
provider['context_cache'] = provider['adapter'] == 'gemini'
# Omit new defaults so frozen operations keep their configuration hash. # Omit new defaults so frozen operations keep their configuration hash.
for name, default in {'cached_input_cost_per_million': None, for name, default in {'cached_input_cost_per_million': None,
'cache_storage_cost_per_million_hour': None, 'cache_storage_cost_per_million_hour': None,
+1 -1
View File
@@ -109,7 +109,7 @@ class GeminiProvider:
cache_http_status = None cache_http_status = None
fingerprint = hashlib.sha256(json_bytes([base, config.id, config.model, secret, fingerprint = hashlib.sha256(json_bytes([base, config.id, config.model, secret,
system, prefix, config.cache_ttl_seconds])).hexdigest() system, prefix, config.cache_ttl_seconds])).hexdigest()
if config.context_cache: if config.use_context_cache:
status = 'below_minimum' status = 'below_minimum'
if len(json_bytes([system, prefix]).decode('utf-8')) >= config.cache_min_prefix_chars: if len(json_bytes([system, prefix]).decode('utf-8')) >= config.cache_min_prefix_chars:
with self._lock: with self._lock:
+17
View File
@@ -156,6 +156,23 @@ def test_config_defaults_keep_frozen_snapshot_compatible():
assert GatewayConfig((config(),)).snapshot()['providers'][0]['context_cache'] is True assert GatewayConfig((config(),)).snapshot()['providers'][0]['context_cache'] is True
def test_capable_adapter_defaults_to_explicit_caching_and_pins_choice():
cfg=ProviderConfig('native','gemini-test','http://localhost/v1beta',adapter='gemini')
assert cfg.use_context_cache
assert GatewayConfig((cfg,)).snapshot()['providers'][0]['context_cache'] is True
assert not replace(cfg,context_cache=False).use_context_cache
assert not ProviderConfig('other','m','http://localhost').use_context_cache
assert replace(cfg,cache_required=True).use_context_cache
def test_default_native_configuration_creates_cache(monkeypatch):
adapter=GeminiProvider()
cfg=replace(config(),context_cache=None)
seen=scripted(adapter,monkeypatch,[CACHE,native()])
result=asyncio.run(adapter.generate(cfg,request(),max_response_bytes=10000))
assert result.diagnostics['cache_status']=='created' and len(seen)==2
@pytest.mark.parametrize('changes', [{'adapter':'openai_compatible'}, {'cache_ttl_seconds':True}, @pytest.mark.parametrize('changes', [{'adapter':'openai_compatible'}, {'cache_ttl_seconds':True},
{'cache_min_prefix_chars':0}, {'cached_input_cost_per_million':-1}, {'model':'../other'}, {'reasoning_effort':'none'}]) {'cache_min_prefix_chars':0}, {'cached_input_cost_per_million':-1}, {'model':'../other'}, {'reasoning_effort':'none'}])
def test_bad_cache_configuration_rejected(changes): def test_bad_cache_configuration_rejected(changes):