Enable explicit caching by default on capable Gemini adapter
This commit is contained in:
1 parent
02a7364c58
commit
d5dac27f56
5 files changed
+53
-14
No files matched your search
@@ -1,4 +1,4 @@
|
||||
# Optional separate configuration. Copy to an ignored *.local.toml file.
|
||||
# Native Gemini configuration. Copy to an ignored *.local.toml file.
|
||||
# This does not replace the existing OpenAI-compatible configuration.
|
||||
[llm]
|
||||
privacy_profile = "cloud_allowed"
|
||||
@@ -19,7 +19,8 @@ max_output_tokens = 16384
|
||||
reasoning_effort = "medium"
|
||||
entity_resolution_reasoning_effort = "low"
|
||||
|
||||
# Opt-in. Instruction/schema prefix only; live evidence is always complete.
|
||||
# Enabled by default on this adapter. Instruction/schema prefix only;
|
||||
# live evidence is always complete. Explicit false is an operational override.
|
||||
context_cache = true
|
||||
cache_ttl_seconds = 300
|
||||
# A character heuristic, not a tokenizer or a format limit. Provider minimums
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# Optional context caching and usage accounting
|
||||
# Context caching and usage accounting
|
||||
|
||||
The package remains the complete, authoritative record. Saved requests, evidence,
|
||||
responses, checkpoints and provider configuration references remain portable.
|
||||
@@ -14,14 +14,14 @@ The existing `openai_compatible` adapter now reads
|
||||
hits when the provider actually reports them, without changing existing requests
|
||||
or enabling explicit caching. An absent count means unknown, not zero.
|
||||
|
||||
The optional `gemini` adapter uses the native GenerateContent API. It moves the
|
||||
The `gemini` adapter uses the native GenerateContent API. It moves the
|
||||
unchanged system instructions and authoritative output schema to the beginning
|
||||
of the wire request. Only an exact top-level schema duplicate is removed from
|
||||
the live user JSON; no case facts, identifiers, evidence or assessments are
|
||||
removed. The archived request is not changed. This is deterministic transport
|
||||
preparation, not another LLM summary.
|
||||
|
||||
With `context_cache = true`, this instruction/schema prefix can be stored as a
|
||||
By default on the native Gemini adapter, this instruction/schema prefix is stored as a
|
||||
Gemini `cachedContents` resource. Case-specific input and all research history
|
||||
are still sent in full. The prefix is independent of the currently selected
|
||||
hypothesis. Reuse requires an exact match of endpoint, provider ID, model,
|
||||
@@ -41,12 +41,16 @@ improvement.
|
||||
## Configuration and fallback
|
||||
|
||||
See `config/llm.gemini-cache.example.toml`. Existing local configuration files
|
||||
are not edited. The native base URL ends in `/v1beta`, **not** `/v1beta/openai`.
|
||||
`context_cache` defaults to false and is allowed only for the native adapter.
|
||||
use the adapter selected in each provider entry. The native base URL ends in `/v1beta`, **not** `/v1beta/openai`.
|
||||
Omitting `context_cache` enables explicit caching on the native Gemini adapter.
|
||||
Adapters without explicit-cache support remain usable without it. Explicit true
|
||||
is allowed only for the native adapter; false remains an operational override.
|
||||
`cached_input_cost_per_million` can also be set for OpenAI-compatible providers
|
||||
that report cache usage. Prices remain optional, model-specific configuration.
|
||||
New default fields are omitted from configuration snapshots to preserve frozen
|
||||
operation hashes and replay of old cases.
|
||||
The effective enabled choice is persisted for native Gemini, so a change from
|
||||
uncached processing is visible to frozen-operation configuration checks. Other
|
||||
adapters retain their old default snapshots. Existing pending operations must
|
||||
keep their original configuration or be tested in a fresh workspace.
|
||||
|
||||
`cache_min_prefix_chars` is an eligibility heuristic, not a token count or API
|
||||
minimum. A short prefix may therefore obtain no explicit cache saving. The
|
||||
@@ -80,7 +84,8 @@ positive, consistent provider-reported cached token count
|
||||
(`LLM_CACHE_USAGE_UNCONFIRMED` otherwise). This is a test assertion, not a
|
||||
promise that Gemini will accept a prefix. A rejected/too-small cache ends the
|
||||
test explicitly. A response without confirmed usage may already have incurred
|
||||
API costs. Default behaviour remains optional; old operation hashes stay valid.
|
||||
API costs. Default behaviour enables caching where supported with graceful fallback;
|
||||
strict enforcement is a separate test mode.
|
||||
Strict cache creation may use half of the configured request timeout, instead
|
||||
of the five-second optional-optimization allowance. The overall deadline still
|
||||
applies; a creation HTTP status is retained for diagnosing a rejected cache.
|
||||
@@ -145,3 +150,8 @@ for this inspection; it is not a new quality or cost benchmark.
|
||||
- [Thinking configuration](https://ai.google.dev/gemini-api/docs/generate-content/thinking)
|
||||
- [Model-specific pricing](https://ai.google.dev/gemini-api/docs/pricing)
|
||||
- [Interactions session API, for a separate future adapter](https://ai.google.dev/gemini-api/docs/interactions-overview)
|
||||
|
||||
The regular local Gemini configurations were switched to the native adapter for
|
||||
this policy. Small prefixes still respect the configured eligibility heuristic;
|
||||
API minimums are not bypassed by padding. Transient cache failures retain the
|
||||
full-input fallback so an optimization outage does not stop normal analysis.
|
||||
@@ -36,7 +36,7 @@ class ProviderConfig:
|
||||
entity_resolution_reasoning_effort: str | None = None
|
||||
cached_input_cost_per_million: float | None = None
|
||||
cache_storage_cost_per_million_hour: float | None = None
|
||||
context_cache: bool = False
|
||||
context_cache: bool | None = None
|
||||
cache_ttl_seconds: int = 300
|
||||
cache_min_prefix_chars: int = 16000
|
||||
cache_required: bool = False
|
||||
@@ -70,9 +70,11 @@ class ProviderConfig:
|
||||
raise LLMError("LLM_CONFIG")
|
||||
if not isinstance(self.adapter, str) or not re.fullmatch(r"[A-Za-z0-9_-]{1,64}", self.adapter):
|
||||
raise LLMError("LLM_CONFIG")
|
||||
for value in (self.enabled, self.allow_insecure_http, self.context_cache, self.cache_required):
|
||||
for value in (self.enabled, self.allow_insecure_http, self.cache_required):
|
||||
if not isinstance(value, bool):
|
||||
raise LLMError("LLM_CONFIG")
|
||||
if self.context_cache is not None and type(self.context_cache) is not bool:
|
||||
raise LLMError('LLM_CONFIG')
|
||||
for effort in (self.reasoning_effort, self.entity_resolution_reasoning_effort):
|
||||
if effort is not None and (not isinstance(effort, str) or effort not in {"none", "minimal", "low", "medium", "high"}):
|
||||
raise LLMError("LLM_CONFIG")
|
||||
@@ -82,7 +84,7 @@ class ProviderConfig:
|
||||
positive(self.cache_min_prefix_chars, integer=True)
|
||||
if self.context_cache and self.adapter != 'gemini':
|
||||
raise LLMError('LLM_CONFIG')
|
||||
if self.cache_required and not self.context_cache:
|
||||
if self.cache_required and not self.use_context_cache:
|
||||
raise LLMError('LLM_CONFIG')
|
||||
if self.cache_required_prompt_prefix is not None and (not self.cache_required
|
||||
or not isinstance(self.cache_required_prompt_prefix, str)
|
||||
@@ -98,6 +100,11 @@ class ProviderConfig:
|
||||
if cost is not None and (isinstance(cost, bool) or not isinstance(cost, (int, float)) or not math.isfinite(cost) or cost < 0):
|
||||
raise LLMError("LLM_CONFIG")
|
||||
|
||||
@property
|
||||
def use_context_cache(self):
|
||||
"""Default to explicit caching on adapters that implement the protocol."""
|
||||
return self.adapter == 'gemini' and self.context_cache is not False
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class GatewayConfig:
|
||||
@@ -138,6 +145,10 @@ class GatewayConfig:
|
||||
# Only configuration references; never resolve credentials for persistence.
|
||||
result = asdict(self)
|
||||
for provider in result["providers"]:
|
||||
if provider['context_cache'] is None:
|
||||
# Persist the effective choice for capable adapters. This makes
|
||||
# the policy change visible to frozen-operation hash checks.
|
||||
provider['context_cache'] = provider['adapter'] == 'gemini'
|
||||
# Omit new defaults so frozen operations keep their configuration hash.
|
||||
for name, default in {'cached_input_cost_per_million': None,
|
||||
'cache_storage_cost_per_million_hour': None,
|
||||
|
||||
@@ -109,7 +109,7 @@ class GeminiProvider:
|
||||
cache_http_status = None
|
||||
fingerprint = hashlib.sha256(json_bytes([base, config.id, config.model, secret,
|
||||
system, prefix, config.cache_ttl_seconds])).hexdigest()
|
||||
if config.context_cache:
|
||||
if config.use_context_cache:
|
||||
status = 'below_minimum'
|
||||
if len(json_bytes([system, prefix]).decode('utf-8')) >= config.cache_min_prefix_chars:
|
||||
with self._lock:
|
||||
|
||||
@@ -156,6 +156,23 @@ def test_config_defaults_keep_frozen_snapshot_compatible():
|
||||
assert GatewayConfig((config(),)).snapshot()['providers'][0]['context_cache'] is True
|
||||
|
||||
|
||||
def test_capable_adapter_defaults_to_explicit_caching_and_pins_choice():
|
||||
cfg=ProviderConfig('native','gemini-test','http://localhost/v1beta',adapter='gemini')
|
||||
assert cfg.use_context_cache
|
||||
assert GatewayConfig((cfg,)).snapshot()['providers'][0]['context_cache'] is True
|
||||
assert not replace(cfg,context_cache=False).use_context_cache
|
||||
assert not ProviderConfig('other','m','http://localhost').use_context_cache
|
||||
assert replace(cfg,cache_required=True).use_context_cache
|
||||
|
||||
|
||||
def test_default_native_configuration_creates_cache(monkeypatch):
|
||||
adapter=GeminiProvider()
|
||||
cfg=replace(config(),context_cache=None)
|
||||
seen=scripted(adapter,monkeypatch,[CACHE,native()])
|
||||
result=asyncio.run(adapter.generate(cfg,request(),max_response_bytes=10000))
|
||||
assert result.diagnostics['cache_status']=='created' and len(seen)==2
|
||||
|
||||
|
||||
@pytest.mark.parametrize('changes', [{'adapter':'openai_compatible'}, {'cache_ttl_seconds':True},
|
||||
{'cache_min_prefix_chars':0}, {'cached_input_cost_per_million':-1}, {'model':'../other'}, {'reasoning_effort':'none'}])
|
||||
def test_bad_cache_configuration_rejected(changes):
|
||||
|
||||
Reference in new issue
Block a user