mirror of
https://github.com/vegu-ai/talemate.git
synced 2026-09-01 19:48:52 +02:00
fix: enable prompt caching for Anthropic
This commit is contained in:
@@ -19,6 +19,7 @@
|
||||
- "Autocomplete Hints: Dialogue, narrative, and contextual autocomplete now accept a free-form `{...}` hint block at the end of the input. Anything inside the curly braces is passed to the LLM as directional guidance for the continuation (tone, beats, sensory detail, character reactions, etc.) and the brace block itself is stripped from the field when the suggestion is accepted. Example: typing `\"Kaira!?\" he yelled {dark corridor, no response, ship shakes}` cues the model on what to weave into the completion without those tokens ending up in the scene text. Works in the scene input, character description / details / attributes, scene intro, and inline character / narrator / context-investigation message editing. Toggle via the new `Enable Hints` setting on the Creator agent's Autocomplete config (default on); trailing braces only, mid-text `{...}` is left alone."
|
||||
- "Node Editor Log: Log entries now show a right-aligned `HH:MM:SS.mmm` timestamp derived from the node execution's `start_time`, making it easier to correlate log entries with what was happening in the scene."
|
||||
fixes:
|
||||
- "Anthropic / OpenRouter Clients: Fixed the 'Optimize for Prompt Caching' client toggle not actually enabling caching on Anthropic. The setting previously only reordered prompt sections so the prefix would be cacheable, but the required `cache_control` API parameter was never sent — so requests still missed the cache. The Anthropic client now sends top-level ephemeral `cache_control` when the toggle is on, and the OpenRouter client does the same for `anthropic/*` models (OpenAI, DeepSeek, and Gemini cache automatically on OpenRouter, so no parameter is needed there). Note: enabling caching for Anthropic models via OpenRouter forces routing to direct Anthropic, excluding Bedrock/Vertex endpoints for that request."
|
||||
- "Game Loop Event: Fixed an internal scene-loop event being constructed with the wrong scene reference, surfaced by the pydantic migration."
|
||||
- "Conversation Agent: Stopped injecting `#` into the LLM stop-sequence list on every conversation turn. The conversation agent's prompt-parameter hook now only wipes character-name stop sequences when the `inject_character_names_into_stop` setting is disabled, and is a no-op otherwise."
|
||||
- "Scene-Group Templates: Templates created via the prompt manager under the `scene` agent prefix (stored at `{scene}/templates/scene/`) are now resolvable from Jinja `{% include %}` calls regardless of the active agent type."
|
||||
|
||||
@@ -334,6 +334,14 @@ class AnthropicClient(ConcurrentInferenceMixin, EndpointOverrideMixin, ClientBas
|
||||
if "max_tokens" not in parameters:
|
||||
parameters["max_tokens"] = self.api_max_output_tokens
|
||||
|
||||
# Prompt caching is opt-in on the Anthropic API — without cache_control
|
||||
# the request is never cached. Top-level cache_control auto-places the
|
||||
# breakpoint on the last cacheable block, which pairs with the
|
||||
# after_history volatile-context placement that optimize_prompt_caching
|
||||
# already enables in the prompt builder.
|
||||
if self.optimize_prompt_caching:
|
||||
parameters["cache_control"] = {"type": "ephemeral"}
|
||||
|
||||
self.log.debug(
|
||||
"generate",
|
||||
model=self.model_name,
|
||||
|
||||
@@ -180,6 +180,23 @@ PROVIDER_FIELD_GROUP = FieldGroup(
|
||||
MIN_THINKING_TOKENS = 256
|
||||
|
||||
|
||||
def cache_control_for_model(model_name: str) -> dict | None:
|
||||
"""
|
||||
Return the cache_control parameter needed to enable prompt caching for
|
||||
the given OpenRouter model, or None if no parameter is required.
|
||||
|
||||
OpenAI / DeepSeek / Gemini cache automatically on OpenRouter. Anthropic
|
||||
models require explicit cache_control or the request is never cached.
|
||||
For Anthropic, top-level ephemeral cache_control mirrors the native SDK's
|
||||
automatic-cache mode — the breakpoint is auto-placed on the last cacheable
|
||||
block. Note: this forces routing to direct Anthropic, excluding Bedrock
|
||||
and Vertex endpoints.
|
||||
"""
|
||||
if model_name.startswith("anthropic/"):
|
||||
return {"type": "ephemeral"}
|
||||
return None
|
||||
|
||||
|
||||
@register()
|
||||
class OpenRouterClient(ConcurrentInferenceMixin, ClientBase):
|
||||
"""
|
||||
@@ -369,6 +386,11 @@ class OpenRouterClient(ConcurrentInferenceMixin, ClientBase):
|
||||
if provider:
|
||||
parameters["provider"] = provider
|
||||
|
||||
if self.optimize_prompt_caching:
|
||||
cache_control = cache_control_for_model(self.model_name)
|
||||
if cache_control is not None:
|
||||
parameters["cache_control"] = cache_control
|
||||
|
||||
# Prepare request payload
|
||||
payload = {
|
||||
"model": self.model_name,
|
||||
|
||||
Reference in New Issue
Block a user