mirror of
https://github.com/vegu-ai/talemate.git
synced 2026-09-02 03:59:12 +02:00
* fix: weight find_docs by token rarity and normalize for summary length (#169) * fix: dilute the summary phrase bonus by summary length (#169 review) * fix: dilute the phrase bonus by summary excess, not total length (#169 review round 2) * docs: state the phrase-dilution deadband honestly; make its test non-vacuous (#169 review round 3)
This commit is contained in:
veguAI
@@ -61,6 +61,7 @@
|
||||
- "Character Card Import: Attribute extraction no longer runs blind in the default (non-Fast) import mode — the character being profiled was missing from its own extraction prompt, so attributes were generated from the greeting and character book alone, without the character's description. Imported attributes now reflect the description."
|
||||
- "Character Creation: The director's 'Limit character attributes' setting now delivers the number of attributes it promises. The character's own name is written into the generated character sheet as a `Name` line, and it was counted against the limit — so every value delivered one attribute fewer than configured, and a limit of 1 produced no attributes at all. The name no longer costs a slot, in generated sheets as well as in sheets supplied to the Persist Character node."
|
||||
- "Fast Character Creation: The one-shot generation prompt no longer repeats the character's name as an attribute — the name is already generated as its own aspect, so a `Name` entry in the character sheet was duplication, and under a configured attribute limit it consumed one of the allowed lines. This applies at every value of the director's 'Limit character attributes' setting, including the default of 0: Fast mode no longer asks for a `Name` attribute, and one written anyway does not cost a slot."
|
||||
- "Help Agent: Documentation lookups now find the page that actually answers the question. Every word counted the same, so asking about 'koboldcpp settings' returned five different agent settings pages and no KoboldCpp page at all — the common word decided the match and the rest were ties broken alphabetically. Distinctive words now count for far more than ubiquitous ones. Page length also buys far less rank: a longer description of a page used to be a strictly better one, every extra word another free chance to match, so describing a page more thoroughly made it surface for topics it only mentions in passing. Those matches are now diluted by how much the description covers, so a page that mentions a topic in passing no longer outranks the page about it."
|
||||
|
||||
0.38.0:
|
||||
features:
|
||||
|
||||
@@ -3,14 +3,17 @@ Documentation access tools for the help agent.
|
||||
|
||||
Exposes the bundled markdown documentation (docs/ in the talemate root) to the
|
||||
LLM through four focal callbacks: look up pages by topic (find_docs, a
|
||||
keyword-scored match over the generated index), full-text search, read a full
|
||||
document, and read a single section of a document. The generated index
|
||||
keyword-scored match over the generated index, weighted by token rarity and
|
||||
diluted by how many distinct tokens a summary has), full-text search, read a
|
||||
full document, and read a single section of a document. The generated index
|
||||
(docs-index.yaml, shipped next to this module) provides path/title/summary for
|
||||
every page; only a compact section overview of it is injected into the prompt.
|
||||
"""
|
||||
|
||||
import math
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import NamedTuple
|
||||
|
||||
import structlog
|
||||
import yaml
|
||||
@@ -97,6 +100,29 @@ FIND_DOCS_LIMIT = 5
|
||||
_index_cache: list[dict] | None = None
|
||||
|
||||
|
||||
class _IndexedEntry(NamedTuple):
|
||||
"""An index entry with its fields tokenized for scoring."""
|
||||
|
||||
entry: dict
|
||||
title_tokens: set[str]
|
||||
path_tokens: set[str]
|
||||
summary_tokens: set[str]
|
||||
title_phrase: str
|
||||
path_phrase: str
|
||||
summary_phrase: str
|
||||
|
||||
|
||||
class _ScoringIndex(NamedTuple):
|
||||
"""The index prepared for scoring, with its corpus-wide statistics."""
|
||||
|
||||
entries: list[_IndexedEntry]
|
||||
idf: dict[str, float]
|
||||
average_summary_tokens: float
|
||||
|
||||
|
||||
_scoring_cache: tuple[list[dict], _ScoringIndex] | None = None
|
||||
|
||||
|
||||
def docs_available() -> bool:
|
||||
return DOCS_DIR.is_dir()
|
||||
|
||||
@@ -171,14 +197,79 @@ def _token_set(text: str) -> set[str]:
|
||||
return raw | set(_token_seq(text))
|
||||
|
||||
|
||||
def _phrase_text(text: str) -> str:
|
||||
"""The token sequence as a padded string, so phrases match on word boundaries."""
|
||||
return f" {' '.join(_token_seq(text))} "
|
||||
|
||||
|
||||
def _scoring_index() -> _ScoringIndex:
|
||||
"""
|
||||
The loaded index prepared for scoring: per-entry token sets and phrase
|
||||
strings, the inverse document frequency of every token, and the average
|
||||
number of distinct summary tokens. Rebuilt whenever a different index is
|
||||
loaded.
|
||||
"""
|
||||
global _scoring_cache
|
||||
index = load_docs_index()
|
||||
if _scoring_cache is not None and _scoring_cache[0] is index:
|
||||
return _scoring_cache[1]
|
||||
|
||||
entries = [
|
||||
_IndexedEntry(
|
||||
entry=entry,
|
||||
title_tokens=_token_set(entry["title"]),
|
||||
path_tokens=_token_set(entry["path"]),
|
||||
summary_tokens=_token_set(entry["summary"]),
|
||||
title_phrase=_phrase_text(entry["title"]),
|
||||
path_phrase=_phrase_text(entry["path"]),
|
||||
summary_phrase=_phrase_text(entry["summary"]),
|
||||
)
|
||||
for entry in index
|
||||
]
|
||||
|
||||
document_frequency: dict[str, int] = {}
|
||||
for indexed in entries:
|
||||
for token in (
|
||||
indexed.title_tokens | indexed.path_tokens | indexed.summary_tokens
|
||||
):
|
||||
document_frequency[token] = document_frequency.get(token, 0) + 1
|
||||
|
||||
total = len(entries)
|
||||
scoring = _ScoringIndex(
|
||||
entries=entries,
|
||||
# BM25's idf: a token in every page is worth almost nothing, a token
|
||||
# in one page is worth several times an average one
|
||||
idf={
|
||||
token: math.log(1 + (total - count + 0.5) / (count + 0.5))
|
||||
for token, count in document_frequency.items()
|
||||
},
|
||||
# floored at one token so an index whose summaries all tokenize to
|
||||
# nothing cannot divide by zero
|
||||
average_summary_tokens=max(
|
||||
1.0, sum(len(indexed.summary_tokens) for indexed in entries) / max(total, 1)
|
||||
),
|
||||
)
|
||||
_scoring_cache = (index, scoring)
|
||||
return scoring
|
||||
|
||||
|
||||
def find_docs(query: str, limit: int = FIND_DOCS_LIMIT) -> list[dict] | str:
|
||||
"""
|
||||
Look up documentation pages by topic.
|
||||
|
||||
Keyword-scored match over the full index (path, title, summary) -
|
||||
deterministic, no LLM involved. Tokens match on word boundaries (a
|
||||
query token "set" does not match "settings"). Returns the best
|
||||
matches with their path, title, summary and manual URL.
|
||||
query token "set" does not match "settings") and are weighted by how
|
||||
rare they are across the index, so a distinctive term ("koboldcpp")
|
||||
counts for far more than a ubiquitous one ("context"). Token hits in a
|
||||
summary are diluted by the whole of its length; the phrase bonus is
|
||||
diluted only once a summary runs past twice the typical length, and an
|
||||
ordinary-length summary keeps it in full - deliberately, since an exact
|
||||
phrase in a summary of normal length is a real signal, and taxing it
|
||||
demotes pages that are long because they genuinely cover a lot. So a
|
||||
summary well past typical cannot buy rank with a passing mention; one
|
||||
just over it still can. Returns the best matches with their path, title,
|
||||
summary and manual URL.
|
||||
"""
|
||||
query_seq = _token_seq(query or "")
|
||||
if not query_seq:
|
||||
@@ -188,29 +279,42 @@ def find_docs(query: str, limit: int = FIND_DOCS_LIMIT) -> list[dict] | str:
|
||||
# word-boundary phrase, only meaningful for multi-word queries
|
||||
phrase = f" {' '.join(query_seq)} " if len(query_seq) > 1 else None
|
||||
|
||||
scoring = _scoring_index()
|
||||
scored: list[tuple[float, dict]] = []
|
||||
for entry in load_docs_index():
|
||||
title_set = _token_set(entry["title"])
|
||||
path_set = _token_set(entry["path"])
|
||||
summary_set = _token_set(entry["summary"])
|
||||
for indexed in scoring.entries:
|
||||
# a longer-than-average summary dilutes its own matches; a shorter
|
||||
# one is not rewarded for its brevity
|
||||
dilution = max(
|
||||
1.0, len(indexed.summary_tokens) / scoring.average_summary_tokens
|
||||
)
|
||||
# a phrase hit is a single event, not one chance per token, so it is
|
||||
# diluted by how far the summary runs PAST typical length rather than
|
||||
# by the whole of it: an ordinary summary pays nothing, and only one
|
||||
# that has bought enough text for a passing mention to be likely does.
|
||||
# This leaves a deadband below 2x that the generator's summary cap
|
||||
# keeps most entries inside - the cap is the primary control here and
|
||||
# this is the backstop against a pathological entry, not a full fix
|
||||
phrase_dilution = max(1.0, dilution - 1.0)
|
||||
score = 0.0
|
||||
for token in tokens:
|
||||
if token in title_set:
|
||||
score += 3
|
||||
if token in path_set:
|
||||
score += 2
|
||||
if token in summary_set:
|
||||
score += 1
|
||||
weight = scoring.idf.get(token, 0.0)
|
||||
if token in indexed.title_tokens:
|
||||
score += 3 * weight
|
||||
if token in indexed.path_tokens:
|
||||
score += 2 * weight
|
||||
if token in indexed.summary_tokens:
|
||||
score += weight / dilution
|
||||
if phrase:
|
||||
if phrase in f" {' '.join(_token_seq(entry['title']))} ":
|
||||
if phrase in indexed.title_phrase:
|
||||
score += 5
|
||||
elif (
|
||||
phrase in f" {' '.join(_token_seq(entry['path']))} "
|
||||
or phrase in f" {' '.join(_token_seq(entry['summary']))} "
|
||||
):
|
||||
elif phrase in indexed.path_phrase:
|
||||
score += 3
|
||||
elif phrase in indexed.summary_phrase:
|
||||
# titles and paths are not length-variable, summaries are: an
|
||||
# undiluted bonus here is rank a long summary buys outright
|
||||
score += 3 / phrase_dilution
|
||||
if score > 0:
|
||||
scored.append((score, entry))
|
||||
scored.append((score, indexed.entry))
|
||||
scored.sort(key=lambda item: (-item[0], item[1]["path"]))
|
||||
if not scored:
|
||||
return (
|
||||
|
||||
@@ -836,9 +836,14 @@ async def test_process_characters_split_mode_sheet_capped_by_max_attributes(
|
||||
_options_all_disabled(extract_attributes=True),
|
||||
)
|
||||
|
||||
# enforced twice: prompt instruction + parser truncation
|
||||
# enforced twice: prompt instruction + parser truncation. the primed Name
|
||||
# line does not spend a slot (#162), so 2 attributes survive beside it
|
||||
assert "at most 2 attributes" in str(agents.client.prompt_history[0]["prompt"])
|
||||
assert character.base_attributes == {"Name": "Hero", "Age": "20"}
|
||||
assert character.base_attributes == {
|
||||
"Name": "Hero",
|
||||
"Age": "20",
|
||||
"Occupation": "knight",
|
||||
}
|
||||
|
||||
|
||||
async def test_process_characters_split_mode_zero_max_attributes_uncapped(agents):
|
||||
|
||||
@@ -233,6 +233,139 @@ def test_find_docs_real_index_smoke():
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"query,expected",
|
||||
[
|
||||
("koboldcpp client setup", "user-guide/clients/types/koboldcpp.md"),
|
||||
("openrouter api key", "user-guide/apis/openrouter.md"),
|
||||
("context db", "user-guide/world-editor/context-db.md"),
|
||||
("tracking a state", "user-guide/tracking-a-state.md"),
|
||||
(
|
||||
"collector nodes",
|
||||
"user-guide/node-editor/core-concepts/collector_nodes.md",
|
||||
),
|
||||
("character card import", "user-guide/character-card-import.md"),
|
||||
# a thorough summary must keep its phrase bonus: this page is the only
|
||||
# one about the pi coding agent, and competes with node-reference
|
||||
# pages that match nothing but the token "agent"
|
||||
("coding agent", "user-guide/clients/types/pi-bridge.md"),
|
||||
],
|
||||
)
|
||||
def test_find_docs_canonical_pages(query, expected):
|
||||
# pinned against the shipped index: regenerating docs-index.yaml must not
|
||||
# silently displace the page that answers these questions
|
||||
results = docs.find_docs(query)
|
||||
assert isinstance(results, list)
|
||||
assert results[0]["path"] == expected
|
||||
|
||||
|
||||
def test_find_docs_rare_tokens_outweigh_common_ones():
|
||||
# against the shipped index, where many pages are titled "Settings" and
|
||||
# only a handful mention KoboldCpp at all - the distinctive token has to
|
||||
# decide the match, or the query drowns in generic settings pages. Any
|
||||
# KoboldCpp page is a right answer here; the point is that one wins.
|
||||
results = docs.find_docs("koboldcpp settings")
|
||||
assert isinstance(results, list)
|
||||
assert "koboldcpp" in results[0]["path"]
|
||||
|
||||
|
||||
# the real #169 case: a summary hand-extended to describe a documentation
|
||||
# section accurately, which then surfaced for topics the page barely mentions
|
||||
BLOAT = (
|
||||
" Covers per-step toggles for content context, description, attributes, "
|
||||
"dialogue instructions, example dialogue and story intent, Full/Minimal "
|
||||
"presets, and card-data fallback when a step is off or fails."
|
||||
)
|
||||
BLOATED_PATH = "user-guide/character-card-import.md"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"query",
|
||||
[
|
||||
"dialogue instructions",
|
||||
"example dialogue",
|
||||
"story intent",
|
||||
"content context",
|
||||
],
|
||||
)
|
||||
def test_find_docs_long_summary_cannot_displace_better_matches(query, monkeypatch):
|
||||
# #169: padding a summary with topics the page only mentions in passing
|
||||
# must not buy it rank. It may still reach the last slot - the words are
|
||||
# genuinely there, and a page whose token score plus its diluted phrase
|
||||
# bonus still earns that slot keeps it - but it may never take a slot
|
||||
# above the one it enters at.
|
||||
index = docs.load_docs_index()
|
||||
assert any(entry["path"] == BLOATED_PATH for entry in index)
|
||||
bloated_index = [
|
||||
{**entry, "summary": entry["summary"] + BLOAT}
|
||||
if entry["path"] == BLOATED_PATH
|
||||
else entry
|
||||
for entry in index
|
||||
]
|
||||
|
||||
trimmed = [entry["path"] for entry in docs.find_docs(query)]
|
||||
monkeypatch.setattr(docs, "load_docs_index", lambda: bloated_index)
|
||||
bloated = [entry["path"] for entry in docs.find_docs(query)]
|
||||
|
||||
assert BLOATED_PATH not in trimmed
|
||||
if BLOATED_PATH in bloated:
|
||||
assert bloated[-1] == BLOATED_PATH
|
||||
assert bloated[:-1] == trimmed[:-1]
|
||||
else:
|
||||
assert bloated == trimmed
|
||||
|
||||
|
||||
def test_find_docs_summary_phrase_bonus_is_diluted(monkeypatch):
|
||||
# the phrase bonus is the one component that could still be bought with
|
||||
# length: an exact phrase planted in a padded summary used to add a flat
|
||||
# +3 and evict a page whose token score alone outscored the intruder
|
||||
index = docs.load_docs_index()
|
||||
bloated_index = [
|
||||
{**entry, "summary": entry["summary"] + BLOAT}
|
||||
if entry["path"] == BLOATED_PATH
|
||||
else entry
|
||||
for entry in index
|
||||
]
|
||||
monkeypatch.setattr(docs, "load_docs_index", lambda: bloated_index)
|
||||
|
||||
# "dialogue instructions" is in BLOAT verbatim, so the padded entry takes
|
||||
# the summary-phrase branch - and must still stay out of the results
|
||||
paths = [entry["path"] for entry in docs.find_docs("dialogue instructions")]
|
||||
assert BLOATED_PATH not in paths
|
||||
assert "user-guide/agents/summarizer/settings.md" in paths
|
||||
|
||||
|
||||
def test_find_docs_phrase_dilution_bites_past_typical_length(monkeypatch):
|
||||
# the other side of the trade from the canonical "coding agent" pin: that
|
||||
# page wins its own topic at its real summary length, and has to lose once
|
||||
# its summary runs well past typical.
|
||||
#
|
||||
# Padded to 2.5x typical deliberately. Pad much further and the page loses
|
||||
# on summary-token dilution alone, which would leave this test passing with
|
||||
# the phrase dilution deleted entirely; at 2.5x it still wins on an
|
||||
# undiluted bonus, so only the dilution can decide it.
|
||||
index = docs.load_docs_index()
|
||||
target = "user-guide/clients/types/pi-bridge.md"
|
||||
scoring = docs._scoring_index()
|
||||
indexed = next(e for e in scoring.entries if e.entry["path"] == target)
|
||||
# "zz0 zz1 ..." - distinct filler tokens that match no real query
|
||||
filler_tokens = int(scoring.average_summary_tokens * 2.5) - len(
|
||||
indexed.summary_tokens
|
||||
)
|
||||
assert filler_tokens > 0
|
||||
filler = " " + " ".join(f"zz{i}" for i in range(filler_tokens))
|
||||
padded = [
|
||||
{**entry, "summary": entry["summary"] + filler}
|
||||
if entry["path"] == target
|
||||
else entry
|
||||
for entry in index
|
||||
]
|
||||
|
||||
assert docs.find_docs("coding agent")[0]["path"] == target
|
||||
monkeypatch.setattr(docs, "load_docs_index", lambda: padded)
|
||||
assert docs.find_docs("coding agent")[0]["path"] != target
|
||||
|
||||
|
||||
def test_docs_section_overview(stub_docs_index):
|
||||
overview = docs.docs_section_overview()
|
||||
by_prefix = {entry["prefix"]: entry for entry in overview}
|
||||
|
||||
Reference in New Issue
Block a user