fix: reduce chunk size to prevent embedding context overflow

nomic-embed-text tokenizes Chinese characters differently from the
estimateTokenCount heuristic (3-5 real tokens per character vs 1
estimated). Large chunks overflow the 8192 token context. Reduce
TargetTokens/MaxTokens to keep chunks within limits.
This commit is contained in:
Aibo117
2026-05-26 14:56:24 +08:00
parent be91feea63
commit fa9f64752b
+3 -3
View File
@@ -38,9 +38,9 @@ const knowledgeCollectionName = "knowledge_chunks"
var Index = &index{ var Index = &index{
chunkConfig: ChunkingConfig{ chunkConfig: ChunkingConfig{
Provider: string(enums.KnowledgeChunkProviderStructured), Provider: string(enums.KnowledgeChunkProviderStructured),
TargetTokens: 300, TargetTokens: 100,
MaxTokens: 400, MaxTokens: 150,
OverlapTokens: 40, OverlapTokens: 20,
EnableFallback: true, EnableFallback: true,
}, },
registry: ragchunk.NewDefaultRegistry(), registry: ragchunk.NewDefaultRegistry(),