marek/MtoRagSystem
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

optimize retriever

Browse Source
This commit is contained in:
team2
2026-02-27 19:27:13 +01:00
parent 44be40a24d
commit 5e004c99f4
6 changed files with 41 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
src/Knowledge/Retrieval/NdjsonHybridRetriever.php
View File

@@ -14,16 +14,18 @@ use App\Vector\VectorSearchClient;
final class NdjsonHybridRetriever implements RetrieverInterface
{
private const VECTOR_SCORE_THRESHOLD = 0.75;
private const VECTOR_SCORE_THRESHOLD = 0.72;
private const HARD_MAX_CHUNKS = 200;
private const HARD_MAX_VECTORK = 200;
private const HARD_MAX_CHUNKS = 120;
private const HARD_MAX_VECTORK = 250;
private const LIST_BONUS = 1.5;
/**
* Tags dürfen nur ein kleiner Bonus sein (kein Gate/Filter).
* Enterprise Default: klein halten, sonst dominieren Tags wieder.
* Tags must only provide a small bonus (never act as a gate/filter).
* Enterprise default: keep it low, otherwise tags will dominate ranking again.
*/
private const TAG_SCORE_BONUS = 0.5;
private const TAG_SCORE_BONUS = 0.1 * (1 - self::VECTOR_SCORE_THRESHOLD);
public function __construct(
private readonly NdjsonChunkLookup $lookup,
@@ -47,21 +49,18 @@ final class NdjsonHybridRetriever implements RetrieverInterface
return $this->retrieveInternal($prompt, $config);
}
public function retrieveForConfig(string $prompt, ModelGenerationConfig $config): array
{
return $this->retrieveInternal($prompt, $config);
}
private function retrieveInternal(string $prompt, ModelGenerationConfig $config): array
public function retrieveInternal(string $prompt, ModelGenerationConfig $config): array
{
$limit = max(1, min($config->getRetrievalMaxChunks(), self::HARD_MAX_CHUNKS));
$vectorTopKBase = max(1, min($config->getRetrievalVectorTopK(), self::HARD_MAX_VECTORK));
// Wichtig: List-Detection bleibt auf Originalprompt (sonst entfernst du "zeige/liste" etc.)
// Important: list-intent detection must run on the original prompt
// (cleaning might remove "show/list" etc.).
$isListQuery = $this->intentLite->isListQuery($prompt);
// -------------------------------------------------
// CLEAN QUERY (nur für Retrieval: Tags + Vector)
// CLEAN QUERY (retrieval-only: tag routing + vector search)
// -------------------------------------------------
$cleanQuery = $this->queryCleaner->clean($prompt);
if ($cleanQuery === '') {
@@ -69,7 +68,7 @@ final class NdjsonHybridRetriever implements RetrieverInterface
}
// -------------------------------------------------
// 1) Tag Routing (bereinigte Query) -> NUR Bonus
// 1) Tag routing (cleaned query) -> bonus only
// -------------------------------------------------
$candidateDocIds = $this->tagRouting->route($cleanQuery);
$candidateSet = null;
@@ -79,29 +78,29 @@ final class NdjsonHybridRetriever implements RetrieverInterface
}
// -------------------------------------------------
// 2) TopK bestimmen
// 2) Determine TopK
// -------------------------------------------------
$topK = $vectorTopKBase;
// List mode: höhere Abdeckung, um mehr Dokumente zu ranken
// List mode: increase coverage to rank more documents
if ($isListQuery) {
$topK = (int)round($vectorTopKBase * 2.5);
$topK = (int)round($vectorTopKBase * self::LIST_BONUS);
}
$topK = max(1, min($topK, self::HARD_MAX_VECTORK));
// -------------------------------------------------
// 3) Vector Search (immer GLOBAL; Tags sind KEIN Filter)
// 3) Vector search (always GLOBAL; tags are NOT a filter)
// -------------------------------------------------
$hits = $this->vectorClient->search($cleanQuery, $topK);
if ($hits === []) {
// Tags dürfen NICHT als Fallback wirken (sonst wieder zu mächtig)
// Tags must NOT act as a fallback (otherwise they become too powerful again).
return [];
}
// -------------------------------------------------
// 4) ChunkIds + Scores sammeln (raw)
// 4) Collect chunkIds + scores (raw)
// -------------------------------------------------
/** @var array<string,float> $rawScoreByChunkId */
$rawScoreByChunkId = [];
@@ -113,14 +112,14 @@ final class NdjsonHybridRetriever implements RetrieverInterface
$raw = (float)$hit['score'];
// Threshold wird auf RAW Score angewendet (Qualitätsgate)
// Apply the threshold to the RAW score (quality gate)
if ($raw < self::VECTOR_SCORE_THRESHOLD) {
continue;
}
$chunkId = (string)$hit['chunk_id'];
// Falls mehrfach: den besten raw score behalten
// If a chunk appears multiple times, keep the best raw score
if (!isset($rawScoreByChunkId[$chunkId]) || $raw > $rawScoreByChunkId[$chunkId]) {
$rawScoreByChunkId[$chunkId] = $raw;
}
@@ -130,11 +129,11 @@ final class NdjsonHybridRetriever implements RetrieverInterface
return [];
}
// Lookup liefert docId + Text etc.
// Lookup returns document_id + text etc.
$rows = $this->lookup->findByChunkIds(array_keys($rawScoreByChunkId));
// -------------------------------------------------
// 5) Adjusted Score (Tag Bonus) + Ranking
// 5) Adjusted score (tag bonus) + ranking
// -------------------------------------------------
/** @var array<string,float> $adjScoreByChunkId */
$adjScoreByChunkId = [];
@@ -174,7 +173,7 @@ final class NdjsonHybridRetriever implements RetrieverInterface
$rankedChunkIds = array_keys($adjScoreByChunkId);
// -------------------------------------------------
// 6) Listenmodus → Dokument-Ranking (mit Tag-Bonus in Scores)
// 6) List mode -> document ranking (with tag bonus in scores)
// -------------------------------------------------
if ($isListQuery) {
$rankedDocIds = $this->rankDocumentsFromAdjustedScores($adjScoreByChunkId, $rows);
@@ -189,7 +188,7 @@ final class NdjsonHybridRetriever implements RetrieverInterface
}
// -------------------------------------------------
// 7) Normaler Chunk-Modus (nach adjusted Ranking)
// 7) Normal chunk mode (by adjusted ranking)
// -------------------------------------------------
return $this->collectTexts($rankedChunkIds, $rows, $limit);
}
@@ -199,7 +198,7 @@ final class NdjsonHybridRetriever implements RetrieverInterface
// =========================================================
// =========================================================
// DOCUMENT RANKING (Adjusted Scores incl. Tag Bonus)
// DOCUMENT RANKING (Adjusted scores incl. tag bonus)
// =========================================================
/**