marek/MtoRagSystem
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

optimize debug retrieval

Browse Source
This commit is contained in:
team2
2026-03-02 07:20:48 +01:00
parent e6858d51d4
commit a1acc3e106
2 changed files with 217 additions and 246 deletions
Download Patch File Download Diff File Expand all files Collapse all files

355
src/Knowledge/Retrieval/NdjsonHybridRetriever.php
View File

@@ -18,18 +18,15 @@ final class NdjsonHybridRetriever implements RetrieverInterface
{
private const VECTOR_SCORE_THRESHOLD = 0.82;
// Guardrails
private const HARD_MAX_CHUNKS = 90;
private const HARD_MAX_VECTORK = 250;
private const LIST_BONUS = 1.25;
// Selection / Fusion
private const MAX_CHUNKS_PER_DOC = 2;
private const MIN_CHUNK_DISTANCE = 2;
private const RRF_K = 60;
// Hardening (nur Edge-Cases; Standardverhalten bleibt gleich)
private const THRESHOLD_FLOOR = 0.65;
private const THRESHOLD_CEIL = 0.90;
private const EMPTY_RRF_FALLBACK_TOPN = 5;
@@ -45,161 +42,140 @@ final class NdjsonHybridRetriever implements RetrieverInterface
private readonly CatalogIntentLite $catalogIntent,
private readonly IntentRouteResolver $routeResolver,
private readonly EntityCatalogService $entityCatalogService
)
{
}
) {}
// =========================================================
// PRODUCTION
// PUBLIC API
// =========================================================
public function retrieve(string $prompt): array
{
$config = $this->configRepository->findActiveForModel();
$config = $this->requireConfig();
$result = $this->execute($prompt, $config, false);
if ($config === null) {
throw new \RuntimeException('No active ModelGenerationConfig found.');
if ($result['catalogBlock'] !== null) {
return [$result['catalogBlock']];
}
return $this->retrieveInternal($prompt, $config);
return $this->collectTextsFromIds(
$result['selectedChunkIds'],
$result['rows']
);
}
public function retrieveInternal(string $prompt, ModelGenerationConfig $config): array
{
// ------------------------------------------------------------
// ROUTING-MATRIX (minimal, ohne Core zu zerlegen)
// ------------------------------------------------------------
// 1) Entity (semantisch über Tag-Vektor)
$entityLabel = $this->catalogIntent->detect($prompt);
// 2) Intent (regelbasiert)
$salesIntent = $this->detectSalesIntent($prompt);
// 3) Route bestimmen (Intent + Entity)
$route = $this->routeResolver->resolve($salesIntent, $entityLabel);
// 4) Early Exit nur für catalog_list
if ($route === IntentRouteResolver::ROUTE_CATALOG_LIST && $entityLabel !== null) {
$catalogBlock = $this->entityCatalogService->listByTerm($entityLabel);
if ($catalogBlock !== null) {
return [$catalogBlock];
}
}
// ------------------------------------------------------------
// NORMALER CORE
// ------------------------------------------------------------
$core = $this->runCore($prompt, $config, false, $salesIntent);
if ($core['ranked_chunk_ids'] === [] || $core['rows'] === []) {
return [];
}
if (!$core['is_list_query']) {
$selectedIds = $this->selectSalesChunkIds($core['ranked_chunk_ids'], $core['rows'], $core['limit']);
return $this->collectTextsFromIds($selectedIds, $core['rows']);
}
$selectedIds = $this->selectListChunkIds($core['ranked_chunk_ids'], $core['rows'], $core['limit']);
return $this->collectTextsFromIds($selectedIds, $core['rows']);
}
// =========================================================
// DEBUG (deterministisch: gleiche Intent-Bestimmung wie Prod)
// =========================================================
/**
* @return array<int, array{
* rank:int,
* chunk_id:string,
* document_id:(string|null),
* raw_score:(float|null),
* rrf_score:(float|null),
* threshold:float,
* intent:string,
* is_list_query:bool,
* text:string
* }>
*/
public function retrieveDebug(string $prompt, ?ModelGenerationConfig $config = null): array
{
$config = $config ?? $this->configRepository->findActiveForModel();
$config = $config ?? $this->requireConfig();
$result = $this->execute($prompt, $config, true);
if ($config === null) {
throw new \RuntimeException('No active ModelGenerationConfig found.');
}
$salesIntent = $this->detectSalesIntent($prompt);
// Debug zeigt Core ohne Early Exit, aber mit identischem Intent-Input.
$core = $this->runCore($prompt, $config, true, $salesIntent);
if ($core['ranked_chunk_ids'] === [] || $core['rows'] === []) {
return [];
}
$selectedChunkIds = $core['is_list_query']
? $this->selectListChunkIds($core['ranked_chunk_ids'], $core['rows'], $core['limit'])
: $this->selectSalesChunkIds($core['ranked_chunk_ids'], $core['rows'], $core['limit']);
if ($selectedChunkIds === []) {
return [];
if ($result['catalogBlock'] !== null) {
return [[
'rank' => 1,
'chunk_id' => '__CATALOG_LIST__',
'document_id' => null,
'raw_score' => null,
'rrf_score' => null,
'threshold' => 0.0,
'intent' => $result['intent'],
'route' => $result['route'],
'entity_label' => $result['entityLabel'],
'is_list_query' => true,
'text' => $result['catalogBlock'],
]];
}
$out = [];
$rank = 0;
foreach ($selectedChunkIds as $chunkId) {
if (!isset($core['rows'][$chunkId])) {
foreach ($result['selectedChunkIds'] as $chunkId) {
if (!isset($result['rows'][$chunkId])) {
continue;
}
$rank++;
$text = trim((string)($core['rows'][$chunkId]['text'] ?? ''));
$out[] = [
'rank' => $rank,
'chunk_id' => (string)$chunkId,
'document_id' => isset($core['rows'][$chunkId]['document_id']) ? (string)$core['rows'][$chunkId]['document_id'] : null,
'raw_score' => isset($core['raw_scores'][$chunkId]) ? (float)$core['raw_scores'][$chunkId] : null,
'rrf_score' => isset($core['rrf_scores'][$chunkId]) ? (float)$core['rrf_scores'][$chunkId] : null,
'threshold' => (float)$core['threshold'],
'intent' => (string)$core['sales_intent'],
'is_list_query' => (bool)$core['is_list_query'],
'text' => $text,
'chunk_id' => $chunkId,
'document_id' => $result['rows'][$chunkId]['document_id'] ?? null,
'raw_score' => $result['rawScores'][$chunkId] ?? null,
'rrf_score' => $result['rrfScores'][$chunkId] ?? null,
'threshold' => $result['threshold'],
'intent' => $result['intent'],
'route' => $result['route'],
'entity_label' => $result['entityLabel'],
'is_list_query' => $result['isListQuery'],
'text' => trim((string)$result['rows'][$chunkId]['text']),
];
}
return $out;
}
// =========================================================
// CENTRAL ORCHESTRATION
// =========================================================
private function execute(
string $prompt,
ModelGenerationConfig $config,
bool $withScores
): array {
$entityLabel = $this->catalogIntent->detect($prompt);
$salesIntent = $this->detectSalesIntent($prompt);
$route = $this->routeResolver->resolve($salesIntent, $entityLabel);
if ($route === IntentRouteResolver::ROUTE_CATALOG_LIST && $entityLabel !== null) {
$catalogBlock = $this->entityCatalogService->listByTerm($entityLabel);
if ($catalogBlock !== null) {
return [
'route' => $route,
'entityLabel' => $entityLabel,
'intent' => $salesIntent,
'isListQuery' => true,
'selectedChunkIds' => [],
'rows' => [],
'rrfScores' => [],
'rawScores' => [],
'threshold' => 0.0,
'catalogBlock' => trim($catalogBlock),
];
}
}
$core = $this->runCore($prompt, $config, $withScores, $salesIntent);
$selectedChunkIds = $core['is_list_query']
? $this->selectListChunkIds($core['ranked_chunk_ids'], $core['rows'], $core['limit'])
: $this->selectSalesChunkIds($core['ranked_chunk_ids'], $core['rows'], $core['limit']);
return [
'route' => $route,
'entityLabel' => $entityLabel,
'intent' => $salesIntent,
'isListQuery' => $core['is_list_query'],
'selectedChunkIds' => $selectedChunkIds,
'rows' => $core['rows'],
'rrfScores' => $core['rrf_scores'],
'rawScores' => $core['raw_scores'],
'threshold' => $core['threshold'],
'catalogBlock' => null,
];
}
// =========================================================
// CORE PIPELINE
// =========================================================
/**
* @return array{
* limit:int,
* is_list_query:bool,
* sales_intent:string,
* threshold:float,
* topk:int,
* ranked_chunk_ids:string[],
* rows:array<string, array<string,mixed>>,
* rrf_scores:array<string,float>,
* raw_scores:array<string,float>
* }
*/
private function runCore(
string $prompt,
ModelGenerationConfig $config,
bool $withScores,
string $salesIntent
): array
{
): array {
$limit = max(1, min($config->getRetrievalMaxChunks(), self::HARD_MAX_CHUNKS));
$vectorTopKBase = max(1, min($config->getRetrievalVectorTopK(), self::HARD_MAX_VECTORK));
@@ -210,33 +186,24 @@ final class NdjsonHybridRetriever implements RetrieverInterface
$cleanQuery = $prompt;
}
[$threshold, $topK] = $this->computeThresholdAndTopK($salesIntent, $isListQuery, $vectorTopKBase);
[$threshold, $topK] = $this->computeThresholdAndTopK(
$salesIntent,
$isListQuery,
$vectorTopKBase
);
// Candidate Routing (keine Set-Map nötig; scoped nur wenn IDs existieren)
$candidateDocIds = $this->tagRouting->route($cleanQuery);
$candidateDocIds = is_array($candidateDocIds) ? array_values(array_unique(array_filter($candidateDocIds, 'is_string'))) : [];
$candidateDocIds = is_array($candidateDocIds)
? array_values(array_unique(array_filter($candidateDocIds, 'is_string')))
: [];
$globalHits = $this->vectorClient->search($cleanQuery, $topK);
$scopedHits = [];
if ($candidateDocIds !== []) {
if (!empty($candidateDocIds)) {
$scopedHits = $this->vectorClient->searchScoped($cleanQuery, $topK, $candidateDocIds);
}
if ($globalHits === [] && $scopedHits === []) {
return [
'limit' => $limit,
'is_list_query' => $isListQuery,
'sales_intent' => $salesIntent,
'threshold' => $threshold,
'topk' => $topK,
'ranked_chunk_ids' => [],
'rows' => [],
'rrf_scores' => [],
'raw_scores' => [],
];
}
$fused = $this->fuseHits(
$globalHits,
$scopedHits,
@@ -248,37 +215,19 @@ final class NdjsonHybridRetriever implements RetrieverInterface
$rrfScores = $fused['rrf_scores'];
$rawScores = $fused['raw_scores'];
// 🛡 Hardening: wenn Threshold alles rausfiltert, aber globale Hits existieren,
// nehmen wir Top-N als minimalen Kontext. Greift nur in Edge-Cases.
if ($rrfScores === [] && $globalHits !== []) {
$rrfScores = $this->fallbackRrfFromHits($globalHits, self::EMPTY_RRF_FALLBACK_TOPN);
}
if ($rrfScores === []) {
return [
'limit' => $limit,
'is_list_query' => $isListQuery,
'sales_intent' => $salesIntent,
'threshold' => $threshold,
'topk' => $topK,
'ranked_chunk_ids' => [],
'rows' => [],
'rrf_scores' => [],
'raw_scores' => $rawScores,
];
}
arsort($rrfScores);
$rankedChunkIds = array_keys($rrfScores);
$rankedChunkIds = array_keys($rrfScores);
$rows = $this->lookup->findByChunkIds($rankedChunkIds);
return [
'limit' => $limit,
'is_list_query' => $isListQuery,
'sales_intent' => $salesIntent,
'threshold' => $threshold,
'topk' => $topK,
'ranked_chunk_ids' => $rankedChunkIds,
'rows' => $rows,
'rrf_scores' => $rrfScores,
@@ -286,36 +235,33 @@ final class NdjsonHybridRetriever implements RetrieverInterface
];
}
/**
* @return array{0: float, 1: int} threshold, topK
*/
// =========================================================
// SUPPORT
// =========================================================
private function requireConfig(): ModelGenerationConfig
{
$config = $this->configRepository->findActiveForModel();
if ($config === null) {
throw new \RuntimeException('No active ModelGenerationConfig found.');
}
return $config;
}
private function detectSalesIntent(string $prompt): string
{
$data = $this->salesIntentLite->detect($prompt);
return (string)($data['intent'] ?? SalesIntentLite::DISCOVERY);
}
private function computeThresholdAndTopK(string $salesIntent, bool $isListQuery, int $vectorTopKBase): array
{
$threshold = self::VECTOR_SCORE_THRESHOLD;
$topK = $vectorTopKBase;
switch ($salesIntent) {
case SalesIntentLite::OBJECTION:
case SalesIntentLite::PRICING:
if ($salesIntent === SalesIntentLite::OBJECTION ||
$salesIntent === SalesIntentLite::PRICING) {
$threshold += 0.02;
break;
case SalesIntentLite::COMPARISON:
$topK = (int)round($vectorTopKBase * 1.4);
break;
case SalesIntentLite::IMPLEMENTATION:
$topK = (int)round($vectorTopKBase * 1.3);
break;
case SalesIntentLite::ROI:
$topK = (int)round($vectorTopKBase * 1.2);
break;
case SalesIntentLite::DISCOVERY:
default:
$threshold += 0;
break;
}
if ($isListQuery) {
@@ -323,39 +269,34 @@ final class NdjsonHybridRetriever implements RetrieverInterface
}
$topK = max(1, min($topK, self::HARD_MAX_VECTORK));
// Enterprise clamp: verhindert Drift, ohne den aktuellen Normalfall zu ändern.
$threshold = max(self::THRESHOLD_FLOOR, min(self::THRESHOLD_CEIL, $threshold));
return [$threshold, $topK];
}
/**
* @return array{
* rrf_scores: array<string,float>,
* raw_scores: array<string,float>
* }
*/
private function fuseHits(
array $globalHits,
array $scopedHits,
float $threshold,
bool $boostScoped,
bool $captureRaw
): array
{
): array {
$rrfScores = [];
$rawScores = [];
$apply = function (array $hits, bool $boost) use (&$rrfScores, &$rawScores, $threshold, $captureRaw): void {
$rank = 0;
foreach ($hits as $hit) {
if (!isset($hit['chunk_id'], $hit['score'])) {
continue;
}
$raw = (float)$hit['score'];
if ($raw < $threshold) {
continue;
}
@@ -386,12 +327,6 @@ final class NdjsonHybridRetriever implements RetrieverInterface
];
}
/**
* Minimaler Fallback: baut RRF nur aus der Reihenfolge (ohne Threshold),
* damit Edge-Cases nicht leer laufen.
*
* @return array<string,float>
*/
private function fallbackRrfFromHits(array $hits, int $topN): array
{
$rrf = [];
@@ -403,8 +338,7 @@ final class NdjsonHybridRetriever implements RetrieverInterface
}
$rank++;
$chunkId = (string)$hit['chunk_id'];
$rrf[$chunkId] = 1.0 / (self::RRF_K + $rank);
$rrf[(string)$hit['chunk_id']] = 1.0 / (self::RRF_K + $rank);
if ($rank >= $topN) {
break;
@@ -414,16 +348,6 @@ final class NdjsonHybridRetriever implements RetrieverInterface
return $rrf;
}
private function detectSalesIntent(string $prompt): string
{
$data = $this->salesIntentLite->detect($prompt);
return (string)($data['intent'] ?? SalesIntentLite::DISCOVERY);
}
// =========================================================
// SELECTION (shared)
// =========================================================
private function selectListChunkIds(array $chunkIds, array $rows, int $limit): array
{
$seen = [];
@@ -439,7 +363,6 @@ final class NdjsonHybridRetriever implements RetrieverInterface
continue;
}
// Dedupe Key (billig & stabil)
$key = md5(mb_strtolower((string)preg_replace('/\s+/u', ' ', $chunk)));
if (isset($seen[$key])) {
@@ -449,7 +372,7 @@ final class NdjsonHybridRetriever implements RetrieverInterface
$seen[$key] = true;
$out[] = (string)$id;
if (\count($out) >= $limit) {
if (count($out) >= $limit) {
break;
}
}
@@ -464,6 +387,7 @@ final class NdjsonHybridRetriever implements RetrieverInterface
$docChunkPositions = [];
foreach ($chunkIds as $chunkId) {
if (!isset($rows[$chunkId]['text'])) {
continue;
}
@@ -480,8 +404,7 @@ final class NdjsonHybridRetriever implements RetrieverInterface
}
if (is_int($chunkIndex)) {
$prev = $docChunkPositions[$docId] ?? [];
foreach ($prev as $prevIdx) {
foreach ($docChunkPositions[$docId] ?? [] as $prevIdx) {
if (abs($prevIdx - $chunkIndex) < self::MIN_CHUNK_DISTANCE) {
continue 2;
}
@@ -497,7 +420,7 @@ final class NdjsonHybridRetriever implements RetrieverInterface
$out[] = (string)$chunkId;
$docCounter[$docId] = ($docCounter[$docId] ?? 0) + 1;
if (\count($out) >= $limit) {
if (count($out) >= $limit) {
break;
}
}
@@ -505,10 +428,6 @@ final class NdjsonHybridRetriever implements RetrieverInterface
return $out;
}
// =========================================================
// COLLECT (shared)
// =========================================================
private function collectTextsFromIds(array $chunkIds, array $rows): array
{
$out = [];
@@ -519,12 +438,10 @@ final class NdjsonHybridRetriever implements RetrieverInterface
}
$text = trim((string)$rows[$id]['text']);
if ($text === '') {
continue;
}
if ($text !== '') {
$out[] = $text;
}
}
return $out;
}

88
templates/admin/model_config/test_retrieval.html.twig
View File

@@ -21,19 +21,16 @@
<div class="card bg-black border-secondary text-light mb-4">
<div class="card-body small">
<div class="row">
<div class="col-md-6">
<strong>Max Chunks:</strong>
{{ config.retrievalMaxChunks }}
</div>
<div class="col-md-6">
<strong>Vector Top K:</strong>
{{ config.retrievalVectorTopK }}
</div>
</div>
</div>
</div>
@@ -77,38 +74,95 @@
</h5>
<div style="max-height: 500px; overflow-y: auto;">
{% for chunk in results %}
<div class="border border-secondary p-3 mb-3 small">
{# ================= META-ZEILE ================= #}
<div class="mb-2 text-warning" style="font-size: 11px; line-height: 1.4;">
<span class="text-info"><strong>rank:</strong> {{ chunk.rank }}</span> |
<span class="text-info"><strong>chunk_id:</strong> {{ chunk.chunk_id }}</span> |
<span class="text-info"><strong>document_id:</strong> <a
class="text-info"
<div class="mb-2 text-warning"
style="font-size: 11px; line-height: 1.5; word-break: break-all;">
<span class="text-info">
<strong>rank:</strong> {{ chunk.rank }}
</span> |
<span class="text-info">
<strong>chunk_id:</strong> {{ chunk.chunk_id }}
</span> |
<span class="text-info">
<strong>document_id:</strong>
{% if chunk.document_id %}
<a class="text-info"
href="{{ path('admin_document_show', { id: chunk.document_id }) }}">
{{ chunk.document_id }}
</a></span> |
<span><strong>rrf_score:</strong> {{ chunk.rrf_score|number_format(6, '.', '') }}</span>
|
<span><strong>raw_score:</strong> {{ chunk.raw_score|number_format(6, '.', '') }}</span>
|
</a>
{% else %}
—
{% endif %}
</span> |
<span><strong>threshold:</strong> {{ chunk.threshold }}</span> |
<span><strong>intent:</strong> {{ chunk.intent }}</span> |
<span>
<strong>is_list_query:</strong>
<strong>route:</strong>
<span class="
{% if chunk.route == 'catalog_list' %}
text-primary
{% elseif chunk.route starts with 'catalog' %}
text-info
{% elseif chunk.route starts with 'sales' %}
text-success
{% else %}
text-secondary
{% endif %}
">
{{ chunk.route ?? '—' }}
</span>
</span> |
<span>
<strong>entity:</strong>
{{ chunk.entity_label ?? '—' }}
</span> |
<span>
<strong>intent:</strong>
{{ chunk.intent ?? '—' }}
</span> |
<span>
<strong>rrf:</strong>
{{ chunk.rrf_score is not null
? chunk.rrf_score|number_format(6, '.', '')
: '—' }}
</span> |
<span>
<strong>raw:</strong>
{{ chunk.raw_score is not null
? chunk.raw_score|number_format(6, '.', '')
: '—' }}
</span> |
<span>
<strong>threshold:</strong>
{{ chunk.threshold ?? '—' }}
</span> |
<span>
<strong>list:</strong>
{{ chunk.is_list_query ? 'true' : 'false' }}
</span>
</div>
{# ================= CHUNK TEXT ================= #}
<div>
<div class="text-light">
{{ chunk.text }}
</div>
</div>
{% endfor %}
</div>
</div>