remove direct chnuk search. only vector search

src/Knowledge/Retrieval/CachedRetriever.php

@@ -5,16 +5,22 @@ declare(strict_types=1);
 namespace App\Knowledge\Retrieval;
 
 use Psr\Cache\CacheItemPoolInterface;
+use Psr\Cache\InvalidArgumentException;
 
-final class CachedRetriever implements RetrieverInterface
+final readonly class CachedRetriever implements RetrieverInterface
 {
     public function __construct(
-        private RetrieverInterface    $inner,
+        private RetrieverInterface     $inner,
         private CacheItemPoolInterface $cache,
-        private int                   $ttlSeconds = 600 // 10 Minuten
-    ) {}
+        private int                    $ttlSeconds
+    )
+    {
+    }
 
-    public function retrieve(string $prompt, int $limit = 3): array
+    /**
+     * @throws InvalidArgumentException
+     */
+    public function retrieve(string $prompt, int $limit = 10): array
     {
         $key = $this->buildCacheKey($prompt, $limit);
 

src/Knowledge/Retrieval/NdjsonHybridRetriever.php

@@ -1,6 +1,5 @@
 <?php
 
-
 declare(strict_types=1);
 
 namespace App\Knowledge\Retrieval;
@@ -9,34 +8,26 @@ use App\Vector\VectorSearchClient;
 
 final class NdjsonHybridRetriever implements RetrieverInterface
 {
-    private const VECTOR_SCORE_THRESHOLD = 0.65;
+    private const VECTOR_SCORE_THRESHOLD = 0.25;
 
     public function __construct(
-        private readonly NdjsonKeywordSearch $keywordSearch,
         private readonly NdjsonChunkLookup   $lookup,
         private readonly VectorSearchClient  $vectorClient,
-        private readonly int                 $maxChunks = 3,
-        private readonly int                 $vectorTopK = 5,
+        private readonly int                 $maxChunks = 10,
+        private readonly int                 $vectorTopK = 10,
     )
     {
     }
 
     public function retrieve(string $prompt, int $limit = null): array
     {
-        $limit ??= $this->maxChunks;
+        $limit = $this->maxChunks;
+        $keywordChunks = [];
 
-        $terms = $this->extractTerms($prompt);
-
-        // 1) Keyword first
-        $keywordChunks = $this->keywordSearch->search($terms, $limit);
-        if (\count($keywordChunks) >= $limit) {
-            return array_slice($keywordChunks, 0, $limit);
-        }
-
-        // 2) Vector fallback / enrichment
+        // Vector / enrichment
         $hits = $this->vectorClient->search($prompt, $this->vectorTopK);
         if ($hits === []) {
-            return $keywordChunks;
+            return $this->diversifyByDevice($keywordChunks, $limit, 1);
         }
 
         $chunkIds = [];
@@ -51,7 +42,7 @@ final class NdjsonHybridRetriever implements RetrieverInterface
         }
 
         if ($chunkIds === []) {
-            return $keywordChunks;
+            return $this->diversifyByDevice($keywordChunks, $limit, 1);
         }
 
         $rows = $this->lookup->findByChunkIds($chunkIds);
@@ -63,9 +54,9 @@ final class NdjsonHybridRetriever implements RetrieverInterface
             $keywordChunks[] = trim($rows[$id]['text']);
         }
 
-        // dedupe + limit
+        // dedupe
         $seen = [];
-        $out = [];
+        $deduped = [];
 
         foreach ($keywordChunks as $chunk) {
             $key = mb_strtolower(preg_replace('/\s+/u', ' ', $chunk));
@@ -73,20 +64,13 @@ final class NdjsonHybridRetriever implements RetrieverInterface
                 continue;
             }
             $seen[$key] = true;
-            $out[] = $chunk;
-            if (\count($out) >= $limit) {
-                break;
-            }
+            $deduped[] = $chunk;
         }
 
-        return $out;
+        // diversify
+        return $this->diversifyByDevice($deduped, $limit, 1);
     }
 
-    /**
-     * minimal term extraction (we keep your old behavior)
-     *
-     * @return string[]
-     */
     private function extractTerms(string $text): array
     {
         $text = mb_strtolower((string)preg_replace('/[^\p{L}\p{N}\s]/u', '', $text));
@@ -96,4 +80,41 @@ final class NdjsonHybridRetriever implements RetrieverInterface
             static fn(string $w) => mb_strlen($w) > 2
         ));
     }
+
+    private function extractDevice(string $chunk): string
+    {
+        $firstLine = explode("\n", $chunk, 2)[0] ?? '';
+        return trim($firstLine);
+    }
+
+    private function diversifyByDevice(array $chunks, int $limit, int $maxPerDevice = 1): array
+    {
+        $seenDevices = [];
+        $out = [];
+
+        foreach ($chunks as $chunk) {
+            $device = $this->extractDevice($chunk);
+
+            if ($device === '') {
+                continue;
+            }
+
+            if (!isset($seenDevices[$device])) {
+                $seenDevices[$device] = 0;
+            }
+
+            if ($seenDevices[$device] >= $maxPerDevice) {
+                continue;
+            }
+
+            $out[] = $chunk;
+            $seenDevices[$device]++;
+
+            if (\count($out) >= $limit) {
+                break;
+            }
+        }
+
+        return $out;
+    }
 }

src/Knowledge/Retrieval/NdjsonKeywordSearch.php

@@ -1,101 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace App\Knowledge\Retrieval;
-
-use App\Knowledge\ChunkManager;
-use App\Knowledge\StopWords;
-
-final class NdjsonKeywordSearch
-{
-    public function __construct(
-        private readonly ChunkManager $chunkManager,
-        private readonly StopWords $stopWords,
-    ) {
-    }
-
-    /**
-     * Streaming Keyword-Search über index.ndjson.
-     *
-     * @param string[] $terms (already lowercased)
-     * @return string[] best chunks
-     */
-    public function search(array $terms, int $limit = 3, int $candidateCap = 200): array
-    {
-        $terms = array_values(array_filter($terms, function (string $t): bool {
-            return $t !== '' && !\in_array($t, $this->stopWords->getStopWords(), true);
-        }));
-
-        if ($terms === []) {
-            return [];
-        }
-
-        // bounded min-heap (score => chunkText)
-        $best = [];
-
-        foreach ($this->chunkManager->streamAll() as $row) {
-            $text = $row['text'] ?? null;
-            if (!is_string($text) || $text === '') {
-                continue;
-            }
-
-            $score = $this->scoreText($text, $terms);
-            if ($score <= 0) {
-                continue;
-            }
-
-            $best[] = ['score' => $score, 'text' => trim($text)];
-
-            // keep array bounded to avoid memory spikes
-            if (\count($best) > $candidateCap) {
-                usort($best, fn($a, $b) => $b['score'] <=> $a['score']);
-                $best = array_slice($best, 0, $candidateCap);
-            }
-        }
-
-        if ($best === []) {
-            return [];
-        }
-
-        usort($best, fn($a, $b) => $b['score'] <=> $a['score']);
-
-        $out = [];
-        $seen = [];
-
-        foreach ($best as $row) {
-            $key = mb_strtolower(preg_replace('/\s+/u', ' ', $row['text']));
-            if (isset($seen[$key])) {
-                continue;
-            }
-            $seen[$key] = true;
-            $out[] = $row['text'];
-
-            if (\count($out) >= $limit) {
-                break;
-            }
-        }
-
-        return $out;
-    }
-
-    /**
-     * Simple scoring: count matches, weight long terms slightly.
-     */
-    private function scoreText(string $text, array $terms): int
-    {
-        $content = mb_strtolower($text);
-        $score = 0;
-
-        foreach ($terms as $term) {
-            if ($term === '') {
-                continue;
-            }
-            if (str_contains($content, $term)) {
-                $score += (mb_strlen($term) >= 10) ? 2 : 1;
-            }
-        }
-
-        return $score;
-    }
-}

src/Knowledge/Retrieval/RetrieverInterface.php

@@ -7,5 +7,5 @@ interface RetrieverInterface
     /**
      * @return string[]  Plain text knowledge chunks
      */
-    public function retrieve(string $prompt, int $limit = 3): array;
+    public function retrieve(string $prompt, int $limit = 10): array;
 }