marek/MtoRagSystem
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

new version ndjson

Browse Source
This commit is contained in:
team 1
2026-02-12 11:22:56 +01:00
parent 0bb0c0b42f
commit 5a52e07edc
10 changed files with 375 additions and 492 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@@ -5,6 +5,8 @@
/src/Vector/vector.index
/src/Vector/vector_meta.json
/var/locks
/var/agent-history
/var/cache
.env.local
test.*
# ---> Symfony

204
config/services.yaml
View File

@@ -1,130 +1,118 @@
# This file is the entry point to configure your own services.
# Files in the packages/ subdirectory configure your dependencies.
#
# https://symfony.com/doc/current/best_practices.html
# ------------------------------------------------------------
# Parameters
# ------------------------------------------------------------
parameters:
mto.index.chunk_size: 800
mto.index.chunk_overlap: 100
mto.index.embedding_model: 'nomic-embed-text'
mto.index.embedding_dimension: 768
mto.index.scoring_version: 1
mto.vector.python_bin: '/var/www/html/src/Vector/.venv/bin/python'
mto.vector.ingest_script: '/src/Vector/vector_ingest.py'
mto.vector.timeout: 600
mto.index.chunk_size: 800
mto.index.chunk_overlap: 100
mto.index.embedding_model: 'nomic-embed-text'
mto.index.embedding_dimension: 768
mto.index.scoring_version: 1
mto.vector.python_bin: '/var/www/html/src/Vector/.venv/bin/python'
mto.vector.ingest_script: '/src/Vector/vector_ingest.py'
mto.vector.timeout: 600
# ------------------------------------------------------------
# Services
# ------------------------------------------------------------
services:
# ------------------------------------------------------------
# Default service configuration
# ------------------------------------------------------------
_defaults:
autowire: true
autoconfigure: true
# ------------------------------------------------------------
# Default service configuration
# ------------------------------------------------------------
_defaults:
autowire: true
autoconfigure: true
# Bind the agent-specific Monolog channel explicitly
bind:
Psr\Log\LoggerInterface $agentLogger: '@monolog.logger.agent'
string $projectDir: '%kernel.project_dir%'
bind:
Psr\Log\LoggerInterface $agentLogger: '@monolog.logger.agent'
string $projectDir: '%kernel.project_dir%'
# ------------------------------------------------------------
# Automatically register all services in src/
# ------------------------------------------------------------
App\:
resource: '../src/'
exclude:
- '../src/DependencyInjection/'
- '../src/Entity/'
- '../src/Kernel.php'
# ------------------------------------------------------------
# Automatically register all services in src/
# ------------------------------------------------------------
App\:
resource: '../src/'
exclude:
- '../src/DependencyInjection/'
- '../src/Entity/'
- '../src/Kernel.php'
# ------------------------------------------------------------
# AI Agent – Infrastructure
# ------------------------------------------------------------
App\Infrastructure\OllamaClient:
arguments:
$apiUrl: '%env(AI_LLM_API_URL)%'
$model: '%env(AI_LLM_MODEL)%'
$timeoutSeconds: '%env(int:AI_LLM_TIMEOUT)%'
App\Controller\:
resource: '../src/Controller/'
tags: ['controller.service_arguments']
# ------------------------------------------------------------
# AI Agent – Context & state
# ------------------------------------------------------------
App\Context\ContextService:
arguments:
$historyDir: '%env(AI_HISTORY_DIR)%'
$projectDir: '%kernel.project_dir%'
# ------------------------------------------------------------
# AI Agent – Infrastructure
# ------------------------------------------------------------
App\Infrastructure\OllamaClient:
arguments:
$apiUrl: '%env(AI_LLM_API_URL)%'
$model: '%env(AI_LLM_MODEL)%'
$timeoutSeconds: '%env(int:AI_LLM_TIMEOUT)%'
# ------------------------------------------------------------
# AI Agent – Debug & logging flags
# ------------------------------------------------------------
App\Agent\AgentRunner:
arguments:
$debug: '%env(bool:AI_DEBUG)%'
$logPrompt: '%env(bool:AI_LOG_PROMPT)%'
$logContext: '%env(bool:AI_LOG_CONTEXT)%'
# ------------------------------------------------------------
# AI Agent – Context & Runner
# ------------------------------------------------------------
App\Context\ContextService:
arguments:
$historyDir: '%env(AI_HISTORY_DIR)%'
$projectDir: '%kernel.project_dir%'
App\Controller\:
resource: '../src/Controller/'
tags: [ 'controller.service_arguments' ]
App\Agent\AgentRunner:
arguments:
$debug: '%env(bool:AI_DEBUG)%'
$logPrompt: '%env(bool:AI_LOG_PROMPT)%'
$logContext: '%env(bool:AI_LOG_CONTEXT)%'
# ------------------------------------------------------------
# AI Agent – Knowledge
# ------------------------------------------------------------
App\Knowledge\Retrieval\ChunkKeywordRetriever:
arguments:
$chunksDir: '%kernel.project_dir%/var/knowledge/chunks'
# ------------------------------------------------------------
# NDJSON Retrieval Stack (FINAL ARCHITECTURE)
# ------------------------------------------------------------
App\Knowledge\Retrieval\CachedRetriever:
arguments:
$inner: '@App\Knowledge\Retrieval\ChunkKeywordRetriever'
$cache: '@cache.app'
$ttlSeconds: 600
App\Knowledge\Retrieval\NdjsonChunkLookup: ~
App\Knowledge\Retrieval\RetrieverInterface:
alias: App\Knowledge\Retrieval\CachedRetriever
App\Knowledge\Retrieval\NdjsonKeywordSearch: ~
App\Knowledge\Ingest\ChunkWriter:
arguments:
$chunksDir: '%kernel.project_dir%/var/knowledge/chunks'
$manifestPath: '%kernel.project_dir%/var/knowledge/manifest.json'
App\Knowledge\Retrieval\NdjsonHybridRetriever:
arguments:
$maxChunks: 3
$vectorTopK: 5
App\Knowledge\Ingest\ChunkIndexWriter:
arguments:
$indexPath: '%kernel.project_dir%/var/knowledge/index.json'
App\Knowledge\Retrieval\CachedRetriever:
arguments:
$inner: '@App\Knowledge\Retrieval\NdjsonHybridRetriever'
$cache: '@cache.app'
$ttlSeconds: 600
App\Knowledge\Retrieval\ChunkIndexLoader:
arguments:
$indexPath: '%kernel.project_dir%/var/knowledge/index.json'
App\Knowledge\Retrieval\RetrieverInterface:
alias: App\Knowledge\Retrieval\CachedRetriever
App\Command\KnowledgeIngestCommand:
arguments:
$uploadsDir: '%kernel.project_dir%/var/knowledge/uploads'
# ------------------------------------------------------------
# Vector Search (FAISS NDJSON-based)
# ------------------------------------------------------------
App\Vector\VectorSearchClient:
arguments:
$vectorDir: '%kernel.project_dir%/src/Vector'
App\Vector\VectorSearchClient:
arguments:
$vectorDir: '%kernel.project_dir%/var/knowledge'
App\Command\VectorIngestCommand:
arguments:
$vectorDir: '%kernel.project_dir%/src/Vector'
$projectDir: '%kernel.project_dir%'
App\Vector\VectorIndexBuilder:
arguments:
$pythonBin: '%mto.vector.python_bin%'
$relativeScriptPath: '%mto.vector.ingest_script%'
$timeoutSeconds: '%mto.vector.timeout%'
App\Command\VectorInstallCommand:
arguments:
$vectorDir: '%kernel.project_dir%/src/Vector'
# ------------------------------------------------------------
# Index Configuration (Guardrails)
# ------------------------------------------------------------
App\Index\IndexConfiguration:
arguments:
$chunkSize: '%mto.index.chunk_size%'
$chunkOverlap: '%mto.index.chunk_overlap%'
$embeddingModel: '%mto.index.embedding_model%'
$embeddingDimension: '%mto.index.embedding_dimension%'
$scoringVersion: '%mto.index.scoring_version%'
$indexFormat: 'ndjson'
$vectorBackend: 'faiss'
App\Vector\VectorIndexBuilder:
arguments:
$pythonBin: '%mto.vector.python_bin%'
$relativeScriptPath: '%mto.vector.ingest_script%'
$timeoutSeconds: '%mto.vector.timeout%'
App\Index\IndexConfiguration:
arguments:
$chunkSize: '%mto.index.chunk_size%'
$chunkOverlap: '%mto.index.chunk_overlap%'
$embeddingModel: '%mto.index.embedding_model%'
$embeddingDimension: '%mto.index.embedding_dimension%'
$scoringVersion: '%mto.index.scoring_version%'
$indexFormat: 'ndjson'
$vectorBackend: 'faiss'

6
public/assets/js/base.js
View File

@@ -113,12 +113,16 @@ document.addEventListener('DOMContentLoaded', () => {
const text = dataLines.join('\n');
if (text === '[DONE]') {
// Finales Rendering mit Normalisierung
// 🔥 Final render flush
if (renderTimer) {
clearTimeout(renderTimer);
renderTimer = null;
}
bubble.innerHTML = renderMarkdown(raw);
chatEl.scrollTop = chatEl.scrollHeight;
abort = true;
return;
}

101
src/Command/KnowledgeIngestCommand.php
View File

@@ -1,28 +1,25 @@
<?php
// src/Command/KnowledgeIngestCommand.php
declare(strict_types=1);
namespace App\Command;
use App\Knowledge\Ingest\KnowledgeIngestService;
use App\Entity\DocumentVersion;
use App\Entity\User;
use App\Ingest\IngestFlow;
use Doctrine\ORM\EntityManagerInterface;
use Symfony\Component\Console\Attribute\AsCommand;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Finder\Finder;
#[AsCommand(
name: 'mto:agent:knowledge:ingest',
description: 'Ingest one or multiple markdown/text documents into file-based knowledge chunks'
)]
final class KnowledgeIngestCommand extends Command
#[AsCommand(name: 'mto:agent:ingest:version')]
class KnowledgeIngestCommand extends Command
{
public function __construct(
private readonly KnowledgeIngestService $ingest,
private readonly string $uploadsDir,
private readonly IngestFlow $ingestFlow,
private readonly EntityManagerInterface $em,
) {
parent::__construct();
}
@@ -30,86 +27,28 @@ final class KnowledgeIngestCommand extends Command
protected function configure(): void
{
$this
->addArgument(
'file',
InputArgument::OPTIONAL,
'Path to a single .txt/.md file'
)
->addOption(
'all',
null,
InputOption::VALUE_NONE,
'Ingest all .md files from the uploads directory'
)
->addOption(
'optimize',
'o',
InputOption::VALUE_NONE,
'Optimize chunks for retrieval quality'
);
->addArgument('versionId', InputArgument::REQUIRED, 'UUID of DocumentVersion')
->addArgument('userId', InputArgument::REQUIRED, 'UUID of user triggering ingest');
}
protected function execute(InputInterface $input, OutputInterface $output): int
{
$files = [];
$optimize = (bool) $input->getOption('optimize');
$versionId = $input->getArgument('versionId');
$userId = $input->getArgument('userId');
if ($input->getOption('all')) {
if (!is_dir($this->uploadsDir)) {
$output->writeln('<error>❌ uploads directory not found</error>');
return Command::FAILURE;
}
$version = $this->em->getRepository(DocumentVersion::class)->find($versionId);
$user = $this->em->getRepository(User::class)->find($userId);
$finder = new Finder();
$finder
->files()
->in($this->uploadsDir)
->name('*.md');
if (!$finder->hasResults()) {
$output->writeln('<comment>ℹ️ No .md files found in uploads/</comment>');
return Command::SUCCESS;
}
foreach ($finder as $file) {
$files[] = $file->getRealPath();
}
$output->writeln(sprintf(
'📂 Ingesting %d markdown files from uploads (%s)',
count($files),
$optimize ? 'optimized' : 'standard'
));
} else {
$file = $input->getArgument('file');
if (!$file) {
$output->writeln('<error>❌ Either provide a file or use --all</error>');
return Command::FAILURE;
}
$files[] = (string) $file;
if (!$version || !$user) {
$output->writeln('<error>Version or User not found.</error>');
return Command::FAILURE;
}
$totalWritten = 0;
$output->writeln('Starting ingest...');
foreach ($files as $filePath) {
$output->writeln('➡️ Ingesting: ' . $filePath);
$this->ingestFlow->ingestDocumentVersion($version, $user);
$written = $this->ingest->ingestFile(
$filePath,
optimize: $optimize
);
$totalWritten += count($written);
foreach ($written as $chunk) {
$output->writeln(' - ' . $chunk);
}
}
$output->writeln('');
$output->writeln('✅ Total written chunks: ' . $totalWritten);
$output->writeln('<info>Ingest completed.</info>');
return Command::SUCCESS;
}

75
src/Command/VectorIngestCommand.php
View File

@@ -4,86 +4,27 @@ declare(strict_types=1);
namespace App\Command;
use App\Vector\VectorIndexBuilder;
use Symfony\Component\Console\Attribute\AsCommand;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
#[AsCommand(
name: 'mto:agent:vector:ingest',
description: 'Builds the FAISS vector index from index.json'
)]
final class VectorIngestCommand extends Command
#[AsCommand(name: 'mto:agent:vector:rebuild')]
class VectorIngestCommand extends Command
{
public function __construct(
private readonly string $vectorDir,
private readonly string $projectDir
private readonly VectorIndexBuilder $builder
) {
parent::__construct();
}
protected function execute(InputInterface $input, OutputInterface $output): int
{
$vectorDir = rtrim($this->vectorDir, '/');
$output->writeln('Rebuilding vector index...');
$this->builder->rebuildFromNdjson();
$output->writeln('Done.');
if (!is_dir($vectorDir)) {
$output->writeln('<error>Vector directory not found</error>');
return Command::FAILURE;
}
$script = $vectorDir . '/vector_ingest.py';
if (!is_file($script)) {
$output->writeln('<error>vector_ingest.py not found</error>');
return Command::FAILURE;
}
// -------------------------------------------------
// Enforce venv usage
// -------------------------------------------------
$venvPython = $vectorDir . '/.venv/bin/python';
if (!is_file($venvPython)) {
$output->writeln('<error>No Python virtual environment found.</error>');
$output->writeln('<comment>Run first:</comment>');
$output->writeln('<info> php bin/console mto:agent:vector:install</info>');
return Command::FAILURE;
}
$knowledgeDir = rtrim($this->projectDir, '/') . '/var/knowledge';
if (!is_dir($knowledgeDir)) {
$output->writeln('<error>Knowledge directory not found:</error>');
$output->writeln($knowledgeDir);
return Command::FAILURE;
}
$output->writeln('<info>Building FAISS vector index…</info>');
$output->writeln(sprintf(
'<comment>Vector dir:</comment> %s',
$vectorDir
));
$output->writeln(sprintf(
'<comment>Knowledge dir:</comment> %s',
$knowledgeDir
));
$cmd = sprintf(
'%s %s %s %s 2>&1',
escapeshellarg($venvPython),
escapeshellarg($script),
escapeshellarg($vectorDir),
escapeshellarg($knowledgeDir)
);
exec($cmd, $out, $exitCode);
foreach ($out as $line) {
$output->writeln($line);
}
return $exitCode === 0
? Command::SUCCESS
: Command::FAILURE;
return Command::SUCCESS;
}
}

114
src/Command/VectorInstallCommand.php
View File

@@ -1,114 +0,0 @@
<?php
declare(strict_types=1);
namespace App\Command;
use Symfony\Component\Console\Attribute\AsCommand;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
/**
* System requirements (once per environment):
* sudo apt update
* sudo apt install -y python3-venv python3-pip
*/
#[AsCommand(
name: 'mto:agent:vector:install',
description: 'Creates a Python venv and installs vector dependencies'
)]
final class VectorInstallCommand extends Command
{
public function __construct(
private readonly string $vectorDir
) {
parent::__construct();
}
protected function execute(InputInterface $input, OutputInterface $output): int
{
if (!is_dir($this->vectorDir)) {
$output->writeln('<error>Vector directory not found</error>');
return Command::FAILURE;
}
$vectorDir = rtrim($this->vectorDir, '/');
$venvDir = $vectorDir . '/.venv';
$venvPython = $venvDir . '/bin/python';
// -------------------------------------------------
// 1) Create venv if missing
// -------------------------------------------------
if (!is_dir($venvDir)) {
$output->writeln('<info>Creating Python virtual environment…</info>');
$cmd = sprintf(
'python3 -m venv %s 2>&1',
escapeshellarg($venvDir)
);
exec($cmd, $out, $exitCode);
foreach ($out as $line) {
$output->writeln($line);
}
if ($exitCode !== 0 || !is_file($venvPython)) {
$output->writeln('');
$output->writeln('<error>Failed to create Python virtual environment.</error>');
$output->writeln('<comment>Ensure that python3-venv is installed on the system.</comment>');
return Command::FAILURE;
}
} else {
$output->writeln('<info>Using existing Python virtual environment</info>');
}
// -------------------------------------------------
// 2) Ensure pip exists inside venv
// -------------------------------------------------
$cmd = sprintf(
'%s -m pip --version 2>&1',
escapeshellarg($venvPython)
);
exec($cmd, $out, $exitCode);
if ($exitCode !== 0) {
$output->writeln('');
$output->writeln('<error>The existing virtual environment has no pip.</error>');
$output->writeln('<comment>This usually means it was created before python3-pip was installed.</comment>');
$output->writeln('<comment>Fix:</comment>');
$output->writeln(sprintf('<info> rm -rf %s</info>', $venvDir));
$output->writeln('<info> php bin/console mto:agent:vector:install</info>');
return Command::FAILURE;
}
// -------------------------------------------------
// 3) Install / update dependencies
// -------------------------------------------------
$output->writeln('<info>Installing vector dependencies…</info>');
$cmd = sprintf(
'%s -m pip install --upgrade faiss-cpu sentence-transformers 2>&1',
escapeshellarg($venvPython)
);
exec($cmd, $out, $exitCode);
foreach ($out as $line) {
$output->writeln($line);
}
if ($exitCode !== 0) {
$output->writeln('<error>Dependency installation failed</error>');
return Command::FAILURE;
}
$output->writeln('');
$output->writeln('<info>Vector dependencies installed successfully</info>');
$output->writeln(sprintf('<comment>venv:</comment> %s', $venvDir));
return Command::SUCCESS;
}
}

44
src/Knowledge/Retrieval/NdjsonChunkLookup.php Normal file
View File

@@ -0,0 +1,44 @@
<?php
declare(strict_types=1);
namespace App\Knowledge\Retrieval;
use App\Knowledge\ChunkManager;
use Symfony\Component\Uid\Uuid;
final class NdjsonChunkLookup
{
public function __construct(
private readonly ChunkManager $chunkManager
)
{
}
/**
* @param string[] $chunkIds RFC4122 UUID strings
* @return array<string,array<string,mixed>> keyed by chunk_id
*/
public function findByChunkIds(array $chunkIds): array
{
$wanted = array_fill_keys($chunkIds, true);
$found = [];
foreach ($this->chunkManager->streamAll() as $row) {
$id = $row['chunk_id'] ?? null;
if (!is_string($id) || !isset($wanted[$id])) {
continue;
}
$found[$id] = $row;
// Early exit sobald alle gefunden
if (\count($found) === \count($wanted)) {
break;
}
}
return $found;
}
}

99
src/Knowledge/Retrieval/NdjsonHybridRetriever.php Normal file
View File

@@ -0,0 +1,99 @@
<?php
declare(strict_types=1);
namespace App\Knowledge\Retrieval;
use App\Vector\VectorSearchClient;
final class NdjsonHybridRetriever implements RetrieverInterface
{
private const VECTOR_SCORE_THRESHOLD = 0.65;
public function __construct(
private readonly NdjsonKeywordSearch $keywordSearch,
private readonly NdjsonChunkLookup $lookup,
private readonly VectorSearchClient $vectorClient,
private readonly int $maxChunks = 3,
private readonly int $vectorTopK = 5,
)
{
}
public function retrieve(string $prompt, int $limit = null): array
{
$limit ??= $this->maxChunks;
$terms = $this->extractTerms($prompt);
// 1) Keyword first
$keywordChunks = $this->keywordSearch->search($terms, $limit);
if (\count($keywordChunks) >= $limit) {
return array_slice($keywordChunks, 0, $limit);
}
// 2) Vector fallback / enrichment
$hits = $this->vectorClient->search($prompt, $this->vectorTopK);
if ($hits === []) {
return $keywordChunks;
}
$chunkIds = [];
foreach ($hits as $hit) {
if (!isset($hit['chunk_id'], $hit['score'])) {
continue;
}
if ((float)$hit['score'] < self::VECTOR_SCORE_THRESHOLD) {
continue;
}
$chunkIds[] = (string)$hit['chunk_id'];
}
if ($chunkIds === []) {
return $keywordChunks;
}
$rows = $this->lookup->findByChunkIds($chunkIds);
foreach ($chunkIds as $id) {
if (!isset($rows[$id]['text']) || !is_string($rows[$id]['text'])) {
continue;
}
$keywordChunks[] = trim($rows[$id]['text']);
}
// dedupe + limit
$seen = [];
$out = [];
foreach ($keywordChunks as $chunk) {
$key = mb_strtolower(preg_replace('/\s+/u', ' ', $chunk));
if (isset($seen[$key])) {
continue;
}
$seen[$key] = true;
$out[] = $chunk;
if (\count($out) >= $limit) {
break;
}
}
return $out;
}
/**
* minimal term extraction (we keep your old behavior)
*
* @return string[]
*/
private function extractTerms(string $text): array
{
$text = mb_strtolower((string)preg_replace('/[^\p{L}\p{N}\s]/u', '', $text));
return array_values(array_filter(
explode(' ', $text),
static fn(string $w) => mb_strlen($w) > 2
));
}
}

101
src/Knowledge/Retrieval/NdjsonKeywordSearch.php Normal file
View File

@@ -0,0 +1,101 @@
<?php
declare(strict_types=1);
namespace App\Knowledge\Retrieval;
use App\Knowledge\ChunkManager;
use App\Knowledge\StopWords;
final class NdjsonKeywordSearch
{
public function __construct(
private readonly ChunkManager $chunkManager,
private readonly StopWords $stopWords,
) {
}
/**
* Streaming Keyword-Search über index.ndjson.
*
* @param string[] $terms (already lowercased)
* @return string[] best chunks
*/
public function search(array $terms, int $limit = 3, int $candidateCap = 200): array
{
$terms = array_values(array_filter($terms, function (string $t): bool {
return $t !== '' && !\in_array($t, $this->stopWords->getStopWords(), true);
}));
if ($terms === []) {
return [];
}
// bounded min-heap (score => chunkText)
$best = [];
foreach ($this->chunkManager->streamAll() as $row) {
$text = $row['text'] ?? null;
if (!is_string($text) || $text === '') {
continue;
}
$score = $this->scoreText($text, $terms);
if ($score <= 0) {
continue;
}
$best[] = ['score' => $score, 'text' => trim($text)];
// keep array bounded to avoid memory spikes
if (\count($best) > $candidateCap) {
usort($best, fn($a, $b) => $b['score'] <=> $a['score']);
$best = array_slice($best, 0, $candidateCap);
}
}
if ($best === []) {
return [];
}
usort($best, fn($a, $b) => $b['score'] <=> $a['score']);
$out = [];
$seen = [];
foreach ($best as $row) {
$key = mb_strtolower(preg_replace('/\s+/u', ' ', $row['text']));
if (isset($seen[$key])) {
continue;
}
$seen[$key] = true;
$out[] = $row['text'];
if (\count($out) >= $limit) {
break;
}
}
return $out;
}
/**
* Simple scoring: count matches, weight long terms slightly.
*/
private function scoreText(string $text, array $terms): int
{
$content = mb_strtolower($text);
$score = 0;
foreach ($terms as $term) {
if ($term === '') {
continue;
}
if (str_contains($content, $term)) {
$score += (mb_strlen($term) >= 10) ? 2 : 1;
}
}
return $score;
}
}

121
src/Knowledge/VectorSearchChunked.php
View File

@@ -1,121 +0,0 @@
<?php
declare(strict_types=1);
namespace App\Knowledge;
use Psr\Log\LoggerInterface;
/**
* VectorSearchChunked
*
* Chunk-based retrieval service for long-form knowledge documents.
* This is a lightweight, deterministic runtime reader for
* precomputed knowledge chunks.
*
* Design principles:
* - No runtime indexing
* - No ML dependencies
* - Deterministic and fast
* - Hard limits to protect prompt size
*
* This service is intentionally simple and can later be replaced
* by a real vector database without changing the AgentRunner.
*/
final class VectorSearchChunked
{
/**
* Directory containing chunked knowledge files.
*/
private string $dataDir = 'var/knowledge/chunks';
/**
* Maximum number of chunks to return.
*/
private int $maxChunks = 3;
public function __construct(
private string $projectDir,
)
{
$this->dataDir = $this->projectDir . '/' . $this->dataDir;
}
/**
* Returns concatenated relevant chunks as plain text.
*
* @param string $prompt
* @return string
*/
public function searchAsText(string $prompt): string
{
if (!is_dir($this->dataDir)) {
return '';
}
$promptLower = mb_strtolower($prompt);
$keywords = $this->extractKeywords($promptLower);
if ($keywords === []) {
return '';
}
$matches = [];
foreach (glob($this->dataDir . '/*.txt') as $file) {
$content = file_get_contents($file);
if ($content === false) {
continue;
}
$contentLower = mb_strtolower($content);
if ($this->matchesKeywords($contentLower, $keywords)) {
$matches[] = trim($content);
}
if (count($matches) >= $this->maxChunks) {
break;
}
}
return implode("\n\n", $matches);
}
/**
* Extracts simple keywords from the prompt.
*
* This is a lightweight heuristic replacement for
* full vector or embedding-based search.
*/
private function extractKeywords(string $prompt): array
{
$words = preg_split('/\W+/u', $prompt, -1, PREG_SPLIT_NO_EMPTY);
if ($words === false) {
return [];
}
$keywords = [];
foreach ($words as $word) {
if (mb_strlen($word) >= 4) {
$keywords[] = $word;
}
}
return array_values(array_unique($keywords));
}
/**
* Checks whether the content matches at least one keyword.
*/
private function matchesKeywords(string $content, array $keywords): bool
{
foreach ($keywords as $keyword) {
if (str_contains($content, $keyword)) {
return true;
}
}
return false;
}
}