marek/MtoRagSystem
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

harden struct

Browse Source
This commit is contained in:
team 1
2026-02-16 12:34:22 +01:00
parent c046a83767
commit 57a35af6c8
6 changed files with 308 additions and 252 deletions
Download Patch File Download Diff File Expand all files Collapse all files

175
src/Vector/VectorIndexBuilder.php
View File

@@ -1,10 +1,10 @@
<?php
declare(strict_types=1);
namespace App\Vector;
use App\Index\IndexConfiguration;
use Symfony\Component\Process\Exception\ProcessFailedException;
use Symfony\Component\Process\Process;
@@ -13,49 +13,32 @@ final class VectorIndexBuilder
private string $pythonBin;
private string $scriptPath;
private string $indexNdjsonPath;
private string $indexMetaPath;
private string $vectorIndexPath;
private string $vectorMetaPath;
private int $timeoutSeconds;
private IndexConfiguration $indexConfiguration;
public function __construct(
string $projectDir,
string $pythonBin = 'python3',
string $relativeScriptPath = '/vector/vector_ingest.py',
string $relativeIndexNdjsonPath = '/var/knowledge/index.ndjson',
string $relativeVectorIndexPath = '/var/knowledge/vector.index',
int $timeoutSeconds = 600
)
{
$base = rtrim($projectDir, '/');
$this->pythonBin = $pythonBin;
$this->scriptPath = $base . $relativeScriptPath;
$this->indexNdjsonPath = $base . $relativeIndexNdjsonPath;
$this->vectorIndexPath = $base . $relativeVectorIndexPath;
$this->timeoutSeconds = $timeoutSeconds;
string $pythonBin,
string $scriptPath,
string $indexNdjsonPath,
string $indexMetaPath,
string $vectorIndexPath,
int $timeoutSeconds,
IndexConfiguration $indexConfiguration
) {
$this->pythonBin = $pythonBin;
$this->scriptPath = $scriptPath;
$this->indexNdjsonPath = $indexNdjsonPath;
$this->indexMetaPath = $indexMetaPath;
$this->vectorIndexPath = $vectorIndexPath;
$this->vectorMetaPath = $vectorIndexPath . '.meta.json';
$this->timeoutSeconds = $timeoutSeconds;
$this->indexConfiguration = $indexConfiguration;
}
public function getIndexNdjsonPath(): string
{
return $this->indexNdjsonPath;
}
public function getVectorIndexPath(): string
{
return $this->vectorIndexPath;
}
public function getScriptPath(): string
{
return $this->scriptPath;
}
/**
* Rebuild FAISS Index deterministisch aus index.ndjson.
*
* Erwartung: Python schreibt in $tmpVectorIndexPath, wir schalten atomar um.
*
* @param string|null $logPath Optional: stdout/stderr dorthin appenden
*/
public function rebuildFromNdjson(?string $logPath = null): void
{
if (!is_file($this->scriptPath)) {
@@ -66,97 +49,97 @@ final class VectorIndexBuilder
throw new \RuntimeException('index.ndjson not found at: ' . $this->indexNdjsonPath);
}
$dir = \dirname($this->vectorIndexPath);
if (!is_dir($dir) && !mkdir($dir, 0777, true) && !is_dir($dir)) {
throw new \RuntimeException('Unable to create vector index directory: ' . $dir);
if (!is_file($this->indexMetaPath)) {
$this->initializeIndexMeta();
}
$indexMeta = json_decode((string) file_get_contents($this->indexMetaPath), true);
if (!is_array($indexMeta) || empty($indexMeta['embedding_model'])) {
throw new \RuntimeException('Invalid index_meta.json');
}
$embeddingModel = (string) $indexMeta['embedding_model'];
$tmpVectorIndexPath = $this->vectorIndexPath . '.tmp';
// Vorheriges tmp entfernen (Sicherheit)
if (is_file($tmpVectorIndexPath)) {
@unlink($tmpVectorIndexPath);
}
// Wichtig: Python erzeugt meta basierend auf endgültigem Namen
$finalMetaPath = $this->vectorMetaPath;
$tmpMetaPath = dirname($this->vectorIndexPath) . '/' . basename($this->vectorIndexPath, '.index') . '.index.meta.json';
@unlink($tmpVectorIndexPath);
@unlink($finalMetaPath);
// ----------------------------
// Python-Aufruf (konservativ)
// ----------------------------
// Wir erwarten/standardisieren (ab jetzt) CLI-Args:
// --index <path-to-index.ndjson>
// --out <path-to-vector.index.tmp>
$cmd = [
$this->pythonBin,
$this->scriptPath,
'--index', $this->indexNdjsonPath,
'--out', $tmpVectorIndexPath,
'--model', 'all-MiniLM-L6-v2',
'--out', $tmpVectorIndexPath,
'--model', $embeddingModel,
];
$process = new Process($cmd);
$process->setTimeout($this->timeoutSeconds);
$process->mustRun();
$this->runProcess($process, $logPath);
// Python muss tmp erzeugt haben
if (!is_file($tmpVectorIndexPath) || filesize($tmpVectorIndexPath) === 0) {
throw new \RuntimeException('Vector index rebuild failed: tmp output missing or empty: ' . $tmpVectorIndexPath);
throw new \RuntimeException('Vector index tmp missing or empty');
}
// Atomarer Switch
$this->atomicSwitch($tmpVectorIndexPath, $this->vectorIndexPath);
// Python erzeugt vector.index.meta.json (nicht tmp.meta!)
if (!is_file($this->vectorMetaPath) || filesize($this->vectorMetaPath) === 0) {
throw new \RuntimeException('Vector meta missing or empty');
}
// Atomarer Switch für Index
if (!rename($tmpVectorIndexPath, $this->vectorIndexPath)) {
throw new \RuntimeException('Atomic switch failed for vector index');
}
}
// -------------------------
// Internals
// -------------------------
private function initializeIndexMeta(): void
{
$dir = dirname($this->indexMetaPath);
if (!is_dir($dir) && !mkdir($dir, 0777, true) && !is_dir($dir)) {
throw new \RuntimeException('Cannot create knowledge directory');
}
$data = [
'index_version' => 1,
'created_at' => (new \DateTimeImmutable())->format(DATE_ATOM),
'embedding_model' => $this->indexConfiguration->getEmbeddingModel(),
'embedding_dimension' => $this->indexConfiguration->getEmbeddingDimension(),
'chunk_size' => $this->indexConfiguration->getChunkSize(),
'chunk_overlap' => $this->indexConfiguration->getChunkOverlap(),
'scoring_version' => $this->indexConfiguration->getScoringVersion(),
'index_format' => 'ndjson',
'vector_backend' => 'faiss',
];
file_put_contents(
$this->indexMetaPath,
json_encode($data, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES)
);
}
private function runProcess(Process $process, ?string $logPath): void
{
if ($logPath !== null) {
$this->appendLog($logPath, "\n=== VectorIndexBuilder START " . (new \DateTimeImmutable())->format(DATE_ATOM) . " ===\n");
$this->appendLog($logPath, "CMD: " . $process->getCommandLine() . "\n");
@file_put_contents($logPath, "=== VectorIndexBuilder START ===\n", FILE_APPEND);
}
$process->run(function (string $type, string $buffer) use ($logPath) {
if ($logPath === null) {
return;
}
// TYPE: Process::OUT / Process::ERR
$this->appendLog($logPath, $buffer);
});
$process->run();
if (!$process->isSuccessful()) {
if ($logPath !== null) {
$this->appendLog($logPath, "\n=== VectorIndexBuilder FAILED ===\n");
$this->appendLog($logPath, "ExitCode: " . $process->getExitCode() . "\n");
$this->appendLog($logPath, "STDERR:\n" . $process->getErrorOutput() . "\n");
@file_put_contents($logPath, $process->getErrorOutput(), FILE_APPEND);
}
throw new ProcessFailedException($process);
}
if ($logPath !== null) {
$this->appendLog($logPath, "\n=== VectorIndexBuilder OK " . (new \DateTimeImmutable())->format(DATE_ATOM) . " ===\n");
}
}
private function appendLog(string $logPath, string $content): void
{
$dir = \dirname($logPath);
if (!is_dir($dir) && !mkdir($dir, 0777, true) && !is_dir($dir)) {
// Wenn Log nicht möglich ist: nicht hart scheitern (Build ist wichtiger)
return;
}
@file_put_contents($logPath, $content, FILE_APPEND);
}
private function atomicSwitch(string $tmp, string $final): void
{
if (!rename($tmp, $final)) {
@unlink($tmp);
throw new \RuntimeException('Atomic switch failed for vector.index');
@file_put_contents($logPath, "=== VectorIndexBuilder OK ===\n", FILE_APPEND);
}
}
}

96
src/Vector/VectorSearchClient.php
View File

@@ -8,48 +8,90 @@ use Psr\Log\LoggerInterface;
final class VectorSearchClient
{
private string $pythonBin;
private string $scriptPath;
private string $vectorIndexPath;
private string $vectorMetaPath;
private string $indexMetaPath;
private LoggerInterface $agentLogger;
public function __construct(
private readonly string $binPythonDir,
private readonly string $vectorSearchPyPath,
private LoggerInterface $agentLogger,
string $pythonBin,
string $scriptPath,
string $vectorIndexPath,
string $vectorMetaPath,
string $indexMetaPath,
LoggerInterface $agentLogger
) {
$this->pythonBin = $pythonBin;
$this->scriptPath = $scriptPath;
$this->vectorIndexPath = $vectorIndexPath;
$this->vectorMetaPath = $vectorMetaPath;
$this->indexMetaPath = $indexMetaPath;
$this->agentLogger = $agentLogger;
}
public function search(string $query, int $limit = 5): array
{
$script = $this->vectorSearchPyPath;
$this->agentLogger->info("Run vector search script $script");
if (!is_file($script)) {
if (!is_file($this->scriptPath)) {
$this->agentLogger->error('vector_search.py not found: ' . $this->scriptPath);
return [];
}
// -------------------------------------------------
// Determine Python interpreter (venv preferred)
// -------------------------------------------------
$venvPython = $this->binPythonDir;
$pythonBin = is_file($venvPython) ? $venvPython : 'python3';
$cmd = sprintf(
'%s %s %s %d 2>&1',
escapeshellarg($pythonBin),
escapeshellarg($script),
escapeshellarg($query),
$limit
);
exec($cmd, $out, $exitCode);
if ($exitCode !== 0 || empty($out)) {
if (!is_file($this->vectorIndexPath)) {
$this->agentLogger->warning('vector.index not found.');
return [];
}
$json = implode("\n", $out);
if (!is_file($this->vectorMetaPath)) {
$this->agentLogger->warning('vector.index.meta.json not found.');
return [];
}
$this->agentLogger->info($json);
if (!is_file($this->indexMetaPath)) {
$this->agentLogger->warning('index_meta.json not found.');
return [];
}
$indexMeta = json_decode((string) file_get_contents($this->indexMetaPath), true);
if (!is_array($indexMeta) || empty($indexMeta['embedding_model'])) {
$this->agentLogger->error('Invalid index_meta.json.');
return [];
}
$embeddingModel = $indexMeta['embedding_model'];
$cmd = [
$this->pythonBin,
$this->scriptPath,
'--query', $query,
'--limit', (string)$limit,
'--index', $this->vectorIndexPath,
'--meta', $this->vectorMetaPath,
'--model', $embeddingModel,
];
$process = new \Symfony\Component\Process\Process($cmd);
$process->setTimeout(30);
$process->run();
if (!$process->isSuccessful()) {
$this->agentLogger->error('Vector search failed: ' . $process->getErrorOutput());
return [];
}
$output = $process->getOutput();
if (trim($output) === '') {
return [];
}
try {
return json_decode($json, true, 512, JSON_THROW_ON_ERROR);
} catch (\Throwable) {
$this->agentLogger->info('vector_search.py is done: ' . $this->scriptPath);
return json_decode($output, true, 512, JSON_THROW_ON_ERROR);
} catch (\Throwable $e) {
$this->agentLogger->error('Invalid JSON from vector_search.py');
return [];
}
}

93
src/Vector/vector_search.py
View File

@@ -2,71 +2,116 @@
import sys
import json
import argparse
from pathlib import Path
# ---------------------------------------------------------
# Argument handling
# ---------------------------------------------------------
if len(sys.argv) < 3:
print("ERROR: Missing arguments (query, limit)")
sys.exit(2)
query = sys.argv[1]
limit = int(sys.argv[2])
vector_dir = Path(__file__).resolve().parent
index_path = vector_dir / "vector.index"
meta_path = vector_dir / "vector.index.meta.json"
# ---------------------------------------------------------
# Dependency checks (controlled)
# Argument parsing (NEW – CLEAN CLI)
# ---------------------------------------------------------
parser = argparse.ArgumentParser(description="FAISS vector search")
parser.add_argument("--query", required=True, help="Search query text")
parser.add_argument("--limit", required=True, type=int, help="Top-K limit")
parser.add_argument("--index", required=True, help="Path to vector.index")
parser.add_argument("--meta", required=True, help="Path to vector.index.meta.json")
parser.add_argument("--model", required=True, help="SentenceTransformer model")
args = parser.parse_args()
query = args.query
limit = args.limit
index_path = Path(args.index).resolve()
meta_path = Path(args.meta).resolve()
embedding_model = args.model
# ---------------------------------------------------------
# Dependency checks (stderr only)
# ---------------------------------------------------------
try:
import faiss # noqa
except Exception:
print("ERROR: Python module 'faiss' not found.")
print("Python module 'faiss' not found.", file=sys.stderr)
sys.exit(10)
try:
from sentence_transformers import SentenceTransformer # noqa
except Exception:
print("ERROR: Python module 'sentence-transformers' not found.")
print("Python module 'sentence-transformers' not found.", file=sys.stderr)
sys.exit(11)
import faiss
from sentence_transformers import SentenceTransformer
# ---------------------------------------------------------
# File checks
# ---------------------------------------------------------
if not index_path.is_file() or not meta_path.is_file():
print("ERROR: Vector index not found. Run vector ingest first.")
if not index_path.is_file():
print(f"vector.index not found at {index_path}", file=sys.stderr)
sys.exit(20)
if not meta_path.is_file():
print(f"vector.index.meta.json not found at {meta_path}", file=sys.stderr)
sys.exit(21)
# ---------------------------------------------------------
# Load model and index
# ---------------------------------------------------------
model = SentenceTransformer("all-MiniLM-L6-v2")
query_vec = model.encode([query], normalize_embeddings=True)
try:
model = SentenceTransformer(embedding_model)
except Exception as e:
print(f"Failed to load embedding model: {embedding_model}", file=sys.stderr)
sys.exit(30)
index = faiss.read_index(str(index_path))
try:
query_vec = model.encode([query], normalize_embeddings=True)
except Exception:
print("Embedding encoding failed.", file=sys.stderr)
sys.exit(31)
try:
index = faiss.read_index(str(index_path))
except Exception:
print("Failed to read FAISS index.", file=sys.stderr)
sys.exit(32)
try:
with open(meta_path, "r", encoding="utf-8") as f:
ids = json.load(f)
except Exception:
print("Failed to read vector meta file.", file=sys.stderr)
sys.exit(33)
with open(meta_path, "r", encoding="utf-8") as f:
ids = json.load(f)
# ---------------------------------------------------------
# Search
# ---------------------------------------------------------
scores, indices = index.search(query_vec, limit)
try:
scores, indices = index.search(query_vec, limit)
except Exception:
print("FAISS search failed.", file=sys.stderr)
sys.exit(40)
results = []
for score, idx in zip(scores[0], indices[0]):
if idx == -1:
continue
if idx < 0 or idx >= len(ids):
continue
results.append({
"chunk_id": ids[idx],
"score": float(score)
})
# ---------------------------------------------------------
# STRICT JSON OUTPUT ONLY
# ---------------------------------------------------------
print(json.dumps(results))
sys.exit(0)