marek/MtoRagSystem
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

new ingest und profile settings

Browse Source
This commit is contained in:
team 1
2026-02-16 14:38:02 +01:00
parent ece93e4cb4
commit 8666b05570
15 changed files with 655 additions and 199 deletions
Download Patch File Download Diff File Expand all files Collapse all files

102
src/Controller/Admin/IngestProfileController.php Normal file
View File

@@ -0,0 +1,102 @@
<?php
declare(strict_types=1);
namespace App\Controller\Admin;
use App\Entity\IngestProfile;
use App\Index\IndexConfigurationProvider;
use App\Index\IndexMetaManager;
use App\Index\IndexStructureComparator;
use App\Repository\IngestProfileRepository;
use Doctrine\ORM\EntityManagerInterface;
use Symfony\Bundle\FrameworkBundle\Controller\AbstractController;
use Symfony\Component\HttpFoundation\Request;
use Symfony\Component\HttpFoundation\Response;
use Symfony\Component\Routing\Attribute\Route;
#[Route('/admin/ingest-profiles')]
class IngestProfileController extends AbstractController
{
#[Route('/', name: 'admin_ingest_profile_list')]
public function list(
IngestProfileRepository $repo,
IndexMetaManager $metaManager,
IndexConfigurationProvider $provider,
IndexStructureComparator $comparator
): Response {
$profiles = $repo->findBy([], ['version' => 'DESC']);
$activeProfile = $repo->findActive();
$meta = $metaManager->readMeta();
$currentStructure = $provider->getConfiguration()->toStructureArray();
$diff = $comparator->compare($meta, $currentStructure);
$structureMismatch = false;
foreach ($diff as $row) {
if (!$row['equal']) {
$structureMismatch = true;
break;
}
}
return $this->render('admin/ingest_profile/list.html.twig', [
'profiles' => $profiles,
'activeProfile' => $activeProfile,
'indexMeta' => $meta,
'diff' => $diff,
'structureMismatch' => $structureMismatch,
]);
}
#[Route('/create', name: 'admin_ingest_profile_create', methods: ['GET', 'POST'])]
public function create(
Request $request,
IngestProfileRepository $repo,
EntityManagerInterface $em
): Response {
if ($request->isMethod('POST')) {
$latest = $repo->findLatestVersion();
$nextVersion = $latest ? $latest->getVersion() + 1 : 1;
$profile = new IngestProfile(
$nextVersion,
(int)$request->request->get('chunk_size'),
(int)$request->request->get('chunk_overlap'),
(string)$request->request->get('embedding_model'),
(int)$request->request->get('embedding_dimension'),
(int)$request->request->get('scoring_version')
);
$em->persist($profile);
$em->flush();
return $this->redirectToRoute('admin_ingest_profile_list');
}
return $this->render('admin/ingest_profile/create.html.twig');
}
#[Route('/activate/{id}', name: 'admin_ingest_profile_activate')]
public function activate(
IngestProfile $profile,
IngestProfileRepository $repo,
EntityManagerInterface $em
): Response {
$active = $repo->findActive();
if ($active) {
$active->deactivate();
}
$profile->activate();
$em->flush();
return $this->redirectToRoute('admin_ingest_profile_list');
}
}

90
src/Entity/IngestProfile.php Normal file
View File

@@ -0,0 +1,90 @@
<?php
declare(strict_types=1);
namespace App\Entity;
use App\Repository\IngestProfileRepository;
use Doctrine\ORM\Mapping as ORM;
use Symfony\Component\Uid\Uuid;
#[ORM\Entity(repositoryClass: IngestProfileRepository::class)]
#[ORM\Table(name: 'ingest_profile')]
class IngestProfile
{
#[ORM\Id]
#[ORM\Column(type: 'uuid', unique: true)]
private Uuid $id;
#[ORM\Column(type: 'integer')]
private int $version;
#[ORM\Column(type: 'integer')]
private int $chunkSize;
#[ORM\Column(type: 'integer')]
private int $chunkOverlap;
#[ORM\Column(type: 'string', length: 255)]
private string $embeddingModel;
#[ORM\Column(type: 'integer')]
private int $embeddingDimension;
#[ORM\Column(type: 'integer')]
private int $scoringVersion;
#[ORM\Column(type: 'boolean')]
private bool $active = false;
#[ORM\Column(type: 'boolean')]
private bool $reindexRequired = true;
#[ORM\Column(type: 'datetime_immutable')]
private \DateTimeImmutable $createdAt;
public function __construct(
int $version,
int $chunkSize,
int $chunkOverlap,
string $embeddingModel,
int $embeddingDimension,
int $scoringVersion
) {
$this->id = Uuid::v4();
$this->version = $version;
$this->chunkSize = $chunkSize;
$this->chunkOverlap = $chunkOverlap;
$this->embeddingModel = $embeddingModel;
$this->embeddingDimension = $embeddingDimension;
$this->scoringVersion = $scoringVersion;
$this->createdAt = new \DateTimeImmutable();
}
public function getId(): Uuid { return $this->id; }
public function getVersion(): int { return $this->version; }
public function getChunkSize(): int { return $this->chunkSize; }
public function getChunkOverlap(): int { return $this->chunkOverlap; }
public function getEmbeddingModel(): string { return $this->embeddingModel; }
public function getEmbeddingDimension(): int { return $this->embeddingDimension; }
public function getScoringVersion(): int { return $this->scoringVersion; }
public function isActive(): bool { return $this->active; }
public function isReindexRequired(): bool { return $this->reindexRequired; }
public function getCreatedAt(): \DateTimeImmutable { return $this->createdAt; }
public function activate(): void
{
$this->active = true;
$this->reindexRequired = true;
}
public function deactivate(): void
{
$this->active = false;
}
public function markReindexDone(): void
{
$this->reindexRequired = false;
}
}

63
src/Index/IndexConfigurationProvider.php Normal file
View File

@@ -0,0 +1,63 @@
<?php
declare(strict_types=1);
namespace App\Index;
use App\Repository\IngestProfileRepository;
final class IndexConfigurationProvider
{
private IngestProfileRepository $repository;
private int $fallbackChunkSize;
private int $fallbackChunkOverlap;
private string $fallbackEmbeddingModel;
private int $fallbackEmbeddingDimension;
private int $fallbackScoringVersion;
public function __construct(
IngestProfileRepository $repository,
int $fallbackChunkSize,
int $fallbackChunkOverlap,
string $fallbackEmbeddingModel,
int $fallbackEmbeddingDimension,
int $fallbackScoringVersion
) {
$this->repository = $repository;
$this->fallbackChunkSize = $fallbackChunkSize;
$this->fallbackChunkOverlap = $fallbackChunkOverlap;
$this->fallbackEmbeddingModel = $fallbackEmbeddingModel;
$this->fallbackEmbeddingDimension = $fallbackEmbeddingDimension;
$this->fallbackScoringVersion = $fallbackScoringVersion;
}
public function getConfiguration(): IndexConfiguration
{
$active = $this->repository->findActive();
if ($active === null) {
// Fallback auf YAML
return new IndexConfiguration(
$this->fallbackChunkSize,
$this->fallbackChunkOverlap,
$this->fallbackEmbeddingModel,
$this->fallbackEmbeddingDimension,
$this->fallbackScoringVersion,
'ndjson',
'faiss'
);
}
return new IndexConfiguration(
$active->getChunkSize(),
$active->getChunkOverlap(),
$active->getEmbeddingModel(),
$active->getEmbeddingDimension(),
$active->getScoringVersion(),
'ndjson',
'faiss'
);
}
}

171
src/Index/IndexMetaManager.php
View File

@@ -7,157 +7,90 @@ namespace App\Index;
final class IndexMetaManager
{
private string $metaPath;
private IndexConfiguration $config;
private IndexConfigurationProvider $provider;
public function __construct(
string $metaPath,
IndexConfiguration $config
IndexConfigurationProvider $provider
) {
$this->metaPath = $metaPath;
$this->config = $config;
$this->provider = $provider;
}
public function getMetaPath(): string
// -----------------------------------------------------
// Public API
// -----------------------------------------------------
public function ensureExists(): void
{
return $this->metaPath;
if (!is_file($this->metaPath)) {
$this->writeMeta(1);
}
}
/**
* @return array<string,mixed>|null
*/
public function readMeta(): ?array
{
if (!is_file($this->metaPath)) {
return null;
}
$raw = file_get_contents($this->metaPath);
if ($raw === false) {
throw new \RuntimeException('Unable to read index_meta.json');
}
$data = json_decode($raw, true);
if (!is_array($data)) {
throw new \RuntimeException('index_meta.json is invalid JSON');
}
return $data;
return json_decode(
(string) file_get_contents($this->metaPath),
true
);
}
/**
* Guardrail:
* - Wenn Meta fehlt → initialisieren
* - Wenn Struktur driftet → Exception
*/
public function validateAgainstCurrent(): void
{
$meta = $this->readMeta();
$current = $this->provider
->getConfiguration()
->toStructureArray();
if ($meta === null) {
$meta = $this->createInitialMeta();
return;
}
$expected = $this->config->toStructureArray();
$diff = $this->diffStructure($meta, $expected);
if ($diff !== []) {
throw new IndexStructureChangedException(
'Index structure changed. Global Reindex required.',
$diff
);
}
}
/**
* Wird beim Global Reindex aufgerufen
*/
public function writeMetaForGlobalReindex(): array
{
$current = $this->readMeta();
$nextVersion = 1;
if (is_array($current) && isset($current['index_version']) && is_int($current['index_version'])) {
$nextVersion = $current['index_version'] + 1;
}
$meta = $this->buildMetaPayload($nextVersion);
$this->atomicWriteJson($meta);
return $meta;
}
public function getConfig(): IndexConfiguration
{
return $this->config;
}
// ---------------------------------------------------------
// Internals
// ---------------------------------------------------------
private function createInitialMeta(): array
{
$meta = $this->buildMetaPayload(1);
$this->atomicWriteJson($meta);
return $meta;
}
private function buildMetaPayload(int $indexVersion): array
{
$structure = $this->config->toStructureArray();
return [
'index_version' => $indexVersion,
'created_at' => (new \DateTimeImmutable())->format(DATE_ATOM),
'embedding_model' => $structure['embedding_model'],
'embedding_dimension' => $structure['embedding_dimension'],
'chunk_size' => $structure['chunk_size'],
'chunk_overlap' => $structure['chunk_overlap'],
'scoring_version' => $structure['scoring_version'],
'index_format' => $structure['index_format'],
'vector_backend' => $structure['vector_backend'],
];
}
private function diffStructure(array $meta, array $expected): array
{
$diff = [];
foreach ($expected as $key => $value) {
$actual = $meta[$key] ?? null;
if ($actual !== $value) {
$diff[$key] = [
'expected' => $value,
'actual' => $actual,
];
foreach ($current as $key => $value) {
if (($meta[$key] ?? null) !== $value) {
throw new \RuntimeException(
'Index structure changed. Global Reindex required.'
);
}
}
return $diff;
}
private function atomicWriteJson(array $payload): void
public function writeMetaForGlobalReindex(): void
{
$current = $this->readMeta();
$nextVersion = ($current['index_version'] ?? 0) + 1;
$this->writeMeta($nextVersion);
}
public function writeMeta(int $indexVersion): void
{
$config = $this->provider->getConfiguration();
$payload = array_merge(
[
'index_version' => $indexVersion,
'created_at' => (new \DateTimeImmutable())->format(DATE_ATOM),
],
$config->toStructureArray()
);
$dir = dirname($this->metaPath);
if (!is_dir($dir) && !mkdir($dir, 0777, true) && !is_dir($dir)) {
throw new \RuntimeException('Unable to create directory: ' . $dir);
if (!is_dir($dir)) {
mkdir($dir, 0777, true);
}
$tmp = $this->metaPath . '.tmp';
$json = json_encode($payload, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES);
if ($json === false) {
throw new \RuntimeException('Unable to encode index_meta.json');
}
if (file_put_contents($tmp, $json . PHP_EOL) === false) {
throw new \RuntimeException('Unable to write temp meta file');
}
if (!rename($tmp, $this->metaPath)) {
@unlink($tmp);
throw new \RuntimeException('Unable to switch meta file atomically');
}
file_put_contents(
$this->metaPath,
json_encode(
$payload,
JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES
)
);
}
}

37
src/Index/IndexStructureComparator.php Normal file
View File

@@ -0,0 +1,37 @@
<?php
declare(strict_types=1);
namespace App\Index;
final class IndexStructureComparator
{
public function compare(?array $meta, array $profileStructure): array
{
$result = [];
$keys = array_unique(
array_merge(
array_keys($profileStructure),
$meta ? array_keys($meta) : []
)
);
foreach ($keys as $key) {
if (in_array($key, ['index_version', 'created_at'], true)) {
continue;
}
$metaValue = $meta[$key] ?? null;
$profileValue = $profileStructure[$key] ?? null;
$result[$key] = [
'meta' => $metaValue,
'profile' => $profileValue,
'equal' => $metaValue === $profileValue,
];
}
return $result;
}
}

34
src/Knowledge/Ingest/SimpleChunker.php
View File

@@ -5,22 +5,31 @@ declare(strict_types=1);
namespace App\Knowledge\Ingest;
use App\Index\IndexConfigurationProvider;
final class SimpleChunker
{
private IndexConfigurationProvider $configurationProvider;
public function __construct(
private int $maxWords = 180,
private int $overlapWords = 30
) {}
IndexConfigurationProvider $configurationProvider
) {
$this->configurationProvider = $configurationProvider;
}
/** @return string[] */
public function chunk(string $text): array
{
$config = $this->configurationProvider->getConfiguration();
$maxWords = $config->getChunkSize();
$overlapWords = $config->getChunkOverlap();
$text = $this->normalize($text);
if ($text === '') {
return [];
}
// Split into tokens: words + whitespace preserved
$tokens = preg_split(
'/(\s+)/u',
$text,
@@ -32,7 +41,6 @@ final class SimpleChunker
return [];
}
// Build word index → token index mapping
$wordTokenIndexes = [];
foreach ($tokens as $i => $token) {
if (!preg_match('/^\s+$/u', $token)) {
@@ -49,12 +57,11 @@ final class SimpleChunker
$wordPos = 0;
while ($wordPos < $totalWords) {
$wordEnd = min($wordPos + $this->maxWords, $totalWords);
$wordEnd = min($wordPos + $maxWords, $totalWords);
$tokenStart = $wordTokenIndexes[$wordPos];
$tokenEnd = $wordTokenIndexes[$wordEnd - 1] + 1;
// Intelligent cut (sentence / paragraph aware)
$tokenEnd = $this->adjustCutToBoundary($tokens, $tokenStart, $tokenEnd);
$chunk = trim(implode('', array_slice(
@@ -71,7 +78,7 @@ final class SimpleChunker
break;
}
$wordPos = max(0, $wordEnd - $this->overlapWords);
$wordPos = max(0, $wordEnd - $overlapWords);
}
return $this->dedupe($chunks);
@@ -86,30 +93,19 @@ final class SimpleChunker
return trim((string) $text);
}
/**
* Move cut backwards to a natural boundary if possible.
* Rules:
* - Never cut inside markdown list items
* - Sentence end only if followed by a line break
* - Paragraph breaks always allowed
*/
private function adjustCutToBoundary(array $tokens, int $start, int $end): int
{
// Detect markdown list context (e.g. "- Foo: Bar")
$startToken = $tokens[$start] ?? '';
if (preg_match('/^- /u', ltrim($startToken))) {
// Keep list blocks intact
return $end;
}
for ($i = $end - 1; $i > $start; $i--) {
// Paragraph boundary
if ($tokens[$i] === "\n\n") {
return $i + 1;
}
// Sentence boundary only if followed by newline
if (
preg_match('/[.!?]\s*$/u', $tokens[$i]) &&
isset($tokens[$i + 1]) &&

31
src/Repository/IngestProfileRepository.php Normal file
View File

@@ -0,0 +1,31 @@
<?php
declare(strict_types=1);
namespace App\Repository;
use App\Entity\IngestProfile;
use Doctrine\Bundle\DoctrineBundle\Repository\ServiceEntityRepository;
use Doctrine\Persistence\ManagerRegistry;
class IngestProfileRepository extends ServiceEntityRepository
{
public function __construct(ManagerRegistry $registry)
{
parent::__construct($registry, IngestProfile::class);
}
public function findLatestVersion(): ?IngestProfile
{
return $this->createQueryBuilder('p')
->orderBy('p.version', 'DESC')
->setMaxResults(1)
->getQuery()
->getOneOrNullResult();
}
public function findActive(): ?IngestProfile
{
return $this->findOneBy(['active' => true]);
}
}

104
src/Vector/VectorIndexBuilder.php
View File

@@ -4,7 +4,7 @@ declare(strict_types=1);
namespace App\Vector;
use App\Index\IndexConfiguration;
use App\Index\IndexConfigurationProvider;
use Symfony\Component\Process\Exception\ProcessFailedException;
use Symfony\Component\Process\Process;
@@ -18,7 +18,7 @@ final class VectorIndexBuilder
private string $vectorMetaPath;
private int $timeoutSeconds;
private IndexConfiguration $indexConfiguration;
private IndexConfigurationProvider $configurationProvider;
public function __construct(
string $pythonBin,
@@ -27,7 +27,7 @@ final class VectorIndexBuilder
string $indexMetaPath,
string $vectorIndexPath,
int $timeoutSeconds,
IndexConfiguration $indexConfiguration
IndexConfigurationProvider $configurationProvider
) {
$this->pythonBin = $pythonBin;
$this->scriptPath = $scriptPath;
@@ -36,39 +36,29 @@ final class VectorIndexBuilder
$this->vectorIndexPath = $vectorIndexPath;
$this->vectorMetaPath = $vectorIndexPath . '.meta.json';
$this->timeoutSeconds = $timeoutSeconds;
$this->indexConfiguration = $indexConfiguration;
$this->configurationProvider = $configurationProvider;
}
/**
* Rebuild FAISS Index deterministisch aus index.ndjson.
*/
public function rebuildFromNdjson(?string $logPath = null): void
{
if (!is_file($this->scriptPath)) {
throw new \RuntimeException('vector_ingest.py not found at: ' . $this->scriptPath);
}
if (!is_file($this->indexNdjsonPath)) {
throw new \RuntimeException('index.ndjson not found at: ' . $this->indexNdjsonPath);
}
$this->assertPreconditions();
if (!is_file($this->indexMetaPath)) {
$this->initializeIndexMeta();
}
$indexMeta = json_decode((string) file_get_contents($this->indexMetaPath), true);
$indexMeta = $this->readIndexMeta();
if (!is_array($indexMeta) || empty($indexMeta['embedding_model'])) {
throw new \RuntimeException('Invalid index_meta.json');
}
$embeddingModel = (string) $indexMeta['embedding_model'];
$embeddingModel = $indexMeta['embedding_model'];
$tmpVectorIndexPath = $this->vectorIndexPath . '.tmp';
// Wichtig: Python erzeugt meta basierend auf endgültigem Namen
$finalMetaPath = $this->vectorMetaPath;
$tmpMetaPath = dirname($this->vectorIndexPath) . '/' . basename($this->vectorIndexPath, '.index') . '.index.meta.json';
// Clean leftovers
@unlink($tmpVectorIndexPath);
@unlink($finalMetaPath);
@unlink($this->vectorMetaPath);
$cmd = [
$this->pythonBin,
@@ -80,21 +70,41 @@ final class VectorIndexBuilder
$process = new Process($cmd);
$process->setTimeout($this->timeoutSeconds);
$process->mustRun();
if (!is_file($tmpVectorIndexPath) || filesize($tmpVectorIndexPath) === 0) {
throw new \RuntimeException('Vector index tmp missing or empty');
$this->runProcess($process, $logPath);
$this->validatePythonOutputs($tmpVectorIndexPath);
$this->atomicSwitch($tmpVectorIndexPath);
}
// -----------------------------------------------------
// Internals
// -----------------------------------------------------
private function assertPreconditions(): void
{
if (!is_file($this->scriptPath)) {
throw new \RuntimeException('vector_ingest.py not found at: ' . $this->scriptPath);
}
// Python erzeugt vector.index.meta.json (nicht tmp.meta!)
if (!is_file($this->vectorMetaPath) || filesize($this->vectorMetaPath) === 0) {
throw new \RuntimeException('Vector meta missing or empty');
if (!is_file($this->indexNdjsonPath)) {
throw new \RuntimeException('index.ndjson not found at: ' . $this->indexNdjsonPath);
}
}
private function readIndexMeta(): array
{
$meta = json_decode(
(string) file_get_contents($this->indexMetaPath),
true
);
if (!is_array($meta) || empty($meta['embedding_model'])) {
throw new \RuntimeException('Invalid index_meta.json');
}
// Atomarer Switch für Index
if (!rename($tmpVectorIndexPath, $this->vectorIndexPath)) {
throw new \RuntimeException('Atomic switch failed for vector index');
}
return $meta;
}
private function initializeIndexMeta(): void
@@ -105,14 +115,16 @@ final class VectorIndexBuilder
throw new \RuntimeException('Cannot create knowledge directory');
}
$config = $this->configurationProvider->getConfiguration();
$data = [
'index_version' => 1,
'created_at' => (new \DateTimeImmutable())->format(DATE_ATOM),
'embedding_model' => $this->indexConfiguration->getEmbeddingModel(),
'embedding_dimension' => $this->indexConfiguration->getEmbeddingDimension(),
'chunk_size' => $this->indexConfiguration->getChunkSize(),
'chunk_overlap' => $this->indexConfiguration->getChunkOverlap(),
'scoring_version' => $this->indexConfiguration->getScoringVersion(),
'embedding_model' => $config->getEmbeddingModel(),
'embedding_dimension' => $config->getEmbeddingDimension(),
'chunk_size' => $config->getChunkSize(),
'chunk_overlap' => $config->getChunkOverlap(),
'scoring_version' => $config->getScoringVersion(),
'index_format' => 'ndjson',
'vector_backend' => 'faiss',
];
@@ -123,6 +135,24 @@ final class VectorIndexBuilder
);
}
private function validatePythonOutputs(string $tmpVectorIndexPath): void
{
if (!is_file($tmpVectorIndexPath) || filesize($tmpVectorIndexPath) === 0) {
throw new \RuntimeException('Vector index tmp missing or empty');
}
if (!is_file($this->vectorMetaPath) || filesize($this->vectorMetaPath) === 0) {
throw new \RuntimeException('Vector meta missing or empty');
}
}
private function atomicSwitch(string $tmpVectorIndexPath): void
{
if (!rename($tmpVectorIndexPath, $this->vectorIndexPath)) {
throw new \RuntimeException('Atomic switch failed for vector index');
}
}
private function runProcess(Process $process, ?string $logPath): void
{
if ($logPath !== null) {