marek/MtoRagSystem
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

optimize auto reload uvicornserver load new vector libs if changed by py

Browse Source
This commit is contained in:
team2
2026-02-26 16:00:24 +01:00
parent 052ff55eda
commit deba7cd06f
4 changed files with 280 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
src/Command/TagsRebuildCommand.php
View File

@@ -4,6 +4,7 @@ declare(strict_types=1);
namespace App\Command;
use App\Index\IndexMetaManager;
use App\Tag\TagNdjsonExporter;
use App\Tag\TagVectorIndexBuilder;
use Symfony\Component\Console\Attribute\AsCommand;
@@ -20,6 +21,7 @@ final class TagsRebuildCommand extends Command
public function __construct(
private readonly TagNdjsonExporter $exporter,
private readonly TagVectorIndexBuilder $builder,
private readonly IndexMetaManager $metaManager,
) {
parent::__construct();
}
@@ -27,15 +29,32 @@ final class TagsRebuildCommand extends Command
protected function execute(InputInterface $input, OutputInterface $output): int
{
try {
// -----------------------------------------
// 1) Export tags.ndjson
// -----------------------------------------
$export = $this->exporter->export();
$output->writeln('<info>1/2 Exported tags.ndjson</info>');
$output->writeln('<info>1/3 Exported tags.ndjson</info>');
$output->writeln('Path: ' . $export['path']);
$output->writeln('Tags: ' . $export['tags']);
$output->writeln('Lines: ' . $export['lines']);
$output->writeln('Bytes: ' . $export['bytes']);
// -----------------------------------------
// 2) Build FAISS tag index
// -----------------------------------------
$this->builder->build();
$output->writeln('<info>2/2 Built vector_tags.index</info>');
$output->writeln('<info>2/3 Built vector_tags.index</info>');
// -----------------------------------------
// 3) Enterprise Commit Marker
// -----------------------------------------
$this->metaManager->touchRuntime([
'last_tags_rebuild_at' => (new \DateTimeImmutable())->format(DATE_ATOM),
]);
$output->writeln('<info>3/3 Runtime commit marker updated</info>');
} catch (\Throwable $e) {
$output->writeln('<error>ERROR: ' . $e->getMessage() . '</error>');
return Command::FAILURE;

54
src/Index/IndexMetaManager.php
View File

@@ -106,6 +106,9 @@ final class IndexMetaManager
$this->atomicWriteJson($this->runtimePath, $payload);
}
/**
* Für Chunk-Rebuilds (harte Zahl + Commit-Timestamp).
*/
public function updateRuntimeStats(int $chunkCount): void
{
$this->ensureRuntimeFileExists();
@@ -118,7 +121,40 @@ final class IndexMetaManager
$this->atomicWriteJson($this->runtimePath, $payload);
}
public function getRuntimeChunkCount(): int
/**
* Enterprise-Commit-Marker für alles, was einen Reload im Vector-Service erfordern soll,
* ohne dass zwingend chunk_count aktualisiert werden kann (z.B. Tag-Rebuild).
*
* Beispiel:
* touchRuntime(['last_tags_rebuild_at' => DATE_ATOM])
*/
public function touchRuntime(array $extra = []): void
{
$this->ensureRuntimeFileExists();
$current = $this->readRuntime() ?? [];
// Grundschema absichern
if (!isset($current['chunk_count'])) {
$current['chunk_count'] = 0;
}
$payload = array_merge(
$current,
$extra,
[
// Commit-Marker IMMER setzen/überschreiben
'last_rebuild_at' => (new \DateTimeImmutable())->format(DATE_ATOM),
]
);
$this->atomicWriteJson($this->runtimePath, $payload);
}
/**
* Runtime-Info lesen (z.B. für Tests, Debug, Polling).
*/
public function readRuntime(): ?array
{
$this->ensureRuntimeFileExists();
@@ -127,9 +163,25 @@ final class IndexMetaManager
true
);
return is_array($data) ? $data : null;
}
public function getRuntimeChunkCount(): int
{
$data = $this->readRuntime();
return (int)($data['chunk_count'] ?? 0);
}
public function getRuntimeLastRebuildAt(): ?string
{
$data = $this->readRuntime();
$v = $data['last_rebuild_at'] ?? null;
return is_string($v) && $v !== '' ? $v : null;
}
// =====================================================
// INTERNAL ATOMIC JSON WRITE
// =====================================================

16
src/Tag/TagVectorIndexBuilder.php
View File

@@ -4,6 +4,7 @@ declare(strict_types=1);
namespace App\Tag;
use App\Index\IndexMetaManager;
use Psr\Log\LoggerInterface;
final class TagVectorIndexBuilder
@@ -16,6 +17,7 @@ final class TagVectorIndexBuilder
private readonly string $embeddingModel,
private readonly int $timeoutSeconds,
private readonly LoggerInterface $agentLogger,
private readonly IndexMetaManager $metaManager, // ✅ NEU
) {}
public function build(): void
@@ -34,18 +36,14 @@ final class TagVectorIndexBuilder
$finalIndex = $this->vectorTagsIndexPath;
$finalMeta = $finalIndex . '.meta.json';
// Ensure output dir exists
$dir = \dirname($finalIndex);
if (!\is_dir($dir)) {
@\mkdir($dir, 0775, true);
}
// Clean tmp leftovers
@\unlink($tmpIndex);
@\unlink($tmpMeta);
// Positional args:
// python vector_ingest_tags.py <tags.ndjson> <out.tmp> <model>
$cmd = sprintf(
'%s %s %s %s %s 2>&1',
escapeshellarg($this->pythonBin),
@@ -73,20 +71,22 @@ final class TagVectorIndexBuilder
throw new \RuntimeException('Tag vector ingest failed (exit=' . $exit . ')');
}
// If no tags -> python may remove outputs and exit 0
if (!is_file($tmpIndex) || !is_file($tmpMeta)) {
// treat as "no index" rather than hard error
@\unlink($tmpIndex);
@\unlink($tmpMeta);
$this->agentLogger->warning('[tags] no tag index produced (maybe 0 tags).');
return;
}
// Atomic switch
$this->atomicReplace($tmpIndex, $finalIndex);
$this->atomicReplace($tmpMeta, $finalMeta);
$this->agentLogger->info('[tags] tag vector index build completed', [
// ✅ ENTERPRISE COMMIT MARKER
$this->metaManager->touchRuntime([
'last_tags_rebuild_at' => (new \DateTimeImmutable())->format(DATE_ATOM),
]);
$this->agentLogger->info('[tags] tag vector index build completed + runtime committed', [
'index' => $finalIndex,
'meta' => $finalMeta,
]);