marek/MtoRagSystem
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

optimize code and ingest docs

Browse Source
This commit is contained in:
team 1
2026-02-17 14:12:24 +01:00
parent f528a1c395
commit 812f2bf265
11 changed files with 156 additions and 282 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
src/Knowledge/Ingest/ChunkIndexWriter.php
View File

@@ -1,58 +0,0 @@
<?php
// src/Knowledge/Ingest/ChunkIndexWriter.php
declare(strict_types=1);
namespace App\Knowledge\Ingest;
final class ChunkIndexWriter
{
public function __construct(
private string $indexPath
) {}
public function add(array $entry): void
{
$index = $this->load();
$index[] = $entry;
$this->save($index);
}
private function load(): array
{
if (!is_file($this->indexPath)) {
return [];
}
$json = file_get_contents($this->indexPath);
$data = $json ? json_decode($json, true) : null;
return is_array($data) ? $data : [];
}
private function save(array $index): void
{
$dir = dirname($this->indexPath);
if (!is_dir($dir)) {
mkdir($dir, 0775, true);
}
file_put_contents(
$this->indexPath,
json_encode($index, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE)
);
}
public function hasSourceHash(string $source, string $hash): bool
{
foreach ($this->load() as $entry) {
if (
($entry['source'] ?? null) === $source &&
($entry['sourceHash'] ?? null) === $hash
) {
return true;
}
}
return false;
}
}

149
src/Knowledge/Ingest/ChunkWriter.php
View File

@@ -1,149 +0,0 @@
<?php
// src/Knowledge/Ingest/ChunkWriter.php
declare(strict_types=1);
namespace App\Knowledge\Ingest;
use App\Knowledge\StopWords;
final class ChunkWriter
{
public function __construct(
private string $chunksDir,
private string $manifestPath,
private ChunkIndexWriter $indexWriter,
private StopWords $stopWords,
)
{
}
/**
* @param string[] $chunks
* @return string[] written filenames
*/
public function write(string $sourceName, array $chunks, string $sourceHash): array
{
if (!is_dir($this->chunksDir)) {
mkdir($this->chunksDir, 0775, true);
}
$manifest = $this->loadManifest();
$written = [];
$base = $this->safeBase($sourceName);
$ts = date('Ymd_His');
foreach ($chunks as $i => $chunk) {
$filename = "{$base}__{$ts}__" . str_pad((string)$i, 4, '0', STR_PAD_LEFT) . ".txt";
$path = rtrim($this->chunksDir, '/') . '/' . $filename;
$header = $this->buildHeader(
source: $sourceName,
index: $i
);
file_put_contents($path, $header . "\n\n" . $chunk);
$written[] = $filename;
$manifest[] = [
'file' => $filename,
'source' => $sourceName,
'index' => $i,
'chars' => mb_strlen($chunk),
'createdAt' => date('c'),
];
$this->indexWriter->add([
'file' => $filename,
'source' => $sourceName,
'sourceHash' => $sourceHash,
'keywords' => $this->extractKeywords($chunk),
'chars' => mb_strlen($chunk),
]);
}
$this->saveManifest($manifest);
return $written;
}
private function safeBase(string $name): string
{
$name = pathinfo($name, PATHINFO_FILENAME);
$name = mb_strtolower($name);
$name = preg_replace('/[^a-z0-9\-_]+/u', '-', $name);
return trim((string)$name, '-');
}
private function loadManifest(): array
{
if (!is_file($this->manifestPath)) {
return [];
}
$json = file_get_contents($this->manifestPath);
$data = $json ? json_decode($json, true) : null;
return is_array($data) ? $data : [];
}
private function saveManifest(array $manifest): void
{
$dir = dirname($this->manifestPath);
if (!is_dir($dir)) {
mkdir($dir, 0775, true);
}
file_put_contents($this->manifestPath, json_encode($manifest, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE));
}
private function buildHeader(string $source, int $index): string
{
return sprintf(
'[Quelle: %s | Abschnitt: Chunk %d]',
$source,
$index + 1
);
}
private function extractKeywords(string $text): array
{
// 1) Lowercase
$text = mb_strtolower($text);
// 2) URLs entfernen (sehr wichtig)
$text = preg_replace('#https?://\S+#u', ' ', $text);
// 3) Newlines & Tabs → Space
$text = str_replace(["\r", "\n", "\t"], ' ', $text);
// 4) Trennzeichen → Space (NICHT löschen!)
$text = preg_replace('/[\/\.\,\:\;\-\_\(\)\[\]\{\}]/u', ' ', $text);
// 5) Alles andere raus
$text = preg_replace('/[^\p{L}\p{N}\s]/u', '', $text);
// 6) Whitespace normalisieren
$text = preg_replace('/\s+/u', ' ', $text);
$text = trim($text);
// 7) Wörter extrahieren
$words = explode(' ', $text);
// 8) Filtern + deduplizieren
$keywords = [];
foreach ($words as $word) {
if (mb_strlen($word) < 4) {
continue;
}
if (in_array($word, $this->stopWords->getStopWords() ?? [], true)) {
continue;
}
$keywords[] = $word;
}
return array_values(array_unique(array_slice($keywords, 0, 25)));
}
}

15
src/Knowledge/Ingest/KnowledgeIngestService.php
View File

@@ -29,18 +29,30 @@ final class KnowledgeIngestService
$chunks = $this->chunker->chunk($text);
$documentId = $version->getDocument()->getId()->toRfc4122();
$doc = $version->getDocument();
$documentId = $doc->getId()->toRfc4122();
$versionId = $version->getId()->toRfc4122();
// ✅ Regel: Wenn title gefüllt ist, kommt er in jeden Chunk
$title = trim((string) $doc->getTitle());
$index = 0;
foreach ($chunks as $chunkText) {
// ✅ Prefix nur wenn title vorhanden; keine Flags, keine Meta-Schalter
if ($title !== '' && !str_starts_with($chunkText, $title)) {
$chunkText = $title . "\n\n" . $chunkText;
}
yield [
'chunk_id' => Uuid::v4()->toRfc4122(),
'document_id' => $documentId,
'version_id' => $versionId,
'chunk_index' => $index++,
'text' => $chunkText,
// ✅ checksum muss den finalen Text abbilden (inkl. Titel)
'checksum' => sha1($chunkText),
'metadata' => $this->buildMetadata($version),
];
@@ -56,7 +68,6 @@ final class KnowledgeIngestService
public function buildAllActiveChunkRecords(): iterable
{
foreach ($this->versionRepo->iterateActiveVersions() as $version) {
// yield from hält das Ganze streamingfähig (Generator-Kaskade)
yield from $this->buildChunkRecords($version);
}
}