add system prompt and chunks index views and edit

2026-02-15 21:33:31 +01:00
parent 59e48242b5
commit 3416678cf4
12 changed files with 743 additions and 7 deletions
--- a/src/Service/Admin/IndexNdjsonInspector.php
+++ b/src/Service/Admin/IndexNdjsonInspector.php
@@ -0,0 +1,132 @@
+<?php
+// src/Service/Admin/IndexNdjsonInspector.php
+
+declare(strict_types=1);
+
+namespace App\Service\Admin;
+
+final class IndexNdjsonInspector
+{
+    private string $ndjsonPath;
+    private string $metaPath;
+
+    public function __construct(string $ndJsonPath, string $indexMetaPath)
+    {
+        // Passe diese Pfade an deine echten Ablageorte an, falls abweichend:
+        // z.B. var/rag/index.ndjson oder storage/rag/index.ndjson etc.
+        $this->ndjsonPath = $ndJsonPath;
+        $this->metaPath   = $indexMetaPath;
+    }
+
+    public function getPaths(): array
+    {
+        return [
+            'ndjson' => $this->ndjsonPath,
+            'meta'   => $this->metaPath,
+        ];
+    }
+
+    public function readMeta(): array
+    {
+        if (!is_file($this->metaPath)) {
+            return [
+                'error' => 'index_meta.json nicht gefunden',
+                'path'  => $this->metaPath,
+            ];
+        }
+
+        $raw = @file_get_contents($this->metaPath);
+        if ($raw === false) {
+            return [
+                'error' => 'index_meta.json konnte nicht gelesen werden',
+                'path'  => $this->metaPath,
+            ];
+        }
+
+        $data = json_decode($raw, true);
+        if (!is_array($data)) {
+            return [
+                'error' => 'index_meta.json ist kein valides JSON-Objekt',
+                'path'  => $this->metaPath,
+            ];
+        }
+
+        return $data;
+    }
+
+    /**
+     * Liest NDJSON "streaming" und gibt nur einen Ausschnitt zurück.
+     * - Keine Voll-Loads
+     * - Maximal limit Zeilen
+     * - Offset = (page-1)*limit
+     */
+    public function readNdjsonPage(int $page, int $limit): array
+    {
+        $page  = max(1, $page);
+        $limit = max(1, min(200, $limit)); // hard cap für Admin-UI
+
+        if (!is_file($this->ndjsonPath)) {
+            return [
+                'error' => 'index.ndjson nicht gefunden',
+                'path'  => $this->ndjsonPath,
+                'items' => [],
+            ];
+        }
+
+        $handle = @fopen($this->ndjsonPath, 'rb');
+        if ($handle === false) {
+            return [
+                'error' => 'index.ndjson konnte nicht geöffnet werden',
+                'path'  => $this->ndjsonPath,
+                'items' => [],
+            ];
+        }
+
+        $offsetLines = ($page - 1) * $limit;
+        $items = [];
+        $lineNo = 0;
+
+        // Sicherheitslimit: wir lesen max. X Bytes pro Line (um Monster-Zeilen abzufangen)
+        $maxLineBytes = 1024 * 1024; // 1MB pro Zeile
+
+        while (!feof($handle)) {
+            $line = fgets($handle, $maxLineBytes);
+            if ($line === false) {
+                break;
+            }
+
+            $lineNo++;
+
+            if ($lineNo <= $offsetLines) {
+                continue;
+            }
+
+            $line = trim($line);
+            if ($line === '') {
+                continue;
+            }
+
+            $decoded = json_decode($line, true);
+            if (!is_array($decoded)) {
+                // Ungültige Zeile ignorieren, aber sichtbar machen wäre auch ok.
+                continue;
+            }
+
+            $items[] = $decoded;
+
+            if (count($items) >= $limit) {
+                break;
+            }
+        }
+
+        fclose($handle);
+
+        return [
+            'error' => null,
+            'path'  => $this->ndjsonPath,
+            'items' => $items,
+            'page'  => $page,
+            'limit' => $limit,
+        ];
+    }
+}