add chunk limiter

This commit is contained in:
team 1
2026-02-16 15:29:31 +01:00
parent b04f972971
commit feff95ffe5
7 changed files with 239 additions and 31 deletions

View File

@@ -22,6 +22,46 @@ final class ChunkManager
return $this->indexPath;
}
// ============================================================
// COUNT (für Guardrails / Limits)
// ============================================================
/**
* Zählt Datensätze (NDJSON-Zeilen) im index.ndjson streaming-basiert.
* Leere / kaputte Zeilen werden ignoriert.
*/
public function countAllChunks(): int
{
if (!is_file($this->indexPath)) {
return 0;
}
$handle = fopen($this->indexPath, 'rb');
if (!$handle) {
throw new \RuntimeException('Unable to open index.ndjson for counting');
}
$count = 0;
try {
while (($line = fgets($handle)) !== false) {
$line = trim($line);
if ($line === '') {
continue;
}
// NDJSON besteht aus JSON-Objekten; wir zählen nur valide Arrays.
$data = json_decode($line, true);
if (is_array($data)) {
$count++;
}
}
} finally {
fclose($handle);
}
return $count;
}
// ============================================================
// APPEND
// ============================================================