p100c

2026-05-12 09:16:09 +02:00
parent 0d55c0a439
commit feaec9bbaf
7 changed files with 405 additions and 17 deletions
--- a/src/Controller/Admin/AdminEvalController.php
+++ b/src/Controller/Admin/AdminEvalController.php
@@ -48,6 +48,8 @@ final class AdminEvalController extends AbstractController

        try {
            $report = $evals->run($type, $caseId !== '' ? $caseId : null);
+            $type = trim((string) ($report['type'] ?? $type));
+
            $this->addFlash(
                ((int) ($report['failed'] ?? 0)) === 0 ? 'success' : 'danger',
                sprintf(
--- a/src/Eval/RetrievalDebugRunner.php
+++ b/src/Eval/RetrievalDebugRunner.php
@@ -33,6 +33,8 @@ final readonly class RetrievalDebugRunner

        $documentIds = $this->extractUniqueStringValues($rows, 'document_id');
        $chunkIds = $this->extractUniqueStringValues($rows, 'chunk_id');
+        $documentRefs = $this->buildDocumentRefs($rows);
+        $resultRows = $this->buildResultRows($rows);
        $joinedText = $this->extractJoinedText($rows);

        $assert = $case->assert;
@@ -220,6 +222,8 @@ final readonly class RetrievalDebugRunner
                'intent' => $intent,
                'document_ids' => $documentIds,
                'chunk_ids' => $chunkIds,
+                'document_refs' => $documentRefs,
+                'result_rows' => $resultRows,
                'matched_any_terms' => $matchedAnyTerms,
                'matched_all_terms' => $matchedAllTerms,
                'forbidden_terms_checked' => $this->normalizeStringList($assert['must_not_include_terms'] ?? []),
@@ -268,6 +272,122 @@ final readonly class RetrievalDebugRunner
        return array_keys($values);
    }

+    /**
+     * @param array<int, array<string, mixed>> $rows
+     * @return array<int, array{id:string,title:string,file_path:string,version_number:string,chunk_ids:array<int,string>,ranks:array<int,int>}>
+     */
+    private function buildDocumentRefs(array $rows): array
+    {
+        $refs = [];
+
+        foreach ($rows as $row) {
+            $documentId = $this->extractNullableString($row, 'document_id');
+
+            if ($documentId === '') {
+                continue;
+            }
+
+            if (!isset($refs[$documentId])) {
+                $refs[$documentId] = [
+                    'id' => $documentId,
+                    'title' => $this->extractNullableString($row, 'document_title'),
+                    'file_path' => $this->extractNullableString($row, 'file_path'),
+                    'version_number' => $this->extractNullableString($row, 'version_number'),
+                    'chunk_ids' => [],
+                    'ranks' => [],
+                ];
+            }
+
+            $chunkId = $this->extractNullableString($row, 'chunk_id');
+            if ($chunkId !== '' && !in_array($chunkId, $refs[$documentId]['chunk_ids'], true)) {
+                $refs[$documentId]['chunk_ids'][] = $chunkId;
+            }
+
+            $rank = $this->extractNullableInt($row, 'rank');
+            if ($rank !== null && !in_array($rank, $refs[$documentId]['ranks'], true)) {
+                $refs[$documentId]['ranks'][] = $rank;
+            }
+        }
+
+        return array_values($refs);
+    }
+
+    /**
+     * @param array<int, array<string, mixed>> $rows
+     * @return array<int, array<string, mixed>>
+     */
+    private function buildResultRows(array $rows): array
+    {
+        $out = [];
+
+        foreach ($rows as $row) {
+            $out[] = [
+                'rank' => $this->extractNullableInt($row, 'rank'),
+                'document_id' => $this->extractNullableString($row, 'document_id'),
+                'document_title' => $this->extractNullableString($row, 'document_title'),
+                'file_path' => $this->extractNullableString($row, 'file_path'),
+                'chunk_id' => $this->extractNullableString($row, 'chunk_id'),
+                'chunk_index' => $this->extractNullableInt($row, 'chunk_index'),
+                'raw_score' => $row['raw_score'] ?? null,
+                'rrf_score' => $row['rrf_score'] ?? null,
+                'text_preview' => $this->previewText($this->extractNullableString($row, 'text')),
+            ];
+        }
+
+        return $out;
+    }
+
+    /**
+     * @param array<string, mixed> $row
+     */
+    private function extractNullableString(array $row, string $key): string
+    {
+        $value = $row[$key] ?? null;
+
+        if ($value === null || is_array($value) || is_object($value)) {
+            return '';
+        }
+
+        return trim((string)$value);
+    }
+
+    /**
+     * @param array<string, mixed> $row
+     */
+    private function extractNullableInt(array $row, string $key): ?int
+    {
+        $value = $row[$key] ?? null;
+
+        if ($value === null || $value === '') {
+            return null;
+        }
+
+        if (is_int($value)) {
+            return $value;
+        }
+
+        if (is_string($value) && preg_match('/^-?\d+$/', trim($value)) === 1) {
+            return (int)$value;
+        }
+
+        return null;
+    }
+
+    private function previewText(string $text, int $limit = 240): string
+    {
+        $text = preg_replace('/\s+/u', ' ', trim($text)) ?? trim($text);
+
+        if ($text === '') {
+            return '';
+        }
+
+        if (mb_strlen($text, 'UTF-8') <= $limit) {
+            return $text;
+        }
+
+        return mb_substr($text, 0, $limit, 'UTF-8') . '...';
+    }
+
    /**
     * @param array<int, array<string, mixed>> $rows
     */
--- a/src/Knowledge/Retrieval/NdjsonHybridRetriever.php
+++ b/src/Knowledge/Retrieval/NdjsonHybridRetriever.php
@@ -133,13 +133,17 @@ final readonly class NdjsonHybridRetriever implements RetrieverInterface
                continue;
            }

+            $row = $result['rows'][$chunkId];
            $rank++;

            $out[] = [
                'rank' => $rank,
                'chunk_id' => $chunkId,
-                'document_id' => $result['rows'][$chunkId]['document_id'] ?? null,
-                'chunk_index' => $result['rows'][$chunkId]['chunk_index'] ?? null,
+                'document_id' => $row['document_id'] ?? null,
+                'document_title' => $this->extractDocumentTitle($row),
+                'file_path' => $this->extractMetadataString($row, 'file_path'),
+                'version_number' => $this->extractMetadataString($row, 'version_number'),
+                'chunk_index' => $row['chunk_index'] ?? null,
                'raw_score' => $result['rawScores'][$chunkId] ?? null,
                'rrf_score' => $result['rrfScores'][$chunkId] ?? null,
                'threshold' => $result['threshold'],
@@ -148,7 +152,7 @@ final readonly class NdjsonHybridRetriever implements RetrieverInterface
                'entity_label' => $result['entityLabel'],
                'is_list_query' => $result['isListQuery'],
                'selection_mode' => $result['selectionMode'],
-                'text' => trim((string)$result['rows'][$chunkId]['text']),
+                'text' => trim((string)($row['text'] ?? '')),
            ];
        }

@@ -1683,6 +1687,20 @@ final readonly class NdjsonHybridRetriever implements RetrieverInterface
        return '';
    }

+    /**
+     * Extracts a scalar metadata value for debug/eval output.
+     */
+    private function extractMetadataString(array $row, string $key): string
+    {
+        $value = $row['metadata'][$key] ?? null;
+
+        if (is_scalar($value)) {
+            return trim((string)$value);
+        }
+
+        return '';
+    }
+
    /**
     * Normalizes text for token-safe product comparisons.
     */
--- a/src/Service/Admin/EvalAdminService.php
+++ b/src/Service/Admin/EvalAdminService.php
@@ -111,14 +111,25 @@ final readonly class EvalAdminService
        $cases = $this->loadCases($type);

        if ($caseId !== '') {
-            $cases = array_values(array_filter(
-                $cases,
-                static fn (EvalCase $case): bool => $case->id === $caseId
-            ));
+            $cases = $this->filterCasesById($cases, $caseId);
+
+            if ($cases === []) {
+                [$type, $cases] = $this->findCasesByIdAcrossTypes($caseId);
+            }
        }

        if ($cases === []) {
-            throw new \RuntimeException('No eval cases selected.');
+            if ($caseId !== '') {
+                throw new \RuntimeException(sprintf(
+                    'Eval case "%s" was not found. Please select a case from the list for the chosen eval type.',
+                    $caseId
+                ));
+            }
+
+            throw new \RuntimeException(sprintf(
+                'No eval cases available for eval type "%s".',
+                $type
+            ));
        }

        $results = $this->runner->runAll($cases);
@@ -133,6 +144,35 @@ final readonly class EvalAdminService
        return $report;
    }

+
+    /**
+     * @param array<int, EvalCase> $cases
+     * @return array<int, EvalCase>
+     */
+    private function filterCasesById(array $cases, string $caseId): array
+    {
+        return array_values(array_filter(
+            $cases,
+            static fn (EvalCase $case): bool => $case->id === $caseId
+        ));
+    }
+
+    /**
+     * @return array{0:string,1:array<int, EvalCase>}
+     */
+    private function findCasesByIdAcrossTypes(string $caseId): array
+    {
+        foreach (array_keys(self::TYPES) as $candidateType) {
+            $cases = $this->filterCasesById($this->loadCases($candidateType), $caseId);
+
+            if ($cases !== []) {
+                return [$candidateType, $cases];
+            }
+        }
+
+        return ['', []];
+    }
+
    /**
     * @return array<string, mixed>|null
     */