first test suite retrieval
This commit is contained in:
team2
@@ -33,9 +33,13 @@ final readonly class RetrievalDebugRunner
|
||||
|
||||
$documentIds = $this->extractUniqueStringValues($rows, 'document_id');
|
||||
$chunkIds = $this->extractUniqueStringValues($rows, 'chunk_id');
|
||||
$joinedText = $this->extractJoinedText($rows);
|
||||
|
||||
$assert = $case->assert;
|
||||
|
||||
// ---------------------------------------------------------
|
||||
// Strict single-value assertions
|
||||
// ---------------------------------------------------------
|
||||
if (isset($assert['selection_mode']) && (string) $assert['selection_mode'] !== $selectionMode) {
|
||||
$failures[] = sprintf(
|
||||
'selection_mode mismatch: expected "%s", got "%s".',
|
||||
@@ -60,6 +64,33 @@ final readonly class RetrievalDebugRunner
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------
|
||||
// Flexible multi-value assertions
|
||||
// ---------------------------------------------------------
|
||||
$this->assertValueInList(
|
||||
failures: $failures,
|
||||
actual: $selectionMode,
|
||||
expectedList: $assert['selection_mode_in'] ?? [],
|
||||
label: 'selection_mode'
|
||||
);
|
||||
|
||||
$this->assertValueInList(
|
||||
failures: $failures,
|
||||
actual: $route,
|
||||
expectedList: $assert['route_in'] ?? [],
|
||||
label: 'route'
|
||||
);
|
||||
|
||||
$this->assertValueInList(
|
||||
failures: $failures,
|
||||
actual: $intent,
|
||||
expectedList: $assert['intent_in'] ?? [],
|
||||
label: 'intent'
|
||||
);
|
||||
|
||||
// ---------------------------------------------------------
|
||||
// Result count assertions
|
||||
// ---------------------------------------------------------
|
||||
if (isset($assert['min_results']) && $resultCount < (int) $assert['min_results']) {
|
||||
$failures[] = sprintf(
|
||||
'result_count too low: expected >= %d, got %d.',
|
||||
@@ -76,6 +107,9 @@ final readonly class RetrievalDebugRunner
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------
|
||||
// ID assertions
|
||||
// ---------------------------------------------------------
|
||||
foreach ($this->normalizeStringList($assert['must_include_document_ids'] ?? []) as $expectedDocumentId) {
|
||||
if (!in_array($expectedDocumentId, $documentIds, true)) {
|
||||
$failures[] = sprintf(
|
||||
@@ -94,6 +128,65 @@ final readonly class RetrievalDebugRunner
|
||||
}
|
||||
}
|
||||
|
||||
$this->assertContainsAtLeastOne(
|
||||
failures: $failures,
|
||||
actualValues: $documentIds,
|
||||
expectedList: $assert['must_include_one_of_document_ids'] ?? [],
|
||||
label: 'document_id'
|
||||
);
|
||||
|
||||
$this->assertContainsAtLeastOne(
|
||||
failures: $failures,
|
||||
actualValues: $chunkIds,
|
||||
expectedList: $assert['must_include_one_of_chunk_ids'] ?? [],
|
||||
label: 'chunk_id'
|
||||
);
|
||||
|
||||
$this->assertContainsNone(
|
||||
failures: $failures,
|
||||
actualValues: $documentIds,
|
||||
forbiddenList: $assert['must_not_include_document_ids'] ?? [],
|
||||
label: 'document_id'
|
||||
);
|
||||
|
||||
$this->assertContainsNone(
|
||||
failures: $failures,
|
||||
actualValues: $chunkIds,
|
||||
forbiddenList: $assert['must_not_include_chunk_ids'] ?? [],
|
||||
label: 'chunk_id'
|
||||
);
|
||||
|
||||
// ---------------------------------------------------------
|
||||
// Text / term assertions
|
||||
// ---------------------------------------------------------
|
||||
$matchedAnyTerms = $this->findMatchingTerms(
|
||||
haystack: $joinedText,
|
||||
terms: $this->normalizeStringList($assert['must_include_any_terms'] ?? [])
|
||||
);
|
||||
|
||||
$matchedAllTerms = $this->findMatchingTerms(
|
||||
haystack: $joinedText,
|
||||
terms: $this->normalizeStringList($assert['must_include_all_terms'] ?? [])
|
||||
);
|
||||
|
||||
$requiredAnyTerms = $this->normalizeStringList($assert['must_include_any_terms'] ?? []);
|
||||
if ($requiredAnyTerms !== [] && $matchedAnyTerms === []) {
|
||||
$failures[] = sprintf(
|
||||
'none of the required any-terms were found in the retrieval text: [%s].',
|
||||
implode(', ', $requiredAnyTerms)
|
||||
);
|
||||
}
|
||||
|
||||
$requiredAllTerms = $this->normalizeStringList($assert['must_include_all_terms'] ?? []);
|
||||
foreach ($requiredAllTerms as $requiredTerm) {
|
||||
if (!$this->containsTerm($joinedText, $requiredTerm)) {
|
||||
$failures[] = sprintf(
|
||||
'required all-term "%s" was not found in the retrieval text.',
|
||||
$requiredTerm
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return new EvalResult(
|
||||
caseId: $case->id,
|
||||
type: $case->type,
|
||||
@@ -108,6 +201,8 @@ final readonly class RetrievalDebugRunner
|
||||
'intent' => $intent,
|
||||
'document_ids' => $documentIds,
|
||||
'chunk_ids' => $chunkIds,
|
||||
'matched_any_terms' => $matchedAnyTerms,
|
||||
'matched_all_terms' => $matchedAllTerms,
|
||||
],
|
||||
);
|
||||
}
|
||||
@@ -153,6 +248,161 @@ final readonly class RetrievalDebugRunner
|
||||
return array_keys($values);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<int, array<string, mixed>> $rows
|
||||
*/
|
||||
private function extractJoinedText(array $rows): string
|
||||
{
|
||||
$parts = [];
|
||||
|
||||
foreach ($rows as $row) {
|
||||
$text = $row['text'] ?? null;
|
||||
|
||||
if (!is_string($text)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$text = trim($text);
|
||||
|
||||
if ($text === '') {
|
||||
continue;
|
||||
}
|
||||
|
||||
$parts[] = $text;
|
||||
}
|
||||
|
||||
return implode("\n\n", $parts);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<int, string> $failures
|
||||
* @param mixed $expectedList
|
||||
*/
|
||||
private function assertValueInList(
|
||||
array &$failures,
|
||||
string $actual,
|
||||
mixed $expectedList,
|
||||
string $label
|
||||
): void {
|
||||
$expected = $this->normalizeStringList($expectedList);
|
||||
|
||||
if ($expected === []) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!in_array($actual, $expected, true)) {
|
||||
$failures[] = sprintf(
|
||||
'%s mismatch: expected one of [%s], got "%s".',
|
||||
$label,
|
||||
implode(', ', $expected),
|
||||
$actual
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<int, string> $failures
|
||||
* @param array<int, string> $actualValues
|
||||
* @param mixed $expectedList
|
||||
*/
|
||||
private function assertContainsAtLeastOne(
|
||||
array &$failures,
|
||||
array $actualValues,
|
||||
mixed $expectedList,
|
||||
string $label
|
||||
): void {
|
||||
$expected = $this->normalizeStringList($expectedList);
|
||||
|
||||
if ($expected === []) {
|
||||
return;
|
||||
}
|
||||
|
||||
foreach ($expected as $candidate) {
|
||||
if (in_array($candidate, $actualValues, true)) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
$failures[] = sprintf(
|
||||
'none of the expected %s values were found. Expected one of [%s], got [%s].',
|
||||
$label,
|
||||
implode(', ', $expected),
|
||||
implode(', ', $actualValues)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<int, string> $failures
|
||||
* @param array<int, string> $actualValues
|
||||
* @param mixed $forbiddenList
|
||||
*/
|
||||
private function assertContainsNone(
|
||||
array &$failures,
|
||||
array $actualValues,
|
||||
mixed $forbiddenList,
|
||||
string $label
|
||||
): void {
|
||||
$forbidden = $this->normalizeStringList($forbiddenList);
|
||||
|
||||
if ($forbidden === []) {
|
||||
return;
|
||||
}
|
||||
|
||||
foreach ($forbidden as $forbiddenValue) {
|
||||
if (in_array($forbiddenValue, $actualValues, true)) {
|
||||
$failures[] = sprintf(
|
||||
'forbidden %s "%s" was present in the retrieval results.',
|
||||
$label,
|
||||
$forbiddenValue
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param array<int, string> $terms
|
||||
* @return array<int, string>
|
||||
*/
|
||||
private function findMatchingTerms(string $haystack, array $terms): array
|
||||
{
|
||||
$matches = [];
|
||||
|
||||
foreach ($terms as $term) {
|
||||
if ($this->containsTerm($haystack, $term)) {
|
||||
$matches[] = $term;
|
||||
}
|
||||
}
|
||||
|
||||
return array_values(array_unique($matches));
|
||||
}
|
||||
|
||||
private function containsTerm(string $haystack, string $term): bool
|
||||
{
|
||||
$haystack = $this->normalizeText($haystack);
|
||||
$term = $this->normalizeText($term);
|
||||
|
||||
if ($term === '') {
|
||||
return false;
|
||||
}
|
||||
|
||||
return str_contains($haystack, $term);
|
||||
}
|
||||
|
||||
private function normalizeText(string $value): string
|
||||
{
|
||||
$value = trim($value);
|
||||
|
||||
if ($value === '') {
|
||||
return '';
|
||||
}
|
||||
|
||||
if (function_exists('mb_strtolower')) {
|
||||
return mb_strtolower($value);
|
||||
}
|
||||
|
||||
return strtolower($value);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param mixed $value
|
||||
* @return array<int, string>
|
||||
|
||||
Reference in New Issue
Block a user