first test suite retrieval

This commit is contained in:
team2
2026-04-22 22:22:27 +02:00
parent 8127d33571
commit 47099a4407
4 changed files with 458 additions and 6 deletions

View File

@@ -8,6 +8,7 @@ use App\Eval\AgentEvalRunner;
use App\Eval\Dto\EvalCase;
use App\Eval\Dto\EvalResult;
use App\Eval\EvalCaseLoader;
use App\Eval\EvalReportWriter;
use Symfony\Component\Console\Attribute\AsCommand;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputArgument;
@@ -25,6 +26,7 @@ final class AgentEvalRunCommand extends Command
public function __construct(
private readonly EvalCaseLoader $loader,
private readonly AgentEvalRunner $runner,
private readonly EvalReportWriter $reportWriter,
) {
parent::__construct();
}
@@ -49,6 +51,12 @@ final class AgentEvalRunCommand extends Command
null,
InputOption::VALUE_NONE,
'Print the full report as JSON'
)
->addOption(
'no-write',
null,
InputOption::VALUE_NONE,
'Do not write the report file'
);
}
@@ -59,6 +67,7 @@ final class AgentEvalRunCommand extends Command
$type = trim((string) $input->getArgument('type'));
$caseId = trim((string) $input->getOption('case'));
$asJson = (bool) $input->getOption('json');
$noWrite = (bool) $input->getOption('no-write');
try {
$cases = $this->loader->load($type);
@@ -97,18 +106,38 @@ final class AgentEvalRunCommand extends Command
$report = [
'type' => $type,
'case_filter' => $caseId !== '' ? $caseId : null,
'total' => count($results),
'passed' => $passed,
'failed' => $failed,
'generated_at' => (new \DateTimeImmutable())->format(\DateTimeInterface::ATOM),
'results' => array_map(
static fn (EvalResult $result): array => $result->toArray(),
$results
),
];
$writtenPath = null;
if (!$noWrite) {
try {
$writtenPath = $this->reportWriter->write($report);
} catch (\Throwable $e) {
$io->error($e->getMessage());
return Command::FAILURE;
}
}
if ($asJson) {
$jsonReport = $report;
if ($writtenPath !== null) {
$jsonReport['written_to'] = $writtenPath;
}
$json = json_encode(
$report,
$jsonReport,
JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE
);
@@ -128,7 +157,8 @@ final class AgentEvalRunCommand extends Command
['type' => $type],
['total' => (string) count($results)],
['passed' => (string) $passed],
['failed' => (string) $failed]
['failed' => (string) $failed],
['report_file' => $writtenPath ?? 'disabled (--no-write)']
);
foreach ($results as $result) {