p23

src/Commerce/CommerceQueryParser.php

@@ -7,6 +7,7 @@ namespace App\Commerce;
 use App\Commerce\Dto\CommerceSearchQuery;
 use App\Config\CommerceIntentConfig;
 use App\Config\CommerceQueryParserConfig;
+use App\Config\LanguageCleanupConfig;
 use App\Knowledge\Retrieval\QueryCleaner;
 use App\Knowledge\Text\TextNormalizer;
 
@@ -17,6 +18,7 @@ final readonly class CommerceQueryParser
         private QueryCleaner $queryCleaner,
         private CommerceQueryParserConfig $config,
         private CommerceIntentConfig $intentConfig,
+        private LanguageCleanupConfig $languageCleanupConfig,
     ) {
     }
 
@@ -177,7 +179,7 @@ final readonly class CommerceQueryParser
 
         $text = $this->wrapForPhraseReplacement($prompt);
 
-        foreach ($this->config->getPhrasesToRemove() as $phrase) {
+        foreach ($this->getCommercePhrasesToRemove() as $phrase) {
             $normalizedPhrase = $this->normalize((string) $phrase);
 
             if ($normalizedPhrase === '') {
@@ -500,6 +502,47 @@ final readonly class CommerceQueryParser
         return preg_replace($this->config->getWhitespaceCollapsePattern(), ' ', $text) ?? $text;
     }
 
+    /** @return string[] */
+    private function getCommercePhrasesToRemove(): array
+    {
+        return $this->mergeUniqueTokens(
+            $this->languageCleanupConfig->getPhrasesForProfile($this->config->getCleanupProfile()),
+            $this->config->getPhrasesToRemove()
+        );
+    }
+
+    /** @return string[] */
+    private function getCommerceFilterSearchTokens(): array
+    {
+        return $this->mergeUniqueTokens(
+            $this->languageCleanupConfig->getStopWordsForProfile($this->config->getCleanupProfile()),
+            $this->config->getFilterSearchTokens()
+        );
+    }
+
+    /**
+     * @param string[] $left
+     * @param string[] $right
+     * @return string[]
+     */
+    private function mergeUniqueTokens(array $left, array $right): array
+    {
+        $out = [];
+
+        foreach ([$left, $right] as $list) {
+            foreach ($list as $token) {
+                $token = trim(mb_strtolower((string) $token, 'UTF-8'));
+                if ($token === '' || in_array($token, $out, true)) {
+                    continue;
+                }
+
+                $out[] = $token;
+            }
+        }
+
+        return $out;
+    }
+
     private function isSearchControlToken(string $token): bool
     {
         $token = trim(mb_strtolower($token));
@@ -508,7 +551,7 @@ final readonly class CommerceQueryParser
             return true;
         }
 
-        if (in_array($token, $this->config->getFilterSearchTokens(), true)) {
+        if (in_array($token, $this->getCommerceFilterSearchTokens(), true)) {
             return true;
         }