optimize ingesting documents

2026-02-28 22:48:01 +01:00
parent 54ce057ef0
commit 509ba83ac0
6 changed files with 335 additions and 21 deletions
--- a/python/vector/vector_ingest.py
+++ b/python/vector/vector_ingest.py
@@ -96,8 +96,8 @@ print("Encoding embeddings...")
 embeddings = model.encode(
    texts,
    normalize_embeddings=True,
-    show_progress_bar=True,
-    batch_size=64
+    show_progress_bar=False,
+    batch_size=128
 )

 embeddings = np.array(embeddings).astype("float32")
--- a/python/vector/vector_ingest_tags.py
+++ b/python/vector/vector_ingest_tags.py
@@ -109,8 +109,8 @@ if not texts:
 embeddings = model.encode(
    texts,
    normalize_embeddings=True,
-    show_progress_bar=False,
-    batch_size=64
+    show_progress_bar=True,
+    batch_size=128
 )

 embeddings = np.array(embeddings).astype("float32")