Extract common code paths in indexing pipeline (#2275)

Major refactoring of indexing pipeline (IndexCollection, IndexHnswDenseVectors, and IndexInvertedDenseVectors), extracting common code paths into AbstractIndexer.
castorini · Dec 13, 2023 · b6a7534 · b6a7534
1 parent 6df6c41
commit b6a7534
Show file tree

Hide file tree

Showing 35 changed files with 749 additions and 1,020 deletions.
diff --git a/docs/regressions/regressions-dl19-passage-cos-dpr-distil-hnsw-onnx.md b/docs/regressions/regressions-dl19-passage-cos-dpr-distil-hnsw-onnx.md
@@ -57,7 +57,7 @@ target/appassembler/bin/IndexHnswDenseVectors \
   -input /path/to/msmarco-passage-cos-dpr-distil \
   -generator HnswDenseVectorDocumentGenerator \
   -index indexes/lucene-hnsw.msmarco-passage-cos-dpr-distil/ \
-  -threads 16 -M 16 -efC 100 \
+  -threads 16 -M 16 -efC 100 -memoryBuffer 65536 \
   >& logs/log.msmarco-passage-cos-dpr-distil &
 ```
 

diff --git a/docs/regressions/regressions-dl19-passage-cos-dpr-distil-hnsw.md b/docs/regressions/regressions-dl19-passage-cos-dpr-distil-hnsw.md
@@ -57,7 +57,7 @@ target/appassembler/bin/IndexHnswDenseVectors \
   -input /path/to/msmarco-passage-cos-dpr-distil \
   -generator HnswDenseVectorDocumentGenerator \
   -index indexes/lucene-hnsw.msmarco-passage-cos-dpr-distil/ \
-  -threads 16 -M 16 -efC 100 \
+  -threads 16 -M 16 -efC 100 -memoryBuffer 65536 \
   >& logs/log.msmarco-passage-cos-dpr-distil &
 ```
 

diff --git a/docs/regressions/regressions-dl19-passage-openai-ada2.md b/docs/regressions/regressions-dl19-passage-openai-ada2.md
@@ -57,7 +57,7 @@ target/appassembler/bin/IndexHnswDenseVectors \
   -input /path/to/msmarco-passage-openai-ada2 \
   -generator HnswDenseVectorDocumentGenerator \
   -index indexes/lucene-hnsw.msmarco-passage-openai-ada2/ \
-  -threads 16 -M 16 -efC 100 -memorybuffer 65536 \
+  -threads 16 -M 16 -efC 100 -memoryBuffer 65536 \
   >& logs/log.msmarco-passage-openai-ada2 &
 ```
 

diff --git a/docs/regressions/regressions-dl20-passage-cos-dpr-distil-hnsw-onnx.md b/docs/regressions/regressions-dl20-passage-cos-dpr-distil-hnsw-onnx.md
@@ -57,7 +57,7 @@ target/appassembler/bin/IndexHnswDenseVectors \
   -input /path/to/msmarco-passage-cos-dpr-distil \
   -generator HnswDenseVectorDocumentGenerator \
   -index indexes/lucene-hnsw.msmarco-passage-cos-dpr-distil/ \
-  -threads 16 -M 16 -efC 100 \
+  -threads 16 -M 16 -efC 100 -memoryBuffer 65536 \
   >& logs/log.msmarco-passage-cos-dpr-distil &
 ```
 

diff --git a/docs/regressions/regressions-dl20-passage-cos-dpr-distil-hnsw.md b/docs/regressions/regressions-dl20-passage-cos-dpr-distil-hnsw.md
@@ -57,7 +57,7 @@ target/appassembler/bin/IndexHnswDenseVectors \
   -input /path/to/msmarco-passage-cos-dpr-distil \
   -generator HnswDenseVectorDocumentGenerator \
   -index indexes/lucene-hnsw.msmarco-passage-cos-dpr-distil/ \
-  -threads 16 -M 16 -efC 100 \
+  -threads 16 -M 16 -efC 100 -memoryBuffer 65536 \
   >& logs/log.msmarco-passage-cos-dpr-distil &
 ```
 

diff --git a/docs/regressions/regressions-dl20-passage-openai-ada2.md b/docs/regressions/regressions-dl20-passage-openai-ada2.md
@@ -57,7 +57,7 @@ target/appassembler/bin/IndexHnswDenseVectors \
   -input /path/to/msmarco-passage-openai-ada2 \
   -generator HnswDenseVectorDocumentGenerator \
   -index indexes/lucene-hnsw.msmarco-passage-openai-ada2/ \
-  -threads 16 -M 16 -efC 100 -memorybuffer 65536 \
+  -threads 16 -M 16 -efC 100 -memoryBuffer 65536 \
   >& logs/log.msmarco-passage-openai-ada2 &
 ```
 

diff --git a/docs/regressions/regressions-mb11.md b/docs/regressions/regressions-mb11.md
@@ -25,7 +25,7 @@ target/appassembler/bin/IndexCollection \
   -input /path/to/mb11 \
   -generator TweetGenerator \
   -index indexes/lucene-index.mb11/ \
-  -threads 44 -storePositions -storeDocvectors -storeRaw -uniqueDocid -tweet.keepUrls -tweet.stemming \
+  -threads 44 -storePositions -storeDocvectors -storeRaw -tweet.keepUrls -tweet.stemming \
   >& logs/log.mb11 &
 ```
 

diff --git a/docs/regressions/regressions-msmarco-passage-cos-dpr-distil-hnsw-onnx.md b/docs/regressions/regressions-msmarco-passage-cos-dpr-distil-hnsw-onnx.md
@@ -54,7 +54,7 @@ target/appassembler/bin/IndexHnswDenseVectors \
   -input /path/to/msmarco-passage-cos-dpr-distil \
   -generator HnswDenseVectorDocumentGenerator \
   -index indexes/lucene-hnsw.msmarco-passage-cos-dpr-distil/ \
-  -threads 16 -M 16 -efC 100 \
+  -threads 16 -M 16 -efC 100 -memoryBuffer 65536 \
   >& logs/log.msmarco-passage-cos-dpr-distil &
 ```
 

diff --git a/docs/regressions/regressions-msmarco-passage-cos-dpr-distil-hnsw.md b/docs/regressions/regressions-msmarco-passage-cos-dpr-distil-hnsw.md
@@ -54,7 +54,7 @@ target/appassembler/bin/IndexHnswDenseVectors \
   -input /path/to/msmarco-passage-cos-dpr-distil \
   -generator HnswDenseVectorDocumentGenerator \
   -index indexes/lucene-hnsw.msmarco-passage-cos-dpr-distil/ \
-  -threads 16 -M 16 -efC 100 \
+  -threads 16 -M 16 -efC 100 -memoryBuffer 65536 \
   >& logs/log.msmarco-passage-cos-dpr-distil &
 ```
 

diff --git a/docs/regressions/regressions-msmarco-passage-openai-ada2.md b/docs/regressions/regressions-msmarco-passage-openai-ada2.md
@@ -54,7 +54,7 @@ target/appassembler/bin/IndexHnswDenseVectors \
   -input /path/to/msmarco-passage-openai-ada2 \
   -generator HnswDenseVectorDocumentGenerator \
   -index indexes/lucene-hnsw.msmarco-passage-openai-ada2/ \
-  -threads 16 -M 16 -efC 100 -memorybuffer 65536 \
+  -threads 16 -M 16 -efC 100 -memoryBuffer 65536 \
   >& logs/log.msmarco-passage-openai-ada2 &
 ```