we should also extrac the tar after download

doupache · Sep 17, 2024 · 0bf209e · 0bf209e
1 parent dced3af
commit 0bf209e
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/benchmarks/bench.sh b/benchmarks/bench.sh
@@ -493,6 +493,10 @@ data_imdb() {
 
             # Download the dataset
             curl -o "${imdb_temp_gz}" "${imdb_url}"
+
+            # Extract the dataset
+            tar -xzvf "${imdb_temp_gz}" -C "${imdb_dir}"
+            $CARGO_COMMAND --bin imdb -- convert --input ${imdb_dir} --output ${imdb_dir} --format parquet
         else 
             echo "IMDB.tgz already exists."