feat: lutra query runner (#4134)

Co-authored-by: Maximilian Roos <m@maxroos.com>

.github/workflows/tests.yaml

@@ -525,8 +525,7 @@ jobs:
         with:
           save-if: ${{ github.ref == 'refs/heads/main' }}
       - name: Verify minimum rust version
-        run: cargo minimal-versions test
-      - run: cargo minimal-versions test --direct
+        run: cargo minimal-versions test --direct
 
   check-ok-to-merge:
     # This indicates to GitHub whether everything in this workflow has passed

Cargo.toml

@@ -11,6 +11,7 @@ members = [
   "prqlc/prqlc-parser",
   "prqlc/prqlc",
   "prqlc/prqlc/examples/compile-files", # An example
+  "lutra/lutra",
   "web/book", # The book / docs
 ]
 resolver = "2"

Taskfile.yml

@@ -21,6 +21,9 @@ includes:
   zig:
     taskfile: ./prqlc/bindings/prqlc-c/examples/minimal-zig
     dir: ./prqlc/bindings/prqlc-c/examples/minimal-zig
+  lutra:
+    taskfile: ./lutra
+    dir: ./lutra
 
 vars:
   # Keep in sync with .vscode/extensions.json

lutra/.gitignore

@@ -0,0 +1,3 @@
+example-project/**/*.parquet
+
+.vscode/

lutra/README.md

@@ -0,0 +1,14 @@
+# Lutra
+
+Query runner for PRQL.
+
+Status: early experimental development.
+
+As prqlc provides conversion from PRQL source to SQL source, Lutra aims to
+provide conversion from PRQL source to relational data.
+
+For this to happen, PRQL source needs to include additional annotations (i.e.
+`@lutra.sqlite`) that define data sources that contain the source data and can
+execute SQL queries.
+
+Lutra can be used as CLI (the binary in `lutra` crate) or as a library.

lutra/Taskfile.yml

@@ -0,0 +1,57 @@
+version: "3"
+
+vars:
+  packages: -p lutra
+
+tasks:
+  run:
+    cmds:
+      - cmd: cargo run --package lutra -- execute example-project
+
+  fmt:
+    desc: Format prqlc source files
+    cmds:
+      - cmd: |
+          # remove trailing whitespace
+          rg '\s+$' --files-with-matches --glob '!*.{rs,snap}' . \
+          | xargs -I _ sh -c "echo Removing trailing whitespace from _ && sd '[\t ]+$' '' _"
+
+      - cmd: |
+          # rustfmt
+          cargo fmt {{.packages}}
+
+      - cmd: |
+          # no dbg
+          rg 'dbg!' --glob '*.rs' . --no-heading
+        ignore_error: true
+
+  test-fast:
+    desc: A fast test used for feedback during development
+    cmds:
+      - cmd: |
+          INSTA_FORCE_PASS=1 cargo nextest run {{.packages}} --no-fail-fast
+
+      - cmd: cargo insta review
+
+      - cmd: cargo clippy --all-targets {{.packages}}
+
+  test:
+    desc: The full test. Generates coverage report.
+    env:
+      # Use a different target dir so we don't poison the cache
+      CARGO_LLVM_COV_TARGET_DIR: ../target-cov
+    cmds:
+      - cmd: |
+          cargo \
+            llvm-cov --lcov --output-path lcov.info \
+            nextest \
+            {{.packages}}
+
+      - cmd:
+          cargo clippy --all-targets {{.packages}} -- -D warnings
+
+  pull-request:
+    desc: Most checks that run within GH actions for a pull request
+    cmds:
+      - task: fmt
+      - task: test

lutra/example-project/Project.prql

@@ -0,0 +1,19 @@
+
+## This module is configured to pull data from an SQLite chinook.db database
+@(lutra.sqlite {file="chinook.db"})
+module chinook {
+  let artists <[{artist_id = int, name = text}]>
+
+  let genres <[{genre_id = int, name = text}]>
+}
+
+let favourite_artists = [
+  {artist_id = 120, last_listen = @2023-05-18},
+  {artist_id = 7, last_listen = @2023-05-16},
+]
+
+from favourite_artists
+join input = (from chinook.artists | select {artist_id, name}) side:left (==artist_id)
+derive aid = favourite_artists.artist_id * 2
+select {aid, input.name, favourite_artists.last_listen}
+sort {-aid}

lutra/example-project/genres.prql

@@ -0,0 +1,7 @@
+let x = [
+  {genre_id = 1, title = "Rock"},
+  {genre_id = 2, title = "Blues"},
+]
+
+x
+into main

lutra/lutra/Cargo.toml

@@ -0,0 +1,33 @@
+[package]
+description = "Query runner for PRQL"
+name = "lutra"
+
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+version.workspace = true
+
+[features]
+cli = ["clap"]
+default = ["cli"]
+
+[[bin]]
+name = "lutra"
+path = "src/cli.rs"
+required-features = ["cli"]
+
+[target.'cfg(not(target_family="wasm"))'.dependencies]
+anyhow = "1.0.79"
+clap = { version = "4.4.18", features = ["derive"], optional = true }
+connector_arrow = { version = "0.1.1", features = ["src_sqlite"] }
+rusqlite = { version = "0.30.0", features = ["bundled"] }
+arrow = { version = "49.0", features = ["prettyprint"] }
+env_logger = "0.10.2"
+log = "0.4.20"
+prqlc = { path = "../../prqlc/prqlc", default-features = false }
+walkdir = "2.4.0"
+
+[target.'cfg(not(target_family="wasm"))'.dev-dependencies]
+insta = { version = "1.34", features = ["colors"] }
+itertools = "0.12.0"