Merge branch 'main' into ronnie/list-contains

.github/workflows/build-docs.yml

@@ -0,0 +1,94 @@
+# This workflow will build the Daft docs and optionally publishes them to Netlify
+# for easy previews.
+
+name: daft-docs
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+    types: [opened, synchronize, reopened, labeled]
+
+jobs:
+  docgen:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    env:
+      python-version: '3.10'
+    steps:
+    - uses: actions/checkout@v4
+    - uses: moonrepo/setup-rust@v1
+      with:
+        cache: false
+    - uses: Swatinem/rust-cache@v2
+      with:
+        key: ${{ runner.os }}-build
+        cache-all-crates: 'true'
+    - name: Set up Python ${{ env.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ env.python-version }}
+        cache: pip
+        cache-dependency-path: |
+          pyproject.toml
+          requirements-dev.txt
+    - name: Setup Virtual Env
+      run: |
+        python -m venv venv
+        echo "$GITHUB_WORKSPACE/venv/bin" >> $GITHUB_PATH
+        pip install uv
+    - name: Install dependencies
+      run: |
+        uv pip install -r requirements-dev.txt
+    - name: Build Daft in development mode and generate docs
+      # TODO: Break on any Sphinx warnings with nitpicks
+      # make html SPHINXOPTS="-W --keep-going -n"
+      run: |
+        source activate
+        maturin develop
+        cd docs/
+        make html SPHINXOPTS="-W --keep-going"
+    - name: Upload built docs
+      uses: actions/upload-artifact@v4
+      with:
+        name: html-docs
+        path: docs/build/html
+  docpublish:
+    runs-on: ubuntu-latest
+    needs: docgen
+    # Only publish to netlify on:
+    # 1. If the previous doc building step didn't fail
+    # 2. If this is a PR from the current repo (since it needs access to secrets)
+    # 3. If it has the `documentation` label
+    if: |
+      success() &&
+      (github.event.pull_request != null) &&
+      (github.event.pull_request.head.repo.full_name == github.repository) &&
+      contains(github.event.pull_request.labels.*.name, 'documentation')
+    steps:
+    - name: Download built HTML docs
+      uses: actions/download-artifact@v4
+      with:
+        name: html-docs
+        path: ./built_html
+    - name: Add a robots.txt to disable indexing of docs
+      run: |
+        echo "User-agent: *" > ./built_html/robots.txt
+        echo "Disallow: /" >> ./built_html/robots.txt
+    - name: Deploy to Netlify
+      uses: nwtgck/actions-netlify@v3.0
+      with:
+        publish-dir: ./built_html
+        production-branch: main
+        github-token: ${{ secrets.GITHUB_TOKEN }}
+        enable-pull-request-comment: true
+        enable-commit-comment: false
+        overwrites-pull-request-comment: true
+        enable-github-deployment: false
+        alias: deploy-preview-${{ github.event.number }}
+        deploy-message: 'Deploy preview for PR #${{ github.event.number }} (current built commit: ${{ github.sha }})'
+    env:
+      NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
+      NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }}
+    timeout-minutes: 1

.github/workflows/python-package.yml

@@ -789,7 +789,6 @@ jobs:
         maturin develop
         pytest --doctest-modules --continue-on-collection-errors daft/dataframe/dataframe.py daft/expressions/expressions.py daft/convert.py daft/udf.py
 
-
   publish-coverage-reports:
     name: Publish coverage reports to CodeCov
     runs-on: ubuntu-latest

.pre-commit-config.yaml

@@ -82,7 +82,7 @@ repos:
     language: system
     types: [rust]
     pass_filenames: false
-    args: [--workspace]
+    args: [--workspace, --all-targets]
 
   - id: cargo-check-all-features
     name: cargo check (all features)
@@ -91,7 +91,7 @@ repos:
     language: system
     types: [rust]
     pass_filenames: false
-    args: [--workspace, --all-features]
+    args: [--workspace, --all-features, --all-targets]
 
   - id: clippy
     name: clippy

Cargo.toml

@@ -5,6 +5,7 @@ common-hashable-float-wrapper = {path = "src/common/hashable-float-wrapper", def
 common-resource-request = {path = "src/common/resource-request", default-features = false}
 common-system-info = {path = "src/common/system-info", default-features = false}
 common-tracing = {path = "src/common/tracing", default-features = false}
+common-version = {path = "src/common/version", default-features = false}
 daft-compression = {path = "src/daft-compression", default-features = false}
 daft-core = {path = "src/daft-core", default-features = false}
 daft-csv = {path = "src/daft-csv", default-features = false}
@@ -164,6 +165,7 @@ snafu = {version = "0.7.4", features = ["futures"]}
 sqlparser = "0.49.0"
 sysinfo = "0.30.12"
 test-log = "0.2.16"
+thiserror = "1.0.63"
 tiktoken-rs = "0.5.9"
 tokio = {version = "1.37.0", features = [
   "net",
@@ -186,6 +188,10 @@ path = "src/arrow2"
 [workspace.dependencies.bincode]
 version = "1.3.3"
 
+[workspace.dependencies.derive_more]
+features = ["display"]
+version = "1.0.0"
+
 [workspace.dependencies.lazy_static]
 version = "1.4.0"