Add file mode to gdal filesystem handler, move more CI to distributio…

…n pipeline instead
duckdb · Nov 2, 2023 · 9266dba · 9266dba
1 parent 98c7b57
commit 9266dba
Show file tree

Hide file tree

Showing 5 changed files with 107 additions and 89 deletions.
diff --git a/.github/workflows/Linux.yml b/.github/workflows/Linux.yml
@@ -23,7 +23,7 @@ jobs:
     strategy:
       matrix:
         # Add commits/tags to build against other DuckDB versions
-        duckdb_version: [ '<submodule_version>', 'v0.9.1' ]
+        duckdb_version: [ '<submodule_version>'  ]
         arch: ['linux_amd64', 'linux_arm64', 'linux_amd64_gcc4']
         vcpkg_version: [ '2023.04.15' ]
         include:
@@ -114,31 +114,31 @@ jobs:
       run: |
         make test_release
 
-    - uses: actions/upload-artifact@v2
-      with:
-        name: ${{matrix.arch}}-extensions-${{ matrix.duckdb_version == '<submodule_version>' && 'latest' || matrix.duckdb_version }}
-        path: |
-          build/release/extension/spatial/spatial.duckdb_extension
-
-    - name: Deploy
-      env:
-        AWS_ACCESS_KEY_ID: ${{ secrets.S3_DEPLOY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DEPLOY_KEY }}
-        AWS_DEFAULT_REGION: ${{ secrets.S3_REGION }}
-        BUCKET_NAME: ${{ secrets.S3_BUCKET }}
-      run: |
-        git config --global --add safe.directory '*'
-        cd duckdb
-        git fetch --tags
-        export DUCKDB_VERSION=`git tag --points-at HEAD`
-        export DUCKDB_VERSION=${DUCKDB_VERSION:=`git log -1 --format=%h`}
-        cd ..
-        if [[ "$AWS_ACCESS_KEY_ID" == "" ]] ; then
-          echo 'No key set, skipping'
-        elif [[ "$GITHUB_REF" =~ ^(refs/tags/v.+)$ ]] ; then
-          python3 -m pip install pip awscli
-          ./scripts/extension-upload.sh spatial ${{ github.ref_name }} $DUCKDB_VERSION ${{matrix.arch}} $BUCKET_NAME true
-        elif [[ "$GITHUB_REF" =~ ^(refs/heads/main)$ ]] ; then
-          python3 -m pip install pip awscli
-          ./scripts/extension-upload.sh spatial `git log -1 --format=%h` $DUCKDB_VERSION ${{matrix.arch}} $BUCKET_NAME false
-        fi
+#    - uses: actions/upload-artifact@v2
+#      with:
+#        name: ${{matrix.arch}}-extensions-${{ matrix.duckdb_version == '<submodule_version>' && 'latest' || matrix.duckdb_version }}
+#        path: |
+#          build/release/extension/spatial/spatial.duckdb_extension
+#
+#    - name: Deploy
+#      env:
+#        AWS_ACCESS_KEY_ID: ${{ secrets.S3_DEPLOY_ID }}
+#        AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DEPLOY_KEY }}
+#        AWS_DEFAULT_REGION: ${{ secrets.S3_REGION }}
+#        BUCKET_NAME: ${{ secrets.S3_BUCKET }}
+#      run: |
+#        git config --global --add safe.directory '*'
+#        cd duckdb
+#        git fetch --tags
+#        export DUCKDB_VERSION=`git tag --points-at HEAD`
+#        export DUCKDB_VERSION=${DUCKDB_VERSION:=`git log -1 --format=%h`}
+#        cd ..
+#        if [[ "$AWS_ACCESS_KEY_ID" == "" ]] ; then
+#          echo 'No key set, skipping'
+#        elif [[ "$GITHUB_REF" =~ ^(refs/tags/v.+)$ ]] ; then
+#          python3 -m pip install pip awscli
+#          ./scripts/extension-upload.sh spatial ${{ github.ref_name }} $DUCKDB_VERSION ${{matrix.arch}} $BUCKET_NAME true
+#        elif [[ "$GITHUB_REF" =~ ^(refs/heads/main)$ ]] ; then
+#          python3 -m pip install pip awscli
+#          ./scripts/extension-upload.sh spatial `git log -1 --format=%h` $DUCKDB_VERSION ${{matrix.arch}} $BUCKET_NAME false
+#        fi
diff --git a/.github/workflows/MacOS.yml b/.github/workflows/MacOS.yml
@@ -22,7 +22,7 @@ jobs:
     strategy:
       matrix:
         # Add commits/tags to build against other DuckDB versions
-        duckdb_version: [ '<submodule_version>', 'v0.9.1' ]
+        duckdb_version: [ '<submodule_version>' ]
         vcpkg_version: [ '2023.04.15' ]
         vcpkg_triplet: [ 'x64-osx', 'arm64-osx' ]
         include:
@@ -83,31 +83,31 @@ jobs:
           make test
 
       # Upload artefact
-      - uses: actions/upload-artifact@v2
-        with:
-          name: osx-${{ matrix.osx_build_arch }}-extension-${{ matrix.duckdb_version == '<submodule_version>' && 'latest' || matrix.duckdb_version }}
-          path: |
-            build/release/extension/spatial/spatial.duckdb_extension
-
-      - name: Deploy
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.S3_DEPLOY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DEPLOY_KEY }}
-          AWS_DEFAULT_REGION: ${{ secrets.S3_REGION }}
-          BUCKET_NAME: ${{ secrets.S3_BUCKET }}
-        run: |
-          git config --global --add safe.directory '*'
-          cd duckdb
-          git fetch --tags
-          export DUCKDB_VERSION=`git tag --points-at HEAD`
-          export DUCKDB_VERSION=${DUCKDB_VERSION:=`git log -1 --format=%h`}
-          cd ..
-          if [[ "$AWS_ACCESS_KEY_ID" == "" ]] ; then
-            echo 'No key set, skipping'
-          elif [[ "$GITHUB_REF" =~ ^(refs/tags/v.+)$ ]] ; then
-            python3 -m pip install pip awscli
-            ./scripts/extension-upload.sh spatial ${{ github.ref_name }} $DUCKDB_VERSION ${{matrix.duckdb_arch}} $BUCKET_NAME true
-          elif [[ "$GITHUB_REF" =~ ^(refs/heads/main)$ ]] ; then
-            python3 -m pip install pip awscli
-            ./scripts/extension-upload.sh spatial `git log -1 --format=%h` $DUCKDB_VERSION ${{matrix.duckdb_arch}} $BUCKET_NAME false
-          fi
+#      - uses: actions/upload-artifact@v2
+#        with:
+#          name: osx-${{ matrix.osx_build_arch }}-extension-${{ matrix.duckdb_version == '<submodule_version>' && 'latest' || matrix.duckdb_version }}
+#          path: |
+#            build/release/extension/spatial/spatial.duckdb_extension
+#
+#      - name: Deploy
+#        env:
+#          AWS_ACCESS_KEY_ID: ${{ secrets.S3_DEPLOY_ID }}
+#          AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DEPLOY_KEY }}
+#          AWS_DEFAULT_REGION: ${{ secrets.S3_REGION }}
+#          BUCKET_NAME: ${{ secrets.S3_BUCKET }}
+#        run: |
+#          git config --global --add safe.directory '*'
+#          cd duckdb
+#          git fetch --tags
+#          export DUCKDB_VERSION=`git tag --points-at HEAD`
+#          export DUCKDB_VERSION=${DUCKDB_VERSION:=`git log -1 --format=%h`}
+#          cd ..
+#          if [[ "$AWS_ACCESS_KEY_ID" == "" ]] ; then
+#            echo 'No key set, skipping'
+#          elif [[ "$GITHUB_REF" =~ ^(refs/tags/v.+)$ ]] ; then
+#            python3 -m pip install pip awscli
+#            ./scripts/extension-upload.sh spatial ${{ github.ref_name }} $DUCKDB_VERSION ${{matrix.duckdb_arch}} $BUCKET_NAME true
+#          elif [[ "$GITHUB_REF" =~ ^(refs/heads/main)$ ]] ; then
+#            python3 -m pip install pip awscli
+#            ./scripts/extension-upload.sh spatial `git log -1 --format=%h` $DUCKDB_VERSION ${{matrix.duckdb_arch}} $BUCKET_NAME false
+#          fi
diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml
@@ -3,11 +3,12 @@
 #
 name: Main Extension Distribution Pipeline
 on:
+  pull_request:
+    paths-ignore:
+      - '**/README.md'
   push:
     paths-ignore:
       - '**/README.md'
-    branches:
-      - main
   workflow_dispatch:
 
 concurrency:

diff --git a/.github/workflows/Windows.yml b/.github/workflows/Windows.yml
@@ -22,7 +22,7 @@ jobs:
     strategy:
       matrix:
         # Add commits/tags to build against other DuckDB versions
-        duckdb_version: [ '<submodule_version>', 'v0.9.1' ]
+        duckdb_version: [ '<submodule_version>' ]
         vcpkg_version: [ '2023.04.15' ]
         vcpkg_triplet: ['x64-windows']
     env:
@@ -68,30 +68,30 @@ jobs:
           # and the build fails. Fixing this is a bit more involved so for now we just disable building the shell on windows.
           BUILD_SHELL: 0
 
-      - uses: actions/upload-artifact@v2
-        with:
-          name: windows-amd64-extensions-${{ matrix.duckdb_version == '<submodule_version>' && 'latest' || matrix.duckdb_version }}
-          path: |
-            build/release/extension/spatial/spatial.duckdb_extension
-
-      - name: Deploy
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.S3_DEPLOY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DEPLOY_KEY }}
-          AWS_DEFAULT_REGION: ${{ secrets.S3_REGION }}
-          BUCKET_NAME: ${{ secrets.S3_BUCKET }}
-        run: |
-          cd duckdb
-          git fetch --tags
-          export DUCKDB_VERSION=`git tag --points-at HEAD`
-          export DUCKDB_VERSION=${DUCKDB_VERSION:=`git log -1 --format=%h`}
-          cd ..
-          if [[ "$AWS_ACCESS_KEY_ID" == "" ]] ; then
-            echo 'No key set, skipping'
-          elif [[ "$GITHUB_REF" =~ ^(refs/tags/v.+)$ ]] ; then
-            python -m pip install awscli
-            ./scripts/extension-upload.sh spatial ${{ github.ref_name }} $DUCKDB_VERSION windows_amd64 $BUCKET_NAME true 
-          elif [[ "$GITHUB_REF" =~ ^(refs/heads/main)$ ]] ; then
-            python -m pip install awscli
-            ./scripts/extension-upload.sh spatial `git log -1 --format=%h` $DUCKDB_VERSION windows_amd64 $BUCKET_NAME false 
-          fi
+#      - uses: actions/upload-artifact@v2
+#        with:
+#          name: windows-amd64-extensions-${{ matrix.duckdb_version == '<submodule_version>' && 'latest' || matrix.duckdb_version }}
+#          path: |
+#            build/release/extension/spatial/spatial.duckdb_extension
+#
+#      - name: Deploy
+#        env:
+#          AWS_ACCESS_KEY_ID: ${{ secrets.S3_DEPLOY_ID }}
+#          AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DEPLOY_KEY }}
+#          AWS_DEFAULT_REGION: ${{ secrets.S3_REGION }}
+#          BUCKET_NAME: ${{ secrets.S3_BUCKET }}
+#        run: |
+#          cd duckdb
+#          git fetch --tags
+#          export DUCKDB_VERSION=`git tag --points-at HEAD`
+#          export DUCKDB_VERSION=${DUCKDB_VERSION:=`git log -1 --format=%h`}
+#          cd ..
+#          if [[ "$AWS_ACCESS_KEY_ID" == "" ]] ; then
+#            echo 'No key set, skipping'
+#          elif [[ "$GITHUB_REF" =~ ^(refs/tags/v.+)$ ]] ; then
+#            python -m pip install awscli
+#            ./scripts/extension-upload.sh spatial ${{ github.ref_name }} $DUCKDB_VERSION windows_amd64 $BUCKET_NAME true
+#          elif [[ "$GITHUB_REF" =~ ^(refs/heads/main)$ ]] ; then
+#            python -m pip install awscli
+#            ./scripts/extension-upload.sh spatial `git log -1 --format=%h` $DUCKDB_VERSION windows_amd64 $BUCKET_NAME false
+#          fi
diff --git a/spatial/src/spatial/gdal/file_handler.cpp b/spatial/src/spatial/gdal/file_handler.cpp
@@ -196,9 +196,26 @@ static int DuckDBStatCallback(void *userData, const char *filename, VSIStatBufL
 	}
 
 	pstatbuf->st_size = static_cast<off_t>(file->GetFileSize());
+
+	auto type = file->GetType();
+	switch (type) {
+	// Thesea re the only three types present on all platforms
+	case FileType::FILE_TYPE_REGULAR:
+		pstatbuf->st_mode = S_IFREG;
+		break;
+	case FileType::FILE_TYPE_DIR:
+		pstatbuf->st_mode = S_IFDIR;
+		break;
+	case FileType::FILE_TYPE_CHARDEV:
+		pstatbuf->st_mode = S_IFCHR;
+		break;
+	default:
+		throw IOException("Unknown file type");
+	}
+
+
 	/* DuckDB doesnt have anything equivalent to these yet... Hopefully thats ok?
 	pstatbuf->st_mtime = file->GetLastModifiedTime();
-	pstatbuf->st_mode = file->GetFileMode();
 	pstatbuf->st_uid = file->GetFileOwner();
 	pstatbuf->st_gid = file->GetFileGroup();
 	pstatbuf->st_ino = file->GetFileInode();