Add configurable query preprocessing #1552
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI Tests | |
| on: [ push, pull_request ] | |
| jobs: | |
| create-archive: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: true | |
| - uses: actions/cache@v4 | |
| with: | |
| path: | | |
| data/country_osm_grid.sql.gz | |
| key: nominatim-country-data-1 | |
| - name: Package tarball | |
| run: | | |
| if [ ! -f data/country_osm_grid.sql.gz ]; then | |
| wget --no-verbose -O data/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz | |
| fi | |
| cd .. | |
| tar czf nominatim-src.tar.bz2 Nominatim | |
| mv nominatim-src.tar.bz2 Nominatim | |
| - name: 'Upload Artifact' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: full-source | |
| path: nominatim-src.tar.bz2 | |
| retention-days: 1 | |
| tests: | |
| needs: create-archive | |
| strategy: | |
| matrix: | |
| flavour: ["ubuntu-20", "ubuntu-24"] | |
| include: | |
| - flavour: ubuntu-20 | |
| ubuntu: 20 | |
| postgresql: 12 | |
| lua: '5.1' | |
| dependencies: pip | |
| - flavour: ubuntu-24 | |
| ubuntu: 24 | |
| postgresql: 17 | |
| lua: '5.3' | |
| dependencies: apt | |
| runs-on: ubuntu-${{ matrix.ubuntu }}.04 | |
| steps: | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: full-source | |
| - name: Unpack Nominatim | |
| run: tar xf nominatim-src.tar.bz2 | |
| - uses: ./Nominatim/.github/actions/setup-postgresql | |
| with: | |
| postgresql-version: ${{ matrix.postgresql }} | |
| - uses: ./Nominatim/.github/actions/build-nominatim | |
| with: | |
| dependencies: ${{ matrix.dependencies }} | |
| - name: Compile osm2pgsql | |
| run: | | |
| sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev libicu-dev liblua${LUA_VERSION}-dev lua-dkjson nlohmann-json3-dev | |
| mkdir osm2pgsql-build | |
| cd osm2pgsql-build | |
| git clone https://github.com/osm2pgsql-dev/osm2pgsql | |
| mkdir build | |
| cd build | |
| cmake ../osm2pgsql | |
| make | |
| sudo make install | |
| cd ../.. | |
| rm -rf osm2pgsql-build | |
| if: matrix.ubuntu == '20' | |
| env: | |
| LUA_VERSION: ${{ matrix.lua }} | |
| - name: Install test prerequisites | |
| run: ./venv/bin/pip install behave==1.2.6 | |
| - name: Install test prerequisites (apt) | |
| run: sudo apt-get install -y -qq python3-pytest python3-pytest-asyncio uvicorn python3-falcon python3-aiosqlite python3-pyosmium | |
| if: matrix.dependencies == 'apt' | |
| - name: Install test prerequisites (pip) | |
| run: ./venv/bin/pip install pytest-asyncio falcon starlette asgi_lifespan aiosqlite osmium uvicorn | |
| if: matrix.dependencies == 'pip' | |
| - name: Install latest flake8 | |
| run: ./venv/bin/pip install -U flake8 | |
| - name: Python linting | |
| run: ../venv/bin/python -m flake8 src | |
| working-directory: Nominatim | |
| - name: Install mypy and typechecking info | |
| run: ./venv/bin/pip install -U mypy types-PyYAML types-jinja2 types-psutil types-requests types-ujson types-Pygments typing-extensions | |
| if: matrix.dependencies == 'pip' | |
| - name: Python static typechecking | |
| run: ../venv/bin/python -m mypy --strict --python-version 3.8 src | |
| working-directory: Nominatim | |
| if: matrix.dependencies == 'pip' | |
| - name: Python unit tests | |
| run: ../venv/bin/python -m pytest test/python | |
| working-directory: Nominatim | |
| - name: BDD tests | |
| run: | | |
| ../../../venv/bin/python -m behave -DREMOVE_TEMPLATE=1 --format=progress3 | |
| working-directory: Nominatim/test/bdd | |
| install: | |
| runs-on: ubuntu-latest | |
| needs: create-archive | |
| strategy: | |
| matrix: | |
| name: [Ubuntu-22, Ubuntu-24] | |
| include: | |
| - name: Ubuntu-22 | |
| image: "ubuntu:22.04" | |
| ubuntu: 22 | |
| install_mode: install-apache | |
| - name: Ubuntu-24 | |
| image: "ubuntu:24.04" | |
| ubuntu: 24 | |
| install_mode: install-apache | |
| container: | |
| image: ${{ matrix.image }} | |
| env: | |
| LANG: en_US.UTF-8 | |
| defaults: | |
| run: | |
| shell: sudo -Hu nominatim bash --noprofile --norc -eo pipefail {0} | |
| steps: | |
| - name: Prepare container (Ubuntu) | |
| run: | | |
| export APT_LISTCHANGES_FRONTEND=none | |
| export DEBIAN_FRONTEND=noninteractive | |
| apt-get update -qq | |
| apt-get install -y git sudo wget | |
| ln -snf /usr/share/zoneinfo/$CONTAINER_TIMEZONE /etc/localtime && echo $CONTAINER_TIMEZONE > /etc/timezone | |
| shell: bash | |
| - name: Setup import user | |
| run: | | |
| useradd -m nominatim | |
| echo 'nominatim ALL=(ALL:ALL) NOPASSWD: ALL' > /etc/sudoers.d/nominiatim | |
| echo "/home/nominatim/Nominatim/vagrant/Install-on-${OS}.sh no $INSTALL_MODE" > /home/nominatim/vagrant.sh | |
| shell: bash | |
| env: | |
| OS: ${{ matrix.name }} | |
| INSTALL_MODE: ${{ matrix.install_mode }} | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: full-source | |
| path: /home/nominatim | |
| - name: Install Nominatim | |
| run: | | |
| export USERNAME=nominatim | |
| export USERHOME=/home/nominatim | |
| export NOSYSTEMD=yes | |
| export HAVE_SELINUX=no | |
| tar xf nominatim-src.tar.bz2 | |
| . vagrant.sh | |
| working-directory: /home/nominatim | |
| - name: Prepare import environment | |
| run: | | |
| mv Nominatim/test/testdb/apidb-test-data.pbf test.pbf | |
| mv Nominatim/settings/flex-base.lua flex-base.lua | |
| mv Nominatim/settings/import-extratags.lua import-extratags.lua | |
| mv Nominatim/settings/taginfo.lua taginfo.lua | |
| rm -rf Nominatim | |
| mkdir data-env-reverse | |
| working-directory: /home/nominatim | |
| - name: Add nominatim to path | |
| run: | | |
| sudo ln -s /home/nominatim/nominatim-venv/bin/nominatim /usr/local/bin/nominatim | |
| - name: Need lua binary | |
| run: | | |
| sudo apt-get install -y lua5.4 lua-dkjson | |
| - name: Print version | |
| run: nominatim --version | |
| working-directory: /home/nominatim/nominatim-project | |
| - name: Print taginfo | |
| run: lua taginfo.lua | |
| working-directory: /home/nominatim | |
| - name: Collect host OS information | |
| run: nominatim admin --collect-os-info | |
| working-directory: /home/nominatim/nominatim-project | |
| - name: Import | |
| run: nominatim import --osm-file ../test.pbf | |
| working-directory: /home/nominatim/nominatim-project | |
| - name: Import special phrases | |
| run: nominatim special-phrases --import-from-wiki | |
| working-directory: /home/nominatim/nominatim-project | |
| - name: Check full import | |
| run: nominatim admin --check-database | |
| working-directory: /home/nominatim/nominatim-project | |
| - name: Warm up database | |
| run: nominatim admin --warm | |
| working-directory: /home/nominatim/nominatim-project | |
| - name: Install osmium | |
| run: | | |
| /home/nominatim/nominatim-venv/bin/pip install osmium | |
| - name: Run update | |
| run: | | |
| nominatim replication --init | |
| NOMINATIM_REPLICATION_MAX_DIFF=1 nominatim replication --once | |
| working-directory: /home/nominatim/nominatim-project | |
| - name: Clean up database | |
| run: nominatim refresh --postcodes --word-tokens | |
| working-directory: /home/nominatim/nominatim-project | |
| - name: Run reverse-only import | |
| run : | | |
| echo 'NOMINATIM_DATABASE_DSN="pgsql:dbname=reverse"' >> .env | |
| nominatim import --osm-file ../test.pbf --reverse-only --no-updates | |
| working-directory: /home/nominatim/data-env-reverse | |
| - name: Check reverse-only import | |
| run: nominatim admin --check-database | |
| working-directory: /home/nominatim/data-env-reverse | |
| - name: Clean up database (reverse-only import) | |
| run: nominatim refresh --postcodes --word-tokens | |
| working-directory: /home/nominatim/nominatim-project | |
| install-no-superuser: | |
| runs-on: ubuntu-24.04 | |
| needs: create-archive | |
| steps: | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: full-source | |
| - name: Unpack Nominatim | |
| run: tar xf nominatim-src.tar.bz2 | |
| - uses: ./Nominatim/.github/actions/setup-postgresql | |
| with: | |
| postgresql-version: 16 | |
| - uses: ./Nominatim/.github/actions/build-nominatim | |
| - name: Prepare import environment | |
| run: | | |
| mv Nominatim/test/testdb/apidb-test-data.pbf test.pbf | |
| rm -rf Nominatim | |
| - name: Prepare Database | |
| run: | | |
| ./venv/bin/nominatim import --prepare-database | |
| - name: Create import user | |
| run: | | |
| sudo -u postgres createuser osm-import | |
| psql -d nominatim -c "ALTER USER \"osm-import\" WITH PASSWORD 'osm-import'" | |
| psql -d nominatim -c 'GRANT CREATE ON SCHEMA public TO "osm-import"' | |
| - name: Run import | |
| run: | | |
| NOMINATIM_DATABASE_DSN="pgsql:host=127.0.0.1;dbname=nominatim;user=osm-import;password=osm-import" ./venv/bin/nominatim import --continue import-from-file --osm-file test.pbf | |
| - name: Check full import | |
| run: ./venv/bin/nominatim admin --check-database | |
| migrate: | |
| runs-on: ubuntu-24.04 | |
| needs: create-archive | |
| steps: | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| name: full-source | |
| - name: Unpack Nominatim | |
| run: tar xf nominatim-src.tar.bz2 | |
| - uses: ./Nominatim/.github/actions/setup-postgresql | |
| with: | |
| postgresql-version: 17 | |
| - name: Install Python dependencies | |
| run: | | |
| sudo apt-get install --no-install-recommends virtualenv osm2pgsql | |
| - name: Install Nominatim master version | |
| run: | | |
| virtualenv master | |
| cd Nominatim | |
| ../master/bin/pip install packaging/nominatim-db | |
| - name: Install Nominatim from pypi | |
| run: | | |
| virtualenv release | |
| ./release/bin/pip install nominatim-db | |
| - name: Import Nominatim database using release | |
| run: | | |
| ./release/bin/nominatim import --osm-file Nominatim/test/testdb/apidb-test-data.pbf | |
| ./release/bin/nominatim add-data --file Nominatim/test/testdb/additional_api_test.data.osm | |
| - name: Migrate to master version | |
| run: | | |
| ./master/bin/nominatim admin --migrate | |
| ./release/bin/nominatim add-data --file Nominatim/test/testdb/additional_api_test.data.osm | |
| codespell: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: codespell-project/actions-codespell@v2 | |
| with: | |
| only_warn: 1 |