.gitlab-ci.yml

#
# Qubes Builder Gitlab CI
#

include:
  - file: /common.yml
    project: QubesOS/qubes-continuous-integration

stages:
  - test
  - prep
  - build

variables:
  DEBUG: "1"
  GIT_SUBMODULE_STRATEGY: recursive
  CI_RUN_PYTEST_JOBS:
    description: "Run all pytest jobs, default: 1"
  CI_RUN_COMPONENT_JOBS:
    description: "Run components jobs, default: 0"
  CI_RUN_TEMPLATE_JOBS:
    description: "Run template jobs, default: 0"
  CI_RUN_ISO_JOBS:
    description: "Run ISO jobs, default: 0"
  CI_RUN_INFRA_JOBS:
    description: "Run all builderv2-github jobs, default: 0"
  CI_RUN_ALL_JOBS:
    description: "Run all jobs, default: 0"

builderv2-github:
  rules:
    - if: $CI_RUN_INFRA_JOBS == "1" || $CI_RUN_ALL_JOBS == "1"
      when: always
    - when: never
  variables:
    CI_QUBES_BUILDER_BRANCH: $CI_COMMIT_BRANCH
  trigger:
    project: QubesOS/qubes-builderv2-github
    strategy: depend

#
# job skeletons
#

.pytest:
  rules:
    - if: $CI_RUN_PYTEST_JOBS != "0" || $CI_RUN_ALL_JOBS == "1"
      when: always
    - when: never
  stage: test
  tags:
    - vm
  artifacts:
    paths:
      - artifacts/htmlcov/
    reports:
      junit: artifacts/qubesbuilder.xml
      coverage_report:
        coverage_format: cobertura
        path: artifacts/coverage.xml
  coverage: '/TOTAL.*\s(\d+)%/'
  variables:
    PYTEST_ARGS: -v --color=yes --showlocals --cov qubesbuilder --cov-report term --cov-report html:artifacts/htmlcov --cov-report xml:artifacts/coverage.xml --junitxml=artifacts/qubesbuilder.xml
  # https://gitlab.com/gitlab-org/gitlab/-/issues/15603
  before_script:
    - sudo dnf install -y python3-pip python3-pytest-cov python3-pytest python3-pytest-mock python3-lxml python3-wheel sequoia-sqv
    - sudo rm -rf ~/pytest-of-user/
    - docker pull registry.gitlab.com/qubesos/docker-images/qubes-builder-fedora:latest
    - docker tag registry.gitlab.com/qubesos/docker-images/qubes-builder-fedora:latest qubes-builder-fedora:latest
    - mkdir ~/gitlab ~/tmp
    - export PYTHONPATH=".:$PYTHONPATH" BASE_ARTIFACTS_DIR=~/gitlab TMPDIR=~/tmp
    - env
  after_script:
  - ci/codecov-wrapper -f artifacts/coverage.xml

.init_cache_job:
  rules:
    - if: $CI_RUN_CACHE_JOBS == "1" || $CI_RUN_COMPONENT_JOBS == "1" || $CI_RUN_ALL_JOBS == "1"
      when: always
    - when: never
  stage: prep
  tags:
    - vm
  artifacts:
    paths:
      - artifacts.tar.xz
  before_script:
    - docker pull registry.gitlab.com/qubesos/docker-images/qubes-builder-fedora:latest
    - docker tag registry.gitlab.com/qubesos/docker-images/qubes-builder-fedora:latest qubes-builder-fedora:latest
    - sudo dnf install -y python3-debian dpkg-dev openssl tree pacman
  script:
    - ./qb --builder-conf tests/builder-ci.yml -d "${CI_JOB_NAME#cache-}" package init-cache
    - tar cJf artifacts.tar.xz --strip-components=1 artifacts

.component_job:
  rules:
    - if: $CI_RUN_COMPONENT_JOBS == "1" || $CI_RUN_ALL_JOBS == "1"
      when: always
    - when: never

.component_docker_job:
  extends: .component_job
  stage: build
  tags:
    - vm
  artifacts:
    paths:
      - artifacts
    when: always
  before_script:
    - cp -Tr tests/gnupg ~/.gnupg
    - chmod 700 ~/.gnupg
    - docker pull registry.gitlab.com/qubesos/docker-images/qubes-builder-fedora:latest
    - docker tag registry.gitlab.com/qubesos/docker-images/qubes-builder-fedora:latest qubes-builder-fedora:latest
    - tar xf artifacts.tar.xz
    - sudo dnf install -y python3-debian dpkg-dev openssl tree pacman
  script:
    - ./qb --builder-conf tests/builder-ci.yml -d "${CI_JOB_NAME#docker-}" ${QB_EXTRA_ARGS} package all
  after_script:
    - rm -rf artifacts/cache
    - tree artifacts

.component_podman_job:
  extends: .component_job
  stage: build
  tags:
    - vm
  artifacts:
    paths:
      - artifacts
    when: always
  before_script:
    - cp -Tr tests/gnupg ~/.gnupg
    - chmod 700 ~/.gnupg
    - sudo sed -i "/$(id -u -n)/d" /etc/subuid
    - sudo sed -i "/$(id -g -n)/d" /etc/subgid
    - echo $(id -u -n):$(( $(id -u)*65536 )):65536 | sudo tee -a /etc/subuid
    - echo $(id -g -n):$(( $(id -g)*65536 )):65536 | sudo tee -a /etc/subgid
    - systemctl --user start podman
    - podman pull docker.io/fepitre/qubes-builder-fedora:latest
    - podman tag docker.io/fepitre/qubes-builder-fedora:latest qubes-builder-fedora:latest
    - tar xf artifacts.tar.xz
    - sudo dnf install -y python3-debian dpkg-dev openssl tree pacman
  script:
    - sed -i 's/docker/podman/' tests/builder-ci.yml
    - ./qb --builder-conf tests/builder-ci.yml -d "${CI_JOB_NAME#podman-}" ${QB_EXTRA_ARGS} package all
  after_script:
    - rm -rf artifacts/cache
    - tree artifacts

.component_qubes_job:
  extends: .component_job
  stage: build
  tags:
    - qubes
  artifacts:
    paths:
      - artifacts
    when: always
  before_script:
    - cp -Tr tests/gnupg ~/.gnupg
    - chmod 700 ~/.gnupg
    - tar xf artifacts.tar.xz
    - sudo dnf install -y python3-debian dpkg-dev openssl tree pacman
  script:
    - sed -i 's/docker/qubes/' tests/builder-ci.yml
    - "sed -i 's/user: gitlab-runner/user: user/; s/group: gitlab-runner/group: user/' tests/builder-ci.yml"
    - ./qb --builder-conf tests/builder-ci.yml -d "${CI_JOB_NAME#qubes-}" --option executor:type=qubes --option executor:options:dispvm=builder-dvm ${QB_EXTRA_ARGS} package all
  after_script:
    - rm -rf artifacts/cache
    - tree artifacts

.component_local_job:
  extends: .component_job
  stage: build
  tags:
    - vm
  artifacts:
    paths:
      - artifacts
    when: always
  before_script:
    - cp -Tr tests/gnupg ~/.gnupg
    - chmod 700 ~/.gnupg
    - tar xf artifacts.tar.xz
    - sudo dnf install -y python3-debian dpkg-dev openssl tree pacman
  script:
    - ./qb --builder-conf tests/builder-ci.yml -d "${CI_JOB_NAME#local-}" --option executor:type=local --option executor:options:directory=/home/gitlab-runner/tmp ${QB_EXTRA_ARGS} package all
  after_script:
    - rm -rf artifacts/cache
    - tree artifacts

.template_qubes_job:
  rules:
    - if: $CI_RUN_TEMPLATE_JOBS == "1" || $CI_RUN_ALL_JOBS == "1"
      when: always
    - when: never
  stage: build
  tags:
    - qubes
  before_script:
    - docker pull registry.gitlab.com/qubesos/docker-images/qubes-builder-fedora:latest
    - docker tag registry.gitlab.com/qubesos/docker-images/qubes-builder-fedora:latest qubes-builder-fedora:latest
    - cp -Tr tests/gnupg ~/.gnupg
    - chmod 700 ~/.gnupg
    - sudo dnf install -y python3-debian dpkg-dev openssl tree pacman
  script:
    - ./qb --builder-conf tests/builder-ci.yml -c builder-rpm -c builder-debian -c template-whonix -c builder-archlinux -c qubes-release package fetch
    - ./qb --builder-conf tests/builder-ci.yml -t "${CI_JOB_NAME#qubes-template-}" --option executor:type=qubes --option executor:options:dispvm=builder-dvm template all

.installer_iso:
  rules:
    - if: $CI_RUN_ISO_JOBS == "1" || $CI_RUN_ALL_JOBS == "1"
      when: always
    - when: never
  stage: build
  tags:
    - vm
  artifacts:
    paths:
      - artifacts/installer
  before_script:
    - cp -Tr tests/gnupg ~/.gnupg
    - chmod 700 ~/.gnupg
    - docker pull registry.gitlab.com/qubesos/docker-images/qubes-builder-fedora:latest
    - docker tag registry.gitlab.com/qubesos/docker-images/qubes-builder-fedora:latest qubes-builder-fedora:latest
    - sudo dnf install -y python3-debian dpkg-dev openssl tree mktorrent
  script:
    - ./qb --builder-conf tests/builder-ci.yml -c qubes-release package fetch
    - 'sh -c "$EXPECT_FAILURE ./qb --builder-conf tests/builder-ci.yml -o iso:kickstart=$KICKSTART_FILE installer init-cache all"'
  after_script:
    - rm -rf artifacts/cache
    - tree artifacts

#
# test
#

lint:
  extends: .lint
  stage: test
  variables:
    DIR: qubesbuilder

mypy:
  stage: test
  image: fedora:40
  tags:
    - docker
  before_script:
    - sudo dnf install -y python3-mypy python3-pip
    - sudo python3 -m pip install types-PyYAML types-python-dateutil
  script:
    - mypy --install-types --non-interactive --junit-xml mypy.xml qubesbuilder
  artifacts:
    reports:
      junit: mypy.xml

shellcheck:
  stage: test
  image: fedora:40
  tags:
    - docker
  before_script:
    - sudo dnf install -y ShellCheck git
  script:
    - shellcheck -x -e SC1117 $(grep -l '^#!/bin/\(ba\)\?sh' $(git ls-files))

pytest:
  extends: .pytest
  tags:
    - qubes
  script:
    - pytest-3 $PYTEST_ARGS tests/test_executors.py tests/test_functions.py tests/test_objects.py tests/test_scripts.py tests/test_log.py

pytest-cli-cleanup:
  extends: .pytest
  script:
    - pytest-3 $PYTEST_ARGS tests/test_cli_cleanup.py -k test_cleanup_

pytest-cli-repository:
  extends: .pytest
  script:
    - pytest-3 $PYTEST_ARGS tests/test_cli_repository.py -k test_repository_

pytest-cli-component-host-fc37:
  extends: .pytest
  script:
    - pytest-3 $PYTEST_ARGS tests/test_cli.py -k test_component_host_fc37_

pytest-cli-component-vm-bookworm:
  extends: .pytest
  script:
    - pytest-3 $PYTEST_ARGS tests/test_cli.py -k test_component_vm_bookworm

pytest-cli-component-vm-archlinux:
  extends: .pytest
  script:
    - pytest-3 $PYTEST_ARGS tests/test_cli.py -k test_component_vm_archlinux

pytest-cli-component:
  extends: .pytest
  script:
    - pytest-3 $PYTEST_ARGS tests/test_cli.py -k test_common_

pytest-cli-increment-component:
  extends: .pytest
  script:
    - pytest-3 $PYTEST_ARGS tests/test_cli.py -k test_increment_component_

pytest-cli-template-fedora:
  extends: .pytest
  script:
    - pytest-3 $PYTEST_ARGS tests/test_cli.py -k test_template_fedora

pytest-cli-template-debian:
  extends: .pytest
  script:
    - pytest-3 $PYTEST_ARGS tests/test_cli.py -k test_template_debian

pytest-cli-installer-cache:
  extends: .pytest
  script:
    - pytest-3 $PYTEST_ARGS tests/test_cli.py -k test_installer_init_cache

#
# Components
#

# Cache

cache-host-fc37:
  extends: .init_cache_job
  script:
    - !reference [.init_cache_job, script ]
    - tests/scripts/check-chroot-content.sh artifacts "${CI_JOB_NAME#cache-}" /usr/bin/gcc

cache-vm-fc40:
  extends: .init_cache_job

cache-vm-fc41:
  extends: .init_cache_job

cache-vm-bookworm:
  extends: .init_cache_job
  script:
    - !reference [.init_cache_job, script ]
    - tests/scripts/check-chroot-content.sh artifacts "${CI_JOB_NAME#cache-}" /usr/bin/quilt

cache-vm-trixie:
  extends: .init_cache_job

cache-vm-archlinux:
  extends: .init_cache_job

cache-vm-jammy:
  extends: .init_cache_job

cache-vm-noble:
  extends: .init_cache_job

# Podman executor

podman-host-fc37:
  extends: .component_podman_job
  needs:
    - cache-host-fc37

podman-vm-fc40:
  extends: .component_podman_job
  needs:
    - cache-vm-fc40

#podman-vm-bookworm:
#  extends: .component_podman_job

# Docker executor

docker-host-fc37:
  extends: .component_docker_job
  needs:
    - cache-host-fc37

docker-vm-fc40:
  extends: .component_docker_job
  needs:
    - cache-vm-fc40

docker-vm-bookworm:
  extends: .component_docker_job
  needs:
    - cache-vm-bookworm

docker-vm-archlinux:
  extends: .component_docker_job
  needs:
    - cache-vm-archlinux

docker-vm-jammy:
  extends: .component_docker_job
  before_script:
    - cp -Tr tests/gnupg ~/.gnupg
    - chmod 700 ~/.gnupg
    - docker pull registry.gitlab.com/qubesos/docker-images/qubes-builder-debian:latest
    - docker tag registry.gitlab.com/qubesos/docker-images/qubes-builder-debian:latest qubes-builder-debian:latest
    - tar xf artifacts.tar.xz
    - sudo dnf install -y python3-debian dpkg-dev openssl tree pacman
    # install an update manually built until https://bugzilla.redhat.com/show_bug.cgi?id=2108872 is handled
    - sudo dnf update -y https://caviar.qubes-os.org/~marmarek/reprepro-5.4.2-1.fc37.x86_64.rpm
  variables:
    QB_EXTRA_ARGS: "-o executor:options:image=qubes-builder-debian:latest -o +components+vmm-xen-guest -c vmm-xen-guest -c core-vchan-xen"
  needs:
    - cache-vm-jammy

docker-vm-noble:
  extends: .component_docker_job
  before_script:
    - cp -Tr tests/gnupg ~/.gnupg
    - chmod 700 ~/.gnupg
    - docker pull registry.gitlab.com/qubesos/docker-images/qubes-builder-ubuntu:latest
    - docker tag registry.gitlab.com/qubesos/docker-images/qubes-builder-ubuntu:latest qubes-builder-ubuntu:latest
    - tar xf artifacts.tar.xz
    - sudo dnf install -y python3-debian dpkg-dev openssl tree pacman
    # install an update manually built until https://bugzilla.redhat.com/show_bug.cgi?id=2108872 is handled
    - sudo dnf update -y https://caviar.qubes-os.org/~marmarek/reprepro-5.4.2-1.fc37.x86_64.rpm
  variables:
    QB_EXTRA_ARGS: "-o executor:options:image=qubes-builder-ubuntu:latest -o +components+vmm-xen-guest -c vmm-xen-guest -c core-vchan-xen"
  needs:
    - cache-vm-noble

# Qubes executor

qubes-host-fc37:
  extends: .component_qubes_job
  needs:
    - cache-host-fc37

qubes-vm-fc40:
  extends: .component_qubes_job
  needs:
    - cache-vm-fc40

qubes-vm-bookworm:
  extends: .component_qubes_job
  needs:
    - cache-vm-bookworm

# Local executor

local-host-fc37:
  extends: .component_local_job
  needs:
    - cache-host-fc37

local-vm-fc40:
  extends: .component_local_job
  needs:
    - cache-vm-fc40

local-vm-bookworm:
  extends: .component_local_job
  needs:
    - cache-vm-bookworm

#
# Templates
#

qubes-template-fedora-40-xfce:
  extends: .template_qubes_job

qubes-template-debian-12:
  extends: .template_qubes_job

qubes-template-whonix-gateway-17:
  extends: .template_qubes_job

qubes-template-whonix-workstation-17:
  extends: .template_qubes_job
  allow_failure: true

qubes-template-archlinux:
  extends: .template_qubes_job

#
# ISO
#

installer-iso-online-testing:
  variables:
    KICKSTART_FILE: conf/iso-online-testing.ks
    EXPECT_FAILURE: ""
  extends: .installer_iso

installer-iso-unsigned:
  variables:
    KICKSTART_FILE: conf/iso-unsigned.ks
    EXPECT_FAILURE: "!"
  extends: .installer_iso

installer-iso-unsigned2:
  variables:
    KICKSTART_FILE: conf/iso-unsigned2.ks
    EXPECT_FAILURE: "!"
  extends: .installer_iso

installer-iso-unknown-key:
  variables:
    KICKSTART_FILE: conf/iso-unknown-key.ks
    EXPECT_FAILURE: "!"
  extends: .installer_iso