.gitlab-ci.yml

stages:
  - build-ai-bsp
  - test-ai-bsp
  - build-sdk

variables:
  BUILD_DIR: ${CI_PROJECT_DIR}/build
  OUTPUT_DIR: ${CI_PROJECT_DIR}/output
  PROP_LIBS_DIR: ${CI_PROJECT_DIR}/prop_libs
  DL_DIR: ${CI_PROJECT_DIR}/downloads
  SSTATE_DIR: ${CI_PROJECT_DIR}/sstate_cache

.selective_rules: &selective_rules
  rules:
    - if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH && $CI_COMMIT_BRANCH
      when: manual
      allow_failure: true
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_TAG
      when: always

.rzg2:
  variables:
    PROP_LIBS_BRANCH: rzg2_v3.0.6_eval

.rzg2l:
  variables:
    PROP_LIBS_BRANCH: rzg2l_v3.0.6_eval

.hihope-rzg2h:
  extends: [".rzg2"]
  variables:
    PLATFORM: hihope-rzg2h

.hihope-rzg2m:
  extends: [".rzg2"]
  variables:
    PLATFORM: hihope-rzg2m

.hihope-rzg2n:
  extends: [".rzg2"]
  variables:
    PLATFORM: hihope-rzg2n

.ek874:
  extends: [".rzg2"]
  variables:
    PLATFORM: ek874

.smarc-rzg2l:
  extends: [".rzg2l"]
  variables:
    PLATFORM: smarc-rzg2l

.smarc-rzg2lc:
  extends: [".rzg2l"]
  variables:
    PLATFORM: smarc-rzg2lc

.smarc-rzg2ul:
  extends: [".rzg2l"]
  variables:
    PLATFORM: smarc-rzg2ul

.smarc-rzv2l:
  extends: [".rzg2l"]
  variables:
    PLATFORM: smarc-rzv2l

.build-meta-renesas-ai:
  image: gitlab.renesas.solutions:5050/spl2/continuous-integration/dockerfiles:ubuntu-20.04-latest
  needs: []
  tags:
    - yocto
  before_script:
    - rm -rf ${OUTPUT_DIR} ${PROP_LIBS_DIR} ${BUILD_DIR}
    - export BUILD_THREADS=$(($(nproc)/2))
    - git config --global --add safe.directory '*'
  script:
    - mkdir -p ${OUTPUT_DIR} ${PROP_LIBS_DIR} ${BUILD_DIR}
    - git clone git@gitlab.renesas.solutions:spl2/continuous-integration/proprietary-libs.git ${PROP_LIBS_DIR}
    - pushd ${PROP_LIBS_DIR} && git checkout ${PROP_LIBS_BRANCH} && popd
    - pushd ${BUILD_DIR}
    - ${CI_PROJECT_DIR}/scripts/build-rzg-ai-bsp.sh -p ${PLATFORM} -l ${PROP_LIBS_DIR} -f ${FRAMEWORK} -n ${BUILD_THREADS} -o ${OUTPUT_DIR} -e -j ${DL_DIR} -k ${SSTATE_DIR} ${EXTRA_BUILD_ARGS}
  after_script:
    - echo "BUILD_JOB_ID=$CI_JOB_ID" >> job.env
  cache:
      - key: "downloads"
        paths:
          - ${DL_DIR}
      - key: "sstate-cache-${PLATFORM}"
        paths:
          - ${SSTATE_DIR}
  artifacts:
    name: "${CI_JOB_NAME}"
    when: always
    expire_in: 2 weeks
    reports:
      dotenv: job.env
    paths:
      - ${BUILD_DIR}/build/conf/bblayers.conf
      - ${BUILD_DIR}/build/conf/local.conf
      - ${OUTPUT_DIR}/

.test-meta-renesas-ai:
  image: gitlab.renesas.solutions:5050/spl2/continuous-integration/dockerfiles:ubuntu-20.04-latest
  stage: test-ai-bsp
  tags:
    - lava-job
  before_script:
    - printenv
  script:
    - mkdir results
    - ${CI_PROJECT_DIR}/scripts/submit-lava-job.sh -b -p ${PLATFORM} -t ${CI_PROJECT_DIR}/scripts/lava_templates/ -k ${CI_PROJECT_DIR}/scripts/known-test-issues.csv -u spl2-bot ${FRAMEWORK_ARG} -r -j results
  artifacts:
    name: "${CI_JOB_NAME}"
    when: always
    expire_in: 1 year
    reports:
      junit: results/results*.xml
    paths:
      - results
      - "benchmark_*.csv"

benchmark:armnn:hihope-rzg2h:
  stage: build-ai-bsp
  extends: [".build-meta-renesas-ai", ".hihope-rzg2h"]
  variables:
    FRAMEWORK: armnn
    EXTRA_BUILD_ARGS: "-b"
  <<: *selective_rules

benchmark:ort:hihope-rzg2h:
  stage: build-ai-bsp
  extends: [".build-meta-renesas-ai", ".hihope-rzg2h"]
  variables:
    FRAMEWORK: onnxruntime
    EXTRA_BUILD_ARGS: "-b"
  <<: *selective_rules

benchmark:tflite:hihope-rzg2h:
  stage: build-ai-bsp
  extends: [".build-meta-renesas-ai", ".hihope-rzg2h"]
  variables:
    FRAMEWORK: tensorflow-lite
    EXTRA_BUILD_ARGS: "-b"
  <<: *selective_rules

benchmark:armnn:hihope-rzg2m:
  stage: build-ai-bsp
  extends: [".build-meta-renesas-ai", ".hihope-rzg2m"]
  variables:
    FRAMEWORK: armnn
    EXTRA_BUILD_ARGS: "-b"

benchmark:ort:hihope-rzg2m:
  stage: build-ai-bsp
  extends: [".build-meta-renesas-ai", ".hihope-rzg2m"]
  variables:
    FRAMEWORK: onnxruntime
    EXTRA_BUILD_ARGS: "-b"

benchmark:tflite:hihope-rzg2m:
  stage: build-ai-bsp
  extends: [".build-meta-renesas-ai", ".hihope-rzg2m"]
  variables:
    FRAMEWORK: tensorflow-lite
    EXTRA_BUILD_ARGS: "-b"

benchmark:armnn:hihope-rzg2n:
  stage: build-ai-bsp
  extends: [".build-meta-renesas-ai", ".hihope-rzg2n"]
  variables:
    FRAMEWORK: armnn
    EXTRA_BUILD_ARGS: "-b"
  <<: *selective_rules

benchmark:ort:hihope-rzg2n:
  stage: build-ai-bsp
  extends: [".build-meta-renesas-ai", ".hihope-rzg2n"]
  variables:
    FRAMEWORK: onnxruntime
    EXTRA_BUILD_ARGS: "-b"
  <<: *selective_rules

benchmark:tflite:hihope-rzg2n:
  stage: build-ai-bsp
  extends: [".build-meta-renesas-ai", ".hihope-rzg2n"]
  variables:
    FRAMEWORK: tensorflow-lite
    EXTRA_BUILD_ARGS: "-b"
  <<: *selective_rules

benchmark:armnn:ek874:
  stage: build-ai-bsp
  extends: [".build-meta-renesas-ai", ".ek874"]
  variables:
    FRAMEWORK: armnn
    EXTRA_BUILD_ARGS: "-b"
  <<: *selective_rules

benchmark:ort:ek874:
  stage: build-ai-bsp
  extends: [".build-meta-renesas-ai", ".ek874"]
  variables:
    FRAMEWORK: onnxruntime
    EXTRA_BUILD_ARGS: "-b"
  <<: *selective_rules

benchmark:tflite:ek874:
  stage: build-ai-bsp
  extends: [".build-meta-renesas-ai", ".ek874"]
  variables:
    FRAMEWORK: tensorflow-lite
    EXTRA_BUILD_ARGS: "-b"
  <<: *selective_rules

benchmark:armnn:smarc-rzg2l:
  stage: build-ai-bsp
  extends: [".build-meta-renesas-ai", ".smarc-rzg2l"]
  variables:
    FRAMEWORK: armnn
    EXTRA_BUILD_ARGS: "-b"

benchmark:ort:smarc-rzg2l:
  stage: build-ai-bsp
  extends: [".build-meta-renesas-ai", ".smarc-rzg2l"]
  variables:
    FRAMEWORK: onnxruntime
    EXTRA_BUILD_ARGS: "-b"

benchmark:tflite:smarc-rzg2l:
  stage: build-ai-bsp
  extends: [".build-meta-renesas-ai", ".smarc-rzg2l"]
  variables:
    FRAMEWORK: tensorflow-lite
    EXTRA_BUILD_ARGS: "-b"

benchmark:armnn:smarc-rzg2lc:
  stage: build-ai-bsp
  extends: [".build-meta-renesas-ai", ".smarc-rzg2lc"]
  variables:
    FRAMEWORK: armnn
    EXTRA_BUILD_ARGS: "-b"
  <<: *selective_rules

benchmark:ort:smarc-rzg2lc:
  stage: build-ai-bsp
  extends: [".build-meta-renesas-ai", ".smarc-rzg2lc"]
  variables:
    FRAMEWORK: onnxruntime
    EXTRA_BUILD_ARGS: "-b"
  <<: *selective_rules

benchmark:tflite:smarc-rzg2lc:
  stage: build-ai-bsp
  extends: [".build-meta-renesas-ai", ".smarc-rzg2lc"]
  variables:
    FRAMEWORK: tensorflow-lite
    EXTRA_BUILD_ARGS: "-b"
  <<: *selective_rules

benchmark:armnn:smarc-rzg2ul:
  stage: build-ai-bsp
  extends: [".build-meta-renesas-ai", ".smarc-rzg2ul"]
  variables:
    FRAMEWORK: armnn
    EXTRA_BUILD_ARGS: "-b"
  <<: *selective_rules

benchmark:ort:smarc-rzg2ul:
  stage: build-ai-bsp
  extends: [".build-meta-renesas-ai", ".smarc-rzg2ul"]
  variables:
    FRAMEWORK: onnxruntime
    EXTRA_BUILD_ARGS: "-b"
  <<: *selective_rules

benchmark:tflite:smarc-rzg2ul:
  stage: build-ai-bsp
  extends: [".build-meta-renesas-ai", ".smarc-rzg2ul"]
  variables:
    FRAMEWORK: tensorflow-lite
    EXTRA_BUILD_ARGS: "-b"
  <<: *selective_rules

benchmark:armnn:smarc-rzv2l:
  stage: build-ai-bsp
  extends: [".build-meta-renesas-ai", ".smarc-rzv2l"]
  variables:
    FRAMEWORK: armnn
    EXTRA_BUILD_ARGS: "-b"
  <<: *selective_rules

benchmark:ort:smarc-rzv2l:
  stage: build-ai-bsp
  extends: [".build-meta-renesas-ai", ".smarc-rzv2l"]
  variables:
    FRAMEWORK: onnxruntime
    EXTRA_BUILD_ARGS: "-b"
  <<: *selective_rules

benchmark:tflite:smarc-rzv2l:
  stage: build-ai-bsp
  extends: [".build-meta-renesas-ai", ".smarc-rzv2l"]
  variables:
    FRAMEWORK: tensorflow-lite
    EXTRA_BUILD_ARGS: "-b"
  <<: *selective_rules


test:benchmark:armnn:hihope-rzg2h:
  needs: ["benchmark:armnn:hihope-rzg2h"]
  extends: [".test-meta-renesas-ai", ".hihope-rzg2h"]
  variables:
    FRAMEWORK_ARG: "-f armnn"

test:benchmark:ort:hihope-rzg2h:
  needs: ["benchmark:ort:hihope-rzg2h"]
  extends: [".test-meta-renesas-ai", ".hihope-rzg2h"]
  variables:
    FRAMEWORK_ARG: "-f onnxruntime"

test:benchmark:tflite:hihope-rzg2h:
  needs: ["benchmark:tflite:hihope-rzg2h"]
  extends: [".test-meta-renesas-ai", ".hihope-rzg2h"]
  variables:
    FRAMEWORK_ARG: "-f tensorflow-lite"

test:benchmark:armnn:hihope-rzg2m:
  needs: ["benchmark:armnn:hihope-rzg2m"]
  extends: [".test-meta-renesas-ai", ".hihope-rzg2m"]
  variables:
    FRAMEWORK_ARG: "-f armnn"

test:benchmark:ort:hihope-rzg2m:
  needs: ["benchmark:ort:hihope-rzg2m"]
  extends: [".test-meta-renesas-ai", ".hihope-rzg2m"]
  variables:
    FRAMEWORK_ARG: "-f onnxruntime"

test:benchmark:tflite:hihope-rzg2m:
  needs: ["benchmark:tflite:hihope-rzg2m"]
  extends: [".test-meta-renesas-ai", ".hihope-rzg2m"]
  variables:
    FRAMEWORK_ARG: "-f tensorflow-lite"

test:benchmark:armnn:hihope-rzg2n:
  needs: ["benchmark:armnn:hihope-rzg2n"]
  extends: [".test-meta-renesas-ai", ".hihope-rzg2n"]
  variables:
    FRAMEWORK_ARG: "-f armnn"

test:benchmark:ort:hihope-rzg2n:
  needs: ["benchmark:ort:hihope-rzg2n"]
  extends: [".test-meta-renesas-ai", ".hihope-rzg2n"]
  variables:
    FRAMEWORK_ARG: "-f onnxruntime"

test:benchmark:tflite:hihope-rzg2n:
  needs: ["benchmark:tflite:hihope-rzg2n"]
  extends: [".test-meta-renesas-ai", ".hihope-rzg2n"]
  variables:
    FRAMEWORK_ARG: "-f tensorflow-lite"

test:benchmark:armnn:ek874:
  needs: ["benchmark:armnn:ek874"]
  extends: [".test-meta-renesas-ai", ".ek874"]
  variables:
    FRAMEWORK_ARG: "-f armnn"

test:benchmark:ort:ek874:
  needs: ["benchmark:ort:ek874"]
  extends: [".test-meta-renesas-ai", ".ek874"]
  variables:
    FRAMEWORK_ARG: "-f onnxruntime"

test:benchmark:tflite:ek874:
  needs: ["benchmark:tflite:ek874"]
  extends: [".test-meta-renesas-ai", ".ek874"]
  variables:
    FRAMEWORK_ARG: "-f tensorflow-lite"

test:benchmark:armnn:smarc-rzg2l:
  needs: ["benchmark:armnn:smarc-rzg2l"]
  extends: [".test-meta-renesas-ai", ".smarc-rzg2l"]
  variables:
    FRAMEWORK_ARG: "-f armnn"

test:benchmark:ort:smarc-rzg2l:
  needs: ["benchmark:ort:smarc-rzg2l"]
  extends: [".test-meta-renesas-ai", ".smarc-rzg2l"]
  variables:
    FRAMEWORK_ARG: "-f onnxruntime"

test:benchmark:tflite:smarc-rzg2l:
  needs: ["benchmark:tflite:smarc-rzg2l"]
  extends: [".test-meta-renesas-ai", ".smarc-rzg2l"]
  variables:
    FRAMEWORK_ARG: "-f tensorflow-lite"

test:benchmark:armnn:smarc-rzg2lc:
  needs: ["benchmark:armnn:smarc-rzg2lc"]
  extends: [".test-meta-renesas-ai", ".smarc-rzg2lc"]
  variables:
    FRAMEWORK_ARG: "-f armnn"

test:benchmark:ort:smarc-rzg2lc:
  needs: ["benchmark:ort:smarc-rzg2lc"]
  extends: [".test-meta-renesas-ai", ".smarc-rzg2lc"]
  variables:
    FRAMEWORK_ARG: "-f onnxruntime"

test:benchmark:tflite:smarc-rzg2lc:
  needs: ["benchmark:tflite:smarc-rzg2lc"]
  extends: [".test-meta-renesas-ai", ".smarc-rzg2lc"]
  variables:
    FRAMEWORK_ARG: "-f tensorflow-lite"

test:benchmark:armnn:smarc-rzg2ul:
  needs: ["benchmark:armnn:smarc-rzg2ul"]
  extends: [".test-meta-renesas-ai", ".smarc-rzg2ul"]
  variables:
    FRAMEWORK_ARG: "-f armnn"

test:benchmark:ort:smarc-rzg2ul:
  needs: ["benchmark:ort:smarc-rzg2ul"]
  extends: [".test-meta-renesas-ai", ".smarc-rzg2ul"]
  variables:
    FRAMEWORK_ARG: "-f onnxruntime"

test:benchmark:tflite:smarc-rzg2ul:
  needs: ["benchmark:tflite:smarc-rzg2ul"]
  extends: [".test-meta-renesas-ai", ".smarc-rzg2ul"]
  variables:
    FRAMEWORK_ARG: "-f tensorflow-lite"

test:benchmark:armnn:smarc-rzv2l:
  needs: ["benchmark:armnn:smarc-rzv2l"]
  extends: [".test-meta-renesas-ai", ".smarc-rzv2l"]
  variables:
    FRAMEWORK_ARG: "-f armnn"

test:benchmark:ort:smarc-rzv2l:
  needs: ["benchmark:ort:smarc-rzv2l"]
  extends: [".test-meta-renesas-ai", ".smarc-rzv2l"]
  variables:
    FRAMEWORK_ARG: "-f onnxruntime"

test:benchmark:tflite:smarc-rzv2l:
  needs: ["benchmark:tflite:smarc-rzv2l"]
  extends: [".test-meta-renesas-ai", ".smarc-rzv2l"]
  variables:
    FRAMEWORK_ARG: "-f tensorflow-lite"


sdk:benchmark:armnn:hihope-rzg2h:
  stage: build-sdk
  extends: [".build-meta-renesas-ai", ".hihope-rzg2h"]
  variables:
    FRAMEWORK: armnn
    EXTRA_BUILD_ARGS: "-b -T"
  <<: *selective_rules

sdk:benchmark:ort:hihope-rzg2h:
  stage: build-sdk
  extends: [".build-meta-renesas-ai", ".hihope-rzg2h"]
  variables:
    FRAMEWORK: onnxruntime
    EXTRA_BUILD_ARGS: "-b -T"
  <<: *selective_rules

sdk:benchmark:tflite:hihope-rzg2h:
  stage: build-sdk
  extends: [".build-meta-renesas-ai", ".hihope-rzg2h"]
  variables:
    FRAMEWORK: tensorflow-lite
    EXTRA_BUILD_ARGS: "-b -T"
  <<: *selective_rules

sdk:benchmark:armnn:hihope-rzg2m:
  stage: build-sdk
  extends: [".build-meta-renesas-ai", ".hihope-rzg2m"]
  variables:
    FRAMEWORK: armnn
    EXTRA_BUILD_ARGS: "-b -T"
  <<: *selective_rules

sdk:benchmark:ort:hihope-rzg2m:
  stage: build-sdk
  extends: [".build-meta-renesas-ai", ".hihope-rzg2m"]
  variables:
    FRAMEWORK: onnxruntime
    EXTRA_BUILD_ARGS: "-b -T"
  <<: *selective_rules

sdk:benchmark:tflite:hihope-rzg2m:
  stage: build-sdk
  extends: [".build-meta-renesas-ai", ".hihope-rzg2m"]
  variables:
    FRAMEWORK: tensorflow-lite
    EXTRA_BUILD_ARGS: "-b -T"
  <<: *selective_rules

sdk:benchmark:armnn:hihope-rzg2n:
  stage: build-sdk
  extends: [".build-meta-renesas-ai", ".hihope-rzg2n"]
  variables:
    FRAMEWORK: armnn
    EXTRA_BUILD_ARGS: "-b -T"
  <<: *selective_rules

sdk:benchmark:ort:hihope-rzg2n:
  stage: build-sdk
  extends: [".build-meta-renesas-ai", ".hihope-rzg2n"]
  variables:
    FRAMEWORK: onnxruntime
    EXTRA_BUILD_ARGS: "-b -T"
  <<: *selective_rules

sdk:benchmark:tflite:hihope-rzg2n:
  stage: build-sdk
  extends: [".build-meta-renesas-ai", ".hihope-rzg2n"]
  variables:
    FRAMEWORK: tensorflow-lite
    EXTRA_BUILD_ARGS: "-b -T"
  <<: *selective_rules

sdk:benchmark:armnn:ek874:
  stage: build-sdk
  extends: [".build-meta-renesas-ai", ".ek874"]
  variables:
    FRAMEWORK: armnn
    EXTRA_BUILD_ARGS: "-b -T"
  <<: *selective_rules

sdk:benchmark:ort:ek874:
  stage: build-sdk
  extends: [".build-meta-renesas-ai", ".ek874"]
  variables:
    FRAMEWORK: onnxruntime
    EXTRA_BUILD_ARGS: "-b -T"
  <<: *selective_rules

sdk:benchmark:tflite:ek874:
  stage: build-sdk
  extends: [".build-meta-renesas-ai", ".ek874"]
  variables:
    FRAMEWORK: tensorflow-lite
    EXTRA_BUILD_ARGS: "-b -T"
  <<: *selective_rules

sdk:benchmark:armnn:smarc-rzg2l:
  stage: build-sdk
  extends: [".build-meta-renesas-ai", ".smarc-rzg2l"]
  variables:
    FRAMEWORK: armnn
    EXTRA_BUILD_ARGS: "-b -T"
  <<: *selective_rules

sdk:benchmark:ort:smarc-rzg2l:
  stage: build-sdk
  extends: [".build-meta-renesas-ai", ".smarc-rzg2l"]
  variables:
    FRAMEWORK: onnxruntime
    EXTRA_BUILD_ARGS: "-b -T"
  <<: *selective_rules

sdk:benchmark:tflite:smarc-rzg2l:
  stage: build-sdk
  extends: [".build-meta-renesas-ai", ".smarc-rzg2l"]
  variables:
    FRAMEWORK: tensorflow-lite
    EXTRA_BUILD_ARGS: "-b -T"
  <<: *selective_rules

sdk:benchmark:armnn:smarc-rzg2lc:
  stage: build-sdk
  extends: [".build-meta-renesas-ai", ".smarc-rzg2lc"]
  variables:
    FRAMEWORK: armnn
    EXTRA_BUILD_ARGS: "-b -T"
  <<: *selective_rules

sdk:benchmark:ort:smarc-rzg2lc:
  stage: build-sdk
  extends: [".build-meta-renesas-ai", ".smarc-rzg2lc"]
  variables:
    FRAMEWORK: onnxruntime
    EXTRA_BUILD_ARGS: "-b -T"
  <<: *selective_rules

sdk:benchmark:tflite:smarc-rzg2lc:
  stage: build-sdk
  extends: [".build-meta-renesas-ai", ".smarc-rzg2lc"]
  variables:
    FRAMEWORK: tensorflow-lite
    EXTRA_BUILD_ARGS: "-b -T"
  <<: *selective_rules

sdk:benchmark:armnn:smarc-rzg2ul:
  stage: build-sdk
  extends: [".build-meta-renesas-ai", ".smarc-rzg2ul"]
  variables:
    FRAMEWORK: armnn
    EXTRA_BUILD_ARGS: "-b -T"
  <<: *selective_rules

sdk:benchmark:ort:smarc-rzg2ul:
  stage: build-sdk
  extends: [".build-meta-renesas-ai", ".smarc-rzg2ul"]
  variables:
    FRAMEWORK: onnxruntime
    EXTRA_BUILD_ARGS: "-b -T"
  <<: *selective_rules

sdk:benchmark:tflite:smarc-rzg2ul:
  stage: build-sdk
  extends: [".build-meta-renesas-ai", ".smarc-rzg2ul"]
  variables:
    FRAMEWORK: tensorflow-lite
    EXTRA_BUILD_ARGS: "-b -T"
  <<: *selective_rules

sdk:benchmark:armnn:smarc-rzv2l:
  stage: build-sdk
  extends: [".build-meta-renesas-ai", ".smarc-rzv2l"]
  variables:
    FRAMEWORK: armnn
    EXTRA_BUILD_ARGS: "-b -T"
  <<: *selective_rules

sdk:benchmark:ort:smarc-rzv2l:
  stage: build-sdk
  extends: [".build-meta-renesas-ai", ".smarc-rzv2l"]
  variables:
    FRAMEWORK: onnxruntime
    EXTRA_BUILD_ARGS: "-b -T"
  <<: *selective_rules

sdk:benchmark:tflite:smarc-rzv2l:
  stage: build-sdk
  extends: [".build-meta-renesas-ai", ".smarc-rzv2l"]
  variables:
    FRAMEWORK: tensorflow-lite
    EXTRA_BUILD_ARGS: "-b -T"
  <<: *selective_rules