[DNM] support "cold" data and code #19584

Workflow file for this run

	---

	name: Zephyr

	# 'workflow_dispatch' allows running this workflow manually from the
	# 'Actions' tab
	# yamllint disable-line rule:truthy
	on: [push, pull_request, workflow_dispatch, workflow_call]

	# Specifies group name that stops previous wokrflows if the name matches
	concurrency:
	# eg. "Zephyr-pull_request-my_fork_branch_to_merge"
	# eg. "Zephyr-push-refs/heads/my_branch_merging"
	group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.head_ref \|\| github.ref }}
	cancel-in-progress: true

	jobs:

	manifest-check:
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	with:
	path: ./workspace/sof
	filter: 'tree:0'

	- name: plain west update
	run: \|

	: This plain 'west update' does not provide 100% certainty that
	: all the manifest revisions make sense but it is quick and
	: will catch many revision problems. Other jobs typically
	: use 'west update --narrow' which is faster but
	: also able to fetch "wild" SHA1s from any random place! --narrow
	: is useful for testing unmerged Zephyr commits but risks
	: accepting "invalid" ones, this will not.

	pip3 install west
	cd workspace/sof/
	west init -l
	west update --fetch-opt=--filter=tree:0

	# Temporary check until we change west.yml and stop nesting rimage
	# and tomlc99 inside sof which will cleanly separate them from
	# XTOS submodules and... temporarily break every CI, which is why
	# it hasn't been done yet.
	- name: git submodules consistency
	run: \|

	cd workspace/sof
	git submodule update --init --recursive

	west update
	if git status --porcelain=v2 \| grep ^ ; then
	git status
	echo 'FAIL: inconsistency between git submodules and west.yml!'
	echo 'See rimage comment in west.yml.'
	echo 'Always use "git status"'
	exit 1
	fi


	# Temporary hacks to compile the very first LP64 configuration early
	# before it becomes supported by the regular
	# sof/scripts/xtensa-build-zephyr.py configuration script. Then this
	# job will be disappear, folded back in the regular build-* jobs below.
	LP64-WIP:
	runs-on: ubuntu-22.04

	steps:
	- uses: actions/checkout@v4
	with:
	path: ./workspace/sof
	filter: 'tree:0'

	- name: west clones
	run: pip3 install west && cd workspace/sof/ && west init -l &&
	west update --narrow --fetch-opt=--filter=tree:0

	# Not strictly necessary but saves a lot of scrolling in the next step
	# Caching a 12G image is unfortunately not possible:
	# https://github.com/ScribeMD/docker-cache/issues/304
	# For faster builds we would have to pay for some persistent runners.
	- name: Download docker image && ls /opt/toolchains/
	run: cd workspace && ./sof/zephyr/docker-run.sh ls -l /opt/toolchains/

	- name: 64 bits build
	run: \|
	cd workspace && ./sof/zephyr/docker-run.sh /bin/sh -c \
	'ln -s /opt/toolchains/zephyr-sdk-* ~/;
	west build --board imx93_evk/mimx9352/a55 sof/app \
	-- -DEXTRA_CFLAGS=-Werror -DEXTRA_CXXFLAGS=-Werror \
	-DEXTRA_AFLAGS=-Werror'


	build-linux:
	runs-on: ubuntu-22.04
	strategy:
	fail-fast: false
	matrix:
	# These keys are kept short because Github's left column is not resizable.
	# Search "zephyr_revision" and see below what they expand to.
	zephyr_revision: [
	mnfst, # special value: don't override sof/west.yml
	zmain, # Zephyr's main branch
	]
	# Using groups to avoid spamming the small results box with too
	# many lines. Pay attention to COMMAS.
	IPC_platforms: [
	# - IPC3 default
	imx8 imx8x imx8m imx8ulp,
	# - IPC4 default
	mtl, lnl, ptl,
	# Temporary testbed for Zephyr development.
	tgl tgl-h,
	]
	build_opts: [""]
	# Sparse matrices are complicated, you must read this page slowly:
	# https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs
	include:
	# specify one extra -d combination without affecting the main matrix
	- build_opts: -d
	zephyr_revision: mnfst
	IPC_platforms: mtl

	# This is "duplication of effort" but it makes sure no one
	# breaks --all, see for instance #9262 and previous commit.
	- zephyr_revision: mnfst
	IPC_platforms: --all

	steps:
	- uses: actions/checkout@v4
	# Download a full clone to fix `git describe`, sof_version.h and
	# build reproducibility. sof.git is still small.
	# This is especially useful for daily builds (but not just).
	with:
	fetch-depth: 0
	filter: 'tree:0'
	path: ./workspace/sof

	- name: west clones

	run: pip3 install west && cd workspace/sof/ && west init -l &&
	time west update --narrow --fetch-opt=--filter=tree:0

	- name: select zephyr revision
	run: \|
	cd workspace
	if [ 'mnfst' = '${{ matrix.zephyr_revision }}' ]; then
	rem_rev=$(git -C zephyr rev-parse HEAD)
	else
	case '${{ matrix.zephyr_revision }}' in
	'zmain') rem_rev='main' ;;
	*) echo 'Unknown matrix.zephyr_revision'; exit 1 ;;
	esac
	( cd sof/submanifests/
	sed -e "s#=sof_zephyr_revision_override=#${rem_rev}#" \
	sof-ci-jenkins/zephyr-override-template.yml > test-zephyr-main.yml
	)
	time west update --narrow --fetch-opt=--filter=tree:0
	fi

	# Because we used git tricks to speed things up, we now have two git
	# problems:
	#
	# 1. git fetch without a refspec fetches zero tag. So `git describe` is
	# different from everyone else. west normally adds `--tags` as a workaround
	# but this is inefficient because it fetches ALL tags; so `west --narrow`
	# does not use --tags.
	#
	# 2. west fetches using the remote URL, not the remote name. So remote
	# branches (if any) are missing from --decorate below
	#
	# => an "empty" and quick fetch _with_ a refspec and the remote name fixes
	# both issues in no time.

	cd zephyr
	time git fetch --filter=tree:0 "$(git remote \|head -n1)" "$rem_rev":_branch_placeholder
	git branch -D _branch_placeholder

	set -x
	west list
	west status
	git log --oneline -n 5 --decorate --graph --no-abbrev-commit

	# Not strictly necessary but saves a lot of scrolling in the next step
	# Caching a 12G image is unfortunately not possible:
	# https://github.com/ScribeMD/docker-cache/issues/304
	# For faster builds we would have to pay for some persistent runners.
	- name: Download docker image && ls /opt/toolchains/
	run: cd workspace && ./sof/zephyr/docker-run.sh ls -l /opt/toolchains/

	# https://github.com/zephyrproject-rtos/docker-image
	# Note: env variables can be passed to the container with
	# -e https_proxy=...
	- name: build
	run: cd workspace && ./sof/zephyr/docker-run.sh
	./sof/zephyr/docker-build.sh --cmake-args=-DEXTRA_CFLAGS=-Werror
	--cmake-args=-DEXTRA_CXXFLAGS=-Werror
	--cmake-args=-DEXTRA_AFLAGS='-Werror -Wa,--fatal-warnings'
	--cmake-args=--warn-uninitialized
	--overlay=sof/app/configs/repro-build.conf
	--no-tarball
	${{ matrix.build_opts }} ${{ matrix.IPC_platforms }}

	- name: Upload build artifacts
	uses: actions/upload-artifact@v4
	if: ${{ matrix.zephyr_revision == 'mnfst' }}
	with:
	name: linux-build ${{ matrix.build_opts }} ${{ matrix.IPC_platforms }}
	if-no-files-found: error
	path: \|
	${{ github.workspace }}/workspace/build-sof-staging
	${{ github.workspace }}/workspace/**/compile_commands.json

	build-windows:
	runs-on: windows-latest
	strategy:
	fail-fast: false
	matrix:
	# Using groups to avoid spamming the small results box with too
	# many lines. Pay attention to COMMAS.
	platforms: [
	# - IPC3 default
	imx8 imx8x imx8m imx8ulp,
	# - IPC4 default
	mtl,
	tgl tgl-h,
	]
	build_opts: [""]
	# Sparse matrices are complicated, see comments on Linux matrix above.
	include:
	- build_opts: -d
	platforms: mtl


	steps:
	- uses: actions/checkout@v4
	with:
	fetch-depth: 0
	filter: 'tree:0'
	path: ./workspace/sof

	# Cache artifacts so we do not overload external servers with downloads
	# Remember to change step key if you change the tools so old cache is not restored,
	# or delete cache manually in Github Actions tab
	- name: Cache unzip
	id: cache-unzip
	uses: actions/cache@v4
	with:
	path: unzip-5.51-1-bin.zip
	key: ${{ runner.os }}-cache-unzip-5-51-1

	- name: Cache wget
	id: cache-wget
	uses: actions/cache@v4
	with:
	path: wget-1.11.4-1-bin.zip
	key: ${{ runner.os }}-cache-wget-1-11-4-1

	# Keep this SDK version identical to the one in
	# sof/zephyr/docker-run.sh
	- name: Cache Zephyr SDK 0.17.0
	id: cache-zephyr-sdk
	uses: actions/cache@v4
	with:
	path: zephyr-sdk-0.17.0_windows-x86_64.7z
	key: ${{ runner.os }}-cache-zephyr-sdk-0-17-0

	# Wget is needed by Zephyr SDK setup.cmd installation script
	- name: Download wget
	if: ${{ steps.cache-wget.outputs.cache-hit != 'true' }}
	run: \|
	curl -L -O http://downloads.sourceforge.net/gnuwin32/wget-1.11.4-1-bin.zip

	- name: Download Zephyr SDK 0.17.0
	if: ${{ steps.cache-zephyr-sdk.outputs.cache-hit != 'true' }}
	run: \| # yamllint disable-line rule:line-length
	curl -L -O `
	https://github.com/zephyrproject-rtos/sdk-ng/releases/download/v0.17.0/zephyr-sdk-0.17.0_windows-x86_64.7z

	# Unzips every .zip package to directory matching its name without extension
	- name: Unzip downloaded packages
	run: \|
	7z x .zip -o
	7z x .7z -o

	- name: Add unzip to system PATH
	run: \|
	echo "${{ github.workspace }}/unzip-5.51-1-bin/bin" \| `
	Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append

	- name: Add wget to system PATH
	run: \|
	echo "${{ github.workspace }}/wget-1.11.4-1-bin/bin" \| `
	Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append

	# Install Zephyr SDK - all toolchains including Host Tools
	# and registering CMake package in the registry
	# setup.cmd may not be called in from msys shell as it does not parse
	# forward slash script input arguments correctly.
	- name: Install Zephyr SDK
	run: zephyr-sdk-0.17.0_windows-x86_64/zephyr-sdk-0.17.0/setup.cmd /t all /h /c

	- name: Setup Python
	uses: actions/setup-python@v5
	with:
	python-version: '3.10'

	- name: West install
	run: pip3 install west

	- name: West clone
	working-directory: ${{ github.workspace }}/workspace
	# Keep in sync with build-linux above
	run: \|
	west init -l sof
	west update --narrow --fetch-opt=--filter=tree:0

	# Get some tags to fix `git describe`, see build-linux comments above.
	cd zephyr
	$_rev = "$(git rev-parse HEAD)"
	git fetch --filter=tree:0 "$(west list -f '{url}' zephyr)" "${_rev}:_branch_placeholder"
	git branch -D _branch_placeholder


	# Call Setup Python again to save the PIP packages in cache
	- name: Setup Python
	uses: actions/setup-python@v5
	id: cache-python
	with:
	python-version: '3.10'
	cache: 'pip'
	cache-dependency-path: workspace/zephyr/scripts/requirements.txt

	# All requirements will be satisfied if the restored cache matches existing state
	- name: Validate python PIP cache
	working-directory: ${{ github.workspace }}/workspace
	run: pip install -r zephyr/scripts/requirements.txt

	# Ninja has been coming and going, see #8250
	- name: choco install ninja
	run: \|
	choco install ninja
	ninja.exe --version

	# MSYS2 provides gcc x64_86 toolchain & openssl
	# Installs in D:/a/_temp/msys64
	#
	# Note there is already C:/msys64/ provided by
	# https://github.com/actions/runner-images/blob/win22/20230918.1/images/win/Windows2022-Readme.md
	# Is it not good enough? Maybe it could save 20-30s.
	- name: Initialize MSYS2
	uses: msys2/setup-msys2@v2
	with:
	msystem: MSYS
	install: gcc openssl-devel
	path-type: inherit

	- name: Build
	shell: msys2 {0}
	working-directory: ${{ github.workspace }}/workspace
	run: python sof/scripts/xtensa-build-zephyr.py
	--no-interactive
	--cmake-args=-DEXTRA_CFLAGS=-Werror
	--cmake-args=-DEXTRA_CXXFLAGS=-Werror
	--cmake-args=-DEXTRA_AFLAGS='-Werror -Wa,--fatal-warnings'
	--cmake-args=--warn-uninitialized
	--overlay=sof/app/configs/repro-build.conf
	--no-tarball
	${{ matrix.build_opts }} ${{ matrix.platforms }}

	- name: Upload build artifacts
	uses: actions/upload-artifact@v4
	with:
	name: windows-build ${{ matrix.build_opts }} ${{ matrix.platforms }}
	if-no-files-found: error
	path: \|
	${{ github.workspace }}/workspace/build-sof-staging
	${{ github.workspace }}/workspace/**/compile_commands.json


	compare-linux-win:

	runs-on: ubuntu-latest

	# - We don't compare _all_ the builds, and
	# - even when some of the ones we compare fail, we still want to compare the rest.
	if: ${{ always() }}
	needs: [build-linux, build-windows]

	steps:
	- uses: actions/checkout@v4
	# we need only one script but it's simpler to get the (last
	# revision of the) whole repo and it takes seconds.
	with:
	# Isolate the clone in a subdirectory to make sure globbing
	# does not catch random SOF files.
	path: ./sof
	filter: 'tree:0'

	- name: Download Windows and Linux builds
	uses: actions/download-artifact@v4

	- name: apt-get dos2unix
	run: sudo apt-get update; sudo apt-get -y install dos2unix

	- name: Delete and fix expected differences
	run: \|
	ls -l
	# run it twice to make sure it's idempotent
	for i in 0 1; do
	./sof/zephyr/scripts/clean-expected-release-differences.sh \
	windows-build* linux-build*
	done

	- name: Compare Linux vs Windows builds
	run: \|
	diffs=0; ls -l

	# Check not too much is missing (it happened!)
	# http://mywiki.wooledge.org/ParsingLs
	for regdir in 'linux-build *mtl' \
	'windows-build *mtl' \
	'windows-build *tgl tgl-h'; do
	find . -maxdepth 1 \| grep -q "\./${regdir}\$" \|\|
	{ >&2 printf 'Missing %s\n' "${regdir}"; exit 1; }
	done

	# The disassembly is one of the most important things in this
	# context, so make sure CONFIG_OUTPUT_DISSASSEMBLY was on.
	ls -l /build-sof-staging/sof-info//zephyr.lst*

	set -x
	for windir in windows-build*; do
	lindir=linux-"${windir#windows-}"
	diff -qr "$lindir" "$windir" \|\| : $((diffs++))
	done
	exit $diffs

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

[DNM] support "cold" data and code #19584

Workflow file

[DNM] support "cold" data and code #19584

Jobs

Run details

Workflow file for this run