Merge pull request thesofproject#10 from nashif/update_may

Update to latest SOF and fix workq

.github/workflows/checkpatch_list.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+# SPDX-License-Identifier: BSD-3-Clause
+
+set -e
+
+# The purpose of this script is not to stop on the first checkpatch
+# failure and report at the end instead.
+#
+# Sample invocation:
+# $0 --codespell --strict < PR_SHAs.txt
+main()
+{
+    local failures=0
+    while read -r sha ; do
+        printf '\n    -------------- \n\n'
+        ./scripts/checkpatch.pl $@ -g $sha || failures=$((failures+1))
+    done
+    printf '\n    -------------- \n\n'
+    return $failures
+}
+
+main "$@"

.github/workflows/installer.yml

@@ -0,0 +1,54 @@
+---
+# Tools that can save round-trips to github and a lot of time:
+#
+# yamllint -f parsable this.yml
+# pip3 install ruamel.yaml.cmd
+# yaml merge-expand this.yml exp.yml && diff -w -u this.yml exp.yml
+#
+# github.com also has a powerful web editor that can be used without
+# committing.
+
+name: installer
+
+# 'workflow_dispatch' allows running this workflow manually from the
+# 'Actions' tab
+# yamllint disable-line rule:truthy
+on: [push, pull_request, workflow_dispatch]
+
+jobs:
+  checktree:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+        # From time to time this will catch a git tag and change SOF_VERSION
+        with: {fetch-depth: 50, submodules: recursive}
+
+      - name: yamllint ourselves
+        run: yamllint .github/workflows/installer.yml
+
+      - name: docker
+        run: docker pull thesofproject/sof && docker tag thesofproject/sof sof
+
+      # FIXME: update the docker image instead
+      - name: HACK add tree to the docker PATH
+        run: |
+           cp -a /usr/bin/tree .
+           printf '%s\n' 'PATH=$PATH:$(pwd)' > stage.sh
+           printf '%s\n' 'make -j3 -C installer/ stage' >> stage.sh
+
+      # -j3 tests the ability to build multiple platforms
+      # concurrently. It makes the build log unreadable but that's OK
+      # because we have other, slower actions with readable logs.
+      - name: build all and stage
+        run: ./scripts/docker-run.sh sh -x ./stage.sh
+
+      - name: check staging tree
+        run: make -C installer/ checktree
+
+      - name: test make cleanall
+        run: |
+           make -C installer cleanall
+           # Make sure there's nothing left
+           rm -rf scripts/kconfig/__pycache__/
+           rm ./tree ./stage.sh # HACK, see above
+           ! git status --porcelain --ignored | grep .

.github/workflows/pull-request.yml

@@ -9,14 +9,23 @@
 # github.com also has a powerful web editor that can be used without
 # committing.
 
-name: Pull Requests
+# This is the name of this workflow and should technically be called
+# something like "Main Workflow" but the place where most people see
+# this name is the Checks window next to other, non-github checks.
+name: GitHub Actions
 
 # yamllint disable-line rule:truthy
 on:
   push:
-    branches: [master]
+    branches:
+      - 'main'
+      - 'stable-**'
+      - '**-stable'
   pull_request:
-    branches: [master]
+    branches:
+      - 'main'
+      - 'stable-**'
+      - '**-stable'
 
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
@@ -64,7 +73,7 @@ jobs:
         run: ./scripts/host-testbench.sh
 
 
-  SOF-build-only:
+  gcc-build-only:
     runs-on: ubuntu-20.04
 
     strategy:
@@ -92,7 +101,7 @@ jobs:
   # before committing it; also deemed overkill for the current amount of
   # duplication.
 
-  SOF-qemu-boot:
+  qemu-boot-test:
     runs-on: ubuntu-20.04
 
     strategy:
@@ -108,20 +117,18 @@ jobs:
       - uses: actions/checkout@v2
         with: {fetch-depth: 0, submodules: recursive}
 
-      # FIXME: could/should probably use ./scripts/xtensa-build-all.sh
-      # -o agent.config instead of hiding this here.
       - name: turn off HAVE_AGENT
-        run: sed -i $(($(grep "config HAVE_AGENT" -n src/platform/Kconfig |
-          cut -f1 -d:)+2))"s/default y/default n/" src/platform/Kconfig
+        run: echo CONFIG_HAVE_AGENT=n >
+          src/arch/xtensa/configs/override/no-agent.config
 
       - name: docker SOF
         run: docker pull thesofproject/sof && docker tag thesofproject/sof sof
 
-      - name: xtensa-build-all
+      - name: xtensa-build-all -o no-agent
         env:
           PLATFORM: ${{ matrix.platform }}
         run: ./scripts/docker-run.sh
-          ./scripts/xtensa-build-all.sh -r ${PLATFORM}
+          ./scripts/xtensa-build-all.sh -o no-agent -r ${PLATFORM}
 
       - name: docker QEMU
         run: docker pull thesofproject/sofqemu &&
@@ -133,7 +140,8 @@ jobs:
         run: ./scripts/docker-qemu.sh
           ../sof.git/scripts/qemu-check.sh ${PLATFORM}
 
-  yamllint-actions:
+  yamllint-gh:
+    name: yamllint /.github/workflows/
     runs-on: ubuntu-20.04
 
     steps:

.github/workflows/scant_fuzzer.yml

@@ -0,0 +1,43 @@
+---
+
+# This is NOT how to properly compile the fuzzer, this is only to detect
+# earlier and faster problems like
+# https://github.com/thesofproject/sof/pull/3824 (288e228) and
+# https://github.com/thesofproject/sof/pull/4118 (758e7be)
+#
+# In other words, when reverting the oss-fuzz parts of these commits
+# this build fails as desired.
+#
+# For the actual fuzzer see tools/oss-fuzz/README.
+
+name: Scant fuzz compile
+
+# 'workflow_dispatch' allows running this workflow manually from the
+# 'Actions' tab
+# yamllint disable-line rule:truthy
+on: [push, pull_request, workflow_dispatch]
+
+jobs:
+  scant-fuzzer-build:
+    runs-on: ubuntu-20.04
+    env:
+      OUT: unused_runtime_output_dir
+    steps:
+      - uses: actions/checkout@v2
+        # From time to time this will catch a git tag and change SOF_VERSION
+        with: {fetch-depth: 10, submodules: recursive}
+
+      - name: yamllint ourselves
+        run: yamllint .github/workflows/scant_fuzzer.yml
+
+      - name: cmake configuration
+        run: cmake -B fuzz-build/ -S tools/oss-fuzz/
+
+      # Ninja does not give us access to the internal fuzz_ipc.o target
+      # so we use make. TODO: isolate the link target in CMakelists.txt
+      - name: build
+        run: |
+            cd fuzz-build/ # make -C is too verbose
+            # We cannot link because we have no main(); we're not the
+            # real fuzzer
+            make -j6 sof_ep fuzz_ipc.o

.github/workflows/shallow_checkpatch.yml

@@ -0,0 +1,53 @@
+---
+# SPDX-License-Identifier: BSD-3-Clause
+# Tools that can save round-trips to github and a lot of time:
+#
+# yamllint -f parsable this.yml
+# pip3 install ruamel.yaml.cmd
+# yaml merge-expand this.yml exp.yml && diff -w -u this.yml exp.yml
+#
+# github.com also has a powerful web editor that can be used without
+# committing.
+
+name: checkpatch
+
+# yamllint disable-line rule:truthy
+on: [pull_request]
+
+jobs:
+  checkpatch:
+    runs-on: ubuntu-20.04
+    env:
+      PR_NUM: ${{github.event.number}}
+      # TODO: reduce duplication with scripts/sof-*-commit-hook.sh
+      # thanks to either some .conf file or some wrapper script
+      CHK_CMD_OPTS: --ignore UNKNOWN_COMMIT_ID --codespell --codespellfile
+          /usr/lib/python3/dist-packages/codespell_lib/data/dictionary.txt
+         --ignore C99_COMMENT_TOLERANCE
+    steps:
+      # depth 2 so:
+      # ^1. we can show the Subject of the current target branch tip
+      # ^2. we reconnect/graft to the later fetch pull/1234/head,
+      - uses: actions/checkout@v2
+        with: {fetch-depth: 2}
+
+      - name: yamllint ourselves
+        run: yamllint .github/workflows/shallow_checkpatch.yml
+
+      - name: install codespell
+        run: sudo apt-get -y install codespell && dpkg -L codespell | grep dict
+
+      # See shallowness issue https://github.com/thesofproject/linux/issues/2556
+      - name: fetch PR commits
+        run: |
+           .github/workflows/shallowfetchPRcommits.sh \
+              ${GITHUB_REPOSITORY} "$PR_NUM"
+           # show what we got
+           git --no-pager log --oneline --graph --decorate --max-count=50
+
+      - name: normal checkpatch
+        run: .github/workflows/checkpatch_list.sh ${CHK_CMD_OPTS} < PR_SHAs.txt
+
+      - name: checkpatch --strict
+        run: .github/workflows/checkpatch_list.sh ${CHK_CMD_OPTS}
+                 --strict < PR_SHAs.txt

.github/workflows/shallowfetchPRcommits.sh

@@ -0,0 +1,69 @@
+#!/bin/sh
+# SPDX-License-Identifier: BSD-3-Clause
+
+set -e
+
+# This script fetches Pull Request commits missing from a shallow clone
+# and creates a PR_SHAs.txt file. This script has a limit of 500 commits but the
+# github API used has a lower limit of 250 commits.
+
+# It does not rely on git merge-bases which basically don't work with
+# shallow clones:
+# https://github.com/thesofproject/linux/issues/2556
+
+# Design goals:
+#
+# - Keep the code short and as simple as possible. No one is interested
+#   in maintaining this sort of script.
+#
+# - Fast and accurate for small Pull Requests
+#
+# - For large Pull Requests _with merges_ the only objective is to
+#   complete in a reasonable time; say less than 10 minutes. It's very
+#   unlikely will look at 250 checkpatch results and time optimizations
+#   should not make this script more complex.
+
+
+# Sample usage:
+#   $0  thesoftproject/linux  2772
+main()
+{
+    local gh_project="$1"
+    local pr_number="$2"
+
+    printf '%s: fetching PR %d for project %s\n' "$0" "$pr_number" "$gh_project"
+
+    # As of March 2021, Github's documented limit is 250 commits
+    # Let's have another cap a 500.
+    # https://docs.github.com/en/rest/reference/pulls#list-commits-on-a-pull-request
+    local pagelen PRlen=0
+    for i in 1 2 3 4 5; do
+        curl -H 'Accept: application/vnd.github.v3+json' \
+         "https://api.github.com/repos/$gh_project/pulls/$pr_number/commits?per_page=100&page=$i" \
+         > commits_"$i".json
+        pagelen=$(jq length < commits_$i.json)
+        if [ "$pagelen" -eq 0 ]; then
+            break
+        fi
+        PRlen=$((PRlen + pagelen))
+    done
+
+    printf 'Found %d commits, SHA1 list is in PR_SHAs.txt\n' "$PRlen"
+
+    # 'cut' removes double quotes
+    cat commits_?.json |
+        jq '.[] | .sha' |
+        cut -b2-41 > PR_SHAs.txt
+
+    # PRlen+1 gets us the merge base for simple, linear histories. For
+    # pull requests with merges, depth=PRLEN goes already much further
+    # than needed and +1 makes little difference. It's not clear when
+    # and for what sort of PRs git fetching individual commits would be
+    # faster so keep a single and simple fetch for now.
+
+    set -x # this command may take a while so show it
+    git fetch --depth "$((PRlen+1))" "https://github.com/$gh_project" "pull/$pr_number/head"
+
+}
+
+main "$@"

.github/workflows/zephyr.yml

@@ -0,0 +1,24 @@
+---
+
+name: Zephyr
+
+# 'workflow_dispatch' allows running this workflow manually from the
+# 'Actions' tab
+# yamllint disable-line rule:truthy
+on: [push, pull_request, workflow_dispatch]
+
+jobs:
+  zephyr-build:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+        # From time to time this will catch a git tag and change SOF_VERSION
+        with: {fetch-depth: 10, submodules: recursive}
+
+      - name: yamllint ourselves
+        run: yamllint .github/workflows/zephyr.yml
+
+      - name: build
+        run: docker run -v "$(pwd)":/workdir
+             docker.io/zephyrprojectrtos/zephyr-build:latest
+             ./zephyr/docker-build.sh

.gitmodules

@@ -1,4 +1,3 @@
 [submodule "rimage"]
 	path = rimage
-	url = https://github.com/thesofproject/rimage.git
-	branch = master
+	url = ../rimage.git

.travis.yml

@@ -57,9 +57,8 @@ jobs:
     - &qemuboottest
       stage: tests
       script:
-        - sed -i $(($(grep "config HAVE_AGENT" -n src/platform/Kconfig |
-             cut -f1 -d:)+2))"s/default y/default n/" src/platform/Kconfig
-        - ./scripts/docker-run.sh ./scripts/xtensa-build-all.sh -r $PLATFORM
+        - echo CONFIG_HAVE_AGENT=n > src/arch/xtensa/configs/override/no-agent.config
+        - ./scripts/docker-run.sh ./scripts/xtensa-build-all.sh -o no-agent -r $PLATFORM
         - ./scripts/docker-qemu.sh ../sof.git/scripts/qemu-check.sh $PLATFORM
       env: PLATFORM='byt cht'
       before_install: