update from origin

.github/workflows/README.md

@@ -47,7 +47,3 @@
  - `NPM_PACKAGE_NAME`: Not a true "secret", but stored here to avoid someone
    pushing bogus packages to NPM during CI testing from a fork
    - In a fork, set to a private name which differs from the production one
- - `SHAKA_BOT_TOKEN`: A GitHub personal access token for the `shaka-bot`
-   account, with `workflow` scope
-   - To generate, visit https://github.com/settings/tokens/new and select the
-     `workflow` scope

.github/workflows/build_and_test.yaml

@@ -38,25 +38,33 @@ jobs:
     needs: lint
     strategy:
       matrix:
-        os: ["ubuntu-latest", "macos-latest", "windows-latest"]
+        os: ["ubuntu-latest", "macos-latest", "windows-latest", "self-hosted-linux-arm64"]
         build_type: ["Debug", "Release"]
         lib_type: ["static", "shared"]
         include:
           - os: ubuntu-latest
             os_name: linux
+            target_arch: x64
             exe_ext: ""
             build_type_suffix: ""
           - os: macos-latest
             os_name: osx
+            target_arch: x64
             exe_ext: ""
             build_type_suffix: ""
           - os: windows-latest
             os_name: win
+            target_arch: x64
             exe_ext: ".exe"
             # 64-bit outputs on Windows go to a different folder name.
             build_type_suffix: "_x64"
+          - os: self-hosted-linux-arm64
+            os_name: linux
+            target_arch: arm64
+            exe_ext: ""
+            build_type_suffix: ""
 
-    name: Build and test ${{ matrix.os_name }} ${{ matrix.build_type }} ${{ matrix.lib_type }}
+    name: Build and test ${{ matrix.os_name }} ${{ matrix.target_arch }} ${{ matrix.build_type }} ${{ matrix.lib_type }}
     runs-on: ${{ matrix.os }}
 
     steps:
@@ -79,6 +87,7 @@ jobs:
         uses: ./src/.github/workflows/custom-actions/build-packager
         with:
           os_name: ${{ matrix.os_name }}
+          target_arch: ${{ matrix.target_arch }}
           lib_type: ${{ matrix.lib_type }}
           build_type: ${{ matrix.build_type }}
           build_type_suffix: ${{ matrix.build_type_suffix }}

.github/workflows/custom-actions/build-packager/action.yaml

@@ -8,6 +8,9 @@ inputs:
   os_name:
     description: The name of the OS (one word).  Appended to artifact filenames.
     required: true
+  target_arch:
+    description: The CPU architecture to target.  We support x64, arm64.
+    required: true
   lib_type:
     description: A library type, either "static" or "shared".
     required: true
@@ -48,6 +51,28 @@ runs:
         echo "${GITHUB_WORKSPACE}/depot_tools" >> $GITHUB_PATH
         echo "::endgroup::"
 
+    - name: Build ninja (arm only)
+      shell: bash
+      run: |
+        # NOTE: There is no prebuilt copy of ninja for the "aarch64"
+        # architecture (as reported by "uname -p" on arm64).  So we must build
+        # our own, as recommended by depot_tools when it fails to fetch a
+        # prebuilt copy for us.
+        # NOTE 2: It turns out that $GITHUB_PATH operates like a stack.
+        # Appending to that file places the new path at the beginning of $PATH
+        # for the next step, so this step must come _after_ installing
+        # depot_tools.
+        if [[ "${{ inputs.target_arch }}" == "arm64" ]]; then
+          echo "::group::Build ninja (arm-only)"
+          git clone https://github.com/ninja-build/ninja.git -b v1.8.2
+          # The --bootstrap option compiles ninja as well as configures it.
+          # This is the exact command prescribed by depot_tools when it fails to
+          # fetch a ninja binary for your platform.
+          (cd ninja && ./configure.py --bootstrap)
+          echo "${GITHUB_WORKSPACE}/ninja" >> $GITHUB_PATH
+          echo "::endgroup::"
+        fi
+
     - name: Configure gclient
       shell: bash
       run: |
@@ -57,14 +82,17 @@ runs:
 
     - name: Sync gclient
       env:
-        MACOSX_DEPLOYMENT_TARGET: "10.10"
-        GYP_DEFINES: "target_arch=x64 libpackager_type=${{ inputs.lib_type }}_library"
-        DEPOT_TOOLS_WIN_TOOLCHAIN: "0"
+        GYP_DEFINES: "target_arch=${{ inputs.target_arch }} libpackager_type=${{ inputs.lib_type }}_library"
         GYP_MSVS_VERSION: "2019"
         GYP_MSVS_OVERRIDE_PATH: "C:/Program Files (x86)/Microsoft Visual Studio/2019/Enterprise"
       shell: bash
       run: |
         echo "::group::Sync gclient"
+        BUILD_CONFIG="${{ inputs.build_type }}-${{ inputs.lib_type }}"
+        if [[ "$BUILD_CONFIG" == "Release-static" && "${{ runner.os }}" == "Linux" ]]; then
+          # For static release builds, set these two additional flags for fully static binaries.
+          export GYP_DEFINES="$GYP_DEFINES disable_fatal_linker_warnings=1 static_link_binaries=1"
+        fi
         gclient sync
         echo "::endgroup::"
 
@@ -83,27 +111,45 @@ runs:
           echo "Skipping artifacts for $BUILD_CONFIG."
           exit 0
         fi
+        if [[ "${{ runner.os }}" == "Linux" ]]; then
+          echo "::group::Check for static executables"
+          (
+            cd src/out/Release${{ inputs.build_type_suffix }}
+            # Prove that we built static executables on Linux.  First, check that
+            # the executables exist, and fail if they do not.  Then check "ldd",
+            # which will fail if the executable is not dynamically linked.  If
+            # "ldd" succeeds, we fail the workflow.  Finally, we call "true" so
+            # that the last executed statement will be a success, and the step
+            # won't be failed if we get that far.
+            ls packager mpd_generator >/dev/null || exit 1
+            ldd packager 2>&1 && exit 1
+            ldd mpd_generator 2>&1 && exit 1
+            true
+          )
+          echo "::endgroup::"
+        fi
         echo "::group::Prepare artifacts folder"
         mkdir artifacts
         ARTIFACTS="$GITHUB_WORKSPACE/artifacts"
         cd src/out/Release${{ inputs.build_type_suffix }}
         echo "::endgroup::"
+        echo "::group::Strip executables"
+        strip packager${{ inputs.exe_ext }}
+        strip mpd_generator${{ inputs.exe_ext }}
+        echo "::endgroup::"
+        SUFFIX="-${{ inputs.os_name }}-${{ inputs.target_arch }}"
+        EXE_SUFFIX="$SUFFIX${{ inputs.exe_ext}}"
         echo "::group::Copy packager"
-        cp packager${{ inputs.exe_ext }} \
-            $ARTIFACTS/packager-${{ inputs.os_name }}${{ inputs.exe_ext }}
+        cp packager${{ inputs.exe_ext }} $ARTIFACTS/packager$EXE_SUFFIX
         echo "::endgroup::"
         echo "::group::Copy mpd_generator"
-        cp mpd_generator${{ inputs.exe_ext }} \
-            $ARTIFACTS/mpd_generator-${{ inputs.os_name }}${{ inputs.exe_ext }}
+        cp mpd_generator${{ inputs.exe_ext }} $ARTIFACTS/mpd_generator$EXE_SUFFIX
         echo "::endgroup::"
-        if [[ '${{ runner.os }}' == 'Windows' ]]; then
-          echo "::group::Zip pssh-box"
-          7z a $ARTIFACTS/pssh-box-${{ inputs.os_name }}.py.zip \
-              pyproto pssh-box.py
-          echo "::endgroup::"
-        else
+        # The pssh-box bundle is OS and architecture independent.  So only do
+        # it on this one OS and architecture, and give it a more generic
+        # filename.
+        if [[ '${{ inputs.os_name }}' == 'linux' && '${{ inputs.target_arch }}' == 'x64' ]]; then
           echo "::group::Tar pssh-box"
-          tar -czf $ARTIFACTS/pssh-box-${{ inputs.os_name }}.py.tar.gz \
-              pyproto pssh-box.py
+          tar -czf $ARTIFACTS/pssh-box.py.tar.gz pyproto pssh-box.py
           echo "::endgroup::"
         fi

.github/workflows/github_release.yaml

@@ -84,7 +84,7 @@ jobs:
         id: draft_release
         uses: actions/create-release@v1
         env:
-          GITHUB_TOKEN: ${{ secrets.SHAKA_BOT_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           tag_name: ${{ needs.setup.outputs.tag }}
           release_name: ${{ needs.setup.outputs.tag }}
@@ -112,25 +112,33 @@ jobs:
     needs: [setup, lint, draft_release]
     strategy:
       matrix:
-        os: ["ubuntu-latest", "macos-latest", "windows-latest"]
+        os: ["ubuntu-latest", "macos-latest", "windows-latest", "self-hosted-linux-arm64"]
         build_type: ["Debug", "Release"]
         lib_type: ["static", "shared"]
         include:
           - os: ubuntu-latest
             os_name: linux
+            target_arch: x64
             exe_ext: ""
             build_type_suffix: ""
           - os: macos-latest
             os_name: osx
+            target_arch: x64
             exe_ext: ""
             build_type_suffix: ""
           - os: windows-latest
             os_name: win
+            target_arch: x64
             exe_ext: ".exe"
             # 64-bit outputs on Windows go to a different folder name.
             build_type_suffix: "_x64"
+          - os: self-hosted-linux-arm64
+            os_name: linux
+            target_arch: arm64
+            exe_ext: ""
+            build_type_suffix: ""
 
-    name: Build and test ${{ matrix.os_name }} ${{ matrix.build_type }} ${{ matrix.lib_type }}
+    name: Build and test ${{ matrix.os_name }} ${{ matrix.target_arch }} ${{ matrix.build_type }} ${{ matrix.lib_type }}
     runs-on: ${{ matrix.os }}
 
     steps:
@@ -153,6 +161,7 @@ jobs:
         uses: ./src/.github/workflows/custom-actions/build-packager
         with:
           os_name: ${{ matrix.os_name }}
+          target_arch: ${{ matrix.target_arch }}
           lib_type: ${{ matrix.lib_type }}
           build_type: ${{ matrix.build_type }}
           build_type_suffix: ${{ matrix.build_type_suffix }}
@@ -170,7 +179,7 @@ jobs:
         if: matrix.build_type == 'Release' && matrix.lib_type == 'static'
         uses: dwenegar/upload-release-assets@v1
         env:
-          GITHUB_TOKEN: ${{ secrets.SHAKA_BOT_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           release_id: ${{ needs.draft_release.outputs.release_id }}
           assets_path: artifacts
@@ -183,6 +192,6 @@ jobs:
       - name: Publish release
         uses: eregon/publish-release@v1
         env:
-          GITHUB_TOKEN: ${{ secrets.SHAKA_BOT_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           release_id: ${{ needs.draft_release.outputs.release_id }}

.github/workflows/update_docs.yaml

@@ -45,6 +45,6 @@ jobs:
       - name: Deploy to gh-pages branch
         uses: peaceiris/actions-gh-pages@v3
         with:
-          github_token: ${{ secrets.SHAKA_BOT_TOKEN }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
           publish_dir: gh-pages
           full_commit_message: Generate docs for ${{ steps.ref.outputs.ref }}

DEPS

@@ -18,28 +18,12 @@ deps = {
   "src/packager/build":
     Var("chromium_git") + "/chromium/src/build@f0243d787961584ac95a86e7dae897b9b60ea674",  #409966
 
-  "src/packager/buildtools/third_party/libc++/trunk":
-    Var("github") + "/llvm-mirror/libcxx.git@8c22696675a2c5ea1c79fc64a4d7dfe1c2f4ca8b",
-
-  "src/packager/buildtools/third_party/libc++abi/trunk":
-    Var("github") + "/llvm-mirror/libcxxabi.git@6092bfa6c153ad712e2fc90c7b9e536420bf3c57",
-
   "src/packager/testing/gmock":
     Var("chromium_git") + "/external/googlemock@0421b6f358139f02e102c9c332ce19a33faf75be",  #566
 
   "src/packager/testing/gtest":
     Var("chromium_git") + "/external/github.com/google/googletest@6f8a66431cb592dad629028a50b3dd418a408c87",
 
-  # Keep bundled binutil scripts but not downloading actual binaries by default.
-  # Automatic downloading of binutils has been causing problems for some users:
-  # #164, #412, #440. Using bundled binutils helps reduce linking time, but
-  # packager codebase is relatively small, so the gain is not significant.
-  # User can still enable the usage of bundled binutils by running
-  # 'python src/packager/third_party/binutils/download.py' and set
-  # 'linux_use_bundled_binutils' and 'linux_use_bundled_gold' to 1 in GYP_DEFINES.
-  "src/packager/third_party/binutils":
-    Var("chromium_git") + "/chromium/src/third_party/binutils@8d77853bc9415bcb7bb4206fa2901de7603387db",
-
   # Make sure the version matches the one in
   # src/packager/third_party/boringssl, which contains perl generated files.
   "src/packager/third_party/boringssl/src":
@@ -70,22 +54,11 @@ deps = {
   "src/packager/third_party/zlib":
     Var("chromium_git") + "/chromium/src/third_party/zlib@830b5c25b5fbe37e032ea09dd011d57042dd94df",  #408157
 
-  "src/packager/tools/clang":
-    Var("chromium_git") + "/chromium/src/tools/clang@723b25997f0aab45fe1776a0f74a14782e350f8f",  #513983
-
   "src/packager/tools/gyp":
     Var("chromium_git") + "/external/gyp@caa60026e223fc501e8b337fd5086ece4028b1c6",
-
-  "src/packager/tools/valgrind":
-    Var("chromium_git") + "/chromium/deps/valgrind@3a97aa8142b6e63f16789b22daafb42d202f91dc",
 }
 
 deps_os = {
-  "unix": {  # Linux, actually.
-    # Linux gold build to build faster.
-    "src/packager/third_party/gold":
-      Var("chromium_git") + "/chromium/deps/gold@29ae7431b4688df544ea840b0b66784e5dd298fe",
-  },
   "win": {
     # Required by boringssl.
     "src/packager/third_party/yasm/source/patched-yasm":
@@ -95,26 +68,14 @@ deps_os = {
 
 hooks = [
   {
-    # Downloads the current stable linux sysroot to build/linux/ if needed.
-    # This script is a no-op except for linux.
-    'name': 'sysroot',
+    # When using CC=clang CXX=clang++, there is a binutils version check that
+    # does not work correctly in common.gypi.  Since we are stuck with a very
+    # old version of chromium/src/build, there is nothing to do but patch it to
+    # remove the check.  Thankfully, this version number does not control
+    # anything critical in the build settings as far as we can tell.
+    'name': 'patch-binutils-version-check',
     'pattern': '.',
-    'action': ['python', 'src/packager/build/linux/sysroot_scripts/install-sysroot.py',
-               '--running-as-hook'],
-  },
-  {
-    # Update the Mac toolchain if necessary.
-    'name': 'mac_toolchain',
-    'pattern': '.',
-    'action': ['python', 'src/packager/build/mac_toolchain.py'],
-  },
-  {
-    # Pull clang if needed or requested via GYP_DEFINES (GYP_DEFINES="clang=1").
-    "name": "clang",
-    # Skip clang updates on Windows, where we don't use clang.
-    "condition": "not checkout_win",
-    "pattern": ".",
-    "action": ["python", "src/packager/tools/clang/scripts/update.py", "--if-needed"],
+    'action': ['sed', '-e', 's/<!pymod_do_main(compiler_version target assembler)/0/', '-i.bk', 'src/packager/build/common.gypi'],
   },
   {
     # A change to a .gyp, .gypi, or to GYP itself should run the generator.

Dockerfile

@@ -14,7 +14,7 @@ RUN sed -i \
     '/malloc_usable_size/a \\nstruct mallinfo {\n  int arena;\n  int hblkhd;\n  int uordblks;\n};' \
     /usr/include/malloc.h
 
-ENV GYP_DEFINES='clang=0 use_experimental_allocator_shim=0 use_allocator=none musl=1'
+ENV GYP_DEFINES='musl=1'
 # Alpine does not support python3 yet, but depot_tools enabled python3
 # by default. Disable python3 explicitly for now.
 # See https://github.com/google/shaka-packager/issues/763 for details.

docs/linux_profiling.md

@@ -1,9 +1,9 @@
 # Linux Profiling
 
 Profiling code is enabled when the `use_allocator` variable in gyp is set to
-`tcmalloc` (currently the default) and `profiling` variable in gyp is set to
-`1`. That will build the tcmalloc library, including the cpu profiling and heap
-profiling code into shaka-packager, e.g.
+`tcmalloc` and `profiling` variable in gyp is set to `1`. That will build the
+tcmalloc library, including the cpu profiling and heap profiling code into
+shaka-packager, e.g.
 
     GYP_DEFINES='profiling=1 use_allocator="tcmalloc"' gclient runhooks
 
@@ -75,6 +75,35 @@ Or you can use gdb to attach at any point:
 3.  The filename will be printed on the console, e.g.
     "`Dumping heap profile to heap.0001.heap (foobar)`"
 
+
+## Thread sanitizer (tsan)
+
+To compile with the thread sanitizer library (tsan), you must set clang as your
+compiler and set the `tsan=1` and `tsan_blacklist` configs:
+
+    CC=clang CXX=clang++ GYP_DEFINES="tsan=1 tsan_blacklist=/path/to/src/packager/tools/memory/tsan_v2/ignores.txt" gclient runhooks
+
+NOTE: tsan and asan cannot be used at the same time.
+
+
+## Adddress sanitizer (asan)
+
+To compile with the address sanitizer library (asan), you must set clang as your
+compiler and set the `asan=1` config:
+
+    CC=clang CXX=clang++ GYP_DEFINES="asan=1" gclient runhooks
+
+NOTE: tsan and asan cannot be used at the same time.
+
+
+## Leak sanitizer (lsan)
+
+To compile with the leak sanitizer library (lsan), you must set clang as your
+compiler and set the `lsan=1` config:
+
+    CC=clang CXX=clang++ GYP_DEFINES="lsan=1" gclient runhooks
+
+
 ## Reference
 
 [Linux Profiling in Chromium](https://chromium.googlesource.com/chromium/src/+/master/docs/linux_profiling.md)