Merge remote-tracking branch 'upstream/master'

* upstream/master: (76 commits) [CI] Automatically refresh docker images (kaldi-asr#4942) make sure nvtx3 is used (kaldi-asr#4937) [CI] Upgrade checkout action (kaldi-asr#4940) Upload logs after build failure (kaldi-asr#4939) upload the error logs to artifact repository (kaldi-asr#4938) fix cuda build with openfst 1.8.3 (kaldi-asr#4936) fix the non-compiling tests, thx to @csukuangfj add support for later openfst versions make nonconst catches const (kaldi-asr#4926) Support openfst-1.8.2 Support openfst-1.8.1 Support openfst-1.8.0 Support openfst-1.7.6 make codefactor happier improve compatibility with C++ standard, esp. C++20 do a full cleanup on apple silicon disable warning about unused flags msse and msse on Apple Silicon fix tests and address comments catch exception by reference so that compiler does not complain Fix reported issues w.r.t python2.7 and some apple silicone quirks ...
tiro-is · Sep 27, 2024 · 406479a · 406479a
2 parents 6637524 + d9ab046
commit 406479a
Show file tree

Hide file tree

Showing 217 changed files with 3,545 additions and 908 deletions.
diff --git a/.github/workflows/c-cpp.yml b/.github/workflows/c-cpp.yml
@@ -16,9 +16,11 @@ jobs:
       CC: "ccache gcc"
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Install sox
       run: sudo apt-get install -y sox intel-mkl
+    - name: Install python2
+      run: sudo apt-get install -y python2
     - name: ccache
       uses: hendrikmuhs/ccache-action@v1.2
       with:
@@ -36,3 +38,9 @@ jobs:
       run: cd src &&  make -j 3
     - name: make test
       run: cd src && make test
+    - name: upload logs if failure
+      if: ${{ failure() }}
+      uses: actions/upload-artifact@v4
+      with:
+         name: fail-logs
+         path: ${{ github.workspace }}/src/**/*testlog
diff --git a/.github/workflows/docker-images.yml b/.github/workflows/docker-images.yml
@@ -3,6 +3,7 @@ name: Docker Image CI
 on:
   schedule:
   - cron: '37 2 * * 1'
+
   workflow_dispatch:
     inputs:
       logLevel:
@@ -14,17 +15,20 @@ on:
           - info
           - warning
           - debug
+
 #  pull_request: #for debugging purposes
 #    branches: [ "master" ]
 
 jobs:
+
   enable_build:
-    if: github.repository == 'kaldi-asr/kaldi'
+    #if: github.repository_owner == 'jtrmal' || github.repository_owner == 'kaldi-asr'
+    if: github.repository_owner == 'kaldi-asr'
     runs-on: ubuntu-latest
     outputs:
       enabled: ${{ steps.set-enabled.outputs.enabled }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Set enabled
@@ -38,22 +42,34 @@ jobs:
             enabled=true
           fi
           echo "enabled: $enabled"
-          echo "::set-output name=enabled::${enabled}"
+          echo "enabled=${enabled}" >> $GITHUB_OUTPUT
 
 
-  docker-buildx-gpu:
+  docker-buildx-gpu-12:
     needs: enable_build
     if: needs.enable_build.outputs.enabled == 'true' || github.event_name == 'push' || github.event_name == 'workflow_dispatch'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - name: Maximize build space
+        uses: AdityaGarg8/remove-unwanted-software@v4.1
+        with:
+          remove-android: 'true'
+          remove-dotnet: 'true'
+          remove-haskell: 'true'
+          remove-codeql: 'true'
+          remove-docker-images: 'true'
+          remove-large-packages: 'true'
+          remove-cached-tools: 'true'
+          remove-swapfile: 'false'
+          verbose: 'true'
+      - uses: actions/checkout@v4
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
         with:
           install: true
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -66,22 +82,34 @@ jobs:
           echo "Platforms: ${{ steps.buildx.outputs.platforms }}"
       - name: Build and push
         run: |
-          cd docker/ubuntu18.04-cuda10.0/
-          docker build --push --tag kaldiasr/kaldi:gpu-latest --tag kaldiasr/kaldi:gpu-ubuntu18.04-cuda10.0 --tag kaldiasr/kaldi:gpu-ubuntu18.04-cuda10.0-$(date +%F) .
+          cd docker/ubuntu22.04-cuda12
+          docker build --push  --tag kaldiasr/kaldi:gpu-latest --tag kaldiasr/kaldi:gpu-ubuntu22.04-cuda12 --tag kaldiasr/kaldi:gpu-ubuntu22.04-cuda12-$(date +%F) .
 
-  docker-buildx-cpu:
+  docker-buildx-gpu-cuda11:
     needs: enable_build
     if: needs.enable_build.outputs.enabled == 'true' || github.event_name == 'push' || github.event_name == 'workflow_dispatch'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - name: Maximize build space
+        uses: AdityaGarg8/remove-unwanted-software@v4.1
+        with:
+          remove-android: 'true'
+          remove-dotnet: 'true'
+          remove-haskell: 'true'
+          remove-codeql: 'true'
+          remove-docker-images: 'true'
+          remove-large-packages: 'true'
+          remove-cached-tools: 'true'
+          remove-swapfile: 'false'
+          verbose: 'true'
+      - uses: actions/checkout@v4
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
         with:
           install: true
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -94,10 +122,71 @@ jobs:
           echo "Platforms: ${{ steps.buildx.outputs.platforms }}"
       - name: Build and push
         run: |
-          cd docker/debian10-cpu/
+          cd docker/ubuntu20.04-cuda11
+          docker build --push  --tag kaldiasr/kaldi:gpu-ubuntu20.04-cuda11 --tag kaldiasr/kaldi:gpu-ubuntu20.04-cuda11-$(date +%F) .
+
+  docker-buildx-cpu-openblas:
+    needs: enable_build
+    if: needs.enable_build.outputs.enabled == 'true' || github.event_name == 'push' || github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          install: true
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Inspect builder
+        run: |
+          echo "Name:      ${{ steps.buildx.outputs.name }}"
+          echo "Endpoint:  ${{ steps.buildx.outputs.endpoint }}"
+          echo "Status:    ${{ steps.buildx.outputs.status }}"
+          echo "Flags:     ${{ steps.buildx.outputs.flags }}"
+          echo "Platforms: ${{ steps.buildx.outputs.platforms }}"
+      - name: Build and push
+        run: |
+          cd docker/debian12-cpu/
           docker build --push \
                        --tag kaldiasr/kaldi:latest \
                        --tag kaldiasr/kaldi:cpu-latest  \
-                       --tag kaldiasr/kaldi:cpu-debian10 \
-                       --tag kaldiasr/kaldi:cpu-debian10-$(date +%F) .
+                       --tag kaldiasr/kaldi:cpu-latest-openblas  \
+                       --tag kaldiasr/kaldi:cpu-debian12-openblas \
+                       --tag kaldiasr/kaldi:cpu-debian12-openblas-$(date +%F) .
 
+  docker-buildx-cpu-mkl:
+    needs: enable_build
+    if: needs.enable_build.outputs.enabled == 'true' || github.event_name == 'push' || github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          install: true
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Inspect builder
+        run: |
+          echo "Name:      ${{ steps.buildx.outputs.name }}"
+          echo "Endpoint:  ${{ steps.buildx.outputs.endpoint }}"
+          echo "Status:    ${{ steps.buildx.outputs.status }}"
+          echo "Flags:     ${{ steps.buildx.outputs.flags }}"
+          echo "Platforms: ${{ steps.buildx.outputs.platforms }}"
+      - name: Build and push
+        run: |
+          cd docker/debian12-cpu-mkl/
+          docker build --push \
+                       --tag kaldiasr/kaldi:cpu-latest-mkl  \
+                       --tag kaldiasr/kaldi:cpu-debian12-mkl \
+                       --tag kaldiasr/kaldi:cpu-debian12-mkl-$(date +%F) .
+ 
+
diff --git a/COPYING b/COPYING
@@ -57,72 +57,72 @@ License v 2.0 are set forth below.
 
 Individual Contributors (in alphabetical order)
 
-      Mohit Agarwal
-      Tanel Alumae
-      Gilles Boulianne
-      Lukas Burget
-      Dogan Can
-      Guoguo Chen
-      Gaofeng Cheng
+      Albert Vernon
+      Alexander Solovets
+      Allen Guo
+      Ariya Rastrow
+      Arnab Ghoshal
       Cisco Corporation
-      Pavel Denisov
-      Ilya Edrenkin
-      Ewald Enzinger
-      Joachim Fainberg
       Daniel Galvez
-      Pegah Ghahremani
-      Arnab Ghoshal
-      Ondrej Glembek
+      Daniel Povey
+      Danijel Korzinek
+      David Snyder
+      Dogan Can
+      Eduardo Silva
+      Ewald Enzinger
+      Gaofeng Cheng
+      Gaurav Kumar
+      Georg Stemmer
+      Gilles Boulianne
       Go Vivace Inc.
-      Allen Guo
-      Hossein Hadian
-      Lv Hang
-      Mirko Hannemann
+      Guoguo Chen
+      Haihua Xu
+      Hainan Xu
       Hendy Irawan
-      Navdeep Jaitly
+      Hossein Hadian
+      Ilya Edrenkin
+      Jan "Yenda" Trmal
+      Jan Silovsky
+      Joachim Fainberg
       Johns Hopkins University
-      Shiyin Kang
-      Kirill Katsnelson
-      Tom Ko
-      Danijel Korzinek
-      Gaurav Kumar
+      Karel Vesely
       Ke Li
+      Kirill Katsnelson
+      Lucas Ondel
+      Lukas Burget
+      Lv Hang
       Matthew Maciejewski
-      Vimal Manohar
-      Yajie Miao
       Microsoft Corporation
+      Minhua Wu
+      Mirko Hannemann
+      Mohit Agarwal
+      Navdeep Jaitly
+      Nickolay V. Shmyrev
+      Omid Sadjadi
+      Ondrej Glembek
+      Ondrej Platek
+      Pavel Denisov
+      Pawel Swietojanski
+      Pegah Ghahremani
+      Peter Smit
       Petr Motlicek
-      Xingyu Na
-      Vincent Nguyen
-      Lucas Ondel
-      Vassil Panayotov
-      Vijayaditya Peddinti
+      Petr Schwarz
       Phonexia s.r.o.
-      Ondrej Platek
-      Daniel Povey
-      Yanmin Qian
-      Ariya Rastrow
       Saarland University
-      Omid Sadjadi
-      Petr Schwarz
-      Yiwen Shao
-      Nickolay V. Shmyrev
-      Jan Silovsky
-      Eduardo Silva
-      Peter Smit
-      David Snyder
-      Alexander Solovets
-      Georg Stemmer
-      Pawel Swietojanski
-      Jan "Yenda" Trmal
-      Albert Vernon
-      Karel Vesely
-      Yiming Wang
       Shinji Watanabe
-      Minhua Wu
-      Haihua Xu
-      Hainan Xu
+      Shiyin Kang
+      Tanel Alumae
+      Tom Ko
+      Vassil Panayotov
+      Vijayaditya Peddinti
+      Vimal Manohar
+      Vincent Nguyen
       Xiaohui Zhang
+      Xingyu Na
+      Yajie Miao
+      Yanmin Qian
+      Yiming Wang
+      Yiwen Shao
 
 Other Source Material
 

diff --git a/cmake/gen_cmake_skeleton.py b/cmake/gen_cmake_skeleton.py
@@ -269,7 +269,7 @@ def gen_code(self):
 
         if len(self.depends) > 0:
             ret.append("target_link_libraries(" + self.target_name + " PUBLIC")
-            for d in self.depends:
+            for d in self.depends + ['-lcblas', '-llapack']:
                 ret.append("    " + d)
             ret.append(")\n")
 

diff --git a/docker/debian10-cpu/Dockerfile → docker/debian12-cpu-mkl/Dockerfile b/docker/debian10-cpu/Dockerfile → docker/debian12-cpu-mkl/Dockerfile
@@ -1,9 +1,10 @@
-FROM debian:10
-LABEL maintainer="rick@scriptix.io"
+FROM debian:12
+LABEL maintainer="jtrmal@apptek.com"
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
         g++ \
+        gfortran \
         make \
         automake \
         autoconf \
@@ -13,29 +14,21 @@ RUN apt-get update && \
         sox \
         libtool \
         git \
-        subversion \
-        python2.7 \
         python3 \
         zlib1g-dev \
         ca-certificates \
-        gfortran \
         patch \
-        ffmpeg \
-	vim && \
+        python-is-python3 && \
     rm -rf /var/lib/apt/lists/*
 
-RUN ln -s /usr/bin/python3 /usr/bin/python
 
 RUN git clone --depth 1 https://github.com/kaldi-asr/kaldi.git /opt/kaldi #EOL
 RUN    cd /opt/kaldi/tools && \
-       ./extras/install_mkl.sh && \
-       make -j $(nproc) && \
+       ./extras/install_mkl.sh  && \
+       make -j 5 && \
        cd /opt/kaldi/src && \
        ./configure --shared && \
        make depend -j $(nproc) && \
-       make -j $(nproc) && \
-       find /opt/kaldi -type f \( -name "*.o" -o -name "*.la" -o -name "*.a" \) -exec rm {} \; && \
-       find /opt/intel -type f -name "*.a" -exec rm {} \; && \
-       find /opt/intel -type f -regex '.*\(_mc.?\|_mic\|_thread\|_ilp64\)\.so' -exec rm {} \; && \
-       rm -rf /opt/kaldi/.git
+       make -j 5 
+
 WORKDIR /opt/kaldi/