Test building with Sox on Windows

.circleci/config.yml

@@ -164,6 +164,7 @@ jobs:
             conda activate base
             conda install -yq conda-build "conda-package-handling!=1.5.0"
             bash packaging/build_conda.sh
+            rm /C/tools/miniconda3/conda-bld/win-64/vs2019*.tar.bz2
       - store_artifacts:
           path: C:/tools/miniconda3/conda-bld/win-64
       - persist_to_workspace:

.circleci/config.yml.in

@@ -164,6 +164,7 @@ jobs:
             conda activate base
             conda install -yq conda-build "conda-package-handling!=1.5.0"
             bash packaging/build_conda.sh
+            rm /C/tools/miniconda3/conda-bld/win-64/vs2019*.tar.bz2
       - store_artifacts:
           path: C:/tools/miniconda3/conda-bld/win-64
       - persist_to_workspace:

.circleci/unittest/windows/scripts/environment.yml

@@ -11,6 +11,5 @@ dependencies:
   - pip:
     - scipy
     - kaldi-io
-    - PySoundFile
     - librosa
     - future

.circleci/unittest/windows/scripts/install.bat

@@ -0,0 +1,20 @@
+@echo on
+
+for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -legacy -products * -version [15^,17^) -property installationPath`) do (
+    if exist "%%i" if exist "%%i\VC\Auxiliary\Build\vcvarsall.bat" (
+        set "VS15INSTALLDIR=%%i"
+        set "VS15VCVARSALL=%%i\VC\Auxiliary\Build\vcvarsall.bat"
+        goto vswhere
+    )
+)
+
+:vswhere
+if "%VSDEVCMD_ARGS%" == "" (
+    call "%VS15VCVARSALL%" x64 || exit /b 1
+) else (
+    call "%VS15VCVARSALL%" x64 %VSDEVCMD_ARGS% || exit /b 1
+)
+
+@echo on
+
+python setup.py develop || exit /b 1

.circleci/unittest/windows/scripts/install.sh

@@ -7,6 +7,11 @@ unset PYTORCH_VERSION
 
 set -e
 
+this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+root_dir="$(git rev-parse --show-toplevel)"
+
+cd "${root_dir}"
+
 eval "$(./conda/Scripts/conda.exe 'shell.bash' 'hook')"
 conda activate ./env
 
@@ -20,4 +25,6 @@ printf "Installing PyTorch with %s\n" "${cudatoolkit}"
 conda install -y -c pytorch-nightly pytorch "${cudatoolkit}"
 
 printf "* Installing torchaudio\n"
-IS_CONDA=true python setup.py develop
+curl --retry 3 https://s3.amazonaws.com/ossci-windows/torchaudio_deps.7z --output /tmp/torchaudio_deps.7z
+7z x /tmp/torchaudio_deps.7z -othird_party
+IS_CONDA=true "$this_dir/install.bat"

packaging/build_conda.sh

@@ -9,4 +9,5 @@ export NO_CUDA_PACKAGE=1
 setup_env 0.6.0
 export SOURCE_ROOT_DIR="$PWD"
 setup_conda_pytorch_constraint
+setup_visual_studio_constraint
 conda build $CONDA_CHANNEL_FLAGS --no-anaconda-upload --python "$PYTHON_VERSION" packaging/torchaudio

packaging/build_wheel.sh

@@ -8,15 +8,12 @@ export BUILD_TYPE="wheel"
 export NO_CUDA_PACKAGE=1
 setup_env 0.6.0
 setup_wheel_python
-if [[ "$OSTYPE" != "msys" ]]; then
+if [[ "$OSTYPE" == "msys" ]]; then
+    "$script_dir/download_deps.sh" "$(pwd)"  # Download static dependencies
+else
     "$script_dir/build_from_source.sh" "$(pwd)"  # Build static dependencies
 fi
 pip_install numpy future
 setup_pip_pytorch_version
 python setup.py clean
-if [[ "$OSTYPE" == "msys" ]]; then
-    python_tag="$(echo "cp$PYTHON_VERSION" | tr -d '.')"
-    IS_WHEEL=1 python setup.py bdist_wheel --plat-name win_amd64 --python-tag $python_tag
-else
-    IS_WHEEL=1 python setup.py bdist_wheel
-fi
+IS_WHEEL=1 python setup.py bdist_wheel

packaging/download_deps.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -ex
+
+# Arguments: PREFIX, specifying where to install dependencies into
+
+PREFIX="$(cygpath $1)"
+
+curl --retry 3 https://s3.amazonaws.com/ossci-windows/torchaudio_deps.7z --output /tmp/torchaudio_deps.7z
+7z x /tmp/torchaudio_deps.7z -o"$PREFIX/third_party"

packaging/pkg_helpers.bash

@@ -219,3 +219,13 @@ setup_conda_cudatoolkit_constraint() {
     esac
   fi
 }
+
+# Build the proper compiler package before building the final package
+setup_visual_studio_constraint() {
+  if [[ "$OSTYPE" == "msys" ]]; then
+      export VSTOOLCHAIN_PACKAGE=vs2019
+      export VSDEVCMD_ARGS=''
+      conda build $CONDA_CHANNEL_FLAGS --no-anaconda-upload packaging/$VSTOOLCHAIN_PACKAGE
+      cp packaging/$VSTOOLCHAIN_PACKAGE/conda_build_config.yaml packaging/torchaudio/conda_build_config.yaml
+  fi
+}

packaging/torchaudio/bld.bat

@@ -1,5 +1,8 @@
 @echo off
 
+sh packaging/download_deps.sh '%CD%'
+if errorlevel 1 exit /b 1
+
 set IS_CONDA=1
 
 python setup.py install --single-version-externally-managed --record=record.txt

packaging/vs2019/activate.bat

@@ -0,0 +1,44 @@
+:: Set env vars that tell distutils to use the compiler that we put on path
+SET DISTUTILS_USE_SDK=1
+SET MSSdk=1
+
+SET "VS_VERSION=16.0"
+SET "VS_MAJOR=16"
+SET "VS_YEAR=2019"
+
+set "MSYS2_ARG_CONV_EXCL=/AI;/AL;/OUT;/out"
+set "MSYS2_ENV_CONV_EXCL=CL"
+
+:: For Python 3.5+, ensure that we link with the dynamic runtime.  See
+:: http://stevedower.id.au/blog/building-for-python-3-5-part-two/ for more info
+set "PY_VCRUNTIME_REDIST=%PREFIX%\\bin\\vcruntime140.dll"
+
+for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -legacy -products * -version [16^,17^) -property installationPath`) do (
+    if exist "%%i" if exist "%%i\VC\Auxiliary\Build\vcvarsall.bat" (
+        set "VSINSTALLDIR=%%i\"
+        goto :vswhere
+    )
+)
+
+:vswhere
+
+:: Shorten PATH to avoid the `input line too long` error.
+SET MyPath=%PATH%
+
+setlocal EnableDelayedExpansion
+
+SET TempPath="%MyPath:;=";"%"
+SET var=
+FOR %%a IN (%TempPath%) DO (
+    IF EXIST %%~sa (
+        SET "var=!var!;%%~sa"
+    )
+)
+
+set "TempPath=!var:~1!"
+endlocal & set "PATH=%TempPath%"
+
+:: Shorten current directory too
+FOR %%A IN (.) DO CD "%%~sA"
+
+:: other things added by install_activate.bat at package build time

packaging/vs2019/conda_build_config.yaml

@@ -0,0 +1,24 @@
+blas_impl:
+  - mkl                        # [x86_64]
+c_compiler:
+  - vs2019                     # [win]
+cxx_compiler:
+  - vs2019                     # [win]
+python:
+  - 3.5
+  - 3.6
+# This differs from target_platform in that it determines what subdir the compiler
+#    will target, not what subdir the compiler package will be itself.
+#    For example, we need a win-64 vs2008_win-32 package, so that we compile win-32
+#    code on win-64 miniconda.
+cross_compiler_target_platform:
+  - win-64                     # [win]
+target_platform:
+  - win-64                     # [win]
+vc:
+  - 14
+zip_keys:
+  -                             # [win]
+    - vc                        # [win]
+    - c_compiler                # [win]
+    - cxx_compiler              # [win]

packaging/vs2019/install_activate.bat

@@ -0,0 +1,30 @@
+set YEAR=2019
+set VER=16
+
+mkdir "%PREFIX%\etc\conda\activate.d"
+COPY "%RECIPE_DIR%\activate.bat" "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
+
+IF "%cross_compiler_target_platform%" == "win-64" (
+  set "target_platform=amd64"
+  echo SET "CMAKE_GENERATOR=Visual Studio %VER% %YEAR% Win64" >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
+  echo pushd "%%VSINSTALLDIR%%" >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
+  IF "%VSDEVCMD_ARGS%" == "" (
+    echo CALL "VC\Auxiliary\Build\vcvarsall.bat" x64 >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
+    echo popd >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
+    echo pushd "%%VSINSTALLDIR%%" >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
+    echo CALL "VC\Auxiliary\Build\vcvarsall.bat" x86_amd64 >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
+  ) ELSE (
+    echo CALL "VC\Auxiliary\Build\vcvarsall.bat" x64 %VSDEVCMD_ARGS% >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
+    echo popd >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
+    echo pushd "%%VSINSTALLDIR%%" >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
+    echo CALL "VC\Auxiliary\Build\vcvarsall.bat" x86_amd64 %VSDEVCMD_ARGS% >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
+  )
+  echo popd >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
+  ) else (
+  set "target_platform=x86"
+  echo SET "CMAKE_GENERATOR=Visual Studio %VER% %YEAR%" >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
+  echo pushd "%%VSINSTALLDIR%%" >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
+  echo CALL "VC\Auxiliary\Build\vcvars32.bat" >> "%PREFIX%\etc\conda\activate.d\vs%YEAR%_compiler_vars.bat"
+  echo popd
+  )
+

packaging/vs2019/install_runtime.bat

@@ -0,0 +1,49 @@
+set VC_PATH=x86
+if "%ARCH%"=="64" (
+   set VC_PATH=x64
+)
+
+set MSC_VER=2019
+
+rem :: This should always be present for VC installed with VS.  Not sure about VC installed with Visual C++ Build Tools 2015
+rem FOR /F "usebackq tokens=3*" %%A IN (`REG QUERY "HKEY_LOCAL_MACHINE\Software\Microsoft\DevDiv\VC\Servicing\14.0\IDE.x64" /v UpdateVersion`) DO (
+rem     set SP=%%A
+rem     )
+
+rem if not "%SP%" == "%PKG_VERSION%" (
+rem    echo "Version detected from registry: %SP%"
+rem    echo    "does not match version of package being built (%PKG_VERSION%)"
+rem    echo "Do you have current updates for VS 2015 installed?"
+rem    exit 1
+rem )
+
+
+REM ========== REQUIRES Win 10 SDK be installed, or files otherwise copied to location below!
+robocopy "C:\Program Files (x86)\Windows Kits\10\Redist\ucrt\DLLs\%VC_PATH%"  "%LIBRARY_BIN%" *.dll /E
+robocopy "C:\Program Files (x86)\Windows Kits\10\Redist\ucrt\DLLs\%VC_PATH%"  "%PREFIX%" *.dll /E
+if %ERRORLEVEL% GEQ 8 exit 1
+
+REM ========== This one comes from visual studio 2019
+set "VC_VER=142"
+
+for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -legacy -products * -version [16^,17^) -property installationPath`) do (
+    if exist "%%i" if exist "%%i\VC\Auxiliary\Build\vcvarsall.bat" (
+        set "VS15VCVARSALL=%%i\VC\Auxiliary\Build\vcvarsall.bat"
+        goto :eof
+    )
+)
+
+@setlocal
+call "%VS15VARSALL%" x64
+
+set "REDIST_ROOT=%VCToolsRedistDir%%VC_PATH%"
+
+robocopy "%REDIST_ROOT%\Microsoft.VC%VC_VER%.CRT" "%LIBRARY_BIN%" *.dll /E
+if %ERRORLEVEL% LSS 8 exit 0
+robocopy "%REDIST_ROOT%\Microsoft.VC%VC_VER%.CRT" "%PREFIX%" *.dll /E
+if %ERRORLEVEL% LSS 8 exit 0
+robocopy "%REDIST_ROOT%\Microsoft.VC%VC_VER%.OpenMP" "%LIBRARY_BIN%" *.dll /E
+if %ERRORLEVEL% LSS 8 exit 0
+robocopy "%REDIST_ROOT%\Microsoft.VC%VC_VER%.OpenMP" "%PREFIX%" *.dll /E
+if %ERRORLEVEL% LSS 8 exit 0
+@endlocal

packaging/vs2019/meta.yaml

@@ -0,0 +1,24 @@
+{% set vcver="14.2" %}
+{% set vcfeature="14" %}
+{% set vsyear="2019" %}
+{% set fullver="15.4.27004.2010" %}
+
+package:
+  name: vs{{ vsyear }}
+  version: {{ fullver }}
+
+build:
+  skip: True  [not win]
+  script_env:
+    - VSDEVCMD_ARGS # [win]
+
+outputs:
+  - name: vs{{ vsyear }}_{{ cross_compiler_target_platform }}
+    script: install_activate.bat
+    track_features:
+      # VS 2019 is binary-compatible with VS 2017/vc 14.1 and 2015/vc14.  Tools are "v142".
+      strong:
+        - vc{{ vcfeature }}
+    about:
+      summary: Activation and version verification of MSVC {{ vcver }} (VS {{ vsyear }}) compiler
+      license: BSD 3-clause

requirements.txt

@@ -1,8 +1,5 @@
 torch>=1.4.0
 
-# Required for Windows because it's the only available backend
-SoundFile; sys_platform == 'win32'
-
 # Optional for torchaudio.kaldi_io
 numpy
 kaldi_io

setup.py

@@ -44,10 +44,19 @@ def check_env_flag(name, default=''):
     # proper link order (sox, mad, flac, lame)
     # (the most important thing is that dependencies come after a libraryl
     # e.g., sox comes first)
-    extra_objects += [os.path.join(audio_path, 'third_party/sox/lib/libsox.a')]
-    extra_objects += [os.path.join(audio_path, 'third_party/mad/lib/libmad.a')]
-    extra_objects += [os.path.join(audio_path, 'third_party/flac/lib/libFLAC.a')]
-    extra_objects += [os.path.join(audio_path, 'third_party/lame/lib/libmp3lame.a')]
+    if platform.system() == 'Windows':
+        extra_objects += [os.path.join(audio_path, 'third_party/sox/lib/sox.lib')]
+        extra_objects += [os.path.join(audio_path, 'third_party/mad/lib/mad.lib')]
+        extra_objects += [os.path.join(audio_path, 'third_party/flac/lib/FLAC.lib')]
+        extra_objects += [os.path.join(audio_path, 'third_party/lame/lib/mp3lame.lib')]
+        extra_objects += [os.path.join(audio_path, 'third_party/ogg/lib/ogg.lib')]
+        extra_objects += [os.path.join(audio_path, 'third_party/sox/lib/lpc10.lib')]
+        extra_objects += [os.path.join(audio_path, 'third_party/sox/lib/gsm.lib')]
+    else:
+        extra_objects += [os.path.join(audio_path, 'third_party/sox/lib/libsox.a')]
+        extra_objects += [os.path.join(audio_path, 'third_party/mad/lib/libmad.a')]
+        extra_objects += [os.path.join(audio_path, 'third_party/flac/lib/libFLAC.a')]
+        extra_objects += [os.path.join(audio_path, 'third_party/lame/lib/libmp3lame.a')]
 else:
     libraries += ['sox']
 
@@ -79,19 +88,16 @@ def check_env_flag(name, default=''):
 if pytorch_package_version is not None:
     pytorch_package_dep += "==" + pytorch_package_version
 
-if platform.system() == 'Windows':
-    ext_modules = None
-else:
-    ext_modules = [
-        CppExtension(
-            'torchaudio._torchaudio',
-            ['torchaudio/csrc/sox.cpp'],
-            libraries=libraries,
-            include_dirs=include_dirs + [cwd],
-            extra_compile_args=eca,
-            extra_objects=extra_objects,
-            extra_link_args=ela),
-    ]
+ext_modules = [
+    CppExtension(
+        'torchaudio._torchaudio',
+        ['torchaudio/csrc/sox.cpp'],
+        libraries=libraries,
+        include_dirs=include_dirs + [cwd],
+        extra_compile_args=eca,
+        extra_objects=extra_objects,
+        extra_link_args=ela),
+]
 
 
 setup(

torchaudio/_backend.py

@@ -8,12 +8,8 @@
 from . import _soundfile_backend, _sox_backend
 
 
-if platform.system() == "Windows":
-    _audio_backend = "soundfile"
-    _audio_backends = {"soundfile": _soundfile_backend}
-else:
-    _audio_backend = "sox"
-    _audio_backends = {"sox": _sox_backend, "soundfile": _soundfile_backend}
+_audio_backend = "sox"
+_audio_backends = {"sox": _sox_backend, "soundfile": _soundfile_backend}
 
 
 def set_audio_backend(backend: str) -> None: