Switch to using Python 3 and move from Ubuntu 16.04 to 18.04 on Azure (…

…#3728)

.azure-pipelines/1-posix-setup.yml

@@ -32,18 +32,17 @@ steps:
       sudo apt-get -q update
       sudo apt-get -yq install \
         git-core cmake ninja-build g++-multilib \
-        libcurl3 libcurl3:i386 \
-        curl gdb p7zip-full tzdata unzip zip
+        libcurl4 libcurl4:i386 \
+        curl gdb p7zip-full tzdata unzip zip python3-pip
       # Download & extract clang
       curl -L -o clang.tar.xz https://github.com/llvm/llvm-project/releases/download/llvmorg-$CLANG_VERSION/clang+llvm-$CLANG_VERSION-x86_64-linux-gnu-ubuntu-16.04.tar.xz
       mkdir clang
       tar -xf clang.tar.xz --strip 1 -C clang
     fi
     # Install lit
-    curl -OL https://bootstrap.pypa.io/pip/2.7/get-pip.py
-    python get-pip.py --user
-    python -m pip install --user lit
-    python -c "import lit.main; lit.main.main();" --version . | head -n 1
+    python3 --version
+    python3 -m pip install --user lit
+    python3 -c "import lit.main; lit.main.main();" --version . | head -n 1
     # Download & extract host LDC
     HOST_OS="$CI_OS"
     if [ "$CI_OS" = "android" ]; then HOST_OS="linux"; fi

.circleci/config.yml

@@ -41,13 +41,10 @@ commonSteps: &commonSteps
             # Add CMake, Ninja and LLVM to PATH for future steps
             echo "export PATH=$PWD/cmake/bin:$PWD/ninja:$PWD/llvm/bin:$PATH" >> $BASH_ENV
           fi
-          # Set python
-          python3 --version
-          ln -sf $(which python3) /usr/local/bin/python
-          python --version
           # Install lit
-          python -m pip install --user lit
-          python -c "import lit.main; lit.main.main();" --version . | head -n 1
+          python3 --version
+          python3 -m pip install --user lit
+          python3 -c "import lit.main; lit.main.main();" --version . | head -n 1
           # Download & extract host LDC
           curl -L -o ldc2.tar.xz https://github.com/ldc-developers/ldc/releases/download/v$HOST_LDC_VERSION/ldc2-$HOST_LDC_VERSION-$CI_OS-x86_64.tar.xz
           mkdir host-ldc
@@ -85,7 +82,7 @@ commonSteps: &commonSteps
     - run:
         name: Run LIT testsuite
         when: always
-        command: cd ../build/tests && python runlit.py -v -j $PARALLELISM .
+        command: cd ../build/tests && python3 runlit.py -v -j $PARALLELISM .
     - run:
         name: Run DMD testsuite
         when: always

.cirrus.yml

@@ -4,8 +4,8 @@
 common_steps_template: &COMMON_STEPS_TEMPLATE
   install_lit_script: |
     # Install lit
-    python -m pip install --user lit
-    python -c "import lit.main; lit.main.main();" --version . | head -n 1
+    python3 -m pip install --user lit
+    python3 -c "import lit.main; lit.main.main();" --version . | head -n 1
   clone_submodules_script: |
     cd $CIRRUS_WORKING_DIR
     git submodule update --init --depth $CIRRUS_CLONE_DEPTH
@@ -33,7 +33,7 @@ common_steps_template: &COMMON_STEPS_TEMPLATE
       ctest --output-on-failure -R ldc2-unittest
     run_lit_testsuite_script: |
       cd $CIRRUS_WORKING_DIR/../build/tests
-      python runlit.py -v -j $PARALLELISM .
+      python3 runlit.py -v -j $PARALLELISM .
     run_dmd_testsuite_script: |
       cd $CIRRUS_WORKING_DIR/../build
       DMD_TESTSUITE_MAKE_ARGS=-j$PARALLELISM ctest -V -R dmd-testsuite
@@ -70,8 +70,6 @@ install_ubuntu_prerequisites_template: &INSTALL_UBUNTU_PREREQUISITES_TEMPLATE
       $libcurl_pkg curl gdb python3 python3-pip tzdata unzip zip \
       $EXTRA_APT_PACKAGES
     python3 --version
-    ln -s $(which python3) /usr/bin/python
-    python --version
     # Download & extract host LDC
     curl --max-time 300 --retry 3 -L -o ldc2.tar.xz https://github.com/ldc-developers/ldc/releases/download/v$HOST_LDC_VERSION/ldc2-$HOST_LDC_VERSION-linux-x86_64.tar.xz
     mkdir host-ldc
@@ -106,8 +104,7 @@ install_macos_prerequisites_template: &INSTALL_MACOS_PREREQUISITES_TEMPLATE
     rm ldc2.tar.xz
     # Install Python3
     brew install python
-    ln -sf $(which python3) /usr/local/bin/python
-    python --version
+    python3 --version
 
 environment:
   CIRRUS_CLONE_DEPTH: 50
@@ -186,8 +183,8 @@ task:
     cd $CIRRUS_WORKING_DIR/..
     sysctl -n hw.ncpu
     pkg install -y git cmake ninja gmake llvm bash gtar p7zip
-    python --version
-    python -m ensurepip
+    python3 --version
+    python3 -m ensurepip
     # Download & extract host LDC
     curl --max-time 300 --retry 3 -L -o ldc2.tar.xz https://github.com/ldc-developers/ldc/releases/download/v$HOST_LDC_VERSION/ldc2-$HOST_LDC_VERSION-freebsd-x86_64.tar.xz
     mkdir pre-ldc

.github/workflows/supported_llvm_versions.yml

@@ -64,8 +64,8 @@ jobs:
       - name: Install lit
         run: |
           set -euxo pipefail
-          python -m pip install --user lit
-          python -c "import lit.main; lit.main.main();" --version . | head -n 1
+          python3 -m pip install --user lit
+          python3 -c "import lit.main; lit.main.main();" --version . | head -n 1
       - name: "Linux: Install gdb"
         if: runner.os == 'Linux'
         run: sudo apt-get update && sudo apt-get install gdb

.travis.yml

@@ -40,8 +40,8 @@ before_install:
 
 install:
   # Install lit
-  - python -m pip install --user lit
-  - python -c "import lit.main; lit.main.main();" --version . | head -n 1
+  - python3 -m pip install --user lit
+  - python3 -c "import lit.main; lit.main.main();" --version . | head -n 1
 
 script:
   - unset LD_LIBRARY_PATH

azure-pipelines.yml

@@ -59,7 +59,7 @@ jobs:
 - job: Linux_multilib
   timeoutInMinutes: 120
   pool:
-    vmImage: 'ubuntu-16.04'
+    vmImage: 'ubuntu-18.04'
   variables:
     CI_OS: linux
     ARCH: x86_64
@@ -153,7 +153,7 @@ jobs:
 - job: Android
   timeoutInMinutes: 120
   pool:
-    vmImage: 'ubuntu-16.04'
+    vmImage: 'ubuntu-18.04'
   variables:
     CI_OS: android
     ANDROID_NDK_VERSION: r21e
@@ -188,7 +188,7 @@ jobs:
 - job: Upload_to_GitHub
   timeoutInMinutes: 30
   pool:
-    vmImage: 'ubuntu-16.04'
+    vmImage: 'ubuntu-18.04'
   dependsOn:
     - Windows_multilib
     - Linux_multilib

shippable.yml

@@ -25,7 +25,7 @@ build:
       apt-get -yq install \
         git-core cmake ninja-build \
         libcurl3 libcurl4-openssl-dev \
-        curl gdb p7zip-full python-pip tzdata unzip zip
+        curl gdb p7zip-full python3-pip tzdata unzip zip
     - pip install --user lit
     # Download & extract host LDC
     - curl -L -o ldc2.tar.xz https://github.com/ldc-developers/ldc/releases/download/v$HOST_LDC_VERSION/ldc2-$HOST_LDC_VERSION-linux-aarch64.tar.xz
@@ -90,7 +90,7 @@ build:
     - rm ../tests/d2/dmd-testsuite/runnable/b18504.d
     # Run LIT testsuite, ignore the errors
     - cd tests
-    - PATH="$PWD/../../llvm/bin:$PATH" python runlit.py -v -j 32 . || true
+    - PATH="$PWD/../../llvm/bin:$PATH" python3 runlit.py -v -j 32 . || true
     - cd ..
     # Run DMD testsuite
     - DMD_TESTSUITE_MAKE_ARGS='-j32 GDB_FLAGS=OFF' ctest -V -R "dmd-testsuite"

tests/CMakeLists.txt

@@ -5,6 +5,10 @@ set( LLVM_TOOLS_DIR    ${LLVM_ROOT_DIR}/bin )
 set( LDC2_BIN_DIR      ${PROJECT_BINARY_DIR}/bin )
 set( LDC2_LIB_DIR      ${PROJECT_BINARY_DIR}/lib${LIB_SUFFIX} )
 set( TESTS_IR_DIR      ${CMAKE_CURRENT_SOURCE_DIR} )
+set( PYTHON_EXE        python3)
+if(WIN32)
+    set(PYTHON_EXE python)
+endif()
 
 if(CMAKE_SIZEOF_VOID_P EQUAL 8)
     set( DEFAULT_TARGET_BITS 64 )
@@ -16,6 +20,6 @@ configure_file(lit.site.cfg.in lit.site.cfg )
 configure_file(runlit.py       runlit.py    COPYONLY)
 
 add_test(NAME lit-tests
-    COMMAND python runlit.py -v .
+    COMMAND ${PYTHON_EXE} runlit.py -v .
 )
 

tests/runlit.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # wrapper to run lit from commandline
 
 from __future__ import print_function