Add python packaging build and deployment with Azure

Z3Prover · Aug 15, 2019 · ec5b148 · ec5b148
1 parent eec550e
commit ec5b148
Show file tree

Hide file tree

Showing 4 changed files with 120 additions and 14 deletions.
diff --git a/scripts/mk_unix_dist.py b/scripts/mk_unix_dist.py
@@ -107,7 +107,7 @@ def check_build_dir(path):
 # Create a build directory using mk_make.py
 def mk_build_dir(path):
     if not check_build_dir(path) or FORCE_MK:
-        opts = ["python", os.path.join('scripts', 'mk_make.py'), "-b", path, "--staticlib"]
+        opts = [sys.executable, os.path.join('scripts', 'mk_make.py'), "-b", path, "--staticlib"]
         if DOTNET_CORE_ENABLED:
             opts.append('--dotnet')
             if not DOTNET_KEY_FILE is None:

diff --git a/scripts/nightly.yaml b/scripts/nightly.yaml
@@ -17,7 +17,6 @@ jobs:
       artifactName: 'Mac'
       targetPath: $(Build.ArtifactStagingDirectory)
 
-
 - job: Ubuntu
   displayName: "Ubuntu build"
   pool:
@@ -35,6 +34,23 @@ jobs:
       artifactName: 'Ubuntu'
       targetPath: $(Build.ArtifactStagingDirectory)
 
+- job: Manylinux
+  displayName: "Manylinux build"
+  pool:
+    vmImage: "ubuntu-16.04"
+  container: "rhelmot/manylinux1_x86_64:latest"
+  variables:
+    python: "/opt/python/cp35-cp35m/bin/python"
+  steps:
+  - script: $(python) scripts/mk_unix_dist.py --nodotnet --nojava
+  - script: git clone https://github.com/z3prover/z3test z3test
+  - script: $(python) z3test/scripts/test_benchmarks.py build-dist/z3 z3test/regressions/smt2
+  - script: cp dist/*.zip $(Build.ArtifactStagingDirectory)/
+  - task: PublishPipelineArtifact@0
+    inputs:
+      artifactName: 'Manylinux'
+      targetPath: $(Build.ArtifactStagingDirectory)
+
 - job: Windows
   displayName: "Windows build"
   pool:
@@ -52,6 +68,36 @@ jobs:
       artifactName: 'Windows'
       targetPath: $(Build.ArtifactStagingDirectory)
 
+- job: Python
+  displayName: "Python packaging"
+  dependsOn:
+  - Manylinux
+  - Windows
+  pool:
+    vmImage: "ubuntu-16.04"
+  steps:
+  - task: DownloadPipelineArtifact@0
+    inputs:
+     artifactName: 'Windows'
+     targetPath: $(Agent.TempDirectory)
+  - task: DownloadPipelineArtifact@0
+    inputs:
+     artifactName: 'Manylinux'
+     targetPath: $(Agent.TempDirectory)
+  - script: cd $(Agent.TempDirectory); mkdir linux-bin; cd linux-bin; unzip ../*centos*.zip
+  - script: cd $(Agent.TempDirectory); mkdir win32-bin; cd win32-bin; unzip ../*x86-win*.zip
+  - script: cd $(Agent.TempDirectory); mkdir win64-bin; cd win64-bin; unzip ../*x64-win*.zip
+  - script: python -m pip install --user -U setuptools wheel
+  - script: cd src/api/python; python setup.py sdist
+  # take a look at this PREMIUM HACK I came up with to get around the fact that the azure variable syntax overloads the bash syntax for subshells
+  - script: cd src/api/python; echo $(Agent.TempDirectory)/linux-bin/* | xargs printf 'PACKAGE_FROM_RELEASE=%s\n' | xargs -I '{}' env '{}' python setup.py bdist_wheel
+  - script: cd src/api/python; echo $(Agent.TempDirectory)/win32-bin/* | xargs printf 'PACKAGE_FROM_RELEASE=%s\n' | xargs -I '{}' env '{}' python setup.py bdist_wheel
+  - script: cd src/api/python; echo $(Agent.TempDirectory)/win64-bin/* | xargs printf 'PACKAGE_FROM_RELEASE=%s\n' | xargs -I '{}' env '{}' python setup.py bdist_wheel
+  - task: PublishPipelineArtifact@0
+    inputs:
+      artifactName: 'Python packages'
+      targetPath: src/api/python/dist
+
 - job: Deploy
   displayName: "Deploy into GitHub"
   dependsOn: 

diff --git a/scripts/release.yml b/scripts/release.yml
@@ -10,14 +10,13 @@ jobs:
       secureFile: 'z3.snk'
   - script: python scripts/mk_unix_dist.py --dotnet-key=$(Agent.TempDirectory)/z3.snk
   - script: git clone https://github.com/z3prover/z3test z3test
-  - script: python z3test/scripts/test_benchmarks.py build-dist/z3 z3test/regressions/smt2    
+  - script: python z3test/scripts/test_benchmarks.py build-dist/z3 z3test/regressions/smt2
   - script: cp dist/*.zip $(Build.ArtifactStagingDirectory)/.
   - task: PublishPipelineArtifact@0
     inputs:
       artifactName: 'Mac'
       targetPath: $(Build.ArtifactStagingDirectory)
 
-
 - job: Ubuntu
   displayName: "Ubuntu build"
   pool:
@@ -28,13 +27,30 @@ jobs:
       secureFile: 'z3.snk'
   - script: python scripts/mk_unix_dist.py --dotnet-key=$(Agent.TempDirectory)/z3.snk
   - script: git clone https://github.com/z3prover/z3test z3test
-  - script: python z3test/scripts/test_benchmarks.py build-dist/z3 z3test/regressions/smt2    
+  - script: python z3test/scripts/test_benchmarks.py build-dist/z3 z3test/regressions/smt2
   - script: cp dist/*.zip $(Build.ArtifactStagingDirectory)/.
   - task: PublishPipelineArtifact@0
     inputs:
       artifactName: 'Ubuntu'
       targetPath: $(Build.ArtifactStagingDirectory)
 
+- job: Manylinux
+  displayName: "Manylinux build"
+  pool:
+    vmImage: "ubuntu-16.04"
+  container: "rhelmot/manylinux1_x86_64:latest"
+  variables:
+    python: "/opt/python/cp35-cp35m/bin/python"
+  steps:
+  - script: $(python) scripts/mk_unix_dist.py --nodotnet --nojava
+  - script: git clone https://github.com/z3prover/z3test z3test
+  - script: $(python) z3test/scripts/test_benchmarks.py build-dist/z3 z3test/regressions/smt2
+  - script: cp dist/*.zip $(Build.ArtifactStagingDirectory)/
+  - task: PublishPipelineArtifact@0
+    inputs:
+      artifactName: 'Manylinux'
+      targetPath: $(Build.ArtifactStagingDirectory)
+
 - job: Windows
   displayName: "Windows build"
   pool:
@@ -43,19 +59,50 @@ jobs:
   - task: DownloadSecureFile@1
     inputs:
       secureFile: 'z3.snk'
-  - script: scripts\mk_win_dist.cmd 
+  - script: scripts\mk_win_dist.cmd
   - script: xcopy dist\*.zip $(Build.ArtifactStagingDirectory)\* /y
   - task: PublishPipelineArtifact@0
     inputs:
       artifactName: 'Windows'
       targetPath: $(Build.ArtifactStagingDirectory)
 
+- job: Python
+  displayName: "Python packaging"
+  dependsOn:
+  - Manylinux
+  - Windows
+  pool:
+    vmImage: "ubuntu-16.04"
+  steps:
+  - task: DownloadPipelineArtifact@0
+    inputs:
+     artifactName: 'Windows'
+     targetPath: $(Agent.TempDirectory)
+  - task: DownloadPipelineArtifact@0
+    inputs:
+     artifactName: 'Manylinux'
+     targetPath: $(Agent.TempDirectory)
+  - script: cd $(Agent.TempDirectory); mkdir linux-bin; cd linux-bin; unzip ../*centos*.zip
+  - script: cd $(Agent.TempDirectory); mkdir win32-bin; cd win32-bin; unzip ../*x86-win*.zip
+  - script: cd $(Agent.TempDirectory); mkdir win64-bin; cd win64-bin; unzip ../*x64-win*.zip
+  - script: python -m pip install --user -U setuptools wheel
+  - script: cd src/api/python; python setup.py sdist
+  # take a look at this PREMIUM HACK I came up with to get around the fact that the azure variable syntax overloads the bash syntax for subshells
+  - script: cd src/api/python; echo $(Agent.TempDirectory)/linux-bin/* | xargs printf 'PACKAGE_FROM_RELEASE=%s\n' | xargs -I '{}' env '{}' python setup.py bdist_wheel
+  - script: cd src/api/python; echo $(Agent.TempDirectory)/win32-bin/* | xargs printf 'PACKAGE_FROM_RELEASE=%s\n' | xargs -I '{}' env '{}' python setup.py bdist_wheel
+  - script: cd src/api/python; echo $(Agent.TempDirectory)/win64-bin/* | xargs printf 'PACKAGE_FROM_RELEASE=%s\n' | xargs -I '{}' env '{}' python setup.py bdist_wheel
+  - task: PublishPipelineArtifact@0
+    inputs:
+      artifactName: 'Python packages'
+      targetPath: src/api/python/dist
+
 - job: Deploy
-  displayName: "Deploy into GitHub"
-  dependsOn: 
+  displayName: "Deploy into GitHub and PyPI"
+  dependsOn:
   - Mac
   - Ubuntu
   - Windows
+  - Python
   steps:
   - task: DownloadPipelineArtifact@0
     inputs:
@@ -69,10 +116,6 @@ jobs:
     inputs:
      artifactName: 'Ubuntu'
      targetPath: tmp
-  - task: CopyFiles@2
-    inputs:
-     sourceFolder: $(Build.ArtifactStagingDirectory)
-     targetFolder: tmp
 # TBD: build NuGet package
 # TBD: this script should build a specific pre-specified tag
   - task: GitHubRelease@0
@@ -89,6 +132,16 @@ jobs:
       assets: 'tmp/*'
       isDraft: true
       isPreRelease: true
+  - task: DownloadPipelineArtifact@0
+    inputs:
+      artifactName: 'Python packages'
+      targetPath: dist
+  - task: DownloadSecureFile@1
+    name: pypirc
+    inputs:
+      secureFile: 'pypirc'
+  - script: python -m pip install --user -U setuptools wheel twine
+  - script: python -m twine upload --config-file $(pypirc.secureFilePath) -r $(pypiReleaseServer) dist/*
 
 
 # TBD: run regression tests on generated binaries.
diff --git a/src/api/python/setup.py b/src/api/python/setup.py
@@ -33,11 +33,13 @@
     RELEASE_METADATA = None
     BUILD_PLATFORM = sys.platform
 else:
+    if not os.path.isdir(RELEASE_DIR):
+        raise Exception("RELEASE_DIR (%s) is not a directory!" % RELEASE_DIR)
     BUILD_DIR = os.path.join(RELEASE_DIR, 'bin')
     HEADER_DIRS = [os.path.join(RELEASE_DIR, 'include')]
     RELEASE_METADATA = os.path.basename(RELEASE_DIR).split('-')
     if RELEASE_METADATA[0] != 'z3' or len(RELEASE_METADATA) not in (4, 5):
-        raise Exception("RELEASE_DIR must be in the format z3-version-arch-os[-osversion] so we can extract metadata from it. Sorry!")
+        raise Exception("RELEASE_DIR (%s) must be in the format z3-version-arch-os[-osversion] so we can extract metadata from it. Sorry!" % RELEASE_DIR)
     RELEASE_METADATA.pop(0)
     BUILD_PLATFORM = RELEASE_METADATA[2]
 
@@ -225,7 +227,12 @@ def run(self):
         distos = RELEASE_METADATA[2]
         if distos in ('debian', 'ubuntu') or 'linux' in distos:
             raise Exception("Linux binary distributions must be built on centos to conform to PEP 513")
-        if distos == 'win':
+        elif distos == 'centos':
+            if arch == 'x64':
+                plat_name = 'manylinux1_x86_64'
+            else:
+                plat_name = 'manylinux1_i686'
+        elif distos == 'win':
             if arch == 'x64':
                 plat_name = 'win_amd64'
             else: