Merge pull request #167 from openedx/feanil/build_multiple_images

feat: Build and test with Python 3.8 and 3.11 for codejail.

.github/workflows/ci.yml

@@ -9,6 +9,13 @@ jobs:
   codejail_ci:
     name: tests
     runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        include:
+          - python_version: '3.8'
+            docker_tag: latest
+          - python_version: '3.11'
+            docker_tag: '3.11'
 
     steps:
       - uses: actions/checkout@v2
@@ -24,15 +31,16 @@ jobs:
         uses: aws-actions/amazon-ecr-login@v1
 
       - name: Parse custom apparmor profile
-        run: sudo apparmor_parser -r -W apparmor-profiles/home.sandbox.codejail_sandbox-python3.8.bin.python
-        
+        run: sudo apparmor_parser -r -W apparmor-profiles/home.sandbox.codejail_sandbox-python3.bin.python
+
       - name: Pull codejail CI image
         run: docker pull 257477529851.dkr.ecr.us-east-1.amazonaws.com/openedx-codejail:latest
 
       - name: Build latest code changes into CI image
         run: |
           docker build --cache-from 257477529851.dkr.ecr.us-east-1.amazonaws.com/openedx-codejail \
-            -t 257477529851.dkr.ecr.us-east-1.amazonaws.com/openedx-codejail .
+            -t 257477529851.dkr.ecr.us-east-1.amazonaws.com/openedx-codejail \
+            --build-arg python_version=${{ matrix.python_version }} .
 
       - name: Run container with custom apparmor profile and codejail CI image
         run: |
@@ -41,7 +49,7 @@ jobs:
 
       - name: Run Non Proxy Tests
         run: docker exec -t codejail bash -c 'make clean && make test_no_proxy'
-          
+
       - name: Run Proxy Tests
         run: docker exec -t codejail bash -c 'make clean && make test_proxy'
 

.github/workflows/push-docker-image.yml

@@ -8,6 +8,13 @@ on:
 jobs:
   push:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - python_version: '3.8'
+            docker_tag: latest
+          - python_version: '3.11'
+            docker_tag: '3.11'
 
     steps:
       - name: Checkout
@@ -28,7 +35,6 @@ jobs:
         env:
           ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
           ECR_REPOSITORY: openedx-codejail
-          IMAGE_TAG: latest
         run: |
-          docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG .
-          docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
+          docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:${{ matrix.docker_tag }} --build-arg ${{ matrix.python_version }} .
+          docker push $ECR_REGISTRY/$ECR_REPOSITORY:${{ matrix.docker_tag }}

Dockerfile

@@ -1,21 +1,28 @@
 FROM ubuntu:focal
 SHELL ["/bin/bash", "-c"]
 
+ARG python_version=3.8
+
 # Install Codejail Packages
-RUN apt-get update && apt-get upgrade -y
-RUN apt-get install -y vim python3-virtualenv python3-pip
-RUN apt-get install -y sudo git
+ENV TZ=Etc/UTC
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get install -y software-properties-common
+RUN add-apt-repository -y ppa:deadsnakes/ppa && apt-get update && apt-get upgrade -y
+RUN apt-get install -y vim python${python_version} python${python_version}-dev python${python_version}-distutils
+RUN apt-get install -y sudo git make curl build-essential
+RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python${python_version}
+RUN pip install virtualenv
 
 # Define Environment Variables
 ENV CODEJAIL_GROUP=sandbox
 ENV CODEJAIL_SANDBOX_CALLER=ubuntu
 ENV CODEJAIL_TEST_USER=sandbox
-ENV CODEJAIL_TEST_VENV=/home/sandbox/codejail_sandbox-python3.8
+ENV CODEJAIL_TEST_VENV=/home/sandbox/codejail_sandbox-python${python_version}
 
 # Create Virtualenv for sandbox user
-RUN virtualenv -p python3.8 --always-copy $CODEJAIL_TEST_VENV
+RUN virtualenv -p python${python_version} --always-copy $CODEJAIL_TEST_VENV
 
-RUN virtualenv -p python3.8 venv
+RUN virtualenv -p python${python_version} venv
 ENV VIRTUAL_ENV=/venv
 
 # Add venv/bin to path
@@ -48,7 +55,7 @@ RUN pip install -r /codejail/requirements/sandbox.txt && pip install -r /codejai
 COPY . /codejail
 
 # Setup sudoers file
-COPY sudoers-file/01-sandbox /etc/sudoers.d/01-sandbox
+COPY sudoers-file/01-sandbox-python-${python_version} /etc/sudoers.d/01-sandbox
 
 # Change Sudoers file permissions
 RUN chmod 0440 /etc/sudoers.d/01-sandbox

apparmor-profiles/home.sandbox.codejail_sandbox-python3.bin.python

@@ -0,0 +1,54 @@
+#include <tunables/global>
+
+profile apparmor_profile /home/sandbox/codejail_sandbox-python{3.[0-9],3.[1-9][0-9]}/bin/python {
+    #include <abstractions/base>
+    #include <abstractions/python>
+
+    /usr/{local/,}lib{,32,64}/python{2.[4-7],3,3.[0-9],3.[1-9][0-9]}/**.{pyc,so,so.*[0-9]} mr,
+    /usr/{local/,}lib{,32,64}/python{2.[4-7],3,3.[0-9],3.[1-9][0-9]}/**.{egg,py,pth}       r,
+    /usr/{local/,}lib{,32,64}/python{2.[4-7],3,3.[0-9],3.[1-9][0-9]}/{site,dist}-packages/ r,
+    /usr/{local/,}lib{,32,64}/python{2.[4-7],3,3.[0-9],3.[1-9][0-9]}/{site,dist}-packages/**/ r,
+    /usr/{local/,}lib{,32,64}/python{2.[4-7],3,3.[0-9],3.[1-9][0-9]}/{site,dist}-packages/*.dist-info/{METADATA,namespace_packages.txt} r,
+    /usr/{local/,}lib{,32,64}/python{2.[4-7],3,3.[0-9],3.[1-9][0-9]}/{site,dist}-packages/*.VERSION r,
+    /usr/{local/,}lib{,32,64}/python{2.[4-7],3,3.[0-9],3.[1-9][0-9]}/{site,dist}-packages/*.egg-info/PKG-INFO r,
+    /usr/{local/,}lib{,32,64}/python3.{1,}[0-9]/lib-dynload/*.so            mr,
+
+    # Site-wide configuration
+    /etc/python{2.[4-7],3.[0-9],3.[1-9][0-9]}/** r,
+
+    # shared python paths
+    /usr/share/{pyshared,pycentral,python-support}/**      r,
+    /{var,usr}/lib/{pyshared,pycentral,python-support}/**  r,
+    /usr/lib/{pyshared,pycentral,python-support}/**.so     mr,
+    /var/lib/{pyshared,pycentral,python-support}/**.pyc    mr,
+    /usr/lib/python3/dist-packages/**.so          mr,
+
+    # wx paths
+    /usr/lib/wx/python/*.pth r,
+
+    # python build configuration and headers
+    /usr/include/python{2.[4-7],3.[0-9],3.[1-9][0-9]}*/pyconfig.h r,
+
+    # Include additions to the abstraction
+    include if exists <abstractions/python.d>
+
+    /home/sandbox/codejail_sandbox-python{3.[0-9],3.[1-9][0-9]}/** mr,
+    /tmp/codejail-*/ rix,
+    /tmp/codejail-*/** wrix,
+
+    # Whitelist particiclar shared objects from the system
+    # python installation
+    #
+    /usr/lib/python{3.[0-9],3.[1-9][0-9]}/lib-dynload/_json.so mr,
+    /usr/lib/python{3.[0-9],3.[1-9][0-9]}/lib-dynload/_ctypes.so mr,
+    /usr/lib/python{3.[0-9],3.[1-9][0-9]}/lib-dynload/_heapq.so mr,
+    /usr/lib/python{3.[0-9],3.[1-9][0-9]}/lib-dynload/_io.so mr,
+    /usr/lib/python{3.[0-9],3.[1-9][0-9]}/lib-dynload/_csv.so mr,
+    /usr/lib/python{3.[0-9],3.[1-9][0-9]}/lib-dynload/datetime.so mr,
+    /usr/lib/python{3.[0-9],3.[1-9][0-9]}/lib-dynload/_elementtree.so mr,
+    /usr/lib/python{3.[0-9],3.[1-9][0-9]}/lib-dynload/pyexpat.so mr,
+    #
+    # Allow access to selections from /proc
+    #
+    /proc/*/mounts r,
+}

codejail/__init__.py

@@ -1,3 +1,3 @@
 """init"""
 
-__version__ = '3.3.3'
+__version__ = '3.4.0'

codejail/tests/test_jail_code.py

@@ -89,12 +89,14 @@ def test_ends_with_exception(self):
         res = jailpy(code="""raise Exception('FAIL')""")
         self.assertNotEqual(res.status, 0)
         self.assertEqual(res.stdout, b"")
-        self.assertEqual(res.stderr, bytes(textwrap.dedent("""\
-            Traceback (most recent call last):
-              File "jailed_code", line 1, in <module>
-                raise Exception('FAIL')
+
+        regex = textwrap.dedent("""
+            (?m)Traceback [(]most recent call last[)]:
+              File "(/.*?/)?jailed_code", line 1, in <module>
+                raise Exception[(]'FAIL'[)]
             Exception: FAIL
-            """), 'utf-8'))
+            """).strip() + "\n"
+        self.assertRegex(res.stderr.decode('utf-8'), regex)
 
     def test_stdin_is_provided(self):
         res = jailpy(

requirements/common_constraints.txt

@@ -13,13 +13,20 @@
 
 
 # using LTS django version
-Django<4.0
+Django<5.0
 
 # elasticsearch>=7.14.0 includes breaking changes in it which caused issues in discovery upgrade process.
 # elastic search changelog: https://www.elastic.co/guide/en/enterprise-search/master/release-notes-7.14.0.html
 elasticsearch<7.14.0
 
-setuptools<60
-
 # django-simple-history>3.0.0 adds indexing and causes a lot of migrations to be affected
 django-simple-history==3.0.0
+
+# opentelemetry requires version 6.x at the moment:
+# https://github.com/open-telemetry/opentelemetry-python/issues/3570
+# Normally this could be added as a constraint in edx-django-utils, where we're
+# adding the opentelemetry dependency. However, when we compile pip-tools.txt,
+# that uses version 7.x, and then there's no undoing that when compiling base.txt.
+# So we need to pin it globally, for now.
+# Ticket for unpinning: https://github.com/openedx/edx-lint/issues/407
+importlib-metadata<7

requirements/development.txt

@@ -1,28 +1,21 @@
 #
-# This file is autogenerated by pip-compile with python 3.8
-# To update, run:
+# This file is autogenerated by pip-compile with Python 3.8
+# by the following command:
 #
 #    make upgrade
 #
-astroid==2.11.6           # via -r requirements/testing.txt, pylint
-attrs==21.4.0             # via -r requirements/testing.txt, pytest
-dill==0.3.5.1             # via -r requirements/testing.txt, pylint
-iniconfig==1.1.1          # via -r requirements/testing.txt, pytest
-isort==5.10.1             # via -r requirements/testing.txt, pylint
-lazy-object-proxy==1.7.1  # via -r requirements/testing.txt, astroid
+astroid==3.1.0            # via -r requirements/testing.txt, pylint
+dill==0.3.8               # via -r requirements/testing.txt, pylint
+exceptiongroup==1.2.1     # via -r requirements/testing.txt, pytest
+iniconfig==2.0.0          # via -r requirements/testing.txt, pytest
+isort==5.13.2             # via -r requirements/testing.txt, pylint
 mccabe==0.7.0             # via -r requirements/testing.txt, pylint
-packaging==21.3           # via -r requirements/testing.txt, pytest
-platformdirs==2.5.2       # via -r requirements/testing.txt, pylint
-pluggy==1.0.0             # via -r requirements/testing.txt, pytest
-py==1.11.0                # via -r requirements/testing.txt, pytest
-pycodestyle==2.8.0        # via -r requirements/testing.txt
-pylint==2.14.3            # via -r requirements/testing.txt
-pyparsing==3.0.9          # via -r requirements/testing.txt, packaging
-pytest==7.1.2             # via -r requirements/testing.txt
+packaging==24.0           # via -r requirements/testing.txt, pytest
+platformdirs==4.2.1       # via -r requirements/testing.txt, pylint
+pluggy==1.5.0             # via -r requirements/testing.txt, pytest
+pycodestyle==2.11.1       # via -r requirements/testing.txt
+pylint==3.1.0             # via -r requirements/testing.txt
+pytest==8.1.1             # via -r requirements/testing.txt
 tomli==2.0.1              # via -r requirements/testing.txt, pylint, pytest
-tomlkit==0.11.0           # via -r requirements/testing.txt, pylint
-typing-extensions==4.2.0  # via -r requirements/testing.txt, astroid, pylint
-wrapt==1.14.1             # via -r requirements/testing.txt, astroid
-
-# The following packages are considered to be unsafe in a requirements file:
-# setuptools
+tomlkit==0.12.4           # via -r requirements/testing.txt, pylint
+typing-extensions==4.11.0  # via -r requirements/testing.txt, astroid, pylint

requirements/pip.txt

@@ -1,11 +1,11 @@
 #
-# This file is autogenerated by pip-compile with python 3.8
-# To update, run:
+# This file is autogenerated by pip-compile with Python 3.8
+# by the following command:
 #
 #    make upgrade
 #
-wheel==0.37.1             # via -r requirements/pip.in
+wheel==0.43.0             # via -r requirements/pip.in
 
 # The following packages are considered to be unsafe in a requirements file:
-pip==22.1.2               # via -r requirements/pip.in
-setuptools==59.8.0        # via -c requirements/common_constraints.txt, -r requirements/pip.in
+pip==24.0                 # via -r requirements/pip.in
+setuptools==69.5.1        # via -r requirements/pip.in

requirements/pip_tools.txt

@@ -1,14 +1,18 @@
 #
-# This file is autogenerated by pip-compile with python 3.8
-# To update, run:
+# This file is autogenerated by pip-compile with Python 3.8
+# by the following command:
 #
 #    make upgrade
 #
-click==8.1.3              # via pip-tools
-pep517==0.12.0            # via pip-tools
-pip-tools==7.3.0          # via -r requirements/pip_tools.in
-tomli==2.0.1              # via pep517
-wheel==0.37.1             # via pip-tools
+build==1.2.1              # via pip-tools
+click==8.1.7              # via pip-tools
+importlib-metadata==6.11.0  # via -c requirements/common_constraints.txt, build
+packaging==24.0           # via build
+pip-tools==7.4.1          # via -r requirements/pip_tools.in
+pyproject-hooks==1.0.0    # via build, pip-tools
+tomli==2.0.1              # via build, pip-tools, pyproject-hooks
+wheel==0.43.0             # via pip-tools
+zipp==3.18.1              # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
 # pip

requirements/sandbox.txt

@@ -1,12 +1,13 @@
 #
 # This file is autogenerated by pip-compile with Python 3.8
-# To update, run:
+# by the following command:
 #
 #    make upgrade
 #
-asgiref==3.5.2            # via django
-django==3.2.17            # via -c requirements/common_constraints.txt, -c requirements/constraints.txt, -r requirements/sandbox.in
-numpy==1.22.4             # via -r requirements/sandbox.in
-pytz==2022.1              # via django
+asgiref==3.8.1            # via django
+django==3.2.25            # via -c requirements/common_constraints.txt, -c requirements/constraints.txt, -r requirements/sandbox.in
+numpy==1.24.4             # via -r requirements/sandbox.in
+pytz==2024.1              # via django
 six==1.16.0               # via -r requirements/sandbox.in
-sqlparse==0.4.2           # via django
+sqlparse==0.5.0           # via django
+typing-extensions==4.11.0  # via asgiref

requirements/testing.txt

@@ -1,28 +1,21 @@
 #
-# This file is autogenerated by pip-compile with python 3.8
-# To update, run:
+# This file is autogenerated by pip-compile with Python 3.8
+# by the following command:
 #
 #    make upgrade
 #
-astroid==2.11.6           # via pylint
-attrs==21.4.0             # via pytest
-dill==0.3.5.1             # via pylint
-iniconfig==1.1.1          # via pytest
-isort==5.10.1             # via -r requirements/testing.in, pylint
-lazy-object-proxy==1.7.1  # via astroid
+astroid==3.1.0            # via pylint
+dill==0.3.8               # via pylint
+exceptiongroup==1.2.1     # via pytest
+iniconfig==2.0.0          # via pytest
+isort==5.13.2             # via -r requirements/testing.in, pylint
 mccabe==0.7.0             # via pylint
-packaging==21.3           # via pytest
-platformdirs==2.5.2       # via pylint
-pluggy==1.0.0             # via pytest
-py==1.11.0                # via pytest
-pycodestyle==2.8.0        # via -r requirements/testing.in
-pylint==2.14.3            # via -r requirements/testing.in
-pyparsing==3.0.9          # via packaging
-pytest==7.1.2             # via -r requirements/testing.in
+packaging==24.0           # via pytest
+platformdirs==4.2.1       # via pylint
+pluggy==1.5.0             # via pytest
+pycodestyle==2.11.1       # via -r requirements/testing.in
+pylint==3.1.0             # via -r requirements/testing.in
+pytest==8.1.1             # via -r requirements/testing.in
 tomli==2.0.1              # via pylint, pytest
-tomlkit==0.11.0           # via pylint
-typing-extensions==4.2.0  # via astroid, pylint
-wrapt==1.14.1             # via astroid
-
-# The following packages are considered to be unsafe in a requirements file:
-# setuptools
+tomlkit==0.12.4           # via pylint
+typing-extensions==4.11.0  # via astroid, pylint