test(storage): add emulator unit tests and cleanup

ci/kokoro/docker/Dockerfile.fedora

@@ -19,8 +19,8 @@ FROM fedora:${DISTRO_VERSION}
 RUN dnf makecache && \
     dnf install -y ccache clang clang-tools-extra cmake diffutils findutils \
         gcc-c++ git libcxx-devel libcxxabi-devel libasan libubsan libtsan \
-        llvm make openssl-devel pkgconfig python3 python3.8 python3-devel \
-        python3-pip tar unzip wget which zlib-devel
+        llvm make openssl-devel pkgconfig python python3.8 python3-devel \
+        python-pip tar unzip wget which zlib-devel
 
 # Sets root's password to the empty string to enable users to get a root shell
 # inside the container with `su -` and no password. Sudo would not work because
@@ -32,8 +32,8 @@ RUN echo 'root:' | chpasswd
 RUN pip3 install --upgrade pip
 RUN pip3 install setuptools wheel
 RUN pip3 install git+git://github.com/googleapis/python-storage@8cf6c62a96ba3fff7e5028d931231e28e5029f1c
-RUN pip3 install grpcio==1.32.0 flask==1.1.2 pytest==6.0.2 httpbin==0.7.0 \
-    scalpl==0.4.0 crc32c==2.1 gevent==20.9.0 gunicorn==20.0.4
+RUN pip3 install flask==1.1.2 httpbin==0.7.0 scalpl==0.4.0 \
+    crc32c==2.1 gunicorn==20.0.4
 
 # Install the Cloud SDK and some of the emulators. We use the emulators to run
 # integration tests for the client libraries.

ci/kokoro/docker/Dockerfile.fedora-install

@@ -23,8 +23,8 @@ RUN dnf makecache && \
     dnf install -y abi-compliance-checker abi-dumper ccache \
         clang clang-tools-extra cmake diffutils doxygen findutils gcc-c++ git \
         grpc-devel grpc-plugins lcov libcxx-devel libcxxabi-devel libcurl-devel \
-        make ninja-build npm openssl-devel pkgconfig protobuf-compiler python3 \
-        python3.8 python3-pip ShellCheck tar unzip w3m wget which zlib-devel
+        make ninja-build npm openssl-devel pkgconfig protobuf-compiler python \
+        python3.8 python-pip ShellCheck tar unzip w3m wget which zlib-devel
 
 # Sets root's password to the empty string to enable users to get a root shell
 # inside the container with `su -` and no password. Sudo would not work because
@@ -57,8 +57,8 @@ RUN pip3 install black==19.3b0
 RUN dnf makecache && dnf install -y python3-devel
 RUN pip3 install setuptools wheel
 RUN pip3 install git+git://github.com/googleapis/python-storage@8cf6c62a96ba3fff7e5028d931231e28e5029f1c
-RUN pip3 install grpcio==1.32.0 flask==1.1.2 pytest==6.0.2 httpbin==0.7.0 \
-    scalpl==0.4.0 crc32c==2.1 gevent==20.9.0 gunicorn==20.0.4
+RUN pip3 install flask==1.1.2 httpbin==0.7.0 scalpl==0.4.0 \
+    crc32c==2.1 gunicorn==20.0.4
 
 # Install cspell for spell checking.
 RUN npm install -g cspell

ci/kokoro/docker/Dockerfile.fedora-libcxx-msan

@@ -19,8 +19,8 @@ FROM fedora:${DISTRO_VERSION}
 # then compile our code.
 RUN dnf makecache && \
     dnf install -y ccache clang clang-tools-extra cmake findutils gcc-c++ \
-        git llvm llvm-devel make ninja-build openssl-devel python3 python3.8 \
-        python3-devel python3-lit python3-pip tar unzip which wget xz
+        git llvm llvm-devel make ninja-build openssl-devel python python3.8 \
+        python3-devel python3-lit python-pip tar unzip which wget xz
 
 # Sets root's password to the empty string to enable users to get a root shell
 # inside the container with `su -` and no password. Sudo would not work because
@@ -32,8 +32,8 @@ RUN echo 'root:' | chpasswd
 RUN pip3 install --upgrade pip
 RUN pip3 install setuptools wheel
 RUN pip3 install git+git://github.com/googleapis/python-storage@8cf6c62a96ba3fff7e5028d931231e28e5029f1c
-RUN pip3 install grpcio==1.32.0 flask==1.1.2 pytest==6.0.2 httpbin==0.7.0 \
-    scalpl==0.4.0 crc32c==2.1 gevent==20.9.0 gunicorn==20.0.4
+RUN pip3 install flask==1.1.2 httpbin==0.7.0 scalpl==0.4.0 \
+    crc32c==2.1 gunicorn==20.0.4
 
 WORKDIR /var/tmp/build
 RUN wget -q https://github.com/llvm/llvm-project/releases/download/llvmorg-10.0.0/libcxx-10.0.0.src.tar.xz

ci/kokoro/docker/Dockerfile.ubuntu

@@ -49,10 +49,11 @@ RUN apt-get update && \
         apt-transport-https
 
 # Install Python packages used in the integration tests.
+RUN update-alternatives --install /usr/bin/python python $(which python3) 10
 RUN pip3 install setuptools wheel
 RUN pip3 install git+git://github.com/googleapis/python-storage@8cf6c62a96ba3fff7e5028d931231e28e5029f1c
-RUN pip3 install grpcio==1.32.0 flask==1.1.2 pytest==6.0.2 httpbin==0.7.0 \
-    scalpl==0.4.0 crc32c==2.1 gevent==20.9.0 gunicorn==20.0.4
+RUN pip3 install flask==1.1.2 httpbin==0.7.0 scalpl==0.4.0 \
+    crc32c==2.1 gunicorn==20.0.4
 
 # Install the Cloud SDK and some of the emulators. We use the emulators to run
 # integration tests for the client libraries.

google/cloud/storage/ci/run_integration_tests_emulator_bazel.sh

@@ -30,6 +30,15 @@ BAZEL_VERB="$1"
 shift
 bazel_test_args=("$@")
 
+# Run the unittests of the emulator before running integration tests.
+"${BAZEL_BIN}" test "${bazel_test_args[@]}" "//google/cloud/storage/emulator:test_utils" \
+  "//google/cloud/storage/emulator:test_gcs"
+exit_status=$?
+
+if [[ "$exit_status" -ne 0 ]]; then
+  exit "${exit_status}"
+fi
+
 # Configure run_testbench_utils.sh to run the GCS testbench.
 source "${PROJECT_ROOT}/google/cloud/storage/tools/run_testbench_utils.sh"
 

google/cloud/storage/emulator/tests_entry.py → google/cloud/storage/emulator/BUILD

@@ -12,10 +12,24 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Tests entry"""
+package(default_visibility = ["//visibility:public"])
 
-from tests import test_gcs, test_utils
+licenses(["notice"])  # Apache 2.0
 
-if __name__ == "__main__":
-    test_utils.run()
-    test_gcs.run()
+py_test(
+    name = "test_gcs",
+    size = "small",
+    srcs = ["tests/test_gcs.py"] + glob(["gcs/*.py"]) + glob(["utils/*.py"]),
+    imports = ["."],
+    main = "tests/test_gcs.py",
+    tags = ["manual"],
+)
+
+py_test(
+    name = "test_utils",
+    size = "small",
+    srcs = ["tests/test_utils.py"] + glob(["gcs/*.py"]) + glob(["utils/*.py"]),
+    imports = ["."],
+    main = "tests/test_utils.py",
+    tags = ["manual"],
+)

google/cloud/storage/emulator/requirements.txt

@@ -1,9 +1,6 @@
 git+git://github.com/googleapis/python-storage@8cf6c62a96ba3fff7e5028d931231e28e5029f1c
-grpcio==1.32.0
 flask==1.1.2
-pytest==6.0.2
 httpbin==0.7.0
 scalpl==0.4.0
 crc32c==2.1
-gevent==20.9.0
 gunicorn==20.0.4