build: update generate-docs to support monorepo (#289)

* build: update generate-docs to support monorepo

* build: update Dockerfile

build: update jinja version

build: update Docker image to py310

build: specify Sphinx version in the package installtion

build: use py3.9 instead

build: update setup.py

* build: fix typo

* build: update to concatenate string var

.kokoro/docker/docs/Dockerfile

@@ -40,8 +40,6 @@ RUN apt-get update \
     libssl-dev \
     libsqlite3-dev \
     portaudio19-dev \
-    python3 \
-    python3-pip \
     redis-server \
     software-properties-common \
     ssh \
@@ -61,6 +59,50 @@ RUN apt-get update \
   && rm -rf /var/lib/apt/lists/* \
   && rm -f /var/cache/apt/archives/*.deb
 
+COPY fetch_gpg_keys.sh /tmp
+# Install the desired versions of Python.
+RUN set -ex \
+  && export GNUPGHOME="$(mktemp -d)" \
+  && echo "disable-ipv6" >> "${GNUPGHOME}/dirmngr.conf" \
+  && /tmp/fetch_gpg_keys.sh \
+  && for PYTHON_VERSION in 3.9.13; do \
+  wget --no-check-certificate -O python-${PYTHON_VERSION}.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz" \
+  && wget --no-check-certificate -O python-${PYTHON_VERSION}.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc" \
+  && gpg --batch --verify python-${PYTHON_VERSION}.tar.xz.asc python-${PYTHON_VERSION}.tar.xz \
+  && rm -r python-${PYTHON_VERSION}.tar.xz.asc \
+  && mkdir -p /usr/src/python-${PYTHON_VERSION} \
+  && tar -xJC /usr/src/python-${PYTHON_VERSION} --strip-components=1 -f python-${PYTHON_VERSION}.tar.xz \
+  && rm python-${PYTHON_VERSION}.tar.xz \
+  && cd /usr/src/python-${PYTHON_VERSION} \
+  && ./configure \
+  --enable-shared \
+  # This works only on Python 2.7 and throws a warning on every other
+  # version, but seems otherwise harmless.
+  --enable-unicode=ucs4 \
+  --with-system-ffi \
+  --without-ensurepip \
+  && make -j$(nproc) \
+  && make install \
+  && ldconfig \
+  ; done \
+  && rm -rf "${GNUPGHOME}" \
+  && rm -rf /usr/src/python* \
+  && rm -rf ~/.cache/
+
+# Install pip on Python 3.9 only.
+# If the environment variable is called "PIP_VERSION", pip explodes with
+# "ValueError: invalid truth value '<VERSION>'"
+ENV PYTHON_PIP_VERSION 23.0
+RUN wget --no-check-certificate -O /tmp/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' \
+  && python3.9 /tmp/get-pip.py "pip==$PYTHON_PIP_VERSION" \
+  # we use "--force-reinstall" for the case where the version of pip we're trying to install is the same as the version bundled with Python
+  # ("Requirement already up-to-date: pip==8.1.2 in /usr/local/lib/python3.10/site-packages")
+  # https://github.com/docker-library/python/pull/143#issuecomment-241032683
+  && pip3 install --no-cache-dir --upgrade --force-reinstall "pip==$PYTHON_PIP_VERSION" \
+  # then we use "pip list" to ensure we don't have more than one pip version installed
+  # https://github.com/docker-library/python/pull/100
+  && [ "$(pip list |tac|tac| awk -F '[ ()]+' '$1 == "pip" { print $2; exit }')" = "$PYTHON_PIP_VERSION" ]
+
 # Install Google Cloud SDK
 RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg  add - && apt-get update -y && apt-get install google-cloud-sdk -y
 

.kokoro/docker/docs/fetch_gpg_keys.sh

@@ -15,31 +15,25 @@
 
 # A script to fetch gpg keys with retry.
 # Avoid jinja parsing the file.
-# 
+#
 
 function retry {
     if [[ "${#}" -le 1 ]]; then
-	echo "Usage: ${0} retry_count commands.."
-	exit 1
+        echo "Usage: ${0} retry_count commands.."
+        exit 1
     fi
     local retries=${1}
     local command="${@:2}"
     until [[ "${retries}" -le 0 ]]; do
-	$command && return 0
-	if [[ $? -ne 0 ]]; then
-	    echo "command failed, retrying"
-	    ((retries--))
-	fi
+        $command && return 0
+        if [[ $? -ne 0 ]]; then
+            echo "command failed, retrying"
+            ((retries--))
+        fi
     done
     return 1
 }
 
-# 3.6.9, 3.7.5 (Ned Deily)
-retry 3 gpg --keyserver ha.pool.sks-keyservers.net --recv-keys \
-      0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D
-
-# 3.8.0 (Łukasz Langa)
-retry 3 gpg --keyserver ha.pool.sks-keyservers.net --recv-keys \
+# 3.8.0, 3.9.0 (Łukasz Langa)
+retry 3 gpg --keyserver keyserver.ubuntu.com --recv-keys \
       E3FF2839C048B25C084DEBE9B26995E310250568
-
-# 

.kokoro/generate-docs.sh

@@ -36,6 +36,8 @@ python3 -m pip install -e .
 python_bucket_items=$(gsutil ls "gs://docs-staging-v2/docfx-python*")
 # Store empty tarballs that did not produce any content to check later.
 empty_packages=""
+# Store monorepo packages to process later
+monorepo_packages=""
 # Retrieve unique repositories to regenerate the YAML with.
 for package in $(echo "${python_bucket_items}" | cut -d "-" -f 5- | rev | cut -d "-" -f 2- | rev | uniq); do
 
@@ -55,6 +57,15 @@ for package in $(echo "${python_bucket_items}" | cut -d "-" -f 5- | rev | cut -d
   tar -zxvf ${tarball}
   repo=$(cat docs.metadata | grep "github_repository:" | cut -d "\"" -f 2 | cut -d "/" -f 2)
 
+  # If the pacakage is part of the monorepo, we'll process this later.
+  if [[ "${repo}" == "google-cloud-python" ]]; then
+    # Add an extra whitespace at the end to be used as a natural separator.
+    monorepo_packages+="$(cat docs.metadata | grep "distribution_name" | cut -d "\"" -f 2) "
+    cd ..
+    rm -rf ${tarball}
+    continue
+  fi
+
   # Clean up the tarball content.
   cd ..
   rm -rf ${tarball}
@@ -152,7 +163,57 @@ for package in $(echo "${python_bucket_items}" | cut -d "-" -f 5- | rev | cut -d
   rm "noxfile.py"
 done
 
-if [ ! ${empty_packages} ]; then
+# Build documentation for monorepo packages
+if [ -n "${monorepo_packages}" ]; then
+  echo "Processing monorepo packages"
+  git clone "https://github.com/googleapis/google-cloud-python.git"
+  cd google-cloud-python/packages
+
+  # TODO (https://github.com/googleapis/sphinx-docfx-yaml/issues/287): support
+  # multi-version build for the monorepo.
+  for monorepo_package in $(echo ${monorepo_packages}); do
+    cd ${monorepo_package}
+
+    # Test running with the plugin version locally.
+    if [[ "${TEST_PLUGIN}" == "true" ]]; then
+      # --no-use-pep517 is required for django-spanner install issue: see https://github.com/pypa/pip/issues/7953
+      python3 -m pip install --user --no-use-pep517 -e .[all]
+      sphinx-build -T -N -D extensions=sphinx.ext.autodoc,sphinx.ext.autosummary,docfx_yaml.extension,sphinx.ext.intersphinx,sphinx.ext.coverage,sphinx.ext.napoleon,sphinx.ext.todo,sphinx.ext.viewcode,recommonmark -b html -d docs/_build/doctrees/ docs/ docs/_build/html/
+      continue
+    fi
+
+    # Build YAML tarballs for Cloud-RAD.
+    nox -s docfx
+
+    # Check that documentation is produced. If not, log and continue.
+    if [ ! "$(ls docs/_build/html/docfx_yaml/)" ]; then
+      empty_packages="${monorepo_package} ${empty_packages}"
+      continue
+    fi
+
+    python3 -m docuploader create-metadata \
+      --name=$(jq --raw-output '.name // empty' .repo-metadata.json) \
+      --version=$(python3 setup.py --version) \
+      --language=$(jq --raw-output '.language // empty' .repo-metadata.json) \
+      --distribution-name=$(python3 setup.py --name) \
+      --product-page=$(jq --raw-output '.product_documentation // empty' .repo-metadata.json) \
+      --github-repository=$(jq --raw-output '.repo // empty' .repo-metadata.json) \
+      --issue-tracker=$(jq --raw-output '.issue_tracker // empty' .repo-metadata.json)
+
+    cat docs.metadata
+
+    # upload docs
+    python3 -m docuploader upload docs/_build/html/docfx_yaml --metadata-file docs.metadata --destination-prefix docfx --staging-bucket "${V2_STAGING_BUCKET}"
+
+    # Clean up the package to make room.
+    cd ../
+    rm -rf ${monorepo_package}
+  done
+  cd ../../
+  rm -rf google-cloud-python
+fi
+
+if [ ! "${empty_packages}" ]; then
   exit
 fi
 

.kokoro/requirements.in

@@ -6,6 +6,7 @@ google-cloud-aiplatform[prediction]
 googleapis-common-protos==1.58.0 # pinned for conflict in range specifier.
 grpcio-status==1.48.2 # must be pinned due to protobuf compatibility.
 ipython
+jinja2<=3.0.3
 
 # General installations.
 gcp-docuploader

.kokoro/requirements.txt

@@ -430,10 +430,12 @@ jeepney==0.8.0 \
     # via
     #   keyring
     #   secretstorage
-jinja2==3.1.2 \
-    --hash=sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852 \
-    --hash=sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61
-    # via gcp-releasetool
+jinja2==3.0.3 \
+    --hash=sha256:077ce6014f7b40d03b47d1f1ca4b0fc8328a692bd284016f806ed0eaca390ad8 \
+    --hash=sha256:611bb273cd68f3b993fabdc4064fc858c5b47a973cb5aa7999ec1ba405c87cd7
+    # via
+    #   -r requirements.in
+    #   gcp-releasetool
 keyring==23.9.0 \
     --hash=sha256:4c32a31174faaee48f43a7e2c7e9c3216ec5e95acf22a2bebfb4a1d05056ee44 \
     --hash=sha256:98f060ec95ada2ab910c195a2d4317be6ef87936a766b239c46aa3c7aac4f0db

setup.py

@@ -22,7 +22,8 @@
     'black',
     'gcp-docuploader',
     'PyYAML',
-    'sphinx',
+    'recommonmark',
+    'sphinx==4.5.0',
     'sphinx-markdown-builder',
     'sphinxcontrib.napoleon',
     'unidecode',