Build cacerts during build from Mozilla's list (#2033)

The cacers keystore is created at build time from a locally stored plain text copy of Mozilla's list of CA certificates.

Fixes adoptium/adoptium-support#13.

README.md

@@ -169,6 +169,9 @@ if using docker, keep the container after the build.
 --make-exploded-image
 creates an exploded image (useful for codesigning jmods). Use --assemble-exploded-image once you have signed the jmods to complete the packaging steps.
 
+--custom-cacerts <true|false>
+If true (default), a custom cacerts file will be generated based on Mozilla's list of CA certificates (see folder security/). If false, the file shipped by OpenJDK will be used. 
+
 -n, --no-colour
 disable colour output.
 
@@ -212,8 +215,7 @@ For reference, OpenJDK version numbers look like 1.8.0_162-b12 (for Java 8) or
 This is typically used in conjunction with -b.
 
 --use-jep319-certs
-Use certs defined in JEP319 in Java 8/9. This will increase the volume of traffic downloaded, however will
-provide an upto date ca cert list.
+Use certs defined in JEP319 in Java 8/9. Deprecated, has no effect.
 
 -v, --version
 specify the OpenJDK version to build e.g. jdk8u.  Left for backwards compatibility.

docker/dockerfile-generator.sh

@@ -243,6 +243,7 @@ printDockerJDKs() {
     printJDK $((JDK_VERSION-1))
     echo "RUN ln -sf /usr/lib/jvm/jdk$((JDK_VERSION-1))/bin/java /usr/bin/java" >> $DOCKERFILE_PATH
     echo "RUN ln -sf /usr/lib/jvm/jdk$((JDK_VERSION-1))/bin/javac /usr/bin/javac" >> $DOCKERFILE_PATH
+    echo "RUN ln -sf /usr/lib/jvm/jdk$((JDK_VERSION-1))/bin/keytool /usr/bin/keytool" >> $DOCKERFILE_PATH
   fi
 
   # Build 'jdk' with the most recent GA release
@@ -254,6 +255,7 @@ printDockerJDKs() {
     printJDK ${JDK_GA}
     echo "RUN ln -sf /usr/lib/jvm/jdk${JDK_GA}/bin/java /usr/bin/java" >> $DOCKERFILE_PATH
     echo "RUN ln -sf /usr/lib/jvm/jdk${JDK_GA}/bin/javac /usr/bin/javac" >> $DOCKERFILE_PATH
+    echo "RUN ln -sf /usr/lib/jvm/jdk${JDK_GA}/bin/keytool /usr/bin/keytool" >> $DOCKERFILE_PATH
   fi
 
   # if JDK_VERSION is 9, another jdk8 doesn't need to be extracted
@@ -274,6 +276,7 @@ RUN sh -c \"mkdir -p /usr/lib/jvm/jdk$JDKVersion && wget 'https://api.adoptopenj
 printCopyFolders(){
   echo "
 COPY sbin /openjdk/sbin
+COPY security /openjdk/security
 COPY workspace/config /openjdk/config
 COPY pipelines /openjdk/pipelines" >> $DOCKERFILE_PATH
 }

makejdk-any-platform.1

@@ -166,8 +166,7 @@ For reference, OpenJDK version numbers look like 1.8.0_162-b12 (for Java 8) or
 This is typically used in conjunction with \fB<-b>\fR.
 .TP
 .BR \-\-use-jep319-certs
-Use certs defined in JEP319 in Java 8/9. This will increase the volume of traffic
-downloaded, however will provide an uptodate ca cert list.
+Use certs defined in JEP319 in Java 8/9. Deprecated, has no effect.
 .TP
 .BR \-v ", " \-\-version " " \fI<version>\fR
 specify the OpenJDK version to build e.g. jdk8u.  Left for backwards compatibility.

sbin/build.sh

@@ -357,6 +357,11 @@ configureCommandParameters()
   echo "Configuring jvm variants if provided"
   addConfigureArgIfValueIsNotEmpty "--with-jvm-variants=" "${BUILD_CONFIG[JVM_VARIANT]}"
 
+  if [ "${BUILD_CONFIG[CUSTOM_CACERTS]}" != "false" ] ; then
+    echo "Configure custom cacerts file security/cacerts"
+    addConfigureArgIfValueIsNotEmpty "--with-cacerts-file=" "$SCRIPT_DIR/../security/cacerts"
+  fi
+
   # Now we add any configure arguments the user has specified on the command line.
   CONFIGURE_ARGS="${CONFIGURE_ARGS} ${BUILD_CONFIG[USER_SUPPLIED_CONFIGURE_ARGS]}"
 

sbin/common/config_init.sh

@@ -47,6 +47,7 @@ CONTAINER_NAME
 COPY_MACOSX_FREE_FONT_LIB_FOR_JDK_FLAG
 COPY_MACOSX_FREE_FONT_LIB_FOR_JRE_FLAG
 COPY_TO_HOST
+CUSTOM_CACERTS
 DEBUG_DOCKER
 DEBUG_IMAGE_PATH
 DISABLE_ADOPT_BRANCH_SAFETY
@@ -206,6 +207,9 @@ function parseConfigurationArguments() {
         "--assemble-exploded-image" )
         BUILD_CONFIG[ASSEMBLE_EXPLODED_IMAGE]=true;;
 
+        "--custom-cacerts" )
+        BUILD_CONFIG[CUSTOM_CACERTS]="$1"; shift;;
+
         "--codesign-identity" )
         BUILD_CONFIG[MACOSX_CODESIGN_IDENTITY]="$1"; shift;;
 
@@ -465,6 +469,9 @@ function configDefaults() {
   BUILD_CONFIG[USER_SUPPLIED_CONFIGURE_ARGS]=${BUILD_CONFIG[USER_SUPPLIED_CONFIGURE_ARGS]:-""}
   BUILD_CONFIG[USER_SUPPLIED_MAKE_ARGS]=${BUILD_CONFIG[USER_SUPPLIED_MAKE_ARGS]:-""}
 
+  # Whether to use AdoptOpenJDK's cacerts file (true) or use the file provided by OpenJDK (false)
+  BUILD_CONFIG[CUSTOM_CACERTS]=${BUILD_CONFIG[CUSTOM_CACERTS]:-"true"}
+
   BUILD_CONFIG[DOCKER]=${BUILD_CONFIG[DOCKER]:-"docker"}
 
   BUILD_CONFIG[TMP_SPACE_BUILD]=${BUILD_CONFIG[TMP_SPACE_BUILD]:-false}

sbin/prepareWorkspace.sh

@@ -505,44 +505,15 @@ checkingAndDownloadingFreeType() {
   fi
 }
 
-# Download our security certificates
-downloadCerts() {
-  local caLink="$1"
+# Generates cacerts file
+prepareCacerts() {
+    echo "Generating cacerts from Mozilla's bundle"
 
-  mkdir -p "security"
-  # Temporary fudge as curl on my windows boxes is exiting with RC=127
-  if [[ "$OSTYPE" == "cygwin" ]] || [[ "$OSTYPE" == "msys" ]]; then
-    wget -O "./security/cacerts" "${caLink}"
-  else
-    curl -L -o "./security/cacerts" "${caLink}"
-  fi
-}
-
-# Certificate Authority Certs (CA Certs)
-checkingAndDownloadCaCerts() {
-  cd "${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[WORKING_DIR]}" || exit
-
-  echo "Retrieving cacerts file if needed"
-  # Ensure it's the latest we pull in
-  rm -rf "cacerts_area"
-  mkdir "cacerts_area" || exit
-  cd "cacerts_area" || exit
-
-  if [ "${BUILD_CONFIG[BUILD_VARIANT]}" == "${BUILD_VARIANT_CORRETTO}" ]; then
-    local caLink="https://github.com/corretto/corretto-8/blob/preview-release/cacerts?raw=true"
-    downloadCerts "$caLink"
-  elif [ "${BUILD_CONFIG[USE_JEP319_CERTS]}" != "true" ]; then
-    git init
-    git remote add origin -f "${BUILD_CONFIG[OPENJDK_BUILD_REPO_URI]}"
-    git config core.sparsecheckout true
-    echo "security/*" >>.git/info/sparse-checkout
-    git pull origin "${BUILD_CONFIG[OPENJDK_BUILD_REPO_BRANCH]}"
-  fi
-
-  cd "${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[WORKING_DIR]}" || exit
+    cd "$SCRIPT_DIR/../security"
+    ./mk-cacerts.sh --keytool "${BUILD_CONFIG[JDK_BOOT_DIR]}/bin/keytool"
 }
 
-# Download all of the dependencies for OpenJDK (Alsa, FreeType, CACerts et al)
+# Download all of the dependencies for OpenJDK (Alsa, FreeType, etc.)
 downloadingRequiredDependencies() {
   if [[ "${BUILD_CONFIG[CLEAN_LIBS]}" == "true" ]]; then
     rm -rf "${BUILD_CONFIG[WORKSPACE_DIR]}/libs/freetype" || true
@@ -582,10 +553,6 @@ downloadingRequiredDependencies() {
   else
     echo "Skipping Freetype"
   fi
-
-  echo "Checking and download CaCerts dependency"
-  checkingAndDownloadCaCerts
-
 }
 
 function moveTmpToWorkspaceLocation() {
@@ -671,5 +638,6 @@ function configureWorkspace() {
     relocateToTmpIfNeeded
     checkoutAndCloneOpenJDKGitRepo
     applyPatches
+    prepareCacerts
   fi
 }

security/.gitignore

@@ -0,0 +1,3 @@
+ca-bundle.crt
+cacerts
+certs/

security/README.md

@@ -1,13 +1,48 @@
 # Security files for OpenJDK
 
-### This repository contains the cacerts file used with OpenJDK
+## CA Certificates
 
-#### Steps we use to create the cacerts file
+The cacerts file that OpenJDK maintains lacks a number of CA certificates that are present in common browsers. As a result, users of OpenJDK cannot connect to servers with Java that they can connect to with their browsers. This causes confusion and [support requests][support-issues]. 
 
-1. Download the following Perl script: https://raw.githubusercontent.com/curl/curl/master/lib/mk-ca-bundle.pl
+In May 2020, [we reached out to OpenJDK to discuss this situation][jdk-dev-thread], but no consensus was reached. Consequently, the [AdoptOpenJDK TSC decided to deviate from OpenJDK and distribute a custom trust store][tsc-decision] based on [Mozilla's list of trusted CA certificates][mozilla-certdata] which is also used by many Linux distributions.
 
-2. Download the following Java application: https://github.com/use-sparingly/keyutil/releases/download/0.4.0/keyutil-0.4.0.jar (source available at https://github.com/use-sparingly/keyutil)
+If you want to build OpenJDK with the original cacerts file, set `--custom-cacerts=false`.
 
-3. Run the provided `GenerateCertsFile.sh` script with: `bash ./GenerateCertsFile.sh` - this will use the above files assuming they're located in the same directory as the script
+### Process
 
-4. Use the cacerts provided: it must be in the `jdk/jre/lib/security` or `jdk/lib/security`folder
+The `cacerts` file is build as part of the regular JDK build from source. The reason is that vetting blobs in PR is difficult. Because `certdata.txt` cannot be converted directly into a Java Key Store, we do it in multiple steps:
+
+1. Convert `certdata.txt` in a PEM file (`ca-bundle.crt`) with [mk-ca-bundle.pl][mk-ca-bundle.pl].
+2. Split `ca-bundle.crt` into individual certificates and import them with `keytool` into a new `cacerts` file.
+
+To generate a new `cacerts` file, run:
+
+    $ ./mk-cacerts.sh
+
+If anybody ever plans to replace `mk-ca-bundle.pl`, be sure to read [Can I use Mozilla's set of CA certificates?][can-i-use-mozilla].
+
+### Updating the List of Certificates
+
+Every time Mozilla updates the list of CA certificates, we have to update our copy of `certdata.txt`. Whether it needs to be updated can be checked on [curl's website][curl-ca-extract]. If it needs updating, the process looks as follows:
+
+1. Download the [current version of certdata.txt][mozilla-certdata].
+2. Replace the existing file in `security`.
+3. Open a pull request to get it merged.
+
+The updated list will be picked up during the next build.
+
+### License
+
+The resulting cacerts file is licensed under the terms of the [source file][mozilla-certdata], the Mozilla Public License, v.2.0.
+
+## Future Work
+
+* Create a GitHub bot that checks whether `certdata.txt` needs updating and automatically creates a PR.
+
+ [support-issues]: https://github.com/AdoptOpenJDK/openjdk-support/issues/13
+ [jdk-dev-thread]: https://mail.openjdk.java.net/pipermail/jdk-dev/2020-May/004305.html
+ [tsc-decision]: https://github.com/AdoptOpenJDK/openjdk-support/issues/13#issuecomment-635400251
+ [mozilla-certdata]: https://hg.mozilla.org/releases/mozilla-release/raw-file/default/security/nss/lib/ckfw/builtins/certdata.txt
+ [mk-ca-bundle.pl]: https://curl.haxx.se/docs/mk-ca-bundle.html
+ [curl-ca-extract]: https://curl.haxx.se/docs/caextract.html
+ [can-i-use-mozilla]: https://wiki.mozilla.org/CA/FAQ#Can_I_use_Mozilla.27s_set_of_CA_certificates.3F