Merge pull request #208 from pyutils/xcookie-ci-update

CI Upgrades
pyutils · Aug 4, 2023 · dbb01ae · dbb01ae
2 parents 7706d56 + 28288d8
commit dbb01ae
Show file tree

Hide file tree

Showing 20 changed files with 1,063 additions and 275 deletions.
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
diff --git a/.readthedocs.yml b/.readthedocs.yml
@@ -1,33 +1,18 @@
 # .readthedocs.yml
 # Read the Docs configuration file
 # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
-# 
+#
 # See Also:
 # https://readthedocs.org/dashboard/line_profiler/advanced/
 
 # Required
 version: 2
-
-# Build documentation in the docs/ directory with Sphinx
 sphinx:
   configuration: docs/source/conf.py
-
-# Build documentation with MkDocs
-#mkdocs:
-#  configuration: mkdocs.yml
-
-# Optionally build your docs in additional formats such as PDF and ePub
 formats: all
-
-# Optionally set the version of Python and requirements required to build your docs
 python:
   version: 3.7
   install:
-    - requirements: requirements/docs.txt
-    - method: pip
-      path: .
-      #extra_requirements:
-      #  - docs
-
-#conda:
-#  environment: environment.yml
+  - requirements: requirements/docs.txt
+  - method: pip
+    path: .
diff --git a/build_wheels.sh b/build_wheels.sh
@@ -1,8 +1,24 @@
 #!/bin/bash
-__doc__="""
+__doc__="
+Runs cibuildwheel to create linux binary wheels.
+
+Requirements:
+    pip install cibuildwheel
+
 SeeAlso:
     pyproject.toml
-"""
+"
+
+if ! which docker ; then
+    echo "Missing requirement: docker. Please install docker before running build_wheels.sh"
+    exit 1
+fi
+if ! which cibuildwheel ; then
+    echo "The cibuildwheel module is not installed. Please pip install cibuildwheel before running build_wheels.sh"
+    exit 1
+fi
+
+
 #pip wheel -w wheelhouse .
 # python -m build --wheel -o wheelhouse  #  line_profiler: +COMMENT_IF(binpy)
 cibuildwheel --config-file pyproject.toml --platform linux --arch x86_64  #  line_profiler: +UNCOMMENT_IF(binpy)
diff --git a/dev/setup_secrets.sh b/dev/setup_secrets.sh
@@ -25,7 +25,7 @@ following CI platforms is used:
 GITHUB ACTION INSTRUCTIONS
 =========================
 
-* `PERSONAL_GITHUB_PUSH_TOKEN` - 
+* `PERSONAL_GITHUB_PUSH_TOKEN` -
     This is only needed if you want to automatically git-tag release branches.
 
     To make a API token go to:
@@ -47,8 +47,8 @@ GITLAB ACTION INSTRUCTIONS
     tee /tmp/repl && colordiff .setup_secrets.sh /tmp/repl
     ```
 
-    * Make sure you add Runners to your project 
-    https://gitlab.org.com/utils/xcookie/-/settings/ci_cd 
+    * Make sure you add Runners to your project
+    https://gitlab.org.com/utils/xcookie/-/settings/ci_cd
     in Runners-> Shared Runners
     and Runners-> Available specific runners
 
@@ -60,16 +60,16 @@ GITLAB ACTION INSTRUCTIONS
     * TWINE_USERNAME - this is your pypi username
         twine info is only needed if you want to automatically publish to pypi
 
-    * TWINE_PASSWORD - this is your pypi password 
+    * TWINE_PASSWORD - this is your pypi password
 
-    * CI_SECRET - We will use this as a secret key to encrypt/decrypt gpg secrets 
+    * CI_SECRET - We will use this as a secret key to encrypt/decrypt gpg secrets
         This is only needed if you want to automatically sign published
         wheels with a gpg key.
 
-    * GITLAB_ORG_PUSH_TOKEN - 
+    * GITLAB_ORG_PUSH_TOKEN -
         This is only needed if you want to automatically git-tag release branches.
 
-        Create a new personal access token in User->Settings->Tokens, 
+        Create a new personal access token in User->Settings->Tokens,
         You can name the token GITLAB_ORG_PUSH_TOKEN_VALUE
         Give it api and write repository permissions
 
@@ -165,8 +165,10 @@ setup_package_environs_github_pyutils(){
 upload_github_secrets(){
     load_secrets
     unset GITHUB_TOKEN
-    #printf "%s" "$GITHUB_TOKEN" | gh auth login --hostname Github.com --with-token 
-    gh auth login 
+    #printf "%s" "$GITHUB_TOKEN" | gh auth login --hostname Github.com --with-token
+    if ! gh auth status ; then
+        gh auth login
+    fi
     source dev/secrets_configuration.sh
     gh secret set "TWINE_USERNAME" -b"${!VARNAME_TWINE_USERNAME}"
     gh secret set "TEST_TWINE_USERNAME" -b"${!VARNAME_TEST_TWINE_USERNAME}"
@@ -223,15 +225,15 @@ upload_gitlab_group_secrets(){
 
     TMP_DIR=$(mktemp -d -t ci-XXXXXXXXXX)
     curl --header "PRIVATE-TOKEN: $PRIVATE_GITLAB_TOKEN" "$HOST/api/v4/groups" > "$TMP_DIR/all_group_info"
-    GROUP_ID=$(cat "$TMP_DIR/all_group_info" | jq ". | map(select(.path==\"$GROUP_NAME\")) | .[0].id")
+    GROUP_ID=$(< "$TMP_DIR/all_group_info" jq ". | map(select(.path==\"$GROUP_NAME\")) | .[0].id")
     echo "GROUP_ID = $GROUP_ID"
 
     curl --header "PRIVATE-TOKEN: $PRIVATE_GITLAB_TOKEN" "$HOST/api/v4/groups/$GROUP_ID" > "$TMP_DIR/group_info"
-    cat "$TMP_DIR/group_info" | jq
+    < "$TMP_DIR/group_info" jq
 
     # Get group-level secret variables
     curl --header "PRIVATE-TOKEN: $PRIVATE_GITLAB_TOKEN" "$HOST/api/v4/groups/$GROUP_ID/variables" > "$TMP_DIR/group_vars"
-    cat "$TMP_DIR/group_vars" | jq '.[] | .key'
+    < "$TMP_DIR/group_vars" jq '.[] | .key'
 
     if [[ "$?" != "0" ]]; then
         echo "Failed to access group level variables. Probably a permission issue"
@@ -244,7 +246,7 @@ upload_gitlab_group_secrets(){
         echo ""
         echo " ---- "
         LOCAL_VALUE=${!SECRET_VARNAME}
-        REMOTE_VALUE=$(cat "$TMP_DIR/group_vars" | jq -r ".[] | select(.key==\"$SECRET_VARNAME\") | .value")
+        REMOTE_VALUE=$(< "$TMP_DIR/group_vars" jq -r ".[] | select(.key==\"$SECRET_VARNAME\") | .value")
 
         # Print current local and remote value of a variable
         echo "SECRET_VARNAME_PTR = $SECRET_VARNAME_PTR"
@@ -264,14 +266,14 @@ upload_gitlab_group_secrets(){
                     --form "protected=true" \
                     --form "masked=true" \
                     --form "environment_scope=*" \
-                    --form "variable_type=env_var" 
+                    --form "variable_type=env_var"
             toggle_setx_exit
         elif [[ "$REMOTE_VALUE" != "$LOCAL_VALUE" ]]; then
             echo "Remove variable does not agree, putting"
             # Update variable value
             toggle_setx_enter
             curl --request PUT --header "PRIVATE-TOKEN: $PRIVATE_GITLAB_TOKEN" "$HOST/api/v4/groups/$GROUP_ID/variables/$SECRET_VARNAME" \
-                    --form "value=${LOCAL_VALUE}" 
+                    --form "value=${LOCAL_VALUE}"
             toggle_setx_exit
         else
             echo "Remote value agrees with local"
@@ -305,23 +307,23 @@ upload_gitlab_repo_secrets(){
     toggle_setx_enter
     curl --header "PRIVATE-TOKEN: $PRIVATE_GITLAB_TOKEN" "$HOST/api/v4/groups" > "$TMP_DIR/all_group_info"
     toggle_setx_exit
-    GROUP_ID=$(cat "$TMP_DIR/all_group_info" | jq ". | map(select(.path==\"$GROUP_NAME\")) | .[0].id")
+    GROUP_ID=$(< "$TMP_DIR/all_group_info" jq ". | map(select(.path==\"$GROUP_NAME\")) | .[0].id")
     echo "GROUP_ID = $GROUP_ID"
 
     toggle_setx_enter
     curl --header "PRIVATE-TOKEN: $PRIVATE_GITLAB_TOKEN" "$HOST/api/v4/groups/$GROUP_ID" > "$TMP_DIR/group_info"
     toggle_setx_exit
-    GROUP_ID=$(cat "$TMP_DIR/all_group_info" | jq ". | map(select(.path==\"$GROUP_NAME\")) | .[0].id")
-    cat "$TMP_DIR/group_info" | jq
+    GROUP_ID=$(< "$TMP_DIR/all_group_info" jq ". | map(select(.path==\"$GROUP_NAME\")) | .[0].id")
+    < "$TMP_DIR/group_info" jq
 
-    PROJECT_ID=$(cat "$TMP_DIR/group_info" | jq ".projects | map(select(.path==\"$PROJECT_NAME\")) | .[0].id")
+    PROJECT_ID=$(< "$TMP_DIR/group_info" jq ".projects | map(select(.path==\"$PROJECT_NAME\")) | .[0].id")
     echo "PROJECT_ID = $PROJECT_ID"
 
     # Get group-level secret variables
     toggle_setx_enter
     curl --header "PRIVATE-TOKEN: $PRIVATE_GITLAB_TOKEN" "$HOST/api/v4/projects/$PROJECT_ID/variables" > "$TMP_DIR/project_vars"
     toggle_setx_exit
-    cat "$TMP_DIR/project_vars" | jq '.[] | .key'
+    < "$TMP_DIR/project_vars" jq '.[] | .key'
     if [[ "$?" != "0" ]]; then
         echo "Failed to access project level variables. Probably a permission issue"
     fi
@@ -334,7 +336,7 @@ upload_gitlab_repo_secrets(){
         echo ""
         echo " ---- "
         LOCAL_VALUE=${!SECRET_VARNAME}
-        REMOTE_VALUE=$(cat "$TMP_DIR/project_vars" | jq -r ".[] | select(.key==\"$SECRET_VARNAME\") | .value")
+        REMOTE_VALUE=$(< "$TMP_DIR/project_vars" jq -r ".[] | select(.key==\"$SECRET_VARNAME\") | .value")
 
         # Print current local and remote value of a variable
         echo "SECRET_VARNAME_PTR = $SECRET_VARNAME_PTR"
@@ -353,7 +355,7 @@ upload_gitlab_repo_secrets(){
                         --form "protected=true" \
                         --form "masked=true" \
                         --form "environment_scope=*" \
-                        --form "variable_type=env_var" 
+                        --form "variable_type=env_var"
             else
                 echo "dry run, not posting"
             fi
@@ -362,7 +364,7 @@ upload_gitlab_repo_secrets(){
             # Update variable value
             if [[ "$LIVE_MODE" == "1" ]]; then
                 curl --request PUT --header "PRIVATE-TOKEN: $PRIVATE_GITLAB_TOKEN" "$HOST/api/v4/projects/$PROJECT_ID/variables/$SECRET_VARNAME" \
-                        --form "value=${LOCAL_VALUE}" 
+                        --form "value=${LOCAL_VALUE}"
             else
                 echo "dry run, not putting"
             fi
@@ -396,12 +398,16 @@ export_encrypted_code_signing_keys(){
     # Careful, if you don't have a subkey, requesting it will export more than you want.
     # Export the main key instead (its better to have subkeys, but this is a lesser evil)
     if [[ "$GPG_SIGN_SUBKEY" == "" ]]; then
+        # NOTE: if you get here this probably means your subkeys expired (and
+        # wont even be visible), so we probably should check for that here and
+        # thrown an error instead of using this hack, which likely wont work
+        # anyway.
         GPG_SIGN_SUBKEY=$(gpg --list-keys --with-subkey-fingerprints "$GPG_IDENTIFIER" | grep "\[C\]" -A 1 | tail -n 1 | awk '{print $1}')
     fi
     echo "MAIN_GPG_KEYID  = $MAIN_GPG_KEYID"
     echo "GPG_SIGN_SUBKEY = $GPG_SIGN_SUBKEY"
 
-    # Only export the signing secret subkey 
+    # Only export the signing secret subkey
     # Export plaintext gpg public keys, private sign key, and trust info
     mkdir -p dev
     gpg --armor --export-options export-backup --export-secret-subkeys "${GPG_SIGN_SUBKEY}!" > dev/ci_secret_gpg_subkeys.pgp
@@ -417,7 +423,7 @@ export_encrypted_code_signing_keys(){
     # Test decrpyt
     GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_public_gpg_key.pgp.enc | gpg --list-packets --verbose
     GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_subkeys.pgp.enc  | gpg --list-packets --verbose
-    GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/gpg_owner_trust.enc 
+    GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/gpg_owner_trust.enc
     cat dev/public_gpg_key
 
     unload_secrets
@@ -447,14 +453,14 @@ _test_gnu(){
     source dev/secrets_configuration.sh
 
     gpg -k
-    
+
     load_secrets
     CI_SECRET="${!VARNAME_CI_SECRET}"
     echo "CI_SECRET = $CI_SECRET"
 
     cat dev/public_gpg_key
-    GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_public_gpg_key.pgp.enc 
-    GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/gpg_owner_trust.enc 
+    GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_public_gpg_key.pgp.enc
+    GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/gpg_owner_trust.enc
     GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_subkeys.pgp.enc
 
     GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_public_gpg_key.pgp.enc | gpg --import