From 1f3ab6c7bed0d7d65d12f0412ea3c9003b22f5a4 Mon Sep 17 00:00:00 2001 From: Damien Duportal Date: Tue, 5 Apr 2022 16:02:33 +0200 Subject: [PATCH] chore(pipeline) Migrate from trusted.ci.jenkins.io to infra.ci.jenkins.io Signed-off-by: Damien Duportal --- JenkinsAgentPodTemplate.yaml | 16 +++ Jenkinsfile | 124 ------------------ README.md | 60 ++++++--- artifactory-users-report/Dockerfile | 8 -- artifactory-users-report/Jenkinsfile | 44 +++++++ artifactory-users-report/user-report.sh | 7 +- fork-report/Dockerfile | 7 - fork-report/Jenkinsfile | 45 +++++++ fork-report/fork-report.rb | 2 +- jira-users-report/Dockerfile | 9 -- jira-users-report/Jenkinsfile | 44 +++++++ jira-users-report/user-report.sh | 1 + maintainers-info-report/Dockerfile | 9 -- maintainers-info-report/Jenkinsfile | 44 +++++++ .../maintainers-info-report.sh | 1 + permissions-report/Dockerfile | 9 -- permissions-report/Jenkinsfile | 46 +++++++ permissions-report/permissions-report.rb | 4 +- 18 files changed, 287 insertions(+), 193 deletions(-) create mode 100644 JenkinsAgentPodTemplate.yaml delete mode 100644 Jenkinsfile delete mode 100644 artifactory-users-report/Dockerfile create mode 100644 artifactory-users-report/Jenkinsfile delete mode 100644 fork-report/Dockerfile create mode 100644 fork-report/Jenkinsfile delete mode 100644 jira-users-report/Dockerfile create mode 100644 jira-users-report/Jenkinsfile delete mode 100644 maintainers-info-report/Dockerfile create mode 100644 maintainers-info-report/Jenkinsfile delete mode 100644 permissions-report/Dockerfile create mode 100644 permissions-report/Jenkinsfile diff --git a/JenkinsAgentPodTemplate.yaml b/JenkinsAgentPodTemplate.yaml new file mode 100644 index 0000000..418b976 --- /dev/null +++ b/JenkinsAgentPodTemplate.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Pod +spec: + containers: + - image: "jenkinsciinfra/helmfile:2.3.23" + imagePullPolicy: "IfNotPresent" + name: "jnlp" + resources: + limits: + memory: "256Mi" + cpu: "1" + requests: + memory: "256Mi" + cpu: "1" + securityContext: + privileged: false diff --git a/Jenkinsfile b/Jenkinsfile deleted file mode 100644 index 6e1e142..0000000 --- a/Jenkinsfile +++ /dev/null @@ -1,124 +0,0 @@ -#!/usr/bin/env groovy - -pipeline { - - triggers { - cron('H * * * *') - } - - options { - // 6 hours timeout combined with lock and inverse precedence to will properly gate the GitHub permissions report - timeout(time: 25, unit: 'HOURS') - } - - agent none - - stages { - - /* Run only on ci.jenkins.io for PR builds */ - stage ('CI Build') { - when { - expression { - !infra.isTrusted() - } - } - agent { - label 'docker&&linux' - } - steps { - sh 'docker build permissions-report -t fork-report' - sh 'docker build permissions-report -t permissions-report' - sh 'docker build artifactory-users-report -t artifactory-users-report' - sh 'docker build jira-users-report -t jira-users-report' - sh 'docker build maintainers-info-report -t maintainers-info-report' - } - } - - /* When running on trusted.ci.jenkins.io, build images and publish reports */ - stage ('Publishing') { - when { - expression { - infra.isTrusted() - } - } - parallel { - stage('GitHub Forks') { - agent { - label 'docker' - } - steps { - withCredentials( - [usernamePassword(credentialsId: 'jenkins-infra-reports', usernameVariable: 'GITHUB_APP', passwordVariable: 'GITHUB_ACCESS_TOKEN')] - ) { - sh 'docker build fork-report -t fork-report' - sh 'docker run -e GITHUB_API_TOKEN=$GITHUB_ACCESS_TOKEN fork-report > github-jenkinsci-fork-report.json' - archiveArtifacts 'github-jenkinsci-fork-report.json' - publishReports ([ 'github-jenkinsci-fork-report.json' ]) - } - } - } - stage('Artifactory Users') { - agent { - label 'docker&&linux' - } - environment { - ARTIFACTORY_AUTH = credentials('artifactoryAdmin') - } - steps { - sh 'docker build artifactory-users-report -t artifactory-users-report' - sh 'docker run -e ARTIFACTORY_AUTH=$ARTIFACTORY_AUTH artifactory-users-report > artifactory-ldap-users-report.json' - archiveArtifacts 'artifactory-ldap-users-report.json' - publishReports ([ 'artifactory-ldap-users-report.json' ]) - } - } - stage('Jira Users') { - agent { - label 'docker' - } - environment { - JIRA_AUTH = credentials('jiraAuth') - } - steps { - sh 'docker build jira-users-report -t jira-users-report' - sh 'docker run -e JIRA_AUTH="${JIRA_AUTH_USR}:${JIRA_AUTH_PSW}" jira-users-report > jira-users-report.json' - archiveArtifacts 'jira-users-report.json' - publishReports ([ 'jira-users-report.json' ]) - } - } - stage('Maintainers Jira Info') { - agent { - label 'docker' - } - environment { - JIRA_AUTH = credentials('jiraAuth') - } - steps { - sh 'docker build maintainers-info-report -t maintainers-info-report' - sh 'docker run -e JIRA_AUTH="${JIRA_AUTH_USR}:${JIRA_AUTH_PSW}" maintainers-info-report > maintainers-info-report.json' - archiveArtifacts 'maintainers-info-report.json' - publishReports ([ 'maintainers-info-report.json' ]) - } - } - stage('GitHub Permissions') { - agent { - label 'docker&&linux' - } - environment { - GITHUB_APP_PRIVATE_KEY_B64 = credentials('githubapp-jenkins-infra-reports-private-key-b64') - GITHUB_APP_ID = credentials('githubapp-jenkins-infra-reports-app-identifier') - GITHUB_ORG_NAME = "jenkinsci" - } - options { - lock(resource: 'github-permissions', inversePrecedence: true) - } - steps { - sh 'docker build permissions-report -t permissions-report' - sh 'docker run -e GITHUB_APP_PRIVATE_KEY_B64 -e GITHUB_APP_ID -e GITHUB_ORG_NAME permissions-report > github-jenkinsci-permissions-report.json' - archiveArtifacts 'github-jenkinsci-permissions-report.json' - publishReports ([ 'github-jenkinsci-permissions-report.json' ]) - } - } - } - } - } -} diff --git a/README.md b/README.md index 27aa145..d821787 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,14 @@ # Jenkins GitHub Reports -Use [octokit.rb](http://octokit.github.io/octokit.rb/) to generate reports about the `jenkinsci` GitHub organization. +If you want to execute these scripts locally, use the Docker image `jenkinsciinfra/helmfile` (source code at ). +It includes all the required dependencies: + +- Ruby +- Ruby Gems for [octokit.rb](http://octokit.github.io/octokit.rb/) to generate reports about the `jenkinsci` GitHub organization and graphql +- Bash +- JQ, Azure and other command lines used by bash scripts + +You can check the exact image version by checking the Jenkins agent pod template specified in `./JenkinsAgentPodTemplate.yaml`. ## Permissions Report @@ -8,18 +16,20 @@ Prints a two-dimensional JSON array optimized for use in [DataTables](https://ww Format example: - [ - [ - "ldap-plugin", - "olamy", - "push" - ], - [ - "ldap-plugin", - "jglick", - "push" - ] - ] +```json +[ + [ + "ldap-plugin", + "olamy", + "push" + ], + [ + "ldap-plugin", + "jglick", + "push" + ] +] +``` ### Usage @@ -29,32 +39,40 @@ We use a Github App for that, you'll need to define the following environment va - GITHUB_APP_ID: The GitHub App's identifier (type integer) set when registering an app - GITHUB_ORG_NAME: The Github organization name (ex: "jenkinsci") - docker build permissions-report -t permissions-report - docker run -e GITHUB_APP_PRIVATE_KEY_B64 -e GITHUB_APP_ID -e GITHUB_ORG_NAME permissions-report +```shell +cd permisions-report/ +ruby ./permisions-report.rb +``` ## Artifactory Users Report Creates a report listing all user accounts in Artifactory. -Consumed by https://github.com/jenkins-infra/repository-permissions-updater/blob/master/src/main/groovy/io/jenkins/infra/repository_permissions_updater/KnownUsers.groovy +Consumed by ### Usage This requires Artifactory admin user credentials. - docker build artifactory-users-report -t artifactory-users-report - docker run -e ARTIFACTORY_AUTH=admin-username:admin-token artifactory-users-report +```bash +cd artifactory-users-report/ +export ARTIFACTORY_AUTH=admin-username:admin-token +bash ./user-report.sh +``` ## Jira Users Report Creates a report listing all user accounts in a Jira group containing plugin maintainers. Currently, we use `jira-users` for that, but may in the future use a more limited group. -Consumed by https://github.com/jenkins-infra/repository-permissions-updater/blob/master/src/main/groovy/io/jenkins/infra/repository_permissions_updater/KnownUsers.groovy +Consumed by ### Usage This requires Jira admin user credentials. - docker build jira-users-report -t jira-users-report - docker run -e JIRA_AUTH=theUser:thePassword jira-users-report +```bash +cd jira-users-report/ +export JIRA_AUTH=admin-username:admin-token +bash ./user-report +``` diff --git a/artifactory-users-report/Dockerfile b/artifactory-users-report/Dockerfile deleted file mode 100644 index 90b1519..0000000 --- a/artifactory-users-report/Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -FROM debian:stable-slim - -RUN apt-get update -RUN apt-get install -y wget curl - -COPY user-report.sh / - -ENTRYPOINT ["sh", "/user-report.sh"] diff --git a/artifactory-users-report/Jenkinsfile b/artifactory-users-report/Jenkinsfile new file mode 100644 index 0000000..f23dc00 --- /dev/null +++ b/artifactory-users-report/Jenkinsfile @@ -0,0 +1,44 @@ +@Library('pipeline-library@pull/348/head') _ + +def cronExpr = env.BRANCH_IS_PRIMARY ? '@hourly' : '' +def reportName = env.BRANCH_IS_PRIMARY ? 'artifactory-ldap-users-report.json' : "artifactory-ldap-users-report-${env.BRANCH_NAME}.json" + +pipeline { + triggers { + cron(cronExpr) + } + options { + // This pipeline takes 1-2 minutes max to execute + timeout(time: 10, unit: 'MINUTES') + lock(resource: 'infra-reports-artifactory-users', inversePrecedence: true) + } + agent { + kubernetes { + yamlFile 'JenkinsAgentPodTemplate.yaml' + } + } + stages { + stage('Generate Artifactory Users Report') { + environment { + ARTIFACTORY_AUTH = credentials('artifactoryAdmin') + REPORT_NAME = "${reportName}" + } + steps { + dir('artifactory-users-report') { + sh 'bash ./user-report.sh > "${REPORT_NAME}"' + archiveArtifacts reportName + } + } + } + stage('Publish Artifactory Users Report') { + when { + expression { env.BRANCH_IS_PRIMARY } + } + steps { + dir('artifactory-users-report') { + publishReports ([reportName]) + } + } + } + } +} diff --git a/artifactory-users-report/user-report.sh b/artifactory-users-report/user-report.sh index 5905823..2476e46 100644 --- a/artifactory-users-report/user-report.sh +++ b/artifactory-users-report/user-report.sh @@ -3,8 +3,9 @@ set -o nounset set -o errexit -wget -O jq https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64 || { echo "Failed to download jq" >&2 ; exit 1; } -chmod +x jq || { echo "Failed to make jq executable" >&2 ; exit 1; } +## Ensure that jq v1.5 (v1.6 and alpine v1.5 are not working as expected) +wget -q -O jq https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64 || { echo "Failed to download jq" >&2 ; exit 1 ; } +chmod +x jq || { echo "Failed to make jq executable" >&2 ; exit 1 ; } -curl -X GET -H 'Content-Length: 0' -u $ARTIFACTORY_AUTH "https://repo.jenkins-ci.org/api/security/users" > artifactory-users-raw.json +curl -X GET -H 'Content-Length: 0' -u "${ARTIFACTORY_AUTH}" "https://repo.jenkins-ci.org/api/security/users" > artifactory-users-raw.json ./jq 'map(select(.realm | test("ldap"))) | [ .[].name ] | sort' artifactory-users-raw.json diff --git a/fork-report/Dockerfile b/fork-report/Dockerfile deleted file mode 100644 index c27fefa..0000000 --- a/fork-report/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM ruby:3.1.0-alpine - -RUN gem install graphql-client httparty - -COPY fork-report.rb / - -ENTRYPOINT ["ruby", "/fork-report.rb"] diff --git a/fork-report/Jenkinsfile b/fork-report/Jenkinsfile new file mode 100644 index 0000000..6f46eb4 --- /dev/null +++ b/fork-report/Jenkinsfile @@ -0,0 +1,45 @@ +@Library('pipeline-library@pull/348/head') _ + +def cronExpr = env.BRANCH_IS_PRIMARY ? '@daily' : '' +def reportName = env.BRANCH_IS_PRIMARY ? 'github-jenkinsci-fork-report.json' : "github-jenkinsci-fork-report-${env.BRANCH_NAME}.json" + +pipeline { + triggers { + cron(cronExpr) + } + options { + // This pipeline takes 1-2 minutes max to execute + timeout(time: 10, unit: 'MINUTES') + lock(resource: 'infra-reports-github-forks', inversePrecedence: true) + } + agent { + kubernetes { + yamlFile 'JenkinsAgentPodTemplate.yaml' + } + } + stages { + stage('Generate GitHub Forks Report') { + environment { + // Requires 'jenkins-infra-reports' to be of type GithubAppCredentials so $GITHUB_AUTH_PSW holds an IAT (Github Installation Access Token) valid for 1 hour + GITHUB_AUTH = credentials('jenkins-infra-reports') + REPORT_NAME = "${reportName}" + } + steps { + dir('fork-report') { + sh 'ruby ./fork-report.rb> "${REPORT_NAME}"' + archiveArtifacts reportName + } + } + } + stage('Publish GitHub Forks Report') { + when { + expression { env.BRANCH_IS_PRIMARY } + } + steps { + dir('fork-report') { + publishReports ([reportName]) + } + } + } + } +} diff --git a/fork-report/fork-report.rb b/fork-report/fork-report.rb index 1cb4b42..e6e9af5 100755 --- a/fork-report/fork-report.rb +++ b/fork-report/fork-report.rb @@ -6,7 +6,7 @@ require 'pp' require 'json' -$auth = "bearer #{ENV['GITHUB_API_TOKEN']}" +$auth = "bearer #{ENV['GITHUB_AUTH_PSW']}" module GitHubGraphQL HTTP = GraphQL::Client::HTTP.new('https://api.github.com/graphql') do diff --git a/jira-users-report/Dockerfile b/jira-users-report/Dockerfile deleted file mode 100644 index 7578b09..0000000 --- a/jira-users-report/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM debian:stable-slim - -RUN apt-get update -RUN apt-get install -y wget curl - -COPY user-report.sh / -RUN chmod +x /user-report.sh - -ENTRYPOINT [ "/user-report.sh" ] diff --git a/jira-users-report/Jenkinsfile b/jira-users-report/Jenkinsfile new file mode 100644 index 0000000..aef937c --- /dev/null +++ b/jira-users-report/Jenkinsfile @@ -0,0 +1,44 @@ +@Library('pipeline-library@pull/348/head') _ + +def cronExpr = env.BRANCH_IS_PRIMARY ? 'H H/2 * * *' : '' +def reportName = env.BRANCH_IS_PRIMARY ? 'jira-users-report.json' : "jira-users-report-${env.BRANCH_NAME}.json" + +pipeline { + triggers { + cron(cronExpr) + } + options { + // This pipeline takes ~1.5 hour to run + timeout(time: 2, unit: 'HOURS') + lock(resource: 'infra-reports-jira-users', inversePrecedence: true) + } + agent { + kubernetes { + yamlFile 'JenkinsAgentPodTemplate.yaml' + } + } + stages { + stage('Generate JIRA Users Report') { + environment { + JIRA_AUTH = credentials('jiraAuth') + REPORT_NAME = "${reportName}" + } + steps { + dir('jira-users-report') { + sh 'bash ./user-report.sh > "${REPORT_NAME}"' + archiveArtifacts reportName + } + } + } + stage('Publish JIRA Users Report') { + when { + expression { env.BRANCH_IS_PRIMARY } + } + steps { + dir('jira-users-report') { + publishReports ([reportName]) + } + } + } + } +} diff --git a/jira-users-report/user-report.sh b/jira-users-report/user-report.sh index ed02857..274fe1b 100644 --- a/jira-users-report/user-report.sh +++ b/jira-users-report/user-report.sh @@ -4,6 +4,7 @@ set -o nounset set -o errexit set -o pipefail +## Ensure that jq v1.5 (v1.6 and alpine v1.5 are not working as expected) wget -q -O jq https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64 || { echo "Failed to download jq" >&2 ; exit 1 ; } chmod +x jq || { echo "Failed to make jq executable" >&2 ; exit 1 ; } diff --git a/maintainers-info-report/Dockerfile b/maintainers-info-report/Dockerfile deleted file mode 100644 index 2cb2c2f..0000000 --- a/maintainers-info-report/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM debian:stable-slim - -RUN apt-get update -RUN apt-get install -y wget curl - -COPY maintainers-info-report.sh / -RUN chmod +x /maintainers-info-report.sh - -ENTRYPOINT [ "/maintainers-info-report.sh" ] diff --git a/maintainers-info-report/Jenkinsfile b/maintainers-info-report/Jenkinsfile new file mode 100644 index 0000000..cf7a183 --- /dev/null +++ b/maintainers-info-report/Jenkinsfile @@ -0,0 +1,44 @@ +@Library('pipeline-library@pull/348/head') _ + +def cronExpr = env.BRANCH_IS_PRIMARY ? '@dahourlyily' : '' +def reportName = env.BRANCH_IS_PRIMARY ? 'maintainers-info-report.json' : "maintainers-info-report-${env.BRANCH_NAME}.json" + +pipeline { + triggers { + cron(cronExpr) + } + options { + // This pipeline takes 30-40 minutes to execute + timeout(time: 1, unit: 'HOURS') + lock(resource: 'infra-reports-maintainers-info', inversePrecedence: true) + } + agent { + kubernetes { + yamlFile 'JenkinsAgentPodTemplate.yaml' + } + } + stages { + stage('Generate Maintainers Info Report') { + environment { + JIRA_AUTH = credentials('jiraAuth') + REPORT_NAME = "${reportName}" + } + steps { + dir('maintainers-info-report') { + sh 'bash ./maintainers-info-report.sh > "${REPORT_NAME}"' + archiveArtifacts reportName + } + } + } + stage('Publish Maintainers Info Report') { + when { + expression { env.BRANCH_IS_PRIMARY } + } + steps { + dir('maintainers-info-report') { + publishReports ([reportName]) + } + } + } + } +} diff --git a/maintainers-info-report/maintainers-info-report.sh b/maintainers-info-report/maintainers-info-report.sh index 47fa315..05c76e6 100755 --- a/maintainers-info-report/maintainers-info-report.sh +++ b/maintainers-info-report/maintainers-info-report.sh @@ -4,6 +4,7 @@ set -o nounset set -o errexit set -o pipefail +## Ensure that jq v1.5 (v1.6 and alpine v1.5 are not working as expected) wget -q -O jq https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64 || { echo "Failed to download jq" >&2 ; exit 1 ; } chmod +x jq || { echo "Failed to make jq executable" >&2 ; exit 1 ; } diff --git a/permissions-report/Dockerfile b/permissions-report/Dockerfile deleted file mode 100644 index e7840ac..0000000 --- a/permissions-report/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM ruby:3.1.1-alpine3.15 - -RUN apk add openssl-dev openssl build-base - -RUN gem install graphql-client httparty jwt time openssl base64 - -COPY permissions-report.rb / - -ENTRYPOINT ["ruby", "/permissions-report.rb"] diff --git a/permissions-report/Jenkinsfile b/permissions-report/Jenkinsfile new file mode 100644 index 0000000..3bb7fae --- /dev/null +++ b/permissions-report/Jenkinsfile @@ -0,0 +1,46 @@ +@Library('pipeline-library@pull/348/head') _ + +def cronExpr = env.BRANCH_IS_PRIMARY ? '@daily' : '' +def reportName = env.BRANCH_IS_PRIMARY ? 'github-jenkinsci-permissions-report.json' : "github-jenkinsci-permissions-report-${env.BRANCH_NAME}.json" + +pipeline { + triggers { + cron(cronExpr) + } + options { + // This pipeline takes 6-7 hours max to execute + timeout(time: 10, unit: 'HOURS') + lock(resource: 'infra-reports-github-permissions', inversePrecedence: true) + } + agent { + kubernetes { + yamlFile 'JenkinsAgentPodTemplate.yaml' + } + } + stages { + stage('Generate GitHub Permissions Report') { + environment { + GITHUB_APP_PRIVATE_KEY_B64 = credentials('githubapp-jenkins-infra-reports-private-key-b64') + GITHUB_APP_ID = credentials('githubapp-jenkins-infra-reports-app-identifier') + GITHUB_ORG_NAME = "jenkinsci" + REPORT_NAME = "${reportName}" + } + steps { + dir('permissions-report') { + sh 'ruby ./permissions-report.rb > "${REPORT_NAME}"' + archiveArtifacts reportName + } + } + } + stage('Publish GitHub Permissions Report') { + when { + expression { env.BRANCH_IS_PRIMARY } + } + steps { + dir('permissions-report') { + publishReports ([reportName]) + } + } + } + } +} diff --git a/permissions-report/permissions-report.rb b/permissions-report/permissions-report.rb index 6e19ff6..d887556 100644 --- a/permissions-report/permissions-report.rb +++ b/permissions-report/permissions-report.rb @@ -57,7 +57,7 @@ def get_auth_token # Your GitHub App's identifier number iss: APP_IDENTIFIER } - + # Cryptographically sign the JWT. jwt = "Bearer #{JWT.encode(payload, PRIVATE_KEY, 'RS256')}" @@ -78,7 +78,7 @@ def get_auth_token else abort "Error: no Github App installation for the organization #{GITHUB_ORG_NAME}" end - + # Retrieve the Installation Access Token of the Github App (ref: https://docs.github.com/en/rest/reference/apps#create-an-installation-access-token-for-an-app) response = HTTParty.post("https://api.github.com/app/installations/#{installationId}/access_tokens", :headers => { 'Authorization' => jwt,