diff --git a/.github/workflows/auto-tag.yml b/.github/workflows/auto-tag.yml new file mode 100644 index 0000000..33446ca --- /dev/null +++ b/.github/workflows/auto-tag.yml @@ -0,0 +1,12 @@ +name: Auto-tag +on: + push: + tags: + - '*.*.*' +jobs: + auto-tag: + name: Auto-tag + runs-on: ubuntu-latest + steps: + - name: Auto-tag + uses: silverstripe/gha-auto-tag@main diff --git a/README.md b/README.md index 97b3080..b22c874 100644 --- a/README.md +++ b/README.md @@ -2,4 +2,4 @@ Create a dynamic Silverstripe CI matrix -See [gha-ci](https://github.com/silverstripe/gha-ci) +Only intended to be used within [gha-ci](https://github.com/silverstripe/gha-ci). See that repository for details on inputs. diff --git a/action.php b/action.php new file mode 100644 index 0000000..52ac606 --- /dev/null +++ b/action.php @@ -0,0 +1,13 @@ +createJson($inputs); diff --git a/action.yml b/action.yml index 9bda70c..20622da 100644 --- a/action.yml +++ b/action.yml @@ -1,12 +1,13 @@ name: Generate Matrix description: GitHub Action to create a dynamic Silverstripe CI matrix + inputs: # extra jobs must be multi-line string, as there's no support for type: array for inputs extra_jobs: type: string required: false default: '' - # simple matrix will only run a single php 7.4 mysql 5.7 job instead of a full matrix + # simple matrix will only run a single job with the lowest supported PHP and mysql versions instead of a full matrix simple_matrix: type: boolean default: false @@ -26,21 +27,26 @@ inputs: js: type: boolean default: true + # https://docs.github.com/en/actions/creating-actions/metadata-syntax-for-github-actions outputs: matrix: description: JSON matrix value: ${{ steps.php-script.outputs.matrix }} + runs: using: composite steps: + - name: Checkout code - uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 #v2 + uses: actions/checkout@7884fcad6b5d53d10323aee724dc68d8b9096a2e # v2.4.2 + - name: Install PHP - uses: shivammathur/setup-php@aa1fe473f9c687b6fb896056d771232c0bc41161 #v2 + uses: shivammathur/setup-php@3eda58347216592f618bb1dff277810b6698e4ca # v2.19.1 with: - php-version: '7.4' + php-version: '8.1' extensions: yaml + - name: Create __inputs.yml shell: bash # Add string inputs to memory instead of using string substituion in shell script @@ -52,7 +58,12 @@ runs: # github.ref_name is the name of the branch on push, and the tag on tag GITHUB_MY_REF: ${{ github.base_ref && github.base_ref || github.ref_name }} run: | - if [ -f __inputs.yml ]; then rm __inputs.yml; fi + # Escape double quotes '"' => '\"' + EXTRA_JOBS=${EXTRA_JOBS//\"/\\\"} + GITHUB_MY_REF=${GITHUB_MY_REF//\"/\\\"} + if [ -f __inputs.yml ]; then + rm __inputs.yml + fi touch __inputs.yml echo "endtoend: ${{ inputs.endtoend }}" >> __inputs.yml echo "js: ${{ inputs.js }}" >> __inputs.yml @@ -62,14 +73,17 @@ runs: echo "simple_matrix: ${{ inputs.simple_matrix }}" >> __inputs.yml echo "github_repository: $GITHUB_REPOSITORY" >> __inputs.yml echo "github_my_ref: $GITHUB_MY_REF" >> __inputs.yml - if [[ "$EXTRA_JOBS" != "" ]]; then echo "extra_jobs:" >> __inputs.yml; fi - if [[ "$EXTRA_JOBS" != "" ]]; then echo "$EXTRA_JOBS" >> __inputs.yml; fi + if [[ "$EXTRA_JOBS" != "" ]]; then + echo "extra_jobs:" >> __inputs.yml + echo "$EXTRA_JOBS" >> __inputs.yml + fi echo "cat __inputs.yml" cat __inputs.yml + - name: Run php script id: php-script shell: bash run: | - MATRIX_JSON=$(php ${{ github.action_path }}/script.php) + MATRIX_JSON=$(php ${{ github.action_path }}/action.php) echo "MATRIX_JSON: $MATRIX_JSON" echo "::set-output name=matrix::${MATRIX_JSON}" diff --git a/consts.php b/consts.php new file mode 100644 index 0000000..4ae9a91 --- /dev/null +++ b/consts.php @@ -0,0 +1,68 @@ + [ + '7.1', + '7.2', + '7.3', + '7.4' + ], + '4.10' => [ + '7.3', + '7.4', + '8.0', + ], + '4.11' => [ + '7.4', + '8.0', + '8.1', + ], + '4' => [ + '7.4', + '8.0', + '8.1', + ], +]; + +const DB_MYSQL_57 = 'mysql57'; +const DB_MYSQL_57_PDO = 'mysql57pdo'; +const DB_MYSQL_80 = 'mysql80'; +const DB_PGSQL = 'pgsql'; + +// Used when determining the version of installer to used. Intentionally doesn't include recipes +const LOCKSTEPED_REPOS = [ + 'silverstripe-admin', + 'silverstripe-asset-admin', + 'silverstripe-assets', + 'silverstripe-campaign-admin', + 'silverstripe-cms', + 'silverstripe-errorpage', + 'silverstripe-framework', + 'silverstripe-reports', + 'silverstripe-siteconfig', + 'silverstripe-versioned', + 'silverstripe-versioned-admin', + // recipe-solr-search doesn't include recipe-cms or recipe-core unlike our other recipes + 'recipe-solr-search', +]; + +// Repositories that do not require silverstripe/installer to be explicitly added as a dependency for testing +const NO_INSTALLER_REPOS = [ + // these are/include recipe-cms or recipe-core, so we don't want to composer require installer + // in .travis.yml they used the 'self' provision rather than 'standard' + 'recipe-authoring-tools', + 'recipe-blog', + 'recipe-ccl', + 'recipe-cms', + 'recipe-collaboration', + 'recipe-content-blocks', + 'recipe-core', + 'recipe-form-building', + 'recipe-kitchen-sink', + 'recipe-reporting-tools', + 'recipe-services', + 'silverstripe-installer', + // vendor-plugin is not a recipe, though we also do not want installer + 'vendor-plugin' +]; diff --git a/job_creator.php b/job_creator.php new file mode 100644 index 0000000..fa5539d --- /dev/null +++ b/job_creator.php @@ -0,0 +1,273 @@ + (int) explode('.', $portions)[1], $installerVersions); + sort($minorPortions); + return '4.' . $minorPortions[count($minorPortions) - 1] . '.x-dev'; + } + } + + public function createJob(int $phpIndex, array $opts): array + { + $installerKey = str_replace('.x-dev', '', $this->installerVersion); + $installerKey = $installerKey ?: '4'; + $phpVersions = INSTALLER_TO_PHP_VERSIONS[$installerKey]; + $default = [ + # ensure there's a default value for all possible return keys + # this allows us to use `if [[ "${{ matrix.key }}" == "true" ]]; then` in github-actions-ci-cd/ci.yml + 'installer_version' => $this->installerVersion, + 'php' => $phpVersions[$phpIndex] ?? $phpVersions[count($phpVersions) - 1], + 'db' => DB_MYSQL_57, + 'composer_require_extra' => '', + 'composer_args' => '', + 'name_suffix' => '', + 'phpunit' => false, + 'phpunit_suite' => 'all', + 'phplinting' => false, + 'phpcoverage' => false, + 'endtoend' => false, + 'endtoend_suite' => 'root', + 'endtoend_config' => '', + 'js' => false, + ]; + return array_merge($default, $opts); + } + + private function parseBoolValue(mixed $value): bool + { + return ($value === true || $value === 'true'); + } + + private function createPhpunitJobs(array $matrix, bool $simpleMatrix, string $suite): array + { + if ($simpleMatrix) { + $matrix['include'][] = $this->createJob(0, [ + 'phpunit' => true, + 'phpunit_suite' => $suite, + ]); + } else { + $matrix['include'][] = $this->createJob(0, [ + 'composer_args' => '--prefer-lowest', + 'phpunit' => true, + 'phpunit_suite' => $suite, + ]); + $matrix['include'][] = $this->createJob(1, [ + 'db' => DB_PGSQL, + 'phpunit' => true, + 'phpunit_suite' => $suite, + ]); + $matrix['include'][] = $this->createJob(3, [ + 'db' => DB_MYSQL_80, + 'phpunit' => true, + 'phpunit_suite' => $suite, + ]); + } + return $matrix; + } + + private function buildDynamicMatrix( + array $matrix, + array $run, + bool $simpleMatrix, + string $githubRepository + ): array { + if ($run['phpunit'] && (file_exists('phpunit.xml') || file_exists('phpunit.xml.dist'))) { + $dom = new DOMDocument(); + $dom->preserveWhiteSpace = false; + $dom->load(file_exists('phpunit.xml') ? 'phpunit.xml' : 'phpunit.xml.dist'); + $xpath = new DOMXPath($dom); + // assume phpunit.xml has defined testsuites + foreach ($xpath->query('//testsuite') as $testsuite) { + if (!$testsuite->hasAttribute('name') || $testsuite->getAttribute('name') == 'Default') { + continue; + } + $matrix = $this->createPhpunitJobs($matrix, $simpleMatrix, $testsuite->getAttribute('name')); + } + // phpunit.xml has no defined testsuites, or only defaults a "Default" + if (count($matrix['include']) == 0) { + $matrix = $this->createPhpunitJobs($matrix, $simpleMatrix, 'all'); + } + } + // skip phpcs on silverstripe-installer which include sample file for use in projects + if ($run['phplinting'] && (file_exists('phpcs.xml') || file_exists('phpcs.xml.dist')) && !preg_match('#/silverstripe-installer$#', $githubRepository)) { + $matrix['include'][] = $this->createJob(0, [ + 'phplinting' => true + ]); + } + // phpcoverage also runs unit tests + // always run on silverstripe account + if ($run['phpcoverage'] || preg_match('#^silverstripe/#', $githubRepository)) { + if ($simpleMatrix) { + $matrix['include'][] = $this->createJob(0, [ + 'phpcoverage' => true + ]); + } else { + $matrix['include'][] = $this->createJob(2, [ + 'db' => DB_MYSQL_57_PDO, + 'phpcoverage' => true + ]); + } + } + // endtoend / behat + if ($run['endtoend'] && file_exists('behat.yml')) { + $matrix['include'][] = $this->createJob(0, [ + 'endtoend' => true, + 'endtoend_suite' => 'root' + ]); + if (!$simpleMatrix) { + $matrix['include'][] = $this->createJob(3, [ + 'db' => DB_MYSQL_80, + 'endtoend' => true, + 'endtoend_suite' => 'root' + ]); + } + } + // javascript tests + if ($run['js'] && file_exists('package.json')) { + $matrix['include'][] = $this->createJob(0, [ + 'js' => true + ]); + } + return $matrix; + } + + public function createJson(array $inputs): string + { + // $myRef will either be a branch for push (i.e cron) and pull-request (target branch), or a semver tag + $myRef = $inputs['github_my_ref']; + $isTag = preg_match('#^[0-9]+\.[0-9]+\.[0-9]+$#', $myRef, $m); + $branch = $isTag ? sprintf('%d.%d', $m[1], $m[2]) : $myRef; + + $githubRepository = $inputs['github_repository']; + $this->installerVersion = $this->getInstallerVersion($githubRepository, $branch); + + $run = []; + $extraJobs = []; + $dynamicMatrix = true; + $simpleMatrix = false; + foreach ($inputs as $input => $value) { + if (in_array($input, ['endtoend', 'js', 'phpunit', 'phpcoverage', 'phplinting'])) { + $run[$input] = $this->parseBoolValue($value); + } else if ($input === 'extra_jobs') { + if ($value === 'none') { + $value = []; + } + $extraJobs = $value; + } else if ($input === 'dynamic_matrix') { + $dynamicMatrix = $this->parseBoolValue($value); + } else if ($input === 'simple_matrix') { + $simpleMatrix = $this->parseBoolValue($value); + } else if (in_array($input, ['github_my_ref', 'github_repository'])) { + continue; + } else { + throw new LogicException("Unhandled input $input"); + } + } + $matrix = ['include' => []]; + + if ($dynamicMatrix) { + $matrix = $this->buildDynamicMatrix($matrix, $run, $simpleMatrix, $githubRepository); + } + + // extra jobs + foreach ($extraJobs as $arr) { + $matrix['include'][] = $this->createJob(0, $arr); + } + + // convert everything to strings and sanatise values + foreach ($matrix['include'] as $i => $job) { + foreach ($job as $key => $val) { + if ($val === true) { + $val = 'true'; + } + if ($val === false) { + $val = 'false'; + } + // all values must be strings + $val = (string) $val; + // remove any dodgy characters + $val = str_replace(["\f", "\r", "\n", "\t", "'", '"', '&', '|'], '', $val); + // only allow visible ascii chars - see [:print:] in the table at https://www.regular-expressions.info/posixbrackets.html#posixbrackets + $val = preg_replace('#[^\x20-\x7E]#', '', $val); + // limit name_suffix length and be strict as it is used in the artifact file name + if ($key === 'name_suffix' && strlen($val) > 20) { + $val = preg_replace('#[^a-zA-Z0-9_\- ]#', '', $val); + $val = substr($val, 0, 20); + } + // composer_require_extra is used in silverstripe/gha-ci `composer require`, so throw an + // exception if there are any dodgy characters + if ($key === 'composer_require_extra' && preg_match('#[^A-Za-z0-9\-\.\^\/~: ]#', $val)) { + throw new InvalidArgumentException("Invalid composer_require_extra $val"); + } + // ensure x.0 versions of PHP retain the minor version + if ($key === 'php' && preg_match('#^[1-9]$#', $val)) { + $val = "$val.0"; + } + // add value back to matrix + $matrix['include'][$i][$key] = $val; + } + } + + // job/artifacts names + foreach ($matrix['include'] as $i => $job) { + $name = [ + $job['php'] + ]; + if (strpos($job['composer_args'], '--prefer-lowest') !== false) { + $name[] = 'prf-low'; + } + $name[] = $job['db']; + if ($job['phpunit'] == 'true') { + $name[] = 'phpunit'; + $name[] = $job['phpunit_suite']; + } + if ($job['endtoend'] == 'true') { + $name[] = 'endtoend'; + $name[] = $job['endtoend_suite'] ?: 'root'; + } + if ($job['js'] == 'true') { + $name[] = 'js'; + } + if ($job['phpcoverage'] == 'true') { + $name[] = 'phpcoverage'; + } + if ($job['phplinting'] == 'true') { + $name[] = 'phplinting'; + } + $name[] = $job['name_suffix']; + $name = array_filter($name); + $matrix['include'][$i]['name'] = implode(' ', $name); + } + + // output json, keep it on a single line so do not use pretty print + $json = json_encode($matrix, JSON_UNESCAPED_SLASHES); + $json = preg_replace("#\n +#", "\n", $json); + $json = str_replace("\n", '', $json); + return trim($json); + } +} \ No newline at end of file diff --git a/script.php b/script.php deleted file mode 100644 index 0e694ce..0000000 --- a/script.php +++ /dev/null @@ -1,300 +0,0 @@ - [ - '7.1', - '7.2', - '7.3', - '7.4' - ], - '4.10' => [ - '7.3', - '7.4', - '8.0', - ], - '4.11' => [ - '7.4', - '8.0', - '8.1', - ], - '4' => [ - '7.4', - '8.0', - '8.1', - ], -]; - -function isLockedStepped($repo) -{ - return in_array($repo, [ - 'silverstripe-admin', - 'silverstripe-asset-admin', - 'silverstripe-assets', - 'silverstripe-campaign-admin', - 'silverstripe-cms', - 'silverstripe-errorpage', - 'silverstripe-framework', - 'silverstripe-reports', - 'silverstripe-siteconfig', - 'silverstripe-versioned', - 'silverstripe-versioned-admin', - // recipe-solr-search is not a true recipe, doesn't include recipe-cms/core - 'recipe-solr-search', - ]); -} - -function shouldNotRequireInstaller($repo) -{ - // these include recipe-cms/core, so we don't want to composer require installer - // in .travis.yml they use the 'self' provision rather than 'standard' - return in_array($repo, [ - 'recipe-authoring-tools', - 'recipe-blog', - 'recipe-ccl', - 'recipe-cms', - 'recipe-collaboration', - 'recipe-content-blocks', - 'recipe-core', - 'recipe-form-building', - 'recipe-kitchen-sink', - 'recipe-reporting-tools', - 'recipe-services', - 'silverstripe-installer', - // vendor-plugin is not a recipe, though we also do not want installer - 'vendor-plugin' - ]); -} - -function getInstallerVersion($githubRepository, $branch) -{ - global $installerToPhpVersions; - $repo = explode('/', $githubRepository)[1]; - if (shouldNotRequireInstaller($repo)) { - return ''; - } - $v = explode('.', $branch); - if (count($v) == 1) { - return '4.x-dev'; - } - if (isLockedStepped($repo)) { - return '4' . $v[1] . 'x-dev'; - } else { - // use the latest minor version of installer - $a = array_keys($installerToPhpVersions); - $a = array_map(fn($k) => (int) $k, $a); - sort($a); - return $a[count($a) - 1] . 'x-dev'; - } -} - -function createJob($phpNum, $opts) -{ - global $installerToPhpVersions, $installerVersion; - $v = str_replace('.x-dev', '', $installerVersion); - $v = $v ?: '4'; - $phpVersions = $installerToPhpVersions[$v]; - $default = [ - # ensure there's a default value for all possible return keys - # this allows use to use `if [ "${{ matrix.key }}" == "true" ]; then` in github-actions-ci-cd/ci.yml - 'installer_version' => $installerVersion, - 'php' => $phpVersions[$phpNum] ?? $phpVersions[count($phpVersions) - 1], - 'db' => DB_MYSQL_57, - 'composer_require_extra' => '', - 'composer_args' => '', - 'name_suffix' => '', - 'phpunit' => false, - 'phpunit_suite' => 'all', - 'phplinting' => false, - 'phpcoverage' => false, - 'endtoend' => false, - 'endtoend_suite' => 'root', - 'endtoend_config' => '', - 'js' => false, - ]; - return array_merge($default, $opts); -} - -function parseBoolValue($value) -{ - return ($value === true || $value === 'true'); -} - -// Reads inputs.yml and creates a new json matrix -$inputs = yaml_parse(file_get_contents('__inputs.yml')); - -// $myRef will either be a branch for push (i.e cron) and pull-request (target branch), or a semver tag -$myRef = $inputs['github_my_ref']; -$isTag = preg_match('#^[0-9]+\.[0-9]+\.[0-9]+$#', $myRef, $m); -$branch = $isTag ? sprintf('%d.%d', $m[1], $m[2]) : $myRef; - -$githubRepository = $inputs['github_repository']; -$installerVersion = getInstallerVersion($githubRepository, $branch); - -$run = []; -$extraJobs = []; -$simpleMatrix = false; -foreach ($inputs as $input => $value) { - if (in_array($input, ['endtoend', 'js', 'phpunit', 'phpcoverage', 'phplinting'])) { - $run[$input] = parseBoolValue($value); - } else if ($input === 'extra_jobs') { - if ($value === 'none') { - $value = []; - } - $extraJobs = $value; - foreach ($extraJobs as $job) { - $job = createJob(3, $job); - } - } else if ($input === 'simple_matrix') { - $simpleMatrix = parseBoolValue($value); - } -} -$matrix = ['include' => []]; -if ((file_exists('phpunit.xml') || file_exists('phpunit.xml.dist')) && $run['phpunit']) { - $d = new DOMDocument(); - $d->preserveWhiteSpace = false; - $fn = file_exists('phpunit.xml') ? 'phpunit.xml' : 'phpunit.xml.dist'; - $d->load($fn); - $x = new DOMXPath($d); - $tss = $x->query('//testsuite'); - // phpunit.xml has defined testsuites - foreach ($tss as $ts) { - if (!$ts->hasAttribute('name') || $ts->getAttribute('name') == 'Default') { - continue; - } - if ($simpleMatrix) { - $matrix['include'][] = createJob(0, [ - 'phpunit' => true, - 'phpunit_suite' => $ts->getAttribute('name'), - ]); - } else { - $matrix['include'][] = createJob(0, [ - 'composer_args' => '--prefer-lowest', - 'phpunit' => true, - 'phpunit_suite' => $ts->getAttribute('name'), - ]); - $matrix['include'][] = createJob(1, [ - 'db' => 'pgsql', - 'phpunit' => true, - 'phpunit_suite' => $ts->getAttribute('name') - ]); - $matrix['include'][] = createJob(3, [ - 'db' => DB_MYSQL_80, - 'phpunit' => true, - 'phpunit_suite' => $ts->getAttribute('name') - ]); - } - } - // phpunit.xml has no defined testsuites - if (count($matrix['include']) == 0) { - if ($simpleMatrix) { - $matrix['include'][] = createJob(0, [ - 'phpunit' => true, - 'phpunit_suite' => 'all' - ]); - } else { - $matrix['include'][] = createJob(0, [ - 'composer_args' => '--prefer-lowest', - 'phpunit' => true, - 'phpunit_suite' => 'all' - ]); - $matrix['include'][] = createJob(1, [ - 'db' => DB_PGSQL, - 'phpunit' => true, - 'phpunit_suite' => 'all' - ]); - $matrix['include'][] = createJob(3, [ - 'db' => DB_MYSQL_80, - 'phpunit' => true, - 'phpunit_suite' => 'all' - ]); - } - } -} -// skip phpcs on silverstripe-installer which include sample file for use in projects -if ((file_exists('phpcs.xml') || file_exists('phpcs.xml.dist')) && !preg_match('#/silverstripe-installer$#', $githubRepository)) { - $matrix['include'][] = createJob(0, [ - 'phplinting' => true - ]); -} -// phpcoverage also runs unit tests -// always run on silverstripe account -if ($run['phpcoverage'] || preg_match('#^silverstripe/#', $githubRepository)) { - if ($simpleMatrix) { - $matrix['include'][] = createJob(0, [ - 'phpcoverage' => true - ]); - } else { - $matrix['include'][] = createJob(2, [ - 'db' => DB_MYSQL_57_PDO, - 'phpcoverage' => true - ]); - } -} -// endtoend / behat -if ($run['endtoend'] && file_exists('behat.yml')) { - $matrix['include'][] = createJob(0, [ - 'endtoend' => true, - 'endtoend_suite' => 'root' - ]); - if (!$simpleMatrix) { - $matrix['include'][] = createJob(3, [ - 'db' => DB_MYSQL_80, - 'endtoend' => true, - 'endtoend_suite' => 'root' - ]); - } -} -// javascript tests -if (file_exists('package.json') && $run['js']) { - $matrix['include'][] = createJob(0, [ - 'js' => true - ]); -} -// extra jobs -foreach ($extraJobs as $arr) { - $matrix['include'][] = createJob(0, $arr); -} - -// convert everything to strings and sanatise values -foreach ($matrix['include'] as $i => $job) { - foreach ($job as $key => $val) { - if ($val === true) { - $val = 'true'; - } - if ($val === false) { - $val = 'false'; - } - // all values must be strings - $val = (string) $val; - // remove any dodgy characters - $val = str_replace(["\r", "\n", "\t", "'", '"', '&', '|'], '', $val); - // ascii chars only - https://www.regular-expressions.info/posixbrackets.html - $val = preg_replace('#[^\x20-\x7E]#', '', $val); - // limit name_suffix length and be strict as it is used in the artifact file name - if ($key === 'name_suffix') { - if (strlen($val) > 20) { - $val = preg_replace('#[^a-zA-Z0-9_\- ]#', '', $val); - $val = substr($val, 0, 20); - } - } - // be strict with composer_require_extra - if ($key === 'composer_require_extra') { - $val = preg_replace('#[^A-Za-z0-9\-\.\^\/~: ]#', '', $val); - } - // add value back to matrix - $matrix['include'][$i][$key] = $val; - } -} - -// output json, keep it on a single line so do not use pretty print -$json = json_encode($matrix); -$json = preg_replace("#\n +#", "\n", $json); -$json = str_replace('\\/', '/', $json); -$json = str_replace("\n", '', $json); -echo trim($json);