From 59d22e947a0002def35a31389332a7a532aad959 Mon Sep 17 00:00:00 2001 From: Levi Whalen Date: Tue, 8 Oct 2024 15:13:20 -0600 Subject: [PATCH 1/7] added pushToAlgoliaWeb ci script --- .github/workflows/push-to-algolia-web.yml | 24 +++++ scripts/ci/codegen/pushToAlgoliaWeb.ts | 120 ++++++++++++++++++++++ scripts/package.json | 1 + 3 files changed, 145 insertions(+) create mode 100644 .github/workflows/push-to-algolia-web.yml create mode 100644 scripts/ci/codegen/pushToAlgoliaWeb.ts diff --git a/.github/workflows/push-to-algolia-web.yml b/.github/workflows/push-to-algolia-web.yml new file mode 100644 index 0000000000..05cd7355a8 --- /dev/null +++ b/.github/workflows/push-to-algolia-web.yml @@ -0,0 +1,24 @@ +name: Push snippets to AlgoliaWeb + +on: workflow_dispatch + +jobs: + release: + name: Scheduled Release + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: main + + - name: Setup + id: setup + uses: ./.github/actions/setup + with: + type: minimal + + - run: yarn workspace scripts pushToAlgoliaWeb + env: + GITHUB_TOKEN: ${{ secrets.ALGOLIA_BOT_TOKEN }} + FORCE: true diff --git a/scripts/ci/codegen/pushToAlgoliaWeb.ts b/scripts/ci/codegen/pushToAlgoliaWeb.ts new file mode 100644 index 0000000000..23fb5f3118 --- /dev/null +++ b/scripts/ci/codegen/pushToAlgoliaWeb.ts @@ -0,0 +1,120 @@ +import fsp from 'fs/promises'; + +import { resolve } from 'path'; + +import { + configureGitHubAuthor, + ensureGitHubToken, + getOctokit, + gitBranchExists, + gitCommit, + OWNER, + run, + setVerbose, + toAbsolutePath, +} from '../../common.js'; +import { getNbGitDiff } from '../utils.js'; + +import { commitStartRelease } from './text.js'; + +const languageFiles = { + csharp: 'guides/csharp/src/saveObjectsMovies.cs', + go: 'guides/go/src/saveObjectsMovies.go', + java: 'guides/java/src/test/java/com/algolia/saveObjectsMovies.java', + javascript: 'guides/javascript/src/saveObjectsMovies.ts', + kotlin: 'guides/kotlin/src/main/kotlin/com/algolia/snippets/saveObjectsMovies.kt', + php: 'guides/php/src/saveObjectsMovies.php', + python: 'guides/python/saveObjectsMovies.py', + ruby: 'guides/ruby/saveObjectsMovies.rb', + scala: 'guides/scala/src/main/scala/saveObjectsMovies.scala', + swift: 'guides/swift/Sources/saveObjectsMovies.swift', +}; +const generateJSON = async (outputFile: string): Promise => { + const filesPromises = Object.entries(languageFiles).map(async (p) => { + const snippet = await fsp.readFile(toAbsolutePath(p[1]), 'utf-8'); + + return [ + [p[0]], + snippet + .replace('ALGOLIA_APPLICATION_ID', 'YourApplicationID') + .replace('ALGOLIA_API_KEY', 'YourWriteAPIKey') + .replace('', 'movies_index'), + ]; + }); + + const files = await Promise.all(filesPromises); + + await fsp.writeFile(outputFile, JSON.stringify(Object.fromEntries(files), null, 2)); +}; + +async function pushToAlgoliaWeb(): Promise { + const githubToken = ensureGitHubToken(); + + const repository = 'AlgoliaWeb'; + const lastCommitMessage = await run('git log -1 --format="%s"'); + const author = (await run('git log -1 --format="Co-authored-by: %an <%ae>"')).trim(); + const coAuthors = (await run('git log -1 --format="%(trailers:key=Co-authored-by)"')) + .split('\n') + .map((coAuthor) => coAuthor.trim()) + .filter(Boolean); + + if (!process.env.FORCE && !lastCommitMessage.startsWith(commitStartRelease)) { + return; + } + + console.log(`Pushing to ${OWNER}/${repository}`); + + const targetBranch = 'feat/automated-update-from-api-clients-automation-repository'; + const githubURL = `https://${githubToken}:${githubToken}@github.com/${OWNER}/${repository}`; + const tempGitDir = resolve(process.env.RUNNER_TEMP! || toAbsolutePath('foo/local/test'), repository); + await fsp.rm(tempGitDir, { force: true, recursive: true }); + await run(`git clone --depth 1 ${githubURL} ${tempGitDir}`); + if (await gitBranchExists(targetBranch, tempGitDir)) { + await run(`git fetch origin ${targetBranch}`, { cwd: tempGitDir }); + await run(`git push -d origin ${targetBranch}`, { cwd: tempGitDir }); + } + await run(`git checkout -B ${targetBranch}`, { cwd: tempGitDir }); + + const pathToSnippets = toAbsolutePath(`${tempGitDir}/_client/src/routes/launchpad/onboarding-snippets.json`); + + await generateJSON(pathToSnippets); + + if ((await getNbGitDiff({ head: null, cwd: tempGitDir })) === 0) { + console.log('❎ Skipping push to AlgoliaWeb because there is no change.'); + + return; + } + + await configureGitHubAuthor(tempGitDir); + + const message = 'feat: update specs and supported versions'; + await run('git add .', { cwd: tempGitDir }); + await gitCommit({ + message, + coAuthors: [author, ...coAuthors], + cwd: tempGitDir, + }); + await run(`git push -f -u origin ${targetBranch}`, { cwd: tempGitDir }); + + console.log(`Creating pull request on ${OWNER}/${repository}...`); + const octokit = getOctokit(); + const { data } = await octokit.pulls.create({ + owner: OWNER, + repo: repository, + title: message, + body: [ + 'This PR is automatically created by https://github.com/algolia/api-clients-automation', + 'It contains the latest generated code snippets.', + ].join('\n\n'), + base: 'master', + head: targetBranch, + }); + + console.log(`Pull request created on ${OWNER}/${repository}`); + console.log(` > ${data.url}`); +} + +if (import.meta.url.endsWith(process.argv[1])) { + setVerbose(false); + pushToAlgoliaWeb(); +} diff --git a/scripts/package.json b/scripts/package.json index 6e1f61c014..e55c6fae84 100644 --- a/scripts/package.json +++ b/scripts/package.json @@ -13,6 +13,7 @@ "pre-commit": "node ./ci/husky/pre-commit.mjs", "pushGeneratedCode": "yarn runScript dist/ci/codegen/pushGeneratedCode.js", "pushToAlgoliaDoc": "yarn runScript dist/ci/codegen/pushToAlgoliaDoc.js", + "pushToAlgoliaWeb": "yarn runScript dist/ci/codegen/pushToAlgoliaWeb.js", "runScript": "NODE_NO_WARNINGS=1 node --enable-source-maps", "setRunVariables": "yarn runScript dist/ci/githubActions/setRunVariables.js", "spreadGeneration": "yarn runScript dist/ci/codegen/spreadGeneration.js", From 6ed658b960db9c56cb028f1171921b1f4e8396b0 Mon Sep 17 00:00:00 2001 From: Levi Whalen Date: Wed, 9 Oct 2024 07:58:38 -0600 Subject: [PATCH 2/7] Update .github/workflows/push-to-algolia-web.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Clément Vannicatte --- .github/workflows/push-to-algolia-web.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/push-to-algolia-web.yml b/.github/workflows/push-to-algolia-web.yml index 05cd7355a8..60d9f4eda8 100644 --- a/.github/workflows/push-to-algolia-web.yml +++ b/.github/workflows/push-to-algolia-web.yml @@ -3,8 +3,8 @@ name: Push snippets to AlgoliaWeb on: workflow_dispatch jobs: - release: - name: Scheduled Release + push: + name: Manual trigger push for onboarding guides runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 From d2e23575af9ba67eea252a337613d82e6551cf06 Mon Sep 17 00:00:00 2001 From: Levi Whalen Date: Wed, 9 Oct 2024 07:58:57 -0600 Subject: [PATCH 3/7] Update scripts/ci/codegen/pushToAlgoliaWeb.ts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Clément Vannicatte --- scripts/ci/codegen/pushToAlgoliaWeb.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ci/codegen/pushToAlgoliaWeb.ts b/scripts/ci/codegen/pushToAlgoliaWeb.ts index 23fb5f3118..6655623a55 100644 --- a/scripts/ci/codegen/pushToAlgoliaWeb.ts +++ b/scripts/ci/codegen/pushToAlgoliaWeb.ts @@ -104,7 +104,7 @@ async function pushToAlgoliaWeb(): Promise { title: message, body: [ 'This PR is automatically created by https://github.com/algolia/api-clients-automation', - 'It contains the latest generated code snippets.', + 'It contains the latest generated guides.', ].join('\n\n'), base: 'master', head: targetBranch, From 3da4814a27c1d7671f7ff2ce839a3711f63ffae6 Mon Sep 17 00:00:00 2001 From: Levi Whalen Date: Wed, 9 Oct 2024 08:27:29 -0600 Subject: [PATCH 4/7] pr review changes --- scripts/ci/codegen/pushToAlgoliaWeb.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/ci/codegen/pushToAlgoliaWeb.ts b/scripts/ci/codegen/pushToAlgoliaWeb.ts index 6655623a55..cafe776808 100644 --- a/scripts/ci/codegen/pushToAlgoliaWeb.ts +++ b/scripts/ci/codegen/pushToAlgoliaWeb.ts @@ -87,7 +87,7 @@ async function pushToAlgoliaWeb(): Promise { await configureGitHubAuthor(tempGitDir); - const message = 'feat: update specs and supported versions'; + const message = 'feat: update generated guides'; await run('git add .', { cwd: tempGitDir }); await gitCommit({ message, @@ -106,7 +106,7 @@ async function pushToAlgoliaWeb(): Promise { 'This PR is automatically created by https://github.com/algolia/api-clients-automation', 'It contains the latest generated guides.', ].join('\n\n'), - base: 'master', + base: 'develop', head: targetBranch, }); From 6c0d9c937e56aaa911a42c8fff2fb873cfcb8de3 Mon Sep 17 00:00:00 2001 From: shortcuts Date: Wed, 9 Oct 2024 17:02:56 +0200 Subject: [PATCH 5/7] chore: programmatic file discovery --- scripts/ci/codegen/pushToAlgoliaWeb.ts | 73 ++++++++++++++------------ 1 file changed, 40 insertions(+), 33 deletions(-) diff --git a/scripts/ci/codegen/pushToAlgoliaWeb.ts b/scripts/ci/codegen/pushToAlgoliaWeb.ts index cafe776808..6e0fc3553c 100644 --- a/scripts/ci/codegen/pushToAlgoliaWeb.ts +++ b/scripts/ci/codegen/pushToAlgoliaWeb.ts @@ -1,13 +1,14 @@ import fsp from 'fs/promises'; - import { resolve } from 'path'; import { configureGitHubAuthor, ensureGitHubToken, + exists, getOctokit, gitBranchExists, gitCommit, + LANGUAGES, OWNER, run, setVerbose, @@ -15,37 +16,41 @@ import { } from '../../common.js'; import { getNbGitDiff } from '../utils.js'; +import {} from 'fs'; +import { getClientsConfigField } from '../../config.js'; import { commitStartRelease } from './text.js'; -const languageFiles = { - csharp: 'guides/csharp/src/saveObjectsMovies.cs', - go: 'guides/go/src/saveObjectsMovies.go', - java: 'guides/java/src/test/java/com/algolia/saveObjectsMovies.java', - javascript: 'guides/javascript/src/saveObjectsMovies.ts', - kotlin: 'guides/kotlin/src/main/kotlin/com/algolia/snippets/saveObjectsMovies.kt', - php: 'guides/php/src/saveObjectsMovies.php', - python: 'guides/python/saveObjectsMovies.py', - ruby: 'guides/ruby/saveObjectsMovies.rb', - scala: 'guides/scala/src/main/scala/saveObjectsMovies.scala', - swift: 'guides/swift/Sources/saveObjectsMovies.swift', -}; -const generateJSON = async (outputFile: string): Promise => { - const filesPromises = Object.entries(languageFiles).map(async (p) => { - const snippet = await fsp.readFile(toAbsolutePath(p[1]), 'utf-8'); - - return [ - [p[0]], - snippet +async function generateJSON(outputFile: string): Promise { + const guides = {}; + for (const language of LANGUAGES) { + if (!(await exists(toAbsolutePath(`docs/guides/${language}`)))) { + continue; + } + + const pathToGuides = toAbsolutePath( + `docs/guides/${language}/${getClientsConfigField(language, ['snippets', 'outputFolder'])}`, + ); + const files = await fsp.readdir(pathToGuides); + for (const file of files) { + const extension = getClientsConfigField(language, ['snippets', 'extension']); + if (!file.endsWith(extension)) { + continue; + } + + const guideName = file.replaceAll(extension, ''); + if (!guides[guideName]) { + guides[guideName] = {}; + } + + guides[guideName][language] = (await fsp.readFile(`${pathToGuides}/${file}`, 'utf-8')) .replace('ALGOLIA_APPLICATION_ID', 'YourApplicationID') .replace('ALGOLIA_API_KEY', 'YourWriteAPIKey') - .replace('', 'movies_index'), - ]; - }); - - const files = await Promise.all(filesPromises); + .replace('', 'movies_index'); + } + } - await fsp.writeFile(outputFile, JSON.stringify(Object.fromEntries(files), null, 2)); -}; + await fsp.writeFile(outputFile, JSON.stringify(guides, null, 2)); +} async function pushToAlgoliaWeb(): Promise { const githubToken = ensureGitHubToken(); @@ -62,23 +67,25 @@ async function pushToAlgoliaWeb(): Promise { return; } - console.log(`Pushing to ${OWNER}/${repository}`); - const targetBranch = 'feat/automated-update-from-api-clients-automation-repository'; const githubURL = `https://${githubToken}:${githubToken}@github.com/${OWNER}/${repository}`; const tempGitDir = resolve(process.env.RUNNER_TEMP! || toAbsolutePath('foo/local/test'), repository); await fsp.rm(tempGitDir, { force: true, recursive: true }); await run(`git clone --depth 1 ${githubURL} ${tempGitDir}`); + + const outputFile = toAbsolutePath(`${tempGitDir}/_client/src/routes/launchpad/onboarding-snippets.json`); + + console.log(`Generating JSON output file from guides at path ${outputFile}`); + + await generateJSON(outputFile); + + console.log(`Pushing to ${OWNER}/${repository}`); if (await gitBranchExists(targetBranch, tempGitDir)) { await run(`git fetch origin ${targetBranch}`, { cwd: tempGitDir }); await run(`git push -d origin ${targetBranch}`, { cwd: tempGitDir }); } await run(`git checkout -B ${targetBranch}`, { cwd: tempGitDir }); - const pathToSnippets = toAbsolutePath(`${tempGitDir}/_client/src/routes/launchpad/onboarding-snippets.json`); - - await generateJSON(pathToSnippets); - if ((await getNbGitDiff({ head: null, cwd: tempGitDir })) === 0) { console.log('❎ Skipping push to AlgoliaWeb because there is no change.'); From cd815ad979045700abb8e268e3e9b4b451f57377 Mon Sep 17 00:00:00 2001 From: shortcuts Date: Wed, 9 Oct 2024 17:03:45 +0200 Subject: [PATCH 6/7] chore: lint --- scripts/ci/codegen/pushToAlgoliaWeb.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/ci/codegen/pushToAlgoliaWeb.ts b/scripts/ci/codegen/pushToAlgoliaWeb.ts index 6e0fc3553c..ae7e6097ed 100644 --- a/scripts/ci/codegen/pushToAlgoliaWeb.ts +++ b/scripts/ci/codegen/pushToAlgoliaWeb.ts @@ -16,7 +16,6 @@ import { } from '../../common.js'; import { getNbGitDiff } from '../utils.js'; -import {} from 'fs'; import { getClientsConfigField } from '../../config.js'; import { commitStartRelease } from './text.js'; From 00c3633add457b23e2d77e90609157e7cdf5afa9 Mon Sep 17 00:00:00 2001 From: shortcuts Date: Wed, 9 Oct 2024 17:24:06 +0200 Subject: [PATCH 7/7] chore: push on release --- .github/workflows/check.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 4f8d311c2d..238f06cdc0 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -689,7 +689,12 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.ALGOLIA_BOT_TOKEN }} - - name: Push generation to the Algolia docs + - name: Push specs and snippets to algolia/doc run: yarn workspace scripts pushToAlgoliaDoc env: GITHUB_TOKEN: ${{ secrets.ALGOLIA_BOT_TOKEN }} + + - name: Push guides to algolia/AlgoliaWeb + run: yarn workspace scripts pushToAlgoliaWeb + env: + GITHUB_TOKEN: ${{ secrets.ALGOLIA_BOT_TOKEN }}