From 603502b9ee24683ae049a202ce2b6d1fe1099a1d Mon Sep 17 00:00:00 2001 From: Samuel Valdes Gutierrez Date: Tue, 16 Apr 2024 15:58:37 -0400 Subject: [PATCH] [OSCI][FEAT] Changelog Project - PoC Changelog and release notes automation tool - OpenSearch Dashboards (#5519) Refactor and Enhance Workflow Management - Added and updated changesets for multiple PRs to improve tracking and documentation of changes. - Removed unnecessary test and dummy files (`test.txt`, various `.yml` fragments) to clean up the repository. - Refactored workflow scripts to streamline changelog generation and fragment handling, moving temporary files to a designated folder. - Updated GitHub Actions workflows by changing event triggers from `pull_request` to `pull_request_target` and vice versa to optimize workflow execution. - Enhanced security and automation by updating token names and adding write permissions to the changeset workflow. - Deleted obsolete workflow file for creating changeset files, now handled by an automated process. - Major clean-up of dummy fragment files and unnecessary changelog entries to maintain clarity and relevancy in documentation. - Implemented minor updates and improvements in codebase, specifically in generating release notes and handling fragments. --------- Signed-off-by: Johnathon Bowers Signed-off-by: CMDWillYang Signed-off-by: Qiwen Li Signed-off-by: qiwen li Signed-off-by: Samuel Valdes Gutierrez Signed-off-by: Ashwin P Chandran Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Johnathon Bowers Co-authored-by: CMDWillYang Co-authored-by: Qiwen Li Co-authored-by: Ashwin P Chandran Co-authored-by: Anan Zhuang Co-authored-by: Josh Romero Co-authored-by: autochangeset[bot] <154024398+autochangeset[bot]@users.noreply.github.com> Co-authored-by: opensearch-bot[bot] <154024398+opensearch-bot[bot]@users.noreply.github.com> Co-authored-by: opensearch-bot-dev[bot] <154634848+opensearch-bot-dev[bot]@users.noreply.github.com> Co-authored-by: Ashwin P Chandran Co-authored-by: Miki Co-authored-by: Kawika Avilla --- .github/pull_request_template.md | 12 ++ .../opensearch_changelog_workflow.yml | 23 +++ changelogs/README.md | 5 + package.json | 3 +- scripts/generate_release_note.js | 8 ++ src/dev/generate_release_note.ts | 134 ++++++++++++++++++ src/dev/generate_release_note_helper.ts | 59 ++++++++ 7 files changed, 243 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/opensearch_changelog_workflow.yml create mode 100644 changelogs/README.md create mode 100644 scripts/generate_release_note.js create mode 100644 src/dev/generate_release_note.ts create mode 100644 src/dev/generate_release_note_helper.ts diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 2616faf425a1..452fa479f98e 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -7,6 +7,18 @@ +## Changelog + + ### Check List - [ ] All tests pass diff --git a/.github/workflows/opensearch_changelog_workflow.yml b/.github/workflows/opensearch_changelog_workflow.yml new file mode 100644 index 000000000000..8af8b0d70b0d --- /dev/null +++ b/.github/workflows/opensearch_changelog_workflow.yml @@ -0,0 +1,23 @@ +name: OpenSearch Changelog Workflow + +on: + pull_request_target: + types: [opened, reopened, edited] + +permissions: + contents: read + issues: write + pull-requests: write + +jobs: + update-changelog: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v4 + - name: Parse changelog entries and submit request for changset creation + uses: BigSamu/OpenSearch_Changelog_Workflow@1.0.0-alpha1 + with: + token: ${{secrets.GITHUB_TOKEN}} + CHANGELOG_PR_BRIDGE_URL_DOMAIN: ${{secrets.CHANGELOG_PR_BRIDGE_URL_DOMAIN}} + CHANGELOG_PR_BRIDGE_API_KEY: ${{secrets.CHANGELOG_PR_BRIDGE_API_KEY}} diff --git a/changelogs/README.md b/changelogs/README.md new file mode 100644 index 000000000000..a4620754cfd1 --- /dev/null +++ b/changelogs/README.md @@ -0,0 +1,5 @@ +# Changelog and Release Notes + +For information regarding the changelog and release notes process, please consult the README in the GitHub Actions repository that this process utilizes. To view this README, follow the link below: + +[GitHub Actions Workflow README](https://github.com/BigSamu/OpenSearch_Change_Set_Create_Action/blob/main/README.md) diff --git a/package.json b/package.json index 30bca1ea69f9..59b2f89d0198 100644 --- a/package.json +++ b/package.json @@ -75,7 +75,8 @@ "docs:acceptApiChanges": "scripts/use_node --max-old-space-size=6144 scripts/check_published_api_changes.js --accept", "osd:bootstrap": "scripts/use_node scripts/build_ts_refs && scripts/use_node scripts/register_git_hook", "spec_to_console": "scripts/use_node scripts/spec_to_console", - "pkg-version": "scripts/use_node -e \"console.log(require('./package.json').version)\"" + "pkg-version": "scripts/use_node -e \"console.log(require('./package.json').version)\"", + "release_note:generate": "scripts/use_node scripts/generate_release_note" }, "repository": { "type": "git", diff --git a/scripts/generate_release_note.js b/scripts/generate_release_note.js new file mode 100644 index 000000000000..4721fe0dec35 --- /dev/null +++ b/scripts/generate_release_note.js @@ -0,0 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +require('../src/setup_node_env'); +require('../src/dev/generate_release_note'); +require('../src/dev/generate_release_note_helper'); diff --git a/src/dev/generate_release_note.ts b/src/dev/generate_release_note.ts new file mode 100644 index 000000000000..4c9eaabf0bf7 --- /dev/null +++ b/src/dev/generate_release_note.ts @@ -0,0 +1,134 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { join, resolve } from 'path'; +import { readFileSync, writeFileSync, Dirent, rm, rename, promises as fsPromises } from 'fs'; +import { load as loadYaml } from 'js-yaml'; +import { readdir } from 'fs/promises'; +import { version as pkgVersion } from '../../package.json'; +import { + validateFragment, + getCurrentDateFormatted, + Changelog, + SECTION_MAPPING, + fragmentDirPath, + SectionKey, + releaseNotesDirPath, + filePath, +} from './generate_release_note_helper'; + +// Function to add content after the 'Unreleased' section in the changelog +function addContentAfterUnreleased(path: string, newContent: string): void { + let fileContent = readFileSync(path, 'utf8'); + const targetString = '## [Unreleased]'; + const targetIndex = fileContent.indexOf(targetString); + + if (targetIndex !== -1) { + const endOfLineIndex = fileContent.indexOf('\n', targetIndex); + if (endOfLineIndex !== -1) { + fileContent = + fileContent.slice(0, endOfLineIndex + 1) + + '\n' + + newContent + + '\n' + + fileContent.slice(endOfLineIndex + 1); + } else { + throw new Error('End of line for "Unreleased" section not found.'); + } + } else { + throw new Error("'## [Unreleased]' not found in the file."); + } + + writeFileSync(path, fileContent); +} + +async function deleteFragments(fragmentTempDirPath: string) { + rm(fragmentTempDirPath, { recursive: true }, (err: any) => { + if (err) { + throw err; + } + }); +} + +// Read fragment files and populate sections +async function readFragments() { + // Initialize sections + const sections: Changelog = (Object.fromEntries( + Object.keys(SECTION_MAPPING).map((key) => [key, []]) + ) as unknown) as Changelog; + + const fragmentPaths = await readdir(fragmentDirPath, { withFileTypes: true }); + for (const fragmentFilename of fragmentPaths) { + // skip non yml or yaml files + if (!/\.ya?ml$/i.test(fragmentFilename.name)) { + // eslint-disable-next-line no-console + console.warn(`Skipping non yml or yaml file ${fragmentFilename.name}`); + continue; + } + + const fragmentPath = join(fragmentDirPath, fragmentFilename.name); + const fragmentContents = readFileSync(fragmentPath, { encoding: 'utf-8' }); + + validateFragment(fragmentContents); + + const fragmentYaml = loadYaml(fragmentContents) as Changelog; + + for (const [sectionKey, entries] of Object.entries(fragmentYaml)) { + sections[sectionKey as SectionKey].push(...entries); + } + } + return { sections, fragmentPaths }; +} + +async function moveFragments(fragmentPaths: Dirent[], fragmentTempDirPath: string): Promise { + // Move fragment files to temp fragments folder + for (const fragmentFilename of fragmentPaths) { + const fragmentPath = resolve(fragmentDirPath, fragmentFilename.name); + const fragmentTempPath = resolve(fragmentTempDirPath, fragmentFilename.name); + rename(fragmentPath, fragmentTempPath, () => {}); + } +} + +function generateChangelog(sections: Changelog) { + // Generate changelog sections + const changelogSections = Object.entries(sections).map(([sectionKey, entries]) => { + const sectionName = SECTION_MAPPING[sectionKey as SectionKey]; + return entries.length === 0 + ? `### ${sectionName}` + : `### ${sectionName}\n\n${entries.map((entry) => ` - ${entry}`).join('\n')}`; + }); + + // Generate full changelog + const currentDate = getCurrentDateFormatted(); + const changelog = `## [${pkgVersion}-${currentDate}](https://github.com/opensearch-project/OpenSearch-Dashboards/releases/tag/${pkgVersion})\n\n${changelogSections.join( + '\n\n' + )}`; + // Update changelog file + addContentAfterUnreleased(filePath, changelog); + return changelogSections; +} + +function generateReleaseNote(changelogSections: string[]) { + // Generate release note + const releaseNoteFilename = `opensearch-dashboards.release-notes-${pkgVersion}.md`; + const releaseNoteHeader = `# VERSION ${pkgVersion} Release Note`; + const releaseNote = `${releaseNoteHeader}\n\n${changelogSections.join('\n\n')}`; + writeFileSync(resolve(releaseNotesDirPath, releaseNoteFilename), releaseNote); +} + +(async () => { + const { sections, fragmentPaths } = await readFragments(); + // create folder for temp fragments + const fragmentTempDirPath = await fsPromises.mkdtemp(join(fragmentDirPath, 'tmp_fragments-')); + // move fragments to temp fragments folder + await moveFragments(fragmentPaths, fragmentTempDirPath); + + const changelogSections = generateChangelog(sections); + + generateReleaseNote(changelogSections); + + // remove temp fragments folder + await deleteFragments(fragmentTempDirPath); +})(); diff --git a/src/dev/generate_release_note_helper.ts b/src/dev/generate_release_note_helper.ts new file mode 100644 index 000000000000..988c0f92d964 --- /dev/null +++ b/src/dev/generate_release_note_helper.ts @@ -0,0 +1,59 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { resolve } from 'path'; + +export const filePath = resolve(__dirname, '..', '..', 'CHANGELOG.md'); +export const fragmentDirPath = resolve(__dirname, '..', '..', 'changelogs', 'fragments'); +export const releaseNotesDirPath = resolve(__dirname, '..', '..', 'release-notes'); + +export function getCurrentDateFormatted(): string { + return new Date().toISOString().slice(0, 10); +} + +export const SECTION_MAPPING = { + breaking: '💥 Breaking Changes', + deprecate: 'Deprecations', + security: '🛡 Security', + feat: '📈 Features/Enhancements', + fix: '🐛 Bug Fixes', + infra: '🚞 Infrastructure', + doc: '📝 Documentation', + chore: '🛠 Maintenance', + refactor: '🪛 Refactoring', + test: '🔩 Tests', +}; + +export type SectionKey = keyof typeof SECTION_MAPPING; +export type Changelog = Record; + +const MAX_ENTRY_LENGTH = 100; +// Each entry must start with '-' and a space, followed by a non-empty string, and be no longer that MAX_ENTRY_LENGTH characters +const entryRegex = new RegExp(`^-.{1,${MAX_ENTRY_LENGTH}}\\(\\[#.+]\\(.+\\)\\)$`); + +// validate format of fragment files +export function validateFragment(content: string) { + const sections = content.split(/(?:\r?\n){2,}/); + + // validate each section + for (const section of sections) { + const lines = section.split('\n'); + const sectionName = lines[0]; + const sectionKey = sectionName.slice(0, -1); + + if (!SECTION_MAPPING[sectionKey as SectionKey] || !sectionName.endsWith(':')) { + throw new Error(`Unknown section ${sectionKey}.`); + } + for (const entry of lines.slice(1)) { + if (entry === '') { + continue; + } + // if (!entryRegex.test(entry)) { + if (!entryRegex.test(entry.trim())) { + throw new Error(`Invalid entry ${entry} in section ${sectionKey}.`); + } + } + } +}