diff --git a/.changeset/sour-mirrors-care.md b/.changeset/sour-mirrors-care.md new file mode 100644 index 0000000..8f41b29 --- /dev/null +++ b/.changeset/sour-mirrors-care.md @@ -0,0 +1,5 @@ +--- +"scan-chart": patch +--- + +Fix capitalization and improve removeStyleTags diff --git a/readme.md b/readme.md index 715687f..89439e6 100644 --- a/readme.md +++ b/readme.md @@ -9,7 +9,7 @@ This has been validated on 40,000 charts, including some that were deliberately /** * Scans `files` as a chart folder, and returns a `ScannedChart` object. */ -function scanChartFolder(files: { filename: string; data: Uint8Array }[]): ScannedChart +function scanChartFolder(files: { fileName: string; data: Uint8Array }[]): ScannedChart function parseChartFile(data: Uint8Array, format: 'chart' | 'mid', iniChartModifiers: IniChartModifiers): ParsedChart function calculateTrackHash(parsedChart: ParsedChart, instrument: Instrument, difficulty: Difficulty): { hash: string, bchart: Uint8Array } @@ -310,38 +310,30 @@ type NoteType = | 'greenTomOrCymbalMarker' type FolderIssueType = - | 'noMetadata' // This chart doesn't have "song.ini" - | 'invalidIni' // .ini file is not named "song.ini" - | 'invalidMetadata' // "song.ini" doesn't have a "[Song]" section - | 'badIniLine' // This line in "song.ini" couldn't be parsed - | 'multipleIniFiles' // This chart has multiple .ini files - | 'noAlbumArt' // This chart doesn't have album art - | 'albumArtSize' // This chart's album art is not 500x500 or 512x512 - | 'badAlbumArt' // This chart's album art couldn't be parsed - | 'multipleAlbumArt' // This chart has multiple album art files - | 'noAudio' // This chart doesn't have an audio file - | 'invalidAudio' // Audio file is not a valid audio stem name - | 'badAudio' // This chart's audio couldn't be parsed - | 'multipleAudio' // This chart has multiple audio files of the same stem - | 'noChart' // This chart doesn't have "notes.chart"/"notes.mid" - | 'invalidChart' // .chart/.mid file is not named "notes.chart"/"notes.mid" - | 'badChart' // This chart's .chart/.mid file couldn't be parsed - | 'multipleChart' // This chart has multiple .chart/.mid files - | 'badVideo' // This chart has a video background that will not work on Linux - | 'multipleVideo' // This chart has multiple video background files + | 'noMetadata' // This chart doesn't have "song.ini" + | 'invalidIni' // .ini file is not named "song.ini" + | 'invalidMetadata' // "song.ini" doesn't have a "[Song]" section + | 'badIniLine' // This line in "song.ini" couldn't be parsed + | 'multipleIniFiles' // This chart has multiple .ini files + | 'noAlbumArt' // This chart doesn't have album art + | 'albumArtSize' // This chart's album art is not 500x500 or 512x512 + | 'badAlbumArt' // This chart's album art couldn't be parsed + | 'multipleAlbumArt' // This chart has multiple album art files + | 'noAudio' // This chart doesn't have an audio file + | 'invalidAudio' // Audio file is not a valid audio stem name + | 'badAudio' // This chart's audio couldn't be parsed + | 'multipleAudio' // This chart has multiple audio files of the same stem + | 'noChart' // This chart doesn't have "notes.chart"/"notes.mid" + | 'invalidChart' // .chart/.mid file is not named "notes.chart"/"notes.mid" + | 'badChart' // This chart's .chart/.mid file couldn't be parsed + | 'multipleChart' // This chart has multiple .chart/.mid files + | 'badVideo' // This chart has a video background that will not work on Linux + | 'multipleVideo' // This chart has multiple video background files type MetadataIssueType = - | 'noName' // Metadata is missing the "name" property - | 'noArtist' // Metadata is missing the "artist" property - | 'noAlbum' // Metadata is missing the "album" property - | 'noGenre' // Metadata is missing the "genre" property - | 'noYear' // Metadata is missing the "year" property - | 'noCharter' // Metadata is missing the "charter" property - | 'missingInstrumentDiff' // Metadata is missing a "diff_" property - | 'extraInstrumentDiff' // Metadata contains a "diff_" property for an uncharted instrument - | 'nonzeroDelay' // Metadata contains a "delay" property that is not zero - | 'drumsSetTo4And5Lane' // Metadata indicates the drum chart is both 4-lane and 5-lane - | 'nonzeroOffset' // Chart file contains an "Offset" property that is not zero + | 'missingValue' // Metadata is missing a required value + | 'invalidValue' // Metadata property was set to an unsupported value + | 'extraValue' // Metadata contains a property that should not be included interface ParsedChart { resolution: number diff --git a/src/audio/audio-scanner.ts b/src/audio/audio-scanner.ts index cc5dcc6..d6edc5d 100644 --- a/src/audio/audio-scanner.ts +++ b/src/audio/audio-scanner.ts @@ -5,7 +5,7 @@ import { getBasename, hasAudioExtension, hasAudioName } from '../utils' // TODO: use _max_threads // eslint-disable-next-line @typescript-eslint/no-unused-vars -export function scanAudio(files: { filename: string; data: Uint8Array }[], _max_threads: number) { +export function scanAudio(files: { fileName: string; data: Uint8Array }[], _max_threads: number) { const folderIssues: { folderIssue: FolderIssueType; description: string }[] = [] const findAudioDataResult = findAudioData(files) @@ -31,20 +31,20 @@ export function scanAudio(files: { filename: string; data: Uint8Array }[], _max_ /** * @returns the audio file(s) in this chart, or `[]` if none were found. */ -function findAudioData(files: { filename: string; data: Uint8Array }[]) { +function findAudioData(files: { fileName: string; data: Uint8Array }[]) { const folderIssues: { folderIssue: FolderIssueType; description: string }[] = [] const audioData: Uint8Array[] = [] const stemNames: string[] = [] for (const file of files) { - if (hasAudioExtension(file.filename)) { - if (hasAudioName(file.filename)) { - stemNames.push(getBasename(file.filename)) - if (!['preview', 'crowd'].includes(getBasename(file.filename).toLowerCase())) { + if (hasAudioExtension(file.fileName)) { + if (hasAudioName(file.fileName)) { + stemNames.push(getBasename(file.fileName)) + if (!['preview', 'crowd'].includes(getBasename(file.fileName).toLowerCase())) { audioData.push(file.data) } } else { - folderIssues.push({ folderIssue: 'invalidAudio', description: `"${file.filename}" is not a valid audio stem name.` }) + folderIssues.push({ folderIssue: 'invalidAudio', description: `"${file.fileName}" is not a valid audio stem name.` }) } } } diff --git a/src/chart/chart-scanner.ts b/src/chart/chart-scanner.ts index eba32ce..304dfe4 100644 --- a/src/chart/chart-scanner.ts +++ b/src/chart/chart-scanner.ts @@ -15,7 +15,7 @@ const MIN_SUSTAIN_GAP_MS = 40 const MIN_SUSTAIN_MS = 100 const NPS_GROUP_SIZE_MS = 1000 -export function scanChart(files: { filename: string; data: Uint8Array }[], iniChartModifiers: IniChartModifiers) { +export function scanChart(files: { fileName: string; data: Uint8Array }[], iniChartModifiers: IniChartModifiers) { const { chartData, format, folderIssues } = findChartData(files) if (chartData) { @@ -105,19 +105,19 @@ export function scanChart(files: { filename: string; data: Uint8Array }[], iniCh return { chartHash: null, notesData: null, metadata: null, folderIssues } } -function findChartData(files: { filename: string; data: Uint8Array }[]) { +function findChartData(files: { fileName: string; data: Uint8Array }[]) { const folderIssues: { folderIssue: FolderIssueType; description: string }[] = [] const chartFiles = _.chain(files) - .filter(f => hasChartExtension(f.filename)) - .orderBy([f => hasChartName(f.filename), f => getExtension(f.filename).toLowerCase() === '.mid'], ['desc', 'desc']) + .filter(f => hasChartExtension(f.fileName)) + .orderBy([f => hasChartName(f.fileName), f => getExtension(f.fileName).toLowerCase() === '.mid'], ['desc', 'desc']) .value() for (const file of chartFiles) { - if (!hasChartName(file.filename)) { + if (!hasChartName(file.fileName)) { folderIssues.push({ folderIssue: 'invalidChart', - description: `"${file.filename}" is not named "notes${getExtension(file.filename).toLowerCase()}".`, + description: `"${file.fileName}" is not named "notes${getExtension(file.fileName).toLowerCase()}".`, }) } } @@ -132,7 +132,7 @@ function findChartData(files: { filename: string; data: Uint8Array }[]) { } else { return { chartData: chartFiles[0].data, - format: (getExtension(chartFiles[0].filename).toLowerCase() === '.mid' ? 'mid' : 'chart') as 'mid' | 'chart', + format: (getExtension(chartFiles[0].fileName).toLowerCase() === '.mid' ? 'mid' : 'chart') as 'mid' | 'chart', folderIssues, } } diff --git a/src/image/image-scanner.ts b/src/image/image-scanner.ts index 9332288..74df789 100644 --- a/src/image/image-scanner.ts +++ b/src/image/image-scanner.ts @@ -3,7 +3,7 @@ import { load } from 'exifreader' import { FolderIssueType } from '../interfaces' import { hasAlbumName } from '../utils' -export function scanImage(files: { filename: string; data: Uint8Array }[]) { +export function scanImage(files: { fileName: string; data: Uint8Array }[]) { const folderIssues: { folderIssue: FolderIssueType; description: string }[] = [] const findAlbumDataResult = findAlbumData(files) @@ -21,13 +21,13 @@ export function scanImage(files: { filename: string; data: Uint8Array }[]) { /** * @returns the album art file data in this chart, or `null` if one wasn't found. */ -function findAlbumData(files: { filename: string; data: Uint8Array }[]) { +function findAlbumData(files: { fileName: string; data: Uint8Array }[]) { const folderIssues: { folderIssue: FolderIssueType; description: string }[] = [] let albumCount = 0 let lastAlbumData: Uint8Array | null = null for (const file of files) { - if (hasAlbumName(file.filename)) { + if (hasAlbumName(file.fileName)) { albumCount++ lastAlbumData = file.data } diff --git a/src/index.ts b/src/index.ts index f5bc46a..f405d92 100644 --- a/src/index.ts +++ b/src/index.ts @@ -17,7 +17,7 @@ export { calculateTrackHash } from './chart/track-hasher' /** * Scans `files` as a chart folder, and returns a `ScannedChart` object. */ -export function scanChartFolder(files: { filename: string; data: Uint8Array }[]): ScannedChart { +export function scanChartFolder(files: { fileName: string; data: Uint8Array }[]): ScannedChart { const chart: RequireMatchingProps, 'folderIssues' | 'metadataIssues' | 'playable'> = { folderIssues: [], metadataIssues: [], @@ -119,10 +119,10 @@ export function scanChartFolder(files: { filename: string; data: Uint8Array }[]) return chart as ScannedChart } -function getChartMD5(files: { filename: string; data: Uint8Array }[]) { +function getChartMD5(files: { fileName: string; data: Uint8Array }[]) { const hash = md5.create() - for (const file of _.orderBy(files, f => f.filename)) { - hash.update(file.filename) + for (const file of _.orderBy(files, f => f.fileName)) { + hash.update(file.fileName) hash.update(file.data) } return hash.hex() diff --git a/src/ini/ini-scanner.ts b/src/ini/ini-scanner.ts index 3112f0b..fb3865e 100644 --- a/src/ini/ini-scanner.ts +++ b/src/ini/ini-scanner.ts @@ -85,7 +85,7 @@ const integerProperties: MetaNumberKey[] = [ ] const requiredProperties: MetaStringKey[] = ['name', 'artist', 'album', 'genre', 'year', 'charter'] -export function scanIni(files: { filename: string; data: Uint8Array }[]) { +export function scanIni(files: { fileName: string; data: Uint8Array }[]) { const folderIssues: { folderIssue: FolderIssueType; description: string }[] = [] const findIniDataResult = findIniData(files) @@ -110,7 +110,7 @@ export function scanIni(files: { filename: string; data: Uint8Array }[]) { /** * @returns the .ini file data in this chart, or `null` if one wasn't found. */ -function findIniData(files: { filename: string; data: Uint8Array }[]): { +function findIniData(files: { fileName: string; data: Uint8Array }[]): { iniData: Uint8Array | null folderIssues: { folderIssue: FolderIssueType; description: string }[] } { @@ -120,11 +120,11 @@ function findIniData(files: { filename: string; data: Uint8Array }[]): { let lastIniData: Uint8Array | null = null for (const file of files) { - if (hasIniExtension(file.filename)) { + if (hasIniExtension(file.fileName)) { iniCount++ lastIniData = file.data - if (!hasIniName(file.filename)) { - folderIssues.push({ folderIssue: 'invalidIni', description: `"${file.filename}" is not named "song.ini".` }) + if (!hasIniName(file.fileName)) { + folderIssues.push({ folderIssue: 'invalidIni', description: `"${file.fileName}" is not named "song.ini".` }) } else { bestIniData = file.data } diff --git a/src/utils.ts b/src/utils.ts index 1e4bfd6..f58ba48 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -44,7 +44,7 @@ export function getEncoding(buffer: Uint8Array) { } /** - * @returns true if the list of filename `extensions` appears to be intended as a chart folder. + * @returns true if the list of fileName `extensions` appears to be intended as a chart folder. */ export function appearsToBeChartFolder(extensions: string[]) { const ext = extensions.map(extension => extension.toLowerCase()) @@ -83,7 +83,7 @@ export function hasIniExtension(name: string) { } /** - * @returns `true` if `name` is a valid ini filename. + * @returns `true` if `name` is a valid ini fileName. */ export function hasIniName(name: string) { return name === 'song.ini' @@ -97,7 +97,7 @@ export function hasChartExtension(name: string) { } /** - * @returns `true` if `name` is a valid chart filename. + * @returns `true` if `name` is a valid chart fileName. */ export function hasChartName(name: string) { return ['notes.chart', 'notes.mid'].includes(name) @@ -111,7 +111,7 @@ export function hasAudioExtension(name: string) { } /** - * @returns `true` if `name` has a valid chart audio filename. + * @returns `true` if `name` has a valid chart audio fileName. */ export function hasAudioName(name: string) { return ( @@ -136,26 +136,67 @@ export function hasAudioName(name: string) { } /** - * @returns `true` if `name` is a valid album filename. + * @returns `true` if `name` is a valid album fileName. */ export function hasAlbumName(name: string) { return ['album.jpg', 'album.jpeg', 'album.png'].includes(name) } /** - * @returns `true` if `name` is a valid video filename. + * @returns `true` if `name` is a valid video fileName. */ export function hasVideoName(name: string) { return getBasename(name) === 'video' && ['.mp4', '.avi', '.webm', '.vp8', '.ogv', '.mpeg'].includes(getExtension(name)) } /** - * @returns `true` if `name` is a video filename that is not supported on Linux. + * @returns `true` if `name` is a video fileName that is not supported on Linux. */ export function hasBadVideoName(name: string) { return getBasename(name) === 'video' && ['.mp4', '.avi', '.mpeg'].includes(getExtension(name)) } +const allowedTags = [ + 'align', + 'allcaps', + 'alpha', + 'b', + 'br', + 'color', + 'cspace', + 'font', + 'font-weight', + 'gradient', + 'i', + 'indent', + 'line-height', + 'line-indent', + 'link', + 'lowercase', + 'margin', + 'mark', + 'mspace', + 'nobr', + 'noparse', + 'page', + 'pos', + 'rotate', + 's', + 'size', + 'smallcaps', + 'space', + 'sprite', + 'strikethrough', + 'style', + 'sub', + 'sup', + 'u', + 'uppercase', + 'voffset', + 'width', + '#', +] +const tagPattern = allowedTags.map(tag => `\\b${tag}\\b`).join('|') /** * @returns `text` with all style tags removed. (e.g. "Aren Eternal & Geo" -> "Aren Eternal & Geo") */ @@ -164,9 +205,8 @@ export function removeStyleTags(text: string) { let newText = text do { oldText = newText - newText = newText.replace(/<\s*[^>]+>(.*?)<\s*\/\s*[^>]+>/g, '$1') - newText = newText.replace(/<\s*\/\s*[^>]+>(.*?)<\s*[^>]+>/g, '$1') - } while (newText != oldText) + newText = newText.replace(new RegExp(`<\\s*\\/?\\s*(?:${tagPattern})[^>]*>`, 'gi'), '').trim() + } while (newText !== oldText) return newText } diff --git a/src/video/video-scanner.ts b/src/video/video-scanner.ts index 3f92fd1..5b0fb6b 100644 --- a/src/video/video-scanner.ts +++ b/src/video/video-scanner.ts @@ -1,7 +1,7 @@ import { FolderIssueType } from '../interfaces' import { hasBadVideoName, hasVideoName } from '../utils' -export function scanVideo(files: { filename: string; data: Uint8Array }[]) { +export function scanVideo(files: { fileName: string; data: Uint8Array }[]) { const folderIssues: { folderIssue: FolderIssueType; description: string }[] = [] const findVideoDataResult = findVideoData(files) @@ -10,20 +10,20 @@ export function scanVideo(files: { filename: string; data: Uint8Array }[]) { return { hasVideoBackground: !!findVideoDataResult.videoData, folderIssues } } -function findVideoData(files: { filename: string; data: Uint8Array }[]) { +function findVideoData(files: { fileName: string; data: Uint8Array }[]) { const folderIssues: { folderIssue: FolderIssueType; description: string }[] = [] let videoCount = 0 let bestVideoData: Uint8Array | null = null let lastVideoData: Uint8Array | null = null for (const file of files) { - if (hasVideoName(file.filename)) { + if (hasVideoName(file.fileName)) { videoCount++ lastVideoData = file.data - if (hasBadVideoName(file.filename)) { + if (hasBadVideoName(file.fileName)) { folderIssues.push({ folderIssue: 'badVideo', - description: `"${file.filename}" will not work on Linux and should be converted to .webm.`, + description: `"${file.fileName}" will not work on Linux and should be converted to .webm.`, }) } else { bestVideoData = file.data