Fix capitalization and improve removeStyleTags

.changeset/sour-mirrors-care.md

@@ -0,0 +1,5 @@
+---
+"scan-chart": patch
+---
+
+Fix capitalization and improve removeStyleTags

readme.md

@@ -9,7 +9,7 @@ This has been validated on 40,000 charts, including some that were deliberately
 /**
  * Scans `files` as a chart folder, and returns a `ScannedChart` object.
  */
-function scanChartFolder(files: { filename: string; data: Uint8Array }[]): ScannedChart
+function scanChartFolder(files: { fileName: string; data: Uint8Array }[]): ScannedChart
 function parseChartFile(data: Uint8Array, format: 'chart' | 'mid', iniChartModifiers: IniChartModifiers): ParsedChart
 function calculateTrackHash(parsedChart: ParsedChart, instrument: Instrument, difficulty: Difficulty): { hash: string, bchart: Uint8Array }
 
@@ -310,38 +310,30 @@ type NoteType =
 	| 'greenTomOrCymbalMarker'
 
 type FolderIssueType =
-	| 'noMetadata' // This chart doesn't have "song.ini"
-	| 'invalidIni' // .ini file is not named "song.ini"
-	| 'invalidMetadata' // "song.ini" doesn't have a "[Song]" section
-	| 'badIniLine' // This line in "song.ini" couldn't be parsed
-	| 'multipleIniFiles' // This chart has multiple .ini files
-	| 'noAlbumArt' // This chart doesn't have album art
-	| 'albumArtSize' // This chart's album art is not 500x500 or 512x512
-	| 'badAlbumArt' // This chart's album art couldn't be parsed
-	| 'multipleAlbumArt' // This chart has multiple album art files
-	| 'noAudio' // This chart doesn't have an audio file
-	| 'invalidAudio' // Audio file is not a valid audio stem name
-	| 'badAudio' // This chart's audio couldn't be parsed
-	| 'multipleAudio' // This chart has multiple audio files of the same stem
-	| 'noChart' // This chart doesn't have "notes.chart"/"notes.mid"
-	| 'invalidChart' // .chart/.mid file is not named "notes.chart"/"notes.mid"
-	| 'badChart' // This chart's .chart/.mid file couldn't be parsed
-	| 'multipleChart' // This chart has multiple .chart/.mid files
-	| 'badVideo' // This chart has a video background that will not work on Linux
-	| 'multipleVideo' // This chart has multiple video background files
+	| 'noMetadata'              // This chart doesn't have "song.ini"
+	| 'invalidIni'              // .ini file is not named "song.ini"
+	| 'invalidMetadata'         // "song.ini" doesn't have a "[Song]" section
+	| 'badIniLine'              // This line in "song.ini" couldn't be parsed
+	| 'multipleIniFiles'        // This chart has multiple .ini files
+	| 'noAlbumArt'              // This chart doesn't have album art
+	| 'albumArtSize'            // This chart's album art is not 500x500 or 512x512
+	| 'badAlbumArt'             // This chart's album art couldn't be parsed
+	| 'multipleAlbumArt'        // This chart has multiple album art files
+	| 'noAudio'                 // This chart doesn't have an audio file
+	| 'invalidAudio'            // Audio file is not a valid audio stem name
+	| 'badAudio'                // This chart's audio couldn't be parsed
+	| 'multipleAudio'           // This chart has multiple audio files of the same stem
+	| 'noChart'                 // This chart doesn't have "notes.chart"/"notes.mid"
+	| 'invalidChart'            // .chart/.mid file is not named "notes.chart"/"notes.mid"
+	| 'badChart'                // This chart's .chart/.mid file couldn't be parsed
+	| 'multipleChart'           // This chart has multiple .chart/.mid files
+	| 'badVideo'                // This chart has a video background that will not work on Linux
+	| 'multipleVideo'           // This chart has multiple video background files
 
 type MetadataIssueType =
-	| 'noName' // Metadata is missing the "name" property
-	| 'noArtist' // Metadata is missing the "artist" property
-	| 'noAlbum' // Metadata is missing the "album" property
-	| 'noGenre' // Metadata is missing the "genre" property
-	| 'noYear' // Metadata is missing the "year" property
-	| 'noCharter' // Metadata is missing the "charter" property
-	| 'missingInstrumentDiff' // Metadata is missing a "diff_" property
-	| 'extraInstrumentDiff' // Metadata contains a "diff_" property for an uncharted instrument
-	| 'nonzeroDelay' // Metadata contains a "delay" property that is not zero
-	| 'drumsSetTo4And5Lane' // Metadata indicates the drum chart is both 4-lane and 5-lane
-	| 'nonzeroOffset' // Chart file contains an "Offset" property that is not zero
+	| 'missingValue'            // Metadata is missing a required value
+	| 'invalidValue'            // Metadata property was set to an unsupported value
+	| 'extraValue'              // Metadata contains a property that should not be included
 
 interface ParsedChart {
 	resolution: number

src/audio/audio-scanner.ts

@@ -5,7 +5,7 @@ import { getBasename, hasAudioExtension, hasAudioName } from '../utils'
 
 // TODO: use _max_threads
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
-export function scanAudio(files: { filename: string; data: Uint8Array }[], _max_threads: number) {
+export function scanAudio(files: { fileName: string; data: Uint8Array }[], _max_threads: number) {
 	const folderIssues: { folderIssue: FolderIssueType; description: string }[] = []
 
 	const findAudioDataResult = findAudioData(files)
@@ -31,20 +31,20 @@ export function scanAudio(files: { filename: string; data: Uint8Array }[], _max_
 /**
  * @returns the audio file(s) in this chart, or `[]` if none were found.
  */
-function findAudioData(files: { filename: string; data: Uint8Array }[]) {
+function findAudioData(files: { fileName: string; data: Uint8Array }[]) {
 	const folderIssues: { folderIssue: FolderIssueType; description: string }[] = []
 	const audioData: Uint8Array[] = []
 	const stemNames: string[] = []
 
 	for (const file of files) {
-		if (hasAudioExtension(file.filename)) {
-			if (hasAudioName(file.filename)) {
-				stemNames.push(getBasename(file.filename))
-				if (!['preview', 'crowd'].includes(getBasename(file.filename).toLowerCase())) {
+		if (hasAudioExtension(file.fileName)) {
+			if (hasAudioName(file.fileName)) {
+				stemNames.push(getBasename(file.fileName))
+				if (!['preview', 'crowd'].includes(getBasename(file.fileName).toLowerCase())) {
 					audioData.push(file.data)
 				}
 			} else {
-				folderIssues.push({ folderIssue: 'invalidAudio', description: `"${file.filename}" is not a valid audio stem name.` })
+				folderIssues.push({ folderIssue: 'invalidAudio', description: `"${file.fileName}" is not a valid audio stem name.` })
 			}
 		}
 	}

src/chart/chart-scanner.ts

@@ -15,7 +15,7 @@ const MIN_SUSTAIN_GAP_MS = 40
 const MIN_SUSTAIN_MS = 100
 const NPS_GROUP_SIZE_MS = 1000
 
-export function scanChart(files: { filename: string; data: Uint8Array }[], iniChartModifiers: IniChartModifiers) {
+export function scanChart(files: { fileName: string; data: Uint8Array }[], iniChartModifiers: IniChartModifiers) {
 	const { chartData, format, folderIssues } = findChartData(files)
 
 	if (chartData) {
@@ -105,19 +105,19 @@ export function scanChart(files: { filename: string; data: Uint8Array }[], iniCh
 	return { chartHash: null, notesData: null, metadata: null, folderIssues }
 }
 
-function findChartData(files: { filename: string; data: Uint8Array }[]) {
+function findChartData(files: { fileName: string; data: Uint8Array }[]) {
 	const folderIssues: { folderIssue: FolderIssueType; description: string }[] = []
 
 	const chartFiles = _.chain(files)
-		.filter(f => hasChartExtension(f.filename))
-		.orderBy([f => hasChartName(f.filename), f => getExtension(f.filename).toLowerCase() === '.mid'], ['desc', 'desc'])
+		.filter(f => hasChartExtension(f.fileName))
+		.orderBy([f => hasChartName(f.fileName), f => getExtension(f.fileName).toLowerCase() === '.mid'], ['desc', 'desc'])
 		.value()
 
 	for (const file of chartFiles) {
-		if (!hasChartName(file.filename)) {
+		if (!hasChartName(file.fileName)) {
 			folderIssues.push({
 				folderIssue: 'invalidChart',
-				description: `"${file.filename}" is not named "notes${getExtension(file.filename).toLowerCase()}".`,
+				description: `"${file.fileName}" is not named "notes${getExtension(file.fileName).toLowerCase()}".`,
 			})
 		}
 	}
@@ -132,7 +132,7 @@ function findChartData(files: { filename: string; data: Uint8Array }[]) {
 	} else {
 		return {
 			chartData: chartFiles[0].data,
-			format: (getExtension(chartFiles[0].filename).toLowerCase() === '.mid' ? 'mid' : 'chart') as 'mid' | 'chart',
+			format: (getExtension(chartFiles[0].fileName).toLowerCase() === '.mid' ? 'mid' : 'chart') as 'mid' | 'chart',
 			folderIssues,
 		}
 	}

src/image/image-scanner.ts

@@ -3,7 +3,7 @@ import { load } from 'exifreader'
 import { FolderIssueType } from '../interfaces'
 import { hasAlbumName } from '../utils'
 
-export function scanImage(files: { filename: string; data: Uint8Array }[]) {
+export function scanImage(files: { fileName: string; data: Uint8Array }[]) {
 	const folderIssues: { folderIssue: FolderIssueType; description: string }[] = []
 
 	const findAlbumDataResult = findAlbumData(files)
@@ -21,13 +21,13 @@ export function scanImage(files: { filename: string; data: Uint8Array }[]) {
 /**
  * @returns the album art file data in this chart, or `null` if one wasn't found.
  */
-function findAlbumData(files: { filename: string; data: Uint8Array }[]) {
+function findAlbumData(files: { fileName: string; data: Uint8Array }[]) {
 	const folderIssues: { folderIssue: FolderIssueType; description: string }[] = []
 	let albumCount = 0
 	let lastAlbumData: Uint8Array | null = null
 
 	for (const file of files) {
-		if (hasAlbumName(file.filename)) {
+		if (hasAlbumName(file.fileName)) {
 			albumCount++
 			lastAlbumData = file.data
 		}

src/index.ts

@@ -17,7 +17,7 @@ export { calculateTrackHash } from './chart/track-hasher'
 /**
  * Scans `files` as a chart folder, and returns a `ScannedChart` object.
  */
-export function scanChartFolder(files: { filename: string; data: Uint8Array }[]): ScannedChart {
+export function scanChartFolder(files: { fileName: string; data: Uint8Array }[]): ScannedChart {
 	const chart: RequireMatchingProps<Subset<ScannedChart>, 'folderIssues' | 'metadataIssues' | 'playable'> = {
 		folderIssues: [],
 		metadataIssues: [],
@@ -119,10 +119,10 @@ export function scanChartFolder(files: { filename: string; data: Uint8Array }[])
 	return chart as ScannedChart
 }
 
-function getChartMD5(files: { filename: string; data: Uint8Array }[]) {
+function getChartMD5(files: { fileName: string; data: Uint8Array }[]) {
 	const hash = md5.create()
-	for (const file of _.orderBy(files, f => f.filename)) {
-		hash.update(file.filename)
+	for (const file of _.orderBy(files, f => f.fileName)) {
+		hash.update(file.fileName)
 		hash.update(file.data)
 	}
 	return hash.hex()

src/ini/ini-scanner.ts

@@ -85,7 +85,7 @@ const integerProperties: MetaNumberKey[] = [
 ]
 const requiredProperties: MetaStringKey[] = ['name', 'artist', 'album', 'genre', 'year', 'charter']
 
-export function scanIni(files: { filename: string; data: Uint8Array }[]) {
+export function scanIni(files: { fileName: string; data: Uint8Array }[]) {
 	const folderIssues: { folderIssue: FolderIssueType; description: string }[] = []
 
 	const findIniDataResult = findIniData(files)
@@ -110,7 +110,7 @@ export function scanIni(files: { filename: string; data: Uint8Array }[]) {
 /**
  * @returns the .ini file data in this chart, or `null` if one wasn't found.
  */
-function findIniData(files: { filename: string; data: Uint8Array }[]): {
+function findIniData(files: { fileName: string; data: Uint8Array }[]): {
 	iniData: Uint8Array | null
 	folderIssues: { folderIssue: FolderIssueType; description: string }[]
 } {
@@ -120,11 +120,11 @@ function findIniData(files: { filename: string; data: Uint8Array }[]): {
 	let lastIniData: Uint8Array | null = null
 
 	for (const file of files) {
-		if (hasIniExtension(file.filename)) {
+		if (hasIniExtension(file.fileName)) {
 			iniCount++
 			lastIniData = file.data
-			if (!hasIniName(file.filename)) {
-				folderIssues.push({ folderIssue: 'invalidIni', description: `"${file.filename}" is not named "song.ini".` })
+			if (!hasIniName(file.fileName)) {
+				folderIssues.push({ folderIssue: 'invalidIni', description: `"${file.fileName}" is not named "song.ini".` })
 			} else {
 				bestIniData = file.data
 			}

src/utils.ts

@@ -44,7 +44,7 @@ export function getEncoding(buffer: Uint8Array) {
 }
 
 /**
- * @returns true if the list of filename `extensions` appears to be intended as a chart folder.
+ * @returns true if the list of fileName `extensions` appears to be intended as a chart folder.
  */
 export function appearsToBeChartFolder(extensions: string[]) {
 	const ext = extensions.map(extension => extension.toLowerCase())
@@ -83,7 +83,7 @@ export function hasIniExtension(name: string) {
 }
 
 /**
- * @returns `true` if `name` is a valid ini filename.
+ * @returns `true` if `name` is a valid ini fileName.
  */
 export function hasIniName(name: string) {
 	return name === 'song.ini'
@@ -97,7 +97,7 @@ export function hasChartExtension(name: string) {
 }
 
 /**
- * @returns `true` if `name` is a valid chart filename.
+ * @returns `true` if `name` is a valid chart fileName.
  */
 export function hasChartName(name: string) {
 	return ['notes.chart', 'notes.mid'].includes(name)
@@ -111,7 +111,7 @@ export function hasAudioExtension(name: string) {
 }
 
 /**
- * @returns `true` if `name` has a valid chart audio filename.
+ * @returns `true` if `name` has a valid chart audio fileName.
  */
 export function hasAudioName(name: string) {
 	return (
@@ -136,26 +136,67 @@ export function hasAudioName(name: string) {
 }
 
 /**
- * @returns `true` if `name` is a valid album filename.
+ * @returns `true` if `name` is a valid album fileName.
  */
 export function hasAlbumName(name: string) {
 	return ['album.jpg', 'album.jpeg', 'album.png'].includes(name)
 }
 
 /**
- * @returns `true` if `name` is a valid video filename.
+ * @returns `true` if `name` is a valid video fileName.
  */
 export function hasVideoName(name: string) {
 	return getBasename(name) === 'video' && ['.mp4', '.avi', '.webm', '.vp8', '.ogv', '.mpeg'].includes(getExtension(name))
 }
 
 /**
- * @returns `true` if `name` is a video filename that is not supported on Linux.
+ * @returns `true` if `name` is a video fileName that is not supported on Linux.
  */
 export function hasBadVideoName(name: string) {
 	return getBasename(name) === 'video' && ['.mp4', '.avi', '.mpeg'].includes(getExtension(name))
 }
 
+const allowedTags = [
+	'align',
+	'allcaps',
+	'alpha',
+	'b',
+	'br',
+	'color',
+	'cspace',
+	'font',
+	'font-weight',
+	'gradient',
+	'i',
+	'indent',
+	'line-height',
+	'line-indent',
+	'link',
+	'lowercase',
+	'margin',
+	'mark',
+	'mspace',
+	'nobr',
+	'noparse',
+	'page',
+	'pos',
+	'rotate',
+	's',
+	'size',
+	'smallcaps',
+	'space',
+	'sprite',
+	'strikethrough',
+	'style',
+	'sub',
+	'sup',
+	'u',
+	'uppercase',
+	'voffset',
+	'width',
+	'#',
+]
+const tagPattern = allowedTags.map(tag => `\\b${tag}\\b`).join('|')
 /**
  * @returns `text` with all style tags removed. (e.g. "<color=#AEFFFF>Aren Eternal</color> & Geo" -> "Aren Eternal & Geo")
  */
@@ -164,9 +205,8 @@ export function removeStyleTags(text: string) {
 	let newText = text
 	do {
 		oldText = newText
-		newText = newText.replace(/<\s*[^>]+>(.*?)<\s*\/\s*[^>]+>/g, '$1')
-		newText = newText.replace(/<\s*\/\s*[^>]+>(.*?)<\s*[^>]+>/g, '$1')
-	} while (newText != oldText)
+		newText = newText.replace(new RegExp(`<\\s*\\/?\\s*(?:${tagPattern})[^>]*>`, 'gi'), '').trim()
+	} while (newText !== oldText)
 	return newText
 }
 

src/video/video-scanner.ts

@@ -1,7 +1,7 @@
 import { FolderIssueType } from '../interfaces'
 import { hasBadVideoName, hasVideoName } from '../utils'
 
-export function scanVideo(files: { filename: string; data: Uint8Array }[]) {
+export function scanVideo(files: { fileName: string; data: Uint8Array }[]) {
 	const folderIssues: { folderIssue: FolderIssueType; description: string }[] = []
 
 	const findVideoDataResult = findVideoData(files)
@@ -10,20 +10,20 @@ export function scanVideo(files: { filename: string; data: Uint8Array }[]) {
 	return { hasVideoBackground: !!findVideoDataResult.videoData, folderIssues }
 }
 
-function findVideoData(files: { filename: string; data: Uint8Array }[]) {
+function findVideoData(files: { fileName: string; data: Uint8Array }[]) {
 	const folderIssues: { folderIssue: FolderIssueType; description: string }[] = []
 	let videoCount = 0
 	let bestVideoData: Uint8Array | null = null
 	let lastVideoData: Uint8Array | null = null
 
 	for (const file of files) {
-		if (hasVideoName(file.filename)) {
+		if (hasVideoName(file.fileName)) {
 			videoCount++
 			lastVideoData = file.data
-			if (hasBadVideoName(file.filename)) {
+			if (hasBadVideoName(file.fileName)) {
 				folderIssues.push({
 					folderIssue: 'badVideo',
-					description: `"${file.filename}" will not work on Linux and should be converted to .webm.`,
+					description: `"${file.fileName}" will not work on Linux and should be converted to .webm.`,
 				})
 			} else {
 				bestVideoData = file.data