From abe6fa3d11584c03f38db9f0a4fe0fc3fc2b1405 Mon Sep 17 00:00:00 2001 From: Elizaveta Egorova Date: Wed, 18 Dec 2024 18:06:26 +0300 Subject: [PATCH 1/2] Calculate checksum for filters and add a '! Checksum' meta to the filter list. #76 Squashed commit of the following: commit a876ffca31c02acc80525f805f4ad58743a66ed5 Author: jellizaveta Date: Fri Dec 13 19:20:40 2024 +0300 add test for calculateChecksum commit c7a0b15f4f6dcd1afeb50baa2fdab8cf8f246feb Author: jellizaveta Date: Fri Dec 13 17:45:29 2024 +0300 fix indents commit 3bdaa4873b1bdcecec422df3d5ff02eeff25a0a5 Author: jellizaveta Date: Fri Dec 13 17:40:12 2024 +0300 add test to check checksum value commit 2e6d05e54ac83fc414c6cafb63c28f12d892ec12 Author: jellizaveta Date: Fri Dec 13 16:59:09 2024 +0300 update changelog commit 44788ec4124fa7795f8364ed044fa8b0001bfb42 Merge: efbff9b 82247f9 Author: jellizaveta Date: Fri Dec 13 16:56:14 2024 +0300 Merge branch 'fix/#76_add_checksum' of ssh://bit.int.agrd.dev:7999/adguard-filters/hostlist-compiler into fix/#76_add_checksum commit efbff9b947db6e509af959f31ee377fddf29ea53 Author: jellizaveta Date: Fri Dec 13 16:56:09 2024 +0300 fix jsDoc comments commit 82247f9cfa2361506618ac3e15fe9dc4b761cb66 Author: Slava Leleka Date: Fri Dec 13 16:55:45 2024 +0300 Applied suggestion commit 4ab778fcf4706f82f59c5f19f0958c2d2811c929 Author: jellizaveta Date: Fri Dec 13 16:41:48 2024 +0300 Calculate checksum for filters and add a '! Checksum' meta to the filter list. #76 --- CHANGELOG.md | 7 +++++++ src/index.d.ts | 16 ++++++++-------- src/index.js | 14 +++++++++----- src/utils.js | 43 +++++++++++++++++++++++++++++++++++++++++++ test/index.test.js | 37 +++++++++++++++++++++++++++++++++++++ test/utils.test.js | 29 +++++++++++++++++++++++++++++ 6 files changed, 133 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 989abd1..bc7292c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased + +### Added + +- Calculation of checksum for filters and `! Checksum` string to the filter list meta [#76] + +[#76]: https://github.com/AdguardTeam/FiltersCompiler/issues/76 ## [1.0.29] - 2024-09-26 diff --git a/src/index.d.ts b/src/index.d.ts index 8e5c74a..9f2a67d 100644 --- a/src/index.d.ts +++ b/src/index.d.ts @@ -1,12 +1,12 @@ declare module '@adguard/hostlist-compiler' { export type SourceType = 'adblock' | 'hosts'; - - export type Transformation = - 'RemoveComments' | - 'Compress' | - 'RemoveModifiers' | - 'Validate' | - 'Deduplicate' | + + export type Transformation = + 'RemoveComments' | + 'Compress' | + 'RemoveModifiers' | + 'Validate' | + 'Deduplicate' | 'InvertAllow' | 'RemoveEmptyLines' | 'TrimLines' | @@ -62,7 +62,7 @@ declare module '@adguard/hostlist-compiler' { * Compiles a filter list using the specified configuration. * * @param {*} configuration - compilation configuration. - See the repo README for the details on it. + * See the repo README for the details on it. * @returns {Promise>} the array of rules. */ declare async function compile(configuration: IConfiguration): Promise; diff --git a/src/index.js b/src/index.js index 149a34e..d7a8189 100644 --- a/src/index.js +++ b/src/index.js @@ -4,17 +4,17 @@ const config = require('./configuration'); const compileSource = require('./compile-source'); const { transform } = require('./transformations/transform'); const packageJson = require('../package.json'); +const { calculateChecksum } = require('./utils'); /** * Prepares list header * * @param {*} configuration - compilation configuration. -See the repo README for the details on it. + * See the repo README for the details on it. * @returns {Array} header lines */ function prepareHeader(configuration) { const lines = [ - '!', `! Title: ${configuration.name}`, ]; @@ -67,7 +67,7 @@ function prepareSourceHeader(source) { * Compiles a filter list using the specified configuration. * * @param {*} configuration - compilation configuration. -See the repo README for the details on it. + * See the repo README for the details on it. * @returns {Promise>} the array of rules. */ async function compile(configuration) { @@ -101,8 +101,12 @@ async function compile(configuration) { // Now prepend the list header and we're good to go const header = prepareHeader(configuration); - consola.info(`Final length of the list is ${header.length + finalList.length}`); - return header.concat(finalList); + // Calculate checksum + const checksum = calculateChecksum(header, finalList); + // Concat everything together + const data = ['!', checksum, ...header, ...finalList]; + consola.info(`Final length of the list is ${data.length}`); + return data; } module.exports = compile; diff --git a/src/utils.js b/src/utils.js index 1486fec..827a725 100644 --- a/src/utils.js +++ b/src/utils.js @@ -1,7 +1,49 @@ const _ = require('lodash'); const fs = require('fs').promises; +const crypto = require('crypto'); const axios = require('axios'); +/** + * Strips specified character from the end of a string. + * + * @param {string} string - The string to strip. + * @param {string} char - The character to strip. + * @returns {string} - The stripped string. + */ +const stripEnd = (str, char) => { + let result = str; + while (result.endsWith(char)) { + result = result.slice(0, -1); + } + return result; +}; + +/** + * Normalizes data by replacing newlines. + * + * @param {string} message - The message to normalize. + * @returns {string} - The normalized message. + */ +const normalizeData = (message) => { + return message.replace(/\r/g, '').replace(/\n+/g, '\n'); +}; + +/** + * Calculates checksum for the given header and rules. + * See: + * https://adblockplus.org/en/filters#special-comments + * https://hg.adblockplus.org/adblockplus/file/tip/addChecksum.py + * + * @param {Array} header - The header lines. + * @param {Array} rules - The rules lines. + * @returns {string} - The calculated checksum. + */ +const calculateChecksum = (header, rules) => { + const content = normalizeData(header.concat(rules).join('\n')); + const checksum = crypto.createHash('md5').update(content).digest('base64'); + return `! Checksum: ${stripEnd(checksum.trim(), '=')}`; +}; + function isURL(str) { try { // eslint-disable-next-line no-unused-vars @@ -169,6 +211,7 @@ class Wildcard { } module.exports = { + calculateChecksum, download, Wildcard, splitByDelimiterWithEscapeCharacter, diff --git a/test/index.test.js b/test/index.test.js index 2cf6f69..59c8c45 100644 --- a/test/index.test.js +++ b/test/index.test.js @@ -1,6 +1,7 @@ const nock = require('nock'); const consola = require('consola'); const compile = require('../src/index'); +const { calculateChecksum } = require('../src/utils'); describe('Hostlist compiler', () => { it('compile from multiple sources', async () => { @@ -32,6 +33,7 @@ describe('Hostlist compiler', () => { const list = await compile(configuration); // assert + expect(list[1].startsWith('! Checksum:')).toBe(true); expect(list).toContain('||example.org'); expect(list).toContain('||example.com'); expect(list).toContain('! Version: 1.0.0.9'); @@ -41,4 +43,39 @@ describe('Hostlist compiler', () => { scope.done(); }); + + it('calculates checksum correctly', async () => { + // Prepare source + const scope = nock('https://example.org') + .get('/source1.txt') + .reply(200, '||example.org'); + + // compiler configuration + const configuration = { + name: 'Test filter', + description: 'Checksum test filter', + version: '1.0.0.1', + sources: [ + { + name: 'source 1', + source: 'https://example.org/source1.txt', + }, + ], + }; + + // compile the final list + const list = await compile(configuration); + + // assert + const checksumLine = list[1]; + expect(checksumLine.startsWith('! Checksum:')).toBe(true); + + const header = list.slice(2, list.indexOf('||example.org')); + const finalList = list.slice(list.indexOf('||example.org')); + + const expectedChecksum = calculateChecksum(header, finalList); + expect(checksumLine).toBe(expectedChecksum); + + scope.done(); + }); }); diff --git a/test/utils.test.js b/test/utils.test.js index 4a2aa85..856d7dd 100644 --- a/test/utils.test.js +++ b/test/utils.test.js @@ -34,3 +34,32 @@ describe('substringBetween', () => { expect(substr).toBe(null); }); }); + +describe('calculateChecksum', () => { + it('calculates checksum with multiple headers and rules', () => { + const header = ['[Adblock Plus 2.0]', '! Title: Example filter list']; + const rules = ['! Checksum: ', '||example.com^', '||test.com^']; + const expectedChecksum = '! Checksum: h0eMYV8/e57vDmGsYZRhsg'; + + const checksum = utils.calculateChecksum(header, rules); + expect(checksum).toBe(expectedChecksum); + }); + + it('calculates checksum with empty rules', () => { + const header = ['[Adblock Plus 2.0]']; + const rules = []; + const expectedChecksum = '! Checksum: 87GLA4VsMnV9PiusIkPkEg'; + + const checksum = utils.calculateChecksum(header, rules); + expect(checksum).toBe(expectedChecksum); + }); + + it('calculates checksum with empty header and rules', () => { + const header = []; + const rules = []; + const expectedChecksum = '! Checksum: 1B2M2Y8AsgTpgAmY7PhCfg'; + + const checksum = utils.calculateChecksum(header, rules); + expect(checksum).toBe(expectedChecksum); + }); +}); From 85e29539762f6c00829cfb82815bf21769c0a894 Mon Sep 17 00:00:00 2001 From: Atlassian Bamboo Date: Wed, 18 Dec 2024 18:06:35 +0300 Subject: [PATCH 2/2] skipci: Automatic increment build number --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index b635ea6..ddff438 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@adguard/hostlist-compiler", - "version": "1.0.29", + "version": "1.0.30", "description": "A simple tool that compiles hosts blocklists from multiple sources", "main": "src/index.js", "types": "src/index.d.ts",