merge master

CHANGELOG.md

@@ -11,8 +11,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Validation of TLD domains [#63]
 
-[#63]: https://github.com/AdguardTeam/HostlistCompiler/issues/63
+### Added
 
+- Calculation of checksum for filters and `! Checksum` string to the filter list meta [#76]
+
+[#63]: https://github.com/AdguardTeam/HostlistCompiler/issues/63
+[#76]: https://github.com/AdguardTeam/FiltersCompiler/issues/76
 
 ## [1.0.29] - 2024-09-26
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adguard/hostlist-compiler",
-  "version": "1.0.29",
+  "version": "1.0.30",
   "description": "A simple tool that compiles hosts blocklists from multiple sources",
   "main": "src/index.js",
   "types": "src/index.d.ts",

src/index.d.ts

@@ -1,12 +1,12 @@
 declare module '@adguard/hostlist-compiler' {
     export type SourceType = 'adblock' | 'hosts';
-    
-    export type Transformation = 
-        'RemoveComments' | 
-        'Compress' | 
-        'RemoveModifiers' | 
-        'Validate' | 
-        'Deduplicate' | 
+
+    export type Transformation =
+        'RemoveComments' |
+        'Compress' |
+        'RemoveModifiers' |
+        'Validate' |
+        'Deduplicate' |
         'InvertAllow' |
         'RemoveEmptyLines' |
         'TrimLines' |
@@ -62,7 +62,7 @@ declare module '@adguard/hostlist-compiler' {
      * Compiles a filter list using the specified configuration.
      *
      * @param {*} configuration - compilation configuration.
-    See the repo README for the details on it.
+     * See the repo README for the details on it.
      * @returns {Promise<Array<string>>} the array of rules.
      */
     declare async function compile(configuration: IConfiguration): Promise<string[]>;

src/index.js

@@ -4,17 +4,17 @@ const config = require('./configuration');
 const compileSource = require('./compile-source');
 const { transform } = require('./transformations/transform');
 const packageJson = require('../package.json');
+const { calculateChecksum } = require('./utils');
 
 /**
  * Prepares list header
  *
  * @param {*} configuration - compilation configuration.
-See the repo README for the details on it.
+ * See the repo README for the details on it.
  * @returns {Array<string>} header lines
  */
 function prepareHeader(configuration) {
     const lines = [
-        '!',
         `! Title: ${configuration.name}`,
     ];
 
@@ -67,7 +67,7 @@ function prepareSourceHeader(source) {
  * Compiles a filter list using the specified configuration.
  *
  * @param {*} configuration - compilation configuration.
-See the repo README for the details on it.
+ * See the repo README for the details on it.
  * @returns {Promise<Array<string>>} the array of rules.
  */
 async function compile(configuration) {
@@ -101,8 +101,12 @@ async function compile(configuration) {
 
     // Now prepend the list header and we're good to go
     const header = prepareHeader(configuration);
-    consola.info(`Final length of the list is ${header.length + finalList.length}`);
-    return header.concat(finalList);
+    // Calculate checksum
+    const checksum = calculateChecksum(header, finalList);
+    // Concat everything together
+    const data = ['!', checksum, ...header, ...finalList];
+    consola.info(`Final length of the list is ${data.length}`);
+    return data;
 }
 
 module.exports = compile;

src/utils.js

@@ -1,7 +1,49 @@
 const _ = require('lodash');
 const fs = require('fs').promises;
+const crypto = require('crypto');
 const axios = require('axios');
 
+/**
+ * Strips specified character from the end of a string.
+ *
+ * @param {string} string - The string to strip.
+ * @param {string} char - The character to strip.
+ * @returns {string} - The stripped string.
+ */
+const stripEnd = (str, char) => {
+    let result = str;
+    while (result.endsWith(char)) {
+        result = result.slice(0, -1);
+    }
+    return result;
+};
+
+/**
+ * Normalizes data by replacing newlines.
+ *
+ * @param {string} message - The message to normalize.
+ * @returns {string} - The normalized message.
+ */
+const normalizeData = (message) => {
+    return message.replace(/\r/g, '').replace(/\n+/g, '\n');
+};
+
+/**
+ * Calculates checksum for the given header and rules.
+ * See:
+ * https://adblockplus.org/en/filters#special-comments
+ * https://hg.adblockplus.org/adblockplus/file/tip/addChecksum.py
+ *
+ * @param {Array<string>} header - The header lines.
+ * @param {Array<string>} rules - The rules lines.
+ * @returns {string} - The calculated checksum.
+ */
+const calculateChecksum = (header, rules) => {
+    const content = normalizeData(header.concat(rules).join('\n'));
+    const checksum = crypto.createHash('md5').update(content).digest('base64');
+    return `! Checksum: ${stripEnd(checksum.trim(), '=')}`;
+};
+
 function isURL(str) {
     try {
         // eslint-disable-next-line no-unused-vars
@@ -169,6 +211,7 @@ class Wildcard {
 }
 
 module.exports = {
+    calculateChecksum,
     download,
     Wildcard,
     splitByDelimiterWithEscapeCharacter,

test/index.test.js

@@ -1,6 +1,7 @@
 const nock = require('nock');
 const consola = require('consola');
 const compile = require('../src/index');
+const { calculateChecksum } = require('../src/utils');
 
 describe('Hostlist compiler', () => {
     it('compile from multiple sources', async () => {
@@ -32,6 +33,7 @@ describe('Hostlist compiler', () => {
         const list = await compile(configuration);
 
         // assert
+        expect(list[1].startsWith('! Checksum:')).toBe(true);
         expect(list).toContain('||example.org');
         expect(list).toContain('||example.com');
         expect(list).toContain('! Version: 1.0.0.9');
@@ -41,4 +43,39 @@ describe('Hostlist compiler', () => {
 
         scope.done();
     });
+
+    it('calculates checksum correctly', async () => {
+        // Prepare source
+        const scope = nock('https://example.org')
+            .get('/source1.txt')
+            .reply(200, '||example.org');
+
+        // compiler configuration
+        const configuration = {
+            name: 'Test filter',
+            description: 'Checksum test filter',
+            version: '1.0.0.1',
+            sources: [
+                {
+                    name: 'source 1',
+                    source: 'https://example.org/source1.txt',
+                },
+            ],
+        };
+
+        // compile the final list
+        const list = await compile(configuration);
+
+        // assert
+        const checksumLine = list[1];
+        expect(checksumLine.startsWith('! Checksum:')).toBe(true);
+
+        const header = list.slice(2, list.indexOf('||example.org'));
+        const finalList = list.slice(list.indexOf('||example.org'));
+
+        const expectedChecksum = calculateChecksum(header, finalList);
+        expect(checksumLine).toBe(expectedChecksum);
+
+        scope.done();
+    });
 });

test/utils.test.js

@@ -34,3 +34,32 @@ describe('substringBetween', () => {
         expect(substr).toBe(null);
     });
 });
+
+describe('calculateChecksum', () => {
+    it('calculates checksum with multiple headers and rules', () => {
+        const header = ['[Adblock Plus 2.0]', '! Title: Example filter list'];
+        const rules = ['! Checksum: ', '||example.com^', '||test.com^'];
+        const expectedChecksum = '! Checksum: h0eMYV8/e57vDmGsYZRhsg';
+
+        const checksum = utils.calculateChecksum(header, rules);
+        expect(checksum).toBe(expectedChecksum);
+    });
+
+    it('calculates checksum with empty rules', () => {
+        const header = ['[Adblock Plus 2.0]'];
+        const rules = [];
+        const expectedChecksum = '! Checksum: 87GLA4VsMnV9PiusIkPkEg';
+
+        const checksum = utils.calculateChecksum(header, rules);
+        expect(checksum).toBe(expectedChecksum);
+    });
+
+    it('calculates checksum with empty header and rules', () => {
+        const header = [];
+        const rules = [];
+        const expectedChecksum = '! Checksum: 1B2M2Y8AsgTpgAmY7PhCfg';
+
+        const checksum = utils.calculateChecksum(header, rules);
+        expect(checksum).toBe(expectedChecksum);
+    });
+});