Add task to sync security advisories (#6926)

dart-lang · Aug 23, 2023 · fbecbb5 · fbecbb5
1 parent e02532d
commit fbecbb5
Show file tree

Hide file tree

Showing 7 changed files with 632 additions and 0 deletions.
diff --git a/app/lib/service/security_advisories/sync_security_advisories.dart b/app/lib/service/security_advisories/sync_security_advisories.dart
@@ -0,0 +1,90 @@
+// Copyright (c) 2023, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:io';
+
+import 'package:gcloud/storage.dart';
+import 'package:http/http.dart';
+import 'package:path/path.dart' as path;
+import 'package:pub_dev/service/security_advisories/backend.dart';
+import 'package:pub_dev/service/security_advisories/models.dart';
+import 'package:pub_dev/shared/storage.dart';
+import 'package:pub_dev/shared/utils.dart';
+
+/// Loads security advisories from osv.dev into [targetDir].
+Future<void> fetchAdvisories(Directory targetDir) async {
+  final bucketName = 'osv-vulnerabilities';
+  final allPubAdvisoriesPath = 'Pub/all.zip';
+  final storage = Storage(Client(), 'project');
+  final bucket = storage.bucket(bucketName);
+  final zipFile = File(path.join(targetDir.path, 'all.zip'));
+
+  final bytes = await bucket.readAsBytes(allPubAdvisoriesPath);
+  zipFile.writeAsBytesSync(bytes);
+
+  ProcessResult processResult;
+  processResult = await Process.run('unzip', [zipFile.path],
+      workingDirectory: targetDir.path);
+
+  if (processResult.exitCode != 0) {
+    throw Exception(
+        'Unzipping advisories failed with exitcode ${processResult.exitCode}.\n'
+        '${processResult.stdout}\n${processResult.stderr}');
+  }
+}
+
+Future<(Map<String, OSV>, List<String>)> loadAdvisoriesFromDir(
+    Directory advisoriesDir) async {
+  final osvs = <String, OSV>{};
+  final failedFiles = <String>[];
+
+  await advisoriesDir.list().forEach((advisory) async {
+    if (advisory.path.endsWith('json')) {
+      OSV osv;
+      try {
+        final file = File(advisory.path).readAsBytesSync();
+        final decoded = utf8JsonDecoder.convert(file) as Map<String, dynamic>;
+        osv = OSV.fromJson(decoded);
+      } catch (e) {
+        failedFiles.add(advisory.path);
+        return;
+      }
+      osvs[osv.id] = osv;
+    }
+  });
+  return (osvs, failedFiles);
+}
+
+Future<void> updateAdvisories(Map<String, OSV> osvs) async {
+  final oldAdvisories = await securityAdvisoryBackend.listAdvisories();
+
+  for (final advisory in oldAdvisories) {
+    if (!osvs.containsKey(advisory.id)) {
+      await securityAdvisoryBackend.deleteAdvisory(advisory.id!);
+    }
+  }
+
+  for (final osv in osvs.values) {
+    await securityAdvisoryBackend.ingestSecurityAdvisory(osv);
+  }
+}
+
+/// Synchronizes the security advisory backend with security advisories from
+/// osv.dev.
+Future<void> syncSecurityAdvisories() async {
+  final tempDir = await Directory.systemTemp.createTemp();
+  try {
+    await fetchAdvisories(tempDir);
+    final (osvs, failedFiles) = await loadAdvisoriesFromDir(tempDir);
+    await updateAdvisories(osvs);
+
+    if (failedFiles.isNotEmpty) {
+      throw Exception(
+          'Advisory ingestion was partial. The following advisories failed '
+          '$failedFiles');
+    }
+  } finally {
+    await tempDir.delete(recursive: true);
+  }
+}
diff --git a/app/lib/tool/neat_task/pub_dev_tasks.dart b/app/lib/tool/neat_task/pub_dev_tasks.dart
@@ -18,6 +18,7 @@ import '../../package/backend.dart';
 import '../../scorecard/backend.dart';
 import '../../search/backend.dart';
 import '../../service/email/backend.dart';
+import '../../service/security_advisories/sync_security_advisories.dart';
 import '../../shared/configuration.dart';
 import '../../shared/count_topics.dart';
 import '../../shared/datastore.dart';
@@ -162,6 +163,11 @@ void _setupGenericPeriodicTasks() {
 
   _daily(name: 'count-topics', isRuntimeVersioned: false, task: countTopics);
 
+  _daily(
+      name: 'sync-security-advisories',
+      isRuntimeVersioned: false,
+      task: syncSecurityAdvisories);
+
   // TODO: setup tasks to remove known obsolete (but now unmapped) fields from entities
 }
 

diff --git a/app/test/service/security_advisory/sync_security_advisories_test.dart b/app/test/service/security_advisory/sync_security_advisories_test.dart
@@ -0,0 +1,52 @@
+// Copyright (c) 2023, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:io';
+
+import 'package:path/path.dart' as path;
+import 'package:pub_dev/service/security_advisories/backend.dart';
+import 'package:pub_dev/service/security_advisories/sync_security_advisories.dart';
+import 'package:test/test.dart';
+
+import '../../shared/test_services.dart';
+
+void main() {
+  testWithProfile('Sync and resync advisories from dircetory', fn: () async {
+    final dataDir1 = Directory(path.join(Directory.current.path, 'test',
+        'service', 'security_advisory', 'testdata', 'adv1'));
+    final (osvs, failedFiles) = await loadAdvisoriesFromDir(dataDir1);
+    expect(failedFiles, isEmpty);
+    expect(osvs.length, 2);
+
+    await updateAdvisories(osvs);
+    var list = await securityAdvisoryBackend.listAdvisories();
+    expect(list.length, 2);
+
+    var adv = await securityAdvisoryBackend.lookupById('GHSA-4rgh-jx4f-qfcq');
+    expect(adv, isNotNull);
+    adv = await securityAdvisoryBackend.lookupById('GHSA-1234-1234-1234');
+    expect(adv, isNotNull);
+    adv = await securityAdvisoryBackend.lookupById('GHSA-5678-5678-5678');
+    expect(adv, isNull);
+
+    final dataDir2 = Directory(path.join(Directory.current.path, 'test',
+        'service', 'security_advisory', 'testdata', 'adv2'));
+    final (updatedOsvs, updatedFailedFiles) =
+        await loadAdvisoriesFromDir(dataDir2);
+    expect(updatedFailedFiles, isEmpty);
+    expect(updatedOsvs.length, 2);
+
+    await updateAdvisories(updatedOsvs);
+
+    list = await securityAdvisoryBackend.listAdvisories();
+    expect(list.length, 2);
+
+    adv = await securityAdvisoryBackend.lookupById('GHSA-4rgh-jx4f-qfcq');
+    expect(adv, isNotNull);
+    adv = await securityAdvisoryBackend.lookupById('GHSA-5678-5678-5678');
+    expect(adv, isNotNull);
+    adv = await securityAdvisoryBackend.lookupById('GHSA-1234-1234-1234');
+    expect(adv, isNull);
+  });
+}
diff --git a/app/test/service/security_advisory/testdata/adv1/GHSA-4rgh-jx4f-qfcq.json b/app/test/service/security_advisory/testdata/adv1/GHSA-4rgh-jx4f-qfcq.json
@@ -0,0 +1,196 @@
+{
+  "id": "GHSA-4rgh-jx4f-qfcq",
+  "summary": "http before 0.13.3 vulnerable to header injection",
+  "details": "An issue was discovered in the http package before 0.13.3 for Dart. If the attacker controls the HTTP method and the app is using Request directly, it's possible to achieve CRLF injection in an HTTP request via HTTP header injection. This issue has been addressed in commit abb2bb182 by validating request methods.",
+  "aliases": [
+    "CVE-2020-35669"
+  ],
+  "modified": "2023-04-11T01:46:51.549596Z",
+  "published": "2022-05-24T17:37:16Z",
+  "database_specific": {
+    "nvd_published_at": "2020-12-24T03:15:00Z",
+    "github_reviewed_at": "2022-08-04T21:05:04Z",
+    "severity": "MODERATE",
+    "github_reviewed": true,
+    "cwe_ids": [
+      "CWE-74"
+    ]
+  },
+  "references": [
+    {
+      "type": "ADVISORY",
+      "url": "https://nvd.nist.gov/vuln/detail/CVE-2020-35669"
+    },
+    {
+      "type": "WEB",
+      "url": "https://github.com/dart-lang/http/issues/511"
+    },
+    {
+      "type": "WEB",
+      "url": "https://github.com/dart-lang/http/pull/512"
+    },
+    {
+      "type": "WEB",
+      "url": "https://github.com/dart-lang/http/commit/abb2bb182fbd7f03aafd1f889b902d7b3bdb8769"
+    },
+    {
+      "type": "PACKAGE",
+      "url": "https://github.com/dart-lang/http"
+    },
+    {
+      "type": "WEB",
+      "url": "https://github.com/dart-lang/http/blob/master/CHANGELOG.md#0133"
+    },
+    {
+      "type": "WEB",
+      "url": "https://pub.dev/packages/http/changelog#0133"
+    }
+  ],
+  "affected": [
+    {
+      "package": {
+        "name": "http",
+        "ecosystem": "Pub",
+        "purl": "pkg:pub/http"
+      },
+      "ranges": [
+        {
+          "type": "ECOSYSTEM",
+          "events": [
+            {
+              "introduced": "0"
+            },
+            {
+              "fixed": "0.13.3"
+            }
+          ]
+        }
+      ],
+      "versions": [
+        "0.10.0",
+        "0.11.0",
+        "0.11.0+1",
+        "0.11.1",
+        "0.11.1+1",
+        "0.11.1+3",
+        "0.11.2",
+        "0.11.3",
+        "0.11.3+1",
+        "0.11.3+11",
+        "0.11.3+12",
+        "0.11.3+13",
+        "0.11.3+14",
+        "0.11.3+15",
+        "0.11.3+16",
+        "0.11.3+17",
+        "0.11.3+2",
+        "0.11.3+3",
+        "0.11.3+4",
+        "0.11.3+5",
+        "0.11.3+6",
+        "0.11.3+7",
+        "0.11.3+8",
+        "0.11.3+9",
+        "0.12.0",
+        "0.12.0+1",
+        "0.12.0+2",
+        "0.12.0+3",
+        "0.12.0+4",
+        "0.12.1",
+        "0.12.2",
+        "0.13.0",
+        "0.13.0-nullsafety.0",
+        "0.13.1",
+        "0.13.2",
+        "0.2.10+1",
+        "0.2.7+0",
+        "0.2.8+2",
+        "0.2.9+7",
+        "0.3.1+1",
+        "0.3.2",
+        "0.3.4",
+        "0.3.5+1",
+        "0.3.7+6",
+        "0.4.0",
+        "0.4.1",
+        "0.4.2",
+        "0.4.3+1",
+        "0.4.4+4",
+        "0.4.5+1",
+        "0.4.7+1",
+        "0.5.0+1",
+        "0.5.1",
+        "0.5.11+1",
+        "0.5.12",
+        "0.5.13",
+        "0.5.14+1",
+        "0.5.14+3",
+        "0.5.15",
+        "0.5.16",
+        "0.5.17",
+        "0.5.20",
+        "0.5.4",
+        "0.5.5",
+        "0.5.6",
+        "0.5.7",
+        "0.5.9",
+        "0.6.1",
+        "0.6.11",
+        "0.6.12",
+        "0.6.13",
+        "0.6.14",
+        "0.6.15+2",
+        "0.6.15+3",
+        "0.6.17",
+        "0.6.17+2",
+        "0.6.19",
+        "0.6.2",
+        "0.6.20+1",
+        "0.6.21+3",
+        "0.6.3+1",
+        "0.6.5",
+        "0.6.6",
+        "0.6.8",
+        "0.6.9",
+        "0.6.9+2",
+        "0.7.0",
+        "0.7.1",
+        "0.7.2",
+        "0.7.2+1",
+        "0.7.3+1",
+        "0.7.4",
+        "0.7.5",
+        "0.7.6",
+        "0.7.6+4",
+        "0.8.0",
+        "0.8.1",
+        "0.8.10",
+        "0.8.10+3",
+        "0.8.10+4",
+        "0.8.2",
+        "0.8.3",
+        "0.8.4",
+        "0.8.5",
+        "0.8.6",
+        "0.8.7",
+        "0.8.8",
+        "0.8.9",
+        "0.9.0",
+        "0.9.1",
+        "0.9.2",
+        "0.9.2+1",
+        "0.9.2+3"
+      ],
+      "database_specific": {
+        "source": "https://github.com/github/advisory-database/blob/main/advisories/github-reviewed/2022/05/GHSA-4rgh-jx4f-qfcq/GHSA-4rgh-jx4f-qfcq.json"
+      }
+    }
+  ],
+  "schema_version": "1.4.0",
+  "severity": [
+    {
+      "type": "CVSS_V3",
+      "score": "CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:C/C:L/I:L/A:N"
+    }
+  ]
+}