Merge pull request #232 from Webperf-se/issue-228

start of splitting out software and CVE lookup, #228
Webperf-se · Oct 1, 2023 · 0a9f437 · 0a9f437
2 parents ece0cf8 + 5113664
commit 0a9f437
Show file tree

Hide file tree

Showing 36 changed files with 24,004 additions and 1,623 deletions.
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -14,6 +14,11 @@ on:
   push:
     paths-ignore:
       - '**.md'
+      - 'software-full.json'
+      - 'software-sources.json'
+      - '**update-software.yml'
+      - 'update_software.py'
+      - 'SAMPLE-software-rules.json'
 #on: [ push, pull_request ]
 jobs:
   analyze:

diff --git a/.github/workflows/regression-test-404.yml b/.github/workflows/regression-test-404.yml
@@ -3,6 +3,11 @@ on:
   push:
     paths-ignore:
       - '**.md'
+      - 'software-full.json'
+      - 'software-sources.json'
+      - '**update-software.yml'
+      - 'update_software.py'
+      - 'SAMPLE-software-rules.json'
 jobs:
   build:
     runs-on: ${{ matrix.os }}

diff --git a/.github/workflows/regression-test-a11y-statement.yml b/.github/workflows/regression-test-a11y-statement.yml
@@ -3,6 +3,11 @@ on:
   push:
     paths-ignore:
       - '**.md'
+      - 'software-full.json'
+      - 'software-sources.json'
+      - '**update-software.yml'
+      - 'update_software.py'
+      - 'SAMPLE-software-rules.json'
 jobs:
   build:
     runs-on: ${{ matrix.os }}

diff --git a/.github/workflows/regression-test-css.yml b/.github/workflows/regression-test-css.yml
@@ -3,6 +3,11 @@ on:
   push:
     paths-ignore:
       - '**.md'
+      - 'software-full.json'
+      - 'software-sources.json'
+      - '**update-software.yml'
+      - 'update_software.py'
+      - 'SAMPLE-software-rules.json'
 jobs:
   build:
     runs-on: ${{ matrix.os }}

diff --git a/.github/workflows/regression-test-email.yml b/.github/workflows/regression-test-email.yml
@@ -3,6 +3,11 @@ on:
   push:
     paths-ignore:
       - '**.md'
+      - 'software-full.json'
+      - 'software-sources.json'
+      - '**update-software.yml'
+      - 'update_software.py'
+      - 'SAMPLE-software-rules.json'
 jobs:
   build:
     runs-on: ${{ matrix.os }}

diff --git a/.github/workflows/regression-test-google-lighthouse-based.yml b/.github/workflows/regression-test-google-lighthouse-based.yml
@@ -3,6 +3,11 @@ on:
   push:
     paths-ignore:
       - '**.md'
+      - 'software-full.json'
+      - 'software-sources.json'
+      - '**update-software.yml'
+      - 'update_software.py'
+      - 'SAMPLE-software-rules.json'
 jobs:
   build:
     runs-on: ${{ matrix.os }}

diff --git a/.github/workflows/regression-test-html.yml b/.github/workflows/regression-test-html.yml
@@ -3,6 +3,11 @@ on:
   push:
     paths-ignore:
       - '**.md'
+      - 'software-full.json'
+      - 'software-sources.json'
+      - '**update-software.yml'
+      - 'update_software.py'
+      - 'SAMPLE-software-rules.json'
 jobs:
   build:
     runs-on: ${{ matrix.os }}

diff --git a/.github/workflows/regression-test-http.yml b/.github/workflows/regression-test-http.yml
@@ -3,6 +3,11 @@ on:
   push:
     paths-ignore:
       - '**.md'
+      - 'software-full.json'
+      - 'software-sources.json'
+      - '**update-software.yml'
+      - 'update_software.py'
+      - 'SAMPLE-software-rules.json'
 jobs:
   build:
     runs-on: ${{ matrix.os }}

diff --git a/.github/workflows/regression-test-pa11y.yml b/.github/workflows/regression-test-pa11y.yml
@@ -3,6 +3,11 @@ on:
   push:
     paths-ignore:
       - '**.md'
+      - 'software-full.json'
+      - 'software-sources.json'
+      - '**update-software.yml'
+      - 'update_software.py'
+      - 'SAMPLE-software-rules.json'
 jobs:
   build:
     runs-on: ${{ matrix.os }}

diff --git a/.github/workflows/regression-test-sitespeed.yml b/.github/workflows/regression-test-sitespeed.yml
@@ -3,6 +3,11 @@ on:
   push:
     paths-ignore:
       - '**.md'
+      - 'software-full.json'
+      - 'software-sources.json'
+      - '**update-software.yml'
+      - 'update_software.py'
+      - 'SAMPLE-software-rules.json'
 jobs:
   build:
     runs-on: ${{ matrix.os }}

diff --git a/.github/workflows/regression-test-software.yml b/.github/workflows/regression-test-software.yml
@@ -3,6 +3,8 @@ on:
   push:
     paths-ignore:
       - '**.md'
+      - 'software-sources.json'
+      - '**update-software.yml'
 jobs:
   build:
     runs-on: ${{ matrix.os }}

diff --git a/.github/workflows/regression-test-standard-files.yml b/.github/workflows/regression-test-standard-files.yml
@@ -3,6 +3,11 @@ on:
   push:
     paths-ignore:
       - '**.md'
+      - 'software-full.json'
+      - 'software-sources.json'
+      - '**update-software.yml'
+      - 'update_software.py'
+      - 'SAMPLE-software-rules.json'
 jobs:
   build:
     runs-on: ${{ matrix.os }}

diff --git a/.github/workflows/regression-test-tracking.yml b/.github/workflows/regression-test-tracking.yml
@@ -7,6 +7,11 @@ on:
   push:
     paths-ignore:
       - '**.md'
+      - 'software-full.json'
+      - 'software-sources.json'
+      - '**update-software.yml'
+      - 'update_software.py'
+      - 'SAMPLE-software-rules.json'
 jobs:
   build:
     runs-on: ${{ matrix.os }}

diff --git a/.github/workflows/regression-test-translations.yml b/.github/workflows/regression-test-translations.yml
@@ -3,6 +3,11 @@ on:
   push:
     paths-ignore:
       - '**.md'
+      - 'software-full.json'
+      - 'software-sources.json'
+      - '**update-software.yml'
+      - 'update_software.py'
+      - 'SAMPLE-software-rules.json'
 jobs:
   build:
     runs-on: ${{ matrix.os }}

diff --git a/.github/workflows/regression-test-webbkoll.yml b/.github/workflows/regression-test-webbkoll.yml
@@ -3,6 +3,11 @@ on:
   push:
     paths-ignore:
       - '**.md'
+      - 'software-full.json'
+      - 'software-sources.json'
+      - '**update-software.yml'
+      - 'update_software.py'
+      - 'SAMPLE-software-rules.json'
 jobs:
   build:
     runs-on: ${{ matrix.os }}

diff --git a/.github/workflows/regression-test-ylt.yml b/.github/workflows/regression-test-ylt.yml
@@ -3,6 +3,11 @@ on:
   push:
     paths-ignore:
       - '**.md'
+      - 'software-full.json'
+      - 'software-sources.json'
+      - '**update-software.yml'
+      - 'update_software.py'
+      - 'SAMPLE-software-rules.json'
 jobs:
   build:
     runs-on: ${{ matrix.os }}

diff --git a/.github/workflows/update-software.yml b/.github/workflows/update-software.yml
@@ -0,0 +1,77 @@
+name: "Software - Update version and security info"
+on:
+  schedule:
+  #  https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#onschedule
+  #  * is a special character in YAML so you have to quote this string
+    - cron: '30 8 14,28 * *'
+  push:
+    paths:
+      - 'software-sources.json'
+      - '**update-software.yml'
+      - '**verify_result.py'
+      - 'update_software.py'
+      - 'SAMPLE-software-rules.json'
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+            matrix:
+                os: [windows-latest]
+                config: ["software_github_adadvisory_database_path=advisory_database"]
+    steps:
+    - name: Check out repository code
+      uses: actions/checkout@v3
+    - name: Setup python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.x' # Version range or exact version of a Python version to use, using SemVer's version range syntax
+        architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified
+    - name: Setup dependencies using pip
+      run: pip install -r requirements.txt
+    - if: ${{ matrix.os == 'ubuntu-latest'  }}
+      name: Setup config (using SAMPLE-config.py as config.py) - LINUX
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: python .github/workflows/verify_result.py -c ${{ matrix.config }},github_api_key=${{ secrets.GITHUB_TOKEN }}
+    - if: ${{ matrix.os == 'windows-latest' }}
+      name: Setup config (using SAMPLE-config.py as config.py) - WINDOWS
+      run: python .github\workflows\verify_result.py -c ${{ matrix.config }},github_api_key=${{ secrets.GITHUB_TOKEN }}
+    - name: Checkout advisory-database repo
+      uses: actions/checkout@v4
+      with:
+        repository: github/advisory-database
+        ref: main
+        path: 
+          advisory_database
+        sparse-checkout: |
+          advisories
+    - if: ${{ matrix.os == 'ubuntu-latest' }}
+      name: Verifing Software(s) - LINUX
+      run: python update_software.py
+    - if: ${{ matrix.os == 'windows-latest' }}
+      name: Verifing Software(s) - WINDOWS
+      run: python update_software.py
+    - if: ${{ matrix.os == 'windows-latest' }}
+      name: Git Fix sort order to Software sources(s) - WINDOWS
+      run: |
+        git config --global user.name 'GitHub Action'
+        git config --global user.email '7h3Rabbit@users.noreply.github.com'
+        git add software-sources.json
+        git commit -m "Added changed software"
+        git push
+    - if: ${{ matrix.os == 'windows-latest' }}
+      name: Git Add any changes to Software(s) - WINDOWS
+      run: |
+        git config --global user.name 'GitHub Action'
+        git config --global user.email '7h3Rabbit@users.noreply.github.com'
+        git add software-full.json
+        git commit -m "Added changed software"
+        git push
+    - if: ${{ matrix.os == 'windows-latest' }}
+      name: Git Add any changes to License(s) - WINDOWS
+      run: |
+        git config --global user.name 'GitHub Action'
+        git config --global user.email '7h3Rabbit@users.noreply.github.com'
+        git add SAMPLE-software-rules.json
+        git commit -m "Added changed licenses"
+        git push        
diff --git a/.github/workflows/verify_result.py b/.github/workflows/verify_result.py
@@ -41,7 +41,11 @@ def prepare_config_file(sample_filename, filename, arguments):
                 value = argument[(index + 1):]
 
                 regex_argument = r'^{0}.*'.format(name)
-                result_argument = r'{0} = {1}'.format(name, value)
+                if value == 'True' or value == 'False' or value == 'None':
+                    result_argument = r'{0} = {1}'.format(name, value)
+                else:
+                    result_argument = r"{0} = '{1}'".format(name, value)
+
 
                 tmp = re.sub(regex_argument, result_argument,
                              tmp, 0, re.MULTILINE)

diff --git a/.gitignore b/.gitignore
@@ -121,3 +121,6 @@ package-lock.json
 package.json
 sitespeed-rules.json
 vnu.jar
+software-unknown-sources.json
+tmp
+cache
diff --git a/SAMPLE-config.py b/SAMPLE-config.py
@@ -51,6 +51,10 @@
 # See https://docs.python.org/3/library/datetime.html#timedelta-objects for possible values
 cache_time_delta = timedelta(hours=1)
 
+# GITHUB API Token, used for calls to github API (to remove call limit)
+github_api_key = None
+
+
 # Tell software test to use stealth mode or not, default is 'True'
 software_use_stealth = True