Merge pull request #29 from tim-stephenson/develop

Removed dead code

environment.yaml

@@ -6,16 +6,13 @@ dependencies:
   - python >=3.11.0,<3.12
   - plyvel >=1.5.0,<2
   - nodejs >=18.12.1,<19
-  - requests >=2.28.2,<3
   - ipykernel >=6.21.1
   - SQLAlchemy >=2.0.4,<3
   - sqlite >=3.40.1,<4
   - pyright >=1.1.295
-  - scikit-learn >=1.2.1
   - pandas >=1.5.3
   - pandas-stubs >=1.5.3.230214
   - pip :
     # Package that is only on PyPI (or need PyPI version)
     - ipython-sql >=0.5.0
-    - jsbeautifier >=1.14.7
 

main/utils/analyzers_utils.py

@@ -1,9 +1,8 @@
 import importlib
-import itertools
 import logging
 from sqlalchemy.engine import Engine
 from types import ModuleType
-from typing import Any, Dict, List, Set, Tuple, Type
+from typing import Any, Dict, List, Tuple, Type
 from analyzers.analyzer import Analyzer
 import analyzers
 
@@ -69,38 +68,6 @@ def run_analyzers(analyzer_objects : List[Analyzer]) -> None:
     for analyzer  in analyzer_objects:
         analyzer.run_analysis()
 
-def get_all_symmetric_differences(analyzer_objects : List[Analyzer], logger : logging.Logger) -> None:
-    grouped_by_fingerprinting_type : Dict[str, List[Analyzer]] = dict()
-    for analyzer in analyzer_objects:
-        if analyzer.fingerprinting_type() in grouped_by_fingerprinting_type:
-            grouped_by_fingerprinting_type[analyzer.fingerprinting_type()].append(analyzer)
-        else:
-            grouped_by_fingerprinting_type[analyzer.fingerprinting_type()] = [analyzer]
-    for analyzer_subgroup in grouped_by_fingerprinting_type.values():
-        for (analyzer1, analyzer2) in  itertools.combinations(analyzer_subgroup, 2):
-            compare(analyzer1,analyzer2,logger)
-
-def compare(analyzer1 : Analyzer, analyzer2 : Analyzer , logger : logging.Logger) -> None:
-        intersection_classified : Set[Tuple[str,str]] = set(analyzer1.get_analysis_results()).intersection( analyzer2.get_analysis_results() )
-        intersection_domain : Set[Tuple[str,str]] = set(analyzer1.analysis_domain()).intersection( analyzer2.analysis_domain() ) 
-
-        logger.info(f"""
-        Fingerprinting method: {analyzer1.fingerprinting_type()}, 
-        classified by {analyzer1.analysis_name()} : {len(analyzer1.get_analysis_results())} / {analyzer1.analysis_domain_size()}
-        classified by {analyzer2.analysis_name()} : {len(analyzer2.get_analysis_results())} / {analyzer2.analysis_domain_size()}
-        intersection : {len(intersection_classified)} / {len(intersection_domain)}
-        """)
-
-
-def load_cache(analyzer_objects : List[Analyzer], cached_results : Dict[str, List[Tuple[str, str]]]) -> None:
-    for analyzer in analyzer_objects:
-        if analyzer.analysis_name() in cached_results:
-            analyzer.set_analysis_results(cached_results[analyzer.analysis_name()])
-
-
-def store_to_cache(analyzer_objects : List[Analyzer]) -> Dict[str, List[Tuple[str, str]]]:
-    return {analyzer.analysis_name() : analyzer.get_analysis_results() for analyzer in analyzer_objects }
-
 
 
 

main/utils/grep_utils.py

@@ -10,11 +10,6 @@ def grepForKeywords(keywords : List[str],  s : str) -> Set[str]:
             results.add(keyword)
     return results
 
-# def compile_regex_from_keywords(keywords : List[str]) -> re.Pattern[str]:
-#     pattern: str = "|".join(map(lambda keyword : "(" + re.escape(keyword) + ")", keywords))
-#     return re.compile(pattern)
-
-
 # un-escape characters escaped via any of the three methods:
 # \xXX                                                       i.e.    \x4E = N          \x21 = !
 # \uXXXX                                                     i.e.    \u265A = ♚        \u269B = ⚛

main/utils/into_table_utils.py

@@ -62,14 +62,6 @@ def analyzerObjects_to_dataframe(analyzer_objects : List[Analyzer]) -> pandas.Da
             df.at[value, analyzer.analysis_name()] = 1 #type: ignore
     return df
 
-def dataframe_to_analyzerObjects(analyzer_objects : List[Analyzer], df : pandas.DataFrame) -> None:
-    for analyzer in analyzer_objects:
-        lst : List[Tuple[str,str]] = []
-        for index, value in df[analyzer.analysis_name()].items(): #type: ignore
-            if value:
-                lst.append(index) #type: ignore
-        analyzer.set_analysis_results(lst)
-
 
 # merge dataframe new and previous, which have the same rows but potentially different columns
 # When a column is shared, the items in previous are discarded and the items in new are used

main/view_results.py

@@ -27,8 +27,8 @@
     parser.add_argument('--leveldb', type=str, action='store',
         help='Name of LevelDB created by OpenWPM', default='leveldb')
     parser.add_argument('--dump_source_code', action='store_true',
-        help="""create 3 folders for each venn diagram area, and dump the source code from the levelDB which 
-        corresponds with the (visit_id,script_url)""", default=False)
+        help="""create folders with the source code (prettified) for each portion of the venn diagram between the analyzers' results,
+        so long at least one of the analyzers classified the (visit_id,script_url) pair as preforming fingerprinting""", default=False)
     parser.add_argument("analyzers_names", nargs="+",
         help="List of analyzers_names to compare. At least one must provided.")
 

node/package.json

@@ -1,25 +1,9 @@
 {
-  "name": "node",
-  "version": "1.0.0",
-  "description": "",
-  "main": "index.js",
-  "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1",
-    "start": "npm run build && npm run execute",
-    "build": "tsc",
-    "execute": "node build/index.js",
+   "scripts": {
     "prettier": "prettier"
   },
-  "author": "",
-  "license": "ISC",
   "dependencies": {
-    "acorn": "^8.8.1",
     "prettier": "^2.8.4"
   },
-  "type": "module",
-  "devDependencies": {
-    "typescript": "^4.9.3",
-    "@tsconfig/node16": "^1.0.3",
-    "@types/node": "^18.11.11"
-  }
+  "type": "module"
 }