Merge pull request #14 from browserup/better-page-data

Better page data
browserup · Sep 11, 2023 · 9ae8a64 · 9ae8a64
2 parents 28e8843 + 74d1055
commit 9ae8a64
Show file tree

Hide file tree

Showing 310 changed files with 17,370 additions and 3,216 deletions.
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -1,5 +1,9 @@
 # Contributing
 
+Please contribute upstream--we follow mitmproxy, but they 
+are the source of truth for the proxy in general.
+
+----------------------------------
 As an open source project, mitmproxy welcomes contributions of all forms. If you would like to bring the project
 forward, please consider contributing in the following areas:
 

diff --git a/README.md b/README.md
@@ -33,7 +33,9 @@ code contributions where possible.
 The main changes for our fork are in the folders below/
 
 mitmproxy/addons/browserup/
+
 test/mitmproxy/addons/browserup
+
 clients/
 
 The standard MITMProxy documentation follows below.

diff --git a/UpgradingMitmProxy.MD b/UpgradingMitmProxy.MD
@@ -0,0 +1,34 @@
+The following instructions are used for updating to a new
+version of mitmproxy.
+
+-----------------------------------------
+
+Fix websocket connection missing issue  for our internal websocket:
+
+mitmproxy/addons/browserup/test/mitmproxy/addons/browserupclients/
+
+mitmproxy/proxy/layers/http/__init__.py
+
+
+elif isinstance(command, SendHttp):    if command.connection in self.connections:        conn = self.connections[command.connection]        yield from self.event_to_child(conn, command.event)    else:        self.debug(f"No connection found for key: {command.connection}")
+
+
+Copy:
+
+mitmproxy/addons/browserup/
+test/mitmproxy/addons/browserup
+clients/
+
+Adjust setup.py to have the browserup executable entrypoint:
+setup.py
+
+
+entry_points={    "console_scripts": [        "mitmproxy = mitmproxy.tools.main:mitmproxy",        "mitmdump = mitmproxy.tools.main:mitmdump",        "mitmweb = mitmproxy.tools.main:mitmweb",        "browserup-proxy = mitmproxy.tools.main:browserupproxy",    ],    "pyinstaller40": [        "hook-dirs = mitmproxy.utils.pyinstaller:hook_dirs",    ]},
+
+Add scripts folder:
+
+scripts/browsertime/browser-data.js
+
+Add (falcon, etc) to:
+
+setup.py
diff --git a/browserup-proxy.schema.json b/browserup-proxy.schema.json
@@ -386,66 +386,66 @@
       "PageTiming": {
         "type": "object",
         "properties": {
+          "onContentLoad": {
+            "type": "number",
+            "description": "onContentLoad per the browser"
+          },
+          "onLoad": {
+            "type": "number",
+            "description": "onLoad per the browser"
+          },
           "_firstInputDelay": {
             "type": "number",
             "description": "firstInputDelay from the browser"
           },
-          "_domInteractive": {
+          "_firstPaint": {
             "type": "number",
-            "description": "domInteractive from the browser"
+            "description": "firstPaint from the browser"
           },
           "_cumulativeLayoutShift": {
             "type": "number",
             "description": "cumulativeLayoutShift metric from the browser"
           },
-          "_dns": {
+          "_largestContentfulPaint": {
             "type": "number",
-            "description": "dns lookup time from the browser"
-          },
-          "_href": {
-            "type": "string",
-            "description": "Top level href, including hashtag, etc per the browser"
+            "description": "largestContentfulPaint from the browser"
           },
-          "_firstPaint": {
+          "_domInteractive": {
             "type": "number",
-            "description": "firstPaint from the browser"
+            "description": "domInteractive from the browser"
           },
-          "_largestContentfulPaint": {
+          "_firstContentfulPaint": {
             "type": "number",
-            "description": "largestContentfulPaint from the browser"
+            "description": "firstContentfulPaint from the browser"
           },
-          "_timeToFirstByte": {
+          "_dns": {
             "type": "number",
-            "description": "Time to first byte of the page's first request per the browser"
+            "description": "dns lookup time from the browser"
           },
           "_ssl": {
             "type": "number",
             "description": "Ssl connect time from the browser"
           },
-          "_firstContentfulPaint": {
-            "type": "number",
-            "description": "firstContentfulPaint from the browser"
-          },
-          "onLoad": {
+          "_timeToFirstByte": {
             "type": "number",
-            "description": "onLoad per the browser"
+            "description": "Time to first byte of the page's first request per the browser"
           },
-          "onContentLoad": {
-            "type": "number",
-            "description": "onContentLoad per the browser"
+          "_href": {
+            "type": "string",
+            "description": "Top level href, including hashtag, etc per the browser"
           }
         }
       },
       "NameValuePair": {
         "type": "object",
         "properties": {
-          "value": {
-            "type": "string",
-            "description": "Value to match"
-          },
           "name": {
             "type": "string",
             "description": "Name to match"
+          },
+          "value": {
+            "type": "string",
+            "description": "Value to match"
           }
         }
       },
@@ -567,44 +567,44 @@
       "VerifyResult": {
         "type": "object",
         "properties": {
-          "type": {
-            "type": "string",
-            "description": "Type"
+          "result": {
+            "type": "boolean",
+            "description": "Result True / False"
           },
           "name": {
             "type": "string",
             "description": "Name"
           },
-          "result": {
-            "type": "boolean",
-            "description": "Result True / False"
+          "type": {
+            "type": "string",
+            "description": "Type"
           }
         }
       },
       "Error": {
         "type": "object",
         "properties": {
-          "details": {
-            "type": "string",
-            "description": "Short details of the error"
-          },
           "name": {
             "type": "string",
             "description": "Name of the Error to add. Stored in har under _errors"
+          },
+          "details": {
+            "type": "string",
+            "description": "Short details of the error"
           }
         }
       },
       "Counter": {
         "type": "object",
         "properties": {
+          "name": {
+            "type": "string",
+            "description": "Name of Custom Counter to add to the page under _counters"
+          },
           "value": {
             "type": "number",
             "format": "double",
             "description": "Value for the counter"
-          },
-          "name": {
-            "type": "string",
-            "description": "Name of Custom Counter to add to the page under _counters"
           }
         }
       },
@@ -675,6 +675,36 @@
           }
         }
       },
+      "Action": {
+        "type": "object",
+        "properties": {
+          "name": {
+            "type": "string"
+          },
+          "id": {
+            "type": "string"
+          },
+          "className": {
+            "type": "string"
+          },
+          "tagName": {
+            "type": "string"
+          },
+          "xpath": {
+            "type": "string"
+          },
+          "dataAttributes": {
+            "type": "string"
+          },
+          "formName": {
+            "type": "string"
+          },
+          "content": {
+            "type": "string"
+          }
+        },
+        "additionalProperties": false
+      },
       "PageTimings": {
         "type": "object",
         "required": [
@@ -1079,6 +1109,54 @@
                   "encoding": {
                     "type": "string"
                   },
+                  "_videoBufferedPercent": {
+                    "type": "integer",
+                    "format": "int64",
+                    "minimum": -1,
+                    "default": -1
+                  },
+                  "_videoStallCount": {
+                    "type": "integer",
+                    "format": "int64",
+                    "minimum": -1,
+                    "default": -1
+                  },
+                  "_videoDecodedByteCount": {
+                    "type": "integer",
+                    "format": "int64",
+                    "minimum": -1,
+                    "default": -1
+                  },
+                  "_videoWaitingCount": {
+                    "type": "integer",
+                    "format": "int64",
+                    "minimum": -1,
+                    "default": -1
+                  },
+                  "_videoErrorCount": {
+                    "type": "integer",
+                    "format": "int64",
+                    "minimum": -1,
+                    "default": -1
+                  },
+                  "_videoDroppedFrames": {
+                    "type": "integer",
+                    "format": "int64",
+                    "minimum": -1,
+                    "default": -1
+                  },
+                  "_videoTotalFrames": {
+                    "type": "integer",
+                    "format": "int64",
+                    "minimum": -1,
+                    "default": -1
+                  },
+                  "_videoAudioBytesDecoded": {
+                    "type": "integer",
+                    "format": "int64",
+                    "minimum": -1,
+                    "default": -1
+                  },
                   "comment": {
                     "type": "string"
                   }

diff --git a/clients/README.md b/clients/README.md
@@ -31,6 +31,11 @@ different. So see them, follow this pattern:
 `openapi-generator config-help -g python`
 
 
+* cd to clients folder
+* run the tests (to regen the schema file browserup-proxy.schema.json)
+* run the commands below to generate the clients
+
+
 ## Generate REST API-Docs
 `openapi-generator generate -g markdown -i /Users/ebeland/apps/mitmproxy/browserup-proxy.schema.json -o markdown`
 

diff --git a/clients/build-clients.sh b/clients/build-clients.sh
@@ -6,6 +6,14 @@ DIR="$(dirname "${BASH_SOURCE[0]}")"
 
 # we need to regenerate the schema in the top level dir
 
+rm -rf markdown && npx openapi-generator-cli generate \
+-g markdown -i "${DIR}/../browserup-proxy.schema.json" \
+-o markdown
+
+rm -rf csharp && npx openapi-generator-cli generate \
+--package-name BrowserUpMitmProxyClient \
+-g csharp-netcore -i "${DIR}/../browserup-proxy.schema.json" \
+-o csharp -c config-csharp.yaml
 
 rm -rf java && npx openapi-generator-cli generate \
 --package-name BrowserUpMitmProxyClient \

diff --git a/clients/config-java.yaml b/clients/config-java.yaml
@@ -10,7 +10,7 @@ artifactId: browserup-mitmproxy-client
 # artifactUrl
 
 # artifact version in generated pom.xml. This also becomes part of the generated library's filename (Default: 1.0.0)
-artifactVersion: 1.0.23-SNAPSHOT
+artifactVersion: 1.1.0-SNAPSHOT
 
 #  package for generated models (Default: org.openapitools.client.model)
 modelPackage: com.browserup.proxy-client