[ci] Comment runtime changes on PRs

This implements apache#12781 but needs to wait for apache#12695 before it can merge

ci/scripts/git_utils.py

@@ -31,24 +31,35 @@ def compress_query(query: str) -> str:
     return query
 
 
-def post(url: str, body: Optional[Any] = None, auth: Optional[Tuple[str, str]] = None):
-    print(f"Requesting POST to", url, "with", body)
+def _request(
+    url: str, method: str, body: Optional[Any] = None, auth: Optional[Tuple[str, str]] = None
+):
+    print(f"Requesting {method} to", url, "with", body)
     headers = {}
-    req = request.Request(url, headers=headers, method="POST")
+    req = request.Request(url, headers=headers, method=method)
     if auth is not None:
         auth_str = base64.b64encode(f"{auth[0]}:{auth[1]}".encode())
         req.add_header("Authorization", f"Basic {auth_str.decode()}")
 
     if body is None:
-        body = ""
+        with request.urlopen(req) as response:
+            return response.read()
+    else:
+        req.add_header("Content-Type", "application/json; charset=utf-8")
+        data = json.dumps(body)
+        data = data.encode("utf-8")
+        req.add_header("Content-Length", len(data))
+
+        with request.urlopen(req, data) as response:
+            return response.read()
+
+
+def post(url: str, body: Optional[Any] = None, auth: Optional[Tuple[str, str]] = None):
+    return _request(url=url, method="POST", body=body, auth=auth)
 
-    req.add_header("Content-Type", "application/json; charset=utf-8")
-    data = json.dumps(body)
-    data = data.encode("utf-8")
-    req.add_header("Content-Length", len(data))
 
-    with request.urlopen(req, data) as response:
-        return response.read()
+def get(url: str):
+    return _request(url=url, method="GET")
 
 
 class GitHubRepo:

ci/scripts/github_ci_runtime_bot.py

@@ -0,0 +1,70 @@
+import json
+import statistics
+from typing import List, Dict, Any
+from git_utils import get
+import scipy.stats
+import logging
+
+MAIN_INFO_URL = "https://ci.tlcpack.ai/blue/rest/organizations/jenkins/pipelines/tvm/runs/?branch=main&start=0&limit=26"
+
+
+def fetch_past_build_times_s() -> List[float]:
+    data = get(MAIN_INFO_URL).decode()
+    data = json.loads(data)
+    build_times_s = []
+    logging.info(f"Fetched {len(data)} builds from main")
+    for item in data:
+        # Only look at completed builds
+        if not can_use_build(item):
+            logging.info("Skipping failed build")
+            continue
+
+        duration = item["durationInMillis"]
+        build_times_s.append(duration / 1000.0)
+
+    return build_times_s
+
+
+def can_use_build(build: Dict[str, Any]):
+    return build["state"] == "FINISHED" and build["result"] == "SUCCESS"
+
+
+def fetch_build_time_s(branch: str, build: str) -> float:
+    build = int(build)
+    info_url = f"https://ci.tlcpack.ai/blue/rest/organizations/jenkins/pipelines/tvm/runs/?branch={branch}&start=0&limit=25"
+    data = get(info_url).decode()
+    data = json.loads(data)
+
+    for item in data:
+        if item["id"] == str(build):
+            if can_use_build(item):
+                return item["durationInMillis"] / 1000.0
+            else:
+                raise RuntimeError(
+                    f"Found build for {branch} with {build} but cannot use it: {item}"
+                )
+
+    raise RuntimeError(f"Unable to find branch {branch} with {build} in {data}")
+
+
+def ci_runtime_comment(branch: str, build: str) -> str:
+    logging.info(f"Calculating CI runtime for {branch} with {build}")
+    main_build_times_s = fetch_past_build_times_s()
+    x = statistics.mean(main_build_times_s)
+    logging.info(f"Sample mean from main: {x}")
+    current_build_time_s = fetch_build_time_s(branch=branch, build=build)
+    build_url = (
+        f"https://ci.tlcpack.ai/blue/organizations/jenkins/tvm/detail/{branch}/{build}/pipeline"
+    )
+    res = scipy.stats.ttest_1samp(main_build_times_s, current_build_time_s)
+    logging.info(f"t-stats: {res}")
+    change = -(x - current_build_time_s) / x * 100.0
+    change = round(change, 2)
+    if res.pvalue < 0.05:
+        return f"This PR **significantly changed [CI runtime]({build_url}): {change}%**"
+    else:
+        return f"This PR had no significant effect on [CI runtime]({build_url}): {change}%"
+
+
+a = ci_runtime_comment(branch="PR-12783", build="2")
+print(a)