install deps for windows based with hashes

.github/workflows/build.yml

@@ -89,7 +89,7 @@ jobs:
           python-version: '3.9'
       - name: Verify requirements
         run: |
-          ./travis/compile_requirements.sh ubuntu_requirements
+          ./travis/compile_requirements.sh ubuntu_requirements ubuntu
       - name: Upload requirements to GitHub artifacts
         uses: actions/upload-artifact@v4
         with:
@@ -195,6 +195,15 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           python-version: '3.9'
+      - name: Verify requirements
+        run: |
+          .\travis\compile_requirements.bat windows_requirements
+      - name: Upload requirements to GitHub artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: windows-requirements
+          path: windows_requirements/
+          retention-days: 3
       - name: Build installers
         shell: bash
         run: |

appveyor/windows_templates/build_windows_templates.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-#!/usr/bin/env python
 """Script to build windows templates."""
 
 import argparse
@@ -11,54 +10,67 @@
 import subprocess
 import sys
 import time
-
-from typing import Callable
+from typing import Callable, Optional
 
 parser = argparse.ArgumentParser(description="Build windows templates.")
 
 parser.add_argument(
-    "--build_dir", default=r"C:\grrbuild", help="GRR build directory.")
+    "--build_dir", default=r"C:\grrbuild", help="GRR build directory."
+)
 
 parser.add_argument(
     "--grr_src",
     default=r"C:\grrbuild\grr",
-    help="Location of the grr src code. If it doesn't exist "
-    " at this path we'll try to check it out from github.")
+    help=(
+        "Location of the grr src code. If it doesn't exist "
+        " at this path we'll try to check it out from github."
+    ),
+)
 
 parser.add_argument(
     "--output_dir",
     default=r"C:\grrbuild\output",
-    help="Destination directory for the templates.")
+    help="Destination directory for the templates.",
+)
 
 parser.add_argument(
     "--test_repack_install",
     action="store_true",
     default=False,
-    help="Test repacking by calling repack on the template after building,"
-    "then try and install the result. For use by integration tests. If you use "
-    "this option you must run as admin.")
+    help=(
+        "Test repacking by calling repack on the template after building,then"
+        " try and install the result. For use by integration tests. If you use"
+        " this option you must run as admin."
+    ),
+)
 
 parser.add_argument(
     "--wheel_dir",
     default=None,
-    help="A directory that will be passed to pip as the wheel-dir parameter.")
+    help="A directory that will be passed to pip as the wheel-dir parameter.",
+)
 
 parser.add_argument(
     "--expect_service_running",
     dest="expect_service_running",
     action="store_true",
-    help="Triggers whether after installation the GRR service should be "
-    "running or not. Used for testing the installation.")
+    help=(
+        "Triggers whether after installation the GRR service should be "
+        "running or not. Used for testing the installation."
+    ),
+)
 parser.add_argument(
     "--noexpect_service_running",
     dest="expect_service_running",
-    action="store_false")
+    action="store_false",
+)
 parser.set_defaults(expect_service_running=True)
 
 parser.add_argument(
     "--config",
     default="",
-    help="Path to the config file to be used when building templates.")
+    help="Path to the config file to be used when building templates.",
+)
 
 
 args = parser.parse_args()
@@ -79,13 +91,16 @@ def _FileRetryLoop(path: str, f: Callable[[], None]) -> None:
       return
     except OSError as e:
       attempts += 1
-      if (e.errno == errno.EACCES and
-          attempts < _FILE_RETRY_LOOP_RETRY_TIME_SECS):
+      if (
+          e.errno == errno.EACCES
+          and attempts < _FILE_RETRY_LOOP_RETRY_TIME_SECS
+      ):
         # The currently installed GRR process may stick around for a few
         # seconds after the service is terminated (keeping the contents of
         # the installation directory locked).
-        logging.info("Permission-denied error while trying to process %s.",
-                     path)
+        logging.info(
+            "Permission-denied error while trying to process %s.", path
+        )
         time.sleep(1)
       else:
         raise
@@ -99,27 +114,6 @@ def _Rename(src: str, dst: str) -> None:
   _FileRetryLoop(src, lambda: os.rename(src, dst))
 
 
-def _RmTreePseudoTransactional(path: str) -> None:
-  """Removes `path`.
-
-  Makes sure that either `path` is gone or that it is still present as
-  it was.
-
-  Args:
-    path: The path to remove.
-  """
-  temp_path = f"{path}_orphaned_{int(time.time())}"
-  logging.info("Trying to rename %s -> %s.", path, temp_path)
-
-  _Rename(path, temp_path)
-
-  try:
-    logging.info("Trying to remove %s.", temp_path)
-    _RmTree(temp_path)
-  except:  # pylint: disable=bare-except
-    logging.info("Failed to remove %s. Ignoring.", temp_path, exc_info=True)
-
-
 def _VerboseCheckCall(params):
   logging.info("Running: %s", params)
 
@@ -141,8 +135,9 @@ def SetupVars(self):
 
     self.virtualenv64 = os.path.join(args.build_dir, "python_64")
     self.grr_client_build64 = "grr_client_build"
-    self.virtualenv_python64 = os.path.join(self.virtualenv64,
-                                            r"Scripts\python.exe")
+    self.virtualenv_python64 = os.path.join(
+        self.virtualenv64, r"Scripts\python.exe"
+    )
 
     self.git = r"git"
 
@@ -180,55 +175,72 @@ def Clean(self):
   def GitCheckoutGRR(self):
     os.chdir(args.build_dir)
     subprocess.check_call(
-        [self.git, "clone", "https://github.com/google/grr.git"])
+        [self.git, "clone", "https://github.com/google/grr.git"]
+    )
 
   def MakeProtoSdist(self):
     os.chdir(os.path.join(args.grr_src, "grr/proto"))
     subprocess.check_call([
-        self.virtualenv_python64, "setup.py", "sdist", "--formats=zip",
-        "--dist-dir=%s" % args.build_dir
+        self.virtualenv_python64,
+        "setup.py",
+        "sdist",
+        "--formats=zip",
+        "--dist-dir=%s" % args.build_dir,
     ])
     return glob.glob(
         os.path.join(args.build_dir, "grr_response_proto-*.zip")
-    ).pop()
+    ).pop(), os.path.join(args.grr_src, "grr", "proto", "requirements", "windows.txt")
 
   def MakeCoreSdist(self):
     os.chdir(os.path.join(args.grr_src, "grr/core"))
     subprocess.check_call([
-        self.virtualenv_python64, "setup.py", "sdist", "--formats=zip",
-        "--dist-dir=%s" % args.build_dir, "--no-sync-artifacts"
+        self.virtualenv_python64,
+        "setup.py",
+        "sdist",
+        "--formats=zip",
+        "--dist-dir=%s" % args.build_dir,
+        "--no-sync-artifacts",
     ])
     return glob.glob(
         os.path.join(args.build_dir, "grr_response_core-*.zip")
-    ).pop()
+    ).pop(), os.path.join(args.grr_src, "grr", "core", "requirements", "windows.txt")
 
   def MakeClientSdist(self):
     os.chdir(os.path.join(args.grr_src, "grr/client/"))
     subprocess.check_call([
-        self.virtualenv_python64, "setup.py", "sdist", "--formats=zip",
-        "--dist-dir=%s" % args.build_dir
+        self.virtualenv_python64,
+        "setup.py",
+        "sdist",
+        "--formats=zip",
+        "--dist-dir=%s" % args.build_dir,
     ])
     return glob.glob(
         os.path.join(args.build_dir, "grr_response_client-*.zip")
-    ).pop()
+    ).pop(), os.path.join(args.grr_src, "grr", "client", "requirements", "windows.txt")
 
   def MakeClientBuilderSdist(self):
     os.chdir(os.path.join(args.grr_src, "grr/client_builder/"))
     subprocess.check_call([
-        self.virtualenv_python64, "setup.py", "sdist", "--formats=zip",
-        "--dist-dir=%s" % args.build_dir
+        self.virtualenv_python64,
+        "setup.py",
+        "sdist",
+        "--formats=zip",
+        "--dist-dir=%s" % args.build_dir,
     ])
     return glob.glob(
         os.path.join(args.build_dir, "grr_response_client_builder-*.zip")
-    ).pop()
+    ).pop(), os.path.join(args.grr_src, "grr", "client_builder", "requirements", "windows.txt")
 
-  def InstallGRR(self, path):
+  def InstallGRR(self, path: str, requirements_file: Optional[str] = None):
     """Installs GRR."""
+    if requirements_file:
+      install_requirements_cmd = ["pip", "install", "--require-hashes", "-r", requirements_file]
+      subprocess.check_call(install_requirements_cmd)
 
-    cmd64 = ["pip", "install"]
+    cmd64 = ["pip", "install", "--no-deps", "--no-index"]
 
     if args.wheel_dir:
-      cmd64 += ["--no-index", r"--find-links=file:///%s" % args.wheel_dir]
+      cmd64 += [r"--find-links=file:///%s" % args.wheel_dir]
 
     cmd64.append(path)
 
@@ -242,8 +254,12 @@ def BuildTemplates(self):
     """
     if args.config:
       build_args = [
-          "--verbose", "--config", args.config, "build", "--output",
-          args.output_dir
+          "--verbose",
+          "--config",
+          args.config,
+          "build",
+          "--output",
+          args.output_dir,
       ]
     else:
       build_args = ["--verbose", "build", "--output", args.output_dir]
@@ -268,9 +284,11 @@ def _WixToolsPath(self) -> str:
   def _RepackTemplates(self):
     """Repack templates with a dummy config."""
     dummy_config = os.path.join(
-        args.grr_src, "grr/test/grr_response_test/test_data/dummyconfig.yaml")
-    template_amd64 = glob.glob(os.path.join(args.output_dir,
-                                            "*_amd64*.zip")).pop()
+        args.grr_src, "grr/test/grr_response_test/test_data/dummyconfig.yaml"
+    )
+    template_amd64 = glob.glob(
+        os.path.join(args.output_dir, "*_amd64*.zip")
+    ).pop()
 
     fleetspeak_config = os.path.join(
         args.grr_src,
@@ -345,8 +363,9 @@ def _CheckInstallSuccess(self):
       raise RuntimeError("Install failed, no files at: %s" % self.install_path)
 
     try:
-      output = subprocess.check_output(["sc", "query", self.service_name],
-                                       encoding="utf-8")
+      output = subprocess.check_output(
+          ["sc", "query", self.service_name], encoding="utf-8"
+      )
       service_running = "RUNNING" in output
     except subprocess.CalledProcessError as e:
       output = e.output
@@ -370,13 +389,15 @@ def _CheckInstallSuccess(self):
     if self.expect_service_running:
       if not service_running:
         raise RuntimeError(
-            "GRR service not running after install, sc query output: %s" %
-            output)
+            "GRR service not running after install, sc query output: %s"
+            % output
+        )
     else:
       if service_running:
         raise RuntimeError(
             "GRR service running after install with expect_service_running == "
-            "False, sc query output: %s" % output)
+            "False, sc query output: %s" % output
+        )
 
   def _InstallInstallers(self):
     """Install the installer built by RepackTemplates."""
@@ -404,15 +425,15 @@ def Build(self):
 
     if not os.path.exists(args.grr_src):
       self.GitCheckoutGRR()
-    proto_sdist = self.MakeProtoSdist()
-    core_sdist = self.MakeCoreSdist()
-    client_sdist = self.MakeClientSdist()
-    client_builder_sdist = self.MakeClientBuilderSdist()
-
-    self.InstallGRR(proto_sdist)
-    self.InstallGRR(core_sdist)
-    self.InstallGRR(client_sdist)
-    self.InstallGRR(client_builder_sdist)
+    proto_sdist, proto_requirements = self.MakeProtoSdist()
+    core_sdist, core_requirements = self.MakeCoreSdist()
+    client_sdist, client_requirements = self.MakeClientSdist()
+    client_builder_sdist, client_builder_requirements = self.MakeClientBuilderSdist()
+
+    self.InstallGRR(proto_sdist, proto_requirements)
+    self.InstallGRR(core_sdist, core_requirements)
+    self.InstallGRR(client_sdist, client_requirements)
+    self.InstallGRR(client_builder_sdist, client_builder_requirements)
     self.BuildTemplates()
     if args.test_repack_install:
       self._RepackTemplates()

grr/server/requirements_lin.in

@@ -0,0 +1,18 @@
+google-api-python-client==1.12.11
+google-auth==2.23.3
+google-cloud-storage==2.13.0
+google-cloud-pubsub==2.18.4
+ipython==7.15.0
+Jinja2==3.1.2
+pexpect==4.8.0
+portpicker==1.6.0b1
+protobuf==4.25.3
+prometheus_client==0.16.0
+pyjwt==2.6.0
+pyOpenSSL==21.0.0 
+python-crontab==2.5.1
+python-debian==0.1.49
+Werkzeug==2.1.2
+# TODO: We currently release fleetspeak-server-bin packages
+# for Linux only.
+fleetspeak-server-bin==0.1.13

travis/compile_requirements.bat

@@ -0,0 +1,20 @@
+
+set DIR=%~dp0
+
+set OUT_FOLDER=%1
+
+
+mkdir "%OUT_FOLDER%"
+
+python -m pip install --require-hashes -r "%DIR%\base_tooling_requirements.txt"
+
+python -m piptools compile --generate-hashes "%DIR%\..\api_client\python\requirements.in" -o "%OUT_FOLDER%\api_client_requirements.txt"
+python -m piptools compile --generate-hashes "%DIR%\..\grr\core\requirements.in" -o "%OUT_FOLDER%\core_requirements.txt"
+python -m piptools compile --generate-hashes "%DIR%\..\grr\server\requirements.in" -o "%OUT_FOLDER%\server_requirements.txt"
+python -m piptools compile --generate-hashes "%DIR%\..\grr\proto\requirements.in" -o "%OUT_FOLDER%\proto_requirements.txt"
+python -m piptools compile --generate-hashes "%DIR%\..\grr\client\requirements.in" -o "%OUT_FOLDER%\client_requirements.txt"
+python -m piptools compile --generate-hashes "%DIR%\..\grr\client\requirements_win.in" -o "%OUT_FOLDER%\client_requirements_win.txt"
+python -m piptools compile --generate-hashes "%DIR%\..\grr\client_builder\requirements.in" -o "%OUT_FOLDER%\client_builder_requirements.txt"
+python -m piptools compile --generate-hashes "%DIR%\..\grr\test\requirements.in" -o "%OUT_FOLDER%\test_requirements.txt"
+
+