install deps for windows based with hashes

.github/workflows/build.yml

@@ -89,7 +89,7 @@ jobs:
           python-version: '3.9'
       - name: Verify requirements
         run: |
-          ./travis/compile_requirements.sh ubuntu_requirements
+          ./travis/compile_requirements.sh ubuntu_requirements ubuntu
       - name: Upload requirements to GitHub artifacts
         uses: actions/upload-artifact@v4
         with:
@@ -126,7 +126,7 @@ jobs:
           virtualenv "${HOME}/INSTALL"
       - name: Verify requirements
         run: |
-          ./travis/compile_requirements.sh osx_requirements
+          ./travis/compile_requirements.sh osx_requirements osx
       - name: Upload requirements to GitHub artifacts
         uses: actions/upload-artifact@v4
         with:
@@ -195,6 +195,15 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           python-version: '3.9'
+      - name: Verify requirements
+        run: |
+          .\travis\compile_requirements.bat windows_requirements
+      - name: Upload requirements to GitHub artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: windows-requirements
+          path: windows_requirements/
+          retention-days: 3
       - name: Build installers
         shell: bash
         run: |

appveyor/windows_templates/build_windows_templates.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-#!/usr/bin/env python
 """Script to build windows templates."""
 
 import argparse
@@ -11,54 +10,67 @@
 import subprocess
 import sys
 import time
-
-from typing import Callable
+from typing import Callable, Optional
 
 parser = argparse.ArgumentParser(description="Build windows templates.")
 
 parser.add_argument(
-    "--build_dir", default=r"C:\grrbuild", help="GRR build directory.")
+    "--build_dir", default=r"C:\grrbuild", help="GRR build directory."
+)
 
 parser.add_argument(
     "--grr_src",
     default=r"C:\grrbuild\grr",
-    help="Location of the grr src code. If it doesn't exist "
-    " at this path we'll try to check it out from github.")
+    help=(
+        "Location of the grr src code. If it doesn't exist "
+        " at this path we'll try to check it out from github."
+    ),
+)
 
 parser.add_argument(
     "--output_dir",
     default=r"C:\grrbuild\output",
-    help="Destination directory for the templates.")
+    help="Destination directory for the templates.",
+)
 
 parser.add_argument(
     "--test_repack_install",
     action="store_true",
     default=False,
-    help="Test repacking by calling repack on the template after building,"
-    "then try and install the result. For use by integration tests. If you use "
-    "this option you must run as admin.")
+    help=(
+        "Test repacking by calling repack on the template after building,then"
+        " try and install the result. For use by integration tests. If you use"
+        " this option you must run as admin."
+    ),
+)
 
 parser.add_argument(
     "--wheel_dir",
     default=None,
-    help="A directory that will be passed to pip as the wheel-dir parameter.")
+    help="A directory that will be passed to pip as the wheel-dir parameter.",
+)
 
 parser.add_argument(
     "--expect_service_running",
     dest="expect_service_running",
     action="store_true",
-    help="Triggers whether after installation the GRR service should be "
-    "running or not. Used for testing the installation.")
+    help=(
+        "Triggers whether after installation the GRR service should be "
+        "running or not. Used for testing the installation."
+    ),
+)
 parser.add_argument(
     "--noexpect_service_running",
     dest="expect_service_running",
-    action="store_false")
+    action="store_false",
+)
 parser.set_defaults(expect_service_running=True)
 
 parser.add_argument(
     "--config",
     default="",
-    help="Path to the config file to be used when building templates.")
+    help="Path to the config file to be used when building templates.",
+)
 
 
 args = parser.parse_args()
@@ -79,13 +91,16 @@ def _FileRetryLoop(path: str, f: Callable[[], None]) -> None:
       return
     except OSError as e:
       attempts += 1
-      if (e.errno == errno.EACCES and
-          attempts < _FILE_RETRY_LOOP_RETRY_TIME_SECS):
+      if (
+          e.errno == errno.EACCES
+          and attempts < _FILE_RETRY_LOOP_RETRY_TIME_SECS
+      ):
         # The currently installed GRR process may stick around for a few
         # seconds after the service is terminated (keeping the contents of
         # the installation directory locked).
-        logging.info("Permission-denied error while trying to process %s.",
-                     path)
+        logging.info(
+            "Permission-denied error while trying to process %s.", path
+        )
         time.sleep(1)
       else:
         raise
@@ -99,27 +114,6 @@ def _Rename(src: str, dst: str) -> None:
   _FileRetryLoop(src, lambda: os.rename(src, dst))
 
 
-def _RmTreePseudoTransactional(path: str) -> None:
-  """Removes `path`.
-
-  Makes sure that either `path` is gone or that it is still present as
-  it was.
-
-  Args:
-    path: The path to remove.
-  """
-  temp_path = f"{path}_orphaned_{int(time.time())}"
-  logging.info("Trying to rename %s -> %s.", path, temp_path)
-
-  _Rename(path, temp_path)
-
-  try:
-    logging.info("Trying to remove %s.", temp_path)
-    _RmTree(temp_path)
-  except:  # pylint: disable=bare-except
-    logging.info("Failed to remove %s. Ignoring.", temp_path, exc_info=True)
-
-
 def _VerboseCheckCall(params):
   logging.info("Running: %s", params)
 
@@ -141,8 +135,9 @@ def SetupVars(self):
 
     self.virtualenv64 = os.path.join(args.build_dir, "python_64")
     self.grr_client_build64 = "grr_client_build"
-    self.virtualenv_python64 = os.path.join(self.virtualenv64,
-                                            r"Scripts\python.exe")
+    self.virtualenv_python64 = os.path.join(
+        self.virtualenv64, r"Scripts\python.exe"
+    )
 
     self.git = r"git"
 
@@ -180,55 +175,72 @@ def Clean(self):
   def GitCheckoutGRR(self):
     os.chdir(args.build_dir)
     subprocess.check_call(
-        [self.git, "clone", "https://github.com/google/grr.git"])
+        [self.git, "clone", "https://github.com/google/grr.git"]
+    )
 
   def MakeProtoSdist(self):
     os.chdir(os.path.join(args.grr_src, "grr/proto"))
     subprocess.check_call([
-        self.virtualenv_python64, "setup.py", "sdist", "--formats=zip",
-        "--dist-dir=%s" % args.build_dir
+        self.virtualenv_python64,
+        "setup.py",
+        "sdist",
+        "--formats=zip",
+        "--dist-dir=%s" % args.build_dir,
     ])
     return glob.glob(
         os.path.join(args.build_dir, "grr_response_proto-*.zip")
-    ).pop()
+    ).pop(), os.path.join(args.grr_src, "grr", "proto", "requirements", "windows.txt")
 
   def MakeCoreSdist(self):
     os.chdir(os.path.join(args.grr_src, "grr/core"))
     subprocess.check_call([
-        self.virtualenv_python64, "setup.py", "sdist", "--formats=zip",
-        "--dist-dir=%s" % args.build_dir, "--no-sync-artifacts"
+        self.virtualenv_python64,
+        "setup.py",
+        "sdist",
+        "--formats=zip",
+        "--dist-dir=%s" % args.build_dir,
+        "--no-sync-artifacts",
     ])
     return glob.glob(
         os.path.join(args.build_dir, "grr_response_core-*.zip")
-    ).pop()
+    ).pop(), os.path.join(args.grr_src, "grr", "core", "requirements", "windows.txt")
 
   def MakeClientSdist(self):
     os.chdir(os.path.join(args.grr_src, "grr/client/"))
     subprocess.check_call([
-        self.virtualenv_python64, "setup.py", "sdist", "--formats=zip",
-        "--dist-dir=%s" % args.build_dir
+        self.virtualenv_python64,
+        "setup.py",
+        "sdist",
+        "--formats=zip",
+        "--dist-dir=%s" % args.build_dir,
     ])
     return glob.glob(
         os.path.join(args.build_dir, "grr_response_client-*.zip")
-    ).pop()
+    ).pop(), os.path.join(args.grr_src, "grr", "client", "requirements", "windows.txt")
 
   def MakeClientBuilderSdist(self):
     os.chdir(os.path.join(args.grr_src, "grr/client_builder/"))
     subprocess.check_call([
-        self.virtualenv_python64, "setup.py", "sdist", "--formats=zip",
-        "--dist-dir=%s" % args.build_dir
+        self.virtualenv_python64,
+        "setup.py",
+        "sdist",
+        "--formats=zip",
+        "--dist-dir=%s" % args.build_dir,
     ])
     return glob.glob(
         os.path.join(args.build_dir, "grr_response_client_builder-*.zip")
-    ).pop()
+    ).pop(), os.path.join(args.grr_src, "grr", "client_builder", "requirements", "windows.txt")
 
-  def InstallGRR(self, path):
+  def InstallGRR(self, path: str, requirements_file: Optional[str] = None):
     """Installs GRR."""
+    if requirements_file:
+      install_requirements_cmd = ["pip", "install", "--require-hashes", "-r", requirements_file]
+      subprocess.check_call(install_requirements_cmd)
 
-    cmd64 = ["pip", "install"]
+    cmd64 = ["pip", "install", "--no-deps", "--no-index"]
 
     if args.wheel_dir:
-      cmd64 += ["--no-index", r"--find-links=file:///%s" % args.wheel_dir]
+      cmd64 += [r"--find-links=file:///%s" % args.wheel_dir]
 
     cmd64.append(path)
 
@@ -242,8 +254,12 @@ def BuildTemplates(self):
     """
     if args.config:
       build_args = [
-          "--verbose", "--config", args.config, "build", "--output",
-          args.output_dir
+          "--verbose",
+          "--config",
+          args.config,
+          "build",
+          "--output",
+          args.output_dir,
       ]
     else:
       build_args = ["--verbose", "build", "--output", args.output_dir]
@@ -268,9 +284,11 @@ def _WixToolsPath(self) -> str:
   def _RepackTemplates(self):
     """Repack templates with a dummy config."""
     dummy_config = os.path.join(
-        args.grr_src, "grr/test/grr_response_test/test_data/dummyconfig.yaml")
-    template_amd64 = glob.glob(os.path.join(args.output_dir,
-                                            "*_amd64*.zip")).pop()
+        args.grr_src, "grr/test/grr_response_test/test_data/dummyconfig.yaml"
+    )
+    template_amd64 = glob.glob(
+        os.path.join(args.output_dir, "*_amd64*.zip")
+    ).pop()
 
     fleetspeak_config = os.path.join(
         args.grr_src,
@@ -345,8 +363,9 @@ def _CheckInstallSuccess(self):
       raise RuntimeError("Install failed, no files at: %s" % self.install_path)
 
     try:
-      output = subprocess.check_output(["sc", "query", self.service_name],
-                                       encoding="utf-8")
+      output = subprocess.check_output(
+          ["sc", "query", self.service_name], encoding="utf-8"
+      )
       service_running = "RUNNING" in output
     except subprocess.CalledProcessError as e:
       output = e.output
@@ -370,13 +389,15 @@ def _CheckInstallSuccess(self):
     if self.expect_service_running:
       if not service_running:
         raise RuntimeError(
-            "GRR service not running after install, sc query output: %s" %
-            output)
+            "GRR service not running after install, sc query output: %s"
+            % output
+        )
     else:
       if service_running:
         raise RuntimeError(
             "GRR service running after install with expect_service_running == "
-            "False, sc query output: %s" % output)
+            "False, sc query output: %s" % output
+        )
 
   def _InstallInstallers(self):
     """Install the installer built by RepackTemplates."""
@@ -404,15 +425,15 @@ def Build(self):
 
     if not os.path.exists(args.grr_src):
       self.GitCheckoutGRR()
-    proto_sdist = self.MakeProtoSdist()
-    core_sdist = self.MakeCoreSdist()
-    client_sdist = self.MakeClientSdist()
-    client_builder_sdist = self.MakeClientBuilderSdist()
-
-    self.InstallGRR(proto_sdist)
-    self.InstallGRR(core_sdist)
-    self.InstallGRR(client_sdist)
-    self.InstallGRR(client_builder_sdist)
+    proto_sdist, proto_requirements = self.MakeProtoSdist()
+    core_sdist, core_requirements = self.MakeCoreSdist()
+    client_sdist, client_requirements = self.MakeClientSdist()
+    client_builder_sdist, client_builder_requirements = self.MakeClientBuilderSdist()
+
+    self.InstallGRR(proto_sdist, proto_requirements)
+    self.InstallGRR(core_sdist, core_requirements)
+    self.InstallGRR(client_sdist, client_requirements)
+    self.InstallGRR(client_builder_sdist, client_builder_requirements)
     self.BuildTemplates()
     if args.test_repack_install:
       self._RepackTemplates()

grr/client/requirements.in

@@ -2,4 +2,3 @@ absl-py==1.4.0
 pytsk3==20230125
 libfsntfs-python==20230606
 fleetspeak-client-bin==0.1.13
-xattr==0.9.7

grr/client/requirements/windows.txt

@@ -0,0 +1,30 @@
+#
+# This file is autogenerated by pip-compile with Python 3.9
+# by the following command:
+#
+#    pip-compile --generate-hashes --output-file='windows_requirements\client_requirements.txt' 'D:\a\grr\grr\travis\\..\grr\client\requirements.in'
+#
+absl-py==1.4.0 \
+    --hash=sha256:0d3fe606adfa4f7db64792dd4c7aee4ee0c38ab75dfd353b7a83ed3e957fcb47 \
+    --hash=sha256:d2c244d01048ba476e7c080bd2c6df5e141d211de80223460d5b3b8a2a58433d
+    # via -r D:\a\grr\grr\travis\\..\grr\client\requirements.in
+fleetspeak-client-bin==0.1.13 \
+    --hash=sha256:8cb051c429fb2eade39bef8c0587d61a2935c7875ca22d3db8f74a4a4ec3f8f4 \
+    --hash=sha256:a5d1be73712278e1c4ad275ea714809fb474929262d245d143e0d0101c084ce9 \
+    --hash=sha256:c1b32540c1d18d71813cfa14454ff39bde3f06ae4ee5ec777e952e16cc94b26b \
+    --hash=sha256:ef28f40757726ffd13b931740be156016d7494f2652a18f2772cba664142ce2e
+    # via -r D:\a\grr\grr\travis\\..\grr\client\requirements.in
+libfsntfs-python==20230606 \
+    --hash=sha256:373f0065888ef9db047d12ecc016b20bf077d0a0bfa9cd6306b564b5071ac370 \
+    --hash=sha256:38dc1e5beb31146ca8fc70003bfc9cc15d98c5405a5d5339767bab8a18049e08 \
+    --hash=sha256:5037a66e562ed03ff57b36dcd8666b555dda4dca1e2045d7f5654764182477a0 \
+    --hash=sha256:6998aa3f09ebdfb6434c035987531edccfb9e3ca35c367b488ccd4e96331fd2e \
+    --hash=sha256:c70df92a678553c52ad98c140fb3f6f6aa335a5103489fbc1dccef577475f7d7 \
+    --hash=sha256:cc4a946073920e7d26d8f299d1b4a7b7ea0ab635bccad3ef5269f2976968c185 \
+    --hash=sha256:d16fb849d3f92840b78b13328484096cccb6a587681353b81b1f2b1c79f452a3 \
+    --hash=sha256:f0df820bd6d91103a36664f228fcc8564715bd9c4f446149d90a1e751bfabdff
+    # via -r D:\a\grr\grr\travis\\..\grr\client\requirements.in
+pytsk3==20230125 \
+    --hash=sha256:4406a88490afcc649cd44aa3e8bd5e8b0767822c17c50cf6c6898fbb9605ac42
+    # via -r D:\a\grr\grr\travis\\..\grr\client\requirements.in
+