From 02080a3bc745e2a5a2fef4f958b0121c6ed74903 Mon Sep 17 00:00:00 2001 From: Susanne Westphal Date: Wed, 19 Jun 2024 13:32:01 +0000 Subject: [PATCH] install deps for windows based with hashes --- .github/workflows/build.yml | 9 + .../build_windows_templates.py | 175 ++++++++++-------- grr/server/requirements_lin.in | 18 ++ travis/compile_requirements.bat | 20 ++ 4 files changed, 145 insertions(+), 77 deletions(-) create mode 100644 grr/server/requirements_lin.in create mode 100755 travis/compile_requirements.bat diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bc037c96f..6269617b0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -195,6 +195,15 @@ jobs: - uses: actions/setup-python@v5 with: python-version: '3.9' + - name: Verify requirements + run: | + .\travis\compile_requirements.bat windows_requirements + - name: Upload requirements to GitHub artifacts + uses: actions/upload-artifact@v4 + with: + name: windows-requirements + path: windows_requirements/ + retention-days: 3 - name: Build installers shell: bash run: | diff --git a/appveyor/windows_templates/build_windows_templates.py b/appveyor/windows_templates/build_windows_templates.py index 456ae19f3..8860d3370 100644 --- a/appveyor/windows_templates/build_windows_templates.py +++ b/appveyor/windows_templates/build_windows_templates.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -#!/usr/bin/env python """Script to build windows templates.""" import argparse @@ -11,54 +10,67 @@ import subprocess import sys import time - -from typing import Callable +from typing import Callable, Optional parser = argparse.ArgumentParser(description="Build windows templates.") parser.add_argument( - "--build_dir", default=r"C:\grrbuild", help="GRR build directory.") + "--build_dir", default=r"C:\grrbuild", help="GRR build directory." +) parser.add_argument( "--grr_src", default=r"C:\grrbuild\grr", - help="Location of the grr src code. If it doesn't exist " - " at this path we'll try to check it out from github.") + help=( + "Location of the grr src code. If it doesn't exist " + " at this path we'll try to check it out from github." + ), +) parser.add_argument( "--output_dir", default=r"C:\grrbuild\output", - help="Destination directory for the templates.") + help="Destination directory for the templates.", +) parser.add_argument( "--test_repack_install", action="store_true", default=False, - help="Test repacking by calling repack on the template after building," - "then try and install the result. For use by integration tests. If you use " - "this option you must run as admin.") + help=( + "Test repacking by calling repack on the template after building,then" + " try and install the result. For use by integration tests. If you use" + " this option you must run as admin." + ), +) parser.add_argument( "--wheel_dir", default=None, - help="A directory that will be passed to pip as the wheel-dir parameter.") + help="A directory that will be passed to pip as the wheel-dir parameter.", +) parser.add_argument( "--expect_service_running", dest="expect_service_running", action="store_true", - help="Triggers whether after installation the GRR service should be " - "running or not. Used for testing the installation.") + help=( + "Triggers whether after installation the GRR service should be " + "running or not. Used for testing the installation." + ), +) parser.add_argument( "--noexpect_service_running", dest="expect_service_running", - action="store_false") + action="store_false", +) parser.set_defaults(expect_service_running=True) parser.add_argument( "--config", default="", - help="Path to the config file to be used when building templates.") + help="Path to the config file to be used when building templates.", +) args = parser.parse_args() @@ -79,13 +91,16 @@ def _FileRetryLoop(path: str, f: Callable[[], None]) -> None: return except OSError as e: attempts += 1 - if (e.errno == errno.EACCES and - attempts < _FILE_RETRY_LOOP_RETRY_TIME_SECS): + if ( + e.errno == errno.EACCES + and attempts < _FILE_RETRY_LOOP_RETRY_TIME_SECS + ): # The currently installed GRR process may stick around for a few # seconds after the service is terminated (keeping the contents of # the installation directory locked). - logging.info("Permission-denied error while trying to process %s.", - path) + logging.info( + "Permission-denied error while trying to process %s.", path + ) time.sleep(1) else: raise @@ -99,27 +114,6 @@ def _Rename(src: str, dst: str) -> None: _FileRetryLoop(src, lambda: os.rename(src, dst)) -def _RmTreePseudoTransactional(path: str) -> None: - """Removes `path`. - - Makes sure that either `path` is gone or that it is still present as - it was. - - Args: - path: The path to remove. - """ - temp_path = f"{path}_orphaned_{int(time.time())}" - logging.info("Trying to rename %s -> %s.", path, temp_path) - - _Rename(path, temp_path) - - try: - logging.info("Trying to remove %s.", temp_path) - _RmTree(temp_path) - except: # pylint: disable=bare-except - logging.info("Failed to remove %s. Ignoring.", temp_path, exc_info=True) - - def _VerboseCheckCall(params): logging.info("Running: %s", params) @@ -141,8 +135,9 @@ def SetupVars(self): self.virtualenv64 = os.path.join(args.build_dir, "python_64") self.grr_client_build64 = "grr_client_build" - self.virtualenv_python64 = os.path.join(self.virtualenv64, - r"Scripts\python.exe") + self.virtualenv_python64 = os.path.join( + self.virtualenv64, r"Scripts\python.exe" + ) self.git = r"git" @@ -180,55 +175,72 @@ def Clean(self): def GitCheckoutGRR(self): os.chdir(args.build_dir) subprocess.check_call( - [self.git, "clone", "https://github.com/google/grr.git"]) + [self.git, "clone", "https://github.com/google/grr.git"] + ) def MakeProtoSdist(self): os.chdir(os.path.join(args.grr_src, "grr/proto")) subprocess.check_call([ - self.virtualenv_python64, "setup.py", "sdist", "--formats=zip", - "--dist-dir=%s" % args.build_dir + self.virtualenv_python64, + "setup.py", + "sdist", + "--formats=zip", + "--dist-dir=%s" % args.build_dir, ]) return glob.glob( os.path.join(args.build_dir, "grr_response_proto-*.zip") - ).pop() + ).pop(), os.path.join(args.grr_src, "grr", "proto", "requirements", "windows.txt") def MakeCoreSdist(self): os.chdir(os.path.join(args.grr_src, "grr/core")) subprocess.check_call([ - self.virtualenv_python64, "setup.py", "sdist", "--formats=zip", - "--dist-dir=%s" % args.build_dir, "--no-sync-artifacts" + self.virtualenv_python64, + "setup.py", + "sdist", + "--formats=zip", + "--dist-dir=%s" % args.build_dir, + "--no-sync-artifacts", ]) return glob.glob( os.path.join(args.build_dir, "grr_response_core-*.zip") - ).pop() + ).pop(), os.path.join(args.grr_src, "grr", "core", "requirements", "windows.txt") def MakeClientSdist(self): os.chdir(os.path.join(args.grr_src, "grr/client/")) subprocess.check_call([ - self.virtualenv_python64, "setup.py", "sdist", "--formats=zip", - "--dist-dir=%s" % args.build_dir + self.virtualenv_python64, + "setup.py", + "sdist", + "--formats=zip", + "--dist-dir=%s" % args.build_dir, ]) return glob.glob( os.path.join(args.build_dir, "grr_response_client-*.zip") - ).pop() + ).pop(), os.path.join(args.grr_src, "grr", "client", "requirements", "windows.txt") def MakeClientBuilderSdist(self): os.chdir(os.path.join(args.grr_src, "grr/client_builder/")) subprocess.check_call([ - self.virtualenv_python64, "setup.py", "sdist", "--formats=zip", - "--dist-dir=%s" % args.build_dir + self.virtualenv_python64, + "setup.py", + "sdist", + "--formats=zip", + "--dist-dir=%s" % args.build_dir, ]) return glob.glob( os.path.join(args.build_dir, "grr_response_client_builder-*.zip") - ).pop() + ).pop(), os.path.join(args.grr_src, "grr", "client_builder", "requirements", "windows.txt") - def InstallGRR(self, path): + def InstallGRR(self, path: str, requirements_file: Optional[str] = None): """Installs GRR.""" + if requirements_file: + install_requirements_cmd = ["pip", "install", "--require-hashes", "-r", requirements_file] + subprocess.check_call(install_requirements_cmd) - cmd64 = ["pip", "install"] + cmd64 = ["pip", "install", "--no-deps", "--no-index"] if args.wheel_dir: - cmd64 += ["--no-index", r"--find-links=file:///%s" % args.wheel_dir] + cmd64 += [r"--find-links=file:///%s" % args.wheel_dir] cmd64.append(path) @@ -242,8 +254,12 @@ def BuildTemplates(self): """ if args.config: build_args = [ - "--verbose", "--config", args.config, "build", "--output", - args.output_dir + "--verbose", + "--config", + args.config, + "build", + "--output", + args.output_dir, ] else: build_args = ["--verbose", "build", "--output", args.output_dir] @@ -268,9 +284,11 @@ def _WixToolsPath(self) -> str: def _RepackTemplates(self): """Repack templates with a dummy config.""" dummy_config = os.path.join( - args.grr_src, "grr/test/grr_response_test/test_data/dummyconfig.yaml") - template_amd64 = glob.glob(os.path.join(args.output_dir, - "*_amd64*.zip")).pop() + args.grr_src, "grr/test/grr_response_test/test_data/dummyconfig.yaml" + ) + template_amd64 = glob.glob( + os.path.join(args.output_dir, "*_amd64*.zip") + ).pop() fleetspeak_config = os.path.join( args.grr_src, @@ -345,8 +363,9 @@ def _CheckInstallSuccess(self): raise RuntimeError("Install failed, no files at: %s" % self.install_path) try: - output = subprocess.check_output(["sc", "query", self.service_name], - encoding="utf-8") + output = subprocess.check_output( + ["sc", "query", self.service_name], encoding="utf-8" + ) service_running = "RUNNING" in output except subprocess.CalledProcessError as e: output = e.output @@ -370,13 +389,15 @@ def _CheckInstallSuccess(self): if self.expect_service_running: if not service_running: raise RuntimeError( - "GRR service not running after install, sc query output: %s" % - output) + "GRR service not running after install, sc query output: %s" + % output + ) else: if service_running: raise RuntimeError( "GRR service running after install with expect_service_running == " - "False, sc query output: %s" % output) + "False, sc query output: %s" % output + ) def _InstallInstallers(self): """Install the installer built by RepackTemplates.""" @@ -404,15 +425,15 @@ def Build(self): if not os.path.exists(args.grr_src): self.GitCheckoutGRR() - proto_sdist = self.MakeProtoSdist() - core_sdist = self.MakeCoreSdist() - client_sdist = self.MakeClientSdist() - client_builder_sdist = self.MakeClientBuilderSdist() - - self.InstallGRR(proto_sdist) - self.InstallGRR(core_sdist) - self.InstallGRR(client_sdist) - self.InstallGRR(client_builder_sdist) + proto_sdist, proto_requirements = self.MakeProtoSdist() + core_sdist, core_requirements = self.MakeCoreSdist() + client_sdist, client_requirements = self.MakeClientSdist() + client_builder_sdist, client_builder_requirements = self.MakeClientBuilderSdist() + + self.InstallGRR(proto_sdist, proto_requirements) + self.InstallGRR(core_sdist, core_requirements) + self.InstallGRR(client_sdist, client_requirements) + self.InstallGRR(client_builder_sdist, client_builder_requirements) self.BuildTemplates() if args.test_repack_install: self._RepackTemplates() diff --git a/grr/server/requirements_lin.in b/grr/server/requirements_lin.in new file mode 100644 index 000000000..9c5f53f96 --- /dev/null +++ b/grr/server/requirements_lin.in @@ -0,0 +1,18 @@ +google-api-python-client==1.12.11 +google-auth==2.23.3 +google-cloud-storage==2.13.0 +google-cloud-pubsub==2.18.4 +ipython==7.15.0 +Jinja2==3.1.2 +pexpect==4.8.0 +portpicker==1.6.0b1 +protobuf==4.25.3 +prometheus_client==0.16.0 +pyjwt==2.6.0 +pyOpenSSL==21.0.0 +python-crontab==2.5.1 +python-debian==0.1.49 +Werkzeug==2.1.2 +# TODO: We currently release fleetspeak-server-bin packages +# for Linux only. +fleetspeak-server-bin==0.1.13 diff --git a/travis/compile_requirements.bat b/travis/compile_requirements.bat new file mode 100755 index 000000000..dd575ef01 --- /dev/null +++ b/travis/compile_requirements.bat @@ -0,0 +1,20 @@ + +set DIR=%~dp0 + +set OUT_FOLDER=%1 + + +mkdir "%OUT_FOLDER%" + +pip install --require-hashes -r "%DIR%\base_tooling_requirements.txt" + +python -m piptools compile --generate-hashes "%DIR%\..\api_client\python\requirements.in" -o "%OUT_FOLDER%\api_client_requirements.txt" +python -m piptools compile --generate-hashes "%DIR%\..\grr\core\requirements.in" -o "%OUT_FOLDER%\core_requirements.txt" +python -m piptools compile --generate-hashes "%DIR%\..\grr\server\requirements.in" -o "%OUT_FOLDER%\server_requirements.txt" +python -m piptools compile --generate-hashes "%DIR%\..\grr\proto\requirements.in" -o "%OUT_FOLDER%\proto_requirements.txt" +python -m piptools compile --generate-hashes "%DIR%\..\grr\client\requirements.in" -o "%OUT_FOLDER%\client_requirements.txt" +python -m piptools compile --generate-hashes "%DIR%\..\grr\server\requirements_win.in" -o "%OUT_FOLDER%\client_requirements_win.txt" +python -m piptools compile --generate-hashes "%DIR%\..\grr\client_builder\requirements.in" -o "%OUT_FOLDER%\client_builder_requirements.txt" +python -m piptools compile --generate-hashes "%DIR%\..\grr\test\requirements.in" -o "%OUT_FOLDER%\test_requirements.txt" + +