add requirements for all packages

.github/workflows/build.yml

@@ -87,14 +87,23 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           python-version: '3.9'
+      - name: Verify requirements
+        run: |
+          ./travis/compile_requirements.sh ubuntu_requirements
+      - name: Upload requirements to GitHub artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: ubuntu-requirements
+          path: ubuntu_requirements/
+          retention-days: 3
       - name: Set up
         run: |
           sudo apt install fakeroot debhelper libffi-dev libssl-dev
           pip install virtualenv
           virtualenv "${HOME}/INSTALL"
       - name: Build
         run: |
-          travis/install_client_builder.sh
+          travis/install_client_builder.sh ubuntu
           travis/build_templates.sh
           ls -la gcs_upload_dir
       - name: Upload installers to GitHub artifacts
@@ -115,9 +124,18 @@ jobs:
         run: |
           pip install --upgrade setuptools virtualenv
           virtualenv "${HOME}/INSTALL"
+      - name: Verify requirements
+        run: |
+          ./travis/compile_requirements.sh osx_requirements
+      - name: Upload requirements to GitHub artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: osx-requirements
+          path: osx_requirements/
+          retention-days: 3
       - name: Build installers
         run: |
-          travis/install_client_builder.sh
+          travis/install_client_builder.sh osx
           travis/build_templates.sh
           ls -la gcs_upload_dir
       - name: Upload installers to GitHub artifacts
@@ -136,6 +154,15 @@ jobs:
       DOCKER_USER: grrbot
     steps:
       - uses: actions/checkout@v4
+      - name: Verify requirements
+        run: |
+          ./travis/compile_requirements.sh centos_requirements
+      - name: Upload requirements to GitHub artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: centos-requirements
+          path: centos_requirements/
+          retention-days: 3
       - name: Build installers
         run: |
           docker run -dit \
@@ -150,7 +177,7 @@ jobs:
           # registered in the environment variables.
           docker exec "${DOCKER_CONTAINER}" bash -l travis/set_up_test_user.sh
           docker exec --user "${DOCKER_USER}" "${DOCKER_CONTAINER}" bash -l -c '/usr/local/bin/python3.9 -m venv "/home/${DOCKER_USER}/INSTALL"'
-          docker exec --user "${DOCKER_USER}" "${DOCKER_CONTAINER}" bash -l travis/install_client_builder.sh
+          docker exec --user "${DOCKER_USER}" "${DOCKER_CONTAINER}" bash -l travis/install_client_builder.sh centos
           docker exec --user "${DOCKER_USER}" "${DOCKER_CONTAINER}" bash -l travis/build_templates.sh
           docker exec "${DOCKER_CONTAINER}" rpm -vih gcs_upload_dir/*.rpm
           ls -la gcs_upload_dir

Dockerfile

@@ -69,16 +69,16 @@ ENV GRR_SOURCE /usr/src/grr
 RUN python -m venv --system-site-packages $VIRTUAL_ENV
 ENV PATH=${VIRTUAL_ENV}/bin:${PATH}
 
+RUN ${VIRTUAL_ENV}/bin/python -m pip install wheel nodeenv grpcio-tools==1.60
+
+RUN ${VIRTUAL_ENV}/bin/nodeenv -p --prebuilt --node=16.13.0
+
 RUN mkdir ${GRR_SOURCE}
 ADD . ${GRR_SOURCE}
 
 WORKDIR ${GRR_SOURCE}
 
-RUN ${VIRTUAL_ENV}/bin/python -m pip install --require-hashes -r requirements.txt
-
-RUN ${VIRTUAL_ENV}/bin/nodeenv -p --prebuilt --node=16.13.0
-
-RUN ${VIRTUAL_ENV}/bin/python -m pip install --no-deps --no-index \
+RUN ${VIRTUAL_ENV}/bin/python -m pip install \
   -e grr/proto \
   -e grr/core \
   -e grr/client \

api_client/python/requirements.in

@@ -0,0 +1,5 @@
+cryptography>=3.3.2
+requests>=2.25.1,<3
+Werkzeug>=2.1.2,<3
+ipython==7.15.0
+setuptools

api_client/python/setup.py

@@ -1,6 +1,8 @@
 #!/usr/bin/env python
 """setup.py file for a GRR API client library."""
 
+from typing import List
+
 import configparser
 import os
 import shutil
@@ -18,34 +20,48 @@
 
 
 def get_config():
-  """Get INI parser with version.ini data."""
-  # TODO(hanuszczak): See comment in `setup.py` for `grr-response-proto`.
-  ini_path = os.path.join(THIS_DIRECTORY, "version.ini")
-  if not os.path.exists(ini_path):
-    ini_path = os.path.join(THIS_DIRECTORY, "../../version.ini")
+    """Get INI parser with version.ini data."""
+    # TODO(hanuszczak): See comment in `setup.py` for `grr-response-proto`.
+    ini_path = os.path.join(THIS_DIRECTORY, "version.ini")
     if not os.path.exists(ini_path):
-      raise RuntimeError("Couldn't find version.ini")
+        ini_path = os.path.join(THIS_DIRECTORY, "../../version.ini")
+        if not os.path.exists(ini_path):
+            raise RuntimeError("Couldn't find version.ini")
 
-  config = configparser.ConfigParser()
-  config.read(ini_path)
-  return config
+    config = configparser.ConfigParser()
+    config.read(ini_path)
+    return config
 
 
 class Sdist(sdist):
-  """Build sdist."""
+    """Build sdist."""
 
-  def make_release_tree(self, base_dir, files):
-    sdist.make_release_tree(self, base_dir, files)
+    def make_release_tree(self, base_dir, files):
+        sdist.make_release_tree(self, base_dir, files)
 
-    sdist_version_ini = os.path.join(base_dir, "version.ini")
-    if os.path.exists(sdist_version_ini):
-      os.unlink(sdist_version_ini)
-    shutil.copy(
-        os.path.join(THIS_DIRECTORY, "../../version.ini"), sdist_version_ini)
+        sdist_version_ini = os.path.join(base_dir, "version.ini")
+        if os.path.exists(sdist_version_ini):
+            os.unlink(sdist_version_ini)
+        shutil.copy(
+            os.path.join(THIS_DIRECTORY, "../../version.ini"), sdist_version_ini
+        )
 
 
 VERSION = get_config()
 
+
+def parse_requirements(filename: str) -> List[str]:
+    requirements = []
+    with open(filename) as file:
+        for line in file:
+            requirement = line.strip()
+            if (comment := requirement.find("#")) >= 0:
+                requirement = requirement[:comment].strip()
+            requirements.append(requirement)
+
+    return requirements
+
+
 setup_args = dict(
     name="grr-api-client",
     version=VERSION.get("Version", "packageversion"),
@@ -65,19 +81,9 @@ def make_release_tree(self, base_dir, files):
     },
     install_requires=[
         "grr_response_proto==%s" % VERSION.get("Version", "packagedepends"),
-        # Note: grr-api-client might very much be used as a library and
-        # therefore shouldn't pin dependencies that might be shared with other
-        # pip packages.
-        "cryptography>=3.3.2",
-        "requests>=2.25.1,<3",
-        "Werkzeug>=2.1.2,<3",
-    ],
-    extra_requires={
-        "shell": [
-            "ipython==7.15.0",
-        ],
-    },
-    data=["version.ini"],
+    ]
+    + parse_requirements("requirements.in"),
+    data=["version.ini", "requirements.in"],
 )
 
 setup(**setup_args)

grr/client/requirements.in

@@ -0,0 +1,5 @@
+absl-py==1.4.0
+pytsk3==20230125
+libfsntfs-python==20230606
+fleetspeak-client-bin==0.1.13
+xattr==0.9.7

grr/client/requirements.txt

@@ -0,0 +1,93 @@
+#
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
+#
+#    pip-compile --generate-hashes ./travis/../grr/client/requirements.in
+#
+absl-py==1.4.0 \
+    --hash=sha256:0d3fe606adfa4f7db64792dd4c7aee4ee0c38ab75dfd353b7a83ed3e957fcb47 \
+    --hash=sha256:d2c244d01048ba476e7c080bd2c6df5e141d211de80223460d5b3b8a2a58433d
+    # via -r ./travis/../grr/client/requirements.in
+cffi==1.16.0 \
+    --hash=sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc \
+    --hash=sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a \
+    --hash=sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417 \
+    --hash=sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab \
+    --hash=sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520 \
+    --hash=sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36 \
+    --hash=sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743 \
+    --hash=sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8 \
+    --hash=sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed \
+    --hash=sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684 \
+    --hash=sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56 \
+    --hash=sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324 \
+    --hash=sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d \
+    --hash=sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235 \
+    --hash=sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e \
+    --hash=sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088 \
+    --hash=sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000 \
+    --hash=sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7 \
+    --hash=sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e \
+    --hash=sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673 \
+    --hash=sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c \
+    --hash=sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe \
+    --hash=sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2 \
+    --hash=sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098 \
+    --hash=sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8 \
+    --hash=sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a \
+    --hash=sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0 \
+    --hash=sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b \
+    --hash=sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896 \
+    --hash=sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e \
+    --hash=sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9 \
+    --hash=sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2 \
+    --hash=sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b \
+    --hash=sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6 \
+    --hash=sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404 \
+    --hash=sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f \
+    --hash=sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0 \
+    --hash=sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4 \
+    --hash=sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc \
+    --hash=sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936 \
+    --hash=sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba \
+    --hash=sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872 \
+    --hash=sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb \
+    --hash=sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614 \
+    --hash=sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1 \
+    --hash=sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d \
+    --hash=sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969 \
+    --hash=sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b \
+    --hash=sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4 \
+    --hash=sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627 \
+    --hash=sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956 \
+    --hash=sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357
+    # via xattr
+fleetspeak-client-bin==0.1.13 \
+    --hash=sha256:8cb051c429fb2eade39bef8c0587d61a2935c7875ca22d3db8f74a4a4ec3f8f4 \
+    --hash=sha256:a5d1be73712278e1c4ad275ea714809fb474929262d245d143e0d0101c084ce9 \
+    --hash=sha256:c1b32540c1d18d71813cfa14454ff39bde3f06ae4ee5ec777e952e16cc94b26b \
+    --hash=sha256:ef28f40757726ffd13b931740be156016d7494f2652a18f2772cba664142ce2e
+    # via -r ./travis/../grr/client/requirements.in
+libfsntfs-python==20230606 \
+    --hash=sha256:373f0065888ef9db047d12ecc016b20bf077d0a0bfa9cd6306b564b5071ac370 \
+    --hash=sha256:38dc1e5beb31146ca8fc70003bfc9cc15d98c5405a5d5339767bab8a18049e08 \
+    --hash=sha256:5037a66e562ed03ff57b36dcd8666b555dda4dca1e2045d7f5654764182477a0 \
+    --hash=sha256:6998aa3f09ebdfb6434c035987531edccfb9e3ca35c367b488ccd4e96331fd2e \
+    --hash=sha256:c70df92a678553c52ad98c140fb3f6f6aa335a5103489fbc1dccef577475f7d7 \
+    --hash=sha256:cc4a946073920e7d26d8f299d1b4a7b7ea0ab635bccad3ef5269f2976968c185 \
+    --hash=sha256:d16fb849d3f92840b78b13328484096cccb6a587681353b81b1f2b1c79f452a3 \
+    --hash=sha256:f0df820bd6d91103a36664f228fcc8564715bd9c4f446149d90a1e751bfabdff
+    # via -r ./travis/../grr/client/requirements.in
+pycparser==2.22 \
+    --hash=sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6 \
+    --hash=sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc
+    # via cffi
+pytsk3==20230125 \
+    --hash=sha256:4406a88490afcc649cd44aa3e8bd5e8b0767822c17c50cf6c6898fbb9605ac42
+    # via -r ./travis/../grr/client/requirements.in
+xattr==0.9.7 \
+    --hash=sha256:1b2cd125150aa9bbfb02929627101b3303920a68487e9c865ddd170188ddd796 \
+    --hash=sha256:1e11ba8ab86dfe74419704c53722ea9b5915833db07416e7c10db5dfb02218bb \
+    --hash=sha256:b0bbca828e04ef2d484a6522ae7b3a7ccad5e43fa1c6f54d78e24bb870f49d44 \
+    --hash=sha256:e2c72a3a501bac715489180ca2b646e48a1ca3a794c1103dd6f0f987d43f570c
+    # via -r ./travis/../grr/client/requirements.in