From acf54c67233d102e47fb9202ebee67d0dfb99fd2 Mon Sep 17 00:00:00 2001 From: "Daniel F. Murcia Rivera" Date: Tue, 5 Sep 2023 14:07:37 -0500 Subject: [PATCH] feat(back): #1146 make python pyproject - add make-python-pyproject buiding from nixpkgs functions - add python-override-utils - add docs and example Signed-off-by: Daniel F. Murcia Rivera --- docs/src/api/extensions/python.md | 111 ++++++++++++++++++ src/args/agnostic.nix | 44 +++---- .../make-python-pyproject-package/default.nix | 9 ++ .../generic_builder/check.nix | 11 ++ .../generic_builder/default.nix | 18 +++ .../generic_builder/env.nix | 14 +++ .../generic_builder/metadata.nix | 7 ++ .../generic_builder/pkg/check/tests.sh | 6 + .../generic_builder/pkg/check/types.sh | 6 + .../generic_builder/pkg/default.nix | 26 ++++ src/args/python-override-utils/default.nix | 48 ++++++++ 11 files changed, 279 insertions(+), 21 deletions(-) create mode 100644 src/args/make-python-pyproject-package/default.nix create mode 100644 src/args/make-python-pyproject-package/generic_builder/check.nix create mode 100644 src/args/make-python-pyproject-package/generic_builder/default.nix create mode 100644 src/args/make-python-pyproject-package/generic_builder/env.nix create mode 100644 src/args/make-python-pyproject-package/generic_builder/metadata.nix create mode 100755 src/args/make-python-pyproject-package/generic_builder/pkg/check/tests.sh create mode 100755 src/args/make-python-pyproject-package/generic_builder/pkg/check/types.sh create mode 100644 src/args/make-python-pyproject-package/generic_builder/pkg/default.nix create mode 100644 src/args/python-override-utils/default.nix diff --git a/docs/src/api/extensions/python.md b/docs/src/api/extensions/python.md index cf874d9e..c629cb4c 100644 --- a/docs/src/api/extensions/python.md +++ b/docs/src/api/extensions/python.md @@ -122,3 +122,114 @@ Example: Refer to [makePythonLock](/api/builtins/utilities/#makepythonlock) to learn how to generate a `sourcesYaml`. + +## makePythonPyprojectPackage + +Create a python package bundle using nixpkgs build functions. +This bundle includes the package itself, some modifications +over the tests and its python environments. + +Types: + +- makePythonPypiEnvironment: (`function Input -> Bundle`): + - Input: `AttrsOf` + - buildEnv: `function {...} -> Derivation` + The nixpkgs buildEnv.override function. + Commonly found at `nixpkgs."${python_version}".buildEnv.override` + - buildPythonPackage: `function {...} -> Derivation` + The nixpkgs buildPythonPackage function. + Commonly found at `nixpkgs."${python_version}".pkgs.buildPythonPackage` + - pkgDeps: `AttrsOf` + The package dependencies. + Usually other python packages build with nix, + but can be also a nix derivation of a binary. + - runtime_deps: `List[Derivation]` + - build_deps: `List[Derivation]` + - test_deps: `List[Derivation]` + - src: `NixPath` + The nix path to the source code of the python package. + i.e. not only be the package itself, it should also contain + a tests folder/module, the pyproject conf and any other meta-package + data that the build or tests requires (e.g. custom mypy conf). + - Bundle: `AttrsOf` + - check: `AttrsOf` + Builds of the package only including one test. + - tests:`Derivation` + - types: `Derivation` + - env: `AttrsOf` + - dev: `Derivation` + The python environment containing only + runtime_deps and test_deps + - runtime: `Derivation` + The python environment containing only + the package itself and its runtime_deps. + - pkg: `Derivation` + The output of the nixpkgs buildPythonPackage function + i.e. the python package + +???+ tip + + The default implemented tests require `mypy` and `pytest` as `test_deps`. + If you do not want the default, you can override the checkPhase + of the package i.e. using `pythonOverrideUtils` or using the + `overridePythonAttrs` function included on the derivation of + nix built python packages. + +Example: + +=== "main.nix" + + ```nix + # /path/to/my/project/makes/example/main.nix + { + inputs, + makeScript, + makePythonPyprojectPackage, + ... + }: let + nixpkgs = inputs.nixpkgs; + python_version = "python311"; + python_pkgs = nixpkgs."${python_version}Packages"; + bundle = makePythonPyprojectPackage { + src = ./.; + buildEnv = nixpkgs."${python_version}".buildEnv.override; + buildPythonPackage = nixpkgs."${python_version}".pkgs.buildPythonPackage; + pkgDeps = { + runtime_deps = with python_pkgs; [click]; + build_deps = with python_pkgs; [flit-core]; + test_deps = with python_pkgs; [ + mypy + pytest + ]; + }; + }; + env = bundle.env.runtime; + in + makeScript { + name = "my-cli"; + searchPaths = { + bin = [ + env + ]; + }; + entrypoint = "my-cli \"\${@}\""; + # Assuming that the pyproject conf has + # a definition of `my-cli` as a cli entrypoint + } + ``` + +???+ tip + + Because env.runtime include the package, + all tests are triggered when building the environment. + If is desirable only to trigger an specific check phase, + then use the check derivations that override this phase. + +???+ tip + + To avoid performance issues use a shared cache + system (e.g. cachix) or an override over the package + to skip tests (unsafe way) to ensure that tests are + executed only once (or never). + This can also help on performance over heavy + compilation/build processes. diff --git a/src/args/agnostic.nix b/src/args/agnostic.nix index 85dc20db..20e81d6b 100644 --- a/src/args/agnostic.nix +++ b/src/args/agnostic.nix @@ -8,9 +8,11 @@ system ? builtins.currentSystem, }: let fix' = __unfix__: let x = __unfix__ x // {inherit __unfix__;}; in x; - - args = fix' (self: { + sources = import ../nix/sources.nix; + args = fix' (self: let __nixpkgs__ = import sources.nixpkgs {inherit system;}; + in { + inherit __nixpkgs__; __nixpkgsSrc__ = sources.nixpkgs; __shellCommands__ = ./shell-commands/template.sh; __shellOptions__ = ./shell-options/template.sh; @@ -19,9 +21,9 @@ __toModuleOutputs__ = import ./to-module-outputs/default.nix self; asContent = import ./as-content/default.nix; attrsGet = import ./attrs-get/default.nix; - attrsMapToList = self.__nixpkgs__.lib.mapAttrsToList; - attrsMerge = builtins.foldl' self.__nixpkgs__.lib.recursiveUpdate {}; - attrsOptional = self.__nixpkgs__.lib.optionalAttrs; + attrsMapToList = __nixpkgs__.lib.mapAttrsToList; + attrsMerge = builtins.foldl' __nixpkgs__.lib.recursiveUpdate {}; + attrsOptional = __nixpkgs__.lib.optionalAttrs; calculateCvss3 = import ./calculate-cvss-3/default.nix self; calculateScorecard = import ./calculate-scorecard/default.nix self; chunks = import ./chunks/default.nix self; @@ -29,17 +31,17 @@ deployContainerImage = import ./deploy-container-image/default.nix self; deployNomad = import ./deploy-nomad/default.nix self; deployTerraform = import ./deploy-terraform/default.nix self; - inherit (self.__nixpkgs__.lib.strings) escapeShellArg; - inherit (self.__nixpkgs__.lib.strings) escapeShellArgs; - inherit (self.__nixpkgs__.lib) fakeSha256; + inherit (__nixpkgs__.lib.strings) escapeShellArg; + inherit (__nixpkgs__.lib.strings) escapeShellArgs; + inherit (__nixpkgs__.lib) fakeSha256; fetchArchive = import ./fetch-archive/default.nix self; fetchGithub = import ./fetch-github/default.nix self; fetchGitlab = import ./fetch-gitlab/default.nix self; fetchNixpkgs = import ./fetch-nixpkgs/default.nix self; fetchRubyGem = import ./fetch-rubygem/default.nix self; fetchUrl = import ./fetch-url/default.nix self; - inherit (self.__nixpkgs__.lib) filterAttrs; - inherit (self.__nixpkgs__.lib.lists) flatten; + inherit (__nixpkgs__.lib) filterAttrs; + inherit (__nixpkgs__.lib.lists) flatten; formatBash = import ./format-bash/default.nix self; formatNix = import ./format-nix/default.nix self; formatPython = import ./format-python/default.nix self; @@ -51,12 +53,12 @@ fromYaml = import ./from-yaml/default.nix self; fromYamlFile = path: self.fromYaml (builtins.readFile path); gitlabCi = import ./gitlab-ci/default.nix; - inherit (self.__nixpkgs__.lib.strings) hasPrefix; - inherit (self.__nixpkgs__.lib.strings) hasSuffix; - inherit (self.__nixpkgs__.stdenv) isDarwin; - inherit (self.__nixpkgs__.stdenv) isLinux; + inherit (__nixpkgs__.lib.strings) hasPrefix; + inherit (__nixpkgs__.lib.strings) hasSuffix; + inherit (__nixpkgs__.stdenv) isDarwin; + inherit (__nixpkgs__.stdenv) isLinux; libGit = import ./lib-git/default.nix self; - listOptional = self.__nixpkgs__.lib.lists.optional; + listOptional = __nixpkgs__.lib.lists.optional; lintClojure = import ./lint-clojure/default.nix self; lintGitCommitMsg = import ./lint-git-commit-msg/default.nix self; lintGitMailMap = import ./lint-git-mailmap/default.nix self; @@ -67,7 +69,7 @@ lintTerraform = import ./lint-terraform/default.nix self; lintWithAjv = import ./lint-with-ajv/default.nix self; lintWithLizard = import ./lint-with-lizard/default.nix self; - inherit (self.__nixpkgs__.lib.filesystem) listFilesRecursive; + inherit (__nixpkgs__.lib.filesystem) listFilesRecursive; makeContainerImage = import ./make-container-image/default.nix self; makeDerivation = import ./make-derivation/default.nix self; makeDerivationParallel = import ./make-derivation-parallel/default.nix self; @@ -79,6 +81,7 @@ makeNodeJsVersion = import ./make-node-js-version/default.nix self; makeNomadEnvironment = import ./make-nomad-environment/default.nix self; makePythonPypiEnvironment = import ./make-python-pypi-environment/default.nix self; + makePythonPyprojectPackage = import ./make-python-pyproject-package/default.nix; makePythonVersion = import ./make-python-version/default.nix self; makeRubyGemsEnvironment = import ./make-ruby-gems-environment/default.nix self; makeRubyGemsInstall = import ./make-ruby-gems-install/default.nix self; @@ -99,11 +102,12 @@ makeWorkspaceForTerraformFromEnv = import ./make-workspace-for-terraform-from-env/default.nix self; managePorts = import ./manage-ports/default.nix self; patchShebangs = import ./patch-shebangs/default.nix self; - inherit (self.__nixpkgs__.lib) removePrefix; + pythonOverrideUtils = import ./python-override-utils/default.nix; + inherit (__nixpkgs__.lib) removePrefix; secureKubernetesWithRbacPolice = import ./secure-kubernetes-with-rbac-police/default.nix self; securePythonWithBandit = import ./secure-python-with-bandit/default.nix self; sortAscii = builtins.sort (a: b: a < b); - sortAsciiCaseless = builtins.sort (a: b: self.__nixpkgs__.lib.toLower a < self.__nixpkgs__.lib.toLower b); + sortAsciiCaseless = builtins.sort (a: b: __nixpkgs__.lib.toLower a < __nixpkgs__.lib.toLower b); stringCapitalize = import ./string-capitalize/default.nix self; sublist = import ./sublist/default.nix self; taintTerraform = import ./taint-terraform/default.nix self; @@ -111,7 +115,7 @@ testTerraform = import ./test-terraform/default.nix self; testPullRequest = import ./test-pull-request/default.nix self; testPython = import ./test-python/default.nix self; - toDerivationName = self.__nixpkgs__.lib.strings.sanitizeDerivationName; + toDerivationName = __nixpkgs__.lib.strings.sanitizeDerivationName; toBashArray = import ./to-bash-array/default.nix self; toBashMap = import ./to-bash-map/default.nix self; toFileJson = import ./to-file-json/default.nix self; @@ -120,7 +124,5 @@ toFileJsonFromFileYaml = import ./to-file-json-from-file-yaml/default.nix self; toFileLst = import ./to-file-lst/default.nix; }); - - sources = import ../nix/sources.nix; in assert args.isDarwin || args.isLinux; args diff --git a/src/args/make-python-pyproject-package/default.nix b/src/args/make-python-pyproject-package/default.nix new file mode 100644 index 00000000..708b7526 --- /dev/null +++ b/src/args/make-python-pyproject-package/default.nix @@ -0,0 +1,9 @@ +{ + buildEnv, + buildPythonPackage, + pkgDeps, + src, +}: +import ./generic_builder { + inherit buildEnv buildPythonPackage pkgDeps src; +} diff --git a/src/args/make-python-pyproject-package/generic_builder/check.nix b/src/args/make-python-pyproject-package/generic_builder/check.nix new file mode 100644 index 00000000..5c53f42a --- /dev/null +++ b/src/args/make-python-pyproject-package/generic_builder/check.nix @@ -0,0 +1,11 @@ +{pkg}: let + build_check = check: + pkg.overridePythonAttrs ( + old: { + checkPhase = [old."${check}"]; + } + ); +in { + tests = build_check "test_check"; + types = build_check "type_check"; +} diff --git a/src/args/make-python-pyproject-package/generic_builder/default.nix b/src/args/make-python-pyproject-package/generic_builder/default.nix new file mode 100644 index 00000000..74eadf8c --- /dev/null +++ b/src/args/make-python-pyproject-package/generic_builder/default.nix @@ -0,0 +1,18 @@ +{ + buildEnv, + buildPythonPackage, + pkgDeps, + src, +}: let + metadata = import ./metadata.nix src; + pkg = import ./pkg { + inherit buildPythonPackage metadata pkgDeps src; + }; + env = import ./env.nix { + inherit buildEnv pkgDeps pkg; + }; + checks = import ./check.nix {inherit pkg;}; +in { + inherit pkg env; + check = checks; +} diff --git a/src/args/make-python-pyproject-package/generic_builder/env.nix b/src/args/make-python-pyproject-package/generic_builder/env.nix new file mode 100644 index 00000000..c31e83f9 --- /dev/null +++ b/src/args/make-python-pyproject-package/generic_builder/env.nix @@ -0,0 +1,14 @@ +{ + buildEnv, + pkgDeps, + pkg, +}: let + build_env = extraLibs: + buildEnv { + inherit extraLibs; + ignoreCollisions = false; + }; +in { + runtime = build_env [pkg]; + dev = build_env (pkgDeps.runtime_deps ++ pkgDeps.test_deps); +} diff --git a/src/args/make-python-pyproject-package/generic_builder/metadata.nix b/src/args/make-python-pyproject-package/generic_builder/metadata.nix new file mode 100644 index 00000000..43658085 --- /dev/null +++ b/src/args/make-python-pyproject-package/generic_builder/metadata.nix @@ -0,0 +1,7 @@ +src: let + _metadata = (builtins.fromTOML (builtins.readFile "${src}/pyproject.toml")).project; + file_str = builtins.readFile "${src}/${_metadata.name}/__init__.py"; + match = builtins.match ".*__version__ *= *\"(.+)\"\n.*" file_str; + version = builtins.elemAt match 0; +in + _metadata // {inherit version;} diff --git a/src/args/make-python-pyproject-package/generic_builder/pkg/check/tests.sh b/src/args/make-python-pyproject-package/generic_builder/pkg/check/tests.sh new file mode 100755 index 00000000..6488b9c1 --- /dev/null +++ b/src/args/make-python-pyproject-package/generic_builder/pkg/check/tests.sh @@ -0,0 +1,6 @@ +# shellcheck shell=bash + +echo "Executing test phase" \ + && pytest --version \ + && pytest ./tests \ + && echo "Finished test phase" diff --git a/src/args/make-python-pyproject-package/generic_builder/pkg/check/types.sh b/src/args/make-python-pyproject-package/generic_builder/pkg/check/types.sh new file mode 100755 index 00000000..8ddb1353 --- /dev/null +++ b/src/args/make-python-pyproject-package/generic_builder/pkg/check/types.sh @@ -0,0 +1,6 @@ +# shellcheck shell=bash + +echo "Executing type check phase" \ + && mypy --version \ + && mypy . --config-file ./mypy.ini \ + && echo "Finished type check phase" diff --git a/src/args/make-python-pyproject-package/generic_builder/pkg/default.nix b/src/args/make-python-pyproject-package/generic_builder/pkg/default.nix new file mode 100644 index 00000000..f9e63d1c --- /dev/null +++ b/src/args/make-python-pyproject-package/generic_builder/pkg/default.nix @@ -0,0 +1,26 @@ +{ + buildPythonPackage, + metadata, + pkgDeps, + src, +}: let + type_check = ./check/types.sh; + test_check = ./check/tests.sh; +in + buildPythonPackage { + inherit src type_check test_check; + inherit (metadata) version; + pname = metadata.name; + format = "pyproject"; + checkPhase = [ + '' + source ${type_check} \ + && source ${test_check} \ + '' + ]; + doCheck = true; + pythonImportsCheck = [metadata.name]; + nativeBuildInputs = pkgDeps.build_deps; + propagatedBuildInputs = pkgDeps.runtime_deps; + nativeCheckInputs = pkgDeps.test_deps; + } diff --git a/src/args/python-override-utils/default.nix b/src/args/python-override-utils/default.nix new file mode 100644 index 00000000..6338dafc --- /dev/null +++ b/src/args/python-override-utils/default.nix @@ -0,0 +1,48 @@ +# python override utils, +# useful when overriding pkgs from an environment to ensure no collisions +let + recursive_python_pkg_override = is_pkg: override: let + # is_pkg: Derivation -> Bool + # override: Derivation -> Derivation + self = recursive_python_pkg_override is_pkg override; + in + pkg: + if is_pkg pkg + then override pkg + else if pkg ? overridePythonAttrs && pkg ? pname + then + pkg.overridePythonAttrs ( + builtins.mapAttrs (_: value: + if builtins.isList value + then map self value + else self value) + ) + else pkg; + + # no_check_override: Derivation -> Derivation + no_check_override = recursive_python_pkg_override (pkg: pkg ? overridePythonAttrs && pkg ? pname) ( + pkg: + pkg.overridePythonAttrs ( + old: + ( + builtins.mapAttrs (_: value: + if builtins.isList value + then map no_check_override value + else no_check_override value) + old + ) + // { + doCheck = false; + } + ) + ); + + # replace_pkg: List[str] -> Derivation -> Derivation + replace_pkg = names: new_pkg: + recursive_python_pkg_override ( + x: x ? overridePythonAttrs && x ? pname && builtins.elem x.pname names + ) (_: new_pkg); +in { + inherit recursive_python_pkg_override no_check_override replace_pkg; + compose = functions: val: builtins.foldl' (x: f: f x) val functions; +}