diff --git a/.mailmap b/.mailmap index 8af8430d..eaee2f81 100644 --- a/.mailmap +++ b/.mailmap @@ -7,6 +7,7 @@ David Acevedo David Acevedo David Arnold David Arnold David Arnold David Arnold Diego Restrepo Diego Restrepo Mesa <36453706+drestrepom@users.noreply.github.com> +Diego Restrepo Diego Restrepo Diego Restrepo Diego Restrepo Mesa <36453706+drestrepom@users.noreply.github.com> Fluid Attacks Fluid Attacks Github Dependabot <49699333+dependabot[bot]@users.noreply.github.com> dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> diff --git a/README.md b/README.md index 426e0909..39752706 100644 --- a/README.md +++ b/README.md @@ -2107,24 +2107,21 @@ and optionally a [Cachix][cachix] cache for reading and writting. Types: - cache: + - extra: (attrsOf (cacheExtra)) - readNixos (`bool`): Optional. Set to `true` in order to add https://cache.nixos.org as a read cache. Defaults to `true`. - - readExtra (`listOf readCacheType`): Optional. - Extra caches to read, if any. - Defaults to `[ ]`. - - readAndWrite: - - enable (`boolean`): Optional. - Defaults to `false`. - - name (`str`): - Name of the [Cachix][cachix] cache. - - pubKey (`str`): - Public key of the [Cachix][cachix] cache. -- readCacheType (`submodule`): +- cacheExtra: + - enable (`str`): The current cache is enabled and the cache + is read on the server. + - priority (`int`): the priority that the cache has when being read. + - pubKey (`str`): public key to read the cache. + - token (`str`): the name of the environment variable that contains the + token tu push the cache. + - type: (`enum [cachix | attic]`): the cache server type. - url (`str`): URL of the cache. - - pubKey (`str`): - Public key of the cache. + - write (`bool`): the cache is enabled to push the binary cache. Required environment variables: @@ -2140,20 +2137,23 @@ Example `makes.nix`: { cache = { readNixos = true; - readExtra = [ - { - url = "https://example.com"; - pubKey = "example.example.org-1:123..."; - } - { - url = "https://example2.com"; - pubKey = "example2.example2.org-1:123..."; - } - ]; - readAndWrite = { - enable = true; - name = "makes"; - pubKey = "makes.cachix.org-1:HbCQcdlYyT/mYuOx6rlgkNkonTGUjVr3D+YpuGRmO+Y="; + extra = { + main = { + enable = true; + pubKey = "makes.cachix.org-1:zO7UjWLTRR8Vfzkgsu1PESjmb6ymy1e4OE9YfMmCQR4="; + token = "CACHIX_AUTH_TOKEN"; + type = "nixos"; + url = "https://makes.cachix.org"; + write = true; + }; + local = { + enable = true; + pubKey = "local:nKOS5sOc0MKPoBJZmY4qWjbcXvoJFaO2S/zN6aUztII="; + token = "ATTIC_AUTH_TOKEN"; + type = "attic"; + url = "http://192.168.1.8:8085/local"; + write = true; + }; }; }; } diff --git a/makes.nix b/makes.nix index 2409aa91..6433458c 100644 --- a/makes.nix +++ b/makes.nix @@ -8,10 +8,16 @@ }: { projectIdentifier = "makes-repo"; cache = { - readAndWrite = { - enable = true; - name = "makes"; - pubKey = "makes.cachix.org-1:zO7UjWLTRR8Vfzkgsu1PESjmb6ymy1e4OE9YfMmCQR4="; + readNixos = true; + extra = { + makes = { + enable = true; + pubKey = "makes.cachix.org-1:zO7UjWLTRR8Vfzkgsu1PESjmb6ymy1e4OE9YfMmCQR4="; + token = "CACHIX_AUTH_TOKEN"; + type = "cachix"; + url = "https://makes.cachix.org"; + write = true; + }; }; }; calculateScorecard = { diff --git a/makes/cli/env/runtime/main.nix b/makes/cli/env/runtime/main.nix index c791fa78..113f95f1 100644 --- a/makes/cli/env/runtime/main.nix +++ b/makes/cli/env/runtime/main.nix @@ -12,6 +12,7 @@ makeSearchPaths { __nixpkgs__.gzip __nixpkgs__.nixStable __nixpkgs__.openssh + # it is necessary to inject the attic client ]; source = [ outputs."/cli/env/runtime/pypi" diff --git a/src/cli/main/cli.py b/src/cli/main/cli.py index 29ad243d..c31f0c29 100644 --- a/src/cli/main/cli.py +++ b/src/cli/main/cli.py @@ -10,7 +10,6 @@ ) import io import json -import operator import os from os import ( environ, @@ -265,6 +264,41 @@ def _clone_src_cache_refresh(head: str, cache_key: str) -> None: shutil.copytree(head, cached) +def _attic_login(caches: List[Dict[str, Any]]) -> None: + for config in caches: + if config["type"] == "attic" and config["token"] in environ: + _run( + args=[ + "attic", + "login", + "local", + config["url"], + environ[config["token"]], + ], + stderr=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + ) + _run( + args=[ + "attic", + "cache", + "create", + config["name"], + ], + stderr=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + ) + _run( + args=[ + "attic", + "use", + config["name"], + ], + stderr=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + ) + + def _nix_build( *, attr: str, @@ -278,9 +312,21 @@ def _nix_build( "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" ) else: - substituters = " ".join(map(operator.itemgetter("url"), cache)) - trusted_pub_keys = " ".join(map(operator.itemgetter("pubKey"), cache)) - + _attic_login(cache) + substituters = " ".join( + [ + item["url"] + for item in cache + if "url" in item and "pubKey" in item and item["url"] + ] + ) + trusted_pub_keys = " ".join( + [ + item["pubKey"] + for item in cache + if "url" in item and "pubKey" in item and item["pubKey"] + ] + ) return [ *_if(NIX_STABLE, f"{__NIX_STABLE__}/bin/nix-build"), *_if(not NIX_STABLE, f"{__NIX_UNSTABLE__}/bin/nix"), @@ -642,18 +688,55 @@ def execute_action(args: List[str], head: str, out: str) -> None: def cache_push(cache: List[Dict[str, str]], out: str) -> None: - once: bool = True - for config in cache: - if config["type"] == "cachix" and "CACHIX_AUTH_TOKEN" in environ: - if once: - CON.rule("Pushing to cache") - once = False + once = True + for config in [item for item in cache if item.get("token", "") in environ]: + if once: + CON.rule("Pushing to cache") + once = False + if config["type"] in "cachix": + _run( + args=["cachix", "authtoken", environ[config["token"]]], + stderr=None, + stdout=sys.stderr.fileno(), + ) _run( args=["cachix", "push", "-c", "0", config["name"], out], stderr=None, stdout=sys.stderr.fileno(), ) - return + elif config["type"] == "attic": + _run( + args=[ + "attic", + "login", + "local", + config["url"], + environ[config["token"]], + ], + stderr=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + ) + _run( + args=[ + "attic", + "cache", + "create", + config["name"], + ], + stderr=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + ) + _run( + args=[ + "attic", + "push", + "--ignore-upstream-cache-filter", + config["name"], + out, + ], + stderr=sys.stderr.fileno(), + stdout=sys.stderr.fileno(), + ) def _get_sys_id() -> str: diff --git a/src/evaluator/modules/cache/default.nix b/src/evaluator/modules/cache/default.nix index 60429ec4..b0c4b889 100644 --- a/src/evaluator/modules/cache/default.nix +++ b/src/evaluator/modules/cache/default.nix @@ -5,28 +5,32 @@ }: { options = { cache = { - readAndWrite = { - enable = lib.mkOption { - default = false; - type = lib.types.bool; - }; - name = lib.mkOption { - type = lib.types.str; - }; - pubKey = lib.mkOption { - type = lib.types.str; - }; - }; - readExtra = lib.mkOption { - default = []; - type = lib.types.listOf (lib.types.submodule (_: { + extra = lib.mkOption { + default = {}; + type = lib.types.attrsOf (lib.types.submodule (_: { options = { + enable = lib.mkOption { + default = false; + type = lib.types.bool; + }; pubKey = lib.mkOption { + default = ""; + type = lib.types.str; + }; + token = lib.mkOption { type = lib.types.str; + default = ""; + }; + type = lib.mkOption { + type = lib.types.enum ["cachix"]; }; url = lib.mkOption { type = lib.types.str; }; + write = lib.mkOption { + default = false; + type = lib.types.bool; + }; }; })); }; @@ -44,19 +48,19 @@ pubKey = "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="; type = "other"; }) - (listOptional config.cache.readAndWrite.enable { - name = config.cache.readAndWrite.name; - url = "https://${config.cache.readAndWrite.name}.cachix.org/"; - pubKey = config.cache.readAndWrite.pubKey; - type = "cachix"; - }) - (builtins.map - (cache: { - inherit (cache) url; - inherit (cache) pubKey; - type = "other"; - }) - config.cache.readExtra) + ( + builtins.filter + (cache: cache.enable) + (builtins.map (name: { + inherit (config.cache.extra.${name}) enable; + inherit (config.cache.extra.${name}) pubKey; + inherit (config.cache.extra.${name}) token; + inherit (config.cache.extra.${name}) type; + inherit (config.cache.extra.${name}) url; + inherit (config.cache.extra.${name}) write; + inherit name; + }) (builtins.attrNames config.cache.extra)) + ) ]; }; };