From c97e088af02ed9fa5d3b6846399e8e86d5ceb3a3 Mon Sep 17 00:00:00 2001 From: Diego Restrepo Date: Mon, 20 Feb 2023 12:23:32 -0500 Subject: [PATCH] refac(back): #1007 redefine cache module - make the cache module generic - add priority to use the different caches - add cache server type - by default the server only read the cache, it must be specified that the cache is also written - add doc - update the cli to use thw new format of caches --- .mailmap | 1 + README.md | 54 ++++++------ makes.nix | 14 +++- makes/cli/env/runtime/main.nix | 47 +++++++---- src/cli/main/cli.py | 104 +++++++++++++++++++++--- src/evaluator/modules/cache/default.nix | 60 +++++++------- 6 files changed, 197 insertions(+), 83 deletions(-) diff --git a/.mailmap b/.mailmap index 8af8430d..eaee2f81 100644 --- a/.mailmap +++ b/.mailmap @@ -7,6 +7,7 @@ David Acevedo David Acevedo David Arnold David Arnold David Arnold David Arnold Diego Restrepo Diego Restrepo Mesa <36453706+drestrepom@users.noreply.github.com> +Diego Restrepo Diego Restrepo Diego Restrepo Diego Restrepo Mesa <36453706+drestrepom@users.noreply.github.com> Fluid Attacks Fluid Attacks Github Dependabot <49699333+dependabot[bot]@users.noreply.github.com> dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> diff --git a/README.md b/README.md index 426e0909..39752706 100644 --- a/README.md +++ b/README.md @@ -2107,24 +2107,21 @@ and optionally a [Cachix][cachix] cache for reading and writting. Types: - cache: + - extra: (attrsOf (cacheExtra)) - readNixos (`bool`): Optional. Set to `true` in order to add https://cache.nixos.org as a read cache. Defaults to `true`. - - readExtra (`listOf readCacheType`): Optional. - Extra caches to read, if any. - Defaults to `[ ]`. - - readAndWrite: - - enable (`boolean`): Optional. - Defaults to `false`. - - name (`str`): - Name of the [Cachix][cachix] cache. - - pubKey (`str`): - Public key of the [Cachix][cachix] cache. -- readCacheType (`submodule`): +- cacheExtra: + - enable (`str`): The current cache is enabled and the cache + is read on the server. + - priority (`int`): the priority that the cache has when being read. + - pubKey (`str`): public key to read the cache. + - token (`str`): the name of the environment variable that contains the + token tu push the cache. + - type: (`enum [cachix | attic]`): the cache server type. - url (`str`): URL of the cache. - - pubKey (`str`): - Public key of the cache. + - write (`bool`): the cache is enabled to push the binary cache. Required environment variables: @@ -2140,20 +2137,23 @@ Example `makes.nix`: { cache = { readNixos = true; - readExtra = [ - { - url = "https://example.com"; - pubKey = "example.example.org-1:123..."; - } - { - url = "https://example2.com"; - pubKey = "example2.example2.org-1:123..."; - } - ]; - readAndWrite = { - enable = true; - name = "makes"; - pubKey = "makes.cachix.org-1:HbCQcdlYyT/mYuOx6rlgkNkonTGUjVr3D+YpuGRmO+Y="; + extra = { + main = { + enable = true; + pubKey = "makes.cachix.org-1:zO7UjWLTRR8Vfzkgsu1PESjmb6ymy1e4OE9YfMmCQR4="; + token = "CACHIX_AUTH_TOKEN"; + type = "nixos"; + url = "https://makes.cachix.org"; + write = true; + }; + local = { + enable = true; + pubKey = "local:nKOS5sOc0MKPoBJZmY4qWjbcXvoJFaO2S/zN6aUztII="; + token = "ATTIC_AUTH_TOKEN"; + type = "attic"; + url = "http://192.168.1.8:8085/local"; + write = true; + }; }; }; } diff --git a/makes.nix b/makes.nix index 2409aa91..6433458c 100644 --- a/makes.nix +++ b/makes.nix @@ -8,10 +8,16 @@ }: { projectIdentifier = "makes-repo"; cache = { - readAndWrite = { - enable = true; - name = "makes"; - pubKey = "makes.cachix.org-1:zO7UjWLTRR8Vfzkgsu1PESjmb6ymy1e4OE9YfMmCQR4="; + readNixos = true; + extra = { + makes = { + enable = true; + pubKey = "makes.cachix.org-1:zO7UjWLTRR8Vfzkgsu1PESjmb6ymy1e4OE9YfMmCQR4="; + token = "CACHIX_AUTH_TOKEN"; + type = "cachix"; + url = "https://makes.cachix.org"; + write = true; + }; }; }; calculateScorecard = { diff --git a/makes/cli/env/runtime/main.nix b/makes/cli/env/runtime/main.nix index c791fa78..67cd6c2e 100644 --- a/makes/cli/env/runtime/main.nix +++ b/makes/cli/env/runtime/main.nix @@ -2,18 +2,37 @@ __nixpkgs__, makeSearchPaths, outputs, + fetchGithub, + __system__, ... -}: -makeSearchPaths { - bin = [ - __nixpkgs__.cachix - __nixpkgs__.git - __nixpkgs__.gnutar - __nixpkgs__.gzip - __nixpkgs__.nixStable - __nixpkgs__.openssh - ]; - source = [ - outputs."/cli/env/runtime/pypi" - ]; -} +}: let + attic = + (import ( + fetchTarball { + url = "https://github.com/edolstra/flake-compat/archive/12c64ca55c1014cdc1b16ed5a804aa8576601ff2.tar.gz"; + sha256 = "0jm6nzb83wa6ai17ly9fzpqc40wg1viib8klq8lby54agpl213w5"; + } + ) { + src = fetchGithub { + owner = "zhaofengli"; + repo = "attic"; + rev = "863f8dcca3efce87a29853f6c842f85de594019e"; + sha256 = "bFzHDHiG5Uwopu/dgje9WNt/KDcxyVinK/k0SYIBtGw="; + }; + }) + .defaultNix; +in + makeSearchPaths { + bin = [ + __nixpkgs__.cachix + __nixpkgs__.git + __nixpkgs__.gnutar + __nixpkgs__.gzip + __nixpkgs__.nixStable + __nixpkgs__.openssh + attic.outputs.packages.${__system__}.attic-client + ]; + source = [ + outputs."/cli/env/runtime/pypi" + ]; + } diff --git a/src/cli/main/cli.py b/src/cli/main/cli.py index 29ad243d..d605ceaa 100644 --- a/src/cli/main/cli.py +++ b/src/cli/main/cli.py @@ -10,7 +10,6 @@ ) import io import json -import operator import os from os import ( environ, @@ -265,6 +264,41 @@ def _clone_src_cache_refresh(head: str, cache_key: str) -> None: shutil.copytree(head, cached) +def _attic_login(caches: List[Dict[str, Any]]) -> None: + for config in caches: + if config["type"] == "attic" and config["token"] in environ: + _run( + args=[ + "attic", + "login", + "local", + config["url"], + environ[config["token"]], + ], + stderr=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + ) + _run( + args=[ + "attic", + "cache", + "create", + config["name"], + ], + stderr=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + ) + _run( + args=[ + "attic", + "use", + config["name"], + ], + stderr=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + ) + + def _nix_build( *, attr: str, @@ -278,9 +312,21 @@ def _nix_build( "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" ) else: - substituters = " ".join(map(operator.itemgetter("url"), cache)) - trusted_pub_keys = " ".join(map(operator.itemgetter("pubKey"), cache)) - + _attic_login(cache) + substituters = " ".join( + [ + item["url"] + for item in cache + if "url" in item and "pubKey" in item and item["url"] + ] + ) + trusted_pub_keys = " ".join( + [ + item["pubKey"] + for item in cache + if "url" in item and "pubKey" in item and item["pubKey"] + ] + ) return [ *_if(NIX_STABLE, f"{__NIX_STABLE__}/bin/nix-build"), *_if(not NIX_STABLE, f"{__NIX_UNSTABLE__}/bin/nix"), @@ -642,17 +688,55 @@ def execute_action(args: List[str], head: str, out: str) -> None: def cache_push(cache: List[Dict[str, str]], out: str) -> None: - once: bool = True - for config in cache: - if config["type"] == "cachix" and "CACHIX_AUTH_TOKEN" in environ: - if once: - CON.rule("Pushing to cache") - once = False + once: Dict[str, bool] = {"cachix": True, "attic": True} + for config in [item for item in cache if item["type"] == "cachix"]: + if config["token"] in environ: + if once["cachix"]: + CON.rule("Pushing to cachix") + once["cachix"] = False _run( args=["cachix", "push", "-c", "0", config["name"], out], stderr=None, stdout=sys.stderr.fileno(), ) + + for config in [item for item in cache if item["type"] == "attic"]: + if config["token"] in environ: + if once["attic"]: + CON.rule("Pushing to attic") + once["attic"] = False + _run( + args=[ + "attic", + "login", + "local", + config["url"], + environ[config["token"]], + ], + stderr=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + ) + _run( + args=[ + "attic", + "cache", + "create", + config["name"], + ], + stderr=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + ) + _run( + args=[ + "attic", + "push", + "--ignore-upstream-cache-filter", + config["name"], + out, + ], + stderr=sys.stderr.fileno(), + stdout=sys.stderr.fileno(), + ) return diff --git a/src/evaluator/modules/cache/default.nix b/src/evaluator/modules/cache/default.nix index 60429ec4..2ba25d74 100644 --- a/src/evaluator/modules/cache/default.nix +++ b/src/evaluator/modules/cache/default.nix @@ -5,28 +5,32 @@ }: { options = { cache = { - readAndWrite = { - enable = lib.mkOption { - default = false; - type = lib.types.bool; - }; - name = lib.mkOption { - type = lib.types.str; - }; - pubKey = lib.mkOption { - type = lib.types.str; - }; - }; - readExtra = lib.mkOption { - default = []; - type = lib.types.listOf (lib.types.submodule (_: { + extra = lib.mkOption { + default = {}; + type = lib.types.attrsOf (lib.types.submodule (_: { options = { + enable = lib.mkOption { + default = false; + type = lib.types.bool; + }; pubKey = lib.mkOption { + default = ""; + type = lib.types.str; + }; + token = lib.mkOption { type = lib.types.str; + default = ""; + }; + type = lib.mkOption { + type = lib.types.enum ["cachix" "attic"]; }; url = lib.mkOption { type = lib.types.str; }; + write = lib.mkOption { + default = false; + type = lib.types.bool; + }; }; })); }; @@ -44,19 +48,19 @@ pubKey = "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="; type = "other"; }) - (listOptional config.cache.readAndWrite.enable { - name = config.cache.readAndWrite.name; - url = "https://${config.cache.readAndWrite.name}.cachix.org/"; - pubKey = config.cache.readAndWrite.pubKey; - type = "cachix"; - }) - (builtins.map - (cache: { - inherit (cache) url; - inherit (cache) pubKey; - type = "other"; - }) - config.cache.readExtra) + ( + builtins.filter + (cache: cache.enable) + (builtins.map (cacheName: { + inherit (config.cache.extra.${cacheName}) enable; + inherit (config.cache.extra.${cacheName}) pubKey; + inherit (config.cache.extra.${cacheName}) token; + inherit (config.cache.extra.${cacheName}) type; + inherit (config.cache.extra.${cacheName}) url; + inherit (config.cache.extra.${cacheName}) write; + name = cacheName; + }) (builtins.attrNames config.cache.extra)) + ) ]; }; };