diff --git a/.mailmap b/.mailmap index 8af8430d..eaee2f81 100644 --- a/.mailmap +++ b/.mailmap @@ -7,6 +7,7 @@ David Acevedo David Acevedo David Arnold David Arnold David Arnold David Arnold Diego Restrepo Diego Restrepo Mesa <36453706+drestrepom@users.noreply.github.com> +Diego Restrepo Diego Restrepo Diego Restrepo Diego Restrepo Mesa <36453706+drestrepom@users.noreply.github.com> Fluid Attacks Fluid Attacks Github Dependabot <49699333+dependabot[bot]@users.noreply.github.com> dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> diff --git a/README.md b/README.md index 426e0909..e8413601 100644 --- a/README.md +++ b/README.md @@ -213,6 +213,16 @@ Real life projects that run entirely on [Makes][makes]: # Contents +- [🦄 Makes](#-makes) + - [Want to get your hands dirty?](#want-to-get-your-hands-dirty) + - [At a glance](#at-a-glance) + - [Cloud native applications with Kubernetes ☸](#cloud-native-applications-with-kubernetes-) + - [Large scale computing on the cloud 🏋](#large-scale-computing-on-the-cloud-) + - [Declarative infra, declarative CI/CD, pure profit](#declarative-infra-declarative-cicd-pure-profit) + - [From dev to prod 🌟](#from-dev-to-prod-) + - [Production ready](#production-ready) + - [Demos](#demos) +- [Contents](#contents) - [Why](#why) - [Goal](#goal) - [Getting started](#getting-started) @@ -2107,24 +2117,21 @@ and optionally a [Cachix][cachix] cache for reading and writting. Types: - cache: + - extra: (attrsOf (cacheExtra)) - readNixos (`bool`): Optional. Set to `true` in order to add https://cache.nixos.org as a read cache. Defaults to `true`. - - readExtra (`listOf readCacheType`): Optional. - Extra caches to read, if any. - Defaults to `[ ]`. - - readAndWrite: - - enable (`boolean`): Optional. - Defaults to `false`. - - name (`str`): - Name of the [Cachix][cachix] cache. - - pubKey (`str`): - Public key of the [Cachix][cachix] cache. -- readCacheType (`submodule`): +- cacheExtra: + - enable (`str`): The current cache is enabled and the cache + is read on the server. + - priority (`int`): the priority that the cache has when being read. + - pubKey (`str`): public key to read the cache. + - token (`str`): the name of the environment variable that contains the + token tu push the cache. + - type: (`enum [cachix | attic]`): the cache server type. - url (`str`): URL of the cache. - - pubKey (`str`): - Public key of the cache. + - write (`bool`): the cache is enabled to push the binary cache. Required environment variables: @@ -2140,20 +2147,25 @@ Example `makes.nix`: { cache = { readNixos = true; - readExtra = [ - { - url = "https://example.com"; - pubKey = "example.example.org-1:123..."; - } - { - url = "https://example2.com"; - pubKey = "example2.example2.org-1:123..."; - } - ]; - readAndWrite = { - enable = true; - name = "makes"; - pubKey = "makes.cachix.org-1:HbCQcdlYyT/mYuOx6rlgkNkonTGUjVr3D+YpuGRmO+Y="; + extra = { + main = { + enable = true; + priority = 20; + pubKey = "makes.cachix.org-1:zO7UjWLTRR8Vfzkgsu1PESjmb6ymy1e4OE9YfMmCQR4="; + token = "CACHIX_AUTH_TOKEN"; + type = "nixos"; + url = "https://makes.cachix.org"; + write = true; + }; + local = { + enable = true; + priority = 10; + pubKey = "local:nKOS5sOc0MKPoBJZmY4qWjbcXvoJFaO2S/zN6aUztII="; + token = "ATTIC_AUTH_TOKEN"; + type = "attic"; + url = "http://192.168.1.8:8085/local"; + write = true; + }; }; }; } diff --git a/makes.nix b/makes.nix index 2409aa91..f98c9256 100644 --- a/makes.nix +++ b/makes.nix @@ -8,10 +8,26 @@ }: { projectIdentifier = "makes-repo"; cache = { - readAndWrite = { - enable = true; - name = "makes"; - pubKey = "makes.cachix.org-1:zO7UjWLTRR8Vfzkgsu1PESjmb6ymy1e4OE9YfMmCQR4="; + readNixos = true; + extra = { + main = { + enable = true; + priority = 20; + pubKey = "makes.cachix.org-1:zO7UjWLTRR8Vfzkgsu1PESjmb6ymy1e4OE9YfMmCQR4="; + token = "CACHIX_AUTH_TOKEN"; + type = "cachix"; + url = "https://makes.cachix.org"; + write = true; + }; + compute = { + enable = true; + priority = 10; + pubKey = "compute:RRi6ocgDbEOVxYwEq9v2gsqyrbVNzflvMr+tuhVu3C0="; + token = "ATTIC_AUTH_TOKEN"; + type = "attic"; + url = "http://192.168.1.11:8085/compute"; + write = true; + }; }; }; calculateScorecard = { diff --git a/makes/cli/env/runtime/main.nix b/makes/cli/env/runtime/main.nix index c791fa78..67cd6c2e 100644 --- a/makes/cli/env/runtime/main.nix +++ b/makes/cli/env/runtime/main.nix @@ -2,18 +2,37 @@ __nixpkgs__, makeSearchPaths, outputs, + fetchGithub, + __system__, ... -}: -makeSearchPaths { - bin = [ - __nixpkgs__.cachix - __nixpkgs__.git - __nixpkgs__.gnutar - __nixpkgs__.gzip - __nixpkgs__.nixStable - __nixpkgs__.openssh - ]; - source = [ - outputs."/cli/env/runtime/pypi" - ]; -} +}: let + attic = + (import ( + fetchTarball { + url = "https://github.com/edolstra/flake-compat/archive/12c64ca55c1014cdc1b16ed5a804aa8576601ff2.tar.gz"; + sha256 = "0jm6nzb83wa6ai17ly9fzpqc40wg1viib8klq8lby54agpl213w5"; + } + ) { + src = fetchGithub { + owner = "zhaofengli"; + repo = "attic"; + rev = "863f8dcca3efce87a29853f6c842f85de594019e"; + sha256 = "bFzHDHiG5Uwopu/dgje9WNt/KDcxyVinK/k0SYIBtGw="; + }; + }) + .defaultNix; +in + makeSearchPaths { + bin = [ + __nixpkgs__.cachix + __nixpkgs__.git + __nixpkgs__.gnutar + __nixpkgs__.gzip + __nixpkgs__.nixStable + __nixpkgs__.openssh + attic.outputs.packages.${__system__}.attic-client + ]; + source = [ + outputs."/cli/env/runtime/pypi" + ]; + } diff --git a/src/cli/main/cli.py b/src/cli/main/cli.py index 29ad243d..8187f987 100644 --- a/src/cli/main/cli.py +++ b/src/cli/main/cli.py @@ -10,7 +10,6 @@ ) import io import json -import operator import os from os import ( environ, @@ -278,8 +277,20 @@ def _nix_build( "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" ) else: - substituters = " ".join(map(operator.itemgetter("url"), cache)) - trusted_pub_keys = " ".join(map(operator.itemgetter("pubKey"), cache)) + substituters = " ".join( + [ + item["url"] + for item in cache + if "url" in item and "pubKey" in item and item["url"] + ] + ) + trusted_pub_keys = " ".join( + [ + item["pubKey"] + for item in cache + if "url" in item and "pubKey" in item and item["pubKey"] + ] + ) return [ *_if(NIX_STABLE, f"{__NIX_STABLE__}/bin/nix-build"), @@ -642,17 +653,55 @@ def execute_action(args: List[str], head: str, out: str) -> None: def cache_push(cache: List[Dict[str, str]], out: str) -> None: - once: bool = True - for config in cache: - if config["type"] == "cachix" and "CACHIX_AUTH_TOKEN" in environ: - if once: - CON.rule("Pushing to cache") - once = False + once: Dict[str, bool] = {"cachix": True, "attic": True} + for config in [item for item in cache if item["type"] == "cachix"]: + if "CACHIX_AUTH_TOKEN" in environ: + if once["cachix"]: + CON.rule("Pushing to cachix") + once["cachix"] = False _run( args=["cachix", "push", "-c", "0", config["name"], out], stderr=None, stdout=sys.stderr.fileno(), ) + + for config in [item for item in cache if item["type"] == "attic"]: + if config["token"] in environ: + if once["attic"]: + CON.rule("Pushing to attic") + once["attic"] = False + _run( + args=[ + "attic", + "login", + "local", + config["url"], + environ[config["token"]], + ], + stderr=sys.stderr.fileno(), + stdout=sys.stderr.fileno(), + ) + _run( + args=[ + "attic", + "cache", + "create", + config["name"], + ], + stderr=None, + stdout=None, + ) + _run( + args=[ + "attic", + "push", + "--ignore-upstream-cache-filter", + config["name"], + out, + ], + stderr=sys.stderr.fileno(), + stdout=sys.stderr.fileno(), + ) return @@ -762,6 +811,11 @@ def write_provenance( def main(args: List[str]) -> None: try: try: + # _run( + # args=["attic", "--help"], + # stderr=sys.stderr.fileno(), + # stdout=sys.stderr.fileno(), + # ) cli(args) except SystemExit as err: CON.out() diff --git a/src/evaluator/modules/cache/default.nix b/src/evaluator/modules/cache/default.nix index 60429ec4..9bde3aa3 100644 --- a/src/evaluator/modules/cache/default.nix +++ b/src/evaluator/modules/cache/default.nix @@ -5,28 +5,35 @@ }: { options = { cache = { - readAndWrite = { - enable = lib.mkOption { - default = false; - type = lib.types.bool; - }; - name = lib.mkOption { - type = lib.types.str; - }; - pubKey = lib.mkOption { - type = lib.types.str; - }; - }; - readExtra = lib.mkOption { - default = []; - type = lib.types.listOf (lib.types.submodule (_: { + extra = lib.mkOption { + default = {}; + type = lib.types.attrsOf (lib.types.submodule (_: { options = { + enable = lib.mkOption { + default = false; + type = lib.types.bool; + }; + priority = lib.mkOption { + type = lib.types.ints.positive; + }; pubKey = lib.mkOption { + default = ""; + type = lib.types.str; + }; + token = lib.mkOption { type = lib.types.str; + default = ""; + }; + type = lib.mkOption { + type = lib.types.enum ["cachix" "attic"]; }; url = lib.mkOption { type = lib.types.str; }; + write = lib.mkOption { + default = false; + type = lib.types.bool; + }; }; })); }; @@ -44,19 +51,20 @@ pubKey = "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="; type = "other"; }) - (listOptional config.cache.readAndWrite.enable { - name = config.cache.readAndWrite.name; - url = "https://${config.cache.readAndWrite.name}.cachix.org/"; - pubKey = config.cache.readAndWrite.pubKey; - type = "cachix"; - }) - (builtins.map - (cache: { - inherit (cache) url; - inherit (cache) pubKey; - type = "other"; - }) - config.cache.readExtra) + ( + builtins.filter + (cache: cache.enable) + (builtins.map (cacheName: { + inherit (config.cache.extra.${cacheName}) enable; + inherit (config.cache.extra.${cacheName}) priority; + inherit (config.cache.extra.${cacheName}) pubKey; + inherit (config.cache.extra.${cacheName}) token; + inherit (config.cache.extra.${cacheName}) type; + inherit (config.cache.extra.${cacheName}) url; + inherit (config.cache.extra.${cacheName}) write; + name = cacheName; + }) (builtins.attrNames config.cache.extra)) + ) ]; }; };