refac(back): fluidattacks#1007 redefine cache module

- make the cache module generic
- add priority to use the different caches
- add cache server type
- by default the server only read the cache,
  it must be specified that the cache is also written
- add doc
- update the cli to use thw new format of caches

.mailmap

@@ -7,6 +7,7 @@ David Acevedo <dacevedoa12@gmail.com> David Acevedo <dacevedoa12@gmail.com>
 David Arnold <david.arnold@iohk.io> David Arnold <david.arnold@iohk.io>
 David Arnold <david.arnold@iohk.io> David Arnold <dgx.arnold@gmail.com>
 Diego Restrepo <drestrepo@fluidattacks.com> Diego Restrepo Mesa <36453706+drestrepom@users.noreply.github.com>
+Diego Restrepo <drestrepo@fluidttacks.com> Diego Restrepo <drestrepo@fluidttacks.com>
 Diego Restrepo <restrepomesadiego@gmail.com> Diego Restrepo Mesa <36453706+drestrepom@users.noreply.github.com>
 Fluid Attacks <help@fluidattacks.com> Fluid Attacks <help@fluidattacks.com>
 Github Dependabot <49699333+dependabot[bot]@users.noreply.github.com> dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

README.md

@@ -213,6 +213,16 @@ Real life projects that run entirely on [Makes][makes]:
 
 # Contents
 
+- [🦄 Makes](#-makes)
+  - [Want to get your hands dirty?](#want-to-get-your-hands-dirty)
+  - [At a glance](#at-a-glance)
+    - [Cloud native applications with Kubernetes ☸](#cloud-native-applications-with-kubernetes-)
+    - [Large scale computing on the cloud 🏋](#large-scale-computing-on-the-cloud-)
+    - [Declarative infra, declarative CI/CD, pure profit](#declarative-infra-declarative-cicd-pure-profit)
+    - [From dev to prod 🌟](#from-dev-to-prod-)
+  - [Production ready](#production-ready)
+    - [Demos](#demos)
+- [Contents](#contents)
 - [Why](#why)
 - [Goal](#goal)
 - [Getting started](#getting-started)
@@ -2107,24 +2117,21 @@ and optionally a [Cachix][cachix] cache for reading and writting.
 Types:
 
 - cache:
+  - extra: (attrsOf (cacheExtra))
   - readNixos (`bool`): Optional.
     Set to `true` in order to add https://cache.nixos.org as a read cache.
     Defaults to `true`.
-  - readExtra (`listOf readCacheType`): Optional.
-    Extra caches to read, if any.
-    Defaults to `[ ]`.
-  - readAndWrite:
-    - enable (`boolean`): Optional.
-      Defaults to `false`.
-    - name (`str`):
-      Name of the [Cachix][cachix] cache.
-    - pubKey (`str`):
-      Public key of the [Cachix][cachix] cache.
-- readCacheType (`submodule`):
+- cacheExtra:
+  - enable (`str`): The current cache is enabled and the cache
+    is read on the server.
+  - priority (`int`): the priority that the cache has when being read.
+  - pubKey (`str`): public key to read the cache.
+  - token (`str`): the name of the environment variable that contains the
+    token tu push the cache.
+  - type: (`enum [cachix | attic]`): the cache server type.
   - url (`str`):
     URL of the cache.
-  - pubKey (`str`):
-    Public key of the cache.
+  - write (`bool`): the cache is enabled to push the binary cache.
 
 Required environment variables:
 
@@ -2140,20 +2147,25 @@ Example `makes.nix`:
 {
   cache = {
     readNixos = true;
-    readExtra = [
-      {
-        url = "https://example.com";
-        pubKey = "example.example.org-1:123...";
-      }
-      {
-        url = "https://example2.com";
-        pubKey = "example2.example2.org-1:123...";
-      }
-    ];
-    readAndWrite = {
-      enable = true;
-      name = "makes";
-      pubKey = "makes.cachix.org-1:HbCQcdlYyT/mYuOx6rlgkNkonTGUjVr3D+YpuGRmO+Y=";
+    extra = {
+      main = {
+        enable = true;
+        priority = 20;
+        pubKey = "makes.cachix.org-1:zO7UjWLTRR8Vfzkgsu1PESjmb6ymy1e4OE9YfMmCQR4=";
+        token = "CACHIX_AUTH_TOKEN";
+        type = "nixos";
+        url = "https://makes.cachix.org";
+        write = true;
+      };
+      local = {
+        enable = true;
+        priority = 10;
+        pubKey = "local:nKOS5sOc0MKPoBJZmY4qWjbcXvoJFaO2S/zN6aUztII=";
+        token = "ATTIC_AUTH_TOKEN";
+        type = "attic";
+        url = "http://192.168.1.8:8085/local";
+        write = true;
+      };
     };
   };
 }

makes.nix

@@ -8,10 +8,26 @@
 }: {
   projectIdentifier = "makes-repo";
   cache = {
-    readAndWrite = {
-      enable = true;
-      name = "makes";
-      pubKey = "makes.cachix.org-1:zO7UjWLTRR8Vfzkgsu1PESjmb6ymy1e4OE9YfMmCQR4=";
+    readNixos = true;
+    extra = {
+      main = {
+        enable = true;
+        priority = 20;
+        pubKey = "makes.cachix.org-1:zO7UjWLTRR8Vfzkgsu1PESjmb6ymy1e4OE9YfMmCQR4=";
+        token = "CACHIX_AUTH_TOKEN";
+        type = "cachix";
+        url = "https://makes.cachix.org";
+        write = true;
+      };
+      compute = {
+        enable = true;
+        priority = 10;
+        pubKey = "compute:RRi6ocgDbEOVxYwEq9v2gsqyrbVNzflvMr+tuhVu3C0=";
+        token = "ATTIC_AUTH_TOKEN";
+        type = "attic";
+        url = "http://192.168.1.11:8085/compute";
+        write = true;
+      };
     };
   };
   calculateScorecard = {

makes/cli/env/runtime/main.nix

@@ -2,18 +2,37 @@
   __nixpkgs__,
   makeSearchPaths,
   outputs,
+  fetchGithub,
+  __system__,
   ...
-}:
-makeSearchPaths {
-  bin = [
-    __nixpkgs__.cachix
-    __nixpkgs__.git
-    __nixpkgs__.gnutar
-    __nixpkgs__.gzip
-    __nixpkgs__.nixStable
-    __nixpkgs__.openssh
-  ];
-  source = [
-    outputs."/cli/env/runtime/pypi"
-  ];
-}
+}: let
+  attic =
+    (import (
+        fetchTarball {
+          url = "https://github.com/edolstra/flake-compat/archive/12c64ca55c1014cdc1b16ed5a804aa8576601ff2.tar.gz";
+          sha256 = "0jm6nzb83wa6ai17ly9fzpqc40wg1viib8klq8lby54agpl213w5";
+        }
+      ) {
+        src = fetchGithub {
+          owner = "zhaofengli";
+          repo = "attic";
+          rev = "863f8dcca3efce87a29853f6c842f85de594019e";
+          sha256 = "bFzHDHiG5Uwopu/dgje9WNt/KDcxyVinK/k0SYIBtGw=";
+        };
+      })
+    .defaultNix;
+in
+  makeSearchPaths {
+    bin = [
+      __nixpkgs__.cachix
+      __nixpkgs__.git
+      __nixpkgs__.gnutar
+      __nixpkgs__.gzip
+      __nixpkgs__.nixStable
+      __nixpkgs__.openssh
+      attic.outputs.packages.${__system__}.attic-client
+    ];
+    source = [
+      outputs."/cli/env/runtime/pypi"
+    ];
+  }

src/cli/main/cli.py

@@ -10,7 +10,6 @@
 )
 import io
 import json
-import operator
 import os
 from os import (
     environ,
@@ -278,8 +277,20 @@ def _nix_build(
             "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="
         )
     else:
-        substituters = " ".join(map(operator.itemgetter("url"), cache))
-        trusted_pub_keys = " ".join(map(operator.itemgetter("pubKey"), cache))
+        substituters = " ".join(
+            [
+                item["url"]
+                for item in cache
+                if "url" in item and "pubKey" in item and item["url"]
+            ]
+        )
+        trusted_pub_keys = " ".join(
+            [
+                item["pubKey"]
+                for item in cache
+                if "url" in item and "pubKey" in item and item["pubKey"]
+            ]
+        )
 
     return [
         *_if(NIX_STABLE, f"{__NIX_STABLE__}/bin/nix-build"),
@@ -642,17 +653,55 @@ def execute_action(args: List[str], head: str, out: str) -> None:
 
 
 def cache_push(cache: List[Dict[str, str]], out: str) -> None:
-    once: bool = True
-    for config in cache:
-        if config["type"] == "cachix" and "CACHIX_AUTH_TOKEN" in environ:
-            if once:
-                CON.rule("Pushing to cache")
-                once = False
+    once: Dict[str, bool] = {"cachix": True, "attic": True}
+    for config in [item for item in cache if item["type"] == "cachix"]:
+        if "CACHIX_AUTH_TOKEN" in environ:
+            if once["cachix"]:
+                CON.rule("Pushing to cachix")
+                once["cachix"] = False
             _run(
                 args=["cachix", "push", "-c", "0", config["name"], out],
                 stderr=None,
                 stdout=sys.stderr.fileno(),
             )
+
+    for config in [item for item in cache if item["type"] == "attic"]:
+        if config["token"] in environ:
+            if once["attic"]:
+                CON.rule("Pushing to attic")
+                once["attic"] = False
+            _run(
+                args=[
+                    "attic",
+                    "login",
+                    "local",
+                    config["url"],
+                    environ[config["token"]],
+                ],
+                stderr=sys.stderr.fileno(),
+                stdout=sys.stderr.fileno(),
+            )
+            _run(
+                args=[
+                    "attic",
+                    "cache",
+                    "create",
+                    config["name"],
+                ],
+                stderr=None,
+                stdout=None,
+            )
+            _run(
+                args=[
+                    "attic",
+                    "push",
+                    "--ignore-upstream-cache-filter",
+                    config["name"],
+                    out,
+                ],
+                stderr=sys.stderr.fileno(),
+                stdout=sys.stderr.fileno(),
+            )
             return
 
 
@@ -762,6 +811,11 @@ def write_provenance(
 def main(args: List[str]) -> None:
     try:
         try:
+            # _run(
+            #     args=["attic", "--help"],
+            #     stderr=sys.stderr.fileno(),
+            #     stdout=sys.stderr.fileno(),
+            # )
             cli(args)
         except SystemExit as err:
             CON.out()

src/evaluator/modules/cache/default.nix

@@ -5,28 +5,35 @@
 }: {
   options = {
     cache = {
-      readAndWrite = {
-        enable = lib.mkOption {
-          default = false;
-          type = lib.types.bool;
-        };
-        name = lib.mkOption {
-          type = lib.types.str;
-        };
-        pubKey = lib.mkOption {
-          type = lib.types.str;
-        };
-      };
-      readExtra = lib.mkOption {
-        default = [];
-        type = lib.types.listOf (lib.types.submodule (_: {
+      extra = lib.mkOption {
+        default = {};
+        type = lib.types.attrsOf (lib.types.submodule (_: {
           options = {
+            enable = lib.mkOption {
+              default = false;
+              type = lib.types.bool;
+            };
+            priority = lib.mkOption {
+              type = lib.types.ints.positive;
+            };
             pubKey = lib.mkOption {
+              default = "";
+              type = lib.types.str;
+            };
+            token = lib.mkOption {
               type = lib.types.str;
+              default = "";
+            };
+            type = lib.mkOption {
+              type = lib.types.enum ["cachix" "attic"];
             };
             url = lib.mkOption {
               type = lib.types.str;
             };
+            write = lib.mkOption {
+              default = false;
+              type = lib.types.bool;
+            };
           };
         }));
       };
@@ -44,19 +51,20 @@
           pubKey = "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=";
           type = "other";
         })
-        (listOptional config.cache.readAndWrite.enable {
-          name = config.cache.readAndWrite.name;
-          url = "https://${config.cache.readAndWrite.name}.cachix.org/";
-          pubKey = config.cache.readAndWrite.pubKey;
-          type = "cachix";
-        })
-        (builtins.map
-          (cache: {
-            inherit (cache) url;
-            inherit (cache) pubKey;
-            type = "other";
-          })
-          config.cache.readExtra)
+        (
+          builtins.filter
+          (cache: cache.enable)
+          (builtins.map (cacheName: {
+            inherit (config.cache.extra.${cacheName}) enable;
+            inherit (config.cache.extra.${cacheName}) priority;
+            inherit (config.cache.extra.${cacheName}) pubKey;
+            inherit (config.cache.extra.${cacheName}) token;
+            inherit (config.cache.extra.${cacheName}) type;
+            inherit (config.cache.extra.${cacheName}) url;
+            inherit (config.cache.extra.${cacheName}) write;
+            name = cacheName;
+          }) (builtins.attrNames config.cache.extra))
+        )
       ];
     };
   };