docker cache deploy

contrib/kubespray/config/config.yaml

@@ -8,6 +8,12 @@ docker_image_tag: v1.5.0
 #                    OpenPAI Customized Settings                      #
 #######################################################################
 # enable_hived_scheduler: true
+# enable_docker_cache: false
+# docker_cache_azure_account_name: ""
+# docker_cache_azure_account_key: ""
+# docker_cache_azure_container_name: "dockerregistry"
+# docker_cache_remote_url: "https://registry-1.docker.io"
+# docker_cache_htpasswd: ""
 # enable_marketplace: false
 
 #############################################

contrib/kubespray/docker-cache-dist.yml

@@ -0,0 +1,11 @@
+- hosts: all
+  become: true
+  become_user: root
+  gather_facts: true
+  roles:
+    - { role: '../roles/docker-cache/install', enable_docker_cache: true }
+  tasks:
+    - name: Restart service docker config from /etc/docker/daemon.json after update
+      ansible.builtin.systemd:
+        name: docker
+        state: reloaded

contrib/kubespray/quick-start/services-configuration.yaml.template

@@ -34,6 +34,16 @@ cluster:
     # Must be lower case, e.g., regsecret.
     secret-name: pai-secret
 
+{% if env["cfg"]["enable_docker_cache"]|default(false) is sameas true %}
+docker-cache:
+  enabled: {{ env["cfg"]["enable_docker_cache"] }}
+  azure_account_name: {{ env["docker_cache"]["azure_account_name"] }}
+  azure_account_key: {{ env["docker_cache"]["azure_account_key"] }}
+  azure_container_name: {{ env["docker_cache"]["azure_container_name"] }}
+  remote_url: {{ env["docker_cache"]["remote_url"] }}
+  registry-htpasswd: {{ env["docker_cache"]["registry_htpasswd"] }}
+{% else %}
+{% endif %}
 
 rest-server:
 #  # launcher type. k8s or yarn

contrib/kubespray/roles/docker-cache/install/defaults/main.yml

@@ -0,0 +1 @@
+enable_docker_cache: false

contrib/kubespray/roles/docker-cache/install/files/daemon-openpai-default-runtime-docker-cache.json

@@ -0,0 +1,9 @@
+{
+  "storage-driver": "overlay2",
+  "registry-mirrors": [
+    "http://localhost:30500"
+  ],
+  "insecure-registries": [
+    "http://localhost:30500"
+  ]
+}

contrib/kubespray/roles/docker-cache/install/files/daemon-openpai-default-runtime.json

@@ -0,0 +1,3 @@
+{
+  "storage-driver": "overlay2"
+}

contrib/kubespray/roles/docker-cache/install/tasks/default-runtime-with-docker-cache.yml

@@ -0,0 +1,12 @@
+
+---
+- name: create docker in /etc
+  file:
+    path: /etc/docker
+    state: directory
+    recurse: yes
+
+- name: copy default runtime configuration file into /etc/docker
+  copy:
+    src: daemon-openpai-default-runtime-docker-cache.json
+    dest: /etc/docker/daemon.json

contrib/kubespray/roles/docker-cache/install/tasks/default-runtime.yml

@@ -0,0 +1,11 @@
+---
+- name: create docker in /etc
+  file:
+    path: /etc/docker
+    state: directory
+    recurse: yes
+
+- name: copy default runtime configuration file into /etc/docker
+  copy:
+    src: daemon-openpai-default-runtime.json
+    dest: /etc/docker/daemon.json

contrib/kubespray/roles/docker-cache/install/tasks/main.yml

@@ -0,0 +1,10 @@
+---
+- name: "Nvidia-smi is not detected, include default docker runtime task"
+  include_tasks: default-runtime.yml
+  when:
+    - not enable_docker_cache
+
+- name: "Nvidia-smi is not detected, docker-cache enabled, include default docker runtime task"
+  include_tasks: default-runtime-with-docker-cache.yml
+  when:
+    - enable_docker_cache

contrib/kubespray/roles/docker-runtime/install/defaults/main.yml

@@ -1 +1 @@
-install_run_time: true
+install_run_time: false

contrib/kubespray/script/k8s_generator.py

@@ -9,6 +9,54 @@
 logger = get_logger(__name__)
 
 
+def get_docker_cache_config_and_mirrors(layout, cluster_config):
+    """
+    generate hived config from layout.yaml and config.yaml
+    Resources (gpu/cpu/mem) specified in layout.yaml is considered as the total resources.
+
+    Parameters:
+    -----------
+    layout: dict
+        layout
+    cluster_config: dict
+        cluster config
+
+    Returns:
+    --------
+    dict, list
+        docker-cache mirrors, used to render docker-cache mirrors template
+        Example:
+        {
+            "azure_account_name": "",
+            "azure_account_key": "",
+            "azure_container_name": "dockerregistry",
+            "remote_url": "",
+            "registry-htpasswd": "",
+        }, [mirror_list]
+    """
+    pai_master_ips = []
+    for machine in layout['machine-list']:
+        if 'pai-master' in machine and machine['pai-master'] == 'true':
+            pai_master_ips.append(machine['hostip'])
+    docker_cache_mirrors = ["http://{}:30500".format(ip) for ip in pai_master_ips]
+
+    if "docker_cache_azure_container_name" not in cluster_config:
+        cluster_config['docker_cache_azure_container_name'] = "dockerregistry"
+    if "docker_cache_remote_url" not in cluster_config:
+        cluster_config['docker_cache_remote_url'] = "https://registry-1.docker.io"
+    if "docker_cache_htpasswd" in cluster_config:
+        cluster_config["docker_cache_htpasswd"] = ""
+    docker_cache_config = {
+        "azure_account_name": cluster_config['docker_cache_azure_account_name'],
+        "azure_account_key": cluster_config['docker_cache_azure_account_key'],
+        "azure_container_name": cluster_config['docker_cache_azure_container_name'],
+        "remote_url": cluster_config['docker_cache_remote_url'],
+        "registry_htpasswd": cluster_config['docker_cache_htpasswd'],
+    }
+
+    return docker_cache_config, docker_cache_mirrors
+
+
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument('-l', '--layout', dest="layout", required=True,
@@ -46,6 +94,24 @@ def main():
             logger.warning("https://docs.projectcalico.org/reference/public-cloud/azure#why-doesnt-azure-support-calico-networking")
             sys.exit(1)
 
+    # Docker-cache is disabled by default.
+    # But if the user sets enable_hived_scheduler to true manually,
+    # we should enable it.
+    if 'enable_docker_cache' in cluster_config and cluster_config['enable_docker_cache'] is True:
+        _, docker_cache_mirrors = get_docker_cache_config_and_mirrors(layout, cluster_config)
+    else:
+        _, docker_cache_mirrors = {}, []
+        cluster_config['enable_docker_cache'] = False
+
+    if "openpai_docker_registry_mirrors" in cluster_config:
+        cluster_config["openpai_docker_registry_mirrors"] += docker_cache_mirrors
+    else:
+        cluster_config["openpai_docker_registry_mirrors"] = docker_cache_mirrors
+    if "openpai_docker_insecure_registries" in cluster_config:
+        cluster_config["openpai_docker_insecure_registries"] += docker_cache_mirrors
+    else:
+        cluster_config["openpai_docker_insecure_registries"] = docker_cache_mirrors
+
     environment = {
         'masters': masters,
         'workers': workers,

contrib/kubespray/script/openpai_generator.py

@@ -233,6 +233,54 @@ def get_hived_config(layout, cluster_config):
     return { "skus": skus }
 
 
+def get_docker_cache_config_and_mirrors(layout, cluster_config):
+    """
+    generate hived config from layout.yaml and config.yaml
+    Resources (gpu/cpu/mem) specified in layout.yaml is considered as the total resources.
+
+    Parameters:
+    -----------
+    layout: dict
+        layout
+    cluster_config: dict
+        cluster config
+
+    Returns:
+    --------
+    dict, list
+        docker-cache mirrors, used to render docker-cache mirrors template
+        Example:
+        {
+            "azure_account_name": "",
+            "azure_account_key": "",
+            "azure_container_name": "dockerregistry",
+            "remote_url": "",
+            "registry-htpasswd": "",
+        }, [mirror_list]
+    """
+    pai_master_ips = []
+    for machine in layout['machine-list']:
+        if 'pai-master' in machine and machine['pai-master'] == 'true':
+            pai_master_ips.append(machine['hostip'])
+    docker_cache_mirrors = ["http://{}:30500".format(ip) for ip in pai_master_ips]
+
+    if "docker_cache_azure_container_name" not in cluster_config:
+        cluster_config['docker_cache_azure_container_name'] = "dockerregistry"
+    if "docker_cache_remote_url" not in cluster_config:
+        cluster_config['docker_cache_remote_url'] = "https://registry-1.docker.io"
+    if "docker_cache_htpasswd" in cluster_config:
+        cluster_config["docker_cache_htpasswd"] = ""
+    docker_cache_config = {
+        "azure_account_name": cluster_config['docker_cache_azure_account_name'],
+        "azure_account_key": cluster_config['docker_cache_azure_account_key'],
+        "azure_container_name": cluster_config['docker_cache_azure_container_name'],
+        "remote_url": cluster_config['docker_cache_remote_url'],
+        "registry_htpasswd": cluster_config['docker_cache_htpasswd'],
+    }
+
+    return docker_cache_config, docker_cache_mirrors
+
+
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument('-l', '--layout', dest="layout", required=True,
@@ -258,12 +306,31 @@ def main():
     else:
         hived_config = get_hived_config(layout, cluster_config)
 
+    # Docker-cache is disabled by default.
+    # But if the user sets enable_hived_scheduler to true manually,
+    # we should enable it.
+    if 'enable_docker_cache' in cluster_config and cluster_config['enable_docker_cache'] is True:
+        docker_cache_config, docker_cache_mirrors = get_docker_cache_config_and_mirrors(layout, cluster_config)
+    else:
+        docker_cache_config, docker_cache_mirrors = {}, []
+        cluster_config['enable_docker_cache'] = False
+
+    if "openpai_docker_registry_mirrors" in cluster_config:
+        cluster_config["openpai_docker_registry_mirrors"] += docker_cache_mirrors
+    else:
+        cluster_config["openpai_docker_registry_mirrors"] = docker_cache_mirrors
+    if "openpai_docker_insecure_registries" in cluster_config:
+        cluster_config["openpai_docker_insecure_registries"] += docker_cache_mirrors
+    else:
+        cluster_config["openpai_docker_insecure_registries"] = docker_cache_mirrors
+
     environment = {
         'masters': masters,
         'workers': workers,
         'cfg': cluster_config,
         'head_node': head_node,
-        'hived': hived_config
+        'hived': hived_config,
+        "docker_cache": docker_cache_config,
     }
 
     map_table = {

src/device-plugin/deploy/service.yaml

@@ -20,6 +20,7 @@ cluster-type:
 
 prerequisite:
   - cluster-configuration
+  - docker-cache
 
 template-list:
   - start.sh

src/docker-cache/config/docker-cache.yaml

@@ -0,0 +1,13 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+enableed: false
+azure_account_name: ""
+azure_account_key: ""
+azure_container_name: dockerregistry
+registry_listener: ":5000"
+container_port: 5000
+service_port: 5000
+service_nodeport: 30500
+remote_url: https://registry-1.docker.io
+registry-htpasswd: dGVzdDokMnkkMDUkRUZiaWphaHovMHl4UC5xMFk1VW52TzljU2hHMThCRzM3QzBHNFhoRmFtTXdTUXdQLjBqQi4KCg==

src/docker-cache/config/docker_cache.py

@@ -0,0 +1,26 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+import copy
+
+class DockerCache(object):
+    def __init__(self, cluster_conf, service_conf, default_service_conf):
+        self.cluster_conf = cluster_conf
+        self.service_conf = dict(default_service_conf, **service_conf)
+
+    def validation_pre(self):
+        machine_list = self.cluster_conf['machine-list']
+        if len([host for host in machine_list if host.get('pai-master') == 'true']) < 1:
+            return False, '"pai-master=true" machine is required to deploy the docker-cache service'
+        return True, None
+
+    def run(self):
+        result = copy.deepcopy(self.service_conf)
+        machine_list = self.cluster_conf['machine-list']
+        server_port = self.service_conf['service_nodeport']
+        master_ip = [host['hostip'] for host in machine_list if host.get('pai-master') == 'true'][0]
+        result['uri'] = 'http://{0}:{1}'.format(master_ip, server_port)
+        return result
+
+    def validation_post(self, conf):
+        return True, None

src/docker-cache/deploy/delete.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+pushd $(dirname "$0") > /dev/null
+
+/bin/bash stop.sh || exit $?
+
+popd > /dev/null

src/docker-cache/deploy/docker-cache-config.yaml.template

@@ -0,0 +1,39 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: registry-config
+  namespace: default
+data:
+  config.yml: |
+    version: 0.1
+    log:
+      fields:
+        service: registry
+    storage:
+      cache:
+        blobdescriptor: inmemory
+      delete:
+        enabled: true
+      azure:
+        accountname: {{ cluster_cfg["docker-cache"]["azure_account_name"] }}
+        accountkey: {{ cluster_cfg["docker-cache"]["azure_account_key"] }}
+        container: {{ cluster_cfg["docker-cache"]["azure_container_name"] }}
+        # realm:  core.windows.net
+    http:
+      addr: {{ cluster_cfg["docker-cache"]["registry_listener"] }}
+      headers:
+        X-Content-Type-Options: [nosniff]
+    # auth:
+    #   htpasswd:
+    #     realm: basic-realm
+    #     path: /auth/htpasswd # /etc/registry
+    proxy:
+      remoteurl: {{ cluster_cfg["docker-cache"]["remote_url"] }}
+    health:
+      storagedriver:
+        enabled: true
+        interval: 10s
+        threshold: 3

src/docker-cache/deploy/docker-cache-secret.yaml.template

@@ -0,0 +1,11 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License
+
+apiVersion: v1
+kind: Secret
+metadata:
+  name: registry-htpasswd
+  namespace: default
+data:
+  htpasswd: | # test test as default, to generate htpasswd please refer to README
+    {{ cluster_cfg["docker-cache"]["registry-htpasswd"] }}