docker cache deploy

contrib/kubespray/config/config.yaml

@@ -8,6 +8,14 @@ docker_image_tag: v1.5.0
 #                    OpenPAI Customized Settings                      #
 #######################################################################
 # enable_hived_scheduler: true
+# enable_docker_cache: false
+# docker_cache_storage_backend: "azure" # or "filesystem"
+# docker_cache_azure_account_name: ""
+# docker_cache_azure_account_key: ""
+# docker_cache_azure_container_name: "dockerregistry"
+# docker_cache_fs_mount_path: "/var/lib/registry"
+# docker_cache_remote_url: "https://registry-1.docker.io"
+# docker_cache_htpasswd: ""
 # enable_marketplace: "false"
 
 #############################################

contrib/kubespray/docker-cache-config-distribute.yml

@@ -0,0 +1,14 @@
+- hosts: all
+  become: true
+  become_user: root
+  gather_facts: true
+  roles:
+    - role: '../roles/docker-cache/install'
+      vars:
+        enable_docker_cache: true
+        docker_cache_host: "{{ hostvars[groups['kube-master'][0]]['ip'] }}"
+  tasks:
+    - name: Restart service docker config from /etc/docker/daemon.json after update
+      ansible.builtin.systemd:
+        name: docker
+        state: restarted

contrib/kubespray/quick-start-kubespray.sh

@@ -38,5 +38,8 @@ ansible-playbook -i ${HOME}/pai-pre-check/pre-check.yml set-host-daemon-port-ran
 echo "Performing pre-installation..."
 ansible-playbook -i ${HOME}/pai-deploy/cluster-cfg/hosts.yml pre-installation.yml || exit $?
 
+echo "Performing docker-cache config distribution..."
+ansible-playbook -i ${HOME}/pai-deploy/cluster-cfg/hosts.yml docker-cache-config-distribute.yml || exit $?
+
 echo "Starting kubernetes..."
 /bin/bash script/kubernetes-boot.sh || exit $?

contrib/kubespray/quick-start/services-configuration.yaml.template

@@ -34,6 +34,28 @@ cluster:
     # Must be lower case, e.g., regsecret.
     secret-name: pai-secret
 
+{% if env["cfg"]["enable_docker_cache"]|default(false) is sameas true %}
+docker-cache:
+  enabled: {{ env["cfg"]["enable_docker_cache"] }}
+  {% if "docker_cache_storage_backend" in env["cfg"] %}
+  storage_backend: {{ env["cfg"]["docker_cache_storage_backend"]}}
+    {% if env["cfg"]["docker_cache_storage_backend"] == "azure" %}
+  azure_account_name: {{ env["cfg"]["docker_cache_azure_account_name"] }}
+  azure_account_key: {{ env["cfg"]["docker_cache_azure_account_key"] }}
+  azure_container_name: {{ env["cfg"]["docker_cache_azure_container_name"] }}
+    {% elif env["cfg"]["docker_cache_storage_backend"] == "filesystem" %}
+  fs_mount_path: {{ env["cfg"]["docker_cache_fs_mount_path"]}}
+    {% else %}
+    {% endif %}
+  {% endif %}
+  {% if "docker_cache_remote_url" in env["cfg"] %}
+  remote_url: {{ env["cfg"]["docker_cache_remote_url"] }}
+  {% endif %}
+  {% if "docker_cache_registry_htpasswd" in env["cfg"] %}
+  registry-htpasswd: {{ env["cfg"]["docker_cache_registry_htpasswd"] }}
+  {% endif %}
+{% else %}
+{% endif %}
 
 rest-server:
 #  # launcher type. k8s or yarn

contrib/kubespray/roles/docker-cache/install/defaults/main.yml

@@ -0,0 +1,2 @@
+enable_docker_cache: false
+docker_cache_host: 'localhost'

contrib/kubespray/roles/docker-cache/install/files/add_docker_cache_config.py

@@ -0,0 +1,46 @@
+from pathlib import Path
+import argparse
+import json
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("host", default="localhost", help="host addr of docker cache registry")
+    return parser.parse_args()
+
+
+def main():
+    args = parse_args()
+
+    folder_path = Path("/etc/docker")
+    target_path = Path("/etc/docker/daemon.json")
+    backup_path = Path("/etc/docker/daemon.json.bk")
+
+    folder_path.mkdir(parents=True, exist_ok=True)
+    target_path.touch(mode=0o666)
+    backup_path.touch(mode=0o666)
+
+    with open(str(target_path)) as f:
+        current_config = json.load(f);
+
+    with open(str(backup_path), 'w') as f:
+        json.dump(current_config, f)
+
+    docker_cache_mirror = "http://{}:30500".format(args.host)
+    if "registry-mirrors" in current_config:
+        if docker_cache_mirror not in current_config["registry-mirrors"]:
+            current_config["registry-mirrors"].append(docker_cache_mirror)
+    else:
+        current_config["registry-mirrors"] = [docker_cache_mirror]
+
+    if "insecure-registries" in current_config:
+        if docker_cache_mirror not in current_config["insecure-registries"]:
+            current_config["insecure-registries"].append(docker_cache_mirror)
+    else:
+        current_config["insecure-registries"] = [docker_cache_mirror]
+
+    with open(str(target_path), 'w') as f:
+        json.dump(current_config, f)
+
+if __name__ == "__main__":
+    main()

contrib/kubespray/roles/docker-cache/install/tasks/add-docker-cache-config.yml

@@ -0,0 +1,13 @@
+---
+- name: copy add_docker_cache_config.py script into /tmp
+  copy:
+    src: add_docker_cache_config.py
+    dest: /tmp/add_docker_cache_config.py
+
+- name: run add_docker_cache_config.py script to add docker cache config
+  ansible.builtin.command:
+    argv:
+      - sudo
+      - python3
+      - /tmp/add_docker_cache_config.py
+      - "{{ docker_cache_host }}"

contrib/kubespray/roles/docker-cache/install/tasks/main.yml

@@ -0,0 +1,5 @@
+---
+- name: "Add docker cache config"
+  include_tasks: add-docker-cache-config.yml
+  when:
+    - enable_docker_cache

contrib/kubespray/roles/docker-runtime/install/defaults/main.yml

@@ -1 +1 @@
-install_run_time: true
+install_run_time: false

contrib/kubespray/script/openpai_generator.py

@@ -263,7 +263,7 @@ def main():
         'workers': workers,
         'cfg': cluster_config,
         'head_node': head_node,
-        'hived': hived_config
+        'hived': hived_config,
     }
 
     map_table = {

src/device-plugin/deploy/service.yaml

@@ -20,6 +20,7 @@ cluster-type:
 
 prerequisite:
   - cluster-configuration
+  - docker-cache
 
 template-list:
   - start.sh

src/docker-cache/config/docker-cache.yaml

@@ -0,0 +1,18 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+enabled: false
+storage_backend: "azure"
+azure_account_name: ""
+azure_account_key: ""
+azure_container_name: dockerregistry
+fs_mount_path: "/var/lib/registry"
+registry_listener: ":5000"
+container_port: 5000
+service_port: 5000
+service_nodeport: 30500
+remote_url: https://registry-1.docker.io
+registry-htpasswd: '' 
+# registry-htpasswd default value can be dGVzdDokMnkkMDUkRUZiaWphaHovMHl4UC5xMFk1VW52TzljU2hHMThCRzM3QzBHNFhoRmFtTXdTUXdQLjBqQi4KCg==,
+# which is "username: test password: test" encoded by htpasswd and base64
+# htpasswd is an apache tool for generate basic auth credentials

src/docker-cache/config/docker_cache.py

@@ -0,0 +1,26 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+import copy
+
+class DockerCache(object):
+    def __init__(self, cluster_conf, service_conf, default_service_conf):
+        self.cluster_conf = cluster_conf
+        self.service_conf = dict(default_service_conf, **service_conf)
+
+    def validation_pre(self):
+        machine_list = self.cluster_conf['machine-list']
+        if len([host for host in machine_list if host.get('pai-master') == 'true']) < 1:
+            return False, '"pai-master=true" machine is required to deploy the docker-cache service'
+        return True, None
+
+    def run(self):
+        result = copy.deepcopy(self.service_conf)
+        machine_list = self.cluster_conf['machine-list']
+        server_port = self.service_conf['service_nodeport']
+        master_ip = [host['hostip'] for host in machine_list if host.get('pai-master') == 'true'][0]
+        result['uri'] = 'http://{0}:{1}'.format(master_ip, server_port)
+        return result
+
+    def validation_post(self, conf):
+        return True, None

src/docker-cache/deploy/delete.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+pushd $(dirname "$0") > /dev/null
+
+/bin/bash stop.sh || exit $?
+
+popd > /dev/null

src/docker-cache/deploy/docker-cache-config.yaml.template

@@ -0,0 +1,47 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: registry-config
+  namespace: default
+data:
+  config.yml: |
+    version: 0.1
+    log:
+      fields:
+        service: registry
+    storage:
+      cache:
+        blobdescriptor: inmemory
+      delete:
+        enabled: true
+      {% if cluster_cfg["docker-cache"]["storage_backend"] == "azure" %}
+      azure:
+        accountname: {{ cluster_cfg["docker-cache"]["azure_account_name"] }}
+        accountkey: {{ cluster_cfg["docker-cache"]["azure_account_key"] }}
+        container: {{ cluster_cfg["docker-cache"]["azure_container_name"] }}
+        # realm:  core.windows.net
+      {% elif cluster_cfg["docker-cache"]["storage_backend"] == "filesystem" %}
+      filesystem:
+        rootdirectory: {{ cluster_cfg["docker-cache"]["fs_mount_path"] }}
+      {% else %}
+      {% endif %}
+    http:
+      addr: {{ cluster_cfg["docker-cache"]["registry_listener"] }}
+      headers:
+        X-Content-Type-Options: [nosniff]
+    {{% if cluster_cfg["docker-cache"]["registry-htpasswd"] != '' %}}
+    auth:
+      htpasswd:
+        realm: basic-realm
+        path: /auth/htpasswd # /etc/registry
+    {{% endif %}}
+    proxy:
+      remoteurl: {{ cluster_cfg["docker-cache"]["remote_url"] }}
+    health:
+      storagedriver:
+        enabled: true
+        interval: 10s
+        threshold: 3

src/docker-cache/deploy/docker-cache-secret.yaml.template

@@ -0,0 +1,11 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License
+
+apiVersion: v1
+kind: Secret
+metadata:
+  name: registry-htpasswd
+  namespace: default
+data:
+  htpasswd: | # test test as default, to generate htpasswd please refer to README
+    {{ cluster_cfg["docker-cache"]["registry-htpasswd"] }}

src/docker-cache/deploy/docker-cache-service.yaml.template

@@ -0,0 +1,19 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: docker-registry-cache-service
+  namespace: default
+  labels:
+    app: docker-registry-cache
+spec:
+  type: NodePort
+  selector:
+    app: docker-registry-cache
+  ports:
+  - port: {{ cluster_cfg["docker-cache"]["service_port"] }}
+    name: http
+    targetPort: {{ cluster_cfg["docker-cache"]["container_port"] }}
+    nodePort: {{ cluster_cfg["docker-cache"]["service_nodeport"] }}

src/docker-cache/deploy/docker-cache.yaml.template

@@ -0,0 +1,40 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: docker-registry-cache
+  namespace: default
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: docker-registry-cache
+  template:
+    metadata:
+      labels:
+        app: docker-registry-cache
+    spec:
+      containers:
+      - name: docker-registry-cache
+        image: registry:2
+        imagePullPolicy: IfNotPresent
+        volumeMounts:
+          - name: registry-htpasswd
+            mountPath: /auth/
+            readOnly: True
+          - name: registry-config
+            mountPath: /etc/docker/registry/
+            readOnly: True
+        ports:
+          - name: http
+            containerPort: {{ cluster_cfg["docker-cache"]["container_port"] }}
+      volumes:
+        - name: registry-htpasswd
+          secret:
+            secretName: registry-htpasswd
+            defaultMode: 0400
+        - name: registry-config
+          configMap:
+            name: registry-config

src/docker-cache/deploy/refresh.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+pushd $(dirname "$0") > /dev/null
+
+bash stop.sh
+bash start.sh
+
+popd > /dev/null

src/docker-cache/deploy/service.yaml

@@ -0,0 +1,23 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+cluster-type:
+  - k8s
+
+prerequisite:
+  - cluster-configuration
+
+template-list:
+  - docker-cache-config.yaml
+  - docker-cache-secret.yaml
+  - docker-cache.yaml
+  - docker-cache-service.yaml
+  - start.sh
+
+start-script: start.sh
+stop-script: stop.sh
+delete-script: delete.sh
+refresh-script: refresh.sh
+
+deploy-rules:
+  - in: pai-master

src/docker-cache/deploy/start.sh.template

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+pushd $(dirname "$0") > /dev/null
+
+{% if cluster_cfg['docker-cache']['enabled'] %}
+kubectl apply --overwrite=true -f docker-cache-config.yaml || exit $?
+kubectl apply --overwrite=true -f docker-cache-secret.yaml || exit $?
+kubectl apply --overwrite=true -f docker-cache.yaml || exit $?
+kubectl apply --overwrite=true -f docker-cache-service.yaml || exit $?
+
+# Wait until the service is ready.
+PYTHONPATH="../../../deployment" python -m  k8sPaiLibrary.monitorTool.check_pod_ready_status -w -k app -v docker-registry-cache || exit $?
+{% else %}
+{% endif %}
+
+popd > /dev/null