Squashed 'tools/' changes from 0d6d4da..74dc626

74dc626 Merge pull request #108 from weaveworks/disable-apt-daily
b4f1d91 Merge pull request #107 from weaveworks/docker-17-update
7436aa1 Override apt daily job to not run immediately on boot
7980f15 Merge pull request #106 from weaveworks/document-docker-install-role
f741e53 Bump to Docker 17.06 from CE repo
61796a1 Update Docker CE Debian repo details
0d86f5e Allow for Docker package to be named docker-ce
065c68d Document selection of Docker installation role.
3809053 Just --porcelain; it defaults to v1
11400ea Merge pull request #105 from weaveworks/remove-weaveplugin-remnants
b8b4d64 remove weaveplugin remnants
35099c9 Merge pull request #104 from weaveworks/pull-docker-py
cdd48fc Pull docker-py to speed tests/builds up.
e1c6c24 Merge pull request #103 from weaveworks/test-build-tags
d5d71e0 Add -tags option so callers can pass in build tags
8949b2b Merge pull request #98 from weaveworks/git-status-tag
ac30687 Merge pull request #100 from weaveworks/python_linting
4b125b5 Pin yapf & flake8 versions
7efb485 Lint python linting function
444755b Swap diff direction to reflect changes required
c5b2434 Install flake8 & yapf
5600eac Lint python in build-tools repo
0b02ca9 Add python linting
c011c0d Merge pull request #79 from kinvolk/schu/python-shebang
6577d07 Merge pull request #99 from weaveworks/shfmt-version
00ce0dc Use git status instead of diff to add 'WIP' tag
411fd13 Use shfmt v1.3.0 instead of latest from master.
31d069d Change Python shebang to `#!/usr/bin/env python`

git-subtree-dir: tools
git-subtree-split: 74dc626b6de3ffb38591510f7cb7bc2db33743c4

build/golang/Dockerfile

@@ -14,7 +14,8 @@ RUN apt-get update && \
       unzip && \
     rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 RUN pip install attrs pyhcl
-RUN curl -fsSL -o shfmt https://github.com/mvdan/sh/releases/download/v1.3.0/shfmt_v1.3.0_linux_amd64 && \
+RUN curl -fsSLo shfmt https://github.com/mvdan/sh/releases/download/v1.3.0/shfmt_v1.3.0_linux_amd64 && \
+	echo "b1925c2c405458811f0c227266402cf1868b4de529f114722c2e3a5af4ac7bb2  shfmt" | sha256sum -c && \
 	chmod +x shfmt && \
 	mv shfmt /usr/bin
 RUN go clean -i net && \

circle.yml

@@ -14,16 +14,18 @@ dependencies:
     - mkdir -p $(dirname $SRCDIR)
     - cp -r $(pwd)/ $SRCDIR
     - |
-      curl -fsSL -o shfmt https://github.com/mvdan/sh/releases/download/v1.3.0/shfmt_v1.3.0_linux_amd64 && \
-      chmod +x shfmt && \
-      sudo mv shfmt /usr/bin
+      curl -fsSLo shfmt https://github.com/mvdan/sh/releases/download/v1.3.0/shfmt_v1.3.0_linux_amd64 && \
+        echo "b1925c2c405458811f0c227266402cf1868b4de529f114722c2e3a5af4ac7bb2  shfmt" | sha256sum -c && \
+        chmod +x shfmt && \
+        sudo mv shfmt /usr/bin
     - |
       cd $SRCDIR;
       go get \
         github.com/fzipp/gocyclo \
         github.com/golang/lint/golint \
         github.com/kisielk/errcheck \
         github.com/fatih/hclfmt
+    - pip install yapf==0.16.2 flake8==3.3.0
 
 test:
   override:

config_management/README.md

@@ -113,6 +113,28 @@ N.B.: `--ssh-extra-args` is used to provide:
 * `StrictHostKeyChecking=no`: as VMs come and go, the same IP can be used by a different machine, so checking the host's SSH key may fail. Note that this introduces a risk of a man-in-the-middle attack.
 * `UserKnownHostsFile=/dev/null`: if you previously connected a VM with the same IP but a different public key, and added it to `~/.ssh/known_hosts`, SSH may still fail to connect, hence we use `/dev/null` instead of `~/.ssh/known_hosts`.
 
+
+### Docker installation role
+
+Various ways to install Docker are provided:
+
+- `docker-from-docker-ce-repo`
+- `docker-from-docker-repo`
+- `docker-from-get.docker.com`
+- `docker-from-tarball`
+
+each producing a slightly different outcome, which can be useful for testing various setup scenarios.
+
+The `docker-install` role selects one of the above ways to install Docker based on the `docker_install_role` variable.
+The default value for this variable is configured in `group_vars/all`.
+You can however override it with whichever role you would want to run by passing the name of the role as a key-value pair in `extra-vars`, e.g.:
+
+```
+ansible-playbook <playbook>.yml \
+    --extra-vars "docker_install_role=docker-from-docker-ce-repo"
+```
+
+
 ## Resources
 
 * [https://www.vagrantup.com/docs/provisioning/ansible.html](https://www.vagrantup.com/docs/provisioning/ansible.html)

config_management/group_vars/all

@@ -1,8 +1,8 @@
 ---
 go_version: 1.8.1
 terraform_version: 0.8.5
-docker_version: 1.11.2
-docker_install_role: 'docker-from-get.docker.com'
+docker_version: 17.06
+docker_install_role: 'docker-from-docker-ce-repo'
 kubernetes_version: 1.6.1
 kubernetes_cni_version: 0.5.1
 kubernetes_token: '123456.0123456789123456'

config_management/roles/dev-tools/files/apt-daily.timer.conf

@@ -0,0 +1,2 @@
+[Timer]
+Persistent=false

config_management/roles/dev-tools/tasks/main.yml

@@ -38,3 +38,11 @@
     dest: /usr/bin
     mode: 0555
     creates: /usr/bin/terraform
+
+# Ubuntu runs an apt update process that will run on first boot from image.
+# This is of questionable value when the machines are only going to live for a few minutes.
+# If you leave them on they will run the process daily.
+# Also we have seen the update process create a 'defunct' process which then throws off Weave Net smoke-test checks.
+# So, we override the 'persistent' setting so it will still run at the scheduled time but will not try to catch up on first boot.
+- name: copy apt daily override
+  copy: src=apt-daily.timer.conf dest=/etc/systemd/system/apt-daily.timer.d/

config_management/roles/docker-configuration/files/docker.conf

@@ -1,3 +1,3 @@
 [Service]
 ExecStart=
-ExecStart=/usr/bin/docker daemon -H fd:// -H unix:///var/run/alt-docker.sock -H tcp://0.0.0.0:2375 -s overlay --insecure-registry "weave-ci-registry:5000"
+ExecStart=/usr/bin/dockerd -H fd:// -H unix:///var/run/alt-docker.sock -H tcp://0.0.0.0:2375 -s overlay --insecure-registry "weave-ci-registry:5000"

config_management/roles/docker-from-docker-ce-repo/tasks/debian.yml

@@ -0,0 +1,35 @@
+---
+# Debian / Ubuntu specific:
+
+- name: install dependencies for docker repository
+  package:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - apt-transport-https
+    - ca-certificates
+
+- name: add apt key for the docker repository
+  apt_key:
+    keyserver: hkp://ha.pool.sks-keyservers.net:80
+    id: 9DC858229FC7DD38854AE2D88D81803C0EBFCD88
+    state: present
+  register: apt_key_docker_repo
+
+- name: add docker's apt repository ({{ ansible_distribution | lower }}-{{ ansible_distribution_release }})
+  apt_repository:
+    repo: deb https://download.docker.com/linux/ubuntu {{ ansible_lsb.codename|lower }} stable
+    state: present
+  register: apt_docker_repo
+
+- name: update apt's cache
+  apt:
+    update_cache: yes
+  when: apt_key_docker_repo.changed or apt_docker_repo.changed
+
+- name: install docker-engine
+  package:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - docker-ce={{ docker_version }}*

config_management/roles/docker-from-docker-ce-repo/tasks/main.yml

@@ -1,29 +1,10 @@
-# Docker installation from Docker's CentOS Community Edition
-# See also: https://docs.docker.com/engine/installation/linux/centos/
+---
+# Set up Docker
+# See also: https://docs.docker.com/engine/installation/linux/ubuntulinux/#install
 
-- name: remove all potentially pre existing packages
-  yum:
-    name: '{{ item }}'
-    state: absent
-  with_items:
-    - docker
-    - docker-common
-    - container-selinux
-    - docker-selinux
-    - docker-engine
+# Distribution-specific tasks:
+- include: debian.yml
+  when: ansible_os_family == "Debian"
 
-- name: install yum-utils
-  yum:
-    name: yum-utils
-    state: present
-
-- name: add docker ce repo
-  command: yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
-
-# Note that Docker CE versions do not follow regular Docker versions, but look
-# like, for example: "17.03.0.el7"
-- name: install docker
-  yum:
-    name: 'docker-ce-{{ docker_version }}'
-    update_cache: yes
-    state: present
+- include: redhat.yml
+  when: ansible_os_family == "RedHat"

config_management/roles/docker-from-docker-ce-repo/tasks/redhat.yml

@@ -0,0 +1,29 @@
+# Docker installation from Docker's CentOS Community Edition
+# See also: https://docs.docker.com/engine/installation/linux/centos/
+
+- name: remove all potentially pre existing packages
+  yum:
+    name: '{{ item }}'
+    state: absent
+  with_items:
+    - docker
+    - docker-common
+    - container-selinux
+    - docker-selinux
+    - docker-engine
+
+- name: install yum-utils
+  yum:
+    name: yum-utils
+    state: present
+
+- name: add docker ce repo
+  command: yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
+
+# Note that Docker CE versions do not follow regular Docker versions, but look
+# like, for example: "17.03.0.el7"
+- name: install docker
+  yum:
+    name: 'docker-ce-{{ docker_version }}'
+    update_cache: yes
+    state: present

config_management/roles/docker-from-get.docker.com/tasks/debian.yml

@@ -5,4 +5,4 @@
   shell: curl -sSL https://get.docker.com/gpg | sudo apt-key add -
 
 - name: install docker
-  shell: 'curl -sSL https://get.docker.com/ | sed -e s/docker-engine/docker-engine={{ docker_version }}*/ | sh'
+  shell: 'curl -sSL https://get.docker.com/ | sed -e s/docker-engine/docker-engine={{ docker_version }}*/ -e s/docker-ce/docker-ce={{ docker_version }}*/ | sh'

config_management/roles/weave-net-utilities/tasks/main.yml

@@ -45,3 +45,12 @@
     - alpine
     - aanand/docker-dnsutils
     - weaveworks/hello-world
+
+- name: docker pull docker-py which is used by tests
+  docker_image:
+    name: joffrey/docker-py
+    tag: '{{ item }}'
+    state: present
+  with_items:
+    - '1.8.1'
+    - '1.9.0-rc2'

dependencies/cross_versions.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 
 # Generate the cross product of latest versions of Weave Net's dependencies:
 # - Go
@@ -20,74 +20,72 @@
 from itertools import product
 
 # See also: /usr/include/sysexits.h
-_ERROR_RUNTIME=1
-_ERROR_ILLEGAL_ARGS=64
+_ERROR_RUNTIME = 1
+_ERROR_ILLEGAL_ARGS = 64
+
 
 def _usage(error_message=None):
-	if error_message:
-		stderr.write('ERROR: ' + error_message + linesep)
-	stdout.write(linesep.join([
-		'Usage:',
-		'    cross_versions.py [OPTION]...',
-		'Examples:',
-		'    cross_versions.py',
-		'    cross_versions.py -r',
-		'    cross_versions.py --rc',
-		'    cross_versions.py -l',
-		'    cross_versions.py --latest',
-		'Options:',
-		'-l/--latest Include only the latest version of each major and minor versions sub-tree.',
-		'-r/--rc     Include release candidate versions.',
-		'-h/--help   Prints this!',
-		''
-	]))
+    if error_message:
+        stderr.write('ERROR: ' + error_message + linesep)
+    stdout.write(
+        linesep.join([
+            'Usage:', '    cross_versions.py [OPTION]...', 'Examples:',
+            '    cross_versions.py', '    cross_versions.py -r',
+            '    cross_versions.py --rc', '    cross_versions.py -l',
+            '    cross_versions.py --latest', 'Options:',
+            '-l/--latest Include only the latest version of each major and'
+            ' minor versions sub-tree.',
+            '-r/--rc     Include release candidate versions.',
+            '-h/--help   Prints this!', ''
+        ]))
+
 
 def _validate_input(argv):
-	try:
-		config = {
-			'rc':     False,
-			'latest': False
-		}
-		opts, args = getopt(argv, 'hlr', ['help', 'latest', 'rc'])
-		for opt, value in opts:
-			if opt in ('-h', '--help'):
-				_usage()
-				exit()
-			if opt in ('-l', '--latest'):
-				config['latest'] = True
-			if opt in ('-r', '--rc'):
-				config['rc'] = True
-		if len(args) != 0:
-			raise ValueError('Unsupported argument(s): %s.' % args)
-		return config
-	except GetoptError as e:
-		_usage(str(e))
-		exit(_ERROR_ILLEGAL_ARGS)
-	except ValueError as e:
-		_usage(str(e))
-		exit(_ERROR_ILLEGAL_ARGS)
+    try:
+        config = {'rc': False, 'latest': False}
+        opts, args = getopt(argv, 'hlr', ['help', 'latest', 'rc'])
+        for opt, value in opts:
+            if opt in ('-h', '--help'):
+                _usage()
+                exit()
+            if opt in ('-l', '--latest'):
+                config['latest'] = True
+            if opt in ('-r', '--rc'):
+                config['rc'] = True
+        if len(args) != 0:
+            raise ValueError('Unsupported argument(s): %s.' % args)
+        return config
+    except GetoptError as e:
+        _usage(str(e))
+        exit(_ERROR_ILLEGAL_ARGS)
+    except ValueError as e:
+        _usage(str(e))
+        exit(_ERROR_ILLEGAL_ARGS)
+
 
 def _versions(dependency, config):
-	return map(str, 
-		filter_versions(
-			get_versions_from(DEPS[dependency]['url'], DEPS[dependency]['re']), 
-			DEPS[dependency]['min'], 
-			**config
-		)
-	)
+    return map(str,
+               filter_versions(
+                   get_versions_from(DEPS[dependency]['url'],
+                                     DEPS[dependency]['re']),
+                   DEPS[dependency]['min'], **config))
+
 
 def cross_versions(config):
-	docker_versions = _versions('docker',     config)
-	k8s_versions    = _versions('kubernetes', config)
-	return product(docker_versions, k8s_versions)
+    docker_versions = _versions('docker', config)
+    k8s_versions = _versions('kubernetes', config)
+    return product(docker_versions, k8s_versions)
+
 
 def main(argv):
-	try:
-		config = _validate_input(argv)
-		print(linesep.join('\t'.join(triple) for triple in cross_versions(config)))
-	except Exception as e:
-		print(str(e))
-		exit(_ERROR_RUNTIME)
+    try:
+        config = _validate_input(argv)
+        print(linesep.join('\t'.join(triple)
+                           for triple in cross_versions(config)))
+    except Exception as e:
+        print(str(e))
+        exit(_ERROR_RUNTIME)
+
 
 if __name__ == '__main__':
-	main(argv[1:])
+    main(argv[1:])