Squashed 'tools/' changes from 41c5622..35679ee (weaveworks#49)

35679ee Merge pull request weaveworks#110 from weaveworks/parallel-push-errors
3ae41b6 Remove unneeded if block
51ff31a Exit on first error
0faad9f Check for errors when pushing images in parallel
74dc626 Merge pull request weaveworks#108 from weaveworks/disable-apt-daily
b4f1d91 Merge pull request weaveworks#107 from weaveworks/docker-17-update
7436aa1 Override apt daily job to not run immediately on boot
7980f15 Merge pull request weaveworks#106 from weaveworks/document-docker-install-role
f741e53 Bump to Docker 17.06 from CE repo
61796a1 Update Docker CE Debian repo details
0d86f5e Allow for Docker package to be named docker-ce
065c68d Document selection of Docker installation role.
3809053 Just --porcelain; it defaults to v1
11400ea Merge pull request weaveworks#105 from weaveworks/remove-weaveplugin-remnants
b8b4d64 remove weaveplugin remnants
35099c9 Merge pull request weaveworks#104 from weaveworks/pull-docker-py
cdd48fc Pull docker-py to speed tests/builds up.
e1c6c24 Merge pull request weaveworks#103 from weaveworks/test-build-tags
d5d71e0 Add -tags option so callers can pass in build tags
8949b2b Merge pull request weaveworks#98 from weaveworks/git-status-tag
ac30687 Merge pull request weaveworks#100 from weaveworks/python_linting
4b125b5 Pin yapf & flake8 versions
7efb485 Lint python linting function
444755b Swap diff direction to reflect changes required
c5b2434 Install flake8 & yapf
5600eac Lint python in build-tools repo
0b02ca9 Add python linting
c011c0d Merge pull request weaveworks#79 from kinvolk/schu/python-shebang
6577d07 Merge pull request weaveworks#99 from weaveworks/shfmt-version
00ce0dc Use git status instead of diff to add 'WIP' tag
411fd13 Use shfmt v1.3.0 instead of latest from master.
0d6d4da Run shfmt 1.3 on the code.
5cdba32 Add sudo
c322ca8 circle.yml: Install shfmt binary.
e59c225 Install shfmt 1.3 binary.
30706e6 Install pyhcl in the build container.
960d222 Merge pull request weaveworks#97 from kinvolk/alban/update-shfmt-3
1d535c7 shellcheck: fix escaping issue
5542498 Merge pull request weaveworks#96 from kinvolk/alban/update-shfmt-2
32f7cc5 shfmt: fix coding style
09f72af lint: print the diff in case of error
571c7d7 Merge pull request weaveworks#95 from kinvolk/alban/update-shfmt
bead6ed Update for latest shfmt
b08dc4d Update for latest shfmt (weaveworks#94)
2ed8aaa Add no-race argument to test script (weaveworks#92)
80dd78e Merge pull request weaveworks#91 from weaveworks/upgrade-go-1.8.1
08dcd0d Please ./lint as shfmt changed its rules between 1.0.0 and 1.3.0.
a8bc9ab Upgrade default Go version to 1.8.1.
31d069d Change Python shebang to `#!/usr/bin/env python`

git-subtree-dir: tools
git-subtree-split: 35679ee5ff17c4edf864b7c43dc70a40337fcd80

tools/build/golang/Dockerfile

@@ -13,7 +13,11 @@ RUN apt-get update && \
       python-yaml \
       unzip && \
     rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
-RUN pip install attrs
+RUN pip install attrs pyhcl
+RUN curl -fsSLo shfmt https://github.com/mvdan/sh/releases/download/v1.3.0/shfmt_v1.3.0_linux_amd64 && \
+	echo "b1925c2c405458811f0c227266402cf1868b4de529f114722c2e3a5af4ac7bb2  shfmt" | sha256sum -c && \
+	chmod +x shfmt && \
+	mv shfmt /usr/bin
 RUN go clean -i net && \
 	go install -tags netgo std && \
 	go install -race -tags netgo std
@@ -29,7 +33,6 @@ RUN go get -tags netgo \
 		github.com/golang/protobuf/protoc-gen-go \
 		github.com/kisielk/errcheck \
 		github.com/mjibson/esc \
-		github.com/mvdan/sh/cmd/shfmt \
 		github.com/prometheus/prometheus/cmd/promtool && \
 		rm -rf /go/pkg /go/src
 RUN mkdir protoc && \

tools/build/golang/build.sh

@@ -13,8 +13,8 @@ fi
 # will have awkward ownership.  So we switch to a user with the
 # same user and group IDs as source directory.  We have to set a
 # few things up so that sudo works without complaining later on.
-uid=$(stat --format="%u" $SRC_PATH)
-gid=$(stat --format="%g" $SRC_PATH)
+uid=$(stat --format="%u" "$SRC_PATH")
+gid=$(stat --format="%g" "$SRC_PATH")
 echo "weave:x:$uid:$gid::$SRC_PATH:/bin/sh" >>/etc/passwd
 echo "weave:*:::::::" >>/etc/shadow
 echo "weave	ALL=(ALL)	NOPASSWD: ALL" >>/etc/sudoers

tools/build/haskell/build.sh

@@ -9,4 +9,4 @@ if [ -z "${SRC_PATH:-}" ]; then
     exit 1
 fi
 
-make -C $SRC_PATH BUILD_IN_CONTAINER=false $*
+make -C "$SRC_PATH" BUILD_IN_CONTAINER=false "$@"

tools/circle.yml

@@ -13,14 +13,19 @@ dependencies:
     - go install -tags netgo std
     - mkdir -p $(dirname $SRCDIR)
     - cp -r $(pwd)/ $SRCDIR
+    - |
+      curl -fsSLo shfmt https://github.com/mvdan/sh/releases/download/v1.3.0/shfmt_v1.3.0_linux_amd64 && \
+        echo "b1925c2c405458811f0c227266402cf1868b4de529f114722c2e3a5af4ac7bb2  shfmt" | sha256sum -c && \
+        chmod +x shfmt && \
+        sudo mv shfmt /usr/bin
     - |
       cd $SRCDIR;
       go get \
         github.com/fzipp/gocyclo \
         github.com/golang/lint/golint \
         github.com/kisielk/errcheck \
-        github.com/fatih/hclfmt \
-        gopkg.in/mvdan/sh.v1/cmd/shfmt
+        github.com/fatih/hclfmt
+    - pip install yapf==0.16.2 flake8==3.3.0
 
 test:
   override:

tools/config_management/README.md

@@ -113,6 +113,28 @@ N.B.: `--ssh-extra-args` is used to provide:
 * `StrictHostKeyChecking=no`: as VMs come and go, the same IP can be used by a different machine, so checking the host's SSH key may fail. Note that this introduces a risk of a man-in-the-middle attack.
 * `UserKnownHostsFile=/dev/null`: if you previously connected a VM with the same IP but a different public key, and added it to `~/.ssh/known_hosts`, SSH may still fail to connect, hence we use `/dev/null` instead of `~/.ssh/known_hosts`.
 
+
+### Docker installation role
+
+Various ways to install Docker are provided:
+
+- `docker-from-docker-ce-repo`
+- `docker-from-docker-repo`
+- `docker-from-get.docker.com`
+- `docker-from-tarball`
+
+each producing a slightly different outcome, which can be useful for testing various setup scenarios.
+
+The `docker-install` role selects one of the above ways to install Docker based on the `docker_install_role` variable.
+The default value for this variable is configured in `group_vars/all`.
+You can however override it with whichever role you would want to run by passing the name of the role as a key-value pair in `extra-vars`, e.g.:
+
+```
+ansible-playbook <playbook>.yml \
+    --extra-vars "docker_install_role=docker-from-docker-ce-repo"
+```
+
+
 ## Resources
 
 * [https://www.vagrantup.com/docs/provisioning/ansible.html](https://www.vagrantup.com/docs/provisioning/ansible.html)

tools/config_management/group_vars/all

@@ -1,8 +1,8 @@
 ---
-go_version: 1.7.4
+go_version: 1.8.1
 terraform_version: 0.8.5
-docker_version: 1.11.2
-docker_install_role: 'docker-from-get.docker.com'
+docker_version: 17.06
+docker_install_role: 'docker-from-docker-ce-repo'
 kubernetes_version: 1.6.1
 kubernetes_cni_version: 0.5.1
 kubernetes_token: '123456.0123456789123456'

tools/config_management/roles/dev-tools/files/apt-daily.timer.conf

@@ -0,0 +1,2 @@
+[Timer]
+Persistent=false

tools/config_management/roles/dev-tools/tasks/main.yml

@@ -38,3 +38,11 @@
     dest: /usr/bin
     mode: 0555
     creates: /usr/bin/terraform
+
+# Ubuntu runs an apt update process that will run on first boot from image.
+# This is of questionable value when the machines are only going to live for a few minutes.
+# If you leave them on they will run the process daily.
+# Also we have seen the update process create a 'defunct' process which then throws off Weave Net smoke-test checks.
+# So, we override the 'persistent' setting so it will still run at the scheduled time but will not try to catch up on first boot.
+- name: copy apt daily override
+  copy: src=apt-daily.timer.conf dest=/etc/systemd/system/apt-daily.timer.d/

tools/config_management/roles/docker-configuration/files/docker.conf

@@ -1,3 +1,3 @@
 [Service]
 ExecStart=
-ExecStart=/usr/bin/docker daemon -H fd:// -H unix:///var/run/alt-docker.sock -H tcp://0.0.0.0:2375 -s overlay --insecure-registry "weave-ci-registry:5000"
+ExecStart=/usr/bin/dockerd -H fd:// -H unix:///var/run/alt-docker.sock -H tcp://0.0.0.0:2375 -s overlay --insecure-registry "weave-ci-registry:5000"

tools/config_management/roles/docker-from-docker-ce-repo/tasks/debian.yml

@@ -0,0 +1,35 @@
+---
+# Debian / Ubuntu specific:
+
+- name: install dependencies for docker repository
+  package:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - apt-transport-https
+    - ca-certificates
+
+- name: add apt key for the docker repository
+  apt_key:
+    keyserver: hkp://ha.pool.sks-keyservers.net:80
+    id: 9DC858229FC7DD38854AE2D88D81803C0EBFCD88
+    state: present
+  register: apt_key_docker_repo
+
+- name: add docker's apt repository ({{ ansible_distribution | lower }}-{{ ansible_distribution_release }})
+  apt_repository:
+    repo: deb https://download.docker.com/linux/ubuntu {{ ansible_lsb.codename|lower }} stable
+    state: present
+  register: apt_docker_repo
+
+- name: update apt's cache
+  apt:
+    update_cache: yes
+  when: apt_key_docker_repo.changed or apt_docker_repo.changed
+
+- name: install docker-engine
+  package:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - docker-ce={{ docker_version }}*

tools/config_management/roles/docker-from-docker-ce-repo/tasks/main.yml

@@ -1,29 +1,10 @@
-# Docker installation from Docker's CentOS Community Edition
-# See also: https://docs.docker.com/engine/installation/linux/centos/
+---
+# Set up Docker
+# See also: https://docs.docker.com/engine/installation/linux/ubuntulinux/#install
 
-- name: remove all potentially pre existing packages
-  yum:
-    name: '{{ item }}'
-    state: absent
-  with_items:
-    - docker
-    - docker-common
-    - container-selinux
-    - docker-selinux
-    - docker-engine
+# Distribution-specific tasks:
+- include: debian.yml
+  when: ansible_os_family == "Debian"
 
-- name: install yum-utils
-  yum:
-    name: yum-utils
-    state: present
-
-- name: add docker ce repo
-  command: yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
-
-# Note that Docker CE versions do not follow regular Docker versions, but look
-# like, for example: "17.03.0.el7"
-- name: install docker
-  yum:
-    name: 'docker-ce-{{ docker_version }}'
-    update_cache: yes
-    state: present
+- include: redhat.yml
+  when: ansible_os_family == "RedHat"

tools/config_management/roles/docker-from-docker-ce-repo/tasks/redhat.yml

@@ -0,0 +1,29 @@
+# Docker installation from Docker's CentOS Community Edition
+# See also: https://docs.docker.com/engine/installation/linux/centos/
+
+- name: remove all potentially pre existing packages
+  yum:
+    name: '{{ item }}'
+    state: absent
+  with_items:
+    - docker
+    - docker-common
+    - container-selinux
+    - docker-selinux
+    - docker-engine
+
+- name: install yum-utils
+  yum:
+    name: yum-utils
+    state: present
+
+- name: add docker ce repo
+  command: yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
+
+# Note that Docker CE versions do not follow regular Docker versions, but look
+# like, for example: "17.03.0.el7"
+- name: install docker
+  yum:
+    name: 'docker-ce-{{ docker_version }}'
+    update_cache: yes
+    state: present

tools/config_management/roles/docker-from-get.docker.com/tasks/debian.yml

@@ -5,4 +5,4 @@
   shell: curl -sSL https://get.docker.com/gpg | sudo apt-key add -
 
 - name: install docker
-  shell: 'curl -sSL https://get.docker.com/ | sed -e s/docker-engine/docker-engine={{ docker_version }}*/ | sh'
+  shell: 'curl -sSL https://get.docker.com/ | sed -e s/docker-engine/docker-engine={{ docker_version }}*/ -e s/docker-ce/docker-ce={{ docker_version }}*/ | sh'

tools/config_management/roles/weave-net-utilities/tasks/main.yml

@@ -45,3 +45,12 @@
     - alpine
     - aanand/docker-dnsutils
     - weaveworks/hello-world
+
+- name: docker pull docker-py which is used by tests
+  docker_image:
+    name: joffrey/docker-py
+    tag: '{{ item }}'
+    state: present
+  with_items:
+    - '1.8.1'
+    - '1.9.0-rc2'

tools/dependencies/cross_versions.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 
 # Generate the cross product of latest versions of Weave Net's dependencies:
 # - Go
@@ -20,74 +20,72 @@
 from itertools import product
 
 # See also: /usr/include/sysexits.h
-_ERROR_RUNTIME=1
-_ERROR_ILLEGAL_ARGS=64
+_ERROR_RUNTIME = 1
+_ERROR_ILLEGAL_ARGS = 64
+
 
 def _usage(error_message=None):
-	if error_message:
-		stderr.write('ERROR: ' + error_message + linesep)
-	stdout.write(linesep.join([
-		'Usage:',
-		'    cross_versions.py [OPTION]...',
-		'Examples:',
-		'    cross_versions.py',
-		'    cross_versions.py -r',
-		'    cross_versions.py --rc',
-		'    cross_versions.py -l',
-		'    cross_versions.py --latest',
-		'Options:',
-		'-l/--latest Include only the latest version of each major and minor versions sub-tree.',
-		'-r/--rc     Include release candidate versions.',
-		'-h/--help   Prints this!',
-		''
-	]))
+    if error_message:
+        stderr.write('ERROR: ' + error_message + linesep)
+    stdout.write(
+        linesep.join([
+            'Usage:', '    cross_versions.py [OPTION]...', 'Examples:',
+            '    cross_versions.py', '    cross_versions.py -r',
+            '    cross_versions.py --rc', '    cross_versions.py -l',
+            '    cross_versions.py --latest', 'Options:',
+            '-l/--latest Include only the latest version of each major and'
+            ' minor versions sub-tree.',
+            '-r/--rc     Include release candidate versions.',
+            '-h/--help   Prints this!', ''
+        ]))
+
 
 def _validate_input(argv):
-	try:
-		config = {
-			'rc':     False,
-			'latest': False
-		}
-		opts, args = getopt(argv, 'hlr', ['help', 'latest', 'rc'])
-		for opt, value in opts:
-			if opt in ('-h', '--help'):
-				_usage()
-				exit()
-			if opt in ('-l', '--latest'):
-				config['latest'] = True
-			if opt in ('-r', '--rc'):
-				config['rc'] = True
-		if len(args) != 0:
-			raise ValueError('Unsupported argument(s): %s.' % args)
-		return config
-	except GetoptError as e:
-		_usage(str(e))
-		exit(_ERROR_ILLEGAL_ARGS)
-	except ValueError as e:
-		_usage(str(e))
-		exit(_ERROR_ILLEGAL_ARGS)
+    try:
+        config = {'rc': False, 'latest': False}
+        opts, args = getopt(argv, 'hlr', ['help', 'latest', 'rc'])
+        for opt, value in opts:
+            if opt in ('-h', '--help'):
+                _usage()
+                exit()
+            if opt in ('-l', '--latest'):
+                config['latest'] = True
+            if opt in ('-r', '--rc'):
+                config['rc'] = True
+        if len(args) != 0:
+            raise ValueError('Unsupported argument(s): %s.' % args)
+        return config
+    except GetoptError as e:
+        _usage(str(e))
+        exit(_ERROR_ILLEGAL_ARGS)
+    except ValueError as e:
+        _usage(str(e))
+        exit(_ERROR_ILLEGAL_ARGS)
+
 
 def _versions(dependency, config):
-	return map(str, 
-		filter_versions(
-			get_versions_from(DEPS[dependency]['url'], DEPS[dependency]['re']), 
-			DEPS[dependency]['min'], 
-			**config
-		)
-	)
+    return map(str,
+               filter_versions(
+                   get_versions_from(DEPS[dependency]['url'],
+                                     DEPS[dependency]['re']),
+                   DEPS[dependency]['min'], **config))
+
 
 def cross_versions(config):
-	docker_versions = _versions('docker',     config)
-	k8s_versions    = _versions('kubernetes', config)
-	return product(docker_versions, k8s_versions)
+    docker_versions = _versions('docker', config)
+    k8s_versions = _versions('kubernetes', config)
+    return product(docker_versions, k8s_versions)
+
 
 def main(argv):
-	try:
-		config = _validate_input(argv)
-		print(linesep.join('\t'.join(triple) for triple in cross_versions(config)))
-	except Exception as e:
-		print(str(e))
-		exit(_ERROR_RUNTIME)
+    try:
+        config = _validate_input(argv)
+        print(linesep.join('\t'.join(triple)
+                           for triple in cross_versions(config)))
+    except Exception as e:
+        print(str(e))
+        exit(_ERROR_RUNTIME)
+
 
 if __name__ == '__main__':
-	main(argv[1:])
+    main(argv[1:])

tools/dependencies/list_os_images.sh

@@ -8,11 +8,11 @@ Dependencies:
   - gcloud, Google Cloud Platform's CLI
   - aws,
 Usage: 
-  $ ./$(basename "$0") PROVIDER OS
+  \$ ./$(basename "$0") PROVIDER OS
   PROVIDER={gcp}
   OS={ubuntu|debian|centos}
 Example: 
-  $ ./$(basename "$0") gcp ubuntu
+  \$ ./$(basename "$0") gcp ubuntu
   ubuntu-os-cloud/ubuntu-1204-lts
   ubuntu-os-cloud/ubuntu-1404-lts
   ubuntu-os-cloud/ubuntu-1604-lts