add states to deal with hashicorp/consul#4977. We create a temporary …

…ACL not registered with dynamicsecrets to solve the chicken/egg problem until the Salt master has a Consul server available to create actual ACL tokens. This is annoying, complex, brittle, but necessary.
jdelic · Dec 2, 2018 · 8f322fc · 8f322fc
1 parent abc0236
commit 8f322fc
Show file tree

Hide file tree

Showing 2 changed files with 113 additions and 23 deletions.
diff --git a/srv/salt/consul/server.sls b/srv/salt/consul/server.sls
@@ -159,8 +159,97 @@ consul-server-service:
             - cmd: consul-sync
 
 
-# on the server, to fix the chicken egg problem of the ACL initialization, we install the server
-# and run it, then install the ACL config and restart the server.
+{% if pillar['dynamicsecrets']['consul-acl-token']['firstrun'] %}
+# on the master server, to fix the chicken egg problem of the ACL initialization, we install the server
+# and run it, then install a temporary ACL config and restart the server.
+consul-tempacl-create-policy:
+    cmd.run:
+        - name: |+
+            cat <<EOT | curl -X PUT -d @- -H "X-Consul-Token: $CONSUL_ACL_MASTER_TOKEN" http://169.254.1.1:8500/v1/acl/policy
+                {
+                    "Name": "tempacl-policy-{{grains['id']|replace('.', '-')}}",
+                    "Description": "Agent policy for {{grains['id']}}",
+                    "Rules": "
+                        key_prefix \"\" {
+                            policy = \"deny\"
+                        }
+
+                        key_prefix \"oauth2-clients\" {
+                            policy = \"write\"
+                        }
+
+                        node \"\" {
+                            policy = \"read\"
+                        }
+
+                        node \"{{grains['id']}}\" {
+                            policy = \"write\"
+                        }
+
+                        service_prefix \"\" {
+                            policy = \"write\"
+                        }
+
+                        agent \"{{grains['id']}}\" {
+                            policy = \"read\"
+                        }
+
+                        event_prefix \"\" {
+                            policy = \"read\"
+                        }
+
+                        query_prefix \"\" {
+                            policy = \"read\"
+                        }
+                    "
+                }
+            EOT
+        - env:
+            CONSUL_ACL_MASTER_TOKEN: {{pillar['dynamicsecrets']['consul-acl-master-token']}}
+        - unless: test -f /etc/consul/conf.d/agent_acl.json
+        - require:
+            - http: consul-server-service
+
+consul-tempacl-server-config:
+    cmd.run:
+        - name: |+
+            cat << EOT | curl -X PUT -d @- -H "X-Consul-Token: $CONSUL_ACL_MASTER_TOKEN" http://169.254.1.1:8500/v1/acl/token/ | \
+                jq -M "{acl: {tokens: { agent: .SecretID, default: .SecretID } } }" > \
+                    /etc/consul/conf.d/agent_acl.json
+                {
+                    "Description": "Temp ACL provisioning token for {{grains['id']}}",
+                    "Policies": [
+                        { "Name": "tempacl-policy-{{grains['id']|replace('.', '-')}}" }
+                    ]
+                }
+            EOT
+        - creates: /etc/consul/conf.d/agent_acl.json
+        - env:
+            CONSUL_ACL_MASTER_TOKEN: {{pillar['dynamicsecrets']['consul-acl-master-token']}}
+        - require:
+            - cmd: consul-tempacl-create-policy
+        - watch_in:
+            - service: consul-server-service-restart
+{% else %}
+# when we have a server, we run it, then
+consul-server-register-acl:
+    event.wait:
+        - name: maurusnet/consul/installed
+        - watch:
+            - service: consul-server-service
+    http.wait_for_successful_query:
+        - name: http://169.254.1.1:8500/v1/acl/info/{{pillar['dynamicsecrets']['consul-acl-token']['accessor_id']}}
+        - wait_for: 10
+        - request_interval: 1
+        - raise_error: False  # only exists in 'tornado' backend
+        - backend: tornado
+        - status: 200
+        - require:
+            - event: consul-server-register-acl
+        - require_in:
+            - cmd: consul-sync
+
+
 consul-acl-server-config:
     file.managed:
         - name: /etc/consul/conf.d/agent_acl.json
@@ -170,9 +259,12 @@ consul-acl-server-config:
         - mode: '0600'
         - template: jinja
         - context:
-              agent_acl_token: {{pillar['dynamicsecrets']['consul-acl-token']['secret_id']}}
+            agent_acl_token: {{pillar['dynamicsecrets']['consul-acl-token']['secret_id']}}
         - require:
-              - http: consul-server-register-acl
+            - http: consul-server-register-acl
+        - watch_in:
+            - service: consul-server-service-restart
+{% endif %}
 
 
 consul-server-service-restart:
@@ -183,7 +275,6 @@ consul-server-service-restart:
         - init_delay: 2
         - watch:
             - file: consul-acl-config
-            - file: consul-acl-server-config
             - file: consul-server-service  # if consul.service changes we want to *restart* (reload: False)
             - file: consul  # restart on a change of the binary
     http.wait_for_successful_query:
@@ -201,24 +292,6 @@ consul-server-service-restart:
             - cmd: consul-sync
 
 
-consul-server-register-acl:
-    event.wait:
-        - name: maurusnet/consul/installed
-        - watch:
-            - service: consul-server-service
-    http.wait_for_successful_query:
-        - name: http://169.254.1.1:8500/v1/acl/info/{{pillar['dynamicsecrets']['consul-acl-token']['accessor_id']}}
-        - wait_for: 10
-        - request_interval: 1
-        - raise_error: False  # only exists in 'tornado' backend
-        - backend: tornado
-        - status: 200
-        - require:
-            - event: consul-server-register-acl
-        - require_in:
-            - cmd: consul-sync
-
-
 {% if pillar['consul-cluster']['number-of-nodes'] == 1 %}
 consul-singlenode-snapshot-timer:
     file.managed:

diff --git a/srv/salt/consul/template.sls b/srv/salt/consul/template.sls
@@ -63,6 +63,23 @@ consul-template-config:
             - file: consul-basedir
 
 
+{% if pillar['dynamicsecrets']['consul-acl-token']['firstrun'] %}
+# work around the insane hoops we have to jump through for
+# https://github.com/hashicorp/consul/issues/4977
+consul-template-firstrun-config:
+    cmd.run:
+        - name: >
+            sed "s#^\(\s*\)token =.*#\1token = \"$(jq -r .acl.tokens.agent /etc/consul/conf.d/agent_acl.json)\"#" \
+                /etc/consul/consul-template.conf > /etc/consul/consul-template.conf.new;
+            mv /etc/consul/consul-template.conf.new /etc/consul/consul-template.conf
+        - onlyif: grep "first run" /etc/consul/consul-template.conf
+        - require:
+            - file: consul-template-config
+        - require_in:
+            - service: consul-template-service
+{% endif %}
+
+
 consul-template-service:
     file.managed:
         - name: /etc/systemd/system/consul-template.service