[api] cluster spec etcd endpts; per-cluster envs

README.md

@@ -74,6 +74,11 @@ spec:
   image: quay.io/m3db/m3dbnode:latest
   replicationFactor: 3
   numberOfShards: 256
+  # Default endpoints if using provided etcd manifests.
+  etcdEndpoints:
+  - http://etcd-0.etcd:2379
+  - http://etcd-1.etcd:2379
+  - http://etcd-2.etcd:2379
   isolationGroups:
     - name: <zone-x>
       numInstances: 1

assets/default-config.tmpl

@@ -0,0 +1,86 @@
+coordinator:
+  listenAddress:
+    type: "config"
+    value: "0.0.0.0:7201"
+  local:
+    namespaces:
+    - namespace: default
+      type: unaggregated
+      retention: 48h
+  metrics:
+    scope:
+      prefix: "coordinator"
+    prometheus:
+      handlerPath: /metrics
+      listenAddress: 0.0.0.0:7203
+    sanitization: prometheus
+    samplingRate: 1.0
+    extended: none
+  tagOptions:
+    idScheme: quoted
+
+db:
+  logging:
+    level: info
+
+  metrics:
+    prometheus:
+      handlerPath: /metrics
+    sanitization: prometheus
+    samplingRate: 1.0
+    extended: detailed
+
+  listenAddress: 0.0.0.0:9000
+  clusterListenAddress: 0.0.0.0:9001
+  httpNodeListenAddress: 0.0.0.0:9002
+  httpClusterListenAddress: 0.0.0.0:9003
+  debugListenAddress: 0.0.0.0:9004
+
+  hostID:
+    resolver: file
+    file:
+      path: /etc/m3db/pod-identity/identity
+      timeout: 5m
+
+
+  client:
+    writeConsistencyLevel: majority
+    readConsistencyLevel: unstrict_majority
+
+  gcPercentage: 100
+
+  writeNewSeriesAsync: true
+  writeNewSeriesLimitPerSecond: 1048576
+  writeNewSeriesBackoffDuration: 2ms
+
+  bootstrap:
+    bootstrappers:
+        - filesystem
+        - commitlog
+        - peers
+        - uninitialized_topology
+    fs:
+        numProcessorsPerCPU: 0.125
+
+  commitlog:
+    flushMaxBytes: 524288
+    flushEvery: 1s
+    queue:
+        calculationType: fixed
+        size: 2097152
+
+  fs:
+    filePathPrefix: /var/lib/m3db
+
+  config:
+    service:
+        env: "{{ .Env }}"
+        zone: embedded
+        service: m3db
+        cacheDir: /var/lib/m3kv
+        etcdClusters:
+        - zone: embedded
+          endpoints:
+{{- range .Endpoints }}
+          - "{{- . }}"
+{{- end }}

assets/default-config.yaml

@@ -2,6 +2,11 @@ coordinator:
   listenAddress:
     type: "config"
     value: "0.0.0.0:7201"
+  local:
+    namespaces:
+    - namespace: default
+      type: unaggregated
+      retention: 48h
   metrics:
     scope:
       prefix: "coordinator"
@@ -37,24 +42,10 @@ db:
       path: /etc/m3db/pod-identity/identity
       timeout: 5m
 
+
   client:
     writeConsistencyLevel: majority
     readConsistencyLevel: unstrict_majority
-    writeTimeout: 10s
-    fetchTimeout: 15s
-    connectTimeout: 20s
-    writeRetry:
-        initialBackoff: 500ms
-        backoffFactor: 3
-        maxRetries: 2
-        jitter: true
-    fetchRetry:
-        initialBackoff: 500ms
-        backoffFactor: 2
-        maxRetries: 3
-        jitter: true
-    backgroundHealthCheckFailLimit: 4
-    backgroundHealthCheckFailThrottleFactor: 0.5
 
   gcPercentage: 100
 
@@ -77,123 +68,10 @@ db:
     queue:
         calculationType: fixed
         size: 2097152
-    blockSize: 10m
 
   fs:
     filePathPrefix: /var/lib/m3db
-    writeBufferSize: 65536
-    dataReadBufferSize: 65536
-    infoReadBufferSize: 128
-    seekReadBufferSize: 4096
-    throughputLimitMbps: 100.0
-    throughputCheckEvery: 128
-
-  repair:
-    enabled: false
-    interval: 2h
-    offset: 30m
-    jitter: 1h
-    throttle: 2m
-    checkInterval: 1m
 
-  pooling:
-    blockAllocSize: 16
-    type: simple
-    seriesPool:
-        size: 262144
-        lowWatermark: 0.7
-        highWatermark: 1.0
-    blockPool:
-        size: 262144
-        lowWatermark: 0.7
-        highWatermark: 1.0
-    encoderPool:
-        size: 262144
-        lowWatermark: 0.7
-        highWatermark: 1.0
-    closersPool:
-        size: 104857
-        lowWatermark: 0.7
-        highWatermark: 1.0
-    contextPool:
-        size: 262144
-        lowWatermark: 0.7
-        highWatermark: 1.0
-    segmentReaderPool:
-        size: 16384
-        lowWatermark: 0.7
-        highWatermark: 1.0
-    iteratorPool:
-        size: 2048
-        lowWatermark: 0.7
-        highWatermark: 1.0
-    fetchBlockMetadataResultsPool:
-        size: 65536
-        capacity: 32
-        lowWatermark: 0.7
-        highWatermark: 1.0
-    fetchBlocksMetadataResultsPool:
-        size: 32
-        capacity: 4096
-        lowWatermark: 0.7
-        highWatermark: 1.0
-    hostBlockMetadataSlicePool:
-        size: 131072
-        capacity: 3
-        lowWatermark: 0.7
-        highWatermark: 1.0
-    blockMetadataPool:
-        size: 65536
-        lowWatermark: 0.7
-        highWatermark: 1.0
-    blockMetadataSlicePool:
-        size: 65536
-        capacity: 32
-        lowWatermark: 0.7
-        highWatermark: 1.0
-    blocksMetadataPool:
-        size: 65536
-        lowWatermark: 0.7
-        highWatermark: 1.0
-    blocksMetadataSlicePool:
-        size: 32
-        capacity: 4096
-        lowWatermark: 0.7
-        highWatermark: 1.0
-    identifierPool:
-        size: 262144
-        lowWatermark: 0.7
-        highWatermark: 1.0
-    bytesPool:
-        buckets:
-            - capacity: 16
-              size: 524288
-              lowWatermark: 0.7
-              highWatermark: 1.0
-            - capacity: 32
-              size: 262144
-              lowWatermark: 0.7
-              highWatermark: 1.0
-            - capacity: 64
-              size: 131072
-              lowWatermark: 0.7
-              highWatermark: 1.0
-            - capacity: 128
-              size: 65536
-              lowWatermark: 0.7
-              highWatermark: 1.0
-            - capacity: 256
-              size: 65536
-              lowWatermark: 0.7
-              highWatermark: 1.0
-            - capacity: 1440
-              size: 16384
-              lowWatermark: 0.7
-              highWatermark: 1.0
-            - capacity: 4096
-              size: 8192
-              lowWatermark: 0.7
-              highWatermark: 1.0
   config:
     service:
         env: default_env

build/ci-test-minikube.sh

@@ -30,7 +30,7 @@ function with_retries() {
   done
 }
 
-sed -i 's#quay.io/m3db/m3db-operator:latest#m3db-operator:local#' manifests/operator.yaml
+sed -i 's#quay.io/m3db/m3db-operator:latest#m3db-operator:local#' bundle.yaml
 
 # TEMP until full e2e suite works
 exit

cmd/m3db-operator/main.go

@@ -53,7 +53,7 @@ import (
 
 const (
 	// Informers will resync on this interval
-	_informerSyncDuration = 30 * time.Second
+	_informerSyncDuration = 5 * time.Minute
 )
 
 var (

docs/api.md

@@ -43,7 +43,8 @@ ClusterSpec defines the desired state for a M3 cluster to be converge to.
 | numberOfShards | NumberOfShards defines how many shards in total | int32 | false |
 | isolationGroups | IsolationGroups specifies a map of key-value pairs. Defines which isolation groups to deploy persistent volumes for data nodes | [][IsolationGroup](#isolationgroup) | false |
 | namespaces | Namespaces specifies the namespaces this cluster will hold. | [][Namespace](#namespace) | false |
-| configMapName | ConfigMapName specifies the ConfigMap to use for this cluster. If unset a sane default will be used. | *string | false |
+| etcdEndpoints | EtcdEndpoints defines the etcd endpoints to use for service discovery. Must be set if no custom configmap is defined. If set, etcd endpoints will be templated in to the default configmap template. | []string | false |
+| configMapName | ConfigMapName specifies the ConfigMap to use for this cluster. If unset a default configmap with template variables for etcd endpoints will be used. See \"Configuring M3DB\" in the docs for more. | *string | false |
 | podIdentityConfig | PodIdentityConfig sets the configuration for pod identity. If unset only pod name and UID will be used. | *PodIdentityConfig | false |
 | containerResources | Resources defines memory / cpu constraints for each container in the cluster. | [corev1.ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#resourcerequirements-v1-core) | false |
 | dataDirVolumeClaimTemplate | DataDirVolumeClaimTemplate is the volume claim template for an M3DB instance's data. It claims PersistentVolumes for cluster storage, volumes are dynamically provisioned by when the StorageClass is defined. | *[corev1.PersistentVolumeClaim](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#persistentvolumeclaim-v1-core) | false |

docs/configuration/configuring_m3db.md

@@ -0,0 +1,10 @@
+# Configuring M3DB
+
+By default the operator will apply a configmap with basic M3DB options and settings for the coordinator to direct
+Prometheus reads/writes to the cluster. This template can be found
+[here](https://github.com/m3db/m3db-operator/blob/master/assets/default-config.tmpl).
+
+To apply custom a configuration for the M3DB cluster, one can set the `configMapName` parameter of the cluster [spec] to
+an existing configmap.
+
+[spec]: ../api

docs/getting_started/create_cluster.md

@@ -42,6 +42,10 @@ spec:
   image: quay.io/m3db/m3dbnode:latest
   replicationFactor: 3
   numberOfShards: 256
+  etcdEndpoints:
+  - http://etcd-0.etcd:2379
+  - http://etcd-1.etcd:2379
+  - http://etcd-2.etcd:2379
   isolationGroups:
     - name: us-east1-b
       numInstances: 1
@@ -116,6 +120,10 @@ spec:
       numInstances: 1
     - name: us-east1-d
       numInstances: 1
+  etcdEndpoints:
+  - http://etcd-0.etcd:2379
+  - http://etcd-1.etcd:2379
+  - http://etcd-2.etcd:2379
   podIdentityConfig:
     sources:
       # - NodeName

example/m3db-cluster-per-zone-storageclass.yaml

@@ -44,6 +44,10 @@ spec:
   - name: us-east1-d
     numInstances: 1
     storageClassName: fast-east1-d
+  etcdEndpoints:
+  - http://etcd-0.etcd:2379
+  - http://etcd-1.etcd:2379
+  - http://etcd-2.etcd:2379
   namespaces:
   - name: metrics-10s:2d
     preset: 10s:2d

example/m3db-cluster.yaml

@@ -16,6 +16,10 @@ spec:
   namespaces:
     - name: metrics-10s:2d
       preset: 10s:2d
+  etcdEndpoints:
+  - http://etcd-0.etcd:2379
+  - http://etcd-1.etcd:2379
+  - http://etcd-2.etcd:2379
   containerResources:
     requests:
       memory: 4Gi

example/simple-cluster.yaml

@@ -6,6 +6,10 @@ spec:
   image: quay.io/m3db/m3dbnode:latest
   replicationFactor: 3
   numberOfShards: 256
+  etcdEndpoints:
+  - http://etcd-0.etcd:2379
+  - http://etcd-1.etcd:2379
+  - http://etcd-2.etcd:2379
   isolationGroups:
     - name: us-east1-b
       numInstances: 1