From f0ea223ad3d831b3ef81aff3354eb7a32bde480d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?K=C3=A9vin=20Gomez?= Date: Mon, 17 Jan 2022 02:15:09 +0100 Subject: [PATCH] Add support for mapping APIKey manifests into secrets --- PROJECT | 8 + api/v1alpha1/apikey_types.go | 50 +++++ api/v1alpha1/zz_generated.deepcopy.go | 89 +++++++++ cmd/controller/main.go | 11 +- .../crd/bases/k8s.kevingomez.fr_apikeys.yaml | 82 +++++++++ config/crd/kustomization.yaml | 3 + .../crd/patches/cainjection_in_apikeys.yaml | 7 + config/crd/patches/webhook_in_apikeys.yaml | 16 ++ config/rbac/apikey_editor_role.yaml | 24 +++ config/rbac/apikey_viewer_role.yaml | 20 ++ config/rbac/role.yaml | 28 +++ config/samples/_v1alpha1_apikey.yaml | 6 + examples/api-keys/simple-api-key.yaml | 6 + go.mod | 4 +- go.sum | 2 + internal/pkg/controllers/apikey_controller.go | 171 ++++++++++++++++++ .../pkg/controllers/datasource_controller.go | 17 +- internal/pkg/grafana/apikeys.go | 134 ++++++++++++++ internal/pkg/kubernetes/secrets.go | 98 ++++++++++ internal/pkg/kubernetes/valueref.go | 37 +--- vendor/github.com/K-Phoen/grabana/client.go | 124 +++++++++++++ .../K-Phoen/grabana/decoder/dashboard.go | 4 + .../K-Phoen/grabana/decoder/logs.go | 141 +++++++++++++++ .../K-Phoen/grabana/decoder/target.go | 63 ++++++- .../K-Phoen/grabana/decoder/yaml.go | 1 + .../github.com/K-Phoen/grabana/logs/logs.go | 168 +++++++++++++++++ vendor/github.com/K-Phoen/grabana/row/row.go | 10 + .../K-Phoen/grabana/target/loki/loki.go | 47 +++++ .../grabana/target/stackdriver/stackdriver.go | 13 ++ .../github.com/K-Phoen/sdk/alert-manager.go | 73 ++++++++ vendor/github.com/K-Phoen/sdk/panel.go | 59 ++++++ vendor/modules.txt | 7 +- 32 files changed, 1471 insertions(+), 52 deletions(-) create mode 100644 api/v1alpha1/apikey_types.go create mode 100644 config/crd/bases/k8s.kevingomez.fr_apikeys.yaml create mode 100644 config/crd/patches/cainjection_in_apikeys.yaml create mode 100644 config/crd/patches/webhook_in_apikeys.yaml create mode 100644 config/rbac/apikey_editor_role.yaml create mode 100644 config/rbac/apikey_viewer_role.yaml create mode 100644 config/samples/_v1alpha1_apikey.yaml create mode 100644 examples/api-keys/simple-api-key.yaml create mode 100644 internal/pkg/controllers/apikey_controller.go create mode 100644 internal/pkg/grafana/apikeys.go create mode 100644 internal/pkg/kubernetes/secrets.go create mode 100644 vendor/github.com/K-Phoen/grabana/decoder/logs.go create mode 100644 vendor/github.com/K-Phoen/grabana/logs/logs.go create mode 100644 vendor/github.com/K-Phoen/grabana/target/loki/loki.go create mode 100644 vendor/github.com/K-Phoen/sdk/alert-manager.go diff --git a/PROJECT b/PROJECT index c0855df0..ccd8a50a 100644 --- a/PROJECT +++ b/PROJECT @@ -20,4 +20,12 @@ resources: kind: Datasource path: github.com/K-Phoen/dark/api/v1alpha1 version: v1alpha1 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: k8s.kevingomez.fr + kind: APIKey + path: github.com/K-Phoen/dark/api/v1alpha1 + version: v1alpha1 version: "3" diff --git a/api/v1alpha1/apikey_types.go b/api/v1alpha1/apikey_types.go new file mode 100644 index 00000000..e2558ad1 --- /dev/null +++ b/api/v1alpha1/apikey_types.go @@ -0,0 +1,50 @@ +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. +// Important: Run "make" to regenerate code after modifying this file + +// APIKeySpec defines the desired state of APIKey +type APIKeySpec struct { + // +kubebuilder:validation:Enum=admin;editor;viewer + // +kubebuilder:validation:Required + Role string `json:"role"` +} + +// APIKeyStatus defines the observed state of APIKey +type APIKeyStatus struct { + Status string `json:"status"` + Message string `json:"message"` +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status +//+kubebuilder:resource:shortName=api-keys;apikeys;api-key;apikey;grafana-api-keys +//+kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.status` +//+kubebuilder:printcolumn:name="Message",type=string,JSONPath=`.status.message` + +// APIKey is the Schema for the apikeys API +type APIKey struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec APIKeySpec `json:"spec,omitempty"` + Status APIKeyStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// APIKeyList contains a list of APIKey +type APIKeyList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []APIKey `json:"items"` +} + +func init() { + SchemeBuilder.Register(&APIKey{}, &APIKeyList{}) +} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index ee06181c..ace06095 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -10,6 +10,95 @@ import ( runtime "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *APIKey) DeepCopyInto(out *APIKey) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new APIKey. +func (in *APIKey) DeepCopy() *APIKey { + if in == nil { + return nil + } + out := new(APIKey) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *APIKey) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *APIKeyList) DeepCopyInto(out *APIKeyList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]APIKey, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new APIKeyList. +func (in *APIKeyList) DeepCopy() *APIKeyList { + if in == nil { + return nil + } + out := new(APIKeyList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *APIKeyList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *APIKeySpec) DeepCopyInto(out *APIKeySpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new APIKeySpec. +func (in *APIKeySpec) DeepCopy() *APIKeySpec { + if in == nil { + return nil + } + out := new(APIKeySpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *APIKeyStatus) DeepCopyInto(out *APIKeyStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new APIKeyStatus. +func (in *APIKeyStatus) DeepCopy() *APIKeyStatus { + if in == nil { + return nil + } + out := new(APIKeyStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BasicAuth) DeepCopyInto(out *BasicAuth) { *out = *in diff --git a/cmd/controller/main.go b/cmd/controller/main.go index e49d6b9e..8980502a 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -11,6 +11,9 @@ import ( // to ensure that exec-entrypoint and run can make use of them. _ "k8s.io/client-go/plugin/pkg/client/auth" + k8skevingomezfrv1 "github.com/K-Phoen/dark/api/v1" + k8skevingomezfrv1alpha1 "github.com/K-Phoen/dark/api/v1alpha1" + "github.com/K-Phoen/dark/internal/pkg/controllers" "github.com/K-Phoen/grabana" "github.com/spf13/pflag" "github.com/spf13/viper" @@ -20,10 +23,6 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" - - k8skevingomezfrv1 "github.com/K-Phoen/dark/api/v1" - k8skevingomezfrv1alpha1 "github.com/K-Phoen/dark/api/v1alpha1" - "github.com/K-Phoen/dark/internal/pkg/controllers" //+kubebuilder:scaffold:imports ) @@ -104,6 +103,10 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "Datasource") os.Exit(1) } + if err = controllers.StartAPIKeyReconciler(logger, mgr, grabanaClient); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "APIKey") + os.Exit(1) + } //+kubebuilder:scaffold:builder // liveness and readiness probes diff --git a/config/crd/bases/k8s.kevingomez.fr_apikeys.yaml b/config/crd/bases/k8s.kevingomez.fr_apikeys.yaml new file mode 100644 index 00000000..57aa6f84 --- /dev/null +++ b/config/crd/bases/k8s.kevingomez.fr_apikeys.yaml @@ -0,0 +1,82 @@ + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + name: apikeys.k8s.kevingomez.fr +spec: + group: k8s.kevingomez.fr + names: + kind: APIKey + listKind: APIKeyList + plural: apikeys + shortNames: + - api-keys + - apikeys + - api-key + - apikey + - grafana-api-keys + singular: apikey + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.status + name: Status + type: string + - jsonPath: .status.message + name: Message + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: APIKey is the Schema for the apikeys API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: APIKeySpec defines the desired state of APIKey + properties: + role: + enum: + - admin + - editor + - viewer + type: string + required: + - role + type: object + status: + description: APIKeyStatus defines the observed state of APIKey + properties: + message: + type: string + status: + type: string + required: + - message + - status + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index bb026d93..a2b551ca 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -4,6 +4,7 @@ resources: - bases/k8s.kevingomez.fr_grafanadashboards.yaml - bases/k8s.kevingomez.fr_datasources.yaml +- bases/k8s.kevingomez.fr_apikeys.yaml #+kubebuilder:scaffold:crdkustomizeresource patchesStrategicMerge: @@ -11,12 +12,14 @@ patchesStrategicMerge: # patches here are for enabling the conversion webhook for each CRD #- patches/webhook_in_grafanadashboards.yaml #- patches/webhook_in_datasources.yaml +#- patches/webhook_in_apikeys.yaml #+kubebuilder:scaffold:crdkustomizewebhookpatch # [CERTMANAGER] To enable cert-operator, uncomment all the sections with [CERTMANAGER] prefix. # patches here are for enabling the CA injection for each CRD #- patches/cainjection_in_grafanadashboards.yaml #- patches/cainjection_in_datasources.yaml +#- patches/cainjection_in_apikeys.yaml #+kubebuilder:scaffold:crdkustomizecainjectionpatch # the following config is for teaching kustomize how to do kustomization for CRDs. diff --git a/config/crd/patches/cainjection_in_apikeys.yaml b/config/crd/patches/cainjection_in_apikeys.yaml new file mode 100644 index 00000000..abe3e694 --- /dev/null +++ b/config/crd/patches/cainjection_in_apikeys.yaml @@ -0,0 +1,7 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: apikeys.k8s.kevingomez.fr diff --git a/config/crd/patches/webhook_in_apikeys.yaml b/config/crd/patches/webhook_in_apikeys.yaml new file mode 100644 index 00000000..650bc721 --- /dev/null +++ b/config/crd/patches/webhook_in_apikeys.yaml @@ -0,0 +1,16 @@ +# The following patch enables a conversion webhook for the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: apikeys.k8s.kevingomez.fr +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + namespace: system + name: webhook-service + path: /convert + conversionReviewVersions: + - v1 diff --git a/config/rbac/apikey_editor_role.yaml b/config/rbac/apikey_editor_role.yaml new file mode 100644 index 00000000..15aa410b --- /dev/null +++ b/config/rbac/apikey_editor_role.yaml @@ -0,0 +1,24 @@ +# permissions for end users to edit apikeys. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: apikey-editor-role +rules: +- apiGroups: + - k8s.kevingomez.fr + resources: + - apikeys + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - k8s.kevingomez.fr + resources: + - apikeys/status + verbs: + - get diff --git a/config/rbac/apikey_viewer_role.yaml b/config/rbac/apikey_viewer_role.yaml new file mode 100644 index 00000000..08232cda --- /dev/null +++ b/config/rbac/apikey_viewer_role.yaml @@ -0,0 +1,20 @@ +# permissions for end users to view apikeys. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: apikey-viewer-role +rules: +- apiGroups: + - k8s.kevingomez.fr + resources: + - apikeys + verbs: + - get + - list + - watch +- apiGroups: + - k8s.kevingomez.fr + resources: + - apikeys/status + verbs: + - get diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index df2b2312..240b5283 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -18,9 +18,37 @@ rules: resources: - secrets verbs: + - create + - delete - get - list - watch +- apiGroups: + - k8s.kevingomez.fr + resources: + - apikeys + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - k8s.kevingomez.fr + resources: + - apikeys/finalizers + verbs: + - update +- apiGroups: + - k8s.kevingomez.fr + resources: + - apikeys/status + verbs: + - get + - patch + - update - apiGroups: - k8s.kevingomez.fr resources: diff --git a/config/samples/_v1alpha1_apikey.yaml b/config/samples/_v1alpha1_apikey.yaml new file mode 100644 index 00000000..7b460be8 --- /dev/null +++ b/config/samples/_v1alpha1_apikey.yaml @@ -0,0 +1,6 @@ +apiVersion: k8s.kevingomez.fr/v1alpha1 +kind: APIKey +metadata: + name: apikey-dark +spec: + role: admin diff --git a/examples/api-keys/simple-api-key.yaml b/examples/api-keys/simple-api-key.yaml new file mode 100644 index 00000000..02692cc5 --- /dev/null +++ b/examples/api-keys/simple-api-key.yaml @@ -0,0 +1,6 @@ +apiVersion: k8s.kevingomez.fr/v1alpha1 +kind: APIKey +metadata: + name: dark-simple-key +spec: + role: admin diff --git a/go.mod b/go.mod index 7390dbda..b5dee8d8 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.17 require ( github.com/K-Phoen/grabana v0.20.6 - github.com/K-Phoen/sdk v0.8.1 + github.com/K-Phoen/sdk v0.8.4 github.com/go-logr/logr v1.2.0 github.com/onsi/ginkgo v1.16.5 github.com/onsi/gomega v1.17.0 @@ -90,3 +90,5 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.2.0 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) + +replace github.com/K-Phoen/grabana => ../grabana diff --git a/go.sum b/go.sum index a50da363..1ae4acb1 100644 --- a/go.sum +++ b/go.sum @@ -71,6 +71,8 @@ github.com/K-Phoen/grabana v0.20.6 h1:MrrupcW1SrmA+UyZQWBKPoz7vfq4whWfXFKjBWRQuU github.com/K-Phoen/grabana v0.20.6/go.mod h1:C07Dzgfy7WM9pt8u8EPP5LlFJnb3MeYNyb3aKTms3uA= github.com/K-Phoen/sdk v0.8.1 h1:Z4niyQJKXZcBbBULLFw55axv56SxlL3REBZjNn1B/HA= github.com/K-Phoen/sdk v0.8.1/go.mod h1:fnbOsbRksULSfcXjOI6W1/HISz5o/u1iEhF/fLedqTg= +github.com/K-Phoen/sdk v0.8.4 h1:KAH0ipC/4iWxslFudjy84Gm8fkaVRfekZK+BPAvoF30= +github.com/K-Phoen/sdk v0.8.4/go.mod h1:fnbOsbRksULSfcXjOI6W1/HISz5o/u1iEhF/fLedqTg= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= diff --git a/internal/pkg/controllers/apikey_controller.go b/internal/pkg/controllers/apikey_controller.go new file mode 100644 index 00000000..48022c0d --- /dev/null +++ b/internal/pkg/controllers/apikey_controller.go @@ -0,0 +1,171 @@ +package controllers + +import ( + "context" + + "github.com/K-Phoen/dark/api/v1alpha1" + "github.com/K-Phoen/dark/internal/pkg/grafana" + "github.com/K-Phoen/dark/internal/pkg/kubernetes" + "github.com/K-Phoen/grabana" + "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +const apiKeysFinalizerName = "apikeys.k8s.kevingomez.fr/finalizer" + +type apiKeyClient interface { + Reconcile(ctx context.Context, key grafana.APIKey) error + Delete(ctx context.Context, name string) error +} + +// APIKeyReconciler reconciles a APIKey object +type APIKeyReconciler struct { + client.Client + + Scheme *runtime.Scheme + Recorder record.EventRecorder + + apiKeyClient apiKeyClient +} + +func StartAPIKeyReconciler(logger logr.Logger, ctrlManager ctrl.Manager, grabanaClient *grabana.Client) error { + apiKeys := grafana.NewAPIKeys(logger, grabanaClient, kubernetes.NewSecrets(logger, ctrlManager.GetClient())) + + reconciler := &APIKeyReconciler{ + Client: ctrlManager.GetClient(), + Scheme: ctrlManager.GetScheme(), + Recorder: ctrlManager.GetEventRecorderFor("api-key-controller"), + apiKeyClient: apiKeys, + } + + return reconciler.SetupWithManager(ctrlManager) +} + +//+kubebuilder:rbac:groups=k8s.kevingomez.fr,resources=apikeys,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=k8s.kevingomez.fr,resources=apikeys/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=k8s.kevingomez.fr,resources=apikeys/finalizers,verbs=update +//+kubebuilder:rbac:groups="",resources=events,verbs=create;patch +//+kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;delete + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// TODO(user): Modify the Reconcile function to compare the state specified by +// the APIKey object against the actual cluster state, and then +// perform operations to make the cluster state reflect the state specified by +// the user. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.10.0/pkg/reconcile +func (r *APIKeyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + logger := log.FromContext(ctx) + + logger.Info("reconciling") + + apiKeyManifest := &v1alpha1.APIKey{} + if err := r.Get(ctx, req.NamespacedName, apiKeyManifest); err != nil { + logger.Error(err, "unable to fetch APIKey") + // we'll ignore not-found errors, since they can't be fixed by an immediate + // requeue (we'll need to wait for a new notification), and we can get them + // on deleted requests. + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + // examine DeletionTimestamp to determine if object is under deletion + if apiKeyManifest.ObjectMeta.DeletionTimestamp.IsZero() { + // The object is not being deleted, so if it does not have our finalizer, + // then lets add the finalizer and update the object. This is equivalent + // registering our finalizer. + if !containsString(apiKeyManifest.GetFinalizers(), apiKeysFinalizerName) { + controllerutil.AddFinalizer(apiKeyManifest, apiKeysFinalizerName) + if err := r.Update(ctx, apiKeyManifest); err != nil { + return ctrl.Result{}, err + } + } + } else { + logger.Info("deleting API key") + + // The object is being deleted + if containsString(apiKeyManifest.GetFinalizers(), apiKeysFinalizerName) { + logger.Info("finalizer found, deleting API key from grafana") + + // our finalizer is present, so lets handle any external dependency + if err := r.apiKeyClient.Delete(ctx, apiKeyManifest.Name); err != nil { + // if fail to delete the external dependency here, return with error + // so that it can be retried + return ctrl.Result{}, err + } + + // remove our finalizer from the list and update it. + controllerutil.RemoveFinalizer(apiKeyManifest, apiKeysFinalizerName) + if err := r.Update(ctx, apiKeyManifest); err != nil { + return ctrl.Result{}, err + } + } + + // Stop reconciliation as the item is being deleted + return ctrl.Result{}, nil + } + + // handle actual reconciliation + return r.doReconcileManifest(ctx, apiKeyManifest) + +} + +func (r *APIKeyReconciler) doReconcileManifest(ctx context.Context, manifest *v1alpha1.APIKey) (ctrl.Result, error) { + logger := log.FromContext(ctx) + + key := grafana.APIKey{ + Name: manifest.Name, + Role: manifest.Spec.Role, + SecretName: manifest.Name, + SecretNamespace: manifest.Namespace, + TokenKey: "token", + } + + if err := r.apiKeyClient.Reconcile(ctx, key); err != nil { + logger.Info("failed reconciling API key") + + r.updateStatus(ctx, manifest, err) + r.Recorder.Event(manifest, "Warning", "Error", "could not reconcile API key with Grafana") + + return ctrl.Result{}, err + } + + r.updateStatus(ctx, manifest, nil) + r.Recorder.Event(manifest, "Normal", "Synchronized", "API key reconciled") + + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *APIKeyReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&v1alpha1.APIKey{}). + Complete(r) +} + +func (r *APIKeyReconciler) updateStatus(ctx context.Context, manifest *v1alpha1.APIKey, err error) { + logger := log.FromContext(ctx) + + // NEVER modify objects from the store. It's a read-only, local cache. + // You can use DeepCopy() to make a deep manifestCopy of original object and modify this manifestCopy + // Or create a manifestCopy manually for better performance + manifestCopy := manifest.DeepCopy() + + if err == nil { + manifestCopy.Status.Status = "OK" + manifestCopy.Status.Message = "Synchronized" + } else { + manifestCopy.Status.Status = "Error" + manifestCopy.Status.Message = err.Error() + } + + if err := r.Status().Update(ctx, manifestCopy); err != nil { + logger.Error(err, "unable to update APIKey status") + } +} diff --git a/internal/pkg/controllers/datasource_controller.go b/internal/pkg/controllers/datasource_controller.go index 9e14b092..3c104377 100644 --- a/internal/pkg/controllers/datasource_controller.go +++ b/internal/pkg/controllers/datasource_controller.go @@ -125,7 +125,8 @@ func (r *DatasourceReconciler) Reconcile(ctx context.Context, req ctrl.Request) //+kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch func StartDatasourceReconciler(logger logr.Logger, ctrlManager ctrl.Manager, grabanaClient *grabana.Client) error { - refReader := kubernetes.NewValueRefReader(logger, ctrlManager.GetClient()) + refReader := kubernetes.NewValueRefReader(logger, kubernetes.NewSecrets(logger, ctrlManager.GetClient())) + reconciler := &DatasourceReconciler{ Client: ctrlManager.GetClient(), Scheme: ctrlManager.GetScheme(), @@ -143,23 +144,23 @@ func (r *DatasourceReconciler) SetupWithManager(mgr ctrl.Manager) error { Complete(r) } -func (r *DatasourceReconciler) updateStatus(ctx context.Context, datasource *v1alpha1.Datasource, err error) { +func (r *DatasourceReconciler) updateStatus(ctx context.Context, manifest *v1alpha1.Datasource, err error) { logger := log.FromContext(ctx) // NEVER modify objects from the store. It's a read-only, local cache. // You can use DeepCopy() to make a deep copy of original object and modify this copy // Or create a copy manually for better performance - dashboardCopy := datasource.DeepCopy() + manifestCopy := manifest.DeepCopy() if err == nil { - dashboardCopy.Status.Status = "OK" - dashboardCopy.Status.Message = "Synchronized" + manifestCopy.Status.Status = "OK" + manifestCopy.Status.Message = "Synchronized" } else { - dashboardCopy.Status.Status = "Error" - dashboardCopy.Status.Message = err.Error() + manifestCopy.Status.Status = "Error" + manifestCopy.Status.Message = err.Error() } - if err := r.Status().Update(ctx, dashboardCopy); err != nil { + if err := r.Status().Update(ctx, manifestCopy); err != nil { logger.Error(err, "unable to update Datasource status") } } diff --git a/internal/pkg/grafana/apikeys.go b/internal/pkg/grafana/apikeys.go new file mode 100644 index 00000000..fcde4e67 --- /dev/null +++ b/internal/pkg/grafana/apikeys.go @@ -0,0 +1,134 @@ +package grafana + +import ( + "context" + "fmt" + + "github.com/K-Phoen/dark/internal/pkg/kubernetes" + "github.com/K-Phoen/grabana" + "github.com/go-logr/logr" + v1 "k8s.io/api/core/v1" +) + +type APIKey struct { + Name string + Role string + + SecretName string + SecretNamespace string + TokenKey string +} + +func (key APIKey) GrabanaRole() (grabana.APIKeyRole, error) { + switch key.Role { + case "admin": + return grabana.AdminRole, nil + case "editor": + return grabana.EditorRole, nil + case "viewer": + return grabana.ViewerRole, nil + } + + return grabana.ViewerRole, fmt.Errorf("invalid role") +} + +type secretsWriter interface { + Read(ctx context.Context, namespace string, ref v1.SecretKeySelector) (string, error) + Upsert(ctx context.Context, request kubernetes.SecretUpsertRequest) error +} + +type APIKeys struct { + logger logr.Logger + grabanaClient *grabana.Client + secrets secretsWriter +} + +func NewAPIKeys(logger logr.Logger, grabanaClient *grabana.Client, secrets secretsWriter) *APIKeys { + return &APIKeys{ + logger: logger, + grabanaClient: grabanaClient, + secrets: secrets, + } +} + +func (keys *APIKeys) Reconcile(ctx context.Context, key APIKey) error { + logger := keys.logger.WithValues("key", key.Name) + + existingGrafanaKeys, err := keys.grabanaClient.APIKeys(ctx) + if err != nil { + logger.Error(err, "could not check existing keys in Grafana") + return err + } + + // the API key does not exist in Grafana, we need to create it and its Kubernetes secret + if _, ok := existingGrafanaKeys[key.Name]; !ok { + return keys.createKey(ctx, key) + } + + // the API key exists, but the secret does not. We need to re-create both + nope := false + _, err = keys.secrets.Read(ctx, key.SecretNamespace, v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: key.SecretName, + }, + Key: key.TokenKey, + Optional: &nope, + }) + + if err == kubernetes.ErrSecretNotFound || err == kubernetes.ErrKeyNotFoundInSecret { + if err := keys.Delete(ctx, key.Name); err != nil { + return err + } + + return keys.createKey(ctx, key) + } + + // api key exists, secret exist = nothing to do (we assume secret is up-to-date) + return nil +} + +func (keys *APIKeys) Delete(ctx context.Context, name string) error { + if err := keys.grabanaClient.DeleteAPIKeyByName(ctx, name); err != nil && err != grabana.ErrAPIKeyNotFound { + keys.logger.Error(err, "could not delete key in Grafana", "key", name) + return err + } + + return nil +} + +func (keys *APIKeys) createKey(ctx context.Context, key APIKey) error { + logger := keys.logger.WithValues("key", key.Name) + + fmt.Printf("creating new api key\n") + + role, err := key.GrabanaRole() + if err != nil { + return err + } + + // create the token in Grafana + token, err := keys.grabanaClient.CreateAPIKey(ctx, grabana.CreateAPIKeyRequest{ + Name: key.Name, + Role: role, + }) + if err != nil { + logger.Error(err, "could not create Grafana API key") + return err + } + + secretPayload := make(map[string][]byte) + secretPayload[key.TokenKey] = []byte(token) + + // map it to a kubernetes secret + err = keys.secrets.Upsert(ctx, kubernetes.SecretUpsertRequest{ + Name: key.SecretName, + Namespace: key.SecretNamespace, + Data: secretPayload, + }) + if err != nil { + logger.Error(err, "could not create Kubernetes secret for API key") + return err + } + + return nil +} diff --git a/internal/pkg/kubernetes/secrets.go b/internal/pkg/kubernetes/secrets.go new file mode 100644 index 00000000..53e4b1b2 --- /dev/null +++ b/internal/pkg/kubernetes/secrets.go @@ -0,0 +1,98 @@ +package kubernetes + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var ErrSecretNotFound = fmt.Errorf("secret not found") +var ErrKeyNotFoundInSecret = fmt.Errorf("key not found") + +type SecretUpsertRequest struct { + Name string + Namespace string + Data map[string][]byte +} + +type Secrets struct { + logger logr.Logger + client client.Client +} + +func NewSecrets(logger logr.Logger, client client.Client) *Secrets { + return &Secrets{ + logger: logger, + client: client, + } +} + +func (secrets *Secrets) Upsert(ctx context.Context, request SecretUpsertRequest) error { + logger := secrets.logger.WithValues("namespace", request.Namespace, "name", request.Name) + logger.Info("upserting secret") + + secret := &v1.Secret{} + + // if a secret for this API key already exists, we delete it + err := secrets.client.Get(ctx, client.ObjectKey{Namespace: request.Namespace, Name: request.Name}, secret) + if err != nil && !apierrors.IsNotFound(err) { + logger.Error(err, "unable to check secret existence") + return err + } + // the secret was found + if err == nil { + if err := secrets.client.Delete(ctx, secret); err != nil { + logger.Error(err, "unable to delete existing secret") + return err + } + } + + // now we can safely re-create the secret with a new value + err = secrets.client.Create(ctx, &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: request.Name, + Namespace: request.Namespace, + Annotations: map[string]string{ + "app.kubernetes.io/managed-by": "dark", + }, + }, + Data: request.Data, + }) + if err != nil { + logger.Error(err, "unable to create secret") + return err + } + + return nil +} + +func (secrets *Secrets) Read(ctx context.Context, namespace string, ref v1.SecretKeySelector) (string, error) { + logger := secrets.logger.WithValues("namespace", namespace, "name", ref.Name) + logger.Info("fetching secret") + + secret := &v1.Secret{} + if err := secrets.client.Get(ctx, client.ObjectKey{Namespace: namespace, Name: ref.Name}, secret); err != nil { + if apierrors.IsNotFound(err) { + return "", ErrSecretNotFound + } + + logger.Error(err, "unable to fetch secret") + return "", err + } + + if _, ok := secret.Data[ref.Key]; !ok { + // key doesn't exist in secret, but the ref was marked as optional + if ref.Optional != nil && *ref.Optional { + return "", nil + } + + return "", fmt.Errorf("key '%s' does not exist: %w", ref.Key, ErrKeyNotFoundInSecret) + } + + return string(secret.Data[ref.Key]), nil +} diff --git a/internal/pkg/kubernetes/valueref.go b/internal/pkg/kubernetes/valueref.go index d97a2243..b0afe081 100644 --- a/internal/pkg/kubernetes/valueref.go +++ b/internal/pkg/kubernetes/valueref.go @@ -7,20 +7,24 @@ import ( "github.com/K-Phoen/dark/api/v1alpha1" "github.com/go-logr/logr" v1 "k8s.io/api/core/v1" - "sigs.k8s.io/controller-runtime/pkg/client" ) var ErrInvalidValueRef = fmt.Errorf("invalid value ref") +type secretsReader interface { + Read(ctx context.Context, namespace string, ref v1.SecretKeySelector) (string, error) +} + type ValueRefReader struct { logger logr.Logger - client client.Reader + + secrets secretsReader } -func NewValueRefReader(logger logr.Logger, client client.Reader) *ValueRefReader { +func NewValueRefReader(logger logr.Logger, secrets secretsReader) *ValueRefReader { return &ValueRefReader{ - logger: logger, - client: client, + logger: logger, + secrets: secrets, } } @@ -33,26 +37,5 @@ func (reader *ValueRefReader) RefToValue(ctx context.Context, namespace string, return ref.Value, nil } - return reader.readSecret(ctx, namespace, ref.ValueRef.SecretKeyRef) -} - -func (reader *ValueRefReader) readSecret(ctx context.Context, namespace string, ref *v1.SecretKeySelector) (string, error) { - reader.logger.Info("fetching secret", "namespace", namespace, "name", ref.Name) - - secret := &v1.Secret{} - if err := reader.client.Get(ctx, client.ObjectKey{Namespace: namespace, Name: ref.Name}, secret); err != nil { - reader.logger.Error(err, "unable to fetch secret") - return "", err - } - - if _, ok := secret.Data[ref.Key]; !ok { - // key doesn't exist in secret, but the ref was marked as optional - if ref.Optional != nil && *ref.Optional { - return "", nil - } - - return "", fmt.Errorf("key '%s' does not exist in secret '%s'", ref.Key, ref.Name) - } - - return string(secret.Data[ref.Key]), nil + return reader.secrets.Read(ctx, namespace, *ref.ValueRef.SecretKeyRef) } diff --git a/vendor/github.com/K-Phoen/grabana/client.go b/vendor/github.com/K-Phoen/grabana/client.go index 0bffd360..4c054c37 100644 --- a/vendor/github.com/K-Phoen/grabana/client.go +++ b/vendor/github.com/K-Phoen/grabana/client.go @@ -26,10 +26,51 @@ var ErrDashboardNotFound = errors.New("dashboard not found") // ErrDatasourceNotFound is returned when the given datasource can not be found. var ErrDatasourceNotFound = errors.New("datasource not found") +// ErrAPIKeyNotFound is returned when the given API key can not be found. +var ErrAPIKeyNotFound = errors.New("API key not found") + // ErrAlertChannelNotFound is returned when the given alert notification // channel can not be found. var ErrAlertChannelNotFound = errors.New("alert channel not found") +// APIKeyRole represents a role given to an API key. +type APIKeyRole uint8 + +const ( + AdminRole APIKeyRole = iota + EditorRole + ViewerRole +) + +func (role APIKeyRole) MarshalJSON() ([]byte, error) { + var s string + switch role { + case ViewerRole: + s = "Viewer" + case EditorRole: + s = "Editor" + case AdminRole: + s = "Admin" + default: + s = "None" + } + + return json.Marshal(s) +} + +// CreateAPIKeyRequest represents a request made to the API key creation endpoint. +type CreateAPIKeyRequest struct { + Name string `json:"name"` + Role APIKeyRole `json:"role"` + SecondsToLive int `json:"secondsToLive"` +} + +// APIKey represents an API key. +type APIKey struct { + ID uint `json:"id"` + Name string `json:"name"` +} + // Dashboard represents a Grafana dashboard. type Dashboard struct { ID uint `json:"id"` @@ -95,6 +136,89 @@ func (client *Client) modifyRequest(request *http.Request) { } } +// CreateAPIKey creates a new API key. +func (client *Client) CreateAPIKey(ctx context.Context, request CreateAPIKeyRequest) (string, error) { + buf, err := json.Marshal(request) + if err != nil { + return "", err + } + + resp, err := client.sendJSON(ctx, http.MethodPost, "/api/auth/keys", buf) + if err != nil { + return "", err + } + + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + return "", client.httpError(resp) + } + + var response struct { + Key string `json:"key"` + } + if err := decodeJSON(resp.Body, &response); err != nil { + return "", err + } + + return response.Key, nil +} + +// DeleteAPIKeyByName deletes an API key given its name. +func (client *Client) DeleteAPIKeyByName(ctx context.Context, name string) error { + apiKeys, err := client.APIKeys(ctx) + if err != nil { + return err + } + + keyToDelete, ok := apiKeys[name] + if !ok { + return ErrAPIKeyNotFound + } + + resp, err := client.delete(ctx, fmt.Sprintf("/api/auth/keys/%d", keyToDelete.ID)) + if err != nil { + return err + } + + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode == http.StatusNotFound { + return ErrAPIKeyNotFound + } + if resp.StatusCode != http.StatusOK { + return client.httpError(resp) + } + + return nil +} + +// APIKeys lists active API keys. +func (client *Client) APIKeys(ctx context.Context) (map[string]APIKey, error) { + resp, err := client.get(ctx, "/api/auth/keys") + if err != nil { + return nil, err + } + + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + return nil, client.httpError(resp) + } + + var keys []APIKey + if err := decodeJSON(resp.Body, &keys); err != nil { + return nil, err + } + + keysMap := make(map[string]APIKey, len(keys)) + for _, key := range keys { + keysMap[key.Name] = key + } + + return keysMap, nil +} + // FindOrCreateFolder returns the folder by its name or creates it if it doesn't exist. func (client *Client) FindOrCreateFolder(ctx context.Context, name string) (*Folder, error) { folder, err := client.GetFolderByTitle(ctx, name) diff --git a/vendor/github.com/K-Phoen/grabana/decoder/dashboard.go b/vendor/github.com/K-Phoen/grabana/decoder/dashboard.go index 9b590939..c68229f5 100644 --- a/vendor/github.com/K-Phoen/grabana/decoder/dashboard.go +++ b/vendor/github.com/K-Phoen/grabana/decoder/dashboard.go @@ -110,6 +110,7 @@ type DashboardPanel struct { Text *DashboardText `yaml:",omitempty"` Heatmap *DashboardHeatmap `yaml:",omitempty"` TimeSeries *DashboardTimeSeries `yaml:"timeseries,omitempty"` + Logs *DashboardLogs `yaml:"logs,omitempty"` } func (panel DashboardPanel) toOption() (row.Option, error) { @@ -131,6 +132,9 @@ func (panel DashboardPanel) toOption() (row.Option, error) { if panel.Heatmap != nil { return panel.Heatmap.toOption() } + if panel.Logs != nil { + return panel.Logs.toOption() + } return nil, ErrPanelNotConfigured } diff --git a/vendor/github.com/K-Phoen/grabana/decoder/logs.go b/vendor/github.com/K-Phoen/grabana/decoder/logs.go new file mode 100644 index 00000000..9e41ad93 --- /dev/null +++ b/vendor/github.com/K-Phoen/grabana/decoder/logs.go @@ -0,0 +1,141 @@ +package decoder + +import ( + "fmt" + + "github.com/K-Phoen/grabana/logs" + "github.com/K-Phoen/grabana/row" +) + +var ErrInvalidSortOrder = fmt.Errorf("invalid sort order") +var ErrInvalidDeduplicationStrategy = fmt.Errorf("invalid deduplication strategy") + +type DashboardLogs struct { + Title string + Description string `yaml:",omitempty"` + Span float32 `yaml:",omitempty"` + Height string `yaml:",omitempty"` + Transparent bool `yaml:",omitempty"` + Datasource string `yaml:",omitempty"` + Repeat string `yaml:",omitempty"` + Targets []LogsTarget `yaml:",omitempty"` + Visualization *LogsVisualization `yaml:",omitempty"` +} + +type LogsTarget struct { + Loki *LokiTarget `yaml:",omitempty"` +} + +func (panel DashboardLogs) toOption() (row.Option, error) { + opts := []logs.Option{} + + if panel.Description != "" { + opts = append(opts, logs.Description(panel.Description)) + } + if panel.Span != 0 { + opts = append(opts, logs.Span(panel.Span)) + } + if panel.Height != "" { + opts = append(opts, logs.Height(panel.Height)) + } + if panel.Transparent { + opts = append(opts, logs.Transparent()) + } + if panel.Datasource != "" { + opts = append(opts, logs.DataSource(panel.Datasource)) + } + if panel.Repeat != "" { + opts = append(opts, logs.Repeat(panel.Repeat)) + } + for _, t := range panel.Targets { + opt, err := panel.target(t) + if err != nil { + return nil, err + } + + opts = append(opts, opt) + } + + vizOpts, err := panel.Visualization.toOptions() + if err != nil { + return nil, err + } + + opts = append(opts, vizOpts...) + + return row.WithLogs(panel.Title, opts...), nil +} + +func (panel DashboardLogs) target(t LogsTarget) (logs.Option, error) { + if t.Loki != nil { + return logs.WithLokiTarget(t.Loki.Query, t.Loki.toOptions()...), nil + } + + return nil, ErrTargetNotConfigured +} + +type LogsVisualization struct { + Time bool `yaml:",omitempty"` + UniqueLabels bool `yaml:"unique_labels,omitempty"` + CommonLabels bool `yaml:"common_labels,omitempty"` + WrapLines bool `yaml:"wrap_lines,omitempty"` + PrettifyJSON bool `yaml:"prettify_json,omitempty"` + HideLogDetails bool `yaml:"hide_log_details,omitempty"` + Order string `yaml:",omitempty"` + Deduplication string `yaml:",omitempty"` +} + +func (viz *LogsVisualization) toOptions() ([]logs.Option, error) { + if viz == nil { + return nil, nil + } + + opts := []logs.Option{} + + if viz.Time { + opts = append(opts, logs.Time()) + } + if viz.UniqueLabels { + opts = append(opts, logs.UniqueLabels()) + } + if viz.CommonLabels { + opts = append(opts, logs.CommonLabels()) + } + if viz.WrapLines { + opts = append(opts, logs.WrapLines()) + } + if viz.PrettifyJSON { + opts = append(opts, logs.PrettifyJSON()) + } + if viz.HideLogDetails { + opts = append(opts, logs.HideLogDetails()) + } + + if viz.Order != "" { + switch viz.Order { + case "asc": + opts = append(opts, logs.Order(logs.Asc)) + case "desc": + opts = append(opts, logs.Order(logs.Desc)) + default: + return nil, ErrInvalidSortOrder + } + } + + if viz.Deduplication != "" { + switch viz.Deduplication { + case "none": + opts = append(opts, logs.Deduplication(logs.None)) + case "exact": + opts = append(opts, logs.Deduplication(logs.Exact)) + case "numbers": + opts = append(opts, logs.Deduplication(logs.Numbers)) + case "signature": + opts = append(opts, logs.Deduplication(logs.Signature)) + default: + return nil, ErrInvalidDeduplicationStrategy + } + } + + return opts, nil +} diff --git a/vendor/github.com/K-Phoen/grabana/decoder/target.go b/vendor/github.com/K-Phoen/grabana/decoder/target.go index 599bcf2c..9bec5f5e 100644 --- a/vendor/github.com/K-Phoen/grabana/decoder/target.go +++ b/vendor/github.com/K-Phoen/grabana/decoder/target.go @@ -5,6 +5,7 @@ import ( "github.com/K-Phoen/grabana/target/graphite" "github.com/K-Phoen/grabana/target/influxdb" + "github.com/K-Phoen/grabana/target/loki" "github.com/K-Phoen/grabana/target/prometheus" "github.com/K-Phoen/grabana/target/stackdriver" ) @@ -12,6 +13,7 @@ import ( var ErrTargetNotConfigured = fmt.Errorf("target not configured") var ErrInvalidStackdriverType = fmt.Errorf("invalid stackdriver target type") var ErrInvalidStackdriverAggregation = fmt.Errorf("invalid stackdriver aggregation type") +var ErrInvalidStackdriverPreprocessor = fmt.Errorf("invalid stackdriver preprocessor") var ErrInvalidStackdriverAlignment = fmt.Errorf("invalid stackdriver alignment method") type Target struct { @@ -60,6 +62,26 @@ func (t PrometheusTarget) toOptions() []prometheus.Option { return opts } +type LokiTarget struct { + Query string + Legend string `yaml:",omitempty"` + Ref string `yaml:",omitempty"` + Hidden bool `yaml:",omitempty"` +} + +func (t LokiTarget) toOptions() []loki.Option { + opts := []loki.Option{ + loki.Legend(t.Legend), + loki.Ref(t.Ref), + } + + if t.Hidden { + opts = append(opts, loki.Hide()) + } + + return opts +} + type GraphiteTarget struct { Query string Ref string `yaml:",omitempty"` @@ -97,16 +119,17 @@ func (t InfluxDBTarget) toOptions() []influxdb.Option { } type StackdriverTarget struct { - Project string - Type string - Metric string - Filters StackdriverFilters `yaml:",omitempty"` - Aggregation string `yaml:",omitempty"` - Alignment *StackdriverAlignment `yaml:",omitempty"` - Legend string `yaml:",omitempty"` - Ref string `yaml:",omitempty"` - Hidden bool `yaml:",omitempty"` - GroupBy []string `yaml:"group_by,omitempty"` + Project string + Type string + Metric string + Filters StackdriverFilters `yaml:",omitempty"` + Aggregation string `yaml:",omitempty"` + Alignment *StackdriverAlignment `yaml:",omitempty"` + Legend string `yaml:",omitempty"` + Preprocessor string `yaml:",omitempty"` + Ref string `yaml:",omitempty"` + Hidden bool `yaml:",omitempty"` + GroupBy []string `yaml:"group_by,omitempty"` } type StackdriverFilters struct { @@ -171,6 +194,15 @@ func (t StackdriverTarget) toOptions() ([]stackdriver.Option, error) { opts = append(opts, opt) } + if t.Preprocessor != "" { + opt, err := t.preprocessor() + if err != nil { + return nil, err + } + + opts = append(opts, opt) + } + if t.Alignment != nil { opt, err := t.Alignment.toOption() if err != nil { @@ -218,6 +250,17 @@ func (t StackdriverTarget) aggregation() (stackdriver.Option, error) { } } +func (t StackdriverTarget) preprocessor() (stackdriver.Option, error) { + switch t.Preprocessor { + case "delta": + return stackdriver.Preprocessor(stackdriver.PreprocessDelta), nil + case "rate": + return stackdriver.Preprocessor(stackdriver.PreprocessRate), nil + default: + return nil, ErrInvalidStackdriverPreprocessor + } +} + func (filters StackdriverFilters) toOptions() []stackdriver.FilterOption { opts := []stackdriver.FilterOption{} diff --git a/vendor/github.com/K-Phoen/grabana/decoder/yaml.go b/vendor/github.com/K-Phoen/grabana/decoder/yaml.go index 5c7ff926..5c8ace1e 100644 --- a/vendor/github.com/K-Phoen/grabana/decoder/yaml.go +++ b/vendor/github.com/K-Phoen/grabana/decoder/yaml.go @@ -9,6 +9,7 @@ import ( func UnmarshalYAML(input io.Reader) (dashboard.Builder, error) { decoder := yaml.NewDecoder(input) + decoder.KnownFields(true) parsed := &DashboardModel{} if err := decoder.Decode(parsed); err != nil { diff --git a/vendor/github.com/K-Phoen/grabana/logs/logs.go b/vendor/github.com/K-Phoen/grabana/logs/logs.go new file mode 100644 index 00000000..310d957d --- /dev/null +++ b/vendor/github.com/K-Phoen/grabana/logs/logs.go @@ -0,0 +1,168 @@ +package logs + +import ( + "github.com/K-Phoen/grabana/target/loki" + "github.com/K-Phoen/sdk" +) + +// DedupStrategy represents a deduplication strategy. +type DedupStrategy string + +const ( + None DedupStrategy = "none" + Exact DedupStrategy = "exact" + Numbers DedupStrategy = "numbers" + Signature DedupStrategy = "signature" +) + +// SortOrder represents a sort order. +type SortOrder string + +const ( + Asc SortOrder = "Ascending" + Desc SortOrder = "Descending" +) + +// Option represents an option that can be used to configure a logs panel. +type Option func(logs *Logs) + +// Logs represents a logs panel. +type Logs struct { + Builder *sdk.Panel +} + +// New creates a new heatmap panel. +func New(title string, options ...Option) *Logs { + panel := &Logs{Builder: sdk.NewLogs(title)} + panel.Builder.IsNew = false + panel.Builder.LogsPanel.Options.EnableLogDetails = true + + for _, opt := range append(defaults(), options...) { + opt(panel) + } + + return panel +} + +func defaults() []Option { + return []Option{ + Span(6), + Order(Desc), + } +} + +// DataSource sets the data source to be used by the panel. +func DataSource(source string) Option { + return func(logs *Logs) { + logs.Builder.Datasource = &source + } +} + +// WithLokiTarget adds a loki query to the graph. +func WithLokiTarget(query string, options ...loki.Option) Option { + target := loki.New(query, options...) + + return func(logs *Logs) { + logs.Builder.AddTarget(&sdk.Target{ + RefID: target.Ref, + Expr: target.Expr, + LegendFormat: target.LegendFormat, + }) + } +} + +// Span sets the width of the panel, in grid units. Should be a positive +// number between 1 and 12. Example: 6. +func Span(span float32) Option { + return func(logs *Logs) { + logs.Builder.Span = span + } +} + +// Height sets the height of the panel, in pixels. Example: "400px". +func Height(height string) Option { + return func(logs *Logs) { + logs.Builder.Height = &height + } +} + +// Description annotates the current visualization with a human-readable description. +func Description(content string) Option { + return func(logs *Logs) { + logs.Builder.Description = &content + } +} + +// Transparent makes the background transparent. +func Transparent() Option { + return func(logs *Logs) { + logs.Builder.Transparent = true + } +} + +// Repeat configures repeating a panel for a variable +func Repeat(repeat string) Option { + return func(logs *Logs) { + logs.Builder.Repeat = &repeat + } +} + +// Time displays the "time" column. This is the timestamp associated with the +// log line as reported from the data source. +func Time() Option { + return func(logs *Logs) { + logs.Builder.LogsPanel.Options.ShowTime = true + } +} + +// UniqueLabels displays the "unique labels" column, which shows only non-common labels. +func UniqueLabels() Option { + return func(logs *Logs) { + logs.Builder.LogsPanel.Options.ShowLabels = true + } +} + +// CommonLabels displays the "common labels". +func CommonLabels() Option { + return func(logs *Logs) { + logs.Builder.LogsPanel.Options.ShowCommonLabels = true + } +} + +// WrapLines enables line wrapping. +func WrapLines() Option { + return func(logs *Logs) { + logs.Builder.LogsPanel.Options.WrapLogMessage = true + } +} + +// PrettifyJSON pretty prints all JSON logs. This setting does not affect logs +// in any format other than JSON. +func PrettifyJSON() Option { + return func(logs *Logs) { + logs.Builder.LogsPanel.Options.PrettifyLogMessage = true + } +} + +// HideLogDetails disables the log details view for each log row. +func HideLogDetails() Option { + return func(logs *Logs) { + logs.Builder.LogsPanel.Options.EnableLogDetails = false + } +} + +// Order display results in descending or ascending time order. +// The default is Descending, showing the newest logs first. +// Set to Ascending to show the oldest log lines first. +func Order(order SortOrder) Option { + return func(logs *Logs) { + logs.Builder.LogsPanel.Options.SortOrder = string(order) + } +} + +// Deduplication sets the deduplication strategy. +func Deduplication(dedup DedupStrategy) Option { + return func(logs *Logs) { + logs.Builder.LogsPanel.Options.DedupStrategy = string(dedup) + } +} diff --git a/vendor/github.com/K-Phoen/grabana/row/row.go b/vendor/github.com/K-Phoen/grabana/row/row.go index 096cfa0b..91f03fa9 100644 --- a/vendor/github.com/K-Phoen/grabana/row/row.go +++ b/vendor/github.com/K-Phoen/grabana/row/row.go @@ -3,6 +3,7 @@ package row import ( "github.com/K-Phoen/grabana/graph" "github.com/K-Phoen/grabana/heatmap" + "github.com/K-Phoen/grabana/logs" "github.com/K-Phoen/grabana/singlestat" "github.com/K-Phoen/grabana/table" "github.com/K-Phoen/grabana/text" @@ -53,6 +54,15 @@ func WithTimeSeries(title string, options ...timeseries.Option) Option { } } +// WithLogs adds a "logs" panel in the row. +func WithLogs(title string, options ...logs.Option) Option { + return func(row *Row) { + panel := logs.New(title, options...) + + row.builder.Add(panel.Builder) + } +} + // WithSingleStat adds a "single stat" panel in the row. func WithSingleStat(title string, options ...singlestat.Option) Option { return func(row *Row) { diff --git a/vendor/github.com/K-Phoen/grabana/target/loki/loki.go b/vendor/github.com/K-Phoen/grabana/target/loki/loki.go new file mode 100644 index 00000000..9cba38ad --- /dev/null +++ b/vendor/github.com/K-Phoen/grabana/target/loki/loki.go @@ -0,0 +1,47 @@ +package loki + +// Option represents an option that can be used to configure a loki query. +type Option func(target *Loki) + +// Loki represents a loki query. +type Loki struct { + Ref string + Hidden bool + Expr string + LegendFormat string +} + +// New creates a new prometheus query. +func New(query string, options ...Option) *Loki { + loki := &Loki{ + Expr: query, + } + + for _, opt := range options { + opt(loki) + } + + return loki +} + +// Legend sets the legend format. +func Legend(legend string) Option { + return func(loki *Loki) { + loki.LegendFormat = legend + } +} + +// Ref sets the reference ID for this query. +func Ref(ref string) Option { + return func(loki *Loki) { + loki.Ref = ref + } +} + +// Hide the query. Grafana does not send hidden queries to the data source, +// but they can still be referenced in alerts. +func Hide() Option { + return func(loki *Loki) { + loki.Hidden = true + } +} diff --git a/vendor/github.com/K-Phoen/grabana/target/stackdriver/stackdriver.go b/vendor/github.com/K-Phoen/grabana/target/stackdriver/stackdriver.go index 6ec5b87f..da97a247 100644 --- a/vendor/github.com/K-Phoen/grabana/target/stackdriver/stackdriver.go +++ b/vendor/github.com/K-Phoen/grabana/target/stackdriver/stackdriver.go @@ -8,6 +8,12 @@ type Option func(target *Stackdriver) const AlignmentStackdriverAuto = "stackdriver-auto" const AlignmentGrafanaAuto = "grafana-auto" +// PreprocessorMethod defines the available pre-processing options. +type PreprocessorMethod string + +const PreprocessRate = "rate" +const PreprocessDelta = "delta" + // Aligner specifies the operation that will be applied to the data points in // each alignment period in a time series. // See https://cloud.google.com/monitoring/api/ref_v3/rest/v3/projects.alertPolicies#Aligner @@ -148,6 +154,13 @@ func Alignment(aligner Aligner, alignmentPeriod string) Option { } } +// Preprocessor defines how the time series should be pre-processed. +func Preprocessor(preprocessor PreprocessorMethod) Option { + return func(stackdriver *Stackdriver) { + stackdriver.Builder.Preprocessor = string(preprocessor) + } +} + // Filter allows to specify which time series will be in the results. func Filter(filters ...FilterOption) Option { return func(stackdriver *Stackdriver) { diff --git a/vendor/github.com/K-Phoen/sdk/alert-manager.go b/vendor/github.com/K-Phoen/sdk/alert-manager.go new file mode 100644 index 00000000..fbb15752 --- /dev/null +++ b/vendor/github.com/K-Phoen/sdk/alert-manager.go @@ -0,0 +1,73 @@ +package sdk + +/* + Copyright 2016 Alexander I.Grafov + Copyright 2016-2019 The Grafana SDK authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + ॐ तारे तुत्तारे तुरे स्व +*/ + +type AlertManager struct { + Config AlertManagerConfig `json:"alertmanager_config"` + TemplateFiles MessageTemplate `json:"template_files"` +} + +type ContactPoint struct { + Name string `json:"name"` + GrafanaManagedReceivers []ContactPointType `json:"grafana_managed_receiver_configs,omitempty"` +} + +type ContactPointType struct { + UID string `json:"uid,omitempty"` + Name string `json:"name"` + Type string `json:"type"` + DisableResolveMessage bool `json:"disableResolveMessage"` + Settings map[string]interface{} `json:"settings"` + SecureSettings map[string]interface{} `json:"secureSettings,omitempty"` +} + +type AlertManagerConfig struct { + Receivers []ContactPoint `json:"receivers"` + Route NotificationPolicies `json:"route"` + Templates []string `json:"templates"` +} + +// Map of template name → template +type MessageTemplate map[string]string + +type NotificationPolicies struct { + // Default alert receiver + Receiver string `json:"receiver"` + // Default timing settings + GroupInterval string `json:"group_interval,omitempty"` + GroupWait string `json:"group_wait,omitempty"` + RepeatInterval string `json:"repeat_interval,omitempty"` + + // Routing policies + Routes []NotificationRoutingPolicy `json:"routes,omitempty"` +} + +type NotificationRoutingPolicy struct { + // Alert receiver + Receiver string `json:"receiver"` + // Default timing settings overrides + GroupInterval string `json:"group_interval,omitempty"` + GroupWait string `json:"group_wait,omitempty"` + RepeatInterval string `json:"repeat_interval,omitempty"` + + ObjectMatchers []AlertObjectMatcher `json:"object_matchers"` +} + +type AlertObjectMatcher [3]string diff --git a/vendor/github.com/K-Phoen/sdk/panel.go b/vendor/github.com/K-Phoen/sdk/panel.go index 4249b11b..5af303f9 100644 --- a/vendor/github.com/K-Phoen/sdk/panel.go +++ b/vendor/github.com/K-Phoen/sdk/panel.go @@ -40,6 +40,7 @@ const ( BarGaugeType HeatmapType TimeseriesType + LogsType ) const MixedSource = "-- Mixed --" @@ -62,6 +63,7 @@ type ( *BarGaugePanel *HeatmapPanel *TimeseriesPanel + *LogsPanel *CustomPanel } panelType int8 @@ -427,6 +429,20 @@ type ( FixedColor string `json:"fixedColor,omitempty"` SeriesBy string `json:"seriesBy,omitempty"` } + LogsPanel struct { + Targets []Target `json:"targets,omitempty"` + Options LogsOptions `json:"options,omitempty"` + } + LogsOptions struct { + DedupStrategy string `json:"dedupStrategy"` + WrapLogMessage bool `json:"wrapLogMessage"` + ShowTime bool `json:"showTime"` + ShowLabels bool `json:"showLabels"` + ShowCommonLabels bool `json:"showCommonLabels"` + PrettifyLogMessage bool `json:"prettifyLogMessage"` + SortOrder string `json:"sortOrder"` + EnableLogDetails bool `json:"enableLogDetails"` + } CustomPanel map[string]interface{} ) @@ -632,6 +648,7 @@ type Target struct { CrossSeriesReducer string `json:"crossSeriesReducer,omitempty"` PerSeriesAligner string `json:"perSeriesAligner,omitempty"` ValueType string `json:"valueType,omitempty"` + Preprocessor string `json:"preprocessor,omitempty"` GroupBys []string `json:"groupBys,omitempty"` Tags []struct { Key string `json:"key,omitempty"` @@ -735,6 +752,24 @@ func NewTimeseries(title string) *Panel { } } +// NewLogs initializes a new panel as a Logs panel. +func NewLogs(title string) *Panel { + if title == "" { + title = "Panel Title" + } + + return &Panel{ + CommonPanel: CommonPanel{ + OfType: LogsType, + Title: title, + Type: "logs", + Span: 12, + IsNew: true, + }, + LogsPanel: &LogsPanel{}, + } +} + // NewTable initializes panel with a table panel. func NewTable(title string) *Panel { if title == "" { @@ -878,6 +913,8 @@ func (p *Panel) ResetTargets() { p.HeatmapPanel.Targets = nil case TimeseriesType: p.TimeseriesPanel.Targets = nil + case LogsType: + p.LogsPanel.Targets = nil } } @@ -899,6 +936,8 @@ func (p *Panel) AddTarget(t *Target) { p.HeatmapPanel.Targets = append(p.HeatmapPanel.Targets, *t) case TimeseriesType: p.TimeseriesPanel.Targets = append(p.TimeseriesPanel.Targets, *t) + case LogsType: + p.LogsPanel.Targets = append(p.LogsPanel.Targets, *t) } // TODO check for existing refID } @@ -928,6 +967,8 @@ func (p *Panel) SetTarget(t *Target) { setTarget(t, &p.HeatmapPanel.Targets) case TimeseriesType: setTarget(t, &p.TimeseriesPanel.Targets) + case LogsType: + setTarget(t, &p.LogsPanel.Targets) } } @@ -961,6 +1002,8 @@ func (p *Panel) RepeatDatasourcesForEachTarget(dsNames ...string) { repeatDS(dsNames, &p.HeatmapPanel.Targets) case TimeseriesType: repeatDS(dsNames, &p.TimeseriesPanel.Targets) + case LogsType: + repeatDS(dsNames, &p.LogsPanel.Targets) } } @@ -997,6 +1040,8 @@ func (p *Panel) RepeatTargetsForDatasources(dsNames ...string) { repeatTarget(dsNames, &p.HeatmapPanel.Targets) case TimeseriesType: repeatTarget(dsNames, &p.TimeseriesPanel.Targets) + case LogsType: + repeatTarget(dsNames, &p.LogsPanel.Targets) } } @@ -1018,6 +1063,8 @@ func (p *Panel) GetTargets() *[]Target { return &p.HeatmapPanel.Targets case TimeseriesType: return &p.TimeseriesPanel.Targets + case LogsType: + return &p.LogsPanel.Targets default: return nil } @@ -1087,6 +1134,12 @@ func (p *Panel) UnmarshalJSON(b []byte) (err error) { if err = json.Unmarshal(b, ×eries); err == nil { p.TimeseriesPanel = ×eries } + case "logs": + var logs LogsPanel + p.OfType = LogsType + if err = json.Unmarshal(b, &logs); err == nil { + p.LogsPanel = &logs + } case "row": var rowpanel RowPanel p.OfType = RowType @@ -1178,6 +1231,12 @@ func (p *Panel) MarshalJSON() ([]byte, error) { TimeseriesPanel }{p.CommonPanel, *p.TimeseriesPanel} return json.Marshal(outTimeseries) + case LogsType: + var outLogs = struct { + CommonPanel + LogsPanel + }{p.CommonPanel, *p.LogsPanel} + return json.Marshal(outLogs) case CustomType: var outCustom = customPanelOutput{ p.CommonPanel, diff --git a/vendor/modules.txt b/vendor/modules.txt index b4566ba8..58b5c8f6 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -20,7 +20,7 @@ github.com/Azure/go-autorest/logger # github.com/Azure/go-autorest/tracing v0.6.0 ## explicit; go 1.12 github.com/Azure/go-autorest/tracing -# github.com/K-Phoen/grabana v0.20.6 +# github.com/K-Phoen/grabana v0.20.6 => ../grabana ## explicit; go 1.16 github.com/K-Phoen/grabana github.com/K-Phoen/grabana/alert @@ -37,11 +37,13 @@ github.com/K-Phoen/grabana/graph github.com/K-Phoen/grabana/graph/series github.com/K-Phoen/grabana/heatmap github.com/K-Phoen/grabana/heatmap/axis +github.com/K-Phoen/grabana/logs github.com/K-Phoen/grabana/row github.com/K-Phoen/grabana/singlestat github.com/K-Phoen/grabana/table github.com/K-Phoen/grabana/target/graphite github.com/K-Phoen/grabana/target/influxdb +github.com/K-Phoen/grabana/target/loki github.com/K-Phoen/grabana/target/prometheus github.com/K-Phoen/grabana/target/stackdriver github.com/K-Phoen/grabana/text @@ -52,7 +54,7 @@ github.com/K-Phoen/grabana/variable/custom github.com/K-Phoen/grabana/variable/datasource github.com/K-Phoen/grabana/variable/interval github.com/K-Phoen/grabana/variable/query -# github.com/K-Phoen/sdk v0.8.1 +# github.com/K-Phoen/sdk v0.8.4 ## explicit; go 1.16 github.com/K-Phoen/sdk # github.com/beorn7/perks v1.0.1 @@ -709,3 +711,4 @@ sigs.k8s.io/structured-merge-diff/v4/value # sigs.k8s.io/yaml v1.3.0 ## explicit; go 1.12 sigs.k8s.io/yaml +# github.com/K-Phoen/grabana => ../grabana