chore: [release-2.9.x] feat: Add ingester_chunks_flush_failures_total (…

…#12925) (#13274) Signed-off-by: George Robinson <george.robinson@grafana.com> Co-authored-by: Trevor Whitney <trevorjwhitney@gmail.com>
grafana · Jul 2, 2024 · 1b7071c · 1b7071c
1 parent 8143fbb
commit 1b7071c
Show file tree

Hide file tree

Showing 6 changed files with 46 additions and 43 deletions.
diff --git a/.github/release-workflows.jsonnet b/.github/release-workflows.jsonnet
@@ -2,11 +2,7 @@ local lokiRelease = import 'workflows/main.jsonnet';
 local build = lokiRelease.build;
 local job = lokiRelease.job;
 
-local releaseLibRef = std.filter(
-  function(dep) dep.source.git.remote == 'https://github.com/grafana/loki-release.git',
-  (import 'jsonnetfile.json').dependencies
-)[0].version;
-
+local releaseLibRef = 'loki-2.9.x';
 local checkTemplate = 'grafana/loki-release/.github/workflows/check.yml@%s' % releaseLibRef;
 
 local imageJobs = {

diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml
@@ -2,8 +2,8 @@
   "check":
     "uses": "grafana/loki-release/.github/workflows/check.yml@loki-2.9.x"
     "with":
-      "build_image": "grafana/loki-build-image:0.33.1"
-      "golang_ci_lint_version": "v1.55.1"
+      "build_image": "grafana/loki-build-image:0.30.1"
+      "golang_ci_lint_version": "v1.51.2"
       "release_lib_ref": "loki-2.9.x"
       "skip_validation": false
       "use_github_app_token": true

diff --git a/.github/workflows/minor-release-pr.yml b/.github/workflows/minor-release-pr.yml
@@ -14,8 +14,8 @@ jobs:
   check:
     uses: "grafana/loki-release/.github/workflows/check.yml@loki-2.9.x"
     with:
-      build_image: "grafana/loki-build-image:0.33.1"
-      golang_ci_lint_version: "v1.55.1"
+      build_image: "grafana/loki-build-image:0.30.1"
+      golang_ci_lint_version: "v1.51.2"
       release_lib_ref: "loki-2.9.x"
       skip_validation: false
       use_github_app_token: true
@@ -89,7 +89,7 @@ jobs:
           --target-branch "${{ steps.extract_branch.outputs.branch }}" \
           --token "${{ steps.github_app_token.outputs.token }}" \
           --versioning-strategy "${{ env.VERSIONING_STRATEGY }}"
-
+        
       working-directory: "lib"
   dist:
     needs:
@@ -136,7 +136,7 @@ jobs:
           --env SKIP_ARM \
           --volume .:/src/loki \
           --workdir /src/loki \
-          --entrypoint /bin/sh "grafana/loki-build-image:0.33.1"
+          --entrypoint /bin/sh "grafana/loki-build-image:0.30.1"
           git config --global --add safe.directory /src/loki
           echo "${NFPM_SIGNING_KEY}" > $NFPM_SIGNING_KEY_FILE
           make dist packages
@@ -181,7 +181,7 @@ jobs:
       name: "parse image platform"
       run: |
         mkdir -p images
-
+        
         platform="$(echo "${{ matrix.platform}}" |  sed  "s/\(.*\)\/\(.*\)/\1-\2/")"
         echo "platform=${platform}" >> $GITHUB_OUTPUT
         echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT
@@ -243,7 +243,7 @@ jobs:
       name: "parse image platform"
       run: |
         mkdir -p images
-
+        
         platform="$(echo "${{ matrix.platform}}" |  sed  "s/\(.*\)\/\(.*\)/\1-\2/")"
         echo "platform=${platform}" >> $GITHUB_OUTPUT
         echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT
@@ -305,7 +305,7 @@ jobs:
       name: "parse image platform"
       run: |
         mkdir -p images
-
+        
         platform="$(echo "${{ matrix.platform}}" |  sed  "s/\(.*\)\/\(.*\)/\1-\2/")"
         echo "platform=${platform}" >> $GITHUB_OUTPUT
         echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT
@@ -369,7 +369,7 @@ jobs:
       name: "parse image platform"
       run: |
         mkdir -p images
-
+        
         platform="$(echo "${{ matrix.platform}}" |  sed  "s/\(.*\)\/\(.*\)/\1-\2/")"
         echo "platform=${platform}" >> $GITHUB_OUTPUT
         echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT
@@ -431,7 +431,7 @@ jobs:
       name: "parse image platform"
       run: |
         mkdir -p images
-
+        
         platform="$(echo "${{ matrix.platform}}" |  sed  "s/\(.*\)\/\(.*\)/\1-\2/")"
         echo "platform=${platform}" >> $GITHUB_OUTPUT
         echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT
@@ -495,7 +495,7 @@ jobs:
       name: "parse image platform"
       run: |
         mkdir -p images
-
+        
         platform="$(echo "${{ matrix.platform}}" |  sed  "s/\(.*\)\/\(.*\)/\1-\2/")"
         echo "platform=${platform}" >> $GITHUB_OUTPUT
         echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT
@@ -559,7 +559,7 @@ jobs:
       name: "parse image platform"
       run: |
         mkdir -p images
-
+        
         platform="$(echo "${{ matrix.platform}}" |  sed  "s/\(.*\)\/\(.*\)/\1-\2/")"
         echo "platform=${platform}" >> $GITHUB_OUTPUT
         echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT
@@ -623,7 +623,7 @@ jobs:
       name: "parse image platform"
       run: |
         mkdir -p images
-
+        
         platform="$(echo "${{ matrix.platform}}" |  sed  "s/\(.*\)\/\(.*\)/\1-\2/")"
         echo "platform=${platform}" >> $GITHUB_OUTPUT
         echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT
@@ -687,7 +687,7 @@ jobs:
       name: "parse image platform"
       run: |
         mkdir -p images
-
+        
         platform="$(echo "${{ matrix.platform}}" |  sed  "s/\(.*\)\/\(.*\)/\1-\2/")"
         echo "platform=${platform}" >> $GITHUB_OUTPUT
         echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT
@@ -778,16 +778,16 @@ jobs:
           --target-branch "${{ steps.extract_branch.outputs.branch }}" \
           --token "${{ steps.github_app_token.outputs.token }}" \
           --versioning-strategy "${{ env.VERSIONING_STRATEGY }}"
-
+        
         cat release.json
-
-        if [[ `jq length release.json` -gt 1 ]]; then
+        
+        if [[ `jq length release.json` -gt 1 ]]; then 
           echo 'release-please would create more than 1 PR, so cannot determine correct version'
           echo "pr_created=false" >> $GITHUB_OUTPUT
           exit 1
         fi
-
-        if [[ `jq length release.json` -eq 0 ]]; then
+        
+        if [[ `jq length release.json` -eq 0 ]]; then 
           echo "pr_created=false" >> $GITHUB_OUTPUT
         else
           version="$(npm run --silent get-version)"

diff --git a/.github/workflows/patch-release-pr.yml b/.github/workflows/patch-release-pr.yml
@@ -14,8 +14,8 @@ jobs:
   check:
     uses: "grafana/loki-release/.github/workflows/check.yml@loki-2.9.x"
     with:
-      build_image: "grafana/loki-build-image:0.33.1"
-      golang_ci_lint_version: "v1.55.1"
+      build_image: "grafana/loki-build-image:0.30.1"
+      golang_ci_lint_version: "v1.51.2"
       release_lib_ref: "loki-2.9.x"
       skip_validation: false
       use_github_app_token: true
@@ -89,7 +89,7 @@ jobs:
           --target-branch "${{ steps.extract_branch.outputs.branch }}" \
           --token "${{ steps.github_app_token.outputs.token }}" \
           --versioning-strategy "${{ env.VERSIONING_STRATEGY }}"
-
+        
       working-directory: "lib"
   dist:
     needs:
@@ -136,7 +136,7 @@ jobs:
           --env SKIP_ARM \
           --volume .:/src/loki \
           --workdir /src/loki \
-          --entrypoint /bin/sh "grafana/loki-build-image:0.33.1"
+          --entrypoint /bin/sh "grafana/loki-build-image:0.30.1"
           git config --global --add safe.directory /src/loki
           echo "${NFPM_SIGNING_KEY}" > $NFPM_SIGNING_KEY_FILE
           make dist packages
@@ -181,7 +181,7 @@ jobs:
       name: "parse image platform"
       run: |
         mkdir -p images
-
+        
         platform="$(echo "${{ matrix.platform}}" |  sed  "s/\(.*\)\/\(.*\)/\1-\2/")"
         echo "platform=${platform}" >> $GITHUB_OUTPUT
         echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT
@@ -243,7 +243,7 @@ jobs:
       name: "parse image platform"
       run: |
         mkdir -p images
-
+        
         platform="$(echo "${{ matrix.platform}}" |  sed  "s/\(.*\)\/\(.*\)/\1-\2/")"
         echo "platform=${platform}" >> $GITHUB_OUTPUT
         echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT
@@ -305,7 +305,7 @@ jobs:
       name: "parse image platform"
       run: |
         mkdir -p images
-
+        
         platform="$(echo "${{ matrix.platform}}" |  sed  "s/\(.*\)\/\(.*\)/\1-\2/")"
         echo "platform=${platform}" >> $GITHUB_OUTPUT
         echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT
@@ -369,7 +369,7 @@ jobs:
       name: "parse image platform"
       run: |
         mkdir -p images
-
+        
         platform="$(echo "${{ matrix.platform}}" |  sed  "s/\(.*\)\/\(.*\)/\1-\2/")"
         echo "platform=${platform}" >> $GITHUB_OUTPUT
         echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT
@@ -431,7 +431,7 @@ jobs:
       name: "parse image platform"
       run: |
         mkdir -p images
-
+        
         platform="$(echo "${{ matrix.platform}}" |  sed  "s/\(.*\)\/\(.*\)/\1-\2/")"
         echo "platform=${platform}" >> $GITHUB_OUTPUT
         echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT
@@ -495,7 +495,7 @@ jobs:
       name: "parse image platform"
       run: |
         mkdir -p images
-
+        
         platform="$(echo "${{ matrix.platform}}" |  sed  "s/\(.*\)\/\(.*\)/\1-\2/")"
         echo "platform=${platform}" >> $GITHUB_OUTPUT
         echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT
@@ -559,7 +559,7 @@ jobs:
       name: "parse image platform"
       run: |
         mkdir -p images
-
+        
         platform="$(echo "${{ matrix.platform}}" |  sed  "s/\(.*\)\/\(.*\)/\1-\2/")"
         echo "platform=${platform}" >> $GITHUB_OUTPUT
         echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT
@@ -623,7 +623,7 @@ jobs:
       name: "parse image platform"
       run: |
         mkdir -p images
-
+        
         platform="$(echo "${{ matrix.platform}}" |  sed  "s/\(.*\)\/\(.*\)/\1-\2/")"
         echo "platform=${platform}" >> $GITHUB_OUTPUT
         echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT
@@ -687,7 +687,7 @@ jobs:
       name: "parse image platform"
       run: |
         mkdir -p images
-
+        
         platform="$(echo "${{ matrix.platform}}" |  sed  "s/\(.*\)\/\(.*\)/\1-\2/")"
         echo "platform=${platform}" >> $GITHUB_OUTPUT
         echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT
@@ -778,16 +778,16 @@ jobs:
           --target-branch "${{ steps.extract_branch.outputs.branch }}" \
           --token "${{ steps.github_app_token.outputs.token }}" \
           --versioning-strategy "${{ env.VERSIONING_STRATEGY }}"
-
+        
         cat release.json
-
-        if [[ `jq length release.json` -gt 1 ]]; then
+        
+        if [[ `jq length release.json` -gt 1 ]]; then 
           echo 'release-please would create more than 1 PR, so cannot determine correct version'
           echo "pr_created=false" >> $GITHUB_OUTPUT
           exit 1
         fi
-
-        if [[ `jq length release.json` -eq 0 ]]; then
+        
+        if [[ `jq length release.json` -eq 0 ]]; then 
           echo "pr_created=false" >> $GITHUB_OUTPUT
         else
           version="$(npm run --silent get-version)"

diff --git a/pkg/ingester/flush.go b/pkg/ingester/flush.go
@@ -363,6 +363,7 @@ func (i *Ingester) encodeChunk(ctx context.Context, ch *chunk.Chunk, desc *chunk
 // chunk to have another opportunity to be flushed.
 func (i *Ingester) flushChunk(ctx context.Context, ch *chunk.Chunk) error {
 	if err := i.store.Put(ctx, []chunk.Chunk{*ch}); err != nil {
+		i.metrics.chunksFlushFailures.Inc()
 		return fmt.Errorf("store put chunk: %w", err)
 	}
 	i.metrics.flushedChunksStats.Inc(1)

diff --git a/pkg/ingester/metrics.go b/pkg/ingester/metrics.go
@@ -46,6 +46,7 @@ type ingesterMetrics struct {
 	chunkSizePerTenant            *prometheus.CounterVec
 	chunkAge                      prometheus.Histogram
 	chunkEncodeTime               prometheus.Histogram
+	chunksFlushFailures           prometheus.Counter
 	chunksFlushedPerReason        *prometheus.CounterVec
 	chunkLifespan                 prometheus.Histogram
 	flushedChunksStats            *analytics.Counter
@@ -229,6 +230,11 @@ func newIngesterMetrics(r prometheus.Registerer) *ingesterMetrics {
 			// 10ms to 10s.
 			Buckets: prometheus.ExponentialBuckets(0.01, 4, 6),
 		}),
+		chunksFlushFailures: promauto.With(r).NewCounter(prometheus.CounterOpts{
+			Namespace: "loki",
+			Name:      "ingester_chunks_flush_failures_total",
+			Help:      "Total number of flush failures.",
+		}),
 		chunksFlushedPerReason: promauto.With(r).NewCounterVec(prometheus.CounterOpts{
 			Namespace: "loki",
 			Name:      "ingester_chunks_flushed_total",