grafana · francoposa · May 7, 2025 · May 2, 2025 · May 2, 2025 · May 2, 2025
@@ -13,13 +13,14 @@ jobs:
   call-update-helm-repo:
     uses: grafana/helm-charts/.github/workflows/update-helm-repo.yaml@main
     permissions:
-      # Give restricted read permissions to "update-helm-repo" action. The action
-      # requests higher permissions for the specific "release" job.
-      contents: read
+      id-token: write
+      contents: write
+      packages: write
     with:
       charts_dir: operations/helm/charts
       cr_configfile: operations/helm/cr.yaml
       ct_configfile: operations/helm/ct.yaml
     secrets:
-      github_app_id: ${{ secrets.MIMIR_HELM_RELEASE_APP_ID }}
-      github_app_pem: ${{ secrets.MIMIR_HELM_RELEASE_APP_KEY_PEM }}
+      # "mimir-helm-release" is a GitHub app, that has permissions to push to grafana/helm-charts.
+      # The app's credentials seat in the mimir repo's vault (ref https://github.com/grafana/shared-workflows/tree/main/actions/get-vault-secrets).
+      vault_repo_secret_name: mimir-helm-release
@@ -28,11 +28,11 @@ jobs:
       - uses: actions/checkout@v4
         with:
           persist-credentials: false
-      - uses: grafana/writers-toolkit/publish-technical-documentation@39cdc38767184996e25d611923f8ce697e33bc70 # publish-technical-documentation/v1.2.0
+      - uses: grafana/writers-toolkit/publish-technical-documentation@6200164349ac8f24bab065ce3689dd8c9a093061 # 
8000
publish-technical-documentation/v1.2.0
         with:
           source_directory: docs/sources/mimir
           website_directory: content/docs/mimir/next
-      - uses: grafana/writers-toolkit/publish-technical-documentation@39cdc38767184996e25d611923f8ce697e33bc70 # publish-technical-documentation/v1.2.0
+      - uses: grafana/writers-toolkit/publish-technical-documentation@6200164349ac8f24bab065ce3689dd8c9a093061 # publish-technical-documentation/v1.2.0
         with:
           source_directory: docs/sources/helm-charts/mimir-distributed
           website_directory: content/docs/helm-charts/mimir-distributed/next
@@ -33,7 +33,7 @@ jobs:
           # Full fetch depth is required to fetch tags. The publishing workflow uses tags to prevent publishing a release branch before it has been formally released, as determined by the presence of a matching tag for the release branch.
           fetch-depth: 0
           persist-credentials: false
-      - uses: grafana/writers-toolkit/publish-technical-documentation-release@d83ba5389fb8de1458b12bcc35ad4a4059883029 # publish-technical-documentation/v1.2.0
+      - uses: grafana/writers-toolkit/publish-technical-documentation-release@6200164349ac8f24bab065ce3689dd8c9a093061 # publish-technical-documentation/v1.2.0
         with:
           release_tag_regexp: "^mimir-distributed-(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)$"
           release_branch_regexp: "^mimir-distributed-release-(0|[1-9]\\d*)\\.(0|[1-9]\\d*)$"

@@ -33,7 +33,7 @@ jobs:
           # The publishing workflow uses tags to prevent publishing a release branch before it has been formally released, as determined by the presence of a matching tag for the release branch.
           fetch-depth: 0
           persist-credentials: false
-      - uses: grafana/writers-toolkit/publish-technical-documentation-release@8cc658b604c6e05c275af30163a1c7728dfe19b2 # publish-technical-documentation-release/v2.2.4
+      - uses: grafana/writers-toolkit/publish-technical-documentation-release@6200164349ac8f24bab065ce3689dd8c9a093061 # publish-technical-documentation-release/v2.2.4
         with:
           release_tag_regexp: "^mimir-(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)$"
           release_branch_regexp: "^release-(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)$"

@@ -170,10 +170,18 @@ jobs:
         run: |
           mkdir -p /go/src/github.com/grafana/mimir
           ln -s $GITHUB_WORKSPACE/* /go/src/github.com/grafana/mimir
-      - name: Run Tests
-        run: |
-          echo "Running unit tests (group ${{ matrix.test_group_id }} of ${{ matrix.test_group_total }}) with Go version: $(go version)"
-          ./.github/workflows/scripts/run-unit-tests-group.sh --index ${{ matrix.test_group_id }} --total ${{ matrix.test_group_total }}
+      # We use retry logic for tests to handle flaky tests that might fail intermittently.
+      # The retry action will run the test command up to max_attempts times if it fails.
+      # This helps reduce CI failures caused by temporary issues, timing problems, or race conditions.
+      - name: Run Tests with retry
+        uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2
+        with:
+          timeout_minutes: 180 # 3 hours so that we rely on the timeout in the makefile
+          max_attempts: 2
+          retry_wait_seconds: 0
+          command: |
+            echo "Running unit tests (group ${{ matrix.test_group_id }} of ${{ matrix.test_group_total }}) with Go version: $(go version)"
+            ./.github/workflows/scripts/run-unit-tests-group.sh --index ${{ matrix.test_group_id }} --total ${{ matrix.test_group_total }}
 
   test-docs:
     uses: ./.github/workflows/test-docs.yml
@@ -282,16 +290,25 @@ jobs:
         # locally and the download time doesn't account in the test execution time, which is subject
         # to a timeout
         run: go run ./tools/pre-pull-images | xargs -n1 -P4 docker pull
-      - name: Integration Tests
-        run: |
-          export IMAGE_TAG_RACE=$(make image-tag-race)
-          export MIMIR_IMAGE="grafana/mimir:$IMAGE_TAG_RACE"
-          export IMAGE_TAG=$(make image-tag)
-          export MIMIRTOOL_IMAGE="grafana/mimirtool:$IMAGE_TAG"
-          export MIMIR_CHECKOUT_DIR="/go/src/github.com/grafana/mimir"
-          echo "Running integration tests with image: $MIMIR_IMAGE (Mimir), $MIMIRTOOL_IMAGE (Mimirtool)"
-          echo "Running integration tests (group ${{ matrix.test_group_id }} of ${{ matrix.test_group_total }}) with Go version: $(go version)"
-          ./.github/workflows/scripts/run-integration-tests-group.sh --index ${{ matrix.test_group_id }} --total ${{ matrix.test_group_total }}
+      # We use retry logic for integration tests to handle flaky tests that might fail intermittently.
+      # The retry action will run the test command up to max_attempts times if it fails.
+      # This helps reduce CI failures caused by temporary issues, network problems, or timing conditions
+      # that are common in integration tests with multiple services interacting.
+      - name: Integration Tests with retry
+        uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3.0.2
+        with:
+          timeout_minutes: 180 # 3 hours so that we rely on the timeout in the makefile
+          max_attempts: 2
+          retry_wait_seconds: 0
+          command: |
+            export IMAGE_TAG_RACE=$(make image-tag-race)
+            export MIMIR_IMAGE="grafana/mimir:$IMAGE_TAG_RACE"
+            export IMAGE_TAG=$(make image-tag)
+            export MIMIRTOOL_IMAGE="grafana/mimirtool:$IMAGE_TAG"
+            export MIMIR_CHECKOUT_DIR="/go/src/github.com/grafana/mimir"
+            echo "Running integration tests with image: $MIMIR_IMAGE (Mimir), $MIMIRTOOL_IMAGE (Mimirtool)"
+            echo "Running integration tests (group ${{ matrix.test_group_id }} of ${{ matrix.test_group_total }}) with Go version: $(go version)"
+            ./.github/workflows/scripts/run-integration-tests-group.sh --index ${{ matrix.test_group_id }} --total ${{ matrix.test_group_total }}
 
   deploy:
     needs: [prepare, build, test, lint, integration]
@@ -300,6 +317,9 @@ jobs:
     runs-on: ubuntu-latest
     container:
       image: ${{ needs.prepare.outputs.build_image }}
+    permissions:
+      contents: read
+      id-token: write
     steps:
       - name: Check out repository
         uses: actions/checkout@v4
@@ -319,12 +339,8 @@ jobs:
           name: Docker Images
       - name: Extract Docker Images from Archive
         run: tar xvf images.tar -C /
+      - name: Login to DockerHub
+        uses: grafana/shared-workflows/actions/dockerhub-login@13fb504e3bfe323c1188bf244970d94b2d336e86 # v1.0.1
       - name: Deploy
         run: |
-          if [ -n "$DOCKER_PASSWORD" ]; then
-            printenv DOCKER_PASSWORD | skopeo login -u "$DOCKER_USERNAME" --password-stdin docker.io
-          fi
           ./.github/workflows/scripts/push-images.sh /tmp/images grafana/ $(make image-tag)
-        env:
-          DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
-          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -13,6 +13,7 @@
 * [CHANGE] gRPC/HTTP clients: Rename metric `cortex_client_request_invalid_cluster_validation_labels_total` to `cortex_client_invalid_cluster_validation_label_requests_total`. #11237
 * [FEATURE] Distributor: Experimental support for Prometheus Remote-Write 2.0 protocol. Limitations: Created timestamp is ignored, per series metadata is merged on metric family level automatically, ingestion might fail if client sends ProtoBuf fields out of order. The label `version` is added to the metric `cortex_distributor_requests_in_total` with a value of either `1.0` or `2.0` depending on the detected Remote-Write protocol. #11100 #11101 #11192 #11143
 * [FEATURE] Query-frontend: expand `query-frontend.cache-errors` and `query-frontend.results-cache-ttl-for-errors` configuration options to cache non-transient response failures for instant queries. #11120
+* [FEATURE] Querier, query-frontend, ruler: Enable experimental support for duration expressions in PromQL, which are simple arithmetics on numbers in offset and range specification. #11344
 * [ENHANCEMENT] Ingester: Add support for exporting native histogram cost attribution metrics (`cortex_ingester_attributed_active_native_histogram_series` and `cortex_ingester_attributed_active_native_histogram_buckets`) with labels specified by customers to a custom Prometheus registry. #10892
 * [ENHANCEMENT] Store-gateway: Download sparse headers uploaded by compactors. Compactors have to be configured with `-compactor.upload-sparse-index-headers=true` option. #10879 #11072.
 * [ENHANCEMENT] Compactor: Upload block index file and multiple segment files concurrently. Concurrency scales linearly with block size up to `-compactor.max-per-block-upload-concurrency`. #10947
@@ -43,6 +44,7 @@
 * [ENHANCEMENT] Ingest-Storage: Add `ingest-storage.kafka.producer-record-version` to allow control Kafka record versioning. #11244
 * [ENHANCEMENT] Ruler: Update `<prometheus-http-prefix>/api/v1/rules` and `<prometheus-http-prefix>/api/v1/alerts` to reply with HTTP error 422 if rule evaluation is completely disabled for the tenant. If only recording rule- or alerting rule evaluation is disabled for the tenant, the response now includes a corresponding warning. #11321
 * [ENHANCEMENT] Add tenant configuration block `ruler_alertmanager_client_config` which allows the Ruler's Alertmanager client options to be specified on a per-tenant basis. #10816
+* [ENHANCEMENT] Store-gateway: Retry querying blocks from store-gateways with dynamic replication until trying all possible store-gateways. #11354 #11398
 * [BUGFIX] OTLP: Fix response body and Content-Type header to align with spec. #10852
 * [BUGFIX] Compactor: fix issue where block becomes permanently stuck when the Compactor's block cleanup job partially deletes a block. #10888
 * [BUGFIX] Storage: fix intermittent failures in S3 upload retries. #10952
@@ -63,7 +65,7 @@
 * [ENHANCEMENT] Alerts: Make `MimirRolloutStuck` a critical alert if it has been firing for 6h. #10890
 * [ENHANCEMENT] Dashboards: Add panels to the `Mimir / Tenants` and `Mimir / Top Tenants` dashboards showing the rate of gateway requests. #10978
 * [ENHANCEMENT] Alerts: Improve `MimirIngesterFailsToProcessRecordsFromKafka` to not fire during forced TSDB head compaction. #11006
-* [ENHANCEMENT] Alerts: Add alerts for invalid cluster validation labels. #11255 #11282
+* [ENHANCEMENT] Alerts: Add alerts for invalid cluster validation labels. #11255 #11282 #11413
 * [CHANGE] Alerts: Update query for `MimirBucketIndexNotUpdated`. Use `max_over_time` to prevent alert firing when pods rotate. #11311
 * [BUGFIX] Dashboards: fix "Mimir / Tenants" legends for non-Kubernetes deployments. #10891
 * [BUGFIX] Recording rules: fix `cluster_namespace_deployment:actual_replicas:count` recording rule when there's a mix on single-zone and multi-zone deployments. #11287
@@ -75,6 +77,7 @@
 * [CHANGE] Update rollout-operator version to 0.26.0. #11232
 * [CHANGE] Memcached: Set a timeout of `500ms` for the `ruler-storage` cache instead of the default `200ms`. #11231
 * [FEATURE] Make ingest storage ingester HPA behavior configurable through `_config.ingest_storage_ingester_hpa_behavior`. #11168
+* [FEATURE] Add an alternate ingest storage HPA trigger that targets maximum owned series per pod. #11356
 
 ### Mimirtool
 

@@ -11,6 +11,7 @@ import (
 
 	"github.com/alecthomas/kingpin/v2"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/prometheus/promql/parser"
 
 	"github.com/grafana/mimir/pkg/mimirtool/commands"
 	"github.com/grafana/mimir/pkg/mimirtool/version"
@@ -35,6 +36,8 @@ var (
 )
 
 func main() {
+	parser.ExperimentalDurationExpr = true
+
 	app := kingpin.New("mimirtool", "A command-line tool to manage Mimir and GEM.")
 
 	envVars := commands.NewEnvVarsWithPrefix("MIMIR")

@@ -42,7 +42,7 @@ services:
     environment:
       - GF_AUTH_ANONYMOUS_ENABLED=true
       - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
-    image: grafana/grafana:11.6.0
+    image: grafana/grafana:11.6.1
     ports:
       - 3000:3000
     volumes:
@@ -63,7 +63,7 @@ services:
   grafana-alloy:
     profiles:
       - grafana-alloy
-    image: grafana/alloy:v1.8.1
+    image: grafana/alloy:v1.8.2
     command: ["run", "--server.http.listen-addr=0.0.0.0:9092", "--storage.path=/var/lib/alloy/data", "/etc/alloy/config.alloy"]
     volumes:
       - ./config/config.alloy:/etc/alloy/config.alloy

@@ -171,6 +171,8 @@ Until deprecation is complete, we need to ensure that a user who is still settin
    }
    ```
 
+   Note this must be added prior to any checks for `t.Cfg.RuntimeConfig.LoadPath` or early exit paths within `initRuntimeConfig`. It's possible to configure Mimir with purely static config and disable runtime config; the mapping needs to happen in either case.
+
 ## Updating Mimir Docs
 
 1. Update the Mimir Config file docs.

@@ -217,6 +217,7 @@ The following features are currently experimental:
   - Enable PromQL experimental functions per-tenant (`-query-frontend.enabled-promql-experimental-functions` and the `enabled_promql_experimental_functions` per-tenant limit)
   - Support for cluster validation via `-query-frontend.client-cluster-validation.label` or `-common.client-cluster-validation.label`.
     Requests with invalid cluster validation labels are tracked via the `cortex_client_invalid_cluster_validation_label_requests_total` metric.
+  - Support for duration expressions in PromQL, which are simple arithmetics on numbers in offset and range specification.
 - Query-scheduler
   - `-query-scheduler.querier-forget-delay`
 - Store-gateway

@@ -7,10 +7,6 @@ weight: 160
 
 # Configure native histograms
 
-{{% admonition type="note" %}}
-Native histograms are an experimental feature of Grafana Mimir.
-{{% /admonition %}}
-
 You can configure native histograms ingestion via the Prometheus [remote write API](/docs/mimir/<MIMIR_VERSION>/references/http-api/#distributor) endpoint globally or per tenant.
 
 {{% admonition type="note" %}}

@@ -111,9 +111,13 @@ service:
 
 OpenTelemetry metrics use resource attributes to describe the set of characteristics associated with a given resource, or entity, producing telemetry data. For example, a host resource might have multiple attributes, including an ID, an image, and a type.
 
-To optimize the storage of and ability to query this data, you can configure Mimir to promote specified OTel resource attributes to labels at the time of ingestion, with periods (`.`) replaced by underscores (`_`).
+To optimize the storage of and ability to query this data, you can use the `-distributor.otel-promote-resource-attributes` option to configure Mimir to promote specified OTel resource attributes to labels at the time of ingestion.
 
-Grafana Cloud automatically promotes the following OTel resource attributes to labels:
+{{< admonition type="note" >}}
+The `-distributor.otel-promote-resource-attributes` option is an experimental feature in Grafana Mimir.
+{{< /admonition >}}
+
+Grafana Cloud automatically promotes the following OTel resource attributes to labels, with periods (`.`) replaced by underscores (`_`):
 
 - `service.instance.id`
 - `service.name`
@@ -135,13 +139,15 @@ Grafana Cloud automatically promotes the following OTel resource attributes to l
 - `k8s.statefulset.name`
 
 {{< admonition type="note" >}}
-To disable this setting or to update this list, contact Grafana Labs Support.
+To disable this option or to update this list, contact Grafana Labs Support.
 {{< /admonition >}}
 
 Mimir stores additional OTel resource attributes in a separate series called `target_info`, which you can query using a join query or the Prometheus `info()` function. Refer to [Functions](https://prometheus.io/docs/prometheus/latest/querying/functions/) in the Prometheus documentation for more information.
 
 To learn more about OpenTelemetry resource attributes, refer to [Resources](https://opentelemetry.io/docs/languages/js/resources/) in the OpenTelemetry documentation.
 
+To learn more about ingesting OpenTelemetry data in Grafana Cloud, refer to [OTLP: OpenTelemetry Protocol format considerations](https://grafana.com/docs/grafana-cloud/send-data/otlp/otlp-format-considerations/).
+
 ## Format considerations
 
 We follow the official [OTLP Metric points to Prometheus](https://opentelemetry.io/docs/reference/specification/compatibility/prometheus_and_openmetrics/#otlp-metric-points-to-prometheus) specification.

@@ -1383,19 +1383,19 @@ How to **investigate**:
 
 ### MimirIngestedDataTooFarInTheFuture
 
-This alert fires when Mimir ingester accepts a sample with timestamp that is too far in the future.
+This alert fires when one or more Mimir ingesters accepts a sample with timestamp that is too far in the future.
 This is typically a result of processing of corrupted message, and it can cause rejection of other samples with timestamp close to "now" (real-world time).
 
 How it **works**:
 
-- The metric exported by ingester computes maximum timestamp from all TSDBs open in ingester.
-- Alert checks this exported metric and fires if maximum timestamp is more than 1h in the future.
+- The metric exported by the ingester computes the maximum timestamp from all TSDBs open in the ingester.
+- The alert checks the metric and fires if the maximum timestamp is more than 1h in the future.
 
 How to **investigate**
 
-- Find the tenant with bad sample on ingester's tenants list, where a warning "TSDB Head max timestamp too far in the future" is displayed.
-- Flush tenant's data to blocks storage.
-- Remove tenant's directory on disk and restart ingester.
+- Find the tenant with a bad sample on an affected ingester's tenants list (obtained via the `/ingester/tenants` endpoint), where a warning "TSDB Head max timestamp too far in the future" is displayed.
+- Flush the tenant's data to blocks storage.
+- Remove the tenant's directory on disk and the restart ingester.
 
 ### MimirStoreGatewayTooManyFailedOperations
 

@@ -15,10 +15,6 @@ weight: 100
 
 # Send native histograms to Mimir
 
-{{% admonition type="note" %}}
-Native histograms are an experimental feature of Grafana Mimir.
-{{% /admonition %}}
-
 Prometheus native histograms is a data type in the Prometheus ecosystem that makes it possible to produce, store, and query a high-resolution [histogram](https://prometheus.io/docs/concepts/metric_types/#histogram) of observations.
 
 Native histograms are different from classic Prometheus histograms in a number of ways:
-Original file line number
+Diff line change
@@ Expand Up @@
        }
        ```
+       Note this must be added prior to any checks for `t.Cfg.RuntimeConfig.LoadPath` or early exit paths within `initRuntimeConfig`. It's possible to configure Mimir with purely static config and disable runtime config; the mapping needs to happen in either case.
     ## Updating Mimir Docs
 . Update the Mimir Config file docs.
@@ Expand Down @@