From c023aad32574719078bc56c58b7b485c24ff5f71 Mon Sep 17 00:00:00 2001 From: Mehrdad Biukian Naeini Date: Sun, 16 Aug 2026 03:34:38 +0400 Subject: [PATCH 1/3] querier: fix active query tracker panic on UTF-8 continuation bytes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit trimStringByBytes scans backwards from the truncation point to find a UTF-8 rune start, but the loop had no lower bound. When a request field consists only of UTF-8 continuation bytes (0x80-0xBF) there is no rune start to land on, so size underflows past zero and bytesStr[-1] panics with 'index out of range [-1]'. Because the active query tracker is enabled by default and the panic happens on a goroutine without recover(), an attacker-supplied match[]/query value can crash the querier. Bound the scan with 'size > 0' so it stops at the start of the string instead of underflowing. When no rune start exists the field is truncated to an empty string, which is safe. Add TestTrimForJsonMarshalContinuationBytes covering the all-continuation- byte case; existing multi-byte tests use valid UTF-8 ('δΈ–') which always terminates the scan at index 0 and therefore never exercised the underflow. Fixes #7729 Signed-off-by: ... Signed-off-by: Mehrdad Biukian Naeini --- CHANGELOG.md | 1 + pkg/util/request_tracker/request_extractor.go | 5 ++++- .../request_tracker/request_tracker_test.go | 17 +++++++++++++++++ 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4fe16f3a8bd..b3129dad108 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Changelog ## master / unreleased +* [BUGFIX] Querier: Fix a panic in the active query tracker's `trimStringByBytes` when a request field consists only of UTF-8 continuation bytes (e.g. an unvalidated `match[]`/`query` value), which caused the rune-boundary scan to underflow and crash the querier. #7729 * [FEATURE] Engine: Add `-querier.selector-batch-size` and `-ruler.selector-batch-size` flags to configure series batching in the Thanos promQL engine. 0 disables batching. #7763 * [CHANGE] Querier: Make query time range configurations per-tenant: `query_ingesters_within`, `query_store_after`, and `shuffle_sharding_ingesters_lookback_period`. Uses `model.Duration` instead of `time.Duration` to support serialization but has minimum unit of 1ms (nanoseconds/microseconds not supported). #7160 * [CHANGE] Cache: Setting `-blocks-storage.bucket-store.metadata-cache.bucket-index-content-ttl` to 0 will disable the bucket-index cache. #7446 diff --git a/pkg/util/request_tracker/request_extractor.go b/pkg/util/request_tracker/request_extractor.go index cbd9f31e8fe..7675d026f8b 100644 --- a/pkg/util/request_tracker/request_extractor.go +++ b/pkg/util/request_tracker/request_extractor.go @@ -83,7 +83,10 @@ func trimStringByBytes(str string, size int) string { bytesStr := []byte(str) trimIndex := len(bytesStr) if size < len(bytesStr) { - for !utf8.RuneStart(bytesStr[size]) { + // Scan backwards to a rune boundary. Bound the scan at size > 0: if the + // string has no rune start (e.g. only UTF-8 continuation bytes) the loop + // must not underflow past zero, which would panic on bytesStr[-1]. + for size > 0 && !utf8.RuneStart(bytesStr[size]) { size-- } trimIndex = size diff --git a/pkg/util/request_tracker/request_tracker_test.go b/pkg/util/request_tracker/request_tracker_test.go index 13ef0a802cd..9842b33ec2f 100644 --- a/pkg/util/request_tracker/request_tracker_test.go +++ b/pkg/util/request_tracker/request_tracker_test.go @@ -191,3 +191,20 @@ func TestRangedQueryExtractorMultiByteTruncation(t *testing.T) { assert.True(t, utf8.Valid(entry), "entry should be valid UTF-8") }) } + +// TestTrimForJsonMarshalContinuationBytes reproduces cortexproject/cortex#7729: +// when the string consists only of UTF-8 continuation bytes (0x80-0xBF) there is +// no rune start to scan back to, so the backwards scan in trimStringByBytes +// underflows past zero and panics with index out of range [-1]. The fix bounds +// the scan at size > 0. +func TestTrimForJsonMarshalContinuationBytes(t *testing.T) { + // 1200 continuation bytes (0x80) with a truncation size below the length. + continuation := strings.Repeat("\x80", 1200) + + require.NotPanics(t, func() { + out := trimForJsonMarshal(continuation, 900) + // Result must always be valid UTF-8 (no partial runes). + assert.True(t, utf8.ValidString(out), "result should be valid UTF-8") + assert.LessOrEqual(t, len(out), len(continuation)) + }) +} From cf7f83c9cf6c36205fea9ed4f9e171a98059294a Mon Sep 17 00:00:00 2001 From: Mehrdad Biukian Naeini Date: Sun, 16 Aug 2026 08:10:15 +0400 Subject: [PATCH 2/3] ingester: fix ingestion delay histogram reset duration (fixes #7731) Signed-off-by: Mehrdad Biukian Naeini --- pkg/ingester/metrics.go | 23 +++++++++++++++-------- pkg/ingester/metrics_test.go | 12 ++++++++++++ 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/pkg/ingester/metrics.go b/pkg/ingester/metrics.go index 3ad21faad6d..54a573da1e6 100644 --- a/pkg/ingester/metrics.go +++ b/pkg/ingester/metrics.go @@ -88,6 +88,20 @@ type ingesterMetrics struct { unoptimizedRegexRejectedTotal *prometheus.CounterVec } +// ingestionDelaySecondsHistogramOpts defines the options for the +// cortex_ingester_ingestion_delay_seconds native histogram. NativeHistogramMinResetDuration +// must be a real duration (time.Hour): a bare integer literal is interpreted as +// nanoseconds, which resets the native histogram on essentially every scrape and +// discards the vast majority of observations (see cortexproject/cortex#7731). +var ingestionDelaySecondsHistogramOpts = prometheus.HistogramOpts{ + Name: "cortex_ingester_ingestion_delay_seconds", + Help: "Delay in seconds between sample ingestion time and sample timestamp.", + NativeHistogramBucketFactor: 1.1, + NativeHistogramMaxBucketNumber: 100, + NativeHistogramMinResetDuration: 1 * time.Hour, + Buckets: []float64{1, 5, 10, 30, 60, 120, 300, 600}, // 1s, 5s, 10s, 30s, 1m, 2m, 5m, 10m +} + func newIngesterMetrics(r prometheus.Registerer, createMetricsConflictingWithTSDB bool, activeSeriesEnabled bool, @@ -151,14 +165,7 @@ func newIngesterMetrics(r prometheus.Registerer, NativeHistogramMinResetDuration: 1 * time.Hour, Buckets: prometheus.ExponentialBuckets(1, 2, 10), // 1 to 512 buckets }, []string{"user"}), - ingestionDelaySeconds: promauto.With(r).NewHistogramVec(prometheus.HistogramOpts{ - Name: "cortex_ingester_ingestion_delay_seconds", - Help: "Delay in seconds between sample ingestion time and sample timestamp.", - NativeHistogramBucketFactor: 1.1, - NativeHistogramMaxBucketNumber: 100, - NativeHistogramMinResetDuration: 1, - Buckets: []float64{1, 5, 10, 30, 60, 120, 300, 600}, // 1s, 5s, 10s, 30s, 1m, 2m, 5m, 10m - }, []string{"user"}), + ingestionDelaySeconds: promauto.With(r).NewHistogramVec(ingestionDelaySecondsHistogramOpts, []string{"user"}), oooLabelsTotal: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ Name: "cortex_ingester_out_of_order_labels_total", Help: "The total number of out of order label found per user.", diff --git a/pkg/ingester/metrics_test.go b/pkg/ingester/metrics_test.go index af09d07e2a3..5a43e420b93 100644 --- a/pkg/ingester/metrics_test.go +++ b/pkg/ingester/metrics_test.go @@ -3,6 +3,7 @@ package ingester import ( "bytes" "testing" + "time" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -1298,3 +1299,14 @@ func populateTSDBMetrics(base float64) *prometheus.Registry { return r } + +// TestIngestionDelaySecondsHistogramResetDuration is a regression test for +// cortexproject/cortex#7731: ingestionDelaySeconds was registered with +// NativeHistogramMinResetDuration: 1, where 1 is interpreted as 1 nanosecond +// (the field is a time.Duration). That reset the native histogram on essentially +// every scrape and discarded ~86% of observations. The reset duration must be a +// real hour-long window. +func TestIngestionDelaySecondsHistogramResetDuration(t *testing.T) { + require.Equal(t, time.Hour, ingestionDelaySecondsHistogramOpts.NativeHistogramMinResetDuration, + "NativeHistogramMinResetDuration must be time.Hour, not a bare integer (interpreted as nanoseconds)") +} From b44e9bc73e1bcaffa5b1a21f577476cff7262cc3 Mon Sep 17 00:00:00 2001 From: Mehrdad Biukian Naeini Date: Sun, 16 Aug 2026 08:10:57 +0400 Subject: [PATCH 3/3] CHANGELOG: add entry for ingestion delay histogram fix (#7731) Signed-off-by: Mehrdad Biukian Naeini --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3129dad108..6359cc3ff0c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # Changelog ## master / unreleased -* [BUGFIX] Querier: Fix a panic in the active query tracker's `trimStringByBytes` when a request field consists only of UTF-8 continuation bytes (e.g. an unvalidated `match[]`/`query` value), which caused the rune-boundary scan to underflow and crash the querier. #7729 +* [BUGFIX] Ingester: Fix `cortex_ingester_ingestion_delay_seconds` native histogram losing ~86% of observations by setting `NativeHistogramMinResetDuration` to 1h instead of the bare integer `1` (interpreted as 1ns). #7731 * [FEATURE] Engine: Add `-querier.selector-batch-size` and `-ruler.selector-batch-size` flags to configure series batching in the Thanos promQL engine. 0 disables batching. #7763 * [CHANGE] Querier: Make query time range configurations per-tenant: `query_ingesters_within`, `query_store_after`, and `shuffle_sharding_ingesters_lookback_period`. Uses `model.Duration` instead of `time.Duration` to support serialization but has minimum unit of 1ms (nanoseconds/microseconds not supported). #7160 * [CHANGE] Cache: Setting `-blocks-storage.bucket-store.metadata-cache.bucket-index-content-ttl` to 0 will disable the bucket-index cache. #7446