From 870fdf71f1e9efe76ef726024e75b9a502c62edc Mon Sep 17 00:00:00 2001 From: Carrie Edwards Date: Fri, 13 Mar 2026 13:34:42 -0700 Subject: [PATCH] Update documentation for st-storage feature Signed-off-by: Carrie Edwards --- cmd/prometheus/main.go | 5 ++--- docs/command-line/prometheus.md | 2 +- docs/feature_flags.md | 29 +++++++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go index 2901707709..35bd0f324f 100644 --- a/cmd/prometheus/main.go +++ b/cmd/prometheus/main.go @@ -282,7 +282,6 @@ func (c *flagConfig) setFeatureListOptions(logger *slog.Logger) error { config.DefaultGlobalConfig.ScrapeProtocols = config.DefaultProtoFirstScrapeProtocols logger.Info("Experimental start timestamp zero ingestion enabled. OpenMetrics 1.0 parsing will parse _created metrics as ST instead of normal sample. Changed default scrape_protocols to prefer PrometheusProto format.", "global.scrape_protocols", fmt.Sprintf("%v", config.DefaultGlobalConfig.ScrapeProtocols)) case "st-storage": - // TODO(bwplotka): Implement ST Storage as per PROM-60 and document this hidden feature flag. c.scrape.ParseST = true c.tsdb.EnableSTStorage = true c.agent.EnableSTStorage = true @@ -290,7 +289,7 @@ func (c *flagConfig) setFeatureListOptions(logger *slog.Logger) error { // Change relevant global variables. Hacky, but it's hard to pass a new option or default to unmarshallers. This is to widen the ST support surface. config.DefaultConfig.GlobalConfig.ScrapeProtocols = config.DefaultProtoFirstScrapeProtocols config.DefaultGlobalConfig.ScrapeProtocols = config.DefaultProtoFirstScrapeProtocols - logger.Info("Experimental start timestamp storage enabled. OpenMetrics 1.0 parsing will parse _created metrics as ST instead of normal sample. Changed default scrape_protocols to prefer PrometheusProto format.", "global.scrape_protocols", fmt.Sprintf("%v", config.DefaultGlobalConfig.ScrapeProtocols)) + logger.Info("Experimental start timestamp storage enabled. OpenMetrics 1.0 parsing will parse _created metrics as ST instead of normal sample. Changed default scrape_protocols to prefer PrometheusProto format. Changed chunk encoding type to XOR2.", "global.scrape_protocols", fmt.Sprintf("%v", config.DefaultGlobalConfig.ScrapeProtocols)) case "delayed-compaction": c.tsdb.EnableDelayedCompaction = true logger.Info("Experimental delayed compaction is enabled.") @@ -603,7 +602,7 @@ func main() { a.Flag("scrape.discovery-reload-interval", "Interval used by scrape manager to throttle target groups updates."). Hidden().Default("5s").SetValue(&cfg.scrape.DiscoveryReloadInterval) - a.Flag("enable-feature", "Comma separated feature names to enable. Valid options: exemplar-storage, expand-external-labels, memory-snapshot-on-shutdown, promql-per-step-stats, promql-experimental-functions, extra-scrape-metrics, auto-gomaxprocs, created-timestamp-zero-ingestion, concurrent-rule-eval, delayed-compaction, old-ui, otlp-deltatocumulative, promql-duration-expr, use-uncached-io, promql-extended-range-selectors, promql-binop-fill-modifiers. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details."). + a.Flag("enable-feature", "Comma separated feature names to enable. Valid options: exemplar-storage, expand-external-labels, memory-snapshot-on-shutdown, promql-per-step-stats, promql-experimental-functions, extra-scrape-metrics, auto-gomaxprocs, created-timestamp-zero-ingestion, st-storage, concurrent-rule-eval, delayed-compaction, old-ui, otlp-deltatocumulative, promql-duration-expr, use-uncached-io, promql-extended-range-selectors, promql-binop-fill-modifiers. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details."). Default("").StringsVar(&cfg.featureList) a.Flag("agent", "Run Prometheus in 'Agent mode'.").BoolVar(&agentMode) diff --git a/docs/command-line/prometheus.md b/docs/command-line/prometheus.md index 251fdfd6a4..481450b32f 100644 --- a/docs/command-line/prometheus.md +++ b/docs/command-line/prometheus.md @@ -59,7 +59,7 @@ The Prometheus monitoring server | --query.timeout | Maximum time a query may take before being aborted. Use with server mode only. | `2m` | | --query.max-concurrency | Maximum number of queries executed concurrently. Use with server mode only. | `20` | | --query.max-samples | Maximum number of samples a single query can load into memory. Note that queries will fail if they try to load more samples than this into memory, so this also limits the number of samples a query can return. Use with server mode only. | `50000000` | -| --enable-feature ... | Comma separated feature names to enable. Valid options: exemplar-storage, expand-external-labels, memory-snapshot-on-shutdown, promql-per-step-stats, promql-experimental-functions, extra-scrape-metrics, auto-gomaxprocs, created-timestamp-zero-ingestion, concurrent-rule-eval, delayed-compaction, old-ui, otlp-deltatocumulative, promql-duration-expr, use-uncached-io, promql-extended-range-selectors, promql-binop-fill-modifiers. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details. | | +| --enable-feature ... | Comma separated feature names to enable. Valid options: exemplar-storage, expand-external-labels, memory-snapshot-on-shutdown, promql-per-step-stats, promql-experimental-functions, extra-scrape-metrics, auto-gomaxprocs, created-timestamp-zero-ingestion, st-storage, concurrent-rule-eval, delayed-compaction, old-ui, otlp-deltatocumulative, promql-duration-expr, use-uncached-io, promql-extended-range-selectors, promql-binop-fill-modifiers. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details. | | | --agent | Run Prometheus in 'Agent mode'. | | | --log.level | Only log messages with the given severity or above. One of: [debug, info, warn, error] | `info` | | --log.format | Output format of log messages. One of: [logfmt, json] | `logfmt` | diff --git a/docs/feature_flags.md b/docs/feature_flags.md index 45d14b72db..afd0c7b28c 100644 --- a/docs/feature_flags.md +++ b/docs/feature_flags.md @@ -77,6 +77,35 @@ Therefore, when `created-timestamp-zero-ingestion` is enabled Prometheus changes Besides enabling this feature in Prometheus, start timestamps need to be exposed by the application being scraped. +## Start timestamp (ST) native storage + +`--enable-feature=st-storage` + +> WARNING: This is a highly experimental and risky setting. +> * The new SamplesV2 WAL records cannot be replayed with Prometheus versions that do not support them. +> * This feature uses XOR2 encoded chunks, which cannot be read by older Prometheus versions that do not support the encoding. +> * XOR2 encoding is new, meaning downstream tools and LTS systems might now support it yet (e.g. Thanos sidecar uploaded blocks). + +> See [PROM-60](https://github.com/prometheus/proposals/pull/60) for the full +design proposal. + +Enables the storage of start timestamps (ST) natively per sample, instead of injecting synthetic 0 valued samples (as `created-timestamp-zero-ingestion` does). +Native storage of start timestamps preserves the exact ST values without adding extra samples. + +Currently, native start timestamp storage is only supported for float samples; support for histograms will be added in the future. +Additionally, start timestamp values are not yet used by the PromQL engine for queries. + +Currently, Prometheus supports start timestamps on: + +* `PrometheusProto` +* `OpenMetrics1.0.0` + +`PrometheusProto` is recommended. + +Besides enabling this feature in Prometheus, start timestamps need to be exposed by the application being scraped. + +Enabling this feature flag automatically enables the xor2-encoding flag. + ## Concurrent evaluation of independent rules `--enable-feature=concurrent-rule-eval`