OPSaid – Cortex

Docs: 配置结构

Sun, 18 Jun 2023 12:08:57 +0800

简要概述

主要展示 cortex 可配置结构。

查看所有模块：

$ ./cortex -modules
alertmanager
all
compactor
configs
distributor *
flusher
ingester *
overrides-exporter
purger *
querier *
query-frontend *
query-scheduler
ruler *
store-gateway *

Modules marked with * are included in target All.

配置示列

最小化

配置结构

target: all
auth_enabled: false
http_prefix: /api/prom

api:
 ...

# server.Config
server:
 ...

# distributor.Config
distributor:
 ...

# querier.Config
querier:
 ...

ingester_client:
 ...

ingester:
 ...

flusher:
 ...

# 数据存储
storage:
 ...

# 资源限制
limits:
 ...

prealloc:
 ...

frontend_worker:
 ...

# Query Frontend
frontend:
 ...

# Query Frontend
query_range:
 ...

# 数据存储
# tsdb.BlocksStorageConfig
blocks_storage:
 ...

# Compactor
# compactor.Config
compactor:
 ...

# storegateway.Config
store_gateway:
 ...

tenant_federation:
 ...

# Ruler
# ruler.Config
ruler:
 ...

# Ruler
# rulestore.Config
ruler_storage:
 ...

configs:
 ...

# Alertmanager
# alertmanager.MultitenantAlertmanagerConfig
alertmanager:
 ...

# Alertmanager
# alertstore.Config
alertmanager_storage:
 ...

runtime_config:
 ...

memberlist:
 ...

# scheduler.Config
query_scheduler:
 ...

tracing:
 ...

数据结构

Config

github.com/cortexproject/cortex/pkg/cortex/cortex.go

// Config is the root config for Cortex.
type Config struct {
 Target flagext.StringSliceCSV `yaml:"target"`
 AuthEnabled bool `yaml:"auth_enabled"`
 PrintConfig bool `yaml:"-"`
 HTTPPrefix string `yaml:"http_prefix"`

 ExternalQueryable prom_storage.Queryable `yaml:"-"`
 ExternalPusher ruler.Pusher `yaml:"-"`

 API api.Config `yaml:"api"`
 Server server.Config `yaml:"server"`
 Distributor distributor.Config `yaml:"distributor"`
 Querier querier.Config `yaml:"querier"`
 IngesterClient client.Config `yaml:"ingester_client"`
 Ingester ingester.Config `yaml:"ingester"`
 Flusher flusher.Config `yaml:"flusher"`
 Storage storage.Config `yaml:"storage"`
 LimitsConfig validation.Limits `yaml:"limits"`
 Prealloc cortexpb.PreallocConfig `yaml:"prealloc" doc:"hidden"`
 Worker querier_worker.Config `yaml:"frontend_worker"`
 Frontend frontend.CombinedFrontendConfig `yaml:"frontend"`
 QueryRange queryrange.Config `yaml:"query_range"`
 BlocksStorage tsdb.BlocksStorageConfig `yaml:"blocks_storage"`
 Compactor compactor.Config `yaml:"compactor"`
 StoreGateway storegateway.Config `yaml:"store_gateway"`
 TenantFederation tenantfederation.Config `yaml:"tenant_federation"`

 Ruler ruler.Config `yaml:"ruler"`
 RulerStorage rulestore.Config `yaml:"ruler_storage"`
 Configs configs.Config `yaml:"configs"`
 Alertmanager alertmanager.MultitenantAlertmanagerConfig `yaml:"alertmanager"`
 AlertmanagerStorage alertstore.Config `yaml:"alertmanager_storage"`
 RuntimeConfig runtimeconfig.Config `yaml:"runtime_config"`
 MemberlistKV memberlist.KVConfig `yaml:"memberlist"`
 QueryScheduler scheduler.Config `yaml:"query_scheduler"`

 Tracing tracing.Config `yaml:"tracing"`
}

api.Config

github.com/cortexproject/cortex/pkg/api/api.go

type Config struct {
 ResponseCompression bool `yaml:"response_compression_enabled"`

 AlertmanagerHTTPPrefix string `yaml:"alertmanager_http_prefix"`
 PrometheusHTTPPrefix string `yaml:"prometheus_http_prefix"`

 // The following configs are injected by the upstream caller.
 ServerPrefix string `yaml:"-"`
 LegacyHTTPPrefix string `yaml:"-"`
 HTTPAuthMiddleware middleware.Interface `yaml:"-"`

 // This allows downstream projects to wrap the distributor push function
 // and access the deserialized write requests before/after they are pushed.
 DistributorPushWrapper DistributorPushWrapper `yaml:"-"`

 // The CustomConfigHandler allows for providing a different handler for the
 // `/config` endpoint. If this field is set _before_ the API module is
 // initialized, the custom config handler will be used instead of
 // DefaultConfigHandler.
 CustomConfigHandler ConfigHandler `yaml:"-"`

 // Allows and is used to configure the addition of HTTP Header fields to logs
 HTTPRequestHeadersToLog flagext.StringSlice `yaml:"http_request_headers_to_log"`
}

server.Config

github.com/weaveworks/common/server

见服务监听

distributor.Config

github.com/cortexproject/cortex/pkg/distributor/distributor.go

见Distributor

querier.Config

github.com/cortexproject/cortex/pkg/querier/querier.go

见Querier

client.Config

github.com/cortexproject/cortex/pkg/ingester/client/client.go

见Ingester

ingester.Config

github.com/cortexproject/cortex/pkg/ingester/ingester.go

见Ingester

flusher.Config

github.com/cortexproject/cortex/pkg/flusher/flusher.go

TODO;

storage.Config

TODO;

validation.Limits

github.com/cortexproject/cortex/pkg/util/validation/limits.go

见资源限制

cortexpb.PreallocConfig

TODO;

querier_worker.Config

github.com/cortexproject/cortex/pkg/querier/worker/worker.go

type Config struct {
 FrontendAddress string `yaml:"frontend_address"`
 SchedulerAddress string `yaml:"scheduler_address"`
 DNSLookupPeriod time.Duration `yaml:"dns_lookup_duration"`

 Parallelism int `yaml:"parallelism"`
 MatchMaxConcurrency bool `yaml:"match_max_concurrent"`
 MaxConcurrentRequests int `yaml:"-"` // Must be same as passed to PromQL Engine.

 QuerierID string `yaml:"id"`

 GRPCClientConfig grpcclient.Config `yaml:"grpc_client_config"`
}

frontend.CombinedFrontendConfig

github.com/cortexproject/cortex/pkg/frontend/config.go

见Frontend

queryrange.Config

github.com/cortexproject/cortex/pkg/querier/tripperware/queryrange/query_range_middlewares.go

// Config for query_range middleware chain.
type Config struct {
 SplitQueriesByInterval time.Duration `yaml:"split_queries_by_interval"`
 AlignQueriesWithStep bool `yaml:"align_queries_with_step"`
 ResultsCacheConfig `yaml:"results_cache"`
 CacheResults bool `yaml:"cache_results"`
 MaxRetries int `yaml:"max_retries"`
 // List of headers which query_range middleware chain would forward to downstream querier.
 ForwardHeaders flagext.StringSlice `yaml:"forward_headers_list"`

 // Populated based on the query configuration
 VerticalShardSize int `yaml:"-"`
}

tsdb.BlocksStorageConfig

github.com/cortexproject/cortex/pkg/storage/tsdb/config.go

见数据存储

compactor.Config

github.com/cortexproject/cortex/pkg/compactor/compactor.go

见Compactor

storegateway.Config

github.com/cortexproject/cortex/pkg/storegateway/

见Store Gateway

tenantfederation.Config

github.com/cortexproject/cortex/pkg/querier/tenantfederation/tenant_federation.go

type Config struct {
 // Enabled switches on support for multi tenant query federation
 Enabled bool `yaml:"enabled"`
}

ruler.Config

github.com/cortexproject/cortex/pkg/ruler/ruler.go

见Ruler

rulestore.Config

github.com/cortexproject/cortex/pkg/ruler/rulestore/config.go

见Ruler

configs.Config

github.com/cortexproject/cortex/pkg/configs/config.go

type Config struct {
 DB db.Config `yaml:"database"`
 API api.Config `yaml:"api"`
}

TODO;

alertmanager.MultitenantAlertmanagerConfig

github.com/cortexproject/cortex/pkg/alertmanager/multitenant.go

见Alertmanager

alertstore.Config

github.com/cortexproject/cortex/pkg/alertmanager/alertstore/config.go

见Alertmanager

runtimeconfig.Config

github.com/cortexproject/cortex/pkg/util/runtimeconfig/manager.go

// Config holds the config for an Manager instance.
// It holds config related to loading per-tenant config.
type Config struct {
 ReloadPeriod time.Duration `yaml:"period"`
 // LoadPath contains the path to the runtime config file, requires an
 // non-empty value
 LoadPath string `yaml:"file"`
 Loader Loader `yaml:"-"`
}

memberlist.KVConfig

github.com/cortexproject/cortex/pkg/ring/kv/memberlist/

见Distributor memberlist.KVConfig。

scheduler.Config

github.com/cortexproject/cortex/pkg/scheduler/scheduler.go

type Config struct {
 MaxOutstandingPerTenant int `yaml:"max_outstanding_requests_per_tenant"`
 QuerierForgetDelay time.Duration `yaml:"querier_forget_delay"`
 GRPCClientConfig grpcclient.Config `yaml:"grpc_client_config" doc:"description=This configures the gRPC client used to report errors back to the query-frontend."`
}

tracing.Config

github.com/cortexproject/cortex/pkg/tracing/tracing.go

type Config struct {
 Type string `yaml:"type" json:"type"`
 Otel Otel `yaml:"otel" json:"otel"`
}

type Otel struct {
 OltpEndpoint string `yaml:"oltp_endpoint" json:"oltp_endpoint" doc:"hidden"`
 OtlpEndpoint string `yaml:"otlp_endpoint" json:"otlp_endpoint"`
 ExporterType string `yaml:"exporter_type" json:"exporter_type"`
 SampleRatio float64 `yaml:"sample_ratio" json:"sample_ratio"`
 TLSEnabled bool `yaml:"tls_enabled"`
 TLS tls.ClientConfig `yaml:"tls"`
 ExtraDetectors []resource.Detector `yaml:"-"`
}

Docs: 服务监听

Mon, 29 May 2023 14:12:40 +0800

简要概述

公共配置如 http、grpc 监听地址等，均使用 github.com/weaveworks/common/server/server.go 中定义的结构体。

配置示列

最小化

server:
 http_listen_port: 9009

默认值

server:
 http_listen_network: tcp
 http_listen_address: ""
 http_listen_port: 9009
 http_listen_conn_limit: 0
 grpc_listen_network: tcp
 grpc_listen_address: ""
 grpc_listen_port: 9095
 grpc_listen_conn_limit: 0
 http_tls_config:
 cert_file: ""
 key_file: ""
 client_auth_type: ""
 client_ca_file: ""
 grpc_tls_config:
 cert_file: ""
 key_file: ""
 client_auth_type: ""
 client_ca_file: ""
 register_instrumentation: true
 graceful_shutdown_timeout: 30s
 http_server_read_timeout: 30s
 http_server_write_timeout: 30s
 http_server_idle_timeout: 2m0s
 grpc_server_max_recv_msg_size: 104857600
 grpc_server_max_send_msg_size: 104857600
 grpc_server_max_concurrent_streams: 1000
 grpc_server_max_connection_idle: 2562047h47m16.854775807s
 grpc_server_max_connection_age: 2562047h47m16.854775807s
 grpc_server_max_connection_age_grace: 2562047h47m16.854775807s
 grpc_server_keepalive_time: 2h0m0s
 grpc_server_keepalive_timeout: 20s
 grpc_server_min_time_between_pings: 10s
 grpc_server_ping_without_stream_allowed: true
 log_format: logfmt
 log_level: info
 log_source_ips_enabled: false
 log_source_ips_header: ""
 log_source_ips_regex: ""
 log_request_at_info_level_enabled: false
 http_path_prefix: ""

参数解析

命令行参数

名称	默认值	说明

数据结构

Config

github.com/weaveworks/common/server/server.go

// Config for a Server
type Config struct {
 MetricsNamespace string `yaml:"-"`
 HTTPListenNetwork string `yaml:"http_listen_network"`
 HTTPListenAddress string `yaml:"http_listen_address"`
 HTTPListenPort int `yaml:"http_listen_port"`
 HTTPConnLimit int `yaml:"http_listen_conn_limit"`
 GRPCListenNetwork string `yaml:"grpc_listen_network"`
 GRPCListenAddress string `yaml:"grpc_listen_address"`
 GRPCListenPort int `yaml:"grpc_listen_port"`
 GRPCConnLimit int `yaml:"grpc_listen_conn_limit"`

 HTTPTLSConfig TLSConfig `yaml:"http_tls_config"`
 GRPCTLSConfig TLSConfig `yaml:"grpc_tls_config"`

 RegisterInstrumentation bool `yaml:"register_instrumentation"`
 ExcludeRequestInLog bool `yaml:"-"`
 DisableRequestSuccessLog bool `yaml:"-"`

 ServerGracefulShutdownTimeout time.Duration `yaml:"graceful_shutdown_timeout"`
 HTTPServerReadTimeout time.Duration `yaml:"http_server_read_timeout"`
 HTTPServerWriteTimeout time.Duration `yaml:"http_server_write_timeout"`
 HTTPServerIdleTimeout time.Duration `yaml:"http_server_idle_timeout"`

 GRPCOptions []grpc.ServerOption `yaml:"-"`
 GRPCMiddleware []grpc.UnaryServerInterceptor `yaml:"-"`
 GRPCStreamMiddleware []grpc.StreamServerInterceptor `yaml:"-"`
 HTTPMiddleware []middleware.Interface `yaml:"-"`
 Router *mux.Router `yaml:"-"`
 DoNotAddDefaultHTTPMiddleware bool `yaml:"-"`

 GPRCServerMaxRecvMsgSize int `yaml:"grpc_server_max_recv_msg_size"`
 GRPCServerMaxSendMsgSize int `yaml:"grpc_server_max_send_msg_size"`
 GPRCServerMaxConcurrentStreams uint `yaml:"grpc_server_max_concurrent_streams"`
 GRPCServerMaxConnectionIdle time.Duration `yaml:"grpc_server_max_connection_idle"`
 GRPCServerMaxConnectionAge time.Duration `yaml:"grpc_server_max_connection_age"`
 GRPCServerMaxConnectionAgeGrace time.Duration `yaml:"grpc_server_max_connection_age_grace"`
 GRPCServerTime time.Duration `yaml:"grpc_server_keepalive_time"`
 GRPCServerTimeout time.Duration `yaml:"grpc_server_keepalive_timeout"`
 GRPCServerMinTimeBetweenPings time.Duration `yaml:"grpc_server_min_time_between_pings"`
 GRPCServerPingWithoutStreamAllowed bool `yaml:"grpc_server_ping_without_stream_allowed"`

 LogFormat logging.Format `yaml:"log_format"`
 LogLevel logging.Level `yaml:"log_level"`
 Log logging.Interface `yaml:"-"`
 LogSourceIPs bool `yaml:"log_source_ips_enabled"`
 LogSourceIPsHeader string `yaml:"log_source_ips_header"`
 LogSourceIPsRegex string `yaml:"log_source_ips_regex"`
 LogRequestAtInfoLevel bool `yaml:"log_request_at_info_level_enabled"`

 // If not set, default signal handler is used.
 SignalHandler SignalHandler `yaml:"-"`

 // If not set, default Prometheus registry is used.
 Registerer prometheus.Registerer `yaml:"-"`
 Gatherer prometheus.Gatherer `yaml:"-"`

 PathPrefix string `yaml:"http_path_prefix"`
}

logging.Format

github.com/weaveworks/common/logging/format.go

// Format is a settable identifier for the output format of logs
type Format struct {
 s string
 Logrus logrus.Formatter
}

// Set updates the value of the output format. Implements flag.Value
func (f *Format) Set(s string) error {
 switch s {
 case "logfmt":
 f.Logrus = &logrus.JSONFormatter{}
 case "json":
 f.Logrus = &logrus.JSONFormatter{}
 default:
 return errors.Errorf("unrecognized log format %q", s)
 }
 f.s = s
 return nil
}

logging.Level

github.com/weaveworks/common/logging/level.go

// Level is a settable identifier for the minimum level a log entry
// must be have.
type Level struct {
 s string
 Logrus logrus.Level
 Gokit level.Option
}

// Set updates the value of the allowed level. Implments flag.Value.
func (l *Level) Set(s string) error {
 switch s {
 case "debug":
 l.Logrus = logrus.DebugLevel
 l.Gokit = level.AllowDebug()
 case "info":
 l.Logrus = logrus.InfoLevel
 l.Gokit = level.AllowInfo()
 case "warn":
 l.Logrus = logrus.WarnLevel
 l.Gokit = level.AllowWarn()
 case "error":
 l.Logrus = logrus.ErrorLevel
 l.Gokit = level.AllowError()
 default:
 return errors.Errorf("unrecognized log level %q", s)
 }

 l.s = s
 return nil
}

Docs: 数据存储

Sun, 18 Jun 2023 12:51:46 +0800

简要概述

对 cortex 数据存储配置。

配置示列

最小化

blocks_storage:
 tsdb:
 dir: ./data/cortex/tsdb
 bucket_store:
 sync_dir: ./data/cortex/tsdb-sync
 backend: filesystem

默认值

storage:
 engine: blocks

blocks_storage:
 backend: filesystem
 s3:
 endpoint: ""
 region: ""
 bucket_name: ""
 secret_access_key: ""
 access_key_id: ""
 insecure: false
 signature_version: v4
 bucket_lookup_type: auto
 sse:
 type: ""
 kms_key_id: ""
 kms_encryption_context: ""
 http:
 idle_conn_timeout: 1m30s
 response_header_timeout: 2m0s
 insecure_skip_verify: false
 tls_handshake_timeout: 10s
 expect_continue_timeout: 1s
 max_idle_connections: 100
 max_idle_connections_per_host: 100
 max_connections_per_host: 0
 gcs:
 ...

 azure:
 ...

 swift:
 ...

 filesystem:
 dir: ./data/tsdb

 bucket_store:
 sync_dir: ./data/cortex/tsdb-sync
 sync_interval: 15m0s
 max_concurrent: 100
 tenant_sync_concurrency: 10
 block_sync_concurrency: 20
 meta_sync_concurrency: 20
 consistency_delay: 0s
 index_cache:
 backend: inmemory
 inmemory:
 max_size_bytes: 1073741824
 memcached:
 addresses: ""
 timeout: 100ms
 max_idle_connections: 16
 max_async_concurrency: 50
 max_async_buffer_size: 10000
 max_get_multi_concurrency: 100
 max_get_multi_batch_size: 0
 max_item_size: 1048576
 auto_discovery: false

 chunks_cache:
 backend: ""
 memcached:
 addresses: ""
 timeout: 100ms
 max_idle_connections: 16
 max_async_concurrency: 50
 max_async_buffer_size: 10000
 max_get_multi_concurrency: 100
 max_get_multi_batch_size: 0
 max_item_size: 1048576
 auto_discovery: false
 subrange_size: 16000
 max_get_range_requests: 3
 attributes_ttl: 168h0m0s
 subrange_ttl: 24h0m0s

 metadata_cache:
 backend: ""
 memcached:
 addresses: ""
 timeout: 100ms
 max_idle_connections: 16
 max_async_concurrency: 50
 max_async_buffer_size: 10000
 max_get_multi_concurrency: 100
 max_get_multi_batch_size: 0
 max_item_size: 1048576
 auto_discovery: false
 tenants_list_ttl: 15m0s
 tenant_blocks_list_ttl: 5m0s
 chunks_list_ttl: 24h0m0s
 metafile_exists_ttl: 2h0m0s
 metafile_doesnt_exist_ttl: 5m0s
 metafile_content_ttl: 24h0m0s
 metafile_max_size_bytes: 1048576
 metafile_attributes_ttl: 168h0m0s
 block_index_attributes_ttl: 168h0m0s
 bucket_index_content_ttl: 5m0s
 bucket_index_max_size_bytes: 1048576
 ignore_deletion_mark_delay: 6h0m0s
 bucket_index:
 enabled: false
 update_on_error_interval: 1m0s
 idle_timeout: 1h0m0s
 max_stale_period: 1h0m0s
 max_chunk_pool_bytes: 2147483648
 chunk_pool_min_bucket_size_bytes: 16000
 chunk_pool_max_bucket_size_bytes: 50000000
 index_header_lazy_loading_enabled: false
 index_header_lazy_loading_idle_timeout: 20m0s
 partitioner_max_gap_bytes: 524288
 postings_offsets_in_mem_sampling: 32

 tsdb:
 dir: ./data/cortex/tsdb
 block_ranges_period:
 - 2h0m0s
 retention_period: 6h0m0s
 ship_interval: 1m0s
 ship_concurrency: 10
 head_compaction_interval: 1m0s
 head_compaction_concurrency: 5
 head_compaction_idle_timeout: 1h0m0s
 head_chunks_write_buffer_size_bytes: 4194304
 stripe_size: 16384
 wal_compression_enabled: false
 wal_segment_size_bytes: 134217728
 flush_blocks_on_shutdown: false
 close_idle_tsdb_timeout: 0s
 max_tsdb_opening_concurrency_on_startup: 10
 max_exemplars: 0

数据结构

Config

github.com/cortexproject/cortex/pkg/storage/tsdb/config.go

// BlocksStorageConfig holds the config information for the blocks storage.
//
//nolint:revive
type BlocksStorageConfig struct {
 Bucket bucket.Config `yaml:",inline"`
 BucketStore BucketStoreConfig `yaml:"bucket_store" doc:"description=This configures how the querier and store-gateway discover and synchronize blocks stored in the bucket."`
 TSDB TSDBConfig `yaml:"tsdb"`
}

BucketStoreConfig

// BucketStoreConfig holds the config information for Bucket Stores used by the querier and store-gateway.
type BucketStoreConfig struct {
 SyncDir string `yaml:"sync_dir"`
 SyncInterval time.Duration `yaml:"sync_interval"`
 MaxConcurrent int `yaml:"max_concurrent"`
 TenantSyncConcurrency int `yaml:"tenant_sync_concurrency"`
 BlockSyncConcurrency int `yaml:"block_sync_concurrency"`
 MetaSyncConcurrency int `yaml:"meta_sync_concurrency"`
 ConsistencyDelay time.Duration `yaml:"consistency_delay"`
 IndexCache IndexCacheConfig `yaml:"index_cache"`
 ChunksCache ChunksCacheConfig `yaml:"chunks_cache"`
 MetadataCache MetadataCacheConfig `yaml:"metadata_cache"`
 IgnoreDeletionMarksDelay time.Duration `yaml:"ignore_deletion_mark_delay"`
 IgnoreBlocksWithin time.Duration `yaml:"ignore_blocks_within"`
 BucketIndex BucketIndexConfig `yaml:"bucket_index"`

 // Chunk pool.
 MaxChunkPoolBytes uint64 `yaml:"max_chunk_pool_bytes"`
 ChunkPoolMinBucketSizeBytes int `yaml:"chunk_pool_min_bucket_size_bytes" doc:"hidden"`
 ChunkPoolMaxBucketSizeBytes int `yaml:"chunk_pool_max_bucket_size_bytes" doc:"hidden"`

 // Controls whether index-header lazy loading is enabled.
 IndexHeaderLazyLoadingEnabled bool `yaml:"index_header_lazy_loading_enabled"`
 IndexHeaderLazyLoadingIdleTimeout time.Duration `yaml:"index_header_lazy_loading_idle_timeout"`

 // Controls the partitioner, used to aggregate multiple GET object API requests.
 // The config option is hidden until experimental.
 PartitionerMaxGapBytes uint64 `yaml:"partitioner_max_gap_bytes" doc:"hidden"`

 // Controls what is the ratio of postings offsets store will hold in memory.
 // Larger value will keep less offsets, which will increase CPU cycles needed for query touching those postings.
 // It's meant for setups that want low baseline memory pressure and where less traffic is expected.
 // On the contrary, smaller value will increase baseline memory usage, but improve latency slightly.
 // 1 will keep all in memory. Default value is the same as in Prometheus which gives a good balance.
 PostingOffsetsInMemSampling int `yaml:"postings_offsets_in_mem_sampling" doc:"hidden"`
}

TSDBConfig

// TSDBConfig holds the config for TSDB opened in the ingesters.
//
//nolint:revive
type TSDBConfig struct {
 Dir string `yaml:"dir"`
 BlockRanges DurationList `yaml:"block_ranges_period"`
 Retention time.Duration `yaml:"retention_period"`
 ShipInterval time.Duration `yaml:"ship_interval"`
 ShipConcurrency int `yaml:"ship_concurrency"`
 HeadCompactionInterval time.Duration `yaml:"head_compaction_interval"`
 HeadCompactionConcurrency int `yaml:"head_compaction_concurrency"`
 HeadCompactionIdleTimeout time.Duration `yaml:"head_compaction_idle_timeout"`
 HeadChunksWriteBufferSize int `yaml:"head_chunks_write_buffer_size_bytes"`
 StripeSize int `yaml:"stripe_size"`
 WALCompressionEnabled bool `yaml:"wal_compression_enabled"`
 WALSegmentSizeBytes int `yaml:"wal_segment_size_bytes"`
 FlushBlocksOnShutdown bool `yaml:"flush_blocks_on_shutdown"`
 CloseIdleTSDBTimeout time.Duration `yaml:"close_idle_tsdb_timeout"`
 // The size of the in-memory queue used before flushing chunks to the disk.
 HeadChunksWriteQueueSize int `yaml:"head_chunks_write_queue_size"`

 // MaxTSDBOpeningConcurrencyOnStartup limits the number of concurrently opening TSDB's during startup.
 MaxTSDBOpeningConcurrencyOnStartup int `yaml:"max_tsdb_opening_concurrency_on_startup"`

 // If true, user TSDBs are not closed on shutdown. Only for testing.
 // If false (default), user TSDBs are closed to make sure all resources are released and closed properly.
 KeepUserTSDBOpenOnShutdown bool `yaml:"-"`

 // How often to check for idle TSDBs for closing. DefaultCloseIdleTSDBInterval is not suitable for testing, so tests can override.
 CloseIdleTSDBInterval time.Duration `yaml:"-"`

 // Positive value enables experimental support for exemplars. 0 or less to disable.
 MaxExemplars int `yaml:"max_exemplars"`

 // Enable snapshotting of in-memory TSDB data on disk when shutting down.
 MemorySnapshotOnShutdown bool `yaml:"memory_snapshot_on_shutdown"`

 // OutOfOrderCapMax is maximum capacity for OOO chunks (in samples).
 OutOfOrderCapMax int64 `yaml:"out_of_order_cap_max"`
}

bucket.Config

github.com/cortexproject/cortex/pkg/storage/bucket/client.go

// Config holds configuration for accessing long-term storage.
type Config struct {
 Backend string `yaml:"backend"`
 // Backends
 S3 s3.Config `yaml:"s3"`
 GCS gcs.Config `yaml:"gcs"`
 Azure azure.Config `yaml:"azure"`
 Swift swift.Config `yaml:"swift"`
 Filesystem filesystem.Config `yaml:"filesystem"`

 // Not used internally, meant to allow callers to wrap Buckets
 // created using this config
 Middlewares []func(objstore.Bucket) (objstore.Bucket, error) `yaml:"-"`

 // Used to inject additional backends into the config. Allows for this config to
 // be embedded in multiple contexts and support non-object storage based backends.
 ExtraBackends []string `yaml:"-"`
}

s3.Config

github.com/cortexproject/cortex/pkg/storage/bucket/s3/config.go

// Config holds the config options for an S3 backend
type Config struct {
 Endpoint string `yaml:"endpoint"`
 Region string `yaml:"region"`
 BucketName string `yaml:"bucket_name"`
 SecretAccessKey flagext.Secret `yaml:"secret_access_key"`
 AccessKeyID string `yaml:"access_key_id"`
 Insecure bool `yaml:"insecure"`
 SignatureVersion string `yaml:"signature_version"`
 BucketLookupType string `yaml:"bucket_lookup_type"`

 SSE SSEConfig `yaml:"sse"`
 HTTP HTTPConfig `yaml:"http"`
}

gcs.Config

TODO;

azure.Config

TODO;

swift.Config

TODO;

filesystem.Config

github.com/cortexproject/cortex/pkg/storage/bucket/filesystem

// Config stores the configuration for storing and accessing objects in the local filesystem.
type Config struct {
 Directory string `yaml:"dir"`
}

SSEConfig

// SSEConfig configures S3 server side encryption
// struct that is going to receive user input (through config file or CLI)
type SSEConfig struct {
 Type string `yaml:"type"`
 KMSKeyID string `yaml:"kms_key_id"`
 KMSEncryptionContext string `yaml:"kms_encryption_context"`
}

const (
 SignatureVersionV4 = "v4"
 SignatureVersionV2 = "v2"

 // SSEKMS config type constant to configure S3 server side encryption using KMS
 // https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html
 SSEKMS = "SSE-KMS"

 // SSES3 config type constant to configure S3 server side encryption with AES-256
 // https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html
 SSES3 = "SSE-S3"

 BucketAutoLookup = "auto"
 BucketVirtualHostLookup = "virtual-hosted"
 BucketPathLookup = "path"
)

IndexCacheConfig

type IndexCacheConfig struct {
 Backend string `yaml:"backend"`
 InMemory InMemoryIndexCacheConfig `yaml:"inmemory"`
 Memcached MemcachedIndexCacheConfig `yaml:"memcached"`
 Redis RedisIndexCacheConfig `yaml:"redis"`
 MultiLevel MultiLevelIndexCacheConfig `yaml:"multilevel"`
}

RedisIndexCacheConfig

type RedisIndexCacheConfig struct {
 ClientConfig RedisClientConfig `yaml:",inline"`
 EnabledItems []string `yaml:"enabled_items"`
}

RedisClientConfig

type RedisClientConfig struct {
 Addresses string `yaml:"addresses"`
 Username string `yaml:"username"`
 Password string `yaml:"password"`
 DB int `yaml:"db"`
 MasterName string `yaml:"master_name"`

 MaxGetMultiConcurrency int `yaml:"max_get_multi_concurrency"`
 GetMultiBatchSize int `yaml:"get_multi_batch_size"`
 MaxSetMultiConcurrency int `yaml:"max_set_multi_concurrency"`
 SetMultiBatchSize int `yaml:"set_multi_batch_size"`
 MaxAsyncConcurrency int `yaml:"max_async_concurrency"`
 MaxAsyncBufferSize int `yaml:"max_async_buffer_size"`

 DialTimeout time.Duration `yaml:"dial_timeout"`
 ReadTimeout time.Duration `yaml:"read_timeout"`
 WriteTimeout time.Duration `yaml:"write_timeout"`

 TLSEnabled bool `yaml:"tls_enabled"`
 TLS tls.ClientConfig `yaml:",inline"`

 // If not zero then client-side caching is enabled.
 // Client-side caching is when data is stored in memory
 // instead of fetching data each time.
 // See https://redis.io/docs/manual/client-side-caching/ for info.
 CacheSize int `yaml:"cache_size"`

 // SetAsyncCircuitBreaker configures the circuit breaker for SetAsync operations.
 SetAsyncCircuitBreaker CircuitBreakerConfig `yaml:"set_async_circuit_breaker_config"`
}

Docs: Distributor

Tue, 30 May 2023 10:11:51 +0800

简要概述

它是性能指标写入路径的第一站。一旦接收到来自 Prometheus 的样本，每个样本都会被验证其正确性，并确保它在配置的租户限制范围内，如果特定租户没有覆盖限制，则回退到默认限制。通过验证的样本然后被分割成批次，并行地发送到多个 Ingester 服务。

分发器进行的验证包括：

指标标签名称的形式正确性。
符合配置的每个指标标签的最大数量。
符合配置的标签名称和值的最大长度。
时间戳不早于/晚于配置的最小/最大时间范围。

组件 Distributor 服务负责处理来自 Prometheus 的样本，它是无状态服务，可弹性扩缩容。

多副本跟踪

见配置项 HATrackerConfig。

比如当一组 k8s 部署了 2 个 prometheus 同时采集数据，此时上报至 cortex 时存在两份相同数据。可通过启用该功能，则会对来自客户端样本数据进行去重。

HA 跟踪器根据集群和副本标签对传入的样本进行去重。集群标签在给定租户中唯一标识冗余的Prometheus服务器集群，而副本标签则唯一标识Prometheus集群内的副本。如果任何副本（不是当前集群中的主副本）接收到重复的传入样本，则认为这些样本是重复的（因此会被丢弃）。

HA跟踪器需要一个键值存储（KV存储）来协调当前选举的副本是哪个。分发器只会接受当前领导者的样本。默认情况下，只有一个或没有副本和集群标签的样本将被接受，并且永远不会被去重处理。

支持的建值存储有：

consul
etcd

这里均以 etcd 为标准。

一致性哈希

使用一致性哈希算法结合配置的复制因子来确定把样本数据发送至哪个 Ingester 实例。

支持两种哈希策略：

根据指标名称与租户ID
根据指标名、标签名、租户ID

默认是第一种策略，通过配置 -distributor.shard-by-all-labels=true 使用第二种策略。

第二种策略在优势在指标写入多个 Ingester 实例之间更加平衡，但每个查询需要与所有 Ingester 实例通信，因为一个指标可能在多个实例之间分布，根据不同的标签集。

哈希环

通过哈希环实现对样本数据在 Ingester 之间的一致性哈希分片和复制，所有 Ingester 实例都会使用一组自己拥有的令牌将自己注册到哈希环中，每个令牌都是一个随机的无符号32位数字，每个传入的样本数据在 Distributor 实例中进行哈希计算，然后将其推送到拥有该系列样本哈希数字范围及哈希环中后续N-1个 Ingester 的拥有者，其中 N 为配置的复制因子（replication_factor）。

哈希环存储在健值存储中，支持的有：

consul
etcd
Gossip memberlist

这里均以 etcd 为标准。

一致性模型

一致性法定人数（Quorum consistency）是分布式系统中一种确保读写操作满足特定法定人数要求的一致性模型。

在这个模型中，法定人数指的是在特定操作上需要达成一致的最小节点或副本数。

由于所有分发器共享对同一哈希环的访问，写请求可以发送到任何一个分发器，并且您可以在其前面设置一个无状态负载均衡器。

为了确保一致的查询结果，Cortex在读写操作上使用了Dynamo风格的法定人数一致性。这意味着在成功响应Prometheus写入请求之前，分发器将等待至少一半加一的摄取器给出积极响应，并将样本发送给它们。

负载均衡

建议对分发器实例进行随机负载均衡，以平衡写入请求。例如，如果您在Kubernetes集群中运行Cortex，可以将分发器作为一个Kubernetes服务来运行。

通过将分发器配置为Kubernetes服务，Kubernetes负责将写入请求随机地分发给可用的分发器实例，这样可以确保负载在不同的实例之间均衡分布，提高系统的可伸缩性和容错性。

配置Kubernetes服务时，可以使用负载均衡策略，例如使用随机算法来分配请求，以实现随机负载均衡。这样可以确保每个分发器实例都能够处理适当的写入请求，从而提高整体性能和可靠性。

配置示例

target: distributor

#auth_enabled: false

distributor:
 pool:
 client_cleanup_period: 15s
 health_check_ingesters: true
 ha_tracker:
 enable_ha_tracker: false
 ha_tracker_update_timeout: 15s
 ha_tracker_update_timeout_jitter_max: 5s
 ha_tracker_failover_timeout: 30s
 kvstore:
 store: etcd
 prefix: ha-tracker/
 etcd:
 endpoints: []
 dial_timeout: 10s
 max_retries: 10
 tls_enabled: false
 tls_cert_path: ""
 tls_key_path: ""
 tls_ca_path: ""
 tls_server_name: ""
 tls_insecure_skip_verify: false
 username: ""
 password: ""
 max_recv_msg_size: 104857600
 remote_timeout: 2s
 extra_queue_delay: 0s
 sharding_strategy: default
 shard_by_all_labels: true
 extend_writes: true
 ring:
 kvstore:
 store: etcd
 prefix: collectors/
 etcd:
 endpoints: []
 dial_timeout: 10s
 max_retries: 10
 tls_enabled: false
 tls_cert_path: ""
 tls_key_path: ""
 tls_ca_path: ""
 tls_server_name: ""
 tls_insecure_skip_verify: false
 username: ""
 password: ""
 heartbeat_period: 5s
 heartbeat_timeout: 1m0s
 instance_id: limiqi-mac2016-01.local
 instance_interface_names:
 - eth0
 - en0
 instance_port: 0
 instance_addr: ""
 instance_limits:
 max_ingestion_rate: 0
 max_inflight_push_requests: 0

应用接口

Remote Write

POST /api/v1/push

X-Prometheus-Remote-Write-Version: 0.1.0

<snappy <protoc buffer> >

用于在 prometheus 中配置 remote write 上传数据。

消息体使用 Protocol Buffer 序列化且使用 Snappy 压缩。

message WriteRequest {
repeated TimeSeries timeseries = 1 [(gogoproto.nullable) = false, (gogoproto.customtype) = "PreallocTimeseries"];
enum SourceEnum {
API = 0;
RULE = 1;
}
SourceEnum Source = 2;
repeated MetricMetadata metadata = 3 [(gogoproto.nullable) = true];
bool skip_label_name_validation = 1000; //set intentionally high to keep WriteRequest compatible with upstream Prometheus
}

Distributor ring status

GET /distributor/ring

Tenants stats

GET /distributor/all_user_stats

HA tracker status

GET /distributor/ha_tracker

数据结构

github.com/cortexproject/cortex/pkg/distributor/distributor.go

Config

// Config contains the configuration required to
// create a Distributor
type Config struct {
 PoolConfig PoolConfig `yaml:"pool"`

 HATrackerConfig HATrackerConfig `yaml:"ha_tracker"`

 MaxRecvMsgSize int `yaml:"max_recv_msg_size"`
 RemoteTimeout time.Duration `yaml:"remote_timeout"`
 ExtraQueryDelay time.Duration `yaml:"extra_queue_delay"`

 ShardingStrategy string `yaml:"sharding_strategy"`
 ShardByAllLabels bool `yaml:"shard_by_all_labels"`
 ExtendWrites bool `yaml:"extend_writes"`

 // Distributors ring
 DistributorRing RingConfig `yaml:"ring"`

 // for testing and for extending the ingester by adding calls to the client
 IngesterClientFactory ring_client.PoolFactory `yaml:"-"`

 // when true the distributor does not validate the label name, Cortex doesn't directly use
 // this (and should never use it) but this feature is used by other projects built on top of it
 SkipLabelNameValidation bool `yaml:"-"`

 // This config is dynamically injected because defined in the querier config.
 ShuffleShardingLookbackPeriod time.Duration `yaml:"-"`

 // Limits for distributor
 InstanceLimits InstanceLimits `yaml:"instance_limits"`
}

PoolConfig

// PoolConfig is config for creating a Pool.
type PoolConfig struct {
 ClientCleanupPeriod time.Duration `yaml:"client_cleanup_period"`
 HealthCheckIngesters bool `yaml:"health_check_ingesters"`
 RemoteTimeout time.Duration `yaml:"-"`
}

HATrackerConfig

// HATrackerConfig contains the configuration require to
// create a HA Tracker.
type HATrackerConfig struct {
 EnableHATracker bool `yaml:"enable_ha_tracker"`
 // We should only update the timestamp if the difference
 // between the stored timestamp and the time we received a sample at
 // is more than this duration.
 UpdateTimeout time.Duration `yaml:"ha_tracker_update_timeout"`
 UpdateTimeoutJitterMax time.Duration `yaml:"ha_tracker_update_timeout_jitter_max"`
 // We should only failover to accepting samples from a replica
 // other than the replica written in the KVStore if the difference
 // between the stored timestamp and the time we received a sample is
 // more than this duration
 FailoverTimeout time.Duration `yaml:"ha_tracker_failover_timeout"`

 KVStore kv.Config `yaml:"kvstore" doc:"description=Backend storage to use for the ring. Please be aware that memberlist is not supported by the HA tracker since gossip propagation is too slow for HA purposes."`
}

kv.Config

github.com/cortexproject/cortex/pkg/ring/kv/client.go

// Config is config for a KVStore currently used by ring and HA tracker,
// where store can be consul or inmemory.
type Config struct {
 Store string `yaml:"store"`
 Prefix string `yaml:"prefix"`
 StoreConfig `yaml:",inline"`

 Mock Client `yaml:"-"`
}

kv.StoreConfig

// StoreConfig is a configuration used for building single store client, either
// Consul, Etcd, Memberlist or MultiClient. It was extracted from Config to keep
// single-client config separate from final client-config (with all the wrappers)
type StoreConfig struct {
 DynamoDB dynamodb.Config `yaml:"dynamodb"`
 Consul consul.Config `yaml:"consul"`
 Etcd etcd.Config `yaml:"etcd"`
 Multi MultiConfig `yaml:"multi"`

 // Function that returns memberlist.KV store to use. By using a function, we can delay
 // initialization of memberlist.KV until it is actually required.
 MemberlistKV func() (*memberlist.KV, error) `yaml:"-"`
}

etcd.Config

github.com/cortexproject/cortex/pkg/ring/kv/etcd/etcd.go

// Config for a new etcd.Client.
type Config struct {
 Endpoints []string `yaml:"endpoints"`
 DialTimeout time.Duration `yaml:"dial_timeout"`
 MaxRetries int `yaml:"max_retries"`
 EnableTLS bool `yaml:"tls_enabled"`
 TLS cortextls.ClientConfig `yaml:",inline"`

 UserName string `yaml:"username"`
 Password string `yaml:"password"`
}

RingConfig

// RingConfig masks the ring lifecycler config which contains
// many options not really required by the distributors ring. This config
// is used to strip down the config to the minimum, and avoid confusion
// to the user.
type RingConfig struct {
 KVStore kv.Config `yaml:"kvstore"`
 HeartbeatPeriod time.Duration `yaml:"heartbeat_period"`
 HeartbeatTimeout time.Duration `yaml:"heartbeat_timeout"`

 // Instance details
 InstanceID string `yaml:"instance_id" doc:"hidden"`
 InstanceInterfaceNames []string `yaml:"instance_interface_names"`
 InstancePort int `yaml:"instance_port" doc:"hidden"`
 InstanceAddr string `yaml:"instance_addr" doc:"hidden"`

 // Injected internally
 ListenPort int `yaml:"-"`
}

InstanceLimits

type InstanceLimits struct {
 MaxIngestionRate float64 `yaml:"max_ingestion_rate"`
 MaxInflightPushRequests int `yaml:"max_inflight_push_requests"`
}

memberlist.KVConfig

github.com/cortexproject/cortex/pkg/ring/kv/memberlist/memberlist_client.go

// KVConfig is a config for memberlist.KV
type KVConfig struct {
 // Memberlist options.
 NodeName string `yaml:"node_name"`
 RandomizeNodeName bool `yaml:"randomize_node_name"`
 StreamTimeout time.Duration `yaml:"stream_timeout"`
 RetransmitMult int `yaml:"retransmit_factor"`
 PushPullInterval time.Duration `yaml:"pull_push_interval"`
 GossipInterval time.Duration `yaml:"gossip_interval"`
 GossipNodes int `yaml:"gossip_nodes"`
 GossipToTheDeadTime time.Duration `yaml:"gossip_to_dead_nodes_time"`
 DeadNodeReclaimTime time.Duration `yaml:"dead_node_reclaim_time"`
 EnableCompression bool `yaml:"compression_enabled"`

 // ip:port to advertise other cluster members. Used for NAT traversal
 AdvertiseAddr string `yaml:"advertise_addr"`
 AdvertisePort int `yaml:"advertise_port"`

 // List of members to join
 JoinMembers flagext.StringSlice `yaml:"join_members"`
 MinJoinBackoff time.Duration `yaml:"min_join_backoff"`
 MaxJoinBackoff time.Duration `yaml:"max_join_backoff"`
 MaxJoinRetries int `yaml:"max_join_retries"`
 AbortIfJoinFails bool `yaml:"abort_if_cluster_join_fails"`
 RejoinInterval time.Duration `yaml:"rejoin_interval"`

 // Remove LEFT ingesters from ring after this timeout.
 LeftIngestersTimeout time.Duration `yaml:"left_ingesters_timeout"`

 // Timeout used when leaving the memberlist cluster.
 LeaveTimeout time.Duration `yaml:"leave_timeout"`

 // How much space to use to keep received and sent messages in memory (for troubleshooting).
 MessageHistoryBufferBytes int `yaml:"message_history_buffer_bytes"`

 TCPTransport TCPTransportConfig `yaml:",inline"`

 // Where to put custom metrics. Metrics are not registered, if this is nil.
 MetricsRegisterer prometheus.Registerer `yaml:"-"`
 MetricsNamespace string `yaml:"-"`

 // Codecs to register. Codecs need to be registered before joining other members.
 Codecs []codec.Codec `yaml:"-"`
}

Docs: Querier

Tue, 30 May 2023 14:27:52 +0800

简要概述

组件 Querier 使用 PromQL 语法查询处理操作。

查询服务从 Ingester 和 Store-Gateway 中获取时间序列样本数据，Ingester 保存了尚未刷新到后端存储的内存中的时间序列样本，由于副本因子的存在，查询服务有可能接收到重复的样本数据，为了解决这个问题，在处理同一时间序列时，查询服务会对具有完全相同时间戳的样本进行内部去重操作。

查询服务是无状态的，并且可以根据需要进行扩展和缩减规模。但是这个注意，如果未开启 “bucket index” 功能，则每个副本启用后不会立刻加入工作。

如何工作

为了能在查询数据的时候找到正确的数据块，组件 Querier 需要知道最新的数据块在对象列表中，通过以下两个方案实现：

定期扫描对象列表（默认行为）；
定期下载对象索引（bucket index 需要开启）；

定期扫描

在启动 Querier 组件时会从所有租户对象存储中的数据块遍历下载 meta.json 文件，在这个阶段组件是不可用的也就是 “/ready” 接口处于不可用，如果在 k8s 中编排则服务是处理非健康状态，不会加入到 Service 中不接收流量。

在运行时，查询器会定期迭代存储桶，以发现新的租户和最近上传的块。查询者不会从块中下载任何内容，除了一个包含块元数据的 meta.json 文件。

当客户端通过 PromQL 语句查询数据时，组件 Querier 根据已下载的元数据计算出需要哪些数据块，然后通过与组件 Store-Gateway 获取实际数据。

开启存储桶索引

开启 “bucket index” 之后，组件 Querier 会在收到给定租户的第一个查询时延迟下载 “bucket-index.json” 索引，将其缓存在内存中并定期更新。“bucket index” 包含租户的块列表和块删除标记，稍后在查询执行过程中使用这些标记来查找给定查询需要查询的块集。

使用 “bucket index” 主要有以下优势：

组件 Querier 在启动后无需等待即可使用；
降低对对象存储的 API 调用量。

查询请求的剖析

当组件 Querier 接收到一个 PromQL 语句时：

{
 "end": "1737950063.305",
 "query": "rate(node_cpu_seconds_total[1m])",
 "start": "1737946463.305",
 "step": "14"
}

将会分析 “start” 至 “end” 时间范围内包含至少一个样本的所有数据块，然后计算包含这些块的数据在哪些 Store-Gateway 实例上，并获取相应序列数据。

从存储网关收到的响应进行一致性检查，以确保所有预期的块都已被查询，如果没有，查询器将重试从不同存储网关的缺失块中获取样本（-store-gateway.sharding-ring.replomation > 1 时），如果所有重试后一致性检查都失败，则查询执行也会失败

如果 “start” 至 “end” 涵盖了持续时间 querier.query-ingesters-within 所配置的值内，则 Querier 组件也会发送请求至所有 Ingester 实例，以防部分数据没有发送至后端对象存储。

当所有需要的数据从 Ingester 与 Store-Gateway 获取到后，Querier 使用 PromQL 引擎计算结果并返回给用户。

如何发现 Store-Gateway 服务：

当开启 store-gateway.sharding-ring.* 时，使用相同的配置共享哈希环；
未开启 store-gateway.sharding-ring.* 时，需配置 querier.store-gateway-addresses 地址用 DNS 轮询。

元数据缓存

组件 Querier 支持 metadata cache 包含内容：

租户列表；
每租户数据库列表；
数据块元数据 meta.json 内容；
数据块标记删除 deletion-mark.json 内容；
租户数据块索引 bucket-index.json.gz 内容；

在通过 blocks-storage.bucket-store.* 开启配置。

配置示例

官方文档

querier:
 max_concurrent: 20
 timeout: 2m0s
 iterators: false
 batch_iterators: true
 ingester_streaming: true
 ingester_metadata_streaming: false
 max_samples: 50000000
 query_ingesters_within: 0s
 query_store_for_labels_enabled: false
 at_modifier_enabled: false
 per_step_stats_enabled: false
 query_store_after: 0s
 max_query_into_future: 10m0s
 default_evaluation_interval: 1m0s
 active_query_tracker_dir: ./active-query-tracker
 lookback_delta: 5m0s
 store_gateway_addresses: ""
 store_gateway_client:
 tls_enabled: false
 tls_cert_path: ""
 tls_key_path: ""
 tls_ca_path: ""
 tls_server_name: ""
 tls_insecure_skip_verify: false
 grpc_compression: ""
 shuffle_sharding_ingesters_lookback_period: 0s

数据结构

querier.Config

github.com/cortexproject/cortex/pkg/querier/querier.go

// Config contains the configuration require to create a querier
type Config struct {
 MaxConcurrent int `yaml:"max_concurrent"`
 Timeout time.Duration `yaml:"timeout"`
 Iterators bool `yaml:"iterators"`
 BatchIterators bool `yaml:"batch_iterators"`
 IngesterStreaming bool `yaml:"ingester_streaming"`
 IngesterMetadataStreaming bool `yaml:"ingester_metadata_streaming"`
 MaxSamples int `yaml:"max_samples"`
 QueryIngestersWithin time.Duration `yaml:"query_ingesters_within"`
 QueryStoreForLabels bool `yaml:"query_store_for_labels_enabled"`
 AtModifierEnabled bool `yaml:"at_modifier_enabled" doc:"hidden"`
 EnablePerStepStats bool `yaml:"per_step_stats_enabled"`

 // QueryStoreAfter the time after which queries should also be sent to the store and not just ingesters.
 QueryStoreAfter time.Duration `yaml:"query_store_after"`
 MaxQueryIntoFuture time.Duration `yaml:"max_query_into_future"`

 // The default evaluation interval for the promql engine.
 // Needs to be configured for subqueries to work as it is the default
 // step if not specified.
 DefaultEvaluationInterval time.Duration `yaml:"default_evaluation_interval"`

 // Directory for ActiveQueryTracker. If empty, ActiveQueryTracker will be disabled and MaxConcurrent will not be applied (!).
 // ActiveQueryTracker logs queries that were active during the last crash, but logs them on the next startup.
 // However, we need to use active query tracker, otherwise we cannot limit Max Concurrent queries in the PromQL
 // engine.
 ActiveQueryTrackerDir string `yaml:"active_query_tracker_dir"`
 // LookbackDelta determines the time since the last sample after which a time
 // series is considered stale.
 LookbackDelta time.Duration `yaml:"lookback_delta"`

 // Blocks storage only.
 StoreGatewayAddresses string `yaml:"store_gateway_addresses"`
 StoreGatewayClient ClientConfig `yaml:"store_gateway_client"`

 ShuffleShardingIngestersLookbackPeriod time.Duration `yaml:"shuffle_sharding_ingesters_lookback_period"`

 // Experimental. Use https://github.com/thanos-community/promql-engine rather than
 // the Prometheus query engine.
 ThanosEngine bool `yaml:"thanos_engine"`
}

ClientConfig

type ClientConfig struct {
 TLSEnabled bool `yaml:"tls_enabled"`
 TLS tls.ClientConfig `yaml:",inline"`
 GRPCCompression string `yaml:"grpc_compression"`
}

Docs: Ingester

Tue, 30 May 2023 11:24:49 +0800

简要概述

该组件负责将传入的样本数据写入存储后端（如对象存储或分布式文件系统等适合长期存储的后端），并在读取路径上返回内存中的系列样本供查询使用。

接收的样本数据并不会立即写入存储，而是保留在内存中，并定期刷新到存储（默认情况下，每隔2小时）。因此，在执行查询时，Querier 组件需要从 Ingester 和后端存储中获取样本。

生命周期

该组件使用生命周期管理器（lifecycler），将 Ingester 当前状态存储在哈希环中。

可以处于以下状态之一：

PENDING（待处理）：摄取器刚启动。在此状态下，摄取器既不接收写入请求也不接收读取请求。
JOINING（加入中）：摄取器正在启动并加入哈希环。在此状态下，摄取器既不接收写入请求也不接收读取请求。摄取器将使用从磁盘加载的令牌（如果配置了-ingester.tokens-file-path）加入环，或者生成一组新的随机令牌。最后，摄取器可以选择观察环以检测令牌冲突，一旦解决任何冲突，摄取器将转移到ACTIVE状态。
ACTIVE（活动状态）：摄取器正在运行。在此状态下，摄取器可以同时接收写入请求和读取请求。
LEAVING（离开中）：摄取器正在关闭并离开哈希环。在此状态下，摄取器不接收写入请求，但仍可接收读取请求。
UNHEALTHY（不健康状态）：摄取器无法向哈希环的KV存储发送心跳信号。在此状态下，分发器在构建传入系列的复制集时会跳过该摄取器，并且摄取器不接收写入或读取请求。

摄取器是半有状态的。

数据丢失

如果进程崩溃或意外退出，所有尚未刷新到长期存储的内存样本数据将会丢失。

有两种主要方式可以缓解这种故障模式：

复制（Replication）

使用复制可以在摄取器中保存每个时间序列的多个副本（通常为3个）

如果 Cortex 集群失去一个 Ingester 实例，被丢失的实例所持有的内存中系列样本也会至少被复制到另一个实例中。在单个实例故障的情况下，不会丢失任何时间序列样本；但在多个摄取器故障的情况下，如果所有持有特定时间序列副本的摄取器都受到故障影响，那么该时间序列可能会丢失。

预写日志（Write-ahead log，WAL）

使用预写日志将所有传入的系列样本写入持久性磁盘，直到它们被刷新到长期存储，在摄取器发生故障的情况下，后续的进程重启将重放预写日志并恢复内存中的系列样本。

与仅使用复制不同，由于持久性磁盘数据不会丢失，在多个摄取器故障的情况下，每个摄取器都将在后续重启时从预写日志中恢复内存中的系列样本。仍然建议使用复制，以确保在单个摄取器故障的情况下，读取路径不会出现临时故障。

因此，在Cortex中，复制和预写日志（WAL）是两种主要的故障处理机制。复制提供冗余备份以减少数据丢失的风险，而预写日志提供了数据持久性和恢复能力。这两种方式可以结合使用，以提供更高的数据可靠性和系统的容错性。

写入解放

摄取器的写入解放（Write de-amplification）是一种机制，用于减少写入操作对长期存储的压力和负载，从而降低系统的成本和复杂性。

摄取器在内存中存储最近接收的样本，以执行写入解放（write de-amplification）。如果摄取器立即将接收到的样本写入长期存储，系统将面临存储压力过大，难以实现扩展性的问题。因此，摄取器会在内存中对样本进行批处理和压缩，并定期将它们刷新到长期存储中。

写入解放是Cortex系统低总拥有成本（Total Cost of Ownership，TCO）的主要原因之一。通过在内存中对样本进行批处理和压缩，摄取器减少了写入长期存储的次数和数据量。这种批处理和压缩的操作减少了对存储系统的负载，降低了硬件资源需求和运维成本。

通过写入解放，Cortex系统能够更有效地利用存储资源，提高了系统的性能和扩展性。通过批处理和压缩，摄取器在将数据刷新到长期存储之前，减少了存储访问的频率和数据量。这不仅减轻了存储系统的负担，还降低了存储成本。写入解放机制是Cortex系统设计中的关键策略，有助于实现更经济高效的数据处理和存储管理，从而降低了系统的总拥有成本。

配置示例

ingester_config

官方文档

ingester:
 lifecycler:
 ring:
 kvstore:
 store: etcd
 prefix: collectors/
 etcd:
 endpoints: []
 dial_timeout: 10s
 max_retries: 10
 tls_enabled: false
 tls_cert_path: ""
 tls_key_path: ""
 tls_ca_path: ""
 tls_server_name: ""
 tls_insecure_skip_verify: false
 username: ""
 password: ""
 heartbeat_timeout: 1m0s
 replication_factor: 1
 zone_awareness_enabled: false
 excluded_zones: ""
 num_tokens: 512
 heartbeat_period: 5s
 observe_period: 0s
 join_after: 0s
 min_ready_duration: 0s
 interface_names:
 - eth0
 - en0
 final_sleep: 0s
 tokens_file_path: ""
 availability_zone: ""
 unregister_on_shutdown: true
 readiness_check_ring_health: true
 address: ""
 port: 0
 id: limiqi-mac2016-01.local
 metadata_retain_period: 10m0s
 rate_update_period: 15s
 active_series_metrics_enabled: true
 active_series_metrics_update_period: 1m0s
 active_series_metrics_idle_timeout: 10m0s
 instance_limits:
 max_ingestion_rate: 0
 max_tenants: 0
 max_series: 0
 max_inflight_push_requests: 0
 ignore_series_limit_for_metric_names: ""

ingester_client_config

官方文档

ingester_client:
 grpc_client_config:
 max_recv_msg_size: 104857600
 max_send_msg_size: 104857600
 grpc_compression: gzip
 rate_limit: 0
 rate_limit_burst: 0
 backoff_on_ratelimits: false
 backoff_config:
 min_period: 100ms
 max_period: 10s
 max_retries: 10
 tls_enabled: false
 tls_cert_path: ""
 tls_key_path: ""
 tls_ca_path: ""
 tls_server_name: ""
 tls_insecure_skip_verify: false

数据结构

github.com/cortexproject/cortex/pkg/ingester/ingester.go

Config

// Config for an Ingester.
type Config struct {
 LifecyclerConfig ring.LifecyclerConfig `yaml:"lifecycler"`

 // Config for metadata purging.
 MetadataRetainPeriod time.Duration `yaml:"metadata_retain_period"`

 RateUpdatePeriod time.Duration `yaml:"rate_update_period"`
 UserTSDBConfigsUpdatePeriod time.Duration `yaml:"user_tsdb_configs_update_period"`

 ActiveSeriesMetricsEnabled bool `yaml:"active_series_metrics_enabled"`
 ActiveSeriesMetricsUpdatePeriod time.Duration `yaml:"active_series_metrics_update_period"`
 ActiveSeriesMetricsIdleTimeout time.Duration `yaml:"active_series_metrics_idle_timeout"`

 // Use blocks storage.
 BlocksStorageConfig cortex_tsdb.BlocksStorageConfig `yaml:"-"`

 // Injected at runtime and read from the distributor config, required
 // to accurately apply global limits.
 DistributorShardingStrategy string `yaml:"-"`
 DistributorShardByAllLabels bool `yaml:"-"`

 // Injected at runtime and read from querier config.
 QueryStoreForLabels bool `yaml:"-"`
 QueryIngestersWithin time.Duration `yaml:"-"`

 DefaultLimits InstanceLimits `yaml:"instance_limits"`
 InstanceLimitsFn func() *InstanceLimits `yaml:"-"`

 IgnoreSeriesLimitForMetricNames string `yaml:"ignore_series_limit_for_metric_names"`

 // For testing, you can override the address and ID of this ingester.
 ingesterClientFactory func(addr string, cfg client.Config) (client.HealthAndIngesterClient, error)
}

ring.LifecyclerConfig

github.com/cortexproject/cortex/pkg/ring/ring.go

// LifecyclerConfig is the config to build a Lifecycler.
type LifecyclerConfig struct {
 RingConfig Config `yaml:"ring"`

 // Config for the ingester lifecycle control
 NumTokens int `yaml:"num_tokens"`
 HeartbeatPeriod time.Duration `yaml:"heartbeat_period"`
 ObservePeriod time.Duration `yaml:"observe_period"`
 JoinAfter time.Duration `yaml:"join_after"`
 MinReadyDuration time.Duration `yaml:"min_ready_duration"`
 InfNames []string `yaml:"interface_names"`
 FinalSleep time.Duration `yaml:"final_sleep"`
 TokensFilePath string `yaml:"tokens_file_path"`
 Zone string `yaml:"availability_zone"`
 UnregisterOnShutdown bool `yaml:"unregister_on_shutdown"`
 ReadinessCheckRingHealth bool `yaml:"readiness_check_ring_health"`

 // For testing, you can override the address and ID of this ingester
 Addr string `yaml:"address" doc:"hidden"`
 Port int `doc:"hidden"`
 ID string `doc:"hidden"`

 // Injected internally
 ListenPort int `yaml:"-"`
}

// Config for a Ring
type Config struct {
 KVStore kv.Config `yaml:"kvstore"`
 HeartbeatTimeout time.Duration `yaml:"heartbeat_timeout"`
 ReplicationFactor int `yaml:"replication_factor"`
 ZoneAwarenessEnabled bool `yaml:"zone_awareness_enabled"`
 ExcludedZones flagext.StringSliceCSV `yaml:"excluded_zones"`

 // Whether the shuffle-sharding subring cache is disabled. This option is set
 // internally and never exposed to the user.
 SubringCacheDisabled bool `yaml:"-"`
}

kv.Config

TODO;

flagext.StringSliceCSV

// StringSliceCSV is a slice of strings that is parsed from a comma-separated string
// It implements flag.Value and yaml Marshalers
type StringSliceCSV []string

InstanceLimits

// InstanceLimits describes limits used by ingester. Reaching any of these will result in Push method to return
// (internal) error.
type InstanceLimits struct {
 MaxIngestionRate float64 `yaml:"max_ingestion_rate"`
 MaxInMemoryTenants int64 `yaml:"max_tenants"`
 MaxInMemorySeries int64 `yaml:"max_series"`
 MaxInflightPushRequests int64 `yaml:"max_inflight_push_requests"`
}

client.Config

// Config is the configuration struct for the ingester client
type Config struct {
 GRPCClientConfig grpcclient.Config `yaml:"grpc_client_config"`
}

grpcclient.Config

// Config for a gRPC client.
type Config struct {
 MaxRecvMsgSize int `yaml:"max_recv_msg_size"`
 MaxSendMsgSize int `yaml:"max_send_msg_size"`
 GRPCCompression string `yaml:"grpc_compression"`
 RateLimit float64 `yaml:"rate_limit"`
 RateLimitBurst int `yaml:"rate_limit_burst"`

 BackoffOnRatelimits bool `yaml:"backoff_on_ratelimits"`
 BackoffConfig backoff.Config `yaml:"backoff_config"`

 TLSEnabled bool `yaml:"tls_enabled"`
 TLS tls.ClientConfig `yaml:",inline"`
}

Docs: Compactor

Tue, 30 May 2023 14:30:23 +0800

简要概述

组件 Compactor 用于将上传至对象存储中的多个数据块合并为一个更大的优化块，帮助减少存储成本并提高查询速度。通过去除重复数据和减小索引大小，能够有效地减少存储需求。同时，通过压缩数据块，查询过程中需要查询的数据块数量也减少，从而提高了查询的效率和速度。

此外，该组件还负责更新每个租户的桶索引。桶索引是 Queriers 服务、Store-gateway 和 rulers 等组件用于发现存储中新数据块的重要工具。通过保持桶索引的更新，同时确保其他组件能够及时发现新的数据块，以便进行查询、存储和规则处理等操作。

注意：使用该组件时，对 alertmanager、ruler 与 block 不能使用相同的 s3 bucket 名称，因为需要从 block 中下载数据解析，如果无法解析程序会异常退出。

如何工作

块压缩有两个主要好处：

垂直压缩：相同时间范围内由所有 Ingester 组件上传的块被压缩；
水平压缩：将具有较小时间范围的块水平压缩为一个较大的块。

垂直压缩

将同一时间范围（默认为2小时）内由 Ingester 组件上传的各个块分别按照租户划分合并为单个块，并且通过去重复样本将原本写入 N 个块（由于复制而产生）。这一步将每个租户在单个2小时时间范围内的块数从 #(Ingester 的数量) 减少到1。

水平压缩

水平压缩在垂直压缩之后触发，将相邻的两个2小时时间范围内的多个块压缩为一个更大的块。尽管此压缩后块块块的总大小不会改变，但它仍然可以显著减少存储网关在内存中保留的索引和索引头的大小。

分片支持

启用分片后，多个实例可以协作，通过租户对块进行分片和分配工作负载。在任何给定时间，一个租户的所有块由单个块压缩器实例处理，但不同租户的压缩可以同时在不同的块压缩器实例上运行。

每当块压缩器池增加或减少时（例如扩容或缩容），租户将在可用的块压缩器实例之间进行重新分片，无需手动干预。

块压缩器的分片功能基于哈希环。在启动时，块压缩器生成随机令牌并将自身注册到环中。运行时，它定期扫描存储桶（每隔 -compactor.compaction-interval）以发现存储中的租户列表，并压缩与其自身在环中分配的令牌范围匹配的每个租户的块。

此功能可以通过设置 -compactor.sharding-enabled=true 启用，并通过 -compactor.ring. 标志（或它们各自的 YAML 配置选项）配置后端哈希环。

数据块删除

当 Compactor 组件成功将几个源数据块压缩成一个大块时，源数据块将从对象存储中删除，这个删除不是立即进行的，按照以下两个步骤：

将数据块标记为删除（软删除）；
当数据块标记删除时间超过 compactor.deletion-delay 参数设置值时，才真正从对象存储中移除（硬删除）。

块压缩器负责标记块进行删除，并在删除延迟到期后进行硬删除。软删除是基于存储桶中块位置内的一个名为 deletion-mark.json 的小文件进行的，查询器和存储网关都会查找该文件。

软删除机制用于给查询器和存储网关足够的时间在旧的源块被删除之前发现新的压缩块。如果块压缩器立即对源块进行硬删除，那么涉及压缩块的一些查询可能会失败，直到查询器和存储网关重新扫描存储桶并找到被删除的源块和新的压缩块。

压缩磁盘评估

当开始压缩时 Compactor 组件需要从对象存储中下载源数据至本地，同时临时在本地生成单一的大块文件，磁盘空间依赖最大的一个租户与参数 compactor.block-ranges 决定。

语法：

min_disk_space_required = compactor.compaction-concurrency * max_compaction_range_blocks_size * 2

大约 1千万的活跃序列需要 150GB 的存储空间，那么假设现在最大租户活跃序列 3千万同时仅允许一个线程 compactor.compaction-concurrency=1 进行操作，同时 compactor.block-ranges=24h 则所需空间为：

450GB = 1 * 24 * 2

配置示例

compactor:
 block_ranges:
 - 2h0m0s
 - 12h0m0s
 - 24h0m0s
 block_sync_concurrency: 20
 meta_sync_concurrency: 20
 consistency_delay: 0s
 data_dir: /data/cortex/compactor
 compaction_interval: 1h0m0s
 compaction_retries: 3
 compaction_concurrency: 1
 cleanup_interval: 15m0s
 cleanup_concurrency: 20
 deletion_delay: 12h0m0s
 tenant_cleanup_delay: 6h0m0s
 skip_blocks_with_out_of_order_chunks_enabled: false
 block_files_concurrency: 10
 blocks_fetch_concurrency: 3
 block_deletion_marks_migration_enabled: false
 sharding_enabled: true
 sharding_strategy: default
 sharding_ring:
 kvstore:
 store: "etcd"
 prefix: "/cortex/collectors/"
 etcd:
 endpoints: ["192.168.31.201:2379","192.168.31.202:2379","192.168.31.203:2379"]
 dial_timeout: 10s
 max_retries: 10
 tls_enabled: true
 tls_cert_path: "/opt/cortex/pki/server.crt"
 tls_key_path: "/opt/cortex/pki/server.key"
 tls_ca_path: "/opt/cortex/pki/ca.crt"
 tls_insecure_skip_verify: true
 heartbeat_period: 5s
 heartbeat_timeout: 1m0s
 instance_id: node1
 instance_port: 9005
 instance_addr: "192.168.31.201"

数据结构

Config

// Config holds the Compactor config.
type Config struct {
 BlockRanges cortex_tsdb.DurationList `yaml:"block_ranges"`
 BlockSyncConcurrency int `yaml:"block_sync_concurrency"`
 MetaSyncConcurrency int `yaml:"meta_sync_concurrency"`
 ConsistencyDelay time.Duration `yaml:"consistency_delay"`
 DataDir string `yaml:"data_dir"`
 CompactionInterval time.Duration `yaml:"compaction_interval"`
 CompactionRetries int `yaml:"compaction_retries"`
 CompactionConcurrency int `yaml:"compaction_concurrency"`
 CleanupInterval time.Duration `yaml:"cleanup_interval"`
 CleanupConcurrency int `yaml:"cleanup_concurrency"`
 DeletionDelay time.Duration `yaml:"deletion_delay"`
 TenantCleanupDelay time.Duration `yaml:"tenant_cleanup_delay"`
 SkipBlocksWithOutOfOrderChunksEnabled bool `yaml:"skip_blocks_with_out_of_order_chunks_enabled"`
 BlockFilesConcurrency int `yaml:"block_files_concurrency"`
 BlocksFetchConcurrency int `yaml:"blocks_fetch_concurrency"`

 // Whether the migration of block deletion marks to the global markers location is enabled.
 BlockDeletionMarksMigrationEnabled bool `yaml:"block_deletion_marks_migration_enabled"`

 EnabledTenants flagext.StringSliceCSV `yaml:"enabled_tenants"`
 DisabledTenants flagext.StringSliceCSV `yaml:"disabled_tenants"`

 // Compactors sharding.
 ShardingEnabled bool `yaml:"sharding_enabled"`
 ShardingStrategy string `yaml:"sharding_strategy"`
 ShardingRing RingConfig `yaml:"sharding_ring"`

 // No need to add options to customize the retry backoff,
 // given the defaults should be fine, but allow to override
 // it in tests.
 retryMinBackoff time.Duration `yaml:"-"`
 retryMaxBackoff time.Duration `yaml:"-"`

 // Allow downstream projects to customise the blocks compactor.
 BlocksGrouperFactory BlocksGrouperFactory `yaml:"-"`
 BlocksCompactorFactory BlocksCompactorFactory `yaml:"-"`

 // Block visit marker file config
 BlockVisitMarkerTimeout time.Duration `yaml:"block_visit_marker_timeout"`
 BlockVisitMarkerFileUpdateInterval time.Duration `yaml:"block_visit_marker_file_update_interval"`
}

cortex_tsdb.DurationList

github.com/cortexproject/cortex/pkg/storage/tsdb

// DurationList is the block ranges for a tsdb
type DurationList []time.Duration

RingConfig

// RingConfig masks the ring lifecycler config which contains
// many options not really required by the compactors ring. This config
// is used to strip down the config to the minimum, and avoid confusion
// to the user.
type RingConfig struct {
 KVStore kv.Config `yaml:"kvstore"`
 HeartbeatPeriod time.Duration `yaml:"heartbeat_period"`
 HeartbeatTimeout time.Duration `yaml:"heartbeat_timeout"`

 // Wait ring stability.
 WaitStabilityMinDuration time.Duration `yaml:"wait_stability_min_duration"`
 WaitStabilityMaxDuration time.Duration `yaml:"wait_stability_max_duration"`

 // Instance details
 InstanceID string `yaml:"instance_id" doc:"hidden"`
 InstanceInterfaceNames []string `yaml:"instance_interface_names"`
 InstancePort int `yaml:"instance_port" doc:"hidden"`
 InstanceAddr string `yaml:"instance_addr" doc:"hidden"`
 TokensFilePath string `yaml:"tokens_file_path"`
 UnregisterOnShutdown bool `yaml:"unregister_on_shutdown"`

 // Injected internally
 ListenPort int `yaml:"-"`

 WaitActiveInstanceTimeout time.Duration `yaml:"wait_active_instance_timeout"`

 ObservePeriod time.Duration `yaml:"-"`
}

kv.Config

github.com/cortexproject/cortex/pkg/ring/kv

见 Distributor/kv.Config

Docs: Store Gateway

Tue, 30 May 2023 14:34:38 +0800

简要概述

组件 Store-gateway 存储网关负责从对象存储中获取数据块提供给 Querier 组件使用，它需要对存储桶具有几乎实时的视图，以便发现属于其分片的数据块。

为了保持对存储桶的视图更新，存储网关可以通过以下两种不同的方式来实现：

定期扫描存储桶（默认方式）：存储网关定期扫描存储桶，以发现新的数据块并更新视图。
定期下载存储桶索引：存储网关可以定期下载存储桶的索引，从而快速获取最新的视图信息。

存储网关是半状态的服务，这意味着它在维护存储桶视图的过程中会保持一些状态信息。它需要跟踪已扫描或已下载的数据块，并根据需要更新存储桶的视图。这种半状态的设计使存储网关能够快速响应查询请求，并提供准确的时间序列数据。

如何工作

定期扫描

在启动 Store-gateway 组件时会从所有租户对象存储中的数据块遍历下载 “meta.json”、“index-header” 文件，在这个阶段组件是不可用的也就是 “/ready” 接口处于不可用，如果在 k8s 中编排则服务是处理非健康状态，不会加入到 Service 中不接收流量。

当正常运行时，会定期重新扫描存储桶去发现是否存在新的数据块或是否有数据块被标记删除，新数据块来自 Ingester 组件与 Compactor 组件上传生成，数据块被标记删除是由 Compactor 标记的。控制多久重新扫描一次由参数 blocks-storage.bucket-store.sync-interval 决定。

存储网关永远不会完全下载块和整个索引，既 “chunks/*” 与 “index” 文件，仅 “index” 的部分内容被下载以文件名 “index-header” 保存，以避免在随后重新启动时重新下载。因此，建议（但不是必需的）使用持久磁盘运行存储网关。例如，如果你在 Kubernetes 中运行 Cortex 集群，你可以为存储网关使用具有持久卷声明的StatefulSet。

开启存储桶索引

启用 bucket 索引后，整体工作流程是相同的，但存储网关不是迭代bucket对象，而是为属于其分片的每个租户获取bucket索引，以发现每个租户的块和块删除标记

配置示例

auth_enabled: false

target: store-gateway

server:
 http_listen_port: 9091

store_gateway:
 sharding_enabled: true
 sharding_ring:
 kvstore:
 store: "etcd"
 prefix: "/cortex/store-gateway/"
 etcd:
 endpoints: ["192.168.30.125:2379"]
 dial_timeout: 10s
 max_retries: 10
 tls_enabled: false
 tls_insecure_skip_verify: false
 heartbeat_period: 5s
 heartbeat_timeout: 1m0s
 instance_id: node1
 instance_interface_names:
 - en0

blocks_storage:
 tsdb:
 dir: ./data/cortex/tsdb1
 bucket_store:
 sync_dir: ./data/cortex/tsdb-sync1
 backend: filesystem

数据结构

Config

// Config holds the store gateway config.
type Config struct {
 ShardingEnabled bool `yaml:"sharding_enabled"`
 ShardingRing RingConfig `yaml:"sharding_ring" doc:"description=The hash ring configuration. This option is required only if blocks sharding is enabled."`
 ShardingStrategy string `yaml:"sharding_strategy"`
}

RingConfig

// RingConfig masks the ring lifecycler config which contains
// many options not really required by the store gateways ring. This config
// is used to strip down the config to the minimum, and avoid confusion
// to the user.
type RingConfig struct {
 KVStore kv.Config `yaml:"kvstore" doc:"description=The key-value store used to share the hash ring across multiple instances. This option needs be set both on the store-gateway and querier when running in microservices mode."`
 HeartbeatPeriod time.Duration `yaml:"heartbeat_period"`
 HeartbeatTimeout time.Duration `yaml:"heartbeat_timeout"`
 ReplicationFactor int `yaml:"replication_factor"`
 TokensFilePath string `yaml:"tokens_file_path"`
 ZoneAwarenessEnabled bool `yaml:"zone_awareness_enabled"`

 // Wait ring stability.
 WaitStabilityMinDuration time.Duration `yaml:"wait_stability_min_duration"`
 WaitStabilityMaxDuration time.Duration `yaml:"wait_stability_max_duration"`

 // Instance details
 InstanceID string `yaml:"instance_id" doc:"hidden"`
 InstanceInterfaceNames []string `yaml:"instance_interface_names"`
 InstancePort int `yaml:"instance_port" doc:"hidden"`
 InstanceAddr string `yaml:"instance_addr" doc:"hidden"`
 InstanceZone string `yaml:"instance_availability_zone"`

 // Injected internally
 ListenPort int `yaml:"-"`
 RingCheckPeriod time.Duration `yaml:"-"`
}

Docs: Frontend

Tue, 30 May 2023 14:27:52 +0800

简要概述

这是可选的服务，提供查询器的API端点，并可以用于加速读取路径。

当使用查询前端时，应将传入的查询请求定向到查询前端而不是查询器，在集群中仍然需要查询器服务，以执行实际的查询操作。

通过使用查询前端，可以将查询请求分配给前端服务，并由前端处理请求的负载均衡和路由。查询前端可以缓存查询结果、处理身份验证和授权、记录查询日志等。这样可以减轻查询器的负载并提高查询的性能和响应速度。

但请注意，查询前端本身并不执行实际的查询操作，它仅用于处理查询请求的转发和管理。实际的查询操作仍由查询器服务执行，因此查询器服务仍然是集群中必需的组件。

要使用查询前端加速读取路径，您需要相应地配置和部署查询前端服务，并确保将查询请求正确地定向到查询前端而不是查询器。具体的配置和部署步骤可能会因您所使用的具体系统和工具而有所不同，建议参考相关的文档和配置指南，以了解如何正确设置和使用查询前端。

总结：查询前端是一个可选的服务，用于加速读取路径。通过将查询请求定向到查询前端而不是查询器，可以提高查询性能和响应速度。查询前端负责请求的负载均衡和路由，但实际的查询操作仍由查询器服务执行。

查询队列

查询前端的排队机制用于以下目的：

确保在查询器出现内存不足错误（OOM错误）时，能够对可能导致该错误的大型查询进行重试。这使管理员可以为查询分配较少的内存，或者乐观地并行运行更多的小型查询，从而有助于降低总拥有成本（TCO）。
通过使用先进先出队列（FIFO）将大量请求分布在所有查询器上，防止多个大型请求在单个查询器上进行队列传送。这样可以平均地将这些请求分配给各个查询器。
通过在不同租户之间公平调度查询，防止单个租户通过服务拒绝攻击（DOS攻击）的方式影响其他租户。
通过查询前端的排队机制，可以实现对大型查询的重试、请求的公平分发和租户之间查询的公平调度。这有助于提高系统的可靠性、资源利用率和用户体验。

水平分割

查询前端将多天的查询拆分为多个单天的查询，并在下游的查询器上并行执行这些查询，然后将结果再次合并在一起。这样可以防止大型（多天）查询在单个查询器中引发内存不足问题，并帮助更快地执行这些查询。

通过将多天的查询拆分为单天的查询并并行执行，查询前端可以更好地处理大型查询，减少内存压力，并提高查询速度。这有助于保持系统的稳定性和性能，并提供更好的用户体验。

缓存数据

查询前端支持缓存查询结果，并在后续查询中重复使用这些结果。如果缓存的结果不完整，查询前端会计算所需的子查询，并在下游的查询器上并行执行它们。查询前端还可以选择将查询与其步长参数对齐，以改善查询结果的可缓存性。结果缓存与任何Cortex缓存后端兼容（目前支持memcached、redis和内存缓存）。

通过查询结果的缓存和重用，查询前端可以提高查询性能和响应时间，特别是对于重复查询或具有相同参数的查询。缓存查询结果可以减少对下游查询器的负载，并提供更快速的查询体验。同时，通过对查询进行步长对齐，可以进一步提高缓存的有效性，增加查询结果的可重用性。这些功能有助于提高系统的性能和可伸缩性，并降低查询成本。

配置示例

导出默认参数

frontend:
 log_queries_longer_than: 0s
 max_body_size: 10485760
 query_stats_enabled: false
 max_outstanding_per_tenant: 100
 querier_forget_delay: 0s
 scheduler_address: ""
 scheduler_dns_lookup_period: 10s
 scheduler_worker_concurrency: 5
 grpc_client_config:
 max_recv_msg_size: 104857600
 max_send_msg_size: 16777216
 grpc_compression: ""
 rate_limit: 0
 rate_limit_burst: 0
 backoff_on_ratelimits: false
 backoff_config:
 min_period: 100ms
 max_period: 10s
 max_retries: 10
 tls_enabled: false
 tls_cert_path: ""
 tls_key_path: ""
 tls_ca_path: ""
 tls_server_name: ""
 tls_insecure_skip_verify: false
 instance_interface_names:
 - eth0
 - en0
 address: ""
 port: 0
 downstream_url: ""

query_range:
 split_queries_by_interval: 0s
 align_queries_with_step: false
 results_cache:
 cache:
 enable_fifocache: false
 default_validity: 0s
 background:
 writeback_goroutines: 10
 writeback_buffer: 10000
 redis:
 endpoint: ""
 master_name: ""
 timeout: 500ms
 expiration: 0s
 db: 0
 pool_size: 0
 password: ""
 tls_enabled: false
 tls_insecure_skip_verify: false
 idle_timeout: 0s
 max_connection_age: 0s
 fifocache:
 max_size_bytes: ""
 max_size_items: 0
 validity: 0s
 size: 0
 prefix: frontend.
 compression: ""
 cache_queryable_samples_stats: false
 cache_results: false
 max_retries: 5
 forward_headers_list: []

使用 redis 缓存

frontend:
 downstream_url: http://192.168.31.201:9009
 log_queries_longer_than: 3s

http_prefix: ""

query_range:
 align_queries_with_step: true
 cache_results: true
 max_retries: 3
 results_cache:
 cache:
 redis:
 db: 1
 endpoint: 192.168.30.125:6379
 password: '********'
 split_queries_by_interval: 24h0m0s

server:
 http_listen_port: 9091

target: query-frontend

数据结构

frontend.CombinedFrontendConfig

github.com/cortexproject/cortex/pkg/frontend/config.go

// This struct combines several configuration options together to preserve backwards compatibility.
type CombinedFrontendConfig struct {
 Handler transport.HandlerConfig `yaml:",inline"`

 // 以下三个均各配置为兼容 prometheus 的后端地址，如组件：Querier
 // 三个同时设置，仅其中一个有效，优先级：DownstreamURL > FrontendV2 > FrontendV1

 FrontendV1 v1.Config `yaml:",inline"`
 FrontendV2 v2.Config `yaml:",inline"`

 DownstreamURL string `yaml:"downstream_url"`
}

InitFrontend

github.com/cortexproject/cortex/pkg/frontend/config.go

transport.HandlerConfig

github.com/cortexproject/cortex/pkg/frontend/transport/handler.go

// Config for a Handler.
type HandlerConfig struct {
 // 记录慢查询日志，设置为0s关闭记录，设置小于0则记录所有，如："3s"
 LogQueriesLongerThan time.Duration `yaml:"log_queries_longer_than"`
 // 允许客户端请求内容的最大字节
 MaxBodySize int64 `yaml:"max_body_size"`
 // 开启查询统计会新增以下四个指标
 // cortex_query_seconds_total
 // cortex_query_fetched_series_total
 // cortex_query_fetched_chunks_bytes_total
 // cortex_query_fetched_data_bytes_total
 QueryStatsEnabled bool `yaml:"query_stats_enabled"`
}

v1.Config

github.com/cortexproject/cortex/pkg/frontend/v1/frontend.go

// Config for a Frontend.
type Config struct {
 MaxOutstandingPerTenant int `yaml:"max_outstanding_per_tenant"`
 QuerierForgetDelay time.Duration `yaml:"querier_forget_delay"`
}

v2.Config

github.com/cortexproject/cortex/pkg/frontend/v2/frontend.go

// Config for a Frontend.
type Config struct {
 SchedulerAddress string `yaml:"scheduler_address"`
 DNSLookupPeriod time.Duration `yaml:"scheduler_dns_lookup_period"`
 WorkerConcurrency int `yaml:"scheduler_worker_concurrency"`
 GRPCClientConfig grpcclient.Config `yaml:"grpc_client_config"`

 // Used to find local IP address, that is sent to scheduler and querier-worker.
 InfNames []string `yaml:"instance_interface_names"`

 // If set, address is not computed from interfaces.
 Addr string `yaml:"address" doc:"hidden"`
 Port int `doc:"hidden"`
}

queryrange.Config

github.com/cortexproject/cortex/pkg/querier/tripperware/queryrange/query_range_middlewares.go

// Config for query_range middleware chain.
type Config struct {
 SplitQueriesByInterval time.Duration `yaml:"split_queries_by_interval"`
 AlignQueriesWithStep bool `yaml:"align_queries_with_step"`
 ResultsCacheConfig `yaml:"results_cache"`
 CacheResults bool `yaml:"cache_results"`
 MaxRetries int `yaml:"max_retries"`
 // List of headers which query_range middleware chain would forward to downstream querier.
 ForwardHeaders flagext.StringSlice `yaml:"forward_headers_list"`

 // Populated based on the query configuration
 VerticalShardSize int `yaml:"-"`
}

ResultsCacheConfig

// ResultsCacheConfig is the config for the results cache.
type ResultsCacheConfig struct {
 CacheConfig cache.Config `yaml:"cache"`
 Compression string `yaml:"compression"`
 CacheQueryableSamplesStats bool `yaml:"cache_queryable_samples_stats"`
}

cache.Config

github.com/cortexproject/cortex/pkg/chunk/cache/cache.go

// Config for building Caches.
type Config struct {
 EnableFifoCache bool `yaml:"enable_fifocache"`

 DefaultValidity time.Duration `yaml:"default_validity"`

 Background BackgroundConfig `yaml:"background"`
 Memcache MemcachedConfig `yaml:"memcached"`
 MemcacheClient MemcachedClientConfig `yaml:"memcached_client"`
 Redis RedisConfig `yaml:"redis"`
 Fifocache FifoCacheConfig `yaml:"fifocache"`

 // This is to name the cache metrics properly.
 Prefix string `yaml:"prefix" doc:"hidden"`

 // For tests to inject specific implementations.
 Cache Cache `yaml:"-"`
}

cache.RedisConfig

// RedisConfig defines how a RedisCache should be constructed.
type RedisConfig struct {
 Endpoint string `yaml:"endpoint"`
 MasterName string `yaml:"master_name"`
 Timeout time.Duration `yaml:"timeout"`
 Expiration time.Duration `yaml:"expiration"`
 DB int `yaml:"db"`
 PoolSize int `yaml:"pool_size"`
 Password flagext.Secret `yaml:"password"`
 EnableTLS bool `yaml:"tls_enabled"`
 InsecureSkipVerify bool `yaml:"tls_insecure_skip_verify"`
 IdleTimeout time.Duration `yaml:"idle_timeout"`
 MaxConnAge time.Duration `yaml:"max_connection_age"`
}

Docs: Ruler

Fri, 30 Jun 2023 17:40:42 +0800

简要概述

用于告警规则

配置示例

ruler:
 evaluation_interval: 60s
 poll_interval: 60s
 rule_path: /data/cortex/rule
 alertmanager_url: "http://192.168.31.201:9009/alertmanager"
 enable_alertmanager_v2: true
 notification_queue_capacity: 10000
 notification_timeout: 10s
 enable_sharding: true
 ring:
 kvstore:
 store: "etcd"
 prefix: "/cortex/rulers/"
 etcd:
 endpoints: ["192.168.31.201:2379","192.168.31.202:2379","192.168.31.203:2379"]
 dial_timeout: 10s
 max_retries: 10
 tls_enabled: true
 tls_cert_path: "/opt/cortex/pki/server.crt"
 tls_key_path: "/opt/cortex/pki/server.key"
 tls_ca_path: "/opt/cortex/pki/ca.crt"
 tls_insecure_skip_verify: false
 heartbeat_period: 5s
 heartbeat_timeout: 1m0s
 instance_id: node1
 instance_port: 9005
 instance_addr: "192.168.31.201"
 enable_api: true
 query_stats_enabled: true

ruler_storage:
 backend: s3
 s3:
 endpoint: "192.168.31.200:9000"
 region: "default"
 bucket_name: "uptime"
 secret_access_key: "minioadmin"
 access_key_id: "minioadmin"
 insecure: false

数据结构

Config

// Config is the configuration for the recording rules server.
type Config struct {
 // This is used for template expansion in alerts; must be a valid URL.
 ExternalURL flagext.URLValue `yaml:"external_url"`
 // Labels to add to all alerts
 ExternalLabels labels.Labels `yaml:"external_labels,omitempty" doc:"nocli|description=Labels to add to all alerts."`
 // GRPC Client configuration.
 ClientTLSConfig grpcclient.Config `yaml:"ruler_client"`
 // How frequently to evaluate rules by default.
 // 检测告警规则表达式的时间，默认 1分钟
 EvaluationInterval time.Duration `yaml:"evaluation_interval"`
 // How frequently to poll for updated rules.
 // 多久同步一次告警规则，默认 1分钟
 PollInterval time.Duration `yaml:"poll_interval"`
 // Path to store rule files for prom manager.
 // 存放告警规则在本地的路径
 RulePath string `yaml:"rule_path"`

 // URL of the Alertmanager to send notifications to.
 // If your are configuring the ruler to send to a Cortex Alertmanager,
 // ensure this includes any path set in the Alertmanager external URL.
 // 告警推送至 alertmanager 实例
 AlertmanagerURL string `yaml:"alertmanager_url"`
 // Whether to use DNS SRV records to discover Alertmanager.
 AlertmanagerDiscovery bool `yaml:"enable_alertmanager_discovery"`
 // How long to wait between refreshing the list of Alertmanager based on DNS service discovery.
 AlertmanagerRefreshInterval time.Duration `yaml:"alertmanager_refresh_interval"`
 // Enables the ruler notifier to use the Alertmananger V2 API.
 // 调用 alertmanager 使用接口版本2
 AlertmanangerEnableV2API bool `yaml:"enable_alertmanager_v2"`
 // Capacity of the queue for notifications to be sent to the Alertmanager.
 NotificationQueueCapacity int `yaml:"notification_queue_capacity"`
 // HTTP timeout duration when sending notifications to the Alertmanager.
 NotificationTimeout time.Duration `yaml:"notification_timeout"`
 // Client configs for interacting with the Alertmanager
 Notifier NotifierConfig `yaml:"alertmanager_client"`

 // Max time to tolerate outage for restoring "for" state of alert.
 OutageTolerance time.Duration `yaml:"for_outage_tolerance"`
 // Minimum duration between alert and restored "for" state. This is maintained only for alerts with configured "for" time greater than grace period.
 ForGracePeriod time.Duration `yaml:"for_grace_period"`
 // Minimum amount of time to wait before resending an alert to Alertmanager.
 ResendDelay time.Duration `yaml:"resend_delay"`

 // Enable sharding rule groups.
 EnableSharding bool `yaml:"enable_sharding"`
 ShardingStrategy string `yaml:"sharding_strategy"`
 SearchPendingFor time.Duration `yaml:"search_pending_for"`
 Ring RingConfig `yaml:"ring"`
 FlushCheckPeriod time.Duration `yaml:"flush_period"`

 EnableAPI bool `yaml:"enable_api"`

 EnabledTenants flagext.StringSliceCSV `yaml:"enabled_tenants"`
 DisabledTenants flagext.StringSliceCSV `yaml:"disabled_tenants"`

 RingCheckPeriod time.Duration `yaml:"-"`

 EnableQueryStats bool `yaml:"query_stats_enabled"`
 DisableRuleGroupLabel bool `yaml:"disable_rule_group_label"`
}

rulestore.Config

github.com/cortexproject/cortex/pkg/ruler/rulestore/config.go

// Config configures a rule store.
type Config struct {
 bucket.Config `yaml:",inline"`
 ConfigDB client.Config `yaml:"configdb"`
 Local local.Config `yaml:"local"`
}

bucket.Config

见数据存储

client.Config

github.com/cortexproject/cortex/pkg/configs/client/client.go

// Config says where we can find the ruler userconfig.
type Config struct {
 ConfigsAPIURL flagext.URLValue `yaml:"configs_api_url"`
 ClientTimeout time.Duration `yaml:"client_timeout"` // HTTP timeout duration for requests made to the Weave Cloud configs service.
 TLS tls_cfg.ClientConfig `yaml:",inline"`
}

local.Config

github.com/cortexproject/cortex/pkg/ruler/rulestore/local/local.go

type Config struct {
 Directory string `yaml:"directory"`
}

RingConfig

github.com/cortexproject/cortex/pkg/ruler/ruler_ring.go

// RingConfig masks the ring lifecycler config which contains
// many options not really required by the rulers ring. This config
// is used to strip down the config to the minimum, and avoid confusion
// to the user.
type RingConfig struct {
 KVStore kv.Config `yaml:"kvstore"`
 HeartbeatPeriod time.Duration `yaml:"heartbeat_period"`
 HeartbeatTimeout time.Duration `yaml:"heartbeat_timeout"`

 // Instance details
 InstanceID string `yaml:"instance_id" doc:"hidden"`
 InstanceInterfaceNames []string `yaml:"instance_interface_names"`
 InstancePort int `yaml:"instance_port" doc:"hidden"`
 InstanceAddr string `yaml:"instance_addr" doc:"hidden"`
 NumTokens int `yaml:"num_tokens"`

 // Injected internally
 ListenPort int `yaml:"-"`

 // Used for testing
 SkipUnregister bool `yaml:"-"`
}

Docs: Alertmanager

Fri, 30 Jun 2023 18:22:08 +0800

简要概述

配置示例

alertmanager:
 data_dir: "/data/cortex/alertmanager"
 retention: 120h0m0s
 external_url: "http://192.168.31.201:9009/alertmanager"
 poll_interval: 15s
 max_recv_msg_size: 16777216
 sharding_enabled: true
 sharding_ring:
 kvstore:
 store: "etcd"
 prefix: "/cortex/alertmanagers/"
 etcd:
 endpoints: ["192.168.31.201:2379","192.168.31.202:2379","192.168.31.203:2379"]
 dial_timeout: 10s
 max_retries: 10
 tls_enabled: true
 tls_cert_path: "/opt/cortex/pki/server.crt"
 tls_key_path: "/opt/cortex/pki/server.key"
 tls_ca_path: "/opt/cortex/pki/ca.crt"
 tls_insecure_skip_verify: true
 heartbeat_period: 5s
 heartbeat_timeout: 1m0s
 replication_factor: 3
 instance_id: node1
 instance_port: 9005
 instance_addr: "192.168.31.201"
 cluster:
 listen_address: 192.168.31.201:9094
 advertise_address: 192.168.31.201:9094
 peers: "192.168.31.201:9094,192.168.31.202:9094,192.168.31.203:9094"
 gossip_interval: 10s
 enable_api: true

alertmanager_storage:
 backend: s3
 s3:
 endpoint: "192.168.31.200:9000"
 insecure: false
 bucket_name: "uptime"
 access_key_id: "minioadmin"
 secret_access_key: "minioadmin"

数据结构

MultitenantAlertmanagerConfig

github.com/cortexproject/cortex/pkg/alertmanager/multitenant.go

// MultitenantAlertmanagerConfig is the configuration for a multitenant Alertmanager.
type MultitenantAlertmanagerConfig struct {
 // 存放告警模版、静默规则等
 DataDir string `yaml:"data_dir"`
 // 静默规则被删除后保留该时间后释放
 Retention time.Duration `yaml:"retention"`
 // 通过 webhook 回调通知中的 externalURL 属性
 ExternalURL flagext.URLValue `yaml:"external_url"`
 // 多久从存储中同步一次数据至本地
 PollInterval time.Duration `yaml:"poll_interval"`
 // 允许传输的最大字节，保留默认 16M 既可
 MaxRecvMsgSize int64 `yaml:"max_recv_msg_size"`

 // Enable sharding for the Alertmanager
 // 是否开启集群分片
 ShardingEnabled bool `yaml:"sharding_enabled"`
 // 配置集群哈希环
 ShardingRing RingConfig `yaml:"sharding_ring"`

 // 如果实例未配置，则默认使用此文件内容
 FallbackConfigFile string `yaml:"fallback_config_file"`
 AutoWebhookRoot string `yaml:"auto_webhook_root"`

 // 集群配置
 Cluster ClusterConfig `yaml:"cluster"`

 // 开启接口
 EnableAPI bool `yaml:"enable_api"`

 // For distributor.
 AlertmanagerClient ClientConfig `yaml:"alertmanager_client"`

 // For the state persister.
 Persister PersisterConfig `yaml:",inline"`

 // 配置哪些租户可以使用 alertmanager 功能
 EnabledTenants flagext.StringSliceCSV `yaml:"enabled_tenants"`
 DisabledTenants flagext.StringSliceCSV `yaml:"disabled_tenants"`
}

RingConfig

// RingConfig masks the ring lifecycler config which contains
// many options not really required by the alertmanager ring. This config
// is used to strip down the config to the minimum, and avoid confusion
// to the user.
type RingConfig struct {
 KVStore kv.Config `yaml:"kvstore" doc:"description=The key-value store used to share the hash ring across multiple instances."`
 HeartbeatPeriod time.Duration `yaml:"heartbeat_period"`
 HeartbeatTimeout time.Duration `yaml:"heartbeat_timeout"`
 ReplicationFactor int `yaml:"replication_factor"`
 ZoneAwarenessEnabled bool `yaml:"zone_awareness_enabled"`

 // Instance details
 InstanceID string `yaml:"instance_id" doc:"hidden"`
 InstanceInterfaceNames []string `yaml:"instance_interface_names"`
 InstancePort int `yaml:"instance_port" doc:"hidden"`
 InstanceAddr string `yaml:"instance_addr" doc:"hidden"`
 InstanceZone string `yaml:"instance_availability_zone"`

 // Injected internally
 ListenPort int `yaml:"-"`
 RingCheckPeriod time.Duration `yaml:"-"`

 // Used for testing
 SkipUnregister bool `yaml:"-"`
}

ClusterConfig

type ClusterConfig struct {
 ListenAddr string `yaml:"listen_address"`
 AdvertiseAddr string `yaml:"advertise_address"`
 Peers flagext.StringSliceCSV `yaml:"peers"`
 PeerTimeout time.Duration `yaml:"peer_timeout"`
 GossipInterval time.Duration `yaml:"gossip_interval"`
 PushPullInterval time.Duration `yaml:"push_pull_interval"`
}

ClientConfig

// ClientConfig is the configuration struct for the alertmanager client.
type ClientConfig struct {
 RemoteTimeout time.Duration `yaml:"remote_timeout"`
 TLSEnabled bool `yaml:"tls_enabled"`
 TLS tls.ClientConfig `yaml:",inline"`
 GRPCCompression string `yaml:"grpc_compression"`
}

PersisterConfig

type PersisterConfig struct {
 Interval time.Duration `yaml:"persist_interval"`
}

kv.Config

见Distributor

tls.ClientConfig

TODO;

alertstore.Config

github.com/cortexproject/cortex/pkg/alertmanager/alertstore/config.go

见Ruler rulestore.Config。

Docs: 资源限制

Tue, 30 May 2023 15:19:32 +0800

简要概述

配置 Cortex 服务（如分发器、摄取器等）的默认限制，这些限制适用于所有租户，例如，您可以设置默认的最大样本数量或最大标签数等。

此外，还允许您为每个租户设置特定的限制，这些特定限制将覆盖默认限制，并且仅适用于相应的租户。您可以根据不同租户的需求设置不同的限制，以确保资源分配的公平性和合理性。

通过配置limits_config，您可以在Cortex中管理和调整各个服务的限制，以满足特定的需求和约束。请注意，这些限制可以帮助保护Cortex集群免受滥用或超负荷使用，同时提供适当的资源分配和性能保证。

配置示例

官方文档

limits:
 # 每租户的速率限制（以每秒样本数为单位）
 # http code=429 resp=ingestion rate limit (25000) exceeded
 ingestion_rate: 25000
 # 应将摄取速率限制应用于每个分发器实例（local）
 # 还是在整个集群中均匀分配（global）
 ingestion_rate_strategy: local
 # 每租户允许的突发大小（以样本数量为单位）
 ingestion_burst_size: 50000
 # 启用所有用户处理带有外部标签的样本的标志，这些外部标签用于标识高可用Prometheus设置中的副本
 accept_ha_samples: false
 # 从指标中提取哪个标签值作为集群标识
 ha_cluster_label: cluster
 # 从指标中提取哪个标签值作为副本标识
 ha_replica_label: __replica__
 ha_max_clusters: 0
 drop_labels: []
 max_label_name_length: 1024
 max_label_value_length: 2048
 max_label_names_per_series: 30
 max_labels_size_bytes: 0
 max_metadata_length: 1024
 reject_old_samples: false
 reject_old_samples_max_age: 2w
 creation_grace_period: 10m
 enforce_metadata_metric_name: true
 enforce_metric_name: true
 ingestion_tenant_shard_size: 0
 max_series_per_query: 100000
 max_series_per_user: 5000000
 max_series_per_metric: 50000
 max_global_series_per_user: 0
 max_global_series_per_metric: 0
 max_metadata_per_user: 8000
 max_metadata_per_metric: 10
 max_global_metadata_per_user: 0
 max_global_metadata_per_metric: 0
 max_fetched_chunks_per_query: 2000000
 max_fetched_series_per_query: 0
 max_fetched_chunk_bytes_per_query: 0
 max_fetched_data_bytes_per_query: 0
 max_query_lookback: 0s
 max_query_length: 0s
 max_query_parallelism: 14
 max_cache_freshness: 1m
 max_queriers_per_tenant: 0
 query_vertical_shard_size: 0
 ruler_evaluation_delay_duration: 0s
 ruler_tenant_shard_size: 0
 ruler_max_rules_per_rule_group: 0
 ruler_max_rule_groups_per_tenant: 0
 store_gateway_tenant_shard_size: 0
 compactor_blocks_retention_period: 0s
 compactor_tenant_shard_size: 0
 s3_sse_type: ""
 s3_sse_kms_key_id: ""
 s3_sse_kms_encryption_context: ""
 alertmanager_receivers_firewall_block_cidr_networks: ""
 alertmanager_receivers_firewall_block_private_addresses: false
 alertmanager_notification_rate_limit: 0
 alertmanager_notification_rate_limit_per_integration: {}
 alertmanager_max_config_size_bytes: 0
 alertmanager_max_templates_count: 0
 alertmanager_max_template_size_bytes: 0
 alertmanager_max_dispatcher_aggregation_groups: 0
 alertmanager_max_alerts_count: 0
 alertmanager_max_alerts_size_bytes: 0

数据结构

github.com/cortexproject/cortex/pkg/util/validation/limits.go

Limits

// Limits describe all the limits for users; can be used to describe global default
// limits via flags, or per-user limits via yaml config.
type Limits struct {
 // Distributor enforced limits.
 IngestionRate float64 `yaml:"ingestion_rate" json:"ingestion_rate"`
 IngestionRateStrategy string `yaml:"ingestion_rate_strategy" json:"ingestion_rate_strategy"`
 IngestionBurstSize int `yaml:"ingestion_burst_size" json:"ingestion_burst_size"`
 AcceptHASamples bool `yaml:"accept_ha_samples" json:"accept_ha_samples"`
 HAClusterLabel string `yaml:"ha_cluster_label" json:"ha_cluster_label"`
 HAReplicaLabel string `yaml:"ha_replica_label" json:"ha_replica_label"`
 HAMaxClusters int `yaml:"ha_max_clusters" json:"ha_max_clusters"`
 DropLabels flagext.StringSlice `yaml:"drop_labels" json:"drop_labels"`
 MaxLabelNameLength int `yaml:"max_label_name_length" json:"max_label_name_length"`
 MaxLabelValueLength int `yaml:"max_label_value_length" json:"max_label_value_length"`
 MaxLabelNamesPerSeries int `yaml:"max_label_names_per_series" json:"max_label_names_per_series"`
 MaxLabelsSizeBytes int `yaml:"max_labels_size_bytes" json:"max_labels_size_bytes"`
 MaxMetadataLength int `yaml:"max_metadata_length" json:"max_metadata_length"`
 RejectOldSamples bool `yaml:"reject_old_samples" json:"reject_old_samples"`
 RejectOldSamplesMaxAge model.Duration `yaml:"reject_old_samples_max_age" json:"reject_old_samples_max_age"`
 CreationGracePeriod model.Duration `yaml:"creation_grace_period" json:"creation_grace_period"`
 EnforceMetadataMetricName bool `yaml:"enforce_metadata_metric_name" json:"enforce_metadata_metric_name"`
 EnforceMetricName bool `yaml:"enforce_metric_name" json:"enforce_metric_name"`
 IngestionTenantShardSize int `yaml:"ingestion_tenant_shard_size" json:"ingestion_tenant_shard_size"`
 MetricRelabelConfigs []*relabel.Config `yaml:"metric_relabel_configs,omitempty" json:"metric_relabel_configs,omitempty" doc:"nocli|description=List of metric relabel configurations. Note that in most situations, it is more effective to use metrics relabeling directly in the Prometheus server, e.g. remote_write.write_relabel_configs."`
 MaxExemplars int `yaml:"max_exemplars" json:"max_exemplars"`

 // Ingester enforced limits.
 // Series
 MaxSeriesPerQuery int `yaml:"max_series_per_query" json:"max_series_per_query"`
 MaxLocalSeriesPerUser int `yaml:"max_series_per_user" json:"max_series_per_user"`
 MaxLocalSeriesPerMetric int `yaml:"max_series_per_metric" json:"max_series_per_metric"`
 MaxGlobalSeriesPerUser int `yaml:"max_global_series_per_user" json:"max_global_series_per_user"`
 MaxGlobalSeriesPerMetric int `yaml:"max_global_series_per_metric" json:"max_global_series_per_metric"`

 // Metadata
 MaxLocalMetricsWithMetadataPerUser int `yaml:"max_metadata_per_user" json:"max_metadata_per_user"`
 MaxLocalMetadataPerMetric int `yaml:"max_metadata_per_metric" json:"max_metadata_per_metric"`
 MaxGlobalMetricsWithMetadataPerUser int `yaml:"max_global_metadata_per_user" json:"max_global_metadata_per_user"`
 MaxGlobalMetadataPerMetric int `yaml:"max_global_metadata_per_metric" json:"max_global_metadata_per_metric"`
 // Out-of-order
 OutOfOrderTimeWindow model.Duration `yaml:"out_of_order_time_window" json:"out_of_order_time_window"`

 // Querier enforced limits.
 MaxChunksPerQuery int `yaml:"max_fetched_chunks_per_query" json:"max_fetched_chunks_per_query"`
 MaxFetchedSeriesPerQuery int `yaml:"max_fetched_series_per_query" json:"max_fetched_series_per_query"`
 MaxFetchedChunkBytesPerQuery int `yaml:"max_fetched_chunk_bytes_per_query" json:"max_fetched_chunk_bytes_per_query"`
 MaxFetchedDataBytesPerQuery int `yaml:"max_fetched_data_bytes_per_query" json:"max_fetched_data_bytes_per_query"`
 MaxQueryLookback model.Duration `yaml:"max_query_lookback" json:"max_query_lookback"`
 MaxQueryLength model.Duration `yaml:"max_query_length" json:"max_query_length"`
 MaxQueryParallelism int `yaml:"max_query_parallelism" json:"max_query_parallelism"`
 MaxCacheFreshness model.Duration `yaml:"max_cache_freshness" json:"max_cache_freshness"`
 MaxQueriersPerTenant int `yaml:"max_queriers_per_tenant" json:"max_queriers_per_tenant"`
 QueryVerticalShardSize int `yaml:"query_vertical_shard_size" json:"query_vertical_shard_size" doc:"hidden"`

 // Query Frontend / Scheduler enforced limits.
 MaxOutstandingPerTenant int `yaml:"max_outstanding_requests_per_tenant" json:"max_outstanding_requests_per_tenant"`

 // Ruler defaults and limits.
 RulerEvaluationDelay model.Duration `yaml:"ruler_evaluation_delay_duration" json:"ruler_evaluation_delay_duration"`
 RulerTenantShardSize int `yaml:"ruler_tenant_shard_size" json:"ruler_tenant_shard_size"`
 RulerMaxRulesPerRuleGroup int `yaml:"ruler_max_rules_per_rule_group" json:"ruler_max_rules_per_rule_group"`
 RulerMaxRuleGroupsPerTenant int `yaml:"ruler_max_rule_groups_per_tenant" json:"ruler_max_rule_groups_per_tenant"`

 // Store-gateway.
 StoreGatewayTenantShardSize int `yaml:"store_gateway_tenant_shard_size" json:"store_gateway_tenant_shard_size"`

 // Compactor.
 CompactorBlocksRetentionPeriod model.Duration `yaml:"compactor_blocks_retention_period" json:"compactor_blocks_retention_period"`
 CompactorTenantShardSize int `yaml:"compactor_tenant_shard_size" json:"compactor_tenant_shard_size"`

 // This config doesn't have a CLI flag registered here because they're registered in
 // their own original config struct.
 S3SSEType string `yaml:"s3_sse_type" json:"s3_sse_type" doc:"nocli|description=S3 server-side encryption type. Required to enable server-side encryption overrides for a specific tenant. If not set, the default S3 client settings are used."`
 S3SSEKMSKeyID string `yaml:"s3_sse_kms_key_id" json:"s3_sse_kms_key_id" doc:"nocli|description=S3 server-side encryption KMS Key ID. Ignored if the SSE type override is not set."`
 S3SSEKMSEncryptionContext string `yaml:"s3_sse_kms_encryption_context" json:"s3_sse_kms_encryption_context" doc:"nocli|description=S3 server-side encryption KMS encryption context. If unset and the key ID override is set, the encryption context will not be provided to S3. Ignored if the SSE type override is not set."`

 // Alertmanager.
 AlertmanagerReceiversBlockCIDRNetworks flagext.CIDRSliceCSV `yaml:"alertmanager_receivers_firewall_block_cidr_networks" json:"alertmanager_receivers_firewall_block_cidr_networks"`
 AlertmanagerReceiversBlockPrivateAddresses bool `yaml:"alertmanager_receivers_firewall_block_private_addresses" json:"alertmanager_receivers_firewall_block_private_addresses"`

 NotificationRateLimit float64 `yaml:"alertmanager_notification_rate_limit" json:"alertmanager_notification_rate_limit"`
 NotificationRateLimitPerIntegration NotificationRateLimitMap `yaml:"alertmanager_notification_rate_limit_per_integration" json:"alertmanager_notification_rate_limit_per_integration"`

 AlertmanagerMaxConfigSizeBytes int `yaml:"alertmanager_max_config_size_bytes" json:"alertmanager_max_config_size_bytes"`
 AlertmanagerMaxTemplatesCount int `yaml:"alertmanager_max_templates_count" json:"alertmanager_max_templates_count"`
 AlertmanagerMaxTemplateSizeBytes int `yaml:"alertmanager_max_template_size_bytes" json:"alertmanager_max_template_size_bytes"`
 AlertmanagerMaxDispatcherAggregationGroups int `yaml:"alertmanager_max_dispatcher_aggregation_groups" json:"alertmanager_max_dispatcher_aggregation_groups"`
 AlertmanagerMaxAlertsCount int `yaml:"alertmanager_max_alerts_count" json:"alertmanager_max_alerts_count"`
 AlertmanagerMaxAlertsSizeBytes int `yaml:"alertmanager_max_alerts_size_bytes" json:"alertmanager_max_alerts_size_bytes"`
}

flagext.StringSlice

github.com/cortexproject/cortex/pkg/util/flagext/

// StringSlice is a slice of strings that implements flag.Value
type StringSlice []string

relabel.Config

github.com/prometheus/prometheus/model/relabel/

flagext.CIDRSliceCSV

// CIDRSliceCSV is a slice of CIDRs that is parsed from a comma-separated string.
// It implements flag.Value and yaml Marshalers.
type CIDRSliceCSV []CIDR

// CIDR is a network CIDR.
type CIDR struct {
 Value *net.IPNet
}

NotificationRateLimitMap

type NotificationRateLimitMap map[string]float64

应用场景

客户端写入高可用

服务端 cortex 的 limit 配置：

distributor:
 ha_tracker:
 enable_ha_tracker: true
 ha_tracker_update_timeout: 15s
 ha_tracker_update_timeout_jitter_max: 5s
 ha_tracker_failover_timeout: 30s
 kvstore:
 store: etcd
 prefix: /cortex/ha-tracker/
 etcd:
 endpoints:
 - 192.168.31.201:2379
 dial_timeout: 10s
 max_retries: 10

limits:
 accept_ha_samples: true
 ha_cluster_label: cluster
 ha_replica_label: __replica__

客户端上报以下数据

pod1 -> metric_name{cluster="cluster1", __replica__="test1"}
pod2 -> metric_name{cluster="cluster1", __replica__="test2"}

由于此时 pod1 与 pod2 处于一个集群 “cluster1”，而分别上报不同的副本 “test1”、“test2”，则同一时间仅允许一个写入。

拒绝日志示例：

replicas did not mach, rejecting sample: replica=test2, elected=test1

允许接收过期数据

服务端 cortex 的 limit 配置

limits:
 out_of_order_time_window: 30m

客户端上报以下数据

服务端 cortex 当前时间： “2023-09-08T11:40:00”

则允许以下数据写入：

metric_name1{} 2023-09-08T11:20:00
metric_name1{} 2023-09-08T11:30:00
metric_name1{} 2023-09-08T11:40:00

则拒绝以下数据写入：

metric_name1{} 2023-09-08T10:50:00
metric_name1{} 2023-09-08T11:10:00
metric_name1{} 2023-09-08T11:20:00