Grafana Mimirtool

curl -fLo mimirtool https://github.com/grafana/mimir/releases/latest/download/mimirtool-linux-amd64
chmod +x mimirtool

mimirtool alertmanager get

mimirtool alertmanager get --output-dir="am"

mimirtool alertmanager load <config_file>
mimirtool alertmanager load <config_file> <template_files>...

mimirtool alertmanager load ./example_alertmanager_config.yaml ./example_alertmanager_template.tpl

route:
  receiver: "example_receiver"
  group_by: ["example_groupby"]
receivers:
  - name: "example_receiver"
templates:
  - example_template.tpl

{{ define "alert_customer_env_message" }}
  [{{ .CommonLabels.alertname }} | {{ .CommonLabels.customer }} | {{ .CommonLabels.environment }}]
{{ end }}

# assuming we have written the files out to the folder am
# mimirtool alertmanager get --output-dir="am"
mimirtool alertmanager load  am/config.yaml am/*.tpl

mimirtool alertmanager delete

mimirtool alertmanager verify <config_file> [template_files...]

mimirtool alertmanager migrate-utf8 <config_file> [template_files...]

mimirtool alertmanager migrate-utf8 <config_file> [template_files...] --output-dir="migrated"

mimirtool alertmanager verify <config_file> [template_files...]

mimirtool alertmanager verify <config_file> [template_files...] --utf8-strict-mode

level=warn msg="UTF-8 strict mode enabled"
mimirtool: error: end of input: expected label value, try --help

mimirtool alertmanager verify <config_file> [template_files...]

mimirtool alerts verify

mimirtool alertmanager render --template.glob 'templates/*' --template.data alert_data1.json --template.text '{{ template "my_message" . }}'

mimirtool rules list

mimirtool rules print

mimirtool rules print --output-dir=rules

mimirtool rules get <namespace> <rule_group_name>

mimirtool rules get <namespace> <rule_group_name> --output-dir=rules

mimirtool rules delete <namespace> <rule_group_name>

mimirtool rules load <file_path>...

mimirtool rules load ./example_rules_one.yaml

namespace: my_namespace
groups:
  - name: example
    interval: 5m
    rules:
      - record: job:http_inprogress_requests:sum
        expr: sum by (job) (http_inprogress_requests)

mimirtool rules load ./example_rules_one.yaml ./example_rules_two.yaml

mimirtool rules delete-namespace <namespace>

mimirtool rules lint <file_path>...

mimirtool rules prepare <file_path>...

mimirtool rules prepare ./example_rules_one.yaml

namespace: my_namespace
groups:
  - name: example
    interval: 5m
    rules:
      - record: job:http_inprogress_requests:sum
        expr: sum by (job) (http_inprogress_requests)

namespace: my_namespace
groups:
  - name: example
    interval: 5m
    rules:
      - record: job:http_inprogress_requests:sum
        # note the added cluster label
        expr: sum by(job, cluster) (http_inprogress_requests)

INFO[0000] SUCCESS: 1 rules found, 0 modified expressions

mimirtool rules check <file_path>...

mimirtool rules check rules.yaml

namespace: my_namespace
groups:
  - name: example
    interval: 5m
    rules:
      - record: job_http_inprogress_requests_sum
        expr: sum by (job) (http_inprogress_requests)

ERRO[0000] bad recording rule name error="recording rule name does not match level:metric:operation format, must contain at least one colon" file=rules.yaml rule=job_http_inprogress_requests_sum ruleGroup=example

mimirtool rules diff <file_path>...

mimirtool rules sync <file_path>...

mimirtool remote-read stats --selector '{job="node"}' --address http://demo.robustperception.io:9090 --remote-read-path /api/v1/read

INFO[0000] Create remote read client using endpoint 'http://demo.robustperception.io:9090/api/v1/read'
INFO[0000] Querying time from=2020-12-30T14:00:00Z to=2020-12-30T15:00:00Z with selector={job="node"}
INFO[0000] MIN TIME                           MAX TIME                           DURATION     NUM SAMPLES  NUM SERIES   NUM STALE NAN VALUES  NUM NAN VALUES
INFO[0000] 2020-12-30 14:00:00.629 +0000 UTC  2020-12-30 14:59:59.629 +0000 UTC  59m59s       159480       425          0                     0

mimirtool remote-read dump --selector 'up{job="node"}' --address http://demo.robustperception.io:9090 --remote-read-path /api/v1/read

{__name__="up", instance="demo.robustperception.io:9100", job="node"} 1 1609336914711
{__name__="up", instance="demo.robustperception.io:9100", job="node"} NaN 1609336924709 # StaleNaN
...

# Use Remote Read API to download all metrics with label job=name into local tsdb
mimirtool remote-read export --selector '{job="node"}' --address http://demo.robustperception.io:9090 --remote-read-path /api/v1/read --tsdb-path ./local-tsdb

INFO[0000] Create remote read client using endpoint 'http://demo.robustperception.io:9090/api/v1/read'
INFO[0000] Created TSDB in path './local-tsdb'
INFO[0000] Using existing TSDB in path './local-tsdb'
INFO[0000] Querying time from=2020-12-30T13:53:59Z to=2020-12-30T14:53:59Z with selector={job="node"}
INFO[0001] Store TSDB blocks in './local-tsdb'
INFO[0001] BLOCK ULID                  MIN TIME                       MAX TIME                       DURATION     NUM SAMPLES  NUM CHUNKS   NUM SERIES   SIZE
INFO[0001] 01ETT28D6B8948J87NZXY8VYD9  2020-12-30 13:53:59 +0000 UTC  2020-12-30 13:59:59 +0000 UTC  6m0.001s     15950        429          425          105KiB867B
INFO[0001] 01ETT28D91Z9SVRYF3DY0KNV41  2020-12-30 14:00:00 +0000 UTC  2020-12-30 14:53:58 +0000 UTC  53m58.001s   143530       1325         425          509KiB679B

promtool tsdb analyze ./local-tsdb

promtool tsdb dump ./local-tsdb

prometheus --storage.tsdb.path ./local-tsdb --config.file=<(echo "")

mimirtool acl generate-header --id=<tenant_id> --rule=<promql_selector>

mimirtool acl generate-header --id=1234 --rule='{namespace="A"}'

The header to set:
X-Prom-Label-Policy: 1234:%7Bnamespace=%22A%22%7D

mimirtool analyze grafana --address=<url>

{
  "metricsUsed": [
    "apiserver_request:availability30d",
    "workqueue_depth",
    "workqueue_queue_duration_seconds_bucket"
  ],
  "dashboards": [
    {
      "slug": "",
      "uid": "09ec8aa1e996d6ffcd6817bbaff4db1b",
      "title": "Kubernetes / API server",
      "metrics": [
        "apiserver_request:availability30d",
        "apiserver_request_total",
        "cluster_quantile:apiserver_request_duration_seconds:histogram_quantile",
        "workqueue_depth",
        "workqueue_queue_duration_seconds_bucket"
      ],
      "parse_errors": ["unsupported panel type: \"news\""]
    }
  ]
}

mimirtool analyze ruler --address=<url> --id=<tenant_id>

{
  "metricsUsed": [
    "apiserver_request_duration_seconds_bucket",
    "container_cpu_usage_seconds_total",
    "scheduler_scheduling_algorithm_duration_seconds_bucket"
  ],
  "ruleGroups": [
    {
      "namspace": "prometheus_rules",
      "name": "kube-apiserver.rules",
      "metrics": [
        "apiserver_request_duration_seconds_bucket",
        "apiserver_request_duration_seconds_count",
        "apiserver_request_total"
      ],
      "parse_errors": null
    }
  ]
}

mimirtool analyze dashboard <file>...

mimirtool analyze rule-file <file>

mimirtool analyze prometheus --address=<url> --id=<tenant_id>

{
  "total_active_series": 38184,
  "in_use_active_series": 14047,
  "additional_active_series": 24137,
  "in_use_metric_counts": [
    {
      "metric": "apiserver_request_duration_seconds_bucket",
      "count": 11400,
      "job_counts": [
        {
          "job": "apiserver",
          "count": 11400
        }
      ]
    },
    {
      "metric": "apiserver_request_total",
      "count": 684,
      "job_counts": [
        {
          "job": "apiserver",
          "count": 684
        }
      ]
    }
  ],
  "additional_metric_counts": [
    {
      "metric": "etcd_request_duration_seconds_bucket",
      "count": 2688,
      "job_counts": [
        {
          "job": "apiserver",
          "count": 2688
        }
      ]
    }
  ]
}

mimirtool bucket-validation

mimirtool config convert --yaml-file=cortex.yaml --flags-file=cortex.flags --yaml-out=mimir.yaml --flags-out=mimir.flags

query_range:
  results_cache:
    cache:
      memcached:
        expiration: 10s # Expiration was removed in Grafana Mimir, so this parameter will be missing from the output YAML
        batch_size: 2048
        parallelism: 10
      memcached_client:
        max_idle_conns: 32

frontend:
  results_cache:
    memcached:
      max_get_multi_batch_size: 2048
      max_get_multi_concurrency: 10
      max_idle_connections: 32

server:
  http_listen_port: 80

 #!/usr/bin/env bash

 set -euf -o pipefail

 function usage {
   cat <<EOF
 Extract the CLI flags from individual components.

 Usage:
   $0 <resources JSON> <component>

 Examples:
   $0 resources.json ingester
   $0 <(tk eval environments/default) distributor
   $0 <(jsonnet environments/default/main.jsonnet) query-frontend
 EOF
 }

 if ! command -v jq &>/dev/null; then
   echo "jq command not found in PATH"
   echo "To download jq, refer to https://stedolan.github.io/jq/download/."
 fi

 if [[ $# -ne 2 ]]; then
   usage
   exit 1
 fi

 jq -rf /dev/stdin -- "$1" <<EOF
 ..
 | if type == "object" and .metadata.name == "$2" then .spec.template.spec.containers[]?.args[] else null end
 | select(. != null)
 EOF

<PATH TO SCRIPT> <(tk eval environments/default) distributor

-consul.hostname=consul.cortex-to-mimir.svc.cluster.local:8500
-distributor.ha-tracker.enable=false
-distributor.ha-tracker.enable-for-all-users=true
-distributor.ha-tracker.etcd.endpoints=etcd-client.cortex-to-mimir.svc.cluster.local.:2379
-distributor.ha-tracker.prefix=prom_ha/
-distributor.ha-tracker.store=etcd
-distributor.health-check-ingesters=true
-distributor.ingestion-burst-size=200000
-distributor.ingestion-rate-limit=10000
-distributor.ingestion-rate-limit-strategy=global
-distributor.remote-timeout=20s
-distributor.replication-factor=3
-distributor.ring.consul.hostname=consul.cortex-to-mimir.svc.cluster.local:8500
-distributor.ring.prefix=
-distributor.shard-by-all-labels=true
-mem-ballast-size-bytes=1073741824
-ring.heartbeat-timeout=10m
-ring.prefix=
-runtime-config.file=/etc/cortex/overrides.yaml
-server.grpc.keepalive.max-connection-age=2m
-server.grpc.keepalive.max-connection-age-grace=5m
-server.grpc.keepalive.max-connection-idle=1m
-server.grpc.keepalive.min-time-between-pings=10s
-server.grpc.keepalive.ping-without-stream-allowed=true
-target=distributor
-validation.reject-old-samples=true
-validation.reject-old-samples.max-age=12h

#!/usr/bin/env bash

set -euf -o pipefail

function usage {
 cat <<EOF
Transform Go flags into JSON key value pairs

Usage:
 $0 <flags file>

Examples:
 $0 flags.flags
EOF
}

if [[ $# -ne 1 ]]; then
 usage
 exit 1
fi

key_values=$(sed -E -e 's/^-*(.*)=(.*)$/  "\1": "\2",/' "$1")
printf "{\n%s\n}" "${key_values::-1}"

mimirtool backfill --address=http://mimir-compactor/ --id=anonymous /var/prometheus/{01G803NFXZ0MVKN71GT91HMV3Z,01G8BQ8PRR4TAP7EXZVBNTRBZ4,01G8CB7GTTC5ZXY23WTXHSYQXQ}

INFO[0000] Backfilling                              blocks="/var/prometheus/01G803NFXZ0MVKN71GT91HMV3Z,/var/prometheus/01G8BQ8PRR4TAP7EXZVBNTRBZ4,/var/prometheus/01G8CB7GTTC5ZXY23WTXHSYQXQ" user=anonymous
INFO[0000] making request to start block upload     block=01G803NFXZ0MVKN71GT91HMV3Z file=meta.json path=/var/prometheus/01G803NFXZ0MVKN71GT91HMV3Z
WARN[0000] block already exists on the server       path=/var/prometheus/01G803NFXZ0MVKN71GT91HMV3Z
INFO[0000] making request to start block upload     block=01G8BQ8PRR4TAP7EXZVBNTRBZ4 file=meta.json path=/var/prometheus/01G8BQ8PRR4TAP7EXZVBNTRBZ4
INFO[0000] uploading block file                     block=01G8BQ8PRR4TAP7EXZVBNTRBZ4 file=index path=/var/prometheus/01G8BQ8PRR4TAP7EXZVBNTRBZ4 size=259867
INFO[0000] uploading block file                     block=01G8BQ8PRR4TAP7EXZVBNTRBZ4 file=chunks/000001 path=/var/prometheus/01G8BQ8PRR4TAP7EXZVBNTRBZ4 size=5024391
INFO[0000] block uploaded successfully              block=01G8BQ8PRR4TAP7EXZVBNTRBZ4 path=/var/prometheus/01G8BQ8PRR4TAP7EXZVBNTRBZ4
INFO[0000] making request to start block upload     block=01G8CB7GTTC5ZXY23WTXHSYQXQ file=meta.json path=/var/prometheus/01G8CB7GTTC5ZXY23WTXHSYQXQ
INFO[0000] uploading block file                     block=01G8CB7GTTC5ZXY23WTXHSYQXQ file=index path=/var/prometheus/01G8CB7GTTC5ZXY23WTXHSYQXQ size=151181
INFO[0000] uploading block file                     block=01G8CB7GTTC5ZXY23WTXHSYQXQ file=chunks/000001 path=/var/prometheus/01G8CB7GTTC5ZXY23WTXHSYQXQ size=1986792
INFO[0000] block uploaded successfully              block=01G8CB7GTTC5ZXY23WTXHSYQXQ path=/var/prometheus/01G8CB7GTTC5ZXY23WTXHSYQXQ
INFO[0001] finished uploading blocks                already_exists=1 failed=0 succeeded=2