为 Tempo 设置监控

app.kubernetes.io/component=distributor
app.kubernetes.io/instance=tempo
app.kubernetes.io/managed-by=Helm
app.kubernetes.io/name=tempo
app.kubernetes.io/part-of=memberlist
app.kubernetes.io/version=2.6.0

discovery.kubernetes "k8s_pods" {
  role = "pod"
  selectors {
    // Only scrape pods with a particular selector.
    role = "pod"
    // The selector is any component that belongs to Tempo.
    label = "app.kubernetes.io/name=tempo"
  }
}

rule {
  source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_component"]
  action = "replace"
  regex = "(.*)"
  replacement = "$1"
  target_label = "k8s_component_name"
}

rule {
  source_labels = ["namespace", "k8s_component_name"]
  action = "replace"
  regex = "(.*?);(.*?)"
  replacement = "$1/$2"
  target_label = "job"
}

alloy:
  configMap:
  create: true
  content: |-
    // Scrape Prometheus metrics for Tempo.
    prometheus.scrape "tempo" {
      // Use Kubernetes discovery to find the relevant pods to scrape.
      targets    = discovery.relabel.k8s_pods.output
      // Forward to the Prometheus relabeling component.
      forward_to = [prometheus.relabel.tempo.receiver]
    }

    // Determine how to select pods to scrape.
    discovery.kubernetes "k8s_pods" {
      // Only scrape pods.
      role = "pod"
      selectors {
        // Only scrape pods with a particular selector.
        role = "pod"
        // The selector is any component that belongs to Tempo.
        label = "app.kubernetes.io/name=tempo"
      }
    }

    // Relabel data from Kubernetes discovery.
    discovery.relabel "k8s_pods" {
      // Relabel from targets scraped by the discovery selection.
      targets = discovery.kubernetes.k8s_pods.targets

      // Create new namespace label based on the discovered kubernetes namespace.
      rule {
        source_labels = ["__meta_kubernetes_namespace"]
        action = "replace"
        regex = "(.*)"
        replacement = "$1"
        target_label = "namespace"
      }

      // Create new component label based on the discovered kubernetes component.
      rule {
        source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_component"]
        action = "replace"
        regex = "(.*)"
        replacement = "$1"
        target_label = "k8s_component_name"
      }

      // Create new cluster label based on the discovered kubernetes instance.
      rule {
        source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_instance"]
        action = "replace"
        regex = "(.*)"
        replacement = "$1"
        target_label = "cluster"
      }
    }

    // Relabel data from Prometheus scraping.
    prometheus.relabel "tempo" {
      // Replace the existing job label with one comprised of the namespace and component.
      rule {
        source_labels = ["namespace", "k8s_component_name"]
        action = "replace"
        regex = "(.*?);(.*?)"
        replacement = "$1/$2"
        target_label = "job"
      }


      // Send the metrics to the Prometheus remote write component.
      forward_to = [prometheus.remote_write.tempo.receiver]
    }

    // Remote write the metrics to a Prometheus compatible endpoint (in this case Mimir).
    prometheus.remote_write "tempo" {
      endpoint {
        url = "https://mimir-cluster.distributor.mimir.svc.cluster.local:9001/api/v1/push"
      }
    }

mimirtool rules load operations/tempo-mixin-compiles/rules.yml --address=https://mimir-cluster.distributor.mimir.svc.cluster.local:9001

mimirtool alertmanager load operations/tempo-mixin-compiles/alerts.yml --address=https://mimir-cluster.distributor.mimir.svc.cluster.local:9001