Create monitors to generate alerts and notifications

    series_conditions:
      defaults:
        critical:
          conditions:
            - op: NOT_EXISTS
              sustain: 60s

api_version: v1/config
kind: Monitor
spec:
    # Required name of the monitor. Can be modified after the monitor is created.
    name: Kubernetes errors in production us-west
    # Logging query to return data for.
    logging_query: severity = "ERROR" AND kubernetes.cluster_name = "production-us-west" | make-series by service
    # Annotations are visible in notifications generated by this monitor.
    # You can template annotations with labels from notifications.
    annotations:
        key_1: "{{ $labels.job }}"
    # Slug of the collection the monitor belongs to.
    collection_slug: production-team
    # Optional setting for configuring how often alerts are evaluated.
    # Defaults to 60 seconds.
    interval_secs: 60
    # Labels are visible in notifications generated by this monitor,
    # and can be used to route alerts with notification overrides.
    labels:
        key_1: kubernetes_cluster
    # Optional notification policy used to route alerts generated by the monitor.
    notification_policy_slug: custom-notification-policy
    schedule:
        # The timezone of the time ranges.
        timezone: UTC
        weekly_schedule:
            monday:
                active: ONLY_DURING_RANGES
                # The time ranges that the monitor is active on this day. Required if
                # active is set to ONLY_DURING_RANGES.
                ranges:
                    - # End time in the in format "<hour>:<minute>", such as "15:30".
                      end_hh_mm: "15:00"
                      # Start time in the in format "<hour>:<minute>", such as "15:30".
                      start_hh_mm: "10:10"
            tuesday:
                active: NEVER
            wednesday:
                active: NEVER
            thursday:
                active: ONLY_DURING_RANGES
                # The time ranges that the monitor is active on this day. Required if
                ranges:
                # active is set to ONLY_DURING_RANGES.
                    - # End time in the in format "<hour>:<minute>", such as "15:30".
                      end_hh_mm: "24:00"
                      # Start time in the in format "<hour>:<minute>", such as "15:30".
                      start_hh_mm: "21:15"
            friday:
                active: NEVER
            saturday:
                active: NEVER
            sunday:
                active: NEVER
    # Conditions evaluated against each queried series to determine the severity of each series.
    series_conditions:
        defaults:
            critical:
                # List of conditions to evaluate against a series.
                # Only one condition must match to assign a severity to a signal.
                conditions:
                   # To alert on missing or no data, change the value for `op` to `NOT_EXISTS`.
                    - op: GT
                    # How long the op operation needs to evaluate for the condition
                    # to evaluate to true.
                    sustain_secs: 300
                    # The value to compare to the metric value using the op operation.
                    value: 60
                    # How long the operation needs to evaluate false to resolve
                    resolve_sustain: 60
            warn:
                # List of conditions to evaluate against a series.
                # Only one condition must match to assign a severity to a signal.
                conditions:
                    - op: GT
                    # How long the op operation needs to evaluate for the condition
                    # to evaluate to true.
                    sustain_secs: 300
                    # The value to compare to the metric value using the op operation.
                    value: 30
                    # How long the operation needs to evaluate false to resolve
                    resolve_sustain: 60
    # Defines how the set of series from the query are split into signals.
    signal_grouping:
        label_names:
            - source
            - service_environment
            # If true, each series will have its own signal and label_names can't be set.
        signal_per_series: false

resource "chronosphere_monitor" "k8s_production" {
  name = "Kubernetes errors in production us-west"
 
  # Reference to the collection the alert belongs to.
  collection_id = chronosphere_collection.k8s_production.id
 
  # Override the notification policy.
  # By default, uses the policy from the collection_id.
  notification_policy_id = chronosphere_collection.k8s_testing.id
 
  # Arbitrary set of labels to assign to the alert.
  labels = {
    "priority" = "sev-1"
  }
 
  # Arbitrary set of annotations to include in alert notifications.
  annotations = {
    "runbook" = "http://default-runbook"
  }
 
  # Interval at which to evaluate the monitor, for example 15s, 30s, or 60s.
  # Defaults to 60s.
  interval = "30s"
 
  query {
    # Logging query to evaluate.
    logging_query = "severity='ERROR' AND kubernetes.cluster_name='production-us-west' | make-series by service"
  }
 
  # The remaining examples are optional signals specifying how to group the
  # series returned from the query.
 
  # No signal_grouping clause = Per monitor
  # signal_grouping with label_names set = Per signal for labels set
  # signal_grouping with signal_per_series set to true = Per series
 
  signal_grouping {
    # Set of labels names used to split series into signals.
    # Each unique combination of labels results in its own signal.
    label_names = ["app", "grpc_service"]
 
    # As an alternative to label_names, signal_per_series creates an alert for
    # every resulting series from the query.
    # signal_per_series = true
  }
 
  # Container for the conditions determining the severity of each series from the query.
  # The highest severity series of a signal determines that signal's severity.
  series_conditions {
    # Condition assigning a warn threshold for series above a certain threshold.
    condition {
      # Severity of the condition, which can be "warn" or "critical".
      severity = "warn"
 
      # Value to compare against each series from the query result.
      # For EXISTS or NOT_EXISTS operators, value must be set to zero or may be omitted.
      value = 5.0
 
      # Operator to use when comparing the query result versus the threshold.
      # Valid values can be one of GT, LT, LEQ, GEQ, EQ, NEQ, EXISTS, NOT_EXISTS.
      op = "GT"
 
      # Amount of time the query needs to fail the condition check before
      # an alert is triggered. Must be an integer. Accepts one of s (seconds), m
      # (minutes), or h (hours) as units. Optional.
      sustain = "240s"
 
      # Amount of time the query needs to no longer fire before resolving. Must be
      # an integer. Accepts one of s (seconds), m (minutes), or h (hours) as units.
      resolve_sustain = "60s"
 
    }
 
    condition {
      severity = "critical"
      value    = 10.0
      op       = "GT"
      sustain  = "120s"
      resolve_sustain = "60s"
    }
 
    # Multiple optional overrides can be defined for different sets of conditions
    # to series with matching labels.
    override {
      # One or more matchers for labels on a series.
      label_matcher {
        # Name of the label
        name = "app"
 
        # How to match the label, which can be "EXACT_MATCHER_TYPE" or
        # "REGEXP_MATCHER_TYPE".
        type = "EXACT_MATCHER_TYPE"
 
        # Value of the label.
        value = "dbmon"
      }
 
      condition {
        severity = "critical"
        value    = 1.0
        op       = "GT"
        sustain  = "60s"
      }
    }
  }
 
# If you define a schedule, Observability Platform evaluates the monitor only during
# the specified time ranges. The monitor is inactive during all unspecified
# time ranges.
# If you define an empty schedule, Observability Platform never evaluates the monitor.
  schedule {
    # Valid values: Any IANA timezone string
    timezone = "UTC"
 
    range {
      # Time range for the monitor schedule. Valid values for day can be full
      # day names, such as "Sunday" or "Monday".
      # Valid time values must be specified in the range of 00:00 to 24:00.
      day   = "Monday"
      start = "07:00"
      end   = "10:10"
    }
 
    range {
      day   = "Monday"
      start = "15:00"
      end   = "22:30"
    }
 
    range {
      day   = "Thursday"
      start = "21:15"
      end   = "24:00"
    }
  }
}