# you can disable an alarm notification by setting the 'to' line to: silent

# -----------------------------------------------------------------------------

# True (1) if the node is experiencing a configuration-related error, false (0) otherwise.

   template: node_config_error
         on: k8s_kubelet.kubelet_node_config_error
       calc: $kubelet_node_config_error
      units: bool
      every: 10s
       warn: $this == 1
      delay: down 1m multiplier 1.5 max 2h
       info: the node is experiencing a configuration-related error
         to: sysadmin

# Failed Token() requests to the alternate token source

   template: token_requests
     lookup: sum -10s of token_fail_count
         on: k8s_kubelet.kubelet_token_requests
      units: failed requests
      every: 10s
       warn: $this > 0
      delay: down 1m multiplier 1.5 max 2h
       info: failed token requests to alternate token source
         to: sysadmin

# Docker and runtime operation errors

   template: kubelet_operations_error
     lookup: sum -1m
         on: k8s_kubelet.kubelet_operations_errors
      units: errors
      every: 10s
       warn: $this > (($status >= $WARNING)  ? (0) : (20))
      delay: up 30s down 1m multiplier 1.5 max 2h
       info: operations error
         to: sysadmin

# -----------------------------------------------------------------------------

# Pod Lifecycle Event Generator Relisting Latency

# 1. calculate the pleg relisting latency for 1m (quantile 0.5, quantile 0.9, quantile 0.99)
# 2. do the same for the last 10s
# 3. raise an alarm if the later is:
# - 2x the first for quantile 0.5
# - 4x the first for quantile 0.9
# - 8x the first for quantile 0.99
#
# we assume the minimum latency is 1000 microseconds

# quantile 0.5

template: 1m_kubelet_pleg_relist_latency_quantile_05
      on: k8s_kubelet.kubelet_pleg_relist_latency_microseconds
  lookup: average -1m unaligned of kubelet_pleg_relist_latency_05
   units: microseconds
   every: 10s
    info: the average value of pleg relisting latency during the last minute (quantile 0.5)

template: 10s_kubelet_pleg_relist_latency_quantile_05
      on: k8s_kubelet.kubelet_pleg_relist_latency_microseconds
  lookup: average -10s unaligned of kubelet_pleg_relist_latency_05
    calc: $this * 100 / (($1m_kubelet_pleg_relist_latency_quantile_05 < 1000)?(1000):($1m_kubelet_pleg_relist_latency_quantile_05))
   every: 10s
   units: %
   warn: $this > (($status >= $WARNING)?(100):(200))
   crit: $this > (($status >= $WARNING)?(200):(400))
  delay: down 1m multiplier 1.5 max 2h
   info: the % of the pleg relisting latency in the last 10 seconds, compared to the last minute (quantile 0.5)
     to: sysadmin

# quantile 0.9

template: 1m_kubelet_pleg_relist_latency_quantile_09
      on: k8s_kubelet.kubelet_pleg_relist_latency_microseconds
  lookup: average -1m unaligned of kubelet_pleg_relist_latency_09
   units: microseconds
   every: 10s
    info: the average value of pleg relisting latency during the last minute (quantile 0.9)

template: 10s_kubelet_pleg_relist_latency_quantile_09
      on: k8s_kubelet.kubelet_pleg_relist_latency_microseconds
  lookup: average -10s unaligned of kubelet_pleg_relist_latency_09
    calc: $this * 100 / (($1m_kubelet_pleg_relist_latency_quantile_09 < 1000)?(1000):($1m_kubelet_pleg_relist_latency_quantile_09))
   every: 10s
   units: %
   warn: $this > (($status >= $WARNING)?(200):(400))
   crit: $this > (($status >= $WARNING)?(400):(800))
  delay: down 1m multiplier 1.5 max 2h
   info: the % of the pleg relisting latency in the last 10 seconds, compared to the last minute (quantile 0.9)
     to: sysadmin

# quantile 0.99

template: 1m_kubelet_pleg_relist_latency_quantile_099
      on: k8s_kubelet.kubelet_pleg_relist_latency_microseconds
  lookup: average -1m unaligned of kubelet_pleg_relist_latency_099
   units: microseconds
   every: 10s
    info: the average value of pleg relisting latency during the last minute (quantile 0.99)

template: 10s_kubelet_pleg_relist_latency_quantile_099
      on: k8s_kubelet.kubelet_pleg_relist_latency_microseconds
  lookup: average -10s unaligned of kubelet_pleg_relist_latency_099
    calc: $this * 100 / (($1m_kubelet_pleg_relist_latency_quantile_099 < 1000)?(1000):($1m_kubelet_pleg_relist_latency_quantile_099))
   every: 10s
   units: %
   warn: $this > (($status >= $WARNING)?(400):(800))
   crit: $this > (($status >= $WARNING)?(800):(1200))
  delay: down 1m multiplier 1.5 max 2h
   info: the % of the pleg relisting latency in the last 10 seconds, compared to the last minute (quantile 0.99)
     to: sysadmin
