owncast/services/metrics/alerting.go

package metrics

import (
	"time"

	"github.com/owncast/owncast/models"
	log "github.com/sirupsen/logrus"
)

const (
	maxCPUAlertingThresholdPCT  = 85
	maxRAMAlertingThresholdPCT  = 85
	maxDiskAlertingThresholdPCT = 90
)

var (
	inCPUAlertingState  = false
	inRAMAlertingState  = false
	inDiskAlertingState = false
)

var errorResetDuration = time.Minute * 5

const alertingError = "The %s utilization of %f%% could cause problems with video generation and delivery. Visit the documentation at http://owncast.online/docs/troubleshooting/ if you are experiencing issues."

func (m *Metrics) handleAlerting() {
	m.handleCPUAlerting()
	m.handleRAMAlerting()
	m.handleDiskAlerting()
}

func (m *Metrics) handleCPUAlerting() {
	if len(m.metrics.CPUUtilizations) < 2 {
		return
	}

	avg := m.recentAverage(m.metrics.CPUUtilizations)
	if avg > maxCPUAlertingThresholdPCT && !inCPUAlertingState {
		log.Warnf(alertingError, "CPU", avg)
		inCPUAlertingState = true

		resetTimer := time.NewTimer(errorResetDuration)
		go func() {
			<-resetTimer.C
			inCPUAlertingState = false
		}()
	}
}

func (m *Metrics) handleRAMAlerting() {
	if len(m.metrics.RAMUtilizations) < 2 {
		return
	}

	avg := m.recentAverage(m.metrics.RAMUtilizations)
	if avg > maxRAMAlertingThresholdPCT && !inRAMAlertingState {
		log.Warnf(alertingError, "memory", avg)
		inRAMAlertingState = true

		resetTimer := time.NewTimer(errorResetDuration)
		go func() {
			<-resetTimer.C
			inRAMAlertingState = false
		}()
	}
}

func (m *Metrics) handleDiskAlerting() {
	if len(m.metrics.DiskUtilizations) < 2 {
		return
	}

	avg := m.recentAverage(m.metrics.DiskUtilizations)

	if avg > maxDiskAlertingThresholdPCT && !inDiskAlertingState {
		log.Warnf(alertingError, "disk", avg)
		inDiskAlertingState = true

		resetTimer := time.NewTimer(errorResetDuration)
		go func() {
			<-resetTimer.C
			inDiskAlertingState = false
		}()
	}
}

func (m *Metrics) recentAverage(values []models.TimestampedValue) float64 {
	return (values[len(values)-1].Value + values[len(values)-2].Value) / 2
}