alerting.go 1.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384
  1. package metrics
  2. import (
  3. "time"
  4. log "github.com/sirupsen/logrus"
  5. )
  6. const maxCPUAlertingThresholdPCT = 85
  7. const maxRAMAlertingThresholdPCT = 85
  8. const maxDiskAlertingThresholdPCT = 90
  9. var inCPUAlertingState = false
  10. var inRAMAlertingState = false
  11. var inDiskAlertingState = false
  12. var errorResetDuration = time.Minute * 5
  13. const alertingError = "The %s utilization of %d%% could cause problems with video generation and delivery. Visit the documentation at http://owncast.online/docs/troubleshooting/ if you are experiencing issues."
  14. func handleAlerting() {
  15. handleCPUAlerting()
  16. handleRAMAlerting()
  17. handleDiskAlerting()
  18. }
  19. func handleCPUAlerting() {
  20. if len(Metrics.CPUUtilizations) < 2 {
  21. return
  22. }
  23. avg := recentAverage(Metrics.CPUUtilizations)
  24. if avg > maxCPUAlertingThresholdPCT && !inCPUAlertingState {
  25. log.Warnf(alertingError, "CPU", avg)
  26. inCPUAlertingState = true
  27. resetTimer := time.NewTimer(errorResetDuration)
  28. go func() {
  29. <-resetTimer.C
  30. inCPUAlertingState = false
  31. }()
  32. }
  33. }
  34. func handleRAMAlerting() {
  35. if len(Metrics.RAMUtilizations) < 2 {
  36. return
  37. }
  38. avg := recentAverage(Metrics.RAMUtilizations)
  39. if avg > maxRAMAlertingThresholdPCT && !inRAMAlertingState {
  40. log.Warnf(alertingError, "memory", avg)
  41. inRAMAlertingState = true
  42. resetTimer := time.NewTimer(errorResetDuration)
  43. go func() {
  44. <-resetTimer.C
  45. inRAMAlertingState = false
  46. }()
  47. }
  48. }
  49. func handleDiskAlerting() {
  50. if len(Metrics.DiskUtilizations) < 2 {
  51. return
  52. }
  53. avg := recentAverage(Metrics.DiskUtilizations)
  54. if avg > maxDiskAlertingThresholdPCT && !inDiskAlertingState {
  55. log.Warnf(alertingError, "disk", avg)
  56. inDiskAlertingState = true
  57. resetTimer := time.NewTimer(errorResetDuration)
  58. go func() {
  59. <-resetTimer.C
  60. inDiskAlertingState = false
  61. }()
  62. }
  63. }
  64. func recentAverage(values []timestampedValue) int {
  65. return (values[len(values)-1].Value + values[len(values)-2].Value) / 2
  66. }