healthOverview.go 8.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279
  1. package metrics
  2. import (
  3. "fmt"
  4. "sort"
  5. "github.com/owncast/owncast/core"
  6. "github.com/owncast/owncast/core/data"
  7. "github.com/owncast/owncast/models"
  8. "github.com/owncast/owncast/utils"
  9. )
  10. const (
  11. healthyPercentageMinValue = 75
  12. maxCPUUsage = 90
  13. minClientCountForDetails = 3
  14. )
  15. // GetStreamHealthOverview will return the stream health overview.
  16. func GetStreamHealthOverview() *models.StreamHealthOverview {
  17. return metrics.streamHealthOverview
  18. }
  19. func generateStreamHealthOverview() {
  20. // Determine what percentage of total players are represented in our overview.
  21. totalPlayerCount := len(core.GetActiveViewers())
  22. if totalPlayerCount == 0 {
  23. metrics.streamHealthOverview = nil
  24. return
  25. }
  26. pct := getClientErrorHeathyPercentage()
  27. if pct < 1 {
  28. metrics.streamHealthOverview = nil
  29. return
  30. }
  31. overview := &models.StreamHealthOverview{
  32. Healthy: pct > healthyPercentageMinValue,
  33. HealthyPercentage: pct,
  34. Message: getStreamHealthOverviewMessage(),
  35. }
  36. if totalPlayerCount > 0 && len(windowedBandwidths) > 0 {
  37. representation := utils.IntPercentage(len(windowedBandwidths), totalPlayerCount)
  38. overview.Representation = representation
  39. }
  40. metrics.streamHealthOverview = overview
  41. }
  42. func getStreamHealthOverviewMessage() string {
  43. if message := wastefulBitrateOverviewMessage(); message != "" {
  44. return message
  45. } else if message := cpuUsageHealthOverviewMessage(); message != "" {
  46. return message
  47. } else if message := networkSpeedHealthOverviewMessage(); message != "" {
  48. return message
  49. } else if message := errorCountHealthOverviewMessage(); message != "" {
  50. return message
  51. }
  52. return ""
  53. }
  54. func networkSpeedHealthOverviewMessage() string {
  55. type singleVariant struct {
  56. isVideoPassthrough bool
  57. bitrate int
  58. }
  59. outputVariants := data.GetStreamOutputVariants()
  60. streamSortVariants := make([]singleVariant, len(outputVariants))
  61. for i, variant := range outputVariants {
  62. variantSort := singleVariant{
  63. bitrate: variant.VideoBitrate,
  64. isVideoPassthrough: variant.IsVideoPassthrough,
  65. }
  66. streamSortVariants[i] = variantSort
  67. }
  68. sort.Slice(streamSortVariants, func(i, j int) bool {
  69. if streamSortVariants[i].isVideoPassthrough && !streamSortVariants[j].isVideoPassthrough {
  70. return true
  71. }
  72. if !streamSortVariants[i].isVideoPassthrough && streamSortVariants[j].isVideoPassthrough {
  73. return false
  74. }
  75. return streamSortVariants[i].bitrate > streamSortVariants[j].bitrate
  76. })
  77. lowestSupportedBitrate := float64(streamSortVariants[len(streamSortVariants)-1].bitrate)
  78. totalNumberOfClients := len(windowedBandwidths)
  79. if totalNumberOfClients == 0 {
  80. return ""
  81. }
  82. // Determine healthy status based on bandwidth speeds of clients.
  83. unhealthyClientCount := 0
  84. for _, speed := range windowedBandwidths {
  85. if int(speed) < int(lowestSupportedBitrate*1.1) {
  86. unhealthyClientCount++
  87. }
  88. }
  89. if unhealthyClientCount == 0 {
  90. return ""
  91. }
  92. return fmt.Sprintf("%d of %d viewers (%d%%) are consuming video slower than, or too close to your bitrate of %d kbps.", unhealthyClientCount, totalNumberOfClients, int((float64(unhealthyClientCount)/float64(totalNumberOfClients))*100), int(lowestSupportedBitrate))
  93. }
  94. // wastefulBitrateOverviewMessage attempts to determine if a streamer is sending to
  95. // Owncast at a bitrate higher than they're streaming to their viewers leading
  96. // to wasted CPU by having to compress it.
  97. func wastefulBitrateOverviewMessage() string {
  98. if len(metrics.CPUUtilizations) < 2 {
  99. return ""
  100. }
  101. // Only return an alert if the CPU usage is around the max cpu threshold.
  102. recentCPUUses := metrics.CPUUtilizations[len(metrics.CPUUtilizations)-2:]
  103. values := make([]float64, len(recentCPUUses))
  104. for i, val := range recentCPUUses {
  105. values[i] = val.Value
  106. }
  107. recentCPUUse := utils.Avg(values)
  108. if recentCPUUse < maxCPUUsage-10 {
  109. return ""
  110. }
  111. currentBroadcast := core.GetCurrentBroadcast()
  112. if currentBroadcast == nil {
  113. return ""
  114. }
  115. currentBroadcaster := core.GetBroadcaster()
  116. if currentBroadcast == nil {
  117. return ""
  118. }
  119. if currentBroadcaster.StreamDetails.VideoBitrate == 0 {
  120. return ""
  121. }
  122. // Not all streams report their inbound bitrate.
  123. inboundBitrate := currentBroadcaster.StreamDetails.VideoBitrate
  124. if inboundBitrate == 0 {
  125. return ""
  126. }
  127. outputVariants := data.GetStreamOutputVariants()
  128. type singleVariant struct {
  129. isVideoPassthrough bool
  130. bitrate int
  131. }
  132. streamSortVariants := make([]singleVariant, len(outputVariants))
  133. for i, variant := range outputVariants {
  134. variantSort := singleVariant{
  135. bitrate: variant.VideoBitrate,
  136. isVideoPassthrough: variant.IsVideoPassthrough,
  137. }
  138. streamSortVariants[i] = variantSort
  139. }
  140. sort.Slice(streamSortVariants, func(i, j int) bool {
  141. if streamSortVariants[i].isVideoPassthrough && !streamSortVariants[j].isVideoPassthrough {
  142. return true
  143. }
  144. if !streamSortVariants[i].isVideoPassthrough && streamSortVariants[j].isVideoPassthrough {
  145. return false
  146. }
  147. return streamSortVariants[i].bitrate > streamSortVariants[j].bitrate
  148. })
  149. maxBitrate := streamSortVariants[0].bitrate
  150. if inboundBitrate > maxBitrate {
  151. return fmt.Sprintf("You're streaming to Owncast at %dkbps but only broadcasting to your viewers at %dkbps, requiring unnecessary work to be performed and possible excessive CPU use. You may want to decrease what you're sending to Owncast or increase what you send to your viewers so the highest bitrate matches.", inboundBitrate, maxBitrate)
  152. }
  153. return ""
  154. }
  155. func cpuUsageHealthOverviewMessage() string {
  156. if len(metrics.CPUUtilizations) < 2 {
  157. return ""
  158. }
  159. recentCPUUses := metrics.CPUUtilizations[len(metrics.CPUUtilizations)-2:]
  160. values := make([]float64, len(recentCPUUses))
  161. for i, val := range recentCPUUses {
  162. values[i] = val.Value
  163. }
  164. recentCPUUse := utils.Avg(values)
  165. if recentCPUUse < maxCPUUsage {
  166. return ""
  167. }
  168. return fmt.Sprintf("The CPU usage on your server is over %d%%. This may cause video to be provided slower than necessary, causing buffering for your viewers. Consider increasing the resources available or reducing the number of output variants you made available.", maxCPUUsage)
  169. }
  170. func errorCountHealthOverviewMessage() string {
  171. totalNumberOfClients := len(windowedBandwidths)
  172. if totalNumberOfClients == 0 {
  173. return ""
  174. }
  175. clientsWithErrors := getClientsWithErrorsCount()
  176. if clientsWithErrors == 0 {
  177. return ""
  178. }
  179. // Only return these detailed values and messages if we feel we have enough
  180. // clients to be able to make a reasonable assessment. This is an arbitrary
  181. // number but 1 out of 1 isn't helpful.
  182. if totalNumberOfClients >= minClientCountForDetails {
  183. healthyPercentage := utils.IntPercentage(clientsWithErrors, totalNumberOfClients)
  184. isUsingPassthrough := false
  185. outputVariants := data.GetStreamOutputVariants()
  186. for _, variant := range outputVariants {
  187. if variant.IsVideoPassthrough {
  188. isUsingPassthrough = true
  189. }
  190. }
  191. if isUsingPassthrough {
  192. return fmt.Sprintf("%d of %d viewers (%d%%) are experiencing errors. You're currently using a video passthrough output, often known for causing playback issues for people. It is suggested you turn it off.", clientsWithErrors, totalNumberOfClients, healthyPercentage)
  193. }
  194. currentBroadcast := core.GetCurrentBroadcast()
  195. if currentBroadcast != nil && currentBroadcast.LatencyLevel.SecondsPerSegment < 3 {
  196. return fmt.Sprintf("%d of %d viewers (%d%%) may be experiencing some issues. You may want to increase your latency buffer level in your video configuration to see if it helps.", clientsWithErrors, totalNumberOfClients, healthyPercentage)
  197. }
  198. return fmt.Sprintf("%d of %d viewers (%d%%) may be experiencing some issues.", clientsWithErrors, totalNumberOfClients, healthyPercentage)
  199. }
  200. return ""
  201. }
  202. func getClientsWithErrorsCount() int {
  203. clientsWithErrors := 0
  204. for _, errors := range windowedErrorCounts {
  205. if errors > 0 {
  206. clientsWithErrors++
  207. }
  208. }
  209. return clientsWithErrors
  210. }
  211. func getClientErrorHeathyPercentage() int {
  212. totalNumberOfClients := len(windowedErrorCounts)
  213. if totalNumberOfClients == 0 {
  214. return -1
  215. }
  216. clientsWithErrors := getClientsWithErrorsCount()
  217. if clientsWithErrors == 0 {
  218. return 100
  219. }
  220. pct := 100 - utils.IntPercentage(clientsWithErrors, totalNumberOfClients)
  221. return pct
  222. }