Skip to content
This repository was archived by the owner on Aug 2, 2023. It is now read-only.

Commit 25130fc

Browse files
committed
New relic timeout handling (#14)
* Get data from API with handled timeout * Add display history to config * Fix and lint * Update doc
1 parent 03b02c7 commit 25130fc

9 files changed

Lines changed: 97 additions & 44 deletions

File tree

README.md

Lines changed: 27 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,18 +2,37 @@
22

33
[![Build Status](https://travis-ci.org/LinMAD/BitAccretion.svg?branch=master)](https://travis-ci.org/LinMAD/BitAccretion)
44

5+
#### Why it's exist?
56
[Project plot on medium](https://medium.com/@artjomnemiro/how-valuable-can-be-visual-monitoring-923e9e865625)
67

78
```text
8-
TODO Restore feature with audio alerts -> plugin for kernel + config
9-
TODO Add more tests for critical code
109
TODO Add system chart how it works
11-
*TIP
10+
Expl:
1211
13-
main <-> kernel
14-
kernel <-> plugin - provider
15-
kernel <-> dashboard
12+
Main -> Loading
13+
Main -> Kernel
14+
Kernel -> Dashboard
15+
Kernel -> Processor
16+
Kernel -> Sound
1617
```
1718

18-
New dashboard prototype
19-
![](resource/example.png)
19+
#### About
20+
BitAccretion it's simple tool to aggregate metrics and visualize it.
21+
22+
Based on Go plugins and allows you to implement specific graph assemblers to show them in the terminal dashboard.
23+
24+
The repository contains implemented New Relic API as a data provider but could be also based on Nginx logs aggregation or other sources.
25+
26+
#### What it can do?
27+
- Displaying terminal dashboard of graph(Systems and metrics) in charts representation.
28+
- Sound alerts with System name if audio file exist.
29+
30+
#### Example how it's looks like
31+
![Demo example](./resource/example.gif)
32+
33+
34+
```text
35+
TODO GOTTY in docker container to access from browser
36+
TODO Remove clock and add degradation chart from exec time (Show error regression from exec time till now)
37+
TODO Add to dashboard name processor name (to displace source of data)
38+
```

config.json.tpl

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
"sound_mode": true,
33
"sound_alert_delay_min": 1,
44
"log_level": 1,
5+
"display_even_log_history": 500,
56
"survey_interval_sec": 10,
67
"interface_update_interval_sec": 1,
78
"health_sensitivity": {

dashboard/announcer.go

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -12,14 +12,13 @@ import (
1212
"github.com/mum4k/termdash/widgets/text"
1313
)
1414

15-
const maxTextHistory = 100
16-
1715
// AnnouncerHandler for dashboard
1816
type AnnouncerHandler struct {
1917
name string
2018
t *text.Text
21-
historyCounter int8
19+
historyCounter int16
2220
sound soundHandler
21+
config *model.Config
2322
}
2423

2524
type soundHandler struct {
@@ -83,7 +82,7 @@ func (anon *AnnouncerHandler) WriteToEventLog(msg string, color cell.Color) {
8382
// handleHistory of logged messages
8483
func (anon *AnnouncerHandler) handleHistory() {
8584
anon.historyCounter++
86-
if anon.historyCounter <= maxTextHistory {
85+
if anon.historyCounter <= anon.config.DisplayEvenLogHistory {
8786
return
8887
}
8988

@@ -113,7 +112,7 @@ func (anon *AnnouncerHandler) playAlter(name string) {
113112
}
114113

115114
// NewAnnouncerWidget creates and returns prepared widget
116-
func NewAnnouncerWidget(sound extension.ISound, delay int, name string) (*AnnouncerHandler, error) {
115+
func NewAnnouncerWidget(sound extension.ISound, c *model.Config, name string) (*AnnouncerHandler, error) {
117116
t, tErr := text.New(text.WrapAtRunes(), text.WrapAtWords(), text.RollContent())
118117
if tErr != nil {
119118
return nil, tErr
@@ -122,12 +121,13 @@ func NewAnnouncerWidget(sound extension.ISound, delay int, name string) (*Announ
122121
now := time.Now().UTC()
123122

124123
return &AnnouncerHandler{
125-
name: name,
126-
t: t,
124+
name: name,
125+
t: t,
126+
config: c,
127127
sound: soundHandler{
128128
player: sound,
129129
isAlertNeeded: false,
130-
soundAlertDelay: time.Duration(delay) * time.Minute,
130+
soundAlertDelay: time.Duration(c.SoundAlertDelayMin) * time.Minute,
131131
lastSoundTriggerTime: now.Add(-42 * time.Hour),
132132
},
133133
},

dashboard/dashboard.go

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ func (m *MonitoringDashboard) GetName() string {
4141
}
4242

4343
// initWidgets for dashboard
44-
func (m *MonitoringDashboard) initWidgets(s extension.ISound, delay int, n []*model.Node) (err error) {
44+
func (m *MonitoringDashboard) initWidgets(s extension.ISound, c *model.Config, n []*model.Node) (err error) {
4545
m.widgetCollection.reqSuccessful, err = NewBarWidget("ok_reqs_bar_widget", cell.ColorGreen, true, n)
4646
if err != nil {
4747
return err
@@ -52,12 +52,12 @@ func (m *MonitoringDashboard) initWidgets(s extension.ISound, delay int, n []*mo
5252
return err
5353
}
5454

55-
m.widgetCollection.reqAggregated, err = NewLineWidget("aggregated_reqs_line_widget")
55+
m.widgetCollection.reqAggregated, err = NewLineWidget("aggregated_reqs_line_widget", c)
5656
if err != nil {
5757
return err
5858
}
5959

60-
m.widgetCollection.eventLog, err = NewAnnouncerWidget(s, delay, "system_error_text_widget")
60+
m.widgetCollection.eventLog, err = NewAnnouncerWidget(s, c, "system_error_text_widget")
6161
if err != nil {
6262
return err
6363
}
@@ -121,15 +121,15 @@ func (m *MonitoringDashboard) createLayout(dashboardName string, t *terminalapi.
121121
}
122122

123123
// NewMonitoringDashboard constructor, will prepare widgets, subscriber's and dependencies
124-
func NewMonitoringDashboard(n string, c *model.Config, s extension.ISound, t terminalapi.Terminal, g model.Graph) (*MonitoringDashboard, error) {
124+
func NewMonitoringDashboard(dashboardName string, c *model.Config, s extension.ISound, t terminalapi.Terminal, g model.Graph) (*MonitoringDashboard, error) {
125125
termDash := &MonitoringDashboard{widgetCollection: &widgets{}}
126126

127-
initErr := termDash.initWidgets(s, c.SoundAlertDelayMin, g.GetAllVertices())
127+
initErr := termDash.initWidgets(s, c, g.GetAllVertices())
128128
if initErr != nil {
129129
return nil, initErr
130130
}
131131

132-
layoutErr := termDash.createLayout(n, &t)
132+
layoutErr := termDash.createLayout(dashboardName, &t)
133133
if layoutErr != nil {
134134
return nil, layoutErr
135135
}

dashboard/line.go

Lines changed: 12 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -7,14 +7,12 @@ import (
77
"github.com/mum4k/termdash/widgets/linechart"
88
)
99

10-
// maxPoints in line chart for one line (control visual overflow and data updates)
11-
const maxPoints = 50
12-
1310
// SparkLineWidgetHandler for dashboard
1411
type SparkLineWidgetHandler struct {
15-
name string
16-
lc *linechart.LineChart
17-
lines seriesData
12+
name string
13+
lc *linechart.LineChart
14+
lines seriesData
15+
config *model.Config
1816
}
1917

2018
// seriesData used to draw points in line chart
@@ -55,14 +53,14 @@ func (s *SparkLineWidgetHandler) updateLineData(g *model.Graph) {
5553
var okPoints, badPoints []float64
5654
nodes := g.GetAllVertices()
5755

58-
if len(s.lines.okData) >= maxPoints {
59-
okPoints = s.lines.okData[1:maxPoints]
56+
if len(s.lines.okData) >= int(s.config.DisplayEvenLogHistory) {
57+
okPoints = s.lines.okData[1:int(s.config.DisplayEvenLogHistory)]
6058
} else {
6159
okPoints = s.lines.okData
6260
}
6361

64-
if len(s.lines.badData) >= maxPoints {
65-
badPoints = s.lines.badData[1:maxPoints]
62+
if len(s.lines.badData) >= int(s.config.DisplayEvenLogHistory) {
63+
badPoints = s.lines.badData[1:int(s.config.DisplayEvenLogHistory)]
6664
} else {
6765
badPoints = s.lines.badData
6866
}
@@ -78,7 +76,7 @@ func (s *SparkLineWidgetHandler) updateLineData(g *model.Graph) {
7876
}
7977

8078
// NewLineWidget creates and returns prepared widget
81-
func NewLineWidget(name string) (*SparkLineWidgetHandler, error) {
79+
func NewLineWidget(name string, c *model.Config) (*SparkLineWidgetHandler, error) {
8280
lc, err := linechart.New(
8381
linechart.AxesCellOpts(cell.FgColor(cell.ColorWhite)),
8482
linechart.YLabelCellOpts(cell.FgColor(cell.ColorWhite)),
@@ -89,8 +87,9 @@ func NewLineWidget(name string) (*SparkLineWidgetHandler, error) {
8987
}
9088

9189
widget := &SparkLineWidgetHandler{
92-
name: name,
93-
lc: lc,
90+
name: name,
91+
lc: lc,
92+
config: c,
9493
lines: seriesData{
9594
okData: make([]float64, 0),
9695
badData: make([]float64, 0),

extension/newrelic/provider.go

Lines changed: 40 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
package main
22

33
import (
4+
"context"
45
"encoding/json"
56
"fmt"
67
"os"
78
"sync"
9+
"time"
810

911
"github.com/LinMAD/BitAccretion/extension"
1012
"github.com/LinMAD/BitAccretion/extension/newrelic/worker"
@@ -15,6 +17,8 @@ import (
1517

1618
// NRConfig addition for config required for New Relic
1719
type NRConfig struct {
20+
// SurveyIntervalSec for data updates
21+
SurveyIntervalSec int `json:"survey_interval_sec"`
1822
// APIKey of NewRelic
1923
APIKey string `json:"api_key"`
2024
// APPSets to survey in NewRelic
@@ -90,11 +94,45 @@ func (nr *ProviderNewRelic) DispatchGraph() (model.Graph, error) {
9094
func (nr *ProviderNewRelic) FetchNewData(log logger.ILogger) (model.Graph, error) {
9195
g := nr.prepareGraph()
9296

97+
log.Debug(fmt.Sprintf("Harvesting data from NewRelic API..."))
98+
99+
ctx, cancel := context.WithCancel(context.Background())
100+
101+
// Get info from API with timeout
102+
ticker := time.NewTicker(time.Duration(nr.Config.SurveyIntervalSec*2+1) * time.Second)
103+
defer ticker.Stop()
104+
isExecuted := false // all only one coroutine
105+
106+
for {
107+
select {
108+
default:
109+
if isExecuted {
110+
continue
111+
}
112+
113+
isExecuted = true
114+
go func(g *model.Graph) {
115+
defer cancel()
116+
nr.fetchMetricsWithGraph(g, log)
117+
}(g)
118+
case <-ticker.C:
119+
cancel()
120+
case <-ctx.Done():
121+
return *g, nil
122+
}
123+
}
124+
}
125+
126+
// ProvideHealth will return health of extension if New Relic API alive\reachable
127+
func (nr *ProviderNewRelic) ProvideHealth() model.HealthState {
128+
return nr.pluginHealth
129+
}
130+
131+
// fetchMetricsWithGraph from API and put in to graph
132+
func (nr *ProviderNewRelic) fetchMetricsWithGraph(g *model.Graph, log logger.ILogger) {
93133
appList := g.GetAllVertices()
94134
appCount := int8(len(appList))
95135

96-
log.Debug(fmt.Sprintf("Harvesting data from NewRelic API..."))
97-
98136
var wg sync.WaitGroup
99137
var w int8
100138
for w = 0; w < appCount; w++ {
@@ -126,13 +164,6 @@ func (nr *ProviderNewRelic) FetchNewData(log logger.ILogger) (model.Graph, error
126164
}
127165

128166
wg.Wait()
129-
130-
return *g, nil
131-
}
132-
133-
// ProvideHealth will return health of extension if New Relic API alive\reachable
134-
func (nr *ProviderNewRelic) ProvideHealth() model.HealthState {
135-
return nr.pluginHealth
136167
}
137168

138169
// prepareGraph from given configuration

extension/newrelic/worker/worker.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ func (w *RelicWorker) CollectApplicationHostMetrics(log logger.ILogger, appID st
4949
hosts := w.relicClient.GetApplicationHost(appID)
5050
hLen := len(hosts.AppsHosts)
5151

52+
// TODO Some where here could be dead lock, mb make timout with context cancel of all go threads
5253
var wg sync.WaitGroup
5354
for group := 0; group < hLen; group++ {
5455
wg.Add(1)

model/config.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,4 +14,6 @@ type Config struct {
1414
InterfaceUpdateIntervalSec int `json:"interface_update_interval_sec"`
1515
// LogLevel message to display in event widget
1616
LogLevel logger.LevelOfLog `json:"log_level"`
17+
// DisplayEvenLogHistory limit to display rendered charts or logs
18+
DisplayEvenLogHistory int16 `json:"display_even_log_history"`
1719
}

resource/example.gif

1.01 MB
Loading

0 commit comments

Comments
 (0)