New relic timeout handling (#14)

* Get data from API with handled timeout

* Add display history to config

* Fix and lint

* Update doc

Author: LinMAD

README.md

@@ -2,18 +2,37 @@
 
 [![Build Status](https://travis-ci.org/LinMAD/BitAccretion.svg?branch=master)](https://travis-ci.org/LinMAD/BitAccretion)
 
+#### Why it's exist?
 [Project plot on medium](https://medium.com/@artjomnemiro/how-valuable-can-be-visual-monitoring-923e9e865625)
 
 ```text
-TODO Restore feature with audio alerts -> plugin for kernel + config
-TODO Add more tests for critical code
 TODO Add system chart how it works
-*TIP
+Expl:
 
-main <-> kernel
-kernel <-> plugin - provider
-kernel <-> dashboard
+Main -> Loading
+Main -> Kernel
+Kernel -> Dashboard
+Kernel -> Processor
+Kernel -> Sound
 ```
 
-New dashboard prototype
-![](resource/example.png)
+#### About
+BitAccretion it's simple tool to aggregate metrics and visualize it. 
+
+Based on Go plugins and allows you to implement specific graph assemblers to show them in the terminal dashboard.
+
+The repository contains implemented New Relic API as a data provider but could be also based on Nginx logs aggregation or other sources. 
+
+#### What it can do?
+- Displaying terminal dashboard of graph(Systems and metrics) in charts representation.
+- Sound alerts with System name if audio file exist.
+
+#### Example how it's looks like
+![Demo example](./resource/example.gif)
+
+
+```text
+TODO GOTTY in docker container to access from browser
+TODO Remove clock and add degradation chart from exec time (Show error regression from exec time till now)
+TODO Add to dashboard name processor name (to displace source of data)
+```

config.json.tpl

@@ -2,6 +2,7 @@
   "sound_mode": true,
   "sound_alert_delay_min": 1,
   "log_level": 1,
+  "display_even_log_history": 500,
   "survey_interval_sec": 10,
   "interface_update_interval_sec": 1,
   "health_sensitivity": {

dashboard/announcer.go

@@ -12,14 +12,13 @@ import (
 	"github.com/mum4k/termdash/widgets/text"
 )
 
-const maxTextHistory = 100
-
 // AnnouncerHandler for dashboard
 type AnnouncerHandler struct {
 	name           string
 	t              *text.Text
-	historyCounter int8
+	historyCounter int16
 	sound          soundHandler
+	config         *model.Config
 }
 
 type soundHandler struct {
@@ -83,7 +82,7 @@ func (anon *AnnouncerHandler) WriteToEventLog(msg string, color cell.Color) {
 // handleHistory of logged messages
 func (anon *AnnouncerHandler) handleHistory() {
 	anon.historyCounter++
-	if anon.historyCounter <= maxTextHistory {
+	if anon.historyCounter <= anon.config.DisplayEvenLogHistory {
 		return
 	}
 
@@ -113,7 +112,7 @@ func (anon *AnnouncerHandler) playAlter(name string) {
 }
 
 // NewAnnouncerWidget creates and returns prepared widget
-func NewAnnouncerWidget(sound extension.ISound, delay int, name string) (*AnnouncerHandler, error) {
+func NewAnnouncerWidget(sound extension.ISound, c *model.Config, name string) (*AnnouncerHandler, error) {
 	t, tErr := text.New(text.WrapAtRunes(), text.WrapAtWords(), text.RollContent())
 	if tErr != nil {
 		return nil, tErr
@@ -122,12 +121,13 @@ func NewAnnouncerWidget(sound extension.ISound, delay int, name string) (*Announ
 	now := time.Now().UTC()
 
 	return &AnnouncerHandler{
-			name: name,
-			t:    t,
+			name:   name,
+			t:      t,
+			config: c,
 			sound: soundHandler{
 				player:               sound,
 				isAlertNeeded:        false,
-				soundAlertDelay:      time.Duration(delay) * time.Minute,
+				soundAlertDelay:      time.Duration(c.SoundAlertDelayMin) * time.Minute,
 				lastSoundTriggerTime: now.Add(-42 * time.Hour),
 			},
 		},

dashboard/dashboard.go

@@ -41,7 +41,7 @@ func (m *MonitoringDashboard) GetName() string {
 }
 
 // initWidgets for dashboard
-func (m *MonitoringDashboard) initWidgets(s extension.ISound, delay int, n []*model.Node) (err error) {
+func (m *MonitoringDashboard) initWidgets(s extension.ISound, c *model.Config, n []*model.Node) (err error) {
 	m.widgetCollection.reqSuccessful, err = NewBarWidget("ok_reqs_bar_widget", cell.ColorGreen, true, n)
 	if err != nil {
 		return err
@@ -52,12 +52,12 @@ func (m *MonitoringDashboard) initWidgets(s extension.ISound, delay int, n []*mo
 		return err
 	}
 
-	m.widgetCollection.reqAggregated, err = NewLineWidget("aggregated_reqs_line_widget")
+	m.widgetCollection.reqAggregated, err = NewLineWidget("aggregated_reqs_line_widget", c)
 	if err != nil {
 		return err
 	}
 
-	m.widgetCollection.eventLog, err = NewAnnouncerWidget(s, delay, "system_error_text_widget")
+	m.widgetCollection.eventLog, err = NewAnnouncerWidget(s, c, "system_error_text_widget")
 	if err != nil {
 		return err
 	}
@@ -121,15 +121,15 @@ func (m *MonitoringDashboard) createLayout(dashboardName string, t *terminalapi.
 }
 
 // NewMonitoringDashboard constructor, will prepare widgets, subscriber's and dependencies
-func NewMonitoringDashboard(n string, c *model.Config, s extension.ISound, t terminalapi.Terminal, g model.Graph) (*MonitoringDashboard, error) {
+func NewMonitoringDashboard(dashboardName string, c *model.Config, s extension.ISound, t terminalapi.Terminal, g model.Graph) (*MonitoringDashboard, error) {
 	termDash := &MonitoringDashboard{widgetCollection: &widgets{}}
 
-	initErr := termDash.initWidgets(s, c.SoundAlertDelayMin, g.GetAllVertices())
+	initErr := termDash.initWidgets(s, c, g.GetAllVertices())
 	if initErr != nil {
 		return nil, initErr
 	}
 
-	layoutErr := termDash.createLayout(n, &t)
+	layoutErr := termDash.createLayout(dashboardName, &t)
 	if layoutErr != nil {
 		return nil, layoutErr
 	}

dashboard/line.go

@@ -7,14 +7,12 @@ import (
 	"github.com/mum4k/termdash/widgets/linechart"
 )
 
-// maxPoints in line chart for one line (control visual overflow and data updates)
-const maxPoints = 50
-
 // SparkLineWidgetHandler for dashboard
 type SparkLineWidgetHandler struct {
-	name  string
-	lc    *linechart.LineChart
-	lines seriesData
+	name   string
+	lc     *linechart.LineChart
+	lines  seriesData
+	config *model.Config
 }
 
 // seriesData used to draw points in line chart
@@ -55,14 +53,14 @@ func (s *SparkLineWidgetHandler) updateLineData(g *model.Graph) {
 	var okPoints, badPoints []float64
 	nodes := g.GetAllVertices()
 
-	if len(s.lines.okData) >= maxPoints {
-		okPoints = s.lines.okData[1:maxPoints]
+	if len(s.lines.okData) >= int(s.config.DisplayEvenLogHistory) {
+		okPoints = s.lines.okData[1:int(s.config.DisplayEvenLogHistory)]
 	} else {
 		okPoints = s.lines.okData
 	}
 
-	if len(s.lines.badData) >= maxPoints {
-		badPoints = s.lines.badData[1:maxPoints]
+	if len(s.lines.badData) >= int(s.config.DisplayEvenLogHistory) {
+		badPoints = s.lines.badData[1:int(s.config.DisplayEvenLogHistory)]
 	} else {
 		badPoints = s.lines.badData
 	}
@@ -78,7 +76,7 @@ func (s *SparkLineWidgetHandler) updateLineData(g *model.Graph) {
 }
 
 // NewLineWidget creates and returns prepared widget
-func NewLineWidget(name string) (*SparkLineWidgetHandler, error) {
+func NewLineWidget(name string, c *model.Config) (*SparkLineWidgetHandler, error) {
 	lc, err := linechart.New(
 		linechart.AxesCellOpts(cell.FgColor(cell.ColorWhite)),
 		linechart.YLabelCellOpts(cell.FgColor(cell.ColorWhite)),
@@ -89,8 +87,9 @@ func NewLineWidget(name string) (*SparkLineWidgetHandler, error) {
 	}
 
 	widget := &SparkLineWidgetHandler{
-		name: name,
-		lc:   lc,
+		name:   name,
+		lc:     lc,
+		config: c,
 		lines: seriesData{
 			okData:  make([]float64, 0),
 			badData: make([]float64, 0),

extension/newrelic/provider.go

@@ -1,10 +1,12 @@
 package main
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"os"
 	"sync"
+	"time"
 
 	"github.com/LinMAD/BitAccretion/extension"
 	"github.com/LinMAD/BitAccretion/extension/newrelic/worker"
@@ -15,6 +17,8 @@ import (
 
 // NRConfig addition for config required for New Relic
 type NRConfig struct {
+	// SurveyIntervalSec for data updates
+	SurveyIntervalSec int `json:"survey_interval_sec"`
 	// APIKey of NewRelic
 	APIKey string `json:"api_key"`
 	// APPSets to survey in NewRelic
@@ -90,11 +94,45 @@ func (nr *ProviderNewRelic) DispatchGraph() (model.Graph, error) {
 func (nr *ProviderNewRelic) FetchNewData(log logger.ILogger) (model.Graph, error) {
 	g := nr.prepareGraph()
 
+	log.Debug(fmt.Sprintf("Harvesting data from NewRelic API..."))
+
+	ctx, cancel := context.WithCancel(context.Background())
+
+	// Get info from API with timeout
+	ticker := time.NewTicker(time.Duration(nr.Config.SurveyIntervalSec*2+1) * time.Second)
+	defer ticker.Stop()
+	isExecuted := false // all only one coroutine
+
+	for {
+		select {
+		default:
+			if isExecuted {
+				continue
+			}
+
+			isExecuted = true
+			go func(g *model.Graph) {
+				defer cancel()
+				nr.fetchMetricsWithGraph(g, log)
+			}(g)
+		case <-ticker.C:
+			cancel()
+		case <-ctx.Done():
+			return *g, nil
+		}
+	}
+}
+
+// ProvideHealth will return health of extension if New Relic API alive\reachable
+func (nr *ProviderNewRelic) ProvideHealth() model.HealthState {
+	return nr.pluginHealth
+}
+
+// fetchMetricsWithGraph from API and put in to graph
+func (nr *ProviderNewRelic) fetchMetricsWithGraph(g *model.Graph, log logger.ILogger) {
 	appList := g.GetAllVertices()
 	appCount := int8(len(appList))
 
-	log.Debug(fmt.Sprintf("Harvesting data from NewRelic API..."))
-
 	var wg sync.WaitGroup
 	var w int8
 	for w = 0; w < appCount; w++ {
@@ -126,13 +164,6 @@ func (nr *ProviderNewRelic) FetchNewData(log logger.ILogger) (model.Graph, error
 	}
 
 	wg.Wait()
-
-	return *g, nil
-}
-
-// ProvideHealth will return health of extension if New Relic API alive\reachable
-func (nr *ProviderNewRelic) ProvideHealth() model.HealthState {
-	return nr.pluginHealth
 }
 
 // prepareGraph from given configuration

extension/newrelic/worker/worker.go

@@ -49,6 +49,7 @@ func (w *RelicWorker) CollectApplicationHostMetrics(log logger.ILogger, appID st
 	hosts := w.relicClient.GetApplicationHost(appID)
 	hLen := len(hosts.AppsHosts)
 
+	// TODO Some where here could be dead lock, mb make timout with context cancel of all go threads
 	var wg sync.WaitGroup
 	for group := 0; group < hLen; group++ {
 		wg.Add(1)

model/config.go

@@ -14,4 +14,6 @@ type Config struct {
 	InterfaceUpdateIntervalSec int `json:"interface_update_interval_sec"`
 	// LogLevel message to display in event widget
 	LogLevel logger.LevelOfLog `json:"log_level"`
+	// DisplayEvenLogHistory limit to display rendered charts or logs
+	DisplayEvenLogHistory int16 `json:"display_even_log_history"`
 }