CP-308049: rrdview tool

Viewing RRDs produced by xcp-rrdd is difficult, because the format is incompatible with rrdtool.
rrdtool has a hardcoded limit of 20 char for RRD names for backward compat with its binary format.

Steps:
* given a directory of xml .gz files containing xcp-rrdd produced rrds
* invokes itself recursively with each file in turn using xargs -P (easy way to parallelize on OCaml 4)
* load all RRDs, and split them into separate files, allowing us to shorten many of their names without conflicts
* some names are still too long, there is a builtin translation table to shorten these
* once split an .rrd file is created using 'rrdtool restore'. This can further be queried/inspected/transformed by rrdtool as needed
* a .sh script is produced that can plot the RRD if desired.

There are many RRDs so plotting isn't done automatically yet.

RRDs contain min/avg/max usually, so this is drawn as a strong line at the average, and an area in a lighter color for min/max
(especially useful for historic data that has been aggregated).

Caveats:
* we don't know the unit name, that is part of the XAPI metadata, but not the XML apparently?
* separate plots are generated for separate intervals, it'd be nice to join all these into the same graph
* the visualization type is not the best for all RRDs, some might benefit from a smoother line, etc.
* for now the tool is just built, but not installed (that'll require a .spec change too and can be done later)

This is just a starting point to be able to visualize this data somehow, and we can improve the actual plotting later.

Signed-off-by: Edwin Török <[email protected]>

Author: edwintorok

ocaml/xcp-rrdd/bin/rrdview/dune

@@ -0,0 +1,17 @@
+(executable
+ (modes byte exe)
+ (name rrdview)
+ ;(public_name rrdview)
+ (libraries
+  threads
+  xapi-rrd.unix
+  bos.setup
+  astring
+  fpath
+  rresult
+  xmlm
+  tyre
+  xapi-rrd
+  result)
+ ;(package xapi-tools)
+ )

ocaml/xcp-rrdd/bin/rrdview/rrdgraph.ml

@@ -0,0 +1,83 @@
+open Rrd
+
+type vname = VName of string
+
+module Rpn = struct
+  module VDef = struct
+    (* see rrdgraph_rpn(3) *)
+    type t = vname * string
+
+    type op = vname -> t
+
+    let op kind vname = (vname, kind)
+
+    let maximum = op "MAXIMUM"
+
+    let minimum = op "MINIMUM"
+
+    let average = op "AVERAGE"
+
+    let stdev = op "STDEV"
+
+    let last = op "LAST"
+
+    let first = op "FIRST"
+
+    let total = op "TOTAL"
+
+    let percent = op "PERCENT"
+
+    let percentnan = op "PERCENTNAN"
+
+    let lsl_slope = op "LSLSLOPE"
+
+    let lsl_intercept = op "LSLSLINT"
+
+    let lsl_correlation = op "LSLCORREL"
+  end
+
+  module CDef = struct
+    type t = string Seq.t (* stores a serialized RPN expression *)
+
+    let to_string r = r |> List.of_seq |> String.concat ","
+
+    let vname (VName vname) = Seq.return vname
+
+    let value f = Printf.sprintf "%g" f |> Seq.return
+
+    (* reverse polish notation: arguments first, operator last *)
+
+    let opn op args = Seq.append (List.to_seq args |> Seq.concat) (Seq.return op)
+
+    let op1 op arg = opn op [arg]
+
+    let op2 op arg1 arg2 = opn op [arg1; arg2]
+
+    let op3 op arg1 arg2 arg3 = opn op [arg1; arg2; arg3]
+  end
+end
+
+module Data = struct
+  type t = string
+
+  (* see rrdgraph_data (3) *)
+
+  let def vname rrdfile rrd rra ds =
+    let step = Int64.mul rrd.timestep @@ Int64.of_int rra.rra_pdp_cnt in
+    ( VName vname
+    , String.concat ":"
+        [
+          "DEF"
+        ; vname ^ "=" ^ Fpath.to_string rrdfile
+        ; ds.ds_name
+        ; Rrd.cf_type_to_string rra.rra_cf
+        ; Printf.sprintf "step=%Lu" step
+        ]
+    )
+
+  let vdef vname (VName var, rpnvdefop) =
+    (VName vname, Printf.sprintf "CDEF:%s=%s,%s" vname var rpnvdefop)
+
+  let cdef vname rpn =
+    (VName vname, Printf.sprintf "CDEF:%s=%s" vname (Rpn.CDef.to_string rpn))
+end

ocaml/xcp-rrdd/bin/rrdview/rrdgraph.mli

@@ -0,0 +1,88 @@
+(** a variable name *)
+type vname
+
+module Rpn : sig
+  (** RPN expressions for VDEF statements, see [rrdgraph_rpn(3)] *)
+  module VDef : sig
+    (** an RPN expression for VDEF, see [rrdgraph_data(3)] *)
+    type t
+
+    (** a VDEF RPN expression, see [rrdgraph_rpn(3)] *)
+    type op = vname -> t
+
+    val maximum : op
+    (** see [rrdgraph_rpn(3)] *)
+
+    val minimum : op
+    (** see [rrdgraph_rpn(3)] *)
+
+    val average : op
+    (** see [rrdgraph_rpn(3)] *)
+
+    val stdev : op
+    (** see [rrdgraph_rpn(3)] *)
+
+    val last : op
+    (** see [rrdgraph_rpn(3)] *)
+
+    val first : op
+    (** see [rrdgraph_rpn(3)] *)
+
+    val total : op
+    (** see [rrdgraph_rpn(3)] *)
+
+    val percent : op
+    (** see [rrdgraph_rpn(3)] *)
+
+    val percentnan : op
+    (** see [rrdgraph_rpn(3)] *)
+
+    val lsl_slope : op
+    (** see [rrdgraph_rpn(3)] *)
+
+    val lsl_intercept : op
+    (** see [rrdgraph_rpn(3)] *)
+
+    val lsl_correlation : op
+    (** see [rrdgraph_rpn(3)] *)
+  end
+
+  module CDef : sig
+    (** an RPN expression for CDEF, see [rrdgraph_data(3)] *)
+    type t
+
+    val vname : vname -> t
+    (** [vname v] is [v] as an RPN expression *)
+
+    val value : float -> t
+    (** [value v] is [v] as an RPN expression *)
+
+    val op1 : string -> t -> t
+    (** [op1 op arg1] is [op arg1]. For valid operators see [rrdgraph_rpn(3)] *)
+
+    val op2 : string -> t -> t -> t
+    (** [op2 op arg1 arg2] is [op arg1 arg2]. For valid operators see [rrdgraph_rpn(3)] *)
+
+    val op3 : string -> t -> t -> t -> t
+    (** [op3 op arg1 arg2 arg3] is [op arg1 arg2 arg3]. For valid operators see [rrdgraph_rpn(3)] *)
+  end
+end
+
+module Data : sig
+  (** an rrd graph data definition, see [rrdgraph_data(3)] *)
+  type t
+
+  val def : string -> Fpath.t -> Rrd.rrd -> Rrd.rra -> Rrd.ds -> vname * t
+  (** [def vname rrdfile rrd rra datasource] is a [DEF] (see [rrdgraph_data(3)]) that loads
+      [datasource.ds_name] from the [rrdfile] and plots it according to the consolidation function in the
+      specified [rra] and timestep calculated based on [rrd]. This data can be refered to as [vname]
+      elsewhere. *)
+
+  val vdef : string -> Rpn.VDef.t -> vname * t
+  (** [vdef vname vdefrpn] defines [vname] through a [VDEF] (see [rrdgraph_data(3)]) using the
+  specified [vdefrpn] expression. Conversion to RPN form is handled internally. *)
+
+  val cdef : string -> Rpn.CDef.t -> vname * t
+  (** [cdef vname cdefrpn] defines [vname] through a [CDEF] (see [rrdgraph_data(3)]) using the
+  specified [cdefrpn] expression. Conversion to RPN form is handled internally. *)
+end