From 7c9a49d35921e9c35ac963683f8ad20cd69b2415 Mon Sep 17 00:00:00 2001 From: lmchilton Date: Wed, 7 Feb 2024 17:22:39 -0500 Subject: [PATCH] Add tool pcp2openmetrics for converting pcp metrics to open metric format including three sets of revisions made from initial PR for pcp2openmetrics tool. Made several changes to qa tests to filter output. --- qa/1131 | 24 +- qa/1131.out | 7 + qa/1589 | 25 +- qa/1589.out | 13 +- qa/1827 | 71 +++ qa/1827.out | 23 + src/pcp2openmetrics/GNUmakefile | 46 ++ src/pcp2openmetrics/pcp2openmetrics.1 | 701 +++++++++++++++++++++++++ src/pcp2openmetrics/pcp2openmetrics.py | 559 ++++++++++++++++++++ src/pcp2openmetrics/pythonserver.py | 54 ++ 10 files changed, 1509 insertions(+), 14 deletions(-) create mode 100755 qa/1827 create mode 100644 qa/1827.out create mode 100644 src/pcp2openmetrics/GNUmakefile create mode 100644 src/pcp2openmetrics/pcp2openmetrics.1 create mode 100755 src/pcp2openmetrics/pcp2openmetrics.py create mode 100644 src/pcp2openmetrics/pythonserver.py diff --git a/qa/1131 b/qa/1131 index 599edbedab..298ef276f8 100755 --- a/qa/1131 +++ b/qa/1131 @@ -1,6 +1,6 @@ #!/bin/sh # PCP QA Test No. 1131 -# Exercise pcp2json. +# Exercise pcp2json and pcp2openmetrics. # # Copyright (c) 2017 Red Hat. # @@ -16,6 +16,7 @@ $python -c "from collections import OrderedDict" >/dev/null 2>&1 [ $? -eq 0 ] || _notrun "python collections OrderedDict module not installed" which pcp2json >/dev/null 2>&1 || _notrun "pcp2json not installed" +which pcp2openmetrics >/dev/null 2>&1 || _notrun "pcp2openmetrics not installed" status=1 # failure is the default! signal=$PCP_BINADM_DIR/pmsignal @@ -23,6 +24,9 @@ $sudo rm -rf $tmp $tmp.* $seq.full trap "cd $here; rm -rf $tmp.*; exit \$status" 0 1 2 3 15 A="$here/archives/rep" +hostname=`hostname` +machineid=`_machine_id` +domainid=`_domain_name` _archive_filter() { @@ -34,6 +38,19 @@ _value_filter() sed 's/\(\.[0-9]\)[0-9]*/\1/g' } +_filter_pcp2openmetrics() +{ + tee -a $here/$seq.full \ + | col -b \ + | sed \ + -e '/domainname=/ s/'$domainid'/DOMAINID/' \ + -e '/machineid=/ s/'$machineid'/MACHINEID/' \ + -e '/groupid=/ s/'$UID'/GROUPID/' \ + -e '/userid=/ s/'$UID'/USERID/' \ + -e '/hostname=/ s/'$hostname'/HOST/' \ + | LC_COLLATE=POSIX sort +} + # real QA test starts here echo "---" pcp2json -a $A -H -I -z "" | _archive_filter @@ -43,6 +60,11 @@ echo "---" pcp2json -a $A -H -I -Z UTC+0 -x "" | _archive_filter echo "---" pcp2json -a $A -H -I -z -X -b GB -P 2 -F $tmp.outfile "" +echo "---" +pcp2openmetrics -s1 -H -z hinv.ncpu | _filter_pcp2openmetrics +echo "---" + + cat $tmp.outfile | _archive_filter which json_verify > /dev/null 2>&1 if [ $? -eq 0 ]; then diff --git a/qa/1131.out b/qa/1131.out index eff0849143..6a18d03ce7 100644 --- a/qa/1131.out +++ b/qa/1131.out @@ -2266,6 +2266,13 @@ QA output created by 1131 } } --- +--- + + +# HELP hinv_ncpu number of CPUs in the system +# TYPE hinv_ncpu gauge +hinv_ncpu{domainname="DOMAINID",groupid="GROUPID",hostname="HOST",machineid="MACHINEID",userid="USERID",agent="linux"} 16 +--- { "@pcp": { "@hosts": [ diff --git a/qa/1589 b/qa/1589 index 28e2b00b12..2f85c931b4 100755 --- a/qa/1589 +++ b/qa/1589 @@ -3,7 +3,9 @@ # Exercise pcp2json HTTP POST functionality. # # Copyright (c) 2023 Red Hat. All Rights Reserved. -# +# + +#set -x seq=`basename $0` echo "QA output created by $seq" @@ -37,7 +39,9 @@ _filter_pcp2json_http() | sed \ -e "s/\(\"@host\"\): \""$hostname"\"/\1:HOST/g" \ -e "s/\(\"value\"\): \""$cpus"\"/\1:NCPU/g" \ - -e "s/\(\"@timestamp\"\): \"[1-9][0-9]*\"/\1:SECS/g" \ + -e 's/\(\"@timestamp\"\): \(.*\)/\1:DATE TIME/' \ + -e 's/- - \[[^]]*\]/- -[DATE]/g' \ + -e 's/.*- -/[IP ADDRESS]/g' \ -e "s/^\(Host: localhost\):$port/\1:PORT/g" \ -e 's/^\(Content-Length:\) [1-9][0-9]*/\1 SIZE/g' \ -e 's/^\(User-Agent: python-requests\).*/\1 VERSION/g' \ @@ -51,21 +55,20 @@ _filter_pcp2json_http() # real QA test starts here port=`_find_free_port` -nc -l localhost $port >$tmp.nc.out 2>$tmp.nc.err & -pid1=$! -sleep 2 # let nc start up - -# in case nc(1) does not exit by itself, e.g. on Ubuntu -( sleep 2; $signal $pid1 ) >>$seq.full 2>&1 & +$PCP_PYTHON_PROG $here/src/pythonserver.py $port &> $tmp.python.out & +pid=$! +sleep 2 # let server start up echo "pcp2json invocation" | tee -a $here/$seq.full -pcp2json -s1 -f%s -ZUTC --url http://localhost:$port/receive hinv.ncpu > $tmp.json.out 2> $tmp.json.err +pcp2json -s1 -ZUTC -u http://localhost:$port hinv.ncpu >$tmp.json.out 2>$tmp.json.err echo "pcp2json HTTP POST (sorted):" -_filter_pcp2json_http <$tmp.nc.out +_filter_pcp2json_http <$tmp.python.out + +($signal $pid ) >>$seq.full 2>&1 & echo "All diagnostics" >> $here/$seq.full -for i in $tmp.json.out $tmp.json.err $tmp.nc.out $tmp.nc.err +for i in $tmp.json.out $tmp.json.err $tmp.python.out do echo "=== $i ===" >>$here/$seq.full cat $i >>$here/$seq.full diff --git a/qa/1589.out b/qa/1589.out index 7509fb6f9d..d4a3d7d1a4 100644 --- a/qa/1589.out +++ b/qa/1589.out @@ -2,11 +2,15 @@ QA output created by 1589 pcp2json invocation pcp2json HTTP POST (sorted): + + + + "value":NCPU "ncpu": { } "@interval": "0", - "@timestamp":SECS, + "@timestamp":DATE TIME "hinv": { } { @@ -23,11 +27,16 @@ pcp2json HTTP POST (sorted): "@pcp": { } Accept: */* +Body: Content-Length: SIZE Content-Type: application/json +Headers: Host: localhost:PORT -POST /receive HTTP/1.1 +INFO:root:POST request, +INFO:root:Starting httpd... +Path: / User-Agent: python-requests VERSION +[IP ADDRESS][DATE] "POST / HTTP/1.1" 200 - { } diff --git a/qa/1827 b/qa/1827 new file mode 100755 index 0000000000..d63c853815 --- /dev/null +++ b/qa/1827 @@ -0,0 +1,71 @@ +#!/bin/sh +# PCP QA Test No. 1589 +# Exercise pcp2openmetrics HTTP POST functionality. +# +# Copyright (c) 2024 Red Hat. All Rights Reserved. +# + +#set -x + +seq=`basename $0` +echo "QA output created by $seq" + +. ./common.python + +which pcp2openmetrics >/dev/null 2>&1 || _notrun "pcp2openmetrics not installed" + +_cleanup() +{ + cd $here + $sudo rm -rf $tmp $tmp.* +} + +status=0 # success is the default! +cpus=`pmprobe -v hinv.ncpu | awk '{print $3}'` +hostname=`hostname` +machineid=`_machine_id` +domainid=`_domain_name` +$sudo rm -rf $tmp $tmp.* $seq.full +trap "_cleanup; exit \$status" 0 1 2 3 15 + + +_filter_pcp2openmetrics_http() +{ + tee -a $here/$seq.full \ + | col -b \ + | sed \ + -e '/domainname=/ s/'$domainid'/DOMAINID/' \ + -e '/machineid=/ s/'$machineid'/MACHINEID/' \ + -e '/groupid=/ s/'$UID'/GROUPID/' \ + -e '/userid=/ s/'$UID'/USERID/' \ + -e '/hostname=/ s/'$hostname'/HOST/' \ + -e 's/- - \[[^]]*\]/- -[DATE]/g' \ + -e 's/.*- -/[IP ADDRESS]/g' \ + -e "s/^\(Host: localhost\):$port/\1:PORT/g" \ + -e 's/^\(Content-Length:\) [1-9][0-9]*/\1 SIZE/g' \ + -e 's/^\(User-Agent: python-requests\).*/\1 VERSION/g' \ + -e 's/^\(Date:\).*/\1 DATE/g' \ + -e 's/\(\"context\":\) [0-9][0-9]*/\1 CTXID/g' \ + -e '/^Accept-Encoding: /d' \ + -e 's/\(\hostname=\): \""$hostname"\"/\1:HOST/g' \ + -e '/^Connection: keep-alive/d' \ + -e '/ using stream socket$/d' \ + | LC_COLLATE=POSIX sort +} + +# real QA test starts here +port=`_find_free_port` +$PCP_PYTHON_PROG $here/src/pythonserver.py $port &> $tmp.python.out & +pid=$! +sleep 2 # let server start up + +echo "pcp2openmetrics invocation" | tee -a $here/$seq.full +pcp2openmetrics -s1 -u http://localhost:$port hinv.ncpu >$tmp.openmetrics.out 2>$tmp.openmetrics.err + +echo "pcp2openmetrics HTTP POST (sorted):" +_filter_pcp2openmetrics_http <$tmp.python.out + +($signal $pid ) >>$seq.full 2>&1 & + +# success, all done +exit \ No newline at end of file diff --git a/qa/1827.out b/qa/1827.out new file mode 100644 index 0000000000..18f742dd1e --- /dev/null +++ b/qa/1827.out @@ -0,0 +1,23 @@ +QA output created by 1827 +pcp2openmetrics invocation +pcp2openmetrics HTTP POST (sorted): + + + + + + +# HELP hinv_ncpu number of CPUs in the system +# TYPE hinv_ncpu gauge +Accept: */* +Body: +Content-Length: SIZE +Content-Type: application/openmetrics-text +Headers: +Host: localhost:PORT +INFO:root:POST request, +INFO:root:Starting httpd... +Path: / +User-Agent: python-requests VERSION +[IP ADDRESS][DATE] "POST / HTTP/1.1" 200 - +hinv_ncpu{domainname="DOMAINID",groupid="GROUPID",hostname="HOST",machineid="MACHINEID",userid="USERID",agent="linux"} 16 diff --git a/src/pcp2openmetrics/GNUmakefile b/src/pcp2openmetrics/GNUmakefile new file mode 100644 index 0000000000..5cc9c2eb66 --- /dev/null +++ b/src/pcp2openmetrics/GNUmakefile @@ -0,0 +1,46 @@ +# +# Copyright (c) 2024 Red Hat. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# + +TOPDIR = ../.. +include $(TOPDIR)/src/include/builddefs + +TARGET = pcp2openmetrics +MAN_SECTION = 1 +MAN_PAGES = $(TARGET).$(MAN_SECTION) +MAN_DEST = $(PCP_MAN_DIR)/man$(MAN_SECTION) +BASHDIR = $(PCP_BASHSHARE_DIR)/completions + +default: $(TARGET).py $(MAN_PAGES) + +default: + +include $(BUILDRULES) + +install: default +ifeq "$(HAVE_PYTHON)" "true" + $(INSTALL) -m 755 $(TARGET).py $(PCP_BIN_DIR)/$(TARGET) + $(INSTALL) -S $(BASHDIR)/pcp $(BASHDIR)/$(TARGET) + @$(INSTALL_MAN) +endif + +default_pcp: default + +install_pcp: install + +check:: $(TARGET).py + $(PYLINT) $^ + +check :: $(MAN_PAGES) + $(MANLINT) $^ + diff --git a/src/pcp2openmetrics/pcp2openmetrics.1 b/src/pcp2openmetrics/pcp2openmetrics.1 new file mode 100644 index 0000000000..a30dbc00bf --- /dev/null +++ b/src/pcp2openmetrics/pcp2openmetrics.1 @@ -0,0 +1,701 @@ +'\"macro stdmacro +.\" +.\" Copyright (C) 2024 Lauren Chilton +.\" Copyright (C) 2024 Red Hat. +.\" +.\" This program is free software; you can redistribute it and/or modify it +.\" under the terms of the GNU General Public License as published by the +.\" Free Software Foundation; either version 2 of the License, or (at your +.\" option) any later version. +.\" +.\" This program is distributed in the hope that it will be useful, but +.\" WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +.\" or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +.\" for more details. +.\" +.\" +.TH PCP2OPENMETRICS 1 "PCP" "Performance Co-Pilot" +.SH NAME +\f3pcp2openmetrics\f1 \- pcp-to-openmetrics exporter +.SH SYNOPSIS +\fBpcp2openmetrics\fP +[\fB\-5CEGHIjLmnrRvVxXz?\fP] +[\fB\-4\fP \fIaction\fP] +[\fB\-8\fP|\fB\-9\fP \fIlimit\fP] +[\fB\-a\fP \fIarchive\fP] +[\fB\-A\fP \fIalign\fP] +[\fB\-\-archive\-folio\fP \fIfolio\fP] +[\fB\-b\fP|\fB\-B\fP \fIspace-scale\fP] +[\fB\-c\fP \fIconfig\fP] +[\fB\-\-container\fP \fIcontainer\fP] +[\fB\-\-daemonize\fP] +[\fB\-e\fP \fIderived\fP] +[\fB\-f\fP \fIformat\fP] +[\fB\-F\fP \fIoutfile\fP] +[\fB\-h\fP \fIhost\fP] +[\fB\-i\fP \fIinstances\fP] +[\fB\-J\fP \fIrank\fP] +[\fB\-K\fP \fIspec\fP] +[\fB\-N\fP \fIpredicate\fP] +[\fB\-o\fP \fItimeout\fP] +[\fB\-O\fP \fIorigin\fP] +[\fB\-p\fP \fIpassword\fP] +[\fB\-P\fP|\fB\-0\fP \fIprecision\fP] +[\fB\-q\fP|\fB\-Q\fP \fIcount-scale\fP] +[\fB\-s\fP \fIsamples\fP] +[\fB\-S\fP \fIstarttime\fP] +[\fB\-t\fP \fIinterval\fP] +[\fB\-T\fP \fIendtime\fP] +[\fB\-u\fP \fIurl\fP] +[\fB\-U\fP \fIusername\fP] +[\fB\-y\fP|\fB\-Y\fP \fItime-scale\fP] +[\fB\-Z\fP \fItimezone\fP] +\fImetricspec\fP +[...] +.SH DESCRIPTION +.B pcp2openmetrics +is a customizable performance metrics exporter tool from PCP to +Open Metrics - +.I https://openmetrics.io +- format. +Any available performance metric, live or archived, system and/or +application, can be selected for exporting using either command line +arguments or a configuration file. +.PP +.B pcp2openmetrics +is a close relative of +.BR pmrep (1). +Refer to +.BR pmrep (1) +for the +.I metricspec +description accepted on +.B pcp2openmetrics +command line. +See +.BR pmrep.conf (5) +for description of the +.B pcp2openmetrics.conf +configuration file syntax. +This page describes +.B pcp2openmetrics +specific options and configuration file differences with +.BR pmrep.conf (5). +.BR pmrep (1) +also lists some usage examples of which most are applicable with +.B pcp2openmetrics +as well. +.PP +Only the command line options listed on this page are supported, +other options available for +.BR pmrep (1) +are not supported. +.PP +Options via environment values (see +.BR pmGetOptions (3)) +override the corresponding built-in default values (if any). +Configuration file options override the corresponding +environment variables (if any). +Command line options override the corresponding configuration +file options (if any). +.SH CONFIGURATION FILE +.B pcp2openmetrics +uses a configuration file with syntax described in +.BR pmrep.conf (5). +The following options are common with +.BR pmrep.conf : +.BR version , +.BR source , +.BR speclocal , +.BR derived , +.BR header , +.BR globals , +.BR samples , +.BR interval , +.BR type , +.BR type_prefer , +.BR ignore_incompat , +.BR names_change , +.BR instances , +.BR live_filter , +.BR rank , +.BR limit_filter , +.BR limit_filter_force , +.BR invert_filter , +.BR predicate , +.BR omit_flat , +.BR include_labels , +.BR precision , +.BR precision_force , +.BR count_scale , +.BR count_scale_force , +.BR space_scale , +.BR space_scale_force , +.BR time_scale , +.BR time_scale_force . +The rest of the +.B pmrep.conf +options are recognized but ignored for compatibility. +.SS pcp2openmetrics specific options +everything (boolean) +.RS 4 +Write everything known about metrics, including PCP internal IDs. +Labels are, however, omitted for backward compatibility. +Enable \fBinclude_labels\fP to include them as well. +Corresponding command line option is \fB\-X\fP. +Defaults to \fBno\fP. +.RE +.PP +exact_types (boolean) +.RS 4 +Write numbers as number data types, not as strings, potentially +losing some precision. +Corresponding command line option is \fB\-E\fP. +Defaults to \fBno\fP. +.RE +.PP +url (string) +.RS 4 +Send OPENMETRICS output as a HTTP POST to the given \fBurl\fP. +Corresponding command line option is \fB\-u\fP. +Defaults to \fBNone\fP. +.RE +.PP +http_pass (string) +.RS 4 +Use given password for Basic Authentication when sending a HTTP POST. +Corresponding command line option is \fB\-p\fP. +Defaults to \fBNone\fP. +.RE +.PP +http_user (string) +.RS 4 +Use given username for Basic Authentication when sending a HTTP POST. +Corresponding command line option is \fB\-U\fP. +Defaults to \fBNone\fP. +.RE +.PP +http_timeout (number) +.RS 4 +Maximum time (in seconds) when sending a HTTP POST. +Corresponding command line option is \fB\-o\fP. +Defaults to \fB2.5\fP seconds. +.RE +.SH OPTIONS +The available command line options are: +.TP 5 +\fB\-0\fR \fIprecision\fR, \fB\-\-precision\-force\fR=\fIprecision\fR +Like +.B \-P +but this option \fIwill\fP override per-metric specifications. +.TP +\fB\-4\fR \fIaction\fR, \fB\-\-names\-change\fR=\fIaction\fR +Specify which +.I action +to take on receiving a metric names change event during sampling. +These events occur when a PMDA discovers new metrics sometime +after starting up, and informs running client tools like +.BR pcp2openmetrics . +Valid values for +.I action +are \fBupdate\fP (refresh metrics being sampled), +\fBignore\fP (do nothing \- the default behaviour) +and \fBabort\fP (exit the program if such an event occurs). +.TP +\fB\-5\fR, \fB\-\-ignore\-unknown\fR +Silently ignore any metric name that cannot be resolved. +At least one metric must be found for the tool to start. +.TP +\fB\-8\fR \fIlimit\fR, \fB\-\-limit\-filter\fR=\fIlimit\fR +Limit results to instances with values above/below +.IR limit . +A positive integer will include instances with values +at or above the limit in reporting. +A negative integer will include instances with values +at or below the limit in reporting. +A value of zero performs no limit filtering. +This option will \fInot\fP override possible per-metric specifications. +See also +.BR \-J " and " +.BR \-N . +.TP +\fB\-9\fR \fIlimit\fR, \fB\-\-limit\-filter\-force\fR=\fIlimit\fR +Like +.B \-8 +but this option \fIwill\fP override per-metric specifications. +.TP +\fB\-a\fR \fIarchive\fR, \fB\-\-archive\fR=\fIarchive\fR +Performance metric values are retrieved from the set of Performance +Co-Pilot (PCP) archive files identified by the +.I archive +argument, which is a comma-separated list of names, each +of which may be the base name of an archive or the name of +a directory containing one or more archives. +.TP +\fB\-A\fR \fIalign\fR, \fB\-\-align\fR=\fIalign\fR +Force the initial sample to be +aligned on the boundary of a natural time unit +.IR align . +Refer to +.BR PCPIntro (1) +for a complete description of the syntax for +.IR align . +.TP +\fB\-\-archive\-folio\fR=\fIfolio\fR +Read metric source archives from the PCP archive +.I folio +created by tools like +.BR pmchart (1) +or, less often, manually with +.BR mkaf (1). +.TP +\fB\-b\fR \fIscale\fR, \fB\-\-space\-scale\fR=\fIscale\fR +.I Unit/scale +for space (byte) metrics, possible values include +.BR bytes , +.BR Kbytes , +.BR KB , +.BR Mbytes , +.BR MB , +and so forth. +This option will \fInot\fP override possible per-metric specifications. +See also +.BR pmParseUnitsStr (3). +.TP +\fB\-B\fR \fIscale\fR, \fB\-\-space\-scale\-force\fR=\fIscale\fR +Like +.B \-b +but this option \fIwill\fP override per-metric specifications. +.TP +\fB\-c\fR \fIconfig\fR, \fB\-\-config\fR=\fIconfig\fR +Specify the +.I config +file or directory to use. +In case \fIconfig\fP is a directory all files in it ending +\fB.conf\fR will be included. +The default is the first found of: +.IR ./pcp2openmetrics.conf , +.IR \f(CR$HOME\fP/.pcp2openmetrics.conf , +.IR \f(CR$HOME\fP/pcp/pcp2openmetrics.conf , +and +.IR \f(CR$PCP_SYSCONF_DIR\fP/pcp2openmetrics.conf . +For details, see the above section and +.BR pmrep.conf (5). +.TP +\fB\-\-container\fR=\fIcontainer\fR +Fetch performance metrics from the specified +.IR container , +either local or remote (see +.BR \-h ). +.TP +\fB\-C\fR, \fB\-\-check\fR +Exit before reporting any values, but after parsing the configuration +and metrics and printing possible headers. +.TP +.B \-\-daemonize +Daemonize on startup. +.TP +\fB\-e\fR \fIderived\fR, \fB\-\-derived\fR=\fIderived\fR +Specify +.I derived +performance metrics. +If +.I derived +starts with a slash (``/'') or with a dot (``.'') it will be +interpreted as a PCP derived metrics configuration file, otherwise it will +be interpreted as comma- or semicolon-separated derived metric expressions. +For complete description of derived metrics and PCP derived metrics +configuration files see +.BR pmLoadDerivedConfig (3) +and +.BR pmRegisterDerived (3). +Alternatively, using +.BR pmrep.conf (5) +configuration syntax allows defining derived metrics as part of metricsets. +.TP +\fB\-E\fR, \fB\-\-exact\-types\fR +Write numbers as number data types, not as strings, potentially +losing some precision. +.TP +\fB\-f\fR \fIformat\fR, \fB\-\-timestamp\-format\fR=\fIformat\fR +Use the +.I format +string for formatting the timestamp. +The format will be used with Python's +.B datetime.strftime +method which is mostly the same as that described in +.BR strftime (3). +The default is +.BR "%Y-%m-%d %H:%M:%S" . +.TP +\fB\-F\fR \fIoutfile\fR, \fB\-\-output\-file\fR=\fIoutfile\fR +Specify the output file +.IR outfile . +.TP +\fB\-G\fR, \fB\-\-no\-globals\fR +Do not include global metrics in reporting (see +.BR pmrep.conf (5)). +.TP +\fB\-h\fR \fIhost\fR, \fB\-\-host\fR=\fIhost\fR +Fetch performance metrics from +.BR pmcd (1) +on +.IR host , +rather than from the default localhost. +.TP +\fB\-H\fR, \fB\-\-no\-header\fR +Do not print any headers. +.TP +\fB\-i\fR \fIinstances\fR, \fB\-\-instances\fR=\fIinstances\fR +Retrieve and report only the specified metric +.IR instances . +By default all instances, present and future, are reported. +.RS +.PP +Refer to +.BR pmrep (1) +for complete description of this option. +.RE +.TP +\fB\-I\fR, \fB\-\-ignore\-incompat\fR +Ignore incompatible metrics. +By default incompatible metrics (that is, +their type is unsupported or they cannot be scaled as requested) +will cause +.B pcp2openmetrics +to terminate with an error message. +With this option all incompatible metrics are silently omitted +from reporting. +This may be especially useful when requesting +non-leaf nodes of the PMNS tree for reporting. +.TP +\fB\-j\fR, \fB\-\-live\-filter\fR +Perform instance live filtering. +This allows capturing all named instances even if processes +are restarted at some point (unlike without live filtering). +Performing live filtering over a huge number of instances will add +some internal overhead so a bit of user caution is advised. +See also +.BR \-n . +.TP +\fB\-J\fR \fIrank\fR, \fB\-\-rank\fR=\fIrank\fR +Limit results to highest/lowest +.IR rank ed +instances of set-valued metrics. +A positive integer will include highest valued instances in reporting. +A negative integer will include lowest valued instances in reporting. +A value of zero performs no ranking. +Ranking does not imply sorting, see +.BR \-6 . +See also +.BR \-8 . +.TP +\fB\-K\fR \fIspec\fR, \fB\-\-spec\-local\fR=\fIspec\fR +When fetching metrics from a local context (see +.BR \-L ), +the +.B \-K +option may be used to control the DSO PMDAs that should be made accessible. +The +.I spec +argument conforms to the syntax described in +.BR pmSpecLocalPMDA (3). +More than one +.B \-K +option may be used. +.TP +\fB\-L\fR, \fB\-\-local\-PMDA\fR +Use a local context to collect metrics from DSO PMDAs on the local host +without PMCD. +See also +.BR \-K . +.TP +\fB\-n\fR, \fB\-\-invert\-filter\fR +Perform ranking before live filtering. +By default instance live filtering (when requested, see +.BR \-j ) +happens before instance ranking (when requested, see +.BR \-J ). +With this option the logic is inverted and ranking happens before +live filtering. +.TP +\fB\-m\fR, \fB\-\-include\-labels\fR +Include PCP metric labels in the output. +.TP +\fB\-N\fR \fIpredicate\fR, \fB\-\-predicate\fR=\fIpredicate\fR +Specify a comma-separated list of +.I predicate +filter reference metrics. +By default ranking (see +.BR \-J ) +happens for each metric individually. +With predicates, ranking is done only for the +specified predicate metrics. +When reporting, rest of the metrics sharing the same +.I instance domain +(see +.BR PCPIntro (1)) +as the predicate will include only the highest/lowest ranking +instances of the corresponding predicate. +Ranking does not imply sorting, see +.BR \-6 . +.RS +.PP +So for example, using \fBproc.memory.rss\fP +(resident memory size of process) +as the +.I predicate +metric together with \fBproc.io.total_bytes\fP and \fBmem.util.used\fP as +metrics to be reported, only the processes using most/least (as per +.BR \-J ) +memory will be included when reporting total bytes written by processes. +Since \fBmem.util.used\fP is a single-valued metric (thus not sharing the +same instance domain as the process related metrics), +it will be reported as usual. +.RE +.TP +\fB\-o\fR, \fB\-\-http-timeout\fR +Timeout (in seconds) when sending a HTTP POST with the +.BR \-u +option. +Default value is \fB2.5\fP seconds. +.TP +\fB\-O\fR \fIorigin\fR, \fB\-\-origin\fR=\fIorigin\fR +When reporting archived metrics, start reporting at +.I origin +within the time window (see +.B \-S +and +.BR \-T ). +Refer to +.BR PCPIntro (1) +for a complete description of the syntax for +.IR origin . +.TP +\fB\-p\fR, \fB\-\-http-pass\fR +Password when using HTTP basic authentication with the +.BR \-u +option. +.TP +\fB\-P\fR \fIprecision\fR, \fB\-\-precision\fR=\fIprecision\fR +Use +.I precision +for numeric non-integer output values. +The default is to use 3 decimal places (when applicable). +This option will \fInot\fP override possible per-metric specifications. +.TP +\fB\-q\fR \fIscale\fR, \fB\-\-count\-scale\fR=\fIscale\fR +.I Unit/scale +for count metrics, possible values include +.BR "count x 10^\-1" , +.BR "count" , +.BR "count x 10" , +.BR "count x 10^2" , +and so forth from +.B 10^\-8 +to +.BR 10^7 . +.\" https://bugzilla.redhat.com/show_bug.cgi?id=1264124 +(These values are currently space-sensitive.) +This option will \fInot\fP override possible per-metric specifications. +See also +.BR pmParseUnitsStr (3). +.TP +\fB\-Q\fR \fIscale\fR, \fB\-\-count\-scale\-force\fR=\fIscale\fR +Like +.B \-q +but this option \fIwill\fP override per-metric specifications. +.TP +\fB\-r\fR, \fB\-\-raw\fR +Output raw metric values, do not convert cumulative counters to rates. +This option \fIwill\fP override possible per-metric specifications. +.TP +\fB\-R\fR, \fB\-\-raw\-prefer\fR +Like +.B \-r +but this option will \fInot\fP override per-metric specifications. +.TP +\fB\-s\fR \fIsamples\fR, \fB\-\-samples\fR=\fIsamples\fR +The +.I samples +argument defines the number of samples to be retrieved and reported. +If +.I samples +is 0 or +.B \-s +is not specified, +.B pcp2openmetrics +will sample and report continuously (in real time mode) or until the end +of the set of PCP archives (in archive mode). +See also +.BR \-T . +.TP +\fB\-S\fR \fIstarttime\fR, \fB\-\-start\fR=\fIstarttime\fR +When reporting archived metrics, the report will be restricted to those +records logged at or after +.IR starttime . +Refer to +.BR PCPIntro (1) +for a complete description of the syntax for +.IR starttime . +.TP +\fB\-t\fR \fIinterval\fR, \fB\-\-interval\fR=\fIinterval\fR +Set the reporting +.I interval +to something other than the default 1 second. +The +.I interval +argument follows the syntax described in +.BR PCPIntro (1), +and in the simplest form may be an unsigned integer +(the implied units in this case are seconds). +See also the +.B \-T +option. +.TP +\fB\-T\fR \fIendtime\fR, \fB\-\-finish\fR=\fIendtime\fR +When reporting archived metrics, the report will be restricted to those +records logged before or at +.IR endtime . +Refer to +.BR PCPIntro (1) +for a complete description of the syntax for +.IR endtime . +.RS +.PP +When used to define the runtime before \fBpcp2openmetrics\fP will exit, +if no \fIsamples\fP is given (see \fB\-s\fP) then the number of +reported samples depends on \fIinterval\fP (see \fB\-t\fP). +If +.I samples +is given then +.I interval +will be adjusted to allow reporting of +.I samples +during runtime. +In case all of +.BR \-T , +.BR \-s , +and +.B \-t +are given, +.I endtime +determines the actual time +.B pcp2openmetrics +will run. +.RE +.TP +\fB\-u\fR, \fB\-\-url\fR +URL for sending an HTTP POST (instead of default standard output). +.TP +\fB\-U\fR, \fB\-\-http-user\fR +Username when using HTTP basic authentication with the +.BR \-u +option. +.TP +\fB\-v\fR, \fB\-\-omit\-flat\fR +Report only set-valued metrics with instances (e.g. disk.dev.read) and +omit single-valued ``flat'' metrics without instances (e.g. +kernel.all.sysfork). +See +.B \-i +and +.BR \-I . +.TP +\fB\-V\fR, \fB\-\-version\fR +Display version number and exit. +.TP +\fB\-x\fR, \fB\-\-with\-extended\fR +Write extended information. +.TP +\fB\-X\fR, \fB\-\-with\-everything\fR +Write everything known about metrics, including PCP internal IDs. +Labels are, however, omitted for backward compatibility, +use \fB\-m\fP to include them as well. +.TP +\fB\-y\fR \fIscale\fR, \fB\-\-time\-scale\fR=\fIscale\fR +.I Unit/scale +for time metrics, possible values include +.BR nanosec , +.BR ns , +.BR microsec , +.BR us , +.BR millisec , +.BR ms , +and so forth up to +.BR hour , +.BR hr . +This option will \fInot\fP override possible per-metric specifications. +See also +.BR pmParseUnitsStr (3). +.TP +\fB\-Y\fR \fIscale\fR, \fB\-\-time\-scale\-force\fR=\fIscale\fR +Like +.B \-y +but this option \fIwill\fP override per-metric specifications. +.TP +\fB\-z\fR, \fB\-\-hostzone\fR +Use the local timezone of the host that is the source of the +performance metrics, as identified by either the +.B \-h +or the +.B \-a +options. +The default is to use the timezone of the local host. +.TP +\fB\-Z\fR \fItimezone\fR, \fB\-\-timezone\fR=\fItimezone\fR +Use +.I timezone +for the date and time. +.I Timezone +is in the format of the environment variable +.B TZ +as described in +.BR environ (7). +Note that when including a timezone string in output, ISO 8601 -style +UTC offsets are used (so something like \-Z EST+5 will become UTC-5). +.TP +\fB\-?\fR, \fB\-\-help\fR +Display usage message and exit. +.SH FILES +.TP 5 +.I pcp2openmetrics.conf +\fBpcp2openmetrics\fP configuration file (see \fB\-c\fP) +.TP +.I \f(CR$PCP_SYSCONF_DIR\fP/pmrep/*.conf +system provided default \fBpmrep\fP configuration files +.SH PCP ENVIRONMENT +Environment variables with the prefix \fBPCP_\fP are used to parameterize +the file and directory names used by PCP. +On each installation, the +file \fI/etc/pcp.conf\fP contains the local values for these variables. +The \fB$PCP_CONF\fP variable may be used to specify an alternative +configuration file, as described in \fBpcp.conf\fP(5). +.PP +For environment variables affecting PCP tools, see \fBpmGetOptions\fP(3). +.SH SEE ALSO +.BR PCPIntro (1), +.BR mkaf (1), +.BR pcp (1), +.BR pcp2elasticsearch (1), +.BR pcp2graphite (1), +.BR pcp2influxdb (1), +.BR pcp2spark (1), +.BR pcp2xlsx (1), +.BR pcp2xml (1), +.BR pcp2json (1), +.BR pcp2zabbix (1), +.BR pmcd (1), +.BR pminfo (1), +.BR pmrep (1), +.BR pmGetOptions (3), +.BR pmLoadDerivedConfig (3), +.BR pmParseUnitsStr (3), +.BR pmRegisterDerived (3), +.BR pmSpecLocalPMDA (3), +.BR LOGARCHIVE (5), +.BR pcp.conf (5), +.BR pmrep.conf (5), +.BR PMNS (5) +and +.BR environ (7). diff --git a/src/pcp2openmetrics/pcp2openmetrics.py b/src/pcp2openmetrics/pcp2openmetrics.py new file mode 100755 index 0000000000..1c8469a4dd --- /dev/null +++ b/src/pcp2openmetrics/pcp2openmetrics.py @@ -0,0 +1,559 @@ +#!/usr/bin/env pmpython +# +# Copyright (C) 2024 Lauren Chilton +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. + + +""" PCP to OPENMETRICS Bridge """ + +# Common imports +from collections import OrderedDict +import errno +import time +import sys + +# Our imports +import requests +import os +import cpmapi + +# PCP Python PMAPI +from pcp import pmapi, pmconfig +from cpmapi import PM_CONTEXT_ARCHIVE, PM_INDOM_NULL, PM_IN_NULL, PM_DEBUG_APPL1, PM_TIME_SEC + +# Default config +DEFAULT_CONFIG = ["./pcp2openmetrics.conf", "$HOME/.pcp2openmetrics.conf", "$HOME/.pcp/pcp2openmetrics.conf", "$PCP_SYSCONF_DIR/pcp2openmetrics.conf"] + +# Defaults +CONFVER = 1 +INDENT = 2 +TIMEFMT = "%Y-%m-%d %H:%M:%S" +TIMEOUT = 2.5 # seconds + +class PCP2OPENMETRICS(object): + """ PCP to OPENMETRICS """ + def __init__(self): + """ Construct object, prepare for command line handling """ + self.context = None + self.daemonize = 0 + self.pmconfig = pmconfig.pmConfig(self) + self.opts = self.options() + + # Configuration directives + self.keys = ('source', 'output', 'derived', 'header', 'globals', + 'samples', 'interval', 'type', 'precision', 'daemonize', + 'timefmt', 'everything', + 'count_scale', 'space_scale', 'time_scale', 'version', + 'count_scale_force', 'space_scale_force', 'time_scale_force', + 'type_prefer', 'precision_force', 'limit_filter', 'limit_filter_force', + 'live_filter', 'rank', 'invert_filter', 'predicate', 'names_change', + 'speclocal', 'instances', 'ignore_incompat', 'ignore_unknown', + 'omit_flat', 'include_labels', 'url', 'http_user', 'http_pass', + 'http_timeout') + + # Ignored for pmrep(1) compatibility + self.keys_ignore = ( + 'timestamp', 'unitinfo', 'colxrow', 'separate_header', 'fixed_header', + 'delay', 'width', 'delimiter', 'extcsv', 'width_force', + 'extheader', 'repeat_header', 'interpol', + 'dynamic_header', 'overall_rank', 'overall_rank_alt', 'sort_metric', + 'instinfo', 'include_texts') + + # The order of preference for options (as present): + # 1 - command line options + # 2 - options from configuration file(s) + # 3 - built-in defaults defined below + self.check = 0 + self.version = CONFVER + self.source = "local:" + self.output = None # For pmrep conf file compat only + self.speclocal = None + self.derived = None + self.header = 1 + self.globals = 1 + self.samples = None # forever + self.interval = pmapi.timeval(10) # 10 sec + self.opts.pmSetOptionInterval(str(10)) # 10 sec + self.delay = 0 + self.type = 0 + self.type_prefer = self.type + self.ignore_incompat = 0 + self.ignore_unknown = 0 + self.names_change = 0 # ignore + self.instances = [] + self.live_filter = 0 + self.rank = 0 + self.limit_filter = 0 + self.limit_filter_force = 0 + self.invert_filter = 0 + self.predicate = None + self.omit_flat = 0 + self.include_labels = 0 + self.precision = 3 # .3f + self.precision_force = None + self.timefmt = TIMEFMT + self.interpol = 0 + self.count_scale = None + self.count_scale_force = None + self.space_scale = None + self.space_scale_force = None + self.time_scale = None + self.time_scale_force = None + + # Not in pcp2openmetrics.conf, won't overwrite + self.outfile = None + + self.everything = 0 + self.url = None + self.http_user = None + self.http_pass = None + self.http_timeout = TIMEOUT + + # Internal + self.runtime = -1 + + self.data = None + self.prev_ts = None + self.writer = None + + # Performance metrics store + # key - metric name + # values - 0:txt label, 1:instance(s), 2:unit/scale, 3:type, + # 4:width, 5:pmfg item, 6:precision, 7:limit + self.metrics = OrderedDict() + self.pmfg = None + self.pmfg_ts = None + + # Read configuration and prepare to connect + self.config = self.pmconfig.set_config_path(DEFAULT_CONFIG) + self.pmconfig.read_options() + self.pmconfig.read_cmd_line() + self.pmconfig.prepare_metrics() + self.pmconfig.set_signal_handler() + + def options(self): + """ Setup default command line argument option handling """ + opts = pmapi.pmOptions() + opts.pmSetOptionCallback(self.option) + opts.pmSetOverrideCallback(self.option_override) + opts.pmSetShortOptions("a:h:LK:c:Ce:D:V?HGA:S:T:O:s:t:rRIi:jJ:4:58:9:nN:vmP:0:q:b:y:Q:B:Y:F:f:Z:zXo:p:U:u:") + opts.pmSetShortUsage("[option...] metricspec [...]") + + opts.pmSetLongOptionHeader("General options") + opts.pmSetLongOptionArchive() # -a/--archive + opts.pmSetLongOptionArchiveFolio() # --archive-folio + opts.pmSetLongOptionContainer() # --container + opts.pmSetLongOptionHost() # -h/--host + opts.pmSetLongOptionLocalPMDA() # -L/--local-PMDA + opts.pmSetLongOptionSpecLocal() # -K/--spec-local + opts.pmSetLongOption("config", 1, "c", "FILE", "config file path") + opts.pmSetLongOption("check", 0, "C", "", "check config and metrics and exit") + opts.pmSetLongOption("output-file", 1, "F", "OUTFILE", "output file") + opts.pmSetLongOption("derived", 1, "e", "FILE|DFNT", "derived metrics definitions") + opts.pmSetLongOption("daemonize", 0, "", "", "daemonize on startup") + opts.pmSetLongOptionDebug() # -D/--debug + opts.pmSetLongOptionVersion() # -V/--version + opts.pmSetLongOptionHelp() # -?/--help + + opts.pmSetLongOptionHeader("Reporting options") + opts.pmSetLongOption("no-header", 0, "H", "", "omit headers") + opts.pmSetLongOption("no-globals", 0, "G", "", "omit global metrics") + opts.pmSetLongOptionAlign() # -A/--align + opts.pmSetLongOptionStart() # -S/--start + opts.pmSetLongOptionFinish() # -T/--finish + opts.pmSetLongOptionOrigin() # -O/--origin + opts.pmSetLongOptionSamples() # -s/--samples + opts.pmSetLongOptionInterval() # -t/--interval + opts.pmSetLongOptionTimeZone() # -Z/--timezone + opts.pmSetLongOptionHostZone() # -z/--hostzone + opts.pmSetLongOption("raw", 0, "r", "", "output raw counter values (no rate conversion)") + opts.pmSetLongOption("raw-prefer", 0, "R", "", "prefer output raw counter values (no rate conversion)") + opts.pmSetLongOption("ignore-incompat", 0, "I", "", "ignore incompatible instances (default: abort)") + opts.pmSetLongOption("ignore-unknown", 0, "5", "", "ignore unknown metrics (default: abort)") + opts.pmSetLongOption("names-change", 1, "4", "ACTION", "update/ignore/abort on PMNS change (default: ignore)") + opts.pmSetLongOption("instances", 1, "i", "STR", "instances to report (default: all current)") + opts.pmSetLongOption("live-filter", 0, "j", "", "perform instance live filtering") + opts.pmSetLongOption("rank", 1, "J", "COUNT", "limit results to COUNT highest/lowest valued instances") + opts.pmSetLongOption("limit-filter", 1, "8", "LIMIT", "default limit for value filtering") + opts.pmSetLongOption("limit-filter-force", 1, "9", "LIMIT", "forced limit for value filtering") + opts.pmSetLongOption("invert-filter", 0, "n", "", "perform ranking before live filtering") + opts.pmSetLongOption("predicate", 1, "N", "METRIC", "set predicate filter reference metric") + opts.pmSetLongOption("omit-flat", 0, "v", "", "omit single-valued metrics") + opts.pmSetLongOption("include-labels", 0, "m", "", "include metric label info") + opts.pmSetLongOption("timestamp-format", 1, "f", "STR", "strftime string for timestamp format") + opts.pmSetLongOption("precision", 1, "P", "N", "prefer N digits after decimal separator (default: 3)") + opts.pmSetLongOption("precision-force", 1, "0", "N", "force N digits after decimal separator") + opts.pmSetLongOption("count-scale", 1, "q", "SCALE", "default count unit") + opts.pmSetLongOption("count-scale-force", 1, "Q", "SCALE", "forced count unit") + opts.pmSetLongOption("space-scale", 1, "b", "SCALE", "default space unit") + opts.pmSetLongOption("space-scale-force", 1, "B", "SCALE", "forced space unit") + opts.pmSetLongOption("time-scale", 1, "y", "SCALE", "default time unit") + opts.pmSetLongOption("time-scale-force", 1, "Y", "SCALE", "forced time unit") + + opts.pmSetLongOption("with-everything", 0, "X", "", "write everything, incl. internal IDs") + + opts.pmSetLongOption("url", 1, "u", "URL", "URL of endpoint to receive HTTP POST") + opts.pmSetLongOption("http-timeout", 1, "o", "SECONDS", "timeout when sending HTTP POST") + opts.pmSetLongOption("http-pass", 1, "p", "PASSWORD", "password for endpoint") + opts.pmSetLongOption("http-user", 1, "U", "USERNAME", "username for endpoint") + + return opts + + def option_override(self, opt): + """ Override standard PCP options """ + if opt in ('g', 'H', 'K', 'n', 'N', 'p'): + return 1 + return 0 + + def option(self, opt, optarg, _index): + """ Perform setup for individual command line option """ + if opt == 'daemonize': + self.daemonize = 1 + elif opt == 'K': + if not self.speclocal or not self.speclocal.startswith(";"): + self.speclocal = ";" + optarg + else: + self.speclocal = self.speclocal + ";" + optarg + elif opt == 'c': + self.config = optarg + elif opt == 'C': + self.check = 1 + elif opt == 'F': + if os.path.exists(optarg): + sys.stderr.write("File %s already exists.\n" % optarg) + sys.exit(1) + self.outfile = optarg + elif opt == 'e': + if not self.derived or not self.derived.startswith(";"): + self.derived = ";" + optarg + else: + self.derived = self.derived + ";" + optarg + elif opt == 'H': + self.header = 0 + elif opt == 'G': + self.globals = 0 + elif opt == 'r': + self.type = 1 + elif opt == 'R': + self.type_prefer = 1 + elif opt == 'I': + self.ignore_incompat = 1 + elif opt == '5': + self.ignore_unknown = 1 + elif opt == '4': + if optarg == 'ignore': + self.names_change = 0 + elif optarg == 'abort': + self.names_change = 1 + elif optarg == 'update': + self.names_change = 2 + else: + sys.stderr.write("Unknown names-change action '%s' specified.\n" % optarg) + sys.exit(1) + elif opt == 'i': + self.instances = self.instances + self.pmconfig.parse_instances(optarg) + elif opt == 'j': + self.live_filter = 1 + elif opt == 'J': + self.rank = optarg + elif opt == '8': + self.limit_filter = optarg + elif opt == '9': + self.limit_filter_force = optarg + elif opt == 'n': + self.invert_filter = 1 + elif opt == 'N': + self.predicate = optarg + elif opt == 'v': + self.omit_flat = 1 + elif opt == 'm': + self.include_labels = 1 + elif opt == 'P': + self.precision = optarg + elif opt == '0': + self.precision_force = optarg + elif opt == 'f': + self.timefmt = optarg + elif opt == 'q': + self.count_scale = optarg + elif opt == 'Q': + self.count_scale_force = optarg + elif opt == 'b': + self.space_scale = optarg + elif opt == 'B': + self.space_scale_force = optarg + elif opt == 'y': + self.time_scale = optarg + elif opt == 'Y': + self.time_scale_force = optarg + elif opt == 'X': + self.everything = 1 + elif opt == 'u': + self.url = optarg + elif opt == 'o': + self.http_timeout = float(optarg) + elif opt == 'U': + self.http_user = optarg + elif opt == 'P': + self.http_pass = optarg + else: + raise pmapi.pmUsageErr() + + def connect(self): + """ Establish PMAPI context """ + context, self.source = pmapi.pmContext.set_connect_options(self.opts, self.source, self.speclocal) + + self.pmfg = pmapi.fetchgroup(context, self.source) + self.pmfg_ts = self.pmfg.extend_timestamp() + self.context = self.pmfg.get_context() + + if pmapi.c_api.pmSetContextOptions(self.context.ctx, self.opts.mode, self.opts.delta): + raise pmapi.pmUsageErr() + + def validate_config(self): + """ Validate configuration options """ + if self.version != CONFVER: + sys.stderr.write("Incompatible configuration file version (read v%s, need v%d).\n" % (self.version, CONFVER)) + sys.exit(1) + + self.pmconfig.validate_common_options() + + self.pmconfig.validate_metrics(curr_insts=not self.live_filter) + self.pmconfig.finalize_options() + + def execute(self): + """ Fetch and report """ + # Debug + if self.context.pmDebug(PM_DEBUG_APPL1): + sys.stdout.write("Known config file keywords: " + str(self.keys) + "\n") + sys.stdout.write("Known metric spec keywords: " + str(self.pmconfig.metricspec) + "\n") + + # Set delay mode, interpolation + if self.context.type != PM_CONTEXT_ARCHIVE: + self.delay = 1 + self.interpol = 1 + + # Common preparations + self.context.prepare_execute(self.opts, False, self.interpol, self.interval) + + # Headers + if self.header == 1: + self.header = 0 + self.write_header() + + # Just checking + if self.check == 1: + return + + # Daemonize when requested + if self.daemonize == 1: + self.opts.daemonize() + + # Align poll interval to host clock + if self.context.type != PM_CONTEXT_ARCHIVE and self.opts.pmGetOptionAlignment(): + align = float(self.opts.pmGetOptionAlignment()) - (time.time() % float(self.opts.pmGetOptionAlignment())) + time.sleep(align) + + # Main loop + refresh_metrics = 0 + while self.samples != 0: + # Refresh metrics as needed + if refresh_metrics: + refresh_metrics = 0 + self.pmconfig.update_metrics(curr_insts=not self.live_filter) + + # Fetch values + refresh_metrics = self.pmconfig.fetch() + if refresh_metrics < 0: + break + + # Report and prepare for the next round + self.report(self.pmfg_ts()) + if self.samples and self.samples > 0: + self.samples -= 1 + if self.delay and self.interpol and self.samples != 0: + self.pmconfig.pause() + + # Allow to flush buffered values / say goodbye + self.report(None) + + def report(self, tstamp): + """ Report metric values """ + if tstamp is not None: + tstamp = tstamp.strftime(self.timefmt) + + self.write_openmetrics(tstamp) + + def write_header(self): + """ Write info header """ + output = self.outfile if self.outfile else "stdout" + if self.context.type == PM_CONTEXT_ARCHIVE: + sys.stdout.write('{ "//": "Writing %d archived metrics to %s..." }\n{ "//": "(Ctrl-C to stop)" }\n' % (len(self.metrics), output)) + return + + sys.stdout.write('{ "//": "Waiting for %d metrics to be written to %s' % (len(self.metrics), output)) + if self.runtime != -1: + sys.stdout.write(':" }\n{ "//": "%s samples(s) with %.1f sec interval ~ %d sec runtime." }\n' % (self.samples, float(self.interval), self.runtime)) + elif self.samples: + duration = (self.samples - 1) * float(self.interval) + sys.stdout.write(':" }\n{ "//": "%s samples(s) with %.1f sec interval ~ %d sec runtime." }\n' % (self.samples, float(self.interval), duration)) + else: + sys.stdout.write('..." }\n{ "//": "(Ctrl-C to stop)" }\n') + + def write_openmetrics(self, timestamp): + """ Write results in openmetrics format """ + if timestamp is None: + # Silent goodbye, close in finalize() + return + + ts = self.context.datetime_to_secs(self.pmfg_ts(), PM_TIME_SEC) + + if self.prev_ts is None: + self.prev_ts = ts + + if not self.writer and not self.url: + if self.outfile is None: + self.writer = sys.stdout + else: + self.writer = open(self.outfile, 'wt') + + def get_type_string(desc): + """ Get metric type as string """ + if desc.contents.type == pmapi.c_api.PM_TYPE_32: + mtype = "32-bit signed" + elif desc.contents.type == pmapi.c_api.PM_TYPE_U32: + mtype = "32-bit unsigned" + elif desc.contents.type == pmapi.c_api.PM_TYPE_64: + mtype = "64-bit signed" + elif desc.contents.type == pmapi.c_api.PM_TYPE_U64: + mtype = "64-bit unsigned" + elif desc.contents.type == pmapi.c_api.PM_TYPE_FLOAT: + mtype = "32-bit float" + elif desc.contents.type == pmapi.c_api.PM_TYPE_DOUBLE: + mtype = "64-bit float" + elif desc.contents.type == pmapi.c_api.PM_TYPE_STRING: + mtype = "string" + else: + mtype = "unknown" + return mtype + + def openmetrics_name(metric): + """ pcp.io metric name to openmetrics.io name conventions """ + return metric.replace('.','_') + + def openmetrics_type(desc): + """ convert pcp.io metric metadata to openmetrics.io TYPE """ + if desc.sem == cpmapi.PM_SEM_COUNTER: + return 'counter' + return 'gauge' + + def openmetrics_labels(inst, name, desc, labels): + """ filter pcp.io labels here; pick out the labels needed """ + result = '' + if desc.indom != PM_INDOM_NULL: + labels[1].update(instname=name, instid=inst) + new_dict = {} + new_dict['semantics'] = self.context.pmSemStr(desc.contents.sem) + new_dict['type'] = get_type_string(desc) + for key in labels: + new_dict.update(labels[key]) + if self.everything: + subset = list(new_dict.keys()) + else: + subset = ['domainname', 'test', 'groupid', 'hostname', 'machineid', 'userid', 'instname', 'instid', 'agent', 'device_type', 'indom_name'] + for i, key in enumerate(subset): + if key not in new_dict: continue + if i != 0: result += ',' + result += '%s="%s"' % (key, new_dict[key]) + return '{' + result + '}' + + results = self.pmconfig.get_ranked_results(valid_only=True) + + body = '' + for metric in results: + i = list(self.metrics.keys()).index(metric) + desc = self.pmconfig.descs[i] + context = self.pmfg.get_context() + pmid = context.pmLookupName(metric) + labels = context.pmLookupLabels(pmid[0]) + help_dict = {} + help_dict[metric] = context.pmLookupText(pmid[0]).decode() + + body += '# HELP %s %s\n' % (openmetrics_name(metric), help_dict[metric]) + body += '# TYPE %s %s\n' % (openmetrics_name(metric), openmetrics_type(desc)) + + for inst, name, value in results[metric]: + if isinstance(value, float): + fmt = "." + str(self.metrics[metric][6]) + "f" + value = format(value, fmt) + elif isinstance (value, int): + fmt = "d" + value = format(value, fmt) + else: + str(value) + body += '%s%s %s\n' % (openmetrics_name(metric), openmetrics_labels(inst, name, desc, labels), value) + + if self.url: + auth = None + if self.http_user and self.http_pass: + auth = requests.auth.HTTPBasicAuth(self.http_user, self.http_pass) + try: + timeout = self.http_timeout + headers = {'Content-Type': 'application/openmetrics-text'} + res = requests.post(self.url, data=body, auth=auth, headers=headers, timeout=timeout) + if res.status_code > 299: + msg = "Cannot send metrics: HTTP code %s\n" % str(res.status_code) + sys.stderr.write(msg) + except requests.exceptions.ConnectionError as post_error: + msg = "Cannot connect to server at %s: %s\n" % (self.url, str(post_error)) + sys.stderr.write(msg) + else: + print(body) + + def finalize(self): + """ Finalize and clean up """ + if self.writer: + try: + self.writer.write("\n") + self.writer.flush() + except IOError as write_error: + if write_error.errno != errno.EPIPE: + raise + try: + self.writer.close() + except Exception: + pass + self.writer = None + +if __name__ == '__main__': + try: + P = PCP2OPENMETRICS() + P.connect() + P.validate_config() + P.execute() + P.finalize() + except pmapi.pmErr as error: + sys.stderr.write("%s: %s" % (error.progname(), error.message())) + if error.message() == "Connection refused": + sys.stderr.write("; is pmcd running?") + sys.stderr.write("\n") + sys.exit(1) + except pmapi.pmUsageErr as usage: + usage.message() + sys.exit(1) + except IOError as error: + if error.errno != errno.EPIPE: + sys.stderr.write("%s\n" % str(error)) + sys.exit(1) + except KeyboardInterrupt: + sys.stdout.write("\n") + P.finalize() diff --git a/src/pcp2openmetrics/pythonserver.py b/src/pcp2openmetrics/pythonserver.py new file mode 100644 index 0000000000..5ba854ea60 --- /dev/null +++ b/src/pcp2openmetrics/pythonserver.py @@ -0,0 +1,54 @@ +#!/usr/bin/env pmpython +# +# Copyright (C) 2024 Lauren Chilton +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. + +from http.server import BaseHTTPRequestHandler, HTTPServer +import logging +import sys + +class RequestHandler(BaseHTTPRequestHandler): + def _set_response(self): + self.send_response(200) + self.send_header('Content-type', 'text/plain') + self.end_headers() + + def do_GET(self): + logging.info("GET request,\nPath: %s\nHeaders:\n%s\n", str(self.path), str(self.headers)) + self._set_response() + self.wfile.write("OK".encode('utf-8')) + + def do_POST(self): + content_length = int(self.headers['Content-Length']) + post_data = self.rfile.read(content_length) + logging.info("POST request,\nPath: %s\nHeaders:\n%s\n\nBody:\n%s\n", str(self.path), str(self.headers), post_data.decode('utf-8')) + self._set_response() + self.wfile.write("OK".encode('utf-8')) + +def run(server_class=HTTPServer, handler_class=RequestHandler): + if len(sys.argv) > 1: + port = int(sys.argv[1]) + else: + port = 8080 + logging.basicConfig(level=logging.INFO) + server_address = ('', port) + httpd = server_class(server_address, handler_class) + logging.info('Starting httpd...\n') + try: + httpd.serve_forever() + except KeyboardInterrupt: + pass + httpd.server_close() + logging.info('Stopping httpd...\n') + +if __name__ == '__main__': + run()