diff --git a/qa/1131 b/qa/1131 index 599edbedab..d6e7e21dd9 100755 --- a/qa/1131 +++ b/qa/1131 @@ -1,6 +1,6 @@ #!/bin/sh # PCP QA Test No. 1131 -# Exercise pcp2json. +# Exercise pcp2json and pcp2openmetrics. # # Copyright (c) 2017 Red Hat. # @@ -9,6 +9,7 @@ seq=`basename $0` echo "QA output created by $seq" . ./common.python +. ./common.check $python -c "from pcp import pmapi" >/dev/null 2>&1 [ $? -eq 0 ] || _notrun "python pcp pmapi module not installed" @@ -16,6 +17,7 @@ $python -c "from collections import OrderedDict" >/dev/null 2>&1 [ $? -eq 0 ] || _notrun "python collections OrderedDict module not installed" which pcp2json >/dev/null 2>&1 || _notrun "pcp2json not installed" +which pcp2openmetrics >/dev/null 2>&1 || _notrun "pcp2openmetrics not installed" status=1 # failure is the default! signal=$PCP_BINADM_DIR/pmsignal @@ -23,6 +25,9 @@ $sudo rm -rf $tmp $tmp.* $seq.full trap "cd $here; rm -rf $tmp.*; exit \$status" 0 1 2 3 15 A="$here/archives/rep" +hostname=`hostname` +machineid=`_machine_id` +domainid=`_domain_name` _archive_filter() { @@ -34,6 +39,19 @@ _value_filter() sed 's/\(\.[0-9]\)[0-9]*/\1/g' } +_filter_pcp2openmetrics() +{ + tee -a $here/$seq.full \ + | col -b \ + | sed \ + -e '/domainname=/ s/'$domainid'/DOMAINID/' \ + -e '/machineid=/ s/'$machineid'/MACHINEID/' \ + -e '/groupid=/ s/'$UID'/GROUPID/' \ + -e '/userid=/ s/'$UID'/USERID/' \ + -e '/hostname=/ s/'$hostname'/HOST/' \ + | LC_COLLATE=POSIX sort +} + # real QA test starts here echo "---" pcp2json -a $A -H -I -z "" | _archive_filter @@ -43,6 +61,11 @@ echo "---" pcp2json -a $A -H -I -Z UTC+0 -x "" | _archive_filter echo "---" pcp2json -a $A -H -I -z -X -b GB -P 2 -F $tmp.outfile "" +echo "---" +pcp2openmetrics -s1 -H -z hinv.ncpu | _filter_pcp2openmetrics +echo "---" + + cat $tmp.outfile | _archive_filter which json_verify > /dev/null 2>&1 if [ $? -eq 0 ]; then diff --git a/qa/1131.out b/qa/1131.out index eff0849143..6a18d03ce7 100644 --- a/qa/1131.out +++ b/qa/1131.out @@ -2266,6 +2266,13 @@ QA output created by 1131 } } --- +--- + + +# HELP hinv_ncpu number of CPUs in the system +# TYPE hinv_ncpu gauge +hinv_ncpu{domainname="DOMAINID",groupid="GROUPID",hostname="HOST",machineid="MACHINEID",userid="USERID",agent="linux"} 16 +--- { "@pcp": { "@hosts": [ diff --git a/qa/1589 b/qa/1589 index 28e2b00b12..ae9cc97e45 100755 --- a/qa/1589 +++ b/qa/1589 @@ -3,7 +3,9 @@ # Exercise pcp2json HTTP POST functionality. # # Copyright (c) 2023 Red Hat. All Rights Reserved. -# +# + +#set -x seq=`basename $0` echo "QA output created by $seq" @@ -51,21 +53,22 @@ _filter_pcp2json_http() # real QA test starts here port=`_find_free_port` -nc -l localhost $port >$tmp.nc.out 2>$tmp.nc.err & -pid1=$! -sleep 2 # let nc start up - -# in case nc(1) does not exit by itself, e.g. on Ubuntu -( sleep 2; $signal $pid1 ) >>$seq.full 2>&1 & +$PCP_PYTHON_PROG $here/src/pythonserver.py $port &> $tmp.python.out & +pid=$! +sleep 2 # let server start up echo "pcp2json invocation" | tee -a $here/$seq.full -pcp2json -s1 -f%s -ZUTC --url http://localhost:$port/receive hinv.ncpu > $tmp.json.out 2> $tmp.json.err +pcp2json -s1 -ZUTC -u http://localhost:$port hinv.ncpu >$tmp.json.out 2>$tmp.json.err +pid2=$! +sleep 2 echo "pcp2json HTTP POST (sorted):" -_filter_pcp2json_http <$tmp.nc.out +_filter_pcp2json_http <$tmp.python.out + +( sleep 2; $signal $pid ) >>$seq.full 2>&1 & echo "All diagnostics" >> $here/$seq.full -for i in $tmp.json.out $tmp.json.err $tmp.nc.out $tmp.nc.err +for i in $tmp.json.out $tmp.json.err $tmp.python.out do echo "=== $i ===" >>$here/$seq.full cat $i >>$here/$seq.full diff --git a/qa/1589.out b/qa/1589.out index 7509fb6f9d..2eb5d5be4b 100644 --- a/qa/1589.out +++ b/qa/1589.out @@ -2,11 +2,15 @@ QA output created by 1589 pcp2json invocation pcp2json HTTP POST (sorted): + + + + "value":NCPU "ncpu": { } "@interval": "0", - "@timestamp":SECS, + "@timestamp": "2024-02-14 18:31:30", "hinv": { } { @@ -22,11 +26,16 @@ pcp2json HTTP POST (sorted): ] "@pcp": { } +127.0.0.1 - - [14/Feb/2024 13:31:30] "POST / HTTP/1.1" 200 - Accept: */* +Body: Content-Length: SIZE Content-Type: application/json +Headers: Host: localhost:PORT -POST /receive HTTP/1.1 +INFO:root:POST request, +INFO:root:Starting httpd... +Path: / User-Agent: python-requests VERSION { } diff --git a/qa/1827 b/qa/1827 new file mode 100755 index 0000000000..def5eb57e1 --- /dev/null +++ b/qa/1827 @@ -0,0 +1,70 @@ +#!/bin/sh +# PCP QA Test No. 1589 +# Exercise pcp2openmetrics HTTP POST functionality. +# +# Copyright (c) 2024 Red Hat. All Rights Reserved. +# + +#set -x + +seq=`basename $0` +echo "QA output created by $seq" + +. ./common.python +. ./common.check + +which pcp2openmetrics >/dev/null 2>&1 || _notrun "pcp2openmetrics not installed" + +_cleanup() +{ + cd $here + $sudo rm -rf $tmp $tmp.* +} + +status=0 # success is the default! +cpus=`pmprobe -v hinv.ncpu | awk '{print $3}'` +hostname=`hostname` +machineid=`_machine_id` +domainid=`_domain_name` +$sudo rm -rf $tmp $tmp.* $seq.full +trap "_cleanup; exit \$status" 0 1 2 3 15 + + +_filter_pcp2openmetrics_http() +{ + tee -a $here/$seq.full \ + | col -b \ + | sed \ + -e '/domainname=/ s/'$domainid'/DOMAINID/' \ + -e '/machineid=/ s/'$machineid'/MACHINEID/' \ + -e '/groupid=/ s/'$UID'/GROUPID/' \ + -e '/userid=/ s/'$UID'/USERID/' \ + -e '/hostname=/ s/'$hostname'/HOST/' \ + -e "s/^\(Host: localhost\):$port/\1:PORT/g" \ + -e 's/^\(Content-Length:\) [1-9][0-9]*/\1 SIZE/g' \ + -e 's/^\(User-Agent: python-requests\).*/\1 VERSION/g' \ + -e 's/^\(Date:\).*/\1 DATE/g' \ + -e 's/\(\"context\":\) [0-9][0-9]*/\1 CTXID/g' \ + -e '/^Accept-Encoding: /d' \ + -e 's/\(\hostname=\): \""$hostname"\"/\1:HOST/g' \ + -e '/^Connection: keep-alive/d' \ + -e '/ using stream socket$/d' \ + | LC_COLLATE=POSIX sort +} + +# real QA test starts here +port=`_find_free_port` +$PCP_PYTHON_PROG $here/src/pythonserver.py $port &> $tmp.python.out & +pid=$! +sleep 2 # let server start up + +echo "pcp2openmetrics invocation" | tee -a $here/$seq.full +pcp2openmetrics -s1 -u http://localhost:$port hinv.ncpu >$tmp.openmetrics.out 2>$tmp.openmetrics.err + +echo "pcp2openmetrics HTTP POST (sorted):" +_filter_pcp2openmetrics_http <$tmp.python.out + +($signal $pid ) >>$seq.full 2>&1 & + +# success, all done +exit \ No newline at end of file diff --git a/qa/1827.out b/qa/1827.out new file mode 100644 index 0000000000..1f6836e684 --- /dev/null +++ b/qa/1827.out @@ -0,0 +1,23 @@ +QA output created by 1827 +pcp2openmetrics invocation +pcp2openmetrics HTTP POST (sorted): + + + + + + +# HELP hinv_ncpu number of CPUs in the system +# TYPE hinv_ncpu gauge +127.0.0.1 - - [28/Feb/2024 12:02:06] "POST / HTTP/1.1" 200 - +Accept: */* +Body: +Content-Length: SIZE +Content-Type: application/openmetrics-text +Headers: +Host: localhost:PORT +INFO:root:POST request, +INFO:root:Starting httpd... +Path: / +User-Agent: python-requests VERSION +hinv_ncpu{domainname="DOMAINID",groupid="GROUPID",hostname="HOST",machineid="MACHINEID",userid="USERID",agent="linux"} 16 diff --git a/src/pcp2openmetrics/GNUmakefile b/src/pcp2openmetrics/GNUmakefile new file mode 100644 index 0000000000..5cc9c2eb66 --- /dev/null +++ b/src/pcp2openmetrics/GNUmakefile @@ -0,0 +1,46 @@ +# +# Copyright (c) 2024 Red Hat. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# + +TOPDIR = ../.. +include $(TOPDIR)/src/include/builddefs + +TARGET = pcp2openmetrics +MAN_SECTION = 1 +MAN_PAGES = $(TARGET).$(MAN_SECTION) +MAN_DEST = $(PCP_MAN_DIR)/man$(MAN_SECTION) +BASHDIR = $(PCP_BASHSHARE_DIR)/completions + +default: $(TARGET).py $(MAN_PAGES) + +default: + +include $(BUILDRULES) + +install: default +ifeq "$(HAVE_PYTHON)" "true" + $(INSTALL) -m 755 $(TARGET).py $(PCP_BIN_DIR)/$(TARGET) + $(INSTALL) -S $(BASHDIR)/pcp $(BASHDIR)/$(TARGET) + @$(INSTALL_MAN) +endif + +default_pcp: default + +install_pcp: install + +check:: $(TARGET).py + $(PYLINT) $^ + +check :: $(MAN_PAGES) + $(MANLINT) $^ + diff --git a/src/pcp2openmetrics/pcp2openmetrics.1 b/src/pcp2openmetrics/pcp2openmetrics.1 new file mode 100644 index 0000000000..a30dbc00bf --- /dev/null +++ b/src/pcp2openmetrics/pcp2openmetrics.1 @@ -0,0 +1,701 @@ +'\"macro stdmacro +.\" +.\" Copyright (C) 2024 Lauren Chilton +.\" Copyright (C) 2024 Red Hat. +.\" +.\" This program is free software; you can redistribute it and/or modify it +.\" under the terms of the GNU General Public License as published by the +.\" Free Software Foundation; either version 2 of the License, or (at your +.\" option) any later version. +.\" +.\" This program is distributed in the hope that it will be useful, but +.\" WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +.\" or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +.\" for more details. +.\" +.\" +.TH PCP2OPENMETRICS 1 "PCP" "Performance Co-Pilot" +.SH NAME +\f3pcp2openmetrics\f1 \- pcp-to-openmetrics exporter +.SH SYNOPSIS +\fBpcp2openmetrics\fP +[\fB\-5CEGHIjLmnrRvVxXz?\fP] +[\fB\-4\fP \fIaction\fP] +[\fB\-8\fP|\fB\-9\fP \fIlimit\fP] +[\fB\-a\fP \fIarchive\fP] +[\fB\-A\fP \fIalign\fP] +[\fB\-\-archive\-folio\fP \fIfolio\fP] +[\fB\-b\fP|\fB\-B\fP \fIspace-scale\fP] +[\fB\-c\fP \fIconfig\fP] +[\fB\-\-container\fP \fIcontainer\fP] +[\fB\-\-daemonize\fP] +[\fB\-e\fP \fIderived\fP] +[\fB\-f\fP \fIformat\fP] +[\fB\-F\fP \fIoutfile\fP] +[\fB\-h\fP \fIhost\fP] +[\fB\-i\fP \fIinstances\fP] +[\fB\-J\fP \fIrank\fP] +[\fB\-K\fP \fIspec\fP] +[\fB\-N\fP \fIpredicate\fP] +[\fB\-o\fP \fItimeout\fP] +[\fB\-O\fP \fIorigin\fP] +[\fB\-p\fP \fIpassword\fP] +[\fB\-P\fP|\fB\-0\fP \fIprecision\fP] +[\fB\-q\fP|\fB\-Q\fP \fIcount-scale\fP] +[\fB\-s\fP \fIsamples\fP] +[\fB\-S\fP \fIstarttime\fP] +[\fB\-t\fP \fIinterval\fP] +[\fB\-T\fP \fIendtime\fP] +[\fB\-u\fP \fIurl\fP] +[\fB\-U\fP \fIusername\fP] +[\fB\-y\fP|\fB\-Y\fP \fItime-scale\fP] +[\fB\-Z\fP \fItimezone\fP] +\fImetricspec\fP +[...] +.SH DESCRIPTION +.B pcp2openmetrics +is a customizable performance metrics exporter tool from PCP to +Open Metrics - +.I https://openmetrics.io +- format. +Any available performance metric, live or archived, system and/or +application, can be selected for exporting using either command line +arguments or a configuration file. +.PP +.B pcp2openmetrics +is a close relative of +.BR pmrep (1). +Refer to +.BR pmrep (1) +for the +.I metricspec +description accepted on +.B pcp2openmetrics +command line. +See +.BR pmrep.conf (5) +for description of the +.B pcp2openmetrics.conf +configuration file syntax. +This page describes +.B pcp2openmetrics +specific options and configuration file differences with +.BR pmrep.conf (5). +.BR pmrep (1) +also lists some usage examples of which most are applicable with +.B pcp2openmetrics +as well. +.PP +Only the command line options listed on this page are supported, +other options available for +.BR pmrep (1) +are not supported. +.PP +Options via environment values (see +.BR pmGetOptions (3)) +override the corresponding built-in default values (if any). +Configuration file options override the corresponding +environment variables (if any). +Command line options override the corresponding configuration +file options (if any). +.SH CONFIGURATION FILE +.B pcp2openmetrics +uses a configuration file with syntax described in +.BR pmrep.conf (5). +The following options are common with +.BR pmrep.conf : +.BR version , +.BR source , +.BR speclocal , +.BR derived , +.BR header , +.BR globals , +.BR samples , +.BR interval , +.BR type , +.BR type_prefer , +.BR ignore_incompat , +.BR names_change , +.BR instances , +.BR live_filter , +.BR rank , +.BR limit_filter , +.BR limit_filter_force , +.BR invert_filter , +.BR predicate , +.BR omit_flat , +.BR include_labels , +.BR precision , +.BR precision_force , +.BR count_scale , +.BR count_scale_force , +.BR space_scale , +.BR space_scale_force , +.BR time_scale , +.BR time_scale_force . +The rest of the +.B pmrep.conf +options are recognized but ignored for compatibility. +.SS pcp2openmetrics specific options +everything (boolean) +.RS 4 +Write everything known about metrics, including PCP internal IDs. +Labels are, however, omitted for backward compatibility. +Enable \fBinclude_labels\fP to include them as well. +Corresponding command line option is \fB\-X\fP. +Defaults to \fBno\fP. +.RE +.PP +exact_types (boolean) +.RS 4 +Write numbers as number data types, not as strings, potentially +losing some precision. +Corresponding command line option is \fB\-E\fP. +Defaults to \fBno\fP. +.RE +.PP +url (string) +.RS 4 +Send OPENMETRICS output as a HTTP POST to the given \fBurl\fP. +Corresponding command line option is \fB\-u\fP. +Defaults to \fBNone\fP. +.RE +.PP +http_pass (string) +.RS 4 +Use given password for Basic Authentication when sending a HTTP POST. +Corresponding command line option is \fB\-p\fP. +Defaults to \fBNone\fP. +.RE +.PP +http_user (string) +.RS 4 +Use given username for Basic Authentication when sending a HTTP POST. +Corresponding command line option is \fB\-U\fP. +Defaults to \fBNone\fP. +.RE +.PP +http_timeout (number) +.RS 4 +Maximum time (in seconds) when sending a HTTP POST. +Corresponding command line option is \fB\-o\fP. +Defaults to \fB2.5\fP seconds. +.RE +.SH OPTIONS +The available command line options are: +.TP 5 +\fB\-0\fR \fIprecision\fR, \fB\-\-precision\-force\fR=\fIprecision\fR +Like +.B \-P +but this option \fIwill\fP override per-metric specifications. +.TP +\fB\-4\fR \fIaction\fR, \fB\-\-names\-change\fR=\fIaction\fR +Specify which +.I action +to take on receiving a metric names change event during sampling. +These events occur when a PMDA discovers new metrics sometime +after starting up, and informs running client tools like +.BR pcp2openmetrics . +Valid values for +.I action +are \fBupdate\fP (refresh metrics being sampled), +\fBignore\fP (do nothing \- the default behaviour) +and \fBabort\fP (exit the program if such an event occurs). +.TP +\fB\-5\fR, \fB\-\-ignore\-unknown\fR +Silently ignore any metric name that cannot be resolved. +At least one metric must be found for the tool to start. +.TP +\fB\-8\fR \fIlimit\fR, \fB\-\-limit\-filter\fR=\fIlimit\fR +Limit results to instances with values above/below +.IR limit . +A positive integer will include instances with values +at or above the limit in reporting. +A negative integer will include instances with values +at or below the limit in reporting. +A value of zero performs no limit filtering. +This option will \fInot\fP override possible per-metric specifications. +See also +.BR \-J " and " +.BR \-N . +.TP +\fB\-9\fR \fIlimit\fR, \fB\-\-limit\-filter\-force\fR=\fIlimit\fR +Like +.B \-8 +but this option \fIwill\fP override per-metric specifications. +.TP +\fB\-a\fR \fIarchive\fR, \fB\-\-archive\fR=\fIarchive\fR +Performance metric values are retrieved from the set of Performance +Co-Pilot (PCP) archive files identified by the +.I archive +argument, which is a comma-separated list of names, each +of which may be the base name of an archive or the name of +a directory containing one or more archives. +.TP +\fB\-A\fR \fIalign\fR, \fB\-\-align\fR=\fIalign\fR +Force the initial sample to be +aligned on the boundary of a natural time unit +.IR align . +Refer to +.BR PCPIntro (1) +for a complete description of the syntax for +.IR align . +.TP +\fB\-\-archive\-folio\fR=\fIfolio\fR +Read metric source archives from the PCP archive +.I folio +created by tools like +.BR pmchart (1) +or, less often, manually with +.BR mkaf (1). +.TP +\fB\-b\fR \fIscale\fR, \fB\-\-space\-scale\fR=\fIscale\fR +.I Unit/scale +for space (byte) metrics, possible values include +.BR bytes , +.BR Kbytes , +.BR KB , +.BR Mbytes , +.BR MB , +and so forth. +This option will \fInot\fP override possible per-metric specifications. +See also +.BR pmParseUnitsStr (3). +.TP +\fB\-B\fR \fIscale\fR, \fB\-\-space\-scale\-force\fR=\fIscale\fR +Like +.B \-b +but this option \fIwill\fP override per-metric specifications. +.TP +\fB\-c\fR \fIconfig\fR, \fB\-\-config\fR=\fIconfig\fR +Specify the +.I config +file or directory to use. +In case \fIconfig\fP is a directory all files in it ending +\fB.conf\fR will be included. +The default is the first found of: +.IR ./pcp2openmetrics.conf , +.IR \f(CR$HOME\fP/.pcp2openmetrics.conf , +.IR \f(CR$HOME\fP/pcp/pcp2openmetrics.conf , +and +.IR \f(CR$PCP_SYSCONF_DIR\fP/pcp2openmetrics.conf . +For details, see the above section and +.BR pmrep.conf (5). +.TP +\fB\-\-container\fR=\fIcontainer\fR +Fetch performance metrics from the specified +.IR container , +either local or remote (see +.BR \-h ). +.TP +\fB\-C\fR, \fB\-\-check\fR +Exit before reporting any values, but after parsing the configuration +and metrics and printing possible headers. +.TP +.B \-\-daemonize +Daemonize on startup. +.TP +\fB\-e\fR \fIderived\fR, \fB\-\-derived\fR=\fIderived\fR +Specify +.I derived +performance metrics. +If +.I derived +starts with a slash (``/'') or with a dot (``.'') it will be +interpreted as a PCP derived metrics configuration file, otherwise it will +be interpreted as comma- or semicolon-separated derived metric expressions. +For complete description of derived metrics and PCP derived metrics +configuration files see +.BR pmLoadDerivedConfig (3) +and +.BR pmRegisterDerived (3). +Alternatively, using +.BR pmrep.conf (5) +configuration syntax allows defining derived metrics as part of metricsets. +.TP +\fB\-E\fR, \fB\-\-exact\-types\fR +Write numbers as number data types, not as strings, potentially +losing some precision. +.TP +\fB\-f\fR \fIformat\fR, \fB\-\-timestamp\-format\fR=\fIformat\fR +Use the +.I format +string for formatting the timestamp. +The format will be used with Python's +.B datetime.strftime +method which is mostly the same as that described in +.BR strftime (3). +The default is +.BR "%Y-%m-%d %H:%M:%S" . +.TP +\fB\-F\fR \fIoutfile\fR, \fB\-\-output\-file\fR=\fIoutfile\fR +Specify the output file +.IR outfile . +.TP +\fB\-G\fR, \fB\-\-no\-globals\fR +Do not include global metrics in reporting (see +.BR pmrep.conf (5)). +.TP +\fB\-h\fR \fIhost\fR, \fB\-\-host\fR=\fIhost\fR +Fetch performance metrics from +.BR pmcd (1) +on +.IR host , +rather than from the default localhost. +.TP +\fB\-H\fR, \fB\-\-no\-header\fR +Do not print any headers. +.TP +\fB\-i\fR \fIinstances\fR, \fB\-\-instances\fR=\fIinstances\fR +Retrieve and report only the specified metric +.IR instances . +By default all instances, present and future, are reported. +.RS +.PP +Refer to +.BR pmrep (1) +for complete description of this option. +.RE +.TP +\fB\-I\fR, \fB\-\-ignore\-incompat\fR +Ignore incompatible metrics. +By default incompatible metrics (that is, +their type is unsupported or they cannot be scaled as requested) +will cause +.B pcp2openmetrics +to terminate with an error message. +With this option all incompatible metrics are silently omitted +from reporting. +This may be especially useful when requesting +non-leaf nodes of the PMNS tree for reporting. +.TP +\fB\-j\fR, \fB\-\-live\-filter\fR +Perform instance live filtering. +This allows capturing all named instances even if processes +are restarted at some point (unlike without live filtering). +Performing live filtering over a huge number of instances will add +some internal overhead so a bit of user caution is advised. +See also +.BR \-n . +.TP +\fB\-J\fR \fIrank\fR, \fB\-\-rank\fR=\fIrank\fR +Limit results to highest/lowest +.IR rank ed +instances of set-valued metrics. +A positive integer will include highest valued instances in reporting. +A negative integer will include lowest valued instances in reporting. +A value of zero performs no ranking. +Ranking does not imply sorting, see +.BR \-6 . +See also +.BR \-8 . +.TP +\fB\-K\fR \fIspec\fR, \fB\-\-spec\-local\fR=\fIspec\fR +When fetching metrics from a local context (see +.BR \-L ), +the +.B \-K +option may be used to control the DSO PMDAs that should be made accessible. +The +.I spec +argument conforms to the syntax described in +.BR pmSpecLocalPMDA (3). +More than one +.B \-K +option may be used. +.TP +\fB\-L\fR, \fB\-\-local\-PMDA\fR +Use a local context to collect metrics from DSO PMDAs on the local host +without PMCD. +See also +.BR \-K . +.TP +\fB\-n\fR, \fB\-\-invert\-filter\fR +Perform ranking before live filtering. +By default instance live filtering (when requested, see +.BR \-j ) +happens before instance ranking (when requested, see +.BR \-J ). +With this option the logic is inverted and ranking happens before +live filtering. +.TP +\fB\-m\fR, \fB\-\-include\-labels\fR +Include PCP metric labels in the output. +.TP +\fB\-N\fR \fIpredicate\fR, \fB\-\-predicate\fR=\fIpredicate\fR +Specify a comma-separated list of +.I predicate +filter reference metrics. +By default ranking (see +.BR \-J ) +happens for each metric individually. +With predicates, ranking is done only for the +specified predicate metrics. +When reporting, rest of the metrics sharing the same +.I instance domain +(see +.BR PCPIntro (1)) +as the predicate will include only the highest/lowest ranking +instances of the corresponding predicate. +Ranking does not imply sorting, see +.BR \-6 . +.RS +.PP +So for example, using \fBproc.memory.rss\fP +(resident memory size of process) +as the +.I predicate +metric together with \fBproc.io.total_bytes\fP and \fBmem.util.used\fP as +metrics to be reported, only the processes using most/least (as per +.BR \-J ) +memory will be included when reporting total bytes written by processes. +Since \fBmem.util.used\fP is a single-valued metric (thus not sharing the +same instance domain as the process related metrics), +it will be reported as usual. +.RE +.TP +\fB\-o\fR, \fB\-\-http-timeout\fR +Timeout (in seconds) when sending a HTTP POST with the +.BR \-u +option. +Default value is \fB2.5\fP seconds. +.TP +\fB\-O\fR \fIorigin\fR, \fB\-\-origin\fR=\fIorigin\fR +When reporting archived metrics, start reporting at +.I origin +within the time window (see +.B \-S +and +.BR \-T ). +Refer to +.BR PCPIntro (1) +for a complete description of the syntax for +.IR origin . +.TP +\fB\-p\fR, \fB\-\-http-pass\fR +Password when using HTTP basic authentication with the +.BR \-u +option. +.TP +\fB\-P\fR \fIprecision\fR, \fB\-\-precision\fR=\fIprecision\fR +Use +.I precision +for numeric non-integer output values. +The default is to use 3 decimal places (when applicable). +This option will \fInot\fP override possible per-metric specifications. +.TP +\fB\-q\fR \fIscale\fR, \fB\-\-count\-scale\fR=\fIscale\fR +.I Unit/scale +for count metrics, possible values include +.BR "count x 10^\-1" , +.BR "count" , +.BR "count x 10" , +.BR "count x 10^2" , +and so forth from +.B 10^\-8 +to +.BR 10^7 . +.\" https://bugzilla.redhat.com/show_bug.cgi?id=1264124 +(These values are currently space-sensitive.) +This option will \fInot\fP override possible per-metric specifications. +See also +.BR pmParseUnitsStr (3). +.TP +\fB\-Q\fR \fIscale\fR, \fB\-\-count\-scale\-force\fR=\fIscale\fR +Like +.B \-q +but this option \fIwill\fP override per-metric specifications. +.TP +\fB\-r\fR, \fB\-\-raw\fR +Output raw metric values, do not convert cumulative counters to rates. +This option \fIwill\fP override possible per-metric specifications. +.TP +\fB\-R\fR, \fB\-\-raw\-prefer\fR +Like +.B \-r +but this option will \fInot\fP override per-metric specifications. +.TP +\fB\-s\fR \fIsamples\fR, \fB\-\-samples\fR=\fIsamples\fR +The +.I samples +argument defines the number of samples to be retrieved and reported. +If +.I samples +is 0 or +.B \-s +is not specified, +.B pcp2openmetrics +will sample and report continuously (in real time mode) or until the end +of the set of PCP archives (in archive mode). +See also +.BR \-T . +.TP +\fB\-S\fR \fIstarttime\fR, \fB\-\-start\fR=\fIstarttime\fR +When reporting archived metrics, the report will be restricted to those +records logged at or after +.IR starttime . +Refer to +.BR PCPIntro (1) +for a complete description of the syntax for +.IR starttime . +.TP +\fB\-t\fR \fIinterval\fR, \fB\-\-interval\fR=\fIinterval\fR +Set the reporting +.I interval +to something other than the default 1 second. +The +.I interval +argument follows the syntax described in +.BR PCPIntro (1), +and in the simplest form may be an unsigned integer +(the implied units in this case are seconds). +See also the +.B \-T +option. +.TP +\fB\-T\fR \fIendtime\fR, \fB\-\-finish\fR=\fIendtime\fR +When reporting archived metrics, the report will be restricted to those +records logged before or at +.IR endtime . +Refer to +.BR PCPIntro (1) +for a complete description of the syntax for +.IR endtime . +.RS +.PP +When used to define the runtime before \fBpcp2openmetrics\fP will exit, +if no \fIsamples\fP is given (see \fB\-s\fP) then the number of +reported samples depends on \fIinterval\fP (see \fB\-t\fP). +If +.I samples +is given then +.I interval +will be adjusted to allow reporting of +.I samples +during runtime. +In case all of +.BR \-T , +.BR \-s , +and +.B \-t +are given, +.I endtime +determines the actual time +.B pcp2openmetrics +will run. +.RE +.TP +\fB\-u\fR, \fB\-\-url\fR +URL for sending an HTTP POST (instead of default standard output). +.TP +\fB\-U\fR, \fB\-\-http-user\fR +Username when using HTTP basic authentication with the +.BR \-u +option. +.TP +\fB\-v\fR, \fB\-\-omit\-flat\fR +Report only set-valued metrics with instances (e.g. disk.dev.read) and +omit single-valued ``flat'' metrics without instances (e.g. +kernel.all.sysfork). +See +.B \-i +and +.BR \-I . +.TP +\fB\-V\fR, \fB\-\-version\fR +Display version number and exit. +.TP +\fB\-x\fR, \fB\-\-with\-extended\fR +Write extended information. +.TP +\fB\-X\fR, \fB\-\-with\-everything\fR +Write everything known about metrics, including PCP internal IDs. +Labels are, however, omitted for backward compatibility, +use \fB\-m\fP to include them as well. +.TP +\fB\-y\fR \fIscale\fR, \fB\-\-time\-scale\fR=\fIscale\fR +.I Unit/scale +for time metrics, possible values include +.BR nanosec , +.BR ns , +.BR microsec , +.BR us , +.BR millisec , +.BR ms , +and so forth up to +.BR hour , +.BR hr . +This option will \fInot\fP override possible per-metric specifications. +See also +.BR pmParseUnitsStr (3). +.TP +\fB\-Y\fR \fIscale\fR, \fB\-\-time\-scale\-force\fR=\fIscale\fR +Like +.B \-y +but this option \fIwill\fP override per-metric specifications. +.TP +\fB\-z\fR, \fB\-\-hostzone\fR +Use the local timezone of the host that is the source of the +performance metrics, as identified by either the +.B \-h +or the +.B \-a +options. +The default is to use the timezone of the local host. +.TP +\fB\-Z\fR \fItimezone\fR, \fB\-\-timezone\fR=\fItimezone\fR +Use +.I timezone +for the date and time. +.I Timezone +is in the format of the environment variable +.B TZ +as described in +.BR environ (7). +Note that when including a timezone string in output, ISO 8601 -style +UTC offsets are used (so something like \-Z EST+5 will become UTC-5). +.TP +\fB\-?\fR, \fB\-\-help\fR +Display usage message and exit. +.SH FILES +.TP 5 +.I pcp2openmetrics.conf +\fBpcp2openmetrics\fP configuration file (see \fB\-c\fP) +.TP +.I \f(CR$PCP_SYSCONF_DIR\fP/pmrep/*.conf +system provided default \fBpmrep\fP configuration files +.SH PCP ENVIRONMENT +Environment variables with the prefix \fBPCP_\fP are used to parameterize +the file and directory names used by PCP. +On each installation, the +file \fI/etc/pcp.conf\fP contains the local values for these variables. +The \fB$PCP_CONF\fP variable may be used to specify an alternative +configuration file, as described in \fBpcp.conf\fP(5). +.PP +For environment variables affecting PCP tools, see \fBpmGetOptions\fP(3). +.SH SEE ALSO +.BR PCPIntro (1), +.BR mkaf (1), +.BR pcp (1), +.BR pcp2elasticsearch (1), +.BR pcp2graphite (1), +.BR pcp2influxdb (1), +.BR pcp2spark (1), +.BR pcp2xlsx (1), +.BR pcp2xml (1), +.BR pcp2json (1), +.BR pcp2zabbix (1), +.BR pmcd (1), +.BR pminfo (1), +.BR pmrep (1), +.BR pmGetOptions (3), +.BR pmLoadDerivedConfig (3), +.BR pmParseUnitsStr (3), +.BR pmRegisterDerived (3), +.BR pmSpecLocalPMDA (3), +.BR LOGARCHIVE (5), +.BR pcp.conf (5), +.BR pmrep.conf (5), +.BR PMNS (5) +and +.BR environ (7). diff --git a/src/pcp2openmetrics/pcp2openmetrics.py b/src/pcp2openmetrics/pcp2openmetrics.py new file mode 100755 index 0000000000..1c8469a4dd --- /dev/null +++ b/src/pcp2openmetrics/pcp2openmetrics.py @@ -0,0 +1,559 @@ +#!/usr/bin/env pmpython +# +# Copyright (C) 2024 Lauren Chilton +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. + + +""" PCP to OPENMETRICS Bridge """ + +# Common imports +from collections import OrderedDict +import errno +import time +import sys + +# Our imports +import requests +import os +import cpmapi + +# PCP Python PMAPI +from pcp import pmapi, pmconfig +from cpmapi import PM_CONTEXT_ARCHIVE, PM_INDOM_NULL, PM_IN_NULL, PM_DEBUG_APPL1, PM_TIME_SEC + +# Default config +DEFAULT_CONFIG = ["./pcp2openmetrics.conf", "$HOME/.pcp2openmetrics.conf", "$HOME/.pcp/pcp2openmetrics.conf", "$PCP_SYSCONF_DIR/pcp2openmetrics.conf"] + +# Defaults +CONFVER = 1 +INDENT = 2 +TIMEFMT = "%Y-%m-%d %H:%M:%S" +TIMEOUT = 2.5 # seconds + +class PCP2OPENMETRICS(object): + """ PCP to OPENMETRICS """ + def __init__(self): + """ Construct object, prepare for command line handling """ + self.context = None + self.daemonize = 0 + self.pmconfig = pmconfig.pmConfig(self) + self.opts = self.options() + + # Configuration directives + self.keys = ('source', 'output', 'derived', 'header', 'globals', + 'samples', 'interval', 'type', 'precision', 'daemonize', + 'timefmt', 'everything', + 'count_scale', 'space_scale', 'time_scale', 'version', + 'count_scale_force', 'space_scale_force', 'time_scale_force', + 'type_prefer', 'precision_force', 'limit_filter', 'limit_filter_force', + 'live_filter', 'rank', 'invert_filter', 'predicate', 'names_change', + 'speclocal', 'instances', 'ignore_incompat', 'ignore_unknown', + 'omit_flat', 'include_labels', 'url', 'http_user', 'http_pass', + 'http_timeout') + + # Ignored for pmrep(1) compatibility + self.keys_ignore = ( + 'timestamp', 'unitinfo', 'colxrow', 'separate_header', 'fixed_header', + 'delay', 'width', 'delimiter', 'extcsv', 'width_force', + 'extheader', 'repeat_header', 'interpol', + 'dynamic_header', 'overall_rank', 'overall_rank_alt', 'sort_metric', + 'instinfo', 'include_texts') + + # The order of preference for options (as present): + # 1 - command line options + # 2 - options from configuration file(s) + # 3 - built-in defaults defined below + self.check = 0 + self.version = CONFVER + self.source = "local:" + self.output = None # For pmrep conf file compat only + self.speclocal = None + self.derived = None + self.header = 1 + self.globals = 1 + self.samples = None # forever + self.interval = pmapi.timeval(10) # 10 sec + self.opts.pmSetOptionInterval(str(10)) # 10 sec + self.delay = 0 + self.type = 0 + self.type_prefer = self.type + self.ignore_incompat = 0 + self.ignore_unknown = 0 + self.names_change = 0 # ignore + self.instances = [] + self.live_filter = 0 + self.rank = 0 + self.limit_filter = 0 + self.limit_filter_force = 0 + self.invert_filter = 0 + self.predicate = None + self.omit_flat = 0 + self.include_labels = 0 + self.precision = 3 # .3f + self.precision_force = None + self.timefmt = TIMEFMT + self.interpol = 0 + self.count_scale = None + self.count_scale_force = None + self.space_scale = None + self.space_scale_force = None + self.time_scale = None + self.time_scale_force = None + + # Not in pcp2openmetrics.conf, won't overwrite + self.outfile = None + + self.everything = 0 + self.url = None + self.http_user = None + self.http_pass = None + self.http_timeout = TIMEOUT + + # Internal + self.runtime = -1 + + self.data = None + self.prev_ts = None + self.writer = None + + # Performance metrics store + # key - metric name + # values - 0:txt label, 1:instance(s), 2:unit/scale, 3:type, + # 4:width, 5:pmfg item, 6:precision, 7:limit + self.metrics = OrderedDict() + self.pmfg = None + self.pmfg_ts = None + + # Read configuration and prepare to connect + self.config = self.pmconfig.set_config_path(DEFAULT_CONFIG) + self.pmconfig.read_options() + self.pmconfig.read_cmd_line() + self.pmconfig.prepare_metrics() + self.pmconfig.set_signal_handler() + + def options(self): + """ Setup default command line argument option handling """ + opts = pmapi.pmOptions() + opts.pmSetOptionCallback(self.option) + opts.pmSetOverrideCallback(self.option_override) + opts.pmSetShortOptions("a:h:LK:c:Ce:D:V?HGA:S:T:O:s:t:rRIi:jJ:4:58:9:nN:vmP:0:q:b:y:Q:B:Y:F:f:Z:zXo:p:U:u:") + opts.pmSetShortUsage("[option...] metricspec [...]") + + opts.pmSetLongOptionHeader("General options") + opts.pmSetLongOptionArchive() # -a/--archive + opts.pmSetLongOptionArchiveFolio() # --archive-folio + opts.pmSetLongOptionContainer() # --container + opts.pmSetLongOptionHost() # -h/--host + opts.pmSetLongOptionLocalPMDA() # -L/--local-PMDA + opts.pmSetLongOptionSpecLocal() # -K/--spec-local + opts.pmSetLongOption("config", 1, "c", "FILE", "config file path") + opts.pmSetLongOption("check", 0, "C", "", "check config and metrics and exit") + opts.pmSetLongOption("output-file", 1, "F", "OUTFILE", "output file") + opts.pmSetLongOption("derived", 1, "e", "FILE|DFNT", "derived metrics definitions") + opts.pmSetLongOption("daemonize", 0, "", "", "daemonize on startup") + opts.pmSetLongOptionDebug() # -D/--debug + opts.pmSetLongOptionVersion() # -V/--version + opts.pmSetLongOptionHelp() # -?/--help + + opts.pmSetLongOptionHeader("Reporting options") + opts.pmSetLongOption("no-header", 0, "H", "", "omit headers") + opts.pmSetLongOption("no-globals", 0, "G", "", "omit global metrics") + opts.pmSetLongOptionAlign() # -A/--align + opts.pmSetLongOptionStart() # -S/--start + opts.pmSetLongOptionFinish() # -T/--finish + opts.pmSetLongOptionOrigin() # -O/--origin + opts.pmSetLongOptionSamples() # -s/--samples + opts.pmSetLongOptionInterval() # -t/--interval + opts.pmSetLongOptionTimeZone() # -Z/--timezone + opts.pmSetLongOptionHostZone() # -z/--hostzone + opts.pmSetLongOption("raw", 0, "r", "", "output raw counter values (no rate conversion)") + opts.pmSetLongOption("raw-prefer", 0, "R", "", "prefer output raw counter values (no rate conversion)") + opts.pmSetLongOption("ignore-incompat", 0, "I", "", "ignore incompatible instances (default: abort)") + opts.pmSetLongOption("ignore-unknown", 0, "5", "", "ignore unknown metrics (default: abort)") + opts.pmSetLongOption("names-change", 1, "4", "ACTION", "update/ignore/abort on PMNS change (default: ignore)") + opts.pmSetLongOption("instances", 1, "i", "STR", "instances to report (default: all current)") + opts.pmSetLongOption("live-filter", 0, "j", "", "perform instance live filtering") + opts.pmSetLongOption("rank", 1, "J", "COUNT", "limit results to COUNT highest/lowest valued instances") + opts.pmSetLongOption("limit-filter", 1, "8", "LIMIT", "default limit for value filtering") + opts.pmSetLongOption("limit-filter-force", 1, "9", "LIMIT", "forced limit for value filtering") + opts.pmSetLongOption("invert-filter", 0, "n", "", "perform ranking before live filtering") + opts.pmSetLongOption("predicate", 1, "N", "METRIC", "set predicate filter reference metric") + opts.pmSetLongOption("omit-flat", 0, "v", "", "omit single-valued metrics") + opts.pmSetLongOption("include-labels", 0, "m", "", "include metric label info") + opts.pmSetLongOption("timestamp-format", 1, "f", "STR", "strftime string for timestamp format") + opts.pmSetLongOption("precision", 1, "P", "N", "prefer N digits after decimal separator (default: 3)") + opts.pmSetLongOption("precision-force", 1, "0", "N", "force N digits after decimal separator") + opts.pmSetLongOption("count-scale", 1, "q", "SCALE", "default count unit") + opts.pmSetLongOption("count-scale-force", 1, "Q", "SCALE", "forced count unit") + opts.pmSetLongOption("space-scale", 1, "b", "SCALE", "default space unit") + opts.pmSetLongOption("space-scale-force", 1, "B", "SCALE", "forced space unit") + opts.pmSetLongOption("time-scale", 1, "y", "SCALE", "default time unit") + opts.pmSetLongOption("time-scale-force", 1, "Y", "SCALE", "forced time unit") + + opts.pmSetLongOption("with-everything", 0, "X", "", "write everything, incl. internal IDs") + + opts.pmSetLongOption("url", 1, "u", "URL", "URL of endpoint to receive HTTP POST") + opts.pmSetLongOption("http-timeout", 1, "o", "SECONDS", "timeout when sending HTTP POST") + opts.pmSetLongOption("http-pass", 1, "p", "PASSWORD", "password for endpoint") + opts.pmSetLongOption("http-user", 1, "U", "USERNAME", "username for endpoint") + + return opts + + def option_override(self, opt): + """ Override standard PCP options """ + if opt in ('g', 'H', 'K', 'n', 'N', 'p'): + return 1 + return 0 + + def option(self, opt, optarg, _index): + """ Perform setup for individual command line option """ + if opt == 'daemonize': + self.daemonize = 1 + elif opt == 'K': + if not self.speclocal or not self.speclocal.startswith(";"): + self.speclocal = ";" + optarg + else: + self.speclocal = self.speclocal + ";" + optarg + elif opt == 'c': + self.config = optarg + elif opt == 'C': + self.check = 1 + elif opt == 'F': + if os.path.exists(optarg): + sys.stderr.write("File %s already exists.\n" % optarg) + sys.exit(1) + self.outfile = optarg + elif opt == 'e': + if not self.derived or not self.derived.startswith(";"): + self.derived = ";" + optarg + else: + self.derived = self.derived + ";" + optarg + elif opt == 'H': + self.header = 0 + elif opt == 'G': + self.globals = 0 + elif opt == 'r': + self.type = 1 + elif opt == 'R': + self.type_prefer = 1 + elif opt == 'I': + self.ignore_incompat = 1 + elif opt == '5': + self.ignore_unknown = 1 + elif opt == '4': + if optarg == 'ignore': + self.names_change = 0 + elif optarg == 'abort': + self.names_change = 1 + elif optarg == 'update': + self.names_change = 2 + else: + sys.stderr.write("Unknown names-change action '%s' specified.\n" % optarg) + sys.exit(1) + elif opt == 'i': + self.instances = self.instances + self.pmconfig.parse_instances(optarg) + elif opt == 'j': + self.live_filter = 1 + elif opt == 'J': + self.rank = optarg + elif opt == '8': + self.limit_filter = optarg + elif opt == '9': + self.limit_filter_force = optarg + elif opt == 'n': + self.invert_filter = 1 + elif opt == 'N': + self.predicate = optarg + elif opt == 'v': + self.omit_flat = 1 + elif opt == 'm': + self.include_labels = 1 + elif opt == 'P': + self.precision = optarg + elif opt == '0': + self.precision_force = optarg + elif opt == 'f': + self.timefmt = optarg + elif opt == 'q': + self.count_scale = optarg + elif opt == 'Q': + self.count_scale_force = optarg + elif opt == 'b': + self.space_scale = optarg + elif opt == 'B': + self.space_scale_force = optarg + elif opt == 'y': + self.time_scale = optarg + elif opt == 'Y': + self.time_scale_force = optarg + elif opt == 'X': + self.everything = 1 + elif opt == 'u': + self.url = optarg + elif opt == 'o': + self.http_timeout = float(optarg) + elif opt == 'U': + self.http_user = optarg + elif opt == 'P': + self.http_pass = optarg + else: + raise pmapi.pmUsageErr() + + def connect(self): + """ Establish PMAPI context """ + context, self.source = pmapi.pmContext.set_connect_options(self.opts, self.source, self.speclocal) + + self.pmfg = pmapi.fetchgroup(context, self.source) + self.pmfg_ts = self.pmfg.extend_timestamp() + self.context = self.pmfg.get_context() + + if pmapi.c_api.pmSetContextOptions(self.context.ctx, self.opts.mode, self.opts.delta): + raise pmapi.pmUsageErr() + + def validate_config(self): + """ Validate configuration options """ + if self.version != CONFVER: + sys.stderr.write("Incompatible configuration file version (read v%s, need v%d).\n" % (self.version, CONFVER)) + sys.exit(1) + + self.pmconfig.validate_common_options() + + self.pmconfig.validate_metrics(curr_insts=not self.live_filter) + self.pmconfig.finalize_options() + + def execute(self): + """ Fetch and report """ + # Debug + if self.context.pmDebug(PM_DEBUG_APPL1): + sys.stdout.write("Known config file keywords: " + str(self.keys) + "\n") + sys.stdout.write("Known metric spec keywords: " + str(self.pmconfig.metricspec) + "\n") + + # Set delay mode, interpolation + if self.context.type != PM_CONTEXT_ARCHIVE: + self.delay = 1 + self.interpol = 1 + + # Common preparations + self.context.prepare_execute(self.opts, False, self.interpol, self.interval) + + # Headers + if self.header == 1: + self.header = 0 + self.write_header() + + # Just checking + if self.check == 1: + return + + # Daemonize when requested + if self.daemonize == 1: + self.opts.daemonize() + + # Align poll interval to host clock + if self.context.type != PM_CONTEXT_ARCHIVE and self.opts.pmGetOptionAlignment(): + align = float(self.opts.pmGetOptionAlignment()) - (time.time() % float(self.opts.pmGetOptionAlignment())) + time.sleep(align) + + # Main loop + refresh_metrics = 0 + while self.samples != 0: + # Refresh metrics as needed + if refresh_metrics: + refresh_metrics = 0 + self.pmconfig.update_metrics(curr_insts=not self.live_filter) + + # Fetch values + refresh_metrics = self.pmconfig.fetch() + if refresh_metrics < 0: + break + + # Report and prepare for the next round + self.report(self.pmfg_ts()) + if self.samples and self.samples > 0: + self.samples -= 1 + if self.delay and self.interpol and self.samples != 0: + self.pmconfig.pause() + + # Allow to flush buffered values / say goodbye + self.report(None) + + def report(self, tstamp): + """ Report metric values """ + if tstamp is not None: + tstamp = tstamp.strftime(self.timefmt) + + self.write_openmetrics(tstamp) + + def write_header(self): + """ Write info header """ + output = self.outfile if self.outfile else "stdout" + if self.context.type == PM_CONTEXT_ARCHIVE: + sys.stdout.write('{ "//": "Writing %d archived metrics to %s..." }\n{ "//": "(Ctrl-C to stop)" }\n' % (len(self.metrics), output)) + return + + sys.stdout.write('{ "//": "Waiting for %d metrics to be written to %s' % (len(self.metrics), output)) + if self.runtime != -1: + sys.stdout.write(':" }\n{ "//": "%s samples(s) with %.1f sec interval ~ %d sec runtime." }\n' % (self.samples, float(self.interval), self.runtime)) + elif self.samples: + duration = (self.samples - 1) * float(self.interval) + sys.stdout.write(':" }\n{ "//": "%s samples(s) with %.1f sec interval ~ %d sec runtime." }\n' % (self.samples, float(self.interval), duration)) + else: + sys.stdout.write('..." }\n{ "//": "(Ctrl-C to stop)" }\n') + + def write_openmetrics(self, timestamp): + """ Write results in openmetrics format """ + if timestamp is None: + # Silent goodbye, close in finalize() + return + + ts = self.context.datetime_to_secs(self.pmfg_ts(), PM_TIME_SEC) + + if self.prev_ts is None: + self.prev_ts = ts + + if not self.writer and not self.url: + if self.outfile is None: + self.writer = sys.stdout + else: + self.writer = open(self.outfile, 'wt') + + def get_type_string(desc): + """ Get metric type as string """ + if desc.contents.type == pmapi.c_api.PM_TYPE_32: + mtype = "32-bit signed" + elif desc.contents.type == pmapi.c_api.PM_TYPE_U32: + mtype = "32-bit unsigned" + elif desc.contents.type == pmapi.c_api.PM_TYPE_64: + mtype = "64-bit signed" + elif desc.contents.type == pmapi.c_api.PM_TYPE_U64: + mtype = "64-bit unsigned" + elif desc.contents.type == pmapi.c_api.PM_TYPE_FLOAT: + mtype = "32-bit float" + elif desc.contents.type == pmapi.c_api.PM_TYPE_DOUBLE: + mtype = "64-bit float" + elif desc.contents.type == pmapi.c_api.PM_TYPE_STRING: + mtype = "string" + else: + mtype = "unknown" + return mtype + + def openmetrics_name(metric): + """ pcp.io metric name to openmetrics.io name conventions """ + return metric.replace('.','_') + + def openmetrics_type(desc): + """ convert pcp.io metric metadata to openmetrics.io TYPE """ + if desc.sem == cpmapi.PM_SEM_COUNTER: + return 'counter' + return 'gauge' + + def openmetrics_labels(inst, name, desc, labels): + """ filter pcp.io labels here; pick out the labels needed """ + result = '' + if desc.indom != PM_INDOM_NULL: + labels[1].update(instname=name, instid=inst) + new_dict = {} + new_dict['semantics'] = self.context.pmSemStr(desc.contents.sem) + new_dict['type'] = get_type_string(desc) + for key in labels: + new_dict.update(labels[key]) + if self.everything: + subset = list(new_dict.keys()) + else: + subset = ['domainname', 'test', 'groupid', 'hostname', 'machineid', 'userid', 'instname', 'instid', 'agent', 'device_type', 'indom_name'] + for i, key in enumerate(subset): + if key not in new_dict: continue + if i != 0: result += ',' + result += '%s="%s"' % (key, new_dict[key]) + return '{' + result + '}' + + results = self.pmconfig.get_ranked_results(valid_only=True) + + body = '' + for metric in results: + i = list(self.metrics.keys()).index(metric) + desc = self.pmconfig.descs[i] + context = self.pmfg.get_context() + pmid = context.pmLookupName(metric) + labels = context.pmLookupLabels(pmid[0]) + help_dict = {} + help_dict[metric] = context.pmLookupText(pmid[0]).decode() + + body += '# HELP %s %s\n' % (openmetrics_name(metric), help_dict[metric]) + body += '# TYPE %s %s\n' % (openmetrics_name(metric), openmetrics_type(desc)) + + for inst, name, value in results[metric]: + if isinstance(value, float): + fmt = "." + str(self.metrics[metric][6]) + "f" + value = format(value, fmt) + elif isinstance (value, int): + fmt = "d" + value = format(value, fmt) + else: + str(value) + body += '%s%s %s\n' % (openmetrics_name(metric), openmetrics_labels(inst, name, desc, labels), value) + + if self.url: + auth = None + if self.http_user and self.http_pass: + auth = requests.auth.HTTPBasicAuth(self.http_user, self.http_pass) + try: + timeout = self.http_timeout + headers = {'Content-Type': 'application/openmetrics-text'} + res = requests.post(self.url, data=body, auth=auth, headers=headers, timeout=timeout) + if res.status_code > 299: + msg = "Cannot send metrics: HTTP code %s\n" % str(res.status_code) + sys.stderr.write(msg) + except requests.exceptions.ConnectionError as post_error: + msg = "Cannot connect to server at %s: %s\n" % (self.url, str(post_error)) + sys.stderr.write(msg) + else: + print(body) + + def finalize(self): + """ Finalize and clean up """ + if self.writer: + try: + self.writer.write("\n") + self.writer.flush() + except IOError as write_error: + if write_error.errno != errno.EPIPE: + raise + try: + self.writer.close() + except Exception: + pass + self.writer = None + +if __name__ == '__main__': + try: + P = PCP2OPENMETRICS() + P.connect() + P.validate_config() + P.execute() + P.finalize() + except pmapi.pmErr as error: + sys.stderr.write("%s: %s" % (error.progname(), error.message())) + if error.message() == "Connection refused": + sys.stderr.write("; is pmcd running?") + sys.stderr.write("\n") + sys.exit(1) + except pmapi.pmUsageErr as usage: + usage.message() + sys.exit(1) + except IOError as error: + if error.errno != errno.EPIPE: + sys.stderr.write("%s\n" % str(error)) + sys.exit(1) + except KeyboardInterrupt: + sys.stdout.write("\n") + P.finalize() diff --git a/src/pcp2openmetrics/pythonserver.py b/src/pcp2openmetrics/pythonserver.py new file mode 100644 index 0000000000..5ba854ea60 --- /dev/null +++ b/src/pcp2openmetrics/pythonserver.py @@ -0,0 +1,54 @@ +#!/usr/bin/env pmpython +# +# Copyright (C) 2024 Lauren Chilton +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. + +from http.server import BaseHTTPRequestHandler, HTTPServer +import logging +import sys + +class RequestHandler(BaseHTTPRequestHandler): + def _set_response(self): + self.send_response(200) + self.send_header('Content-type', 'text/plain') + self.end_headers() + + def do_GET(self): + logging.info("GET request,\nPath: %s\nHeaders:\n%s\n", str(self.path), str(self.headers)) + self._set_response() + self.wfile.write("OK".encode('utf-8')) + + def do_POST(self): + content_length = int(self.headers['Content-Length']) + post_data = self.rfile.read(content_length) + logging.info("POST request,\nPath: %s\nHeaders:\n%s\n\nBody:\n%s\n", str(self.path), str(self.headers), post_data.decode('utf-8')) + self._set_response() + self.wfile.write("OK".encode('utf-8')) + +def run(server_class=HTTPServer, handler_class=RequestHandler): + if len(sys.argv) > 1: + port = int(sys.argv[1]) + else: + port = 8080 + logging.basicConfig(level=logging.INFO) + server_address = ('', port) + httpd = server_class(server_address, handler_class) + logging.info('Starting httpd...\n') + try: + httpd.serve_forever() + except KeyboardInterrupt: + pass + httpd.server_close() + logging.info('Stopping httpd...\n') + +if __name__ == '__main__': + run()