From ae887ab3fad99c2dcd80bb39ee798ddb413c6211 Mon Sep 17 00:00:00 2001 From: Marco Braga Date: Fri, 7 Jul 2023 13:52:13 -0300 Subject: [PATCH] OPCT-226: cmd/report UX enhancements Several enhancements applied to the command 'report' to provide a better experience in the UI when reviewing artifacts. The alpha version is experimental version for v0.5 introducting the changes from the PR: https://github.com/redhat-openshift-ecosystem/provider-certification-tool/pull/76 --- .gitignore | 1 + Makefile | 33 +- cmd/{ => opct}/root.go | 2 +- data/templates/report/README.md | 14 + data/templates/report/filter.html | 447 +++++++ data/templates/report/report.html | 1067 +++++++++++++++++ docs/README.md | 1 - docs/dev.md | 4 +- docs/dev/report.md | 46 + go.mod | 10 +- go.sum | 27 +- hack/Containerfile | 2 +- hack/Containerfile.ci | 4 +- hack/verify-codegen.sh | 16 - internal/opct/archive/errorcounter.go | 62 + internal/opct/archive/metaconfig.go | 37 + internal/opct/archive/metalog.go | 128 ++ internal/opct/archive/opctconfigmap.go | 31 + internal/opct/archive/pluginsdefinition.go | 5 + internal/opct/archive/runtime.go | 16 + internal/opct/metrics/timers.go | 44 + internal/opct/plugin/plugin.go | 81 ++ internal/opct/plugin/tags.go | 95 ++ internal/opct/plugin/tags_test.go | 46 + internal/opct/plugin/test.go | 99 ++ internal/opct/plugin/testdoc.go | 110 ++ internal/opct/report/checks.go | 480 ++++++++ internal/opct/report/checks_test.go | 7 + internal/opct/report/report.go | 559 +++++++++ internal/opct/summary/consolidated.go | 604 ++++++++++ internal/opct/summary/openshift.go | 236 ++++ internal/{pkg => opct}/summary/result.go | 207 +++- internal/opct/summary/sonobuoy.go | 60 + internal/{pkg => opct}/summary/suite.go | 2 +- internal/{pkg => openshift/ci}/sippy/sippy.go | 16 +- internal/openshift/ci/types.go | 20 + internal/openshift/mustgather/etcd.go | 384 ++++++ internal/openshift/mustgather/mustgather.go | 462 +++++++ .../openshift/mustgather/podnetconcheck.go | 135 +++ internal/pkg/summary/consolidated.go | 557 --------- internal/pkg/summary/opct.go | 52 - internal/pkg/summary/openshift.go | 138 --- internal/pkg/summary/sonobuoy.go | 14 - main.go | 2 +- mkdocs.yml | 28 +- pkg/cmd/report/report.go | 593 +++++++++ pkg/report/cmd.go | 372 ------ pkg/retrieve/retrieve.go | 2 +- pkg/run/run.go | 18 +- pkg/status/printer.go | 3 + pkg/status/status.go | 32 +- pkg/types.go | 2 +- 52 files changed, 6118 insertions(+), 1295 deletions(-) rename cmd/{ => opct}/root.go (99%) create mode 100644 data/templates/report/README.md create mode 100644 data/templates/report/filter.html create mode 100644 data/templates/report/report.html create mode 100644 docs/dev/report.md delete mode 100755 hack/verify-codegen.sh create mode 100644 internal/opct/archive/errorcounter.go create mode 100644 internal/opct/archive/metaconfig.go create mode 100644 internal/opct/archive/metalog.go create mode 100644 internal/opct/archive/opctconfigmap.go create mode 100644 internal/opct/archive/pluginsdefinition.go create mode 100644 internal/opct/archive/runtime.go create mode 100644 internal/opct/metrics/timers.go create mode 100644 internal/opct/plugin/plugin.go create mode 100644 internal/opct/plugin/tags.go create mode 100644 internal/opct/plugin/tags_test.go create mode 100644 internal/opct/plugin/test.go create mode 100644 internal/opct/plugin/testdoc.go create mode 100644 internal/opct/report/checks.go create mode 100644 internal/opct/report/checks_test.go create mode 100644 internal/opct/report/report.go create mode 100644 internal/opct/summary/consolidated.go create mode 100644 internal/opct/summary/openshift.go rename internal/{pkg => opct}/summary/result.go (51%) create mode 100644 internal/opct/summary/sonobuoy.go rename internal/{pkg => opct}/summary/suite.go (96%) rename internal/{pkg => openshift/ci}/sippy/sippy.go (89%) create mode 100644 internal/openshift/ci/types.go create mode 100644 internal/openshift/mustgather/etcd.go create mode 100644 internal/openshift/mustgather/mustgather.go create mode 100644 internal/openshift/mustgather/podnetconcheck.go delete mode 100644 internal/pkg/summary/consolidated.go delete mode 100644 internal/pkg/summary/opct.go delete mode 100644 internal/pkg/summary/openshift.go delete mode 100644 internal/pkg/summary/sonobuoy.go create mode 100644 pkg/cmd/report/report.go delete mode 100644 pkg/report/cmd.go diff --git a/.gitignore b/.gitignore index 77455776..58cf33c4 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ kubeconfig # build files dist/ +build/ # changelog is generated automaticaly by hack/generate-changelog.sh # available only in the rendered webpage (built by mkdocs). diff --git a/Makefile b/Makefile index 4e2d8499..ff2be748 100644 --- a/Makefile +++ b/Makefile @@ -18,10 +18,7 @@ GOARCH ?= amd64 unexport GOFLAGS .PHONY: all -all: build-linux-amd64 -all: build-windows-amd64 -all: build-darwin-amd64 -all: build-darwin-arm64 +all: linux-amd64-container build-windows-amd64 build-darwin-amd64 build-darwin-arm64 .PHONY: build-dep build-dep: @@ -29,8 +26,8 @@ build-dep: .PHONY: build build: build-dep - GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o $(BUILD_DIR)/opct-$(GOOS)-$(GOARCH)$(GOEXT) $(GO_BUILD_FLAGS) - @cd $(BUILD_DIR); md5sum $(BUILD_DIR)/opct-$(GOOS)-$(GOARCH)$(GOEXT) > $(BUILD_DIR)/opct-$(GOOS)-$(GOARCH)$(GOEXT).sum; cd - + GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o $(BUILD_DIR)/opct-$(GOOS)-$(GOARCH) $(GO_BUILD_FLAGS) + @cd $(BUILD_DIR); md5sum $(BUILD_DIR)/opct-$(GOOS)-$(GOARCH) > $(BUILD_DIR)/opct-$(GOOS)-$(GOARCH).sum; cd - .PHONY: build-linux-amd64 build-linux-amd64: GOOS = linux @@ -38,10 +35,9 @@ build-linux-amd64: GOARCH = amd64 build-linux-amd64: build .PHONY: build-windows-amd64 -build-windows-amd64: GOOS = windows -build-windows-amd64: GOARCH = amd64 -build-windows-amd64: GOEXT = .exe -build-windows-amd64: build +build-windows-amd64: build-dep + GOOS=windows GOARCH=amd64 go build -o $(BUILD_DIR)/opct-windows.exe $(GO_BUILD_FLAGS) + @cd $(BUILD_DIR); md5sum $(BUILD_DIR)/opct-windows-amd64 > $(BUILD_DIR)/opct-windows-amd64.sum; cd - .PHONY: build-darwin-amd64 build-darwin-amd64: GOOS = darwin @@ -57,11 +53,18 @@ build-darwin-arm64: build linux-amd64-container: build-linux-amd64 podman build -t $(IMG):latest -f hack/Containerfile --build-arg=RELEASE_TAG=$(RELEASE_TAG) . -# Utils dev -.PHONY: update-go -update-go: - go get -u - go mod tidy +# Publish devel binaries (non-official). Must be used only for troubleshooting in development/support. +.PHONY: publish-amd64-devel +publish-amd64-devel: build-linux-amd64 + aws s3 cp $(BUILD_DIR)/opct-linux-amd64 s3://openshift-provider-certification/bin/opct-linux-amd64-devel + +.PHONY: publish-darwin-arm64-devel +publish-darwin-arm64-devel: build-darwin-arm64 + aws s3 cp $(BUILD_DIR)/opct-darwin-arm64 s3://openshift-provider-certification/bin/opct-darwin-arm64-devel + +.PHONY: publish-devel +publish-devel: publish-amd64-devel +publish-devel: publish-darwin-arm64-devel .PHONY: test test: diff --git a/cmd/root.go b/cmd/opct/root.go similarity index 99% rename from cmd/root.go rename to cmd/opct/root.go index b774dbe3..816f41bd 100644 --- a/cmd/root.go +++ b/cmd/opct/root.go @@ -10,8 +10,8 @@ import ( "github.com/spf13/viper" "github.com/vmware-tanzu/sonobuoy/cmd/sonobuoy/app" + "github.com/redhat-openshift-ecosystem/provider-certification-tool/pkg/cmd/report" "github.com/redhat-openshift-ecosystem/provider-certification-tool/pkg/destroy" - "github.com/redhat-openshift-ecosystem/provider-certification-tool/pkg/report" "github.com/redhat-openshift-ecosystem/provider-certification-tool/pkg/retrieve" "github.com/redhat-openshift-ecosystem/provider-certification-tool/pkg/run" "github.com/redhat-openshift-ecosystem/provider-certification-tool/pkg/status" diff --git a/data/templates/report/README.md b/data/templates/report/README.md new file mode 100644 index 00000000..fd6673ef --- /dev/null +++ b/data/templates/report/README.md @@ -0,0 +1,14 @@ +# Report HTML app + +Report is build upon Vue framework using native browser. + +The pages are reactive, using the opct-report.json as data source. + +The opct-report.json is generated by `report` command when processing +the results. + + +References: + +- https://vuejs.org/guide/extras/ways-of-using-vue.html +- https://markus.oberlehner.net/blog/goodbye-webpack-building-vue-applications-without-webpack/ \ No newline at end of file diff --git a/data/templates/report/filter.html b/data/templates/report/filter.html new file mode 100644 index 00000000..31856d90 --- /dev/null +++ b/data/templates/report/filter.html @@ -0,0 +1,447 @@ + + + + + + + OPCT Filters + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + +
+
+
+ + + +

[[ .Summary.Tests.Archive ]]

+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Clear + + + + + + + + + ID + Name + Status + State + + + + + + + + + + + + + + + + + + + + + + + + + + + 
{{ infoModal.content }}
+ + +
+
+
+
+
+ + + + + + + + + + diff --git a/data/templates/report/report.html b/data/templates/report/report.html new file mode 100644 index 00000000..583122a5 --- /dev/null +++ b/data/templates/report/report.html @@ -0,0 +1,1067 @@ + + + + + + OPCT Report + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + +
+ + +
+ +
+ + +
+

CAMGI, Cluster Autoscaler Must Gather Investigator, is a tool for examining OKD/OpenShift must-gather + records to investigate cluster autoscaler behavior and configuration.

+

Steps to use with OPCT:

+ + 

+# Extract the OPCT result file (artifacts.tar.gz)
+mkdir results && \
+tar xfz artifacts.tar.gz -C results
+
+# Extract the must-gather (requires xz)
+mkdir results/must-gather && \
+tar xfJ results/plugins/99-openshift-artifacts-collector/results/global/artifacts_must-gather.tar.xz -C results/must-gather
+
+ + 
./camgi results/must-gather > results/camgi.html
+
+ +
+

TODO: collect the camgd.html in the artifacts plugin.

+
+
+ + + + + + + + diff --git a/docs/README.md b/docs/README.md index f8891d97..b27fa6cc 100644 --- a/docs/README.md +++ b/docs/README.md @@ -9,6 +9,5 @@ Here you can find the initial steps to use the tool: - [User Guide](./user.md) - [Installation Check List](./user-installation-checklist.md) - [Installation Review](./user-installation-review.md) - - [Results Review](./user-results-review.md) - [Support Guide](./support-guide.md) - [Development Guide](./dev.md) \ No newline at end of file diff --git a/docs/dev.md b/docs/dev.md index 3d7e9aac..1497ae1c 100644 --- a/docs/dev.md +++ b/docs/dev.md @@ -53,14 +53,14 @@ Release process checklist: ~~~bash # Example git tag v0.4.0 -m "Release for OPCT v0.4 related to features on OPCT-XXX" -git push --tags upstream +git push upstream ref/tags/v0.4.0 ~~~ - Open a PR updating the [`PluginsImage` value](https://github.com/redhat-openshift-ecosystem/provider-certification-tool/blob/main/pkg/types.go#LL16C2-L16C14) on the CLI repository, merge it; - Create a tag on [CLI/Tool repository](https://github.com/redhat-openshift-ecosystem/provider-certification-tool) based on the `main` branch (or the commit for the release) ~~~bash # Example git tag v0.4.0 -m "Release for OPCT v0.4 related to features on OPCT-XXX" -git push --tags upstream +git push upstream ref/tags/v0.4.0 ~~~ ### Manual tests diff --git a/docs/dev/report.md b/docs/dev/report.md new file mode 100644 index 00000000..719c3111 --- /dev/null +++ b/docs/dev/report.md @@ -0,0 +1,46 @@ +# opct report | development + +This document describe development details about the report. + +First of all, the report is the core component in the review process. +It will extract all the data needed to the review process, transform +it into business logic, aggregating common data, loading it to the +final report data which is consumed to build CLI and HTML report output. + +The input data is the report tarball file, which should have all required data, including must-gather. + +The possible output channels are: + +- CLI stdout +- HTML report file: stored at `/opct-report.html` (a.k.a frontend) +- JSON dataset: stored at `/opct-report.json` +- Log files: stored at `/failures-${plugin}` +- Minimal HTTP file serveer serving the `` as root directory in TCP port 9090 + +Overview of the flow: + +``` mermaid +%%{init: {"flowchart": {"useMaxWidth": false}}}%% + +sequenceDiagram + autonumber + Reviewer->>Reviewer/Output: + Reviewer->>OPCT/Report: ./opct report [opts] --save-to + OPCT/Report->>OPCT/Archive: Extract artifact + OPCT/Archive->>OPCT/Archive: Extract files/metadata/plugins + OPCT/Archive->>OPCT/MustGather: Extract Must Gather from Artifact + OPCT/MustGather->>OPCT/MustGather: Run preprocessors (counters, aggregators) + OPCT/MustGather->>OPCT/Report: Data Loaded + OPCT/Report->>OPCT/Report: Data Transformer/ Processor/ Aggregator/ Checks + OPCT/Report->>Reviewer/Output: Extract test output files + OPCT/Report->>Reviewer/Output: Show CLI output + OPCT/Report->>Reviewer/Output: Save /opct-report.html/json + OPCT/Report->>Reviewer/Output: HTTP server started at :9090 + Reviewer->>Reviewer/Output: Open http://localhost:9090/opct-report.html + Reviewer/Output->>Reviewer/Output: Browser loads data set report.json + Reviewer->>Reviewer/Output: Navigate/Explore the results +``` + +## Frontend + +> TODO: detail about the frontend construct. \ No newline at end of file diff --git a/go.mod b/go.mod index 77033ff7..12c7e7c7 100644 --- a/go.mod +++ b/go.mod @@ -11,13 +11,14 @@ require ( github.com/spf13/viper v1.11.0 github.com/stretchr/testify v1.8.0 github.com/vmware-tanzu/sonobuoy v0.56.10 - github.com/xuri/excelize/v2 v2.6.1 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c k8s.io/api v0.26.1 k8s.io/apimachinery v0.26.1 k8s.io/client-go v0.26.1 ) +require github.com/ulikunitz/xz v0.5.11 + require ( github.com/briandowns/spinner v1.6.1 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect @@ -51,13 +52,11 @@ require ( github.com/moby/spdystream v0.2.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect + github.com/montanaflynn/stats v0.7.1 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pelletier/go-toml v1.9.4 // indirect github.com/pelletier/go-toml/v2 v2.0.0-beta.8 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/richardlehane/mscfb v1.0.4 // indirect - github.com/richardlehane/msoleps v1.0.3 // indirect github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b // indirect @@ -67,9 +66,6 @@ require ( github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.2.0 // indirect - github.com/xuri/efp v0.0.0-20220603152613-6918739fd470 // indirect - github.com/xuri/nfp v0.0.0-20220409054826-5e722a1d9e22 // indirect - golang.org/x/crypto v0.0.0-20220817201139-bc19a97f63c8 // indirect golang.org/x/net v0.3.1-0.20221206200815-1e63c2f08a10 // indirect golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect golang.org/x/sys v0.3.0 // indirect diff --git a/go.sum b/go.sum index 44df0952..25ed7d5f 100644 --- a/go.sum +++ b/go.sum @@ -212,8 +212,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= -github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= +github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= +github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= @@ -234,11 +234,6 @@ github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qR github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/richardlehane/mscfb v1.0.4 h1:WULscsljNPConisD5hR0+OyZjwK46Pfyr6mPu5ZawpM= -github.com/richardlehane/mscfb v1.0.4/go.mod h1:YzVpcZg9czvAuhk9T+a3avCpcFPMUWm7gK3DypaEsUk= -github.com/richardlehane/msoleps v1.0.1/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= -github.com/richardlehane/msoleps v1.0.3 h1:aznSZzrwYRl3rLKRT3gUk9am7T/mLNSnJINvN0AQoVM= -github.com/richardlehane/msoleps v1.0.3/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 h1:mZHayPoR0lNmnHyvtYjDeq0zlVHn9K/ZXoy17ylucdo= github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5/go.mod h1:GEXHk5HgEKCvEIIrSpFI3ozzG5xOKA2DVlEX/gGnewM= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -277,14 +272,10 @@ github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PK github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= +github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/vmware-tanzu/sonobuoy v0.56.10 h1:ONmnCpdL37BqVFQU5brCA2/t7I7P98cpMwMNi+m2H2M= github.com/vmware-tanzu/sonobuoy v0.56.10/go.mod h1:lwHRx/0isQgbw7uegniKjmGGVF2l2Ivp2S3AXiXKuQM= -github.com/xuri/efp v0.0.0-20220603152613-6918739fd470 h1:6932x8ltq1w4utjmfMPVj09jdMlkY0aiA6+Skbtl3/c= -github.com/xuri/efp v0.0.0-20220603152613-6918739fd470/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI= -github.com/xuri/excelize/v2 v2.6.1 h1:ICBdtw803rmhLN3zfvyEGH3cwSmZv+kde7LhTDT659k= -github.com/xuri/excelize/v2 v2.6.1/go.mod h1:tL+0m6DNwSXj/sILHbQTYsLi9IF4TW59H2EF3Yrx1AU= -github.com/xuri/nfp v0.0.0-20220409054826-5e722a1d9e22 h1:OAmKAfT06//esDdpi/DZ8Qsdt4+M5+ltca05dA5bG2M= -github.com/xuri/nfp v0.0.0-20220409054826-5e722a1d9e22/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -302,8 +293,6 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220817201139-bc19a97f63c8 h1:GIAS/yBem/gq2MUqgNIzUHW7cJMmx3TGZOrnyYaNQ6c= -golang.org/x/crypto v0.0.0-20220817201139-bc19a97f63c8/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -316,8 +305,6 @@ golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EH golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.0.0-20220413100746-70e8d0d3baa9 h1:LRtI4W37N+KFebI/qV0OFiLUv4GLOWeEW5hn/KEJvxE= -golang.org/x/image v0.0.0-20220413100746-70e8d0d3baa9/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -370,9 +357,7 @@ golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220812174116-3211cb980234/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.3.1-0.20221206200815-1e63c2f08a10 h1:Frnccbp+ok2GkUS2tC84yAq/U9Vg+0sIO7aRL3T4Xnc= golang.org/x/net v0.3.1-0.20221206200815-1e63c2f08a10/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -433,13 +418,11 @@ golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -452,7 +435,6 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= @@ -624,7 +606,6 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/hack/Containerfile b/hack/Containerfile index e265bb3a..7ded2cf2 100644 --- a/hack/Containerfile +++ b/hack/Containerfile @@ -16,4 +16,4 @@ COPY --from=builder \ /go/src/github.com/redhat-openshift-ecosystem/provider-certification-tool/build/opct-linux-amd64 \ /usr/bin/ -CMD ["/usr/bin/opct-linux-amd64"] \ No newline at end of file +CMD ["/usr/bin/opct-linux-amd64"] diff --git a/hack/Containerfile.ci b/hack/Containerfile.ci index f0531e6f..8a490fa0 100644 --- a/hack/Containerfile.ci +++ b/hack/Containerfile.ci @@ -4,6 +4,6 @@ LABEL io.k8s.display-name="OPCT" \ io.opct.tags="opct,conformance,openshift,tests,e2e" \ io.opct.os="linux" io.opct.arch="amd64" -COPY ./openshift-provider-cert-linux-amd64 /usr/bin/ +COPY ./build/opct-linux-amd64 /usr/bin/ -CMD ["/usr/bin/openshift-provider-cert-linux-amd64"] \ No newline at end of file +CMD ["/usr/bin/opct-linux-amd64"] \ No newline at end of file diff --git a/hack/verify-codegen.sh b/hack/verify-codegen.sh deleted file mode 100755 index bafc9041..00000000 --- a/hack/verify-codegen.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh - -if [ "$IS_CONTAINER" != "" ]; then - go install github.com/go-bindata/go-bindata/go-bindata@latest - set -xe - ./hack/update-generated-bindata.sh - set +ex - git diff --exit-code -else - podman run --rm \ - --env IS_CONTAINER=TRUE \ - --volume "${PWD}:/go/src/github.com/redhat-openshift-ecosystem/provider-certification-tool:z" \ - --workdir /go/src/github.com/redhat-openshift-ecosystem/provider-certification-tool \ - docker.io/golang:1.19 \ - ./hack/verify-codegen.sh "${@}" -fi diff --git a/internal/opct/archive/errorcounter.go b/internal/opct/archive/errorcounter.go new file mode 100644 index 00000000..99cfa7de --- /dev/null +++ b/internal/opct/archive/errorcounter.go @@ -0,0 +1,62 @@ +package archive + +import ( + "regexp" + + "github.com/redhat-openshift-ecosystem/provider-certification-tool/internal/openshift/ci" +) + +// ErrorCounter is a map to handle a generic error counter, indexed by error pattern. +type ErrorCounter map[string]int + +func NewErrorCounter(buf *string, pattern []string) ErrorCounter { + total := 0 + counters := make(ErrorCounter, len(pattern)+2) + + incError := func(err string, cnt int) { + if _, ok := counters[err]; !ok { + counters[err] = 0 + } + counters[err] += cnt + total += cnt + } + + for _, errName := range append(pattern, `error`) { + reErr := regexp.MustCompile(errName) + // Check occurrences in Failure + if matches := reErr.FindAllStringIndex(*buf, -1); len(matches) != 0 { + incError(errName, len(matches)) + } + } + + if total == 0 { + return nil + } + counters["total"] = total + return counters +} + +func MergeErrorCounters(ec1, ec2 *ErrorCounter) *ErrorCounter { + new := make(ErrorCounter, len(ci.CommonErrorPatterns)) + if ec1 == nil { + return &new + } + if ec2 == nil { + return ec1 + } + for kerr, errName := range *ec1 { + if _, ok := new[kerr]; !ok { + new[kerr] = errName + } else { + new[kerr] += errName + } + } + for kerr, errName := range *ec2 { + if _, ok := new[kerr]; !ok { + new[kerr] = errName + } else { + new[kerr] += errName + } + } + return &new +} diff --git a/internal/opct/archive/metaconfig.go b/internal/opct/archive/metaconfig.go new file mode 100644 index 00000000..c9984d35 --- /dev/null +++ b/internal/opct/archive/metaconfig.go @@ -0,0 +1,37 @@ +/* +Handle items in the file path meta/config.json +*/ +package archive + +import ( + sbconfig "github.com/vmware-tanzu/sonobuoy/pkg/config" +) + +type MetaConfigSonobuoy = sbconfig.Config + +// ParseMetaConfig extract relevant attributes to export to data keeper. +func ParseMetaConfig(cfg *MetaConfigSonobuoy) []*RuntimeInfoItem { + var runtimeConfig []*RuntimeInfoItem + + // General Server + runtimeConfig = append(runtimeConfig, &RuntimeInfoItem{Name: "UUID", Value: cfg.UUID}) + runtimeConfig = append(runtimeConfig, &RuntimeInfoItem{Name: "Version", Value: cfg.Version}) + runtimeConfig = append(runtimeConfig, &RuntimeInfoItem{Name: "ResultsDir", Value: cfg.ResultsDir}) + runtimeConfig = append(runtimeConfig, &RuntimeInfoItem{Name: "Namespace", Value: cfg.Namespace}) + + // Plugins + runtimeConfig = append(runtimeConfig, &RuntimeInfoItem{Name: "WorkerImage", Value: cfg.WorkerImage}) + runtimeConfig = append(runtimeConfig, &RuntimeInfoItem{Name: "ImagePullPolicy", Value: cfg.ImagePullPolicy}) + + // Security Config (customized by OPCT) + runtimeConfig = append(runtimeConfig, &RuntimeInfoItem{Name: "AggregatorPermissions", Value: cfg.AggregatorPermissions}) + runtimeConfig = append(runtimeConfig, &RuntimeInfoItem{Name: "ServiceAccountName", Value: cfg.ServiceAccountName}) + existingSA := "no" + if cfg.ExistingServiceAccount { + existingSA = "yes" + } + runtimeConfig = append(runtimeConfig, &RuntimeInfoItem{Name: "ExistingServiceAccount", Value: existingSA}) + runtimeConfig = append(runtimeConfig, &RuntimeInfoItem{Name: "SecurityContextMode", Value: cfg.SecurityContextMode}) + + return runtimeConfig +} diff --git a/internal/opct/archive/metalog.go b/internal/opct/archive/metalog.go new file mode 100644 index 00000000..7bba17f3 --- /dev/null +++ b/internal/opct/archive/metalog.go @@ -0,0 +1,128 @@ +/* +Handle items in the file path meta/run.log +*/ +package archive + +import ( + "encoding/json" + "fmt" + "strings" + "time" + + log "github.com/sirupsen/logrus" +) + +type MetaLogItem struct { + Level string `json:"level,omitempty"` + Message string `json:"msg,omitempty"` + Time string `json:"time,omitempty"` + Plugin string `json:"plugin,omitempty"` + Method string `json:"method,omitempty"` + PluginName string `json:"plugin_name,omitempty"` +} + +func ParseMetaLogs(logs []string) []*RuntimeInfoItem { + + var serverStartedAt string + runtimeLogs := []*RuntimeInfoItem{} + exists := struct{}{} + mapExists := map[string]struct{}{} + pluginStartedAt := map[string]string{} + pluginFinishedAt := map[string]string{} + + // convert from ISO8601 [returning errors] to: + dateFormat := "2006/01/02 15:04:05" + convertDate := func(t string) string { + new := strings.Replace(t, "-", "/", -1) + new = strings.Replace(new, "T", " ", -1) + new = strings.Replace(new, "Z", "", -1) + return new + } + diffDate := func(strStart string, strEnd string) string { + start, err := time.Parse(dateFormat, convertDate(strStart)) + if err != nil { + fmt.Println("start") + fmt.Println(err) + } + end, err := time.Parse(dateFormat, convertDate(strEnd)) + if err != nil { + fmt.Println("dateEnd") + fmt.Println(err) + } + return end.Sub(start).String() + } + + // parse meta/run.log + for x := range logs { + logitem := MetaLogItem{} + if err := json.Unmarshal([]byte(logs[x]), &logitem); err != nil { + log.Debugf("Erorr: [parser] couldn't parse item in meta/run.log: %v", err) + continue + } + + // server started: msg=Starting server Expected Results + if strings.HasPrefix(logitem.Message, "Starting server Expected Results") { + runtimeLogs = append(runtimeLogs, &RuntimeInfoItem{ + Time: logitem.Time, + Name: "server started", + }) + serverStartedAt = logitem.Time + } + + // marker: plugin started (healthy) + if logitem.Method == "POST" && logitem.Message == "received request" { + // Get only the first message indicating the plugin has been started + if _, ok := mapExists[logitem.PluginName]; ok { + continue + } + mapExists[logitem.PluginName] = exists + runtimeLogs = append(runtimeLogs, &RuntimeInfoItem{ + Time: logitem.Time, + Name: fmt.Sprintf("plugin started %s", logitem.PluginName), + }) + pluginStartedAt[logitem.PluginName] = logitem.Time + } + + // marker: plugin finished + if logitem.Method == "PUT" { + pluginFinishedAt[logitem.PluginName] = logitem.Time + var delta string + switch logitem.PluginName { + case "05-openshift-cluster-upgrade": + delta = diffDate(pluginStartedAt[logitem.PluginName], logitem.Time) + case "10-openshift-kube-conformance": + delta = diffDate(pluginFinishedAt["05-openshift-cluster-upgrade"], logitem.Time) + case "20-openshift-conformance-validated": + delta = diffDate(pluginFinishedAt["10-openshift-kube-conformance"], logitem.Time) + case "99-openshift-artifacts-collector": + delta = diffDate(pluginFinishedAt["20-openshift-conformance-validated"], logitem.Time) + } + runtimeLogs = append(runtimeLogs, &RuntimeInfoItem{ + Name: fmt.Sprintf("plugin finished %s", logitem.PluginName), + Time: logitem.Time, + Total: diffDate(pluginStartedAt[logitem.PluginName], logitem.Time), + Delta: delta, + }) + } + + // marker: plugin cleaned + if logitem.Message == "Invoking plugin cleanup" { + msg := "server finished" + if _, ok := mapExists[msg]; !ok { + runtimeLogs = append(runtimeLogs, &RuntimeInfoItem{ + Name: msg, + Time: logitem.Time, + Total: diffDate(serverStartedAt, logitem.Time), + }) + } + mapExists[msg] = exists + } + + // More events: + // Shutting down aggregation server + // server collector started: Running cluster queries + // server collector finished + } + + return runtimeLogs +} diff --git a/internal/opct/archive/opctconfigmap.go b/internal/opct/archive/opctconfigmap.go new file mode 100644 index 00000000..3839a929 --- /dev/null +++ b/internal/opct/archive/opctconfigmap.go @@ -0,0 +1,31 @@ +/* +Handle items in the file path resources/ns/{opct_namespace}/core_v1_configmaps.json +*/ +package archive + +import ( + v1 "k8s.io/api/core/v1" +) + +func OpctConfigMapNames() []string { + return []string{ + "openshift-provider-certification-version", + "plugins-config", + } +} + +func ParseOpctConfig(cms *v1.ConfigMapList) []*RuntimeInfoItem { + var cmData []*RuntimeInfoItem + for _, cm := range cms.Items { + + switch cm.ObjectMeta.Name { + case "openshift-provider-certification-version", "plugins-config": + default: + continue + } + for k, v := range cm.Data { + cmData = append(cmData, &RuntimeInfoItem{Config: cm.ObjectMeta.Name, Name: k, Value: v}) + } + } + return cmData +} diff --git a/internal/opct/archive/pluginsdefinition.go b/internal/opct/archive/pluginsdefinition.go new file mode 100644 index 00000000..41a429aa --- /dev/null +++ b/internal/opct/archive/pluginsdefinition.go @@ -0,0 +1,5 @@ +/* +Handle items in the file path: plugins/{plugin_name}/definition.json +Summary of content: Plugin Definition containing all information of plugin configuration. +*/ +package archive diff --git a/internal/opct/archive/runtime.go b/internal/opct/archive/runtime.go new file mode 100644 index 00000000..a18e26d4 --- /dev/null +++ b/internal/opct/archive/runtime.go @@ -0,0 +1,16 @@ +package archive + +type RuntimeInfoItem struct { + // Name holds the name of the item/attribute. + Name string `json:"name"` + // Value holds the value of the item/attribute. + Value string `json:"value"` + // Config is a optional field containing extra config information from the item. + Config string `json:"config,omitempty"` + // Time is a optional field with the timestamp in the datetime format. + Time string `json:"time,omitempty"` + // Total is a optional field with the calculation of total time the item take. + Total string `json:"total,omitempty"` + // Detal is a optional field with the calculation of difference between two timers. + Delta string `json:"delta,omitempty"` +} diff --git a/internal/opct/metrics/timers.go b/internal/opct/metrics/timers.go new file mode 100644 index 00000000..8ec957df --- /dev/null +++ b/internal/opct/metrics/timers.go @@ -0,0 +1,44 @@ +package metrics + +import "time" + +type Timers struct { + Timers map[string]*Timer `json:"Timers,omitempty"` + last string +} + +func NewTimers() Timers { + ts := Timers{Timers: make(map[string]*Timer)} + return ts +} + +// set a timer, updating if existing. +func (ts *Timers) set(k string) { + if _, ok := ts.Timers[k]; !ok { + ts.Timers[k] = &Timer{start: time.Now()} + } else { + stop := time.Now() + ts.Timers[k].Total = stop.Sub(ts.Timers[k].start).Seconds() + } +} + +// Set check last timer, stop and add a new one (lap). +func (ts *Timers) Set(k string) { + if ts.last != "" { + ts.set(ts.last) + } + ts.set(k) + ts.last = k +} + +// Add a new timer. +func (ts *Timers) Add(k string) { + ts.set(k) +} + +type Timer struct { + start time.Time + + // Total time in milisseconds + Total float64 `json:"seconds"` +} diff --git a/internal/opct/plugin/plugin.go b/internal/opct/plugin/plugin.go new file mode 100644 index 00000000..c81d5dd1 --- /dev/null +++ b/internal/opct/plugin/plugin.go @@ -0,0 +1,81 @@ +package plugin + +import ( + "github.com/redhat-openshift-ecosystem/provider-certification-tool/internal/opct/archive" + "github.com/redhat-openshift-ecosystem/provider-certification-tool/internal/openshift/ci" +) + +const ( + PluginNameOpenShiftUpgrade = "05-openshift-cluster-upgrade" + PluginNameKubernetesConformance = "10-openshift-kube-conformance" + PluginNameOpenShiftConformance = "20-openshift-conformance-validated" + PluginNameArtifactsCollector = "99-openshift-artifacts-collector" + + // Old Plugin names (prior v0.2). It's used to keep compatibility + PluginOldNameKubernetesConformance = "openshift-kube-conformance" + PluginOldNameOpenShiftConformance = "openshift-conformance-validated" +) + +type PluginDefinition struct { + PluginImage string `json:"pluginImage"` + SonobuoyImage string `json:"sonobuoyImage"` + Name string `json:"name"` +} + +// OPCTPluginSummary handle plugin details +type OPCTPluginSummary struct { + Name string + NameAlias string + Status string + Total int64 + Passed int64 + Failed int64 + Timeout int64 + Skipped int64 + + // FailedItems is the map with details for each failure + Tests Tests + + // FailedList is the list of tests failures on the original execution + FailedList []string + + // FailedFilterSuite is the list of failures (A) included only in the original suite (B): A INTERSECTION B + FailedFilterSuite []string + + // FailedFilterBaseline is the list of failures (A) excluding the baseline(B): A EXCLUDE B + FailedFilterBaseline []string + + // FailedFilterPrio is the priority list of failures - not reporting as flake in OpenShift CI. + FailedFilterPrio []string + + // DocumentationReference + Documentation *TestDocumentation + + // Definition + Definition *PluginDefinition + + ErrorCounters archive.ErrorCounter `json:"errorCounters,omitempty"` +} + +func (ps *OPCTPluginSummary) calculateErrorCounter() *archive.ErrorCounter { + if ps.ErrorCounters == nil { + ps.ErrorCounters = make(archive.ErrorCounter, len(ci.CommonErrorPatterns)) + } + for _, test := range ps.Tests { + if test.ErrorCounters == nil { + continue + } + for kerr, errName := range test.ErrorCounters { + if _, ok := ps.ErrorCounters[kerr]; !ok { + ps.ErrorCounters[kerr] = errName + } else { + ps.ErrorCounters[kerr] += errName + } + } + } + return &ps.ErrorCounters +} + +func (ps *OPCTPluginSummary) GetErrorCounters() *archive.ErrorCounter { + return ps.calculateErrorCounter() +} diff --git a/internal/opct/plugin/tags.go b/internal/opct/plugin/tags.go new file mode 100644 index 00000000..9dba2651 --- /dev/null +++ b/internal/opct/plugin/tags.go @@ -0,0 +1,95 @@ +package plugin + +import ( + "fmt" + "regexp" + "sort" +) + +const tagRegex = `^\[([a-zA-Z0-9-]*)\]` + +// SortedData stores the key/value to be sorted. +type SortedData struct { + Key string + Value int +} + +// SortedList stores the list of key/value map, implementing interfaces +// to sort/rank a map strings with integers as values. +type SortedList []SortedData + +func (p SortedList) Len() int { return len(p) } +func (p SortedList) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p SortedList) Less(i, j int) bool { return p[i].Value < p[j].Value } + +// TestTags stores the test tags map with it's counter. +// The test tag is the work extracted from the first bracket from a test name. +// Example test name: '[sig-provider] test name' the 'sig-provider' is the tag. +type TestTags map[string]int + +// NewTestTagsEmpty creates the TestTags with a specific size, to be populated later. +func NewTestTagsEmpty(size int) TestTags { + tt := make(TestTags, size) + tt["total"] = 0 + return tt +} + +// NewTestTags creates the TestTags populating the tag values and counters. +func NewTestTags(tests []*string) TestTags { + tt := make(TestTags, len(tests)) + tt["total"] = 0 + tt.addBatch(tests) + return tt +} + +// Add extracts tags from test name, store, and increment the counter. +func (tt TestTags) Add(test *string) { + reT := regexp.MustCompile(tagRegex) + match := reT.FindStringSubmatch(*test) + if len(match) > 0 { + if _, ok := tt[match[1]]; !ok { + tt[match[1]] = 1 + } else { + tt[match[1]] += 1 + } + } + tt["total"] += 1 +} + +// AddBatch receive a list of test name (string slice) and stores it. +func (tt TestTags) addBatch(kn []*string) { + for _, test := range kn { + tt.Add(test) + } +} + +// SortRev creates a rank of tags. +func (tt TestTags) sortRev() []SortedData { + tags := make(SortedList, len(tt)) + i := 0 + for k, v := range tt { + tags[i] = SortedData{k, v} + i++ + } + sort.Sort(sort.Reverse(tags)) + return tags +} + +// ShowSorted return an string with the rank of tags. +func (tt TestTags) ShowSorted() string { + tags := tt.sortRev() + msg := "" + for _, k := range tags { + if k.Key == "total" { + msg = fmt.Sprintf("[%v=%v]", k.Key, k.Value) + continue + } + msg = fmt.Sprintf("%s [%v=%s]", msg, k.Key, UtilsCalcPercStr(int64(k.Value), int64(tt["total"]))) + } + return msg +} + +// calcPercStr receives the numerator and denominator and return the numerator and percentage as string. +func UtilsCalcPercStr(num, den int64) string { + return fmt.Sprintf("%d (%.2f%%)", num, (float64(num)/float64(den))*100) +} diff --git a/internal/opct/plugin/tags_test.go b/internal/opct/plugin/tags_test.go new file mode 100644 index 00000000..02ff4363 --- /dev/null +++ b/internal/opct/plugin/tags_test.go @@ -0,0 +1,46 @@ +package plugin + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func validTests(testDesc *string) []*string { + tests := []*string{} + prefix := "tag" + max := 5 + + for i := 1; i <= max; i++ { + for x := (max - i); x >= 0; x-- { + test := fmt.Sprintf("[%s-%d] %s ID %d", prefix, i, *testDesc, i) + tests = append(tests, &test) + } + } + return tests +} + +func TestShowSorted(t *testing.T) { + desc := "TestShowSorted" + cases := []struct { + name string + tests []*string + want string + }{ + { + name: "empty", + tests: validTests(&desc), + want: "[total=15] [tag-1=5 (33.33%)] [tag-2=4 (26.67%)] [tag-3=3 (20.00%)] [tag-4=2 (13.33%)] [tag-5=1 (6.67%)]", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // fmt.Printf("%v\n", tc.tests) + testTags := NewTestTags(tc.tests) + msg := testTags.ShowSorted() + assert.Equal(t, tc.want, msg, "unexpected ,essage") + }) + } +} diff --git a/internal/opct/plugin/test.go b/internal/opct/plugin/test.go new file mode 100644 index 00000000..2508171c --- /dev/null +++ b/internal/opct/plugin/test.go @@ -0,0 +1,99 @@ +package plugin + +import ( + "fmt" + "regexp" + "strings" + + "github.com/redhat-openshift-ecosystem/provider-certification-tool/internal/opct/archive" + "github.com/redhat-openshift-ecosystem/provider-certification-tool/internal/openshift/ci" + "github.com/redhat-openshift-ecosystem/provider-certification-tool/internal/openshift/ci/sippy" +) + +type Tests map[string]*TestItem + +type TestItem struct { + // Name is the name of the e2e test. It is hidden from JSON as Tests is a map, and + // the key can be used. + Name string `json:"-"` + + // ID is the unique identifier of the test within the execution. + ID string `json:"id"` + + // Status store the test result. Valid values: passed, skipped, failed. + Status string `json:"status"` + + // State represents the state of the test. It can be any status value or filter name. + State string `json:"state,omitempty"` + + // Failure contains the failure reason extracted from JUnit field 'item.detials.failure'. + Failure string `json:"-"` + + // SystemOut contains the entire test stdout extracted from JUnit field 'item.detials.system-out'. + SystemOut string `json:"-"` + + // Offset is the offset of failure from the plugin result file. + Offset int `json:"-"` + + // Flaky contains the flake information from OpenShift CI - scraped from Sippy API. + Flake *sippy.SippyTestsResponse `json:"flake,omitempty"` + + // ErrorCounters errors indexed by common error key. + ErrorCounters archive.ErrorCounter `json:"errorCounters,omitempty"` + + // Reference for documentation. + Documentation string `json:"documentation"` +} + +// UpdateErrorCounter reads the failures and stdout looking for error patterns from +// a specific test, accumulating the ErrorCounters structure. +func (pi *TestItem) UpdateErrorCounter() { + total := 0 + counters := make(archive.ErrorCounter, len(ci.CommonErrorPatterns)+1) + + incError := func(err string, cnt int) { + if _, ok := counters[err]; !ok { + counters[err] = 0 + } + counters[err] += cnt + total += cnt + } + + for _, errName := range ci.CommonErrorPatterns { + reErr := regexp.MustCompile(errName) + // Check occurrences in Failure + if matches := reErr.FindAllStringIndex(pi.Failure, -1); len(matches) != 0 { + incError(errName, len(matches)) + } + // Check occurrences in SystemOut + if matches := reErr.FindAllStringIndex(pi.SystemOut, -1); len(matches) != 0 { + incError(errName, len(matches)) + } + } + + if total == 0 { + return + } + pi.ErrorCounters = counters + pi.ErrorCounters["total"] = total +} + +// LookupDocumentation extracts from the test name the expected part (removing '[Conformance]') +// to link to the Documentation URL refereced by the Kubernetes Conformance markdown available +// at https://github.com/cncf/k8s-conformance/blob/master/docs/KubeConformance-.md . +// The test documentation (TestDocumentation) should be indexed prior calling the LookupDocumentation. +func (pi *TestItem) LookupDocumentation(d *TestDocumentation) { + + // origin/openshift-tests appends 'labels' after '[Conformance]' in the + // test name in the kubernetes/conformance, transforming it from the original name from upstream. + // nameIndex will try to recover the original name to lookup in the source docs. + nameIndex := fmt.Sprintf("%s[Conformance]", strings.Split(pi.Name, "[Conformance]")[0]) + + // check if the test name is indexed in the conformance documentation. + if _, ok := d.Tests[nameIndex]; ok { + pi.Documentation = d.Tests[nameIndex].URLFragment + return + } + // When the test is not indexed, no documentation will be added. + pi.Documentation = *d.UserBaseURL +} diff --git a/internal/opct/plugin/testdoc.go b/internal/opct/plugin/testdoc.go new file mode 100644 index 00000000..805e0d27 --- /dev/null +++ b/internal/opct/plugin/testdoc.go @@ -0,0 +1,110 @@ +package plugin + +import ( + "fmt" + "io" + "net/http" + "regexp" + "strings" + + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" +) + +type TestDocumentation struct { + // UserBaseURL is a the User Facing base URL for the documentation. + UserBaseURL *string + + // SourceBaseURL is the raw URL to be indexed. + SourceBaseURL *string + + // Raw stores the data extracted from SourceBaseURL. + Raw *string + + // Tests is the map indexed by test name, with URL fragment (page references) as a value. + // Example: for the e2e test '[sig-machinery] run instance', the following map will be created: + // map['[sig-machinery] run instance']='#sig-machinery--run-instance' + Tests map[string]*TestDocumentationItem +} + +// TestDocumentationItem refers to items documented by +type TestDocumentationItem struct { + Title string + Name string + // URLFragment stores the discovered fragment parsed by the Documentation page, + // indexed by test name, used to mount the Documentation URL for failed tests. + URLFragment string +} + +func NewTestDocumentation(user, source string) *TestDocumentation { + return &TestDocumentation{ + UserBaseURL: &user, + SourceBaseURL: &source, + } +} + +// Load documentation from Suite and save it to further query +func (d *TestDocumentation) Load() error { + app := "Test Documentation" + req, err := http.NewRequest(http.MethodGet, *d.SourceBaseURL, nil) + if err != nil { + return errors.Wrapf(err, "failed to create request to get %s", app) + } + res, err := http.DefaultClient.Do(req) + if err != nil { + return errors.Wrapf(err, "failed to make request to %s", app) + } + defer res.Body.Close() + + if res.StatusCode != http.StatusOK { + return errors.New(fmt.Sprintf("unexpected HTTP status code to %s", app)) + } + + resBody, err := io.ReadAll(res.Body) + if err != nil { + return errors.Wrapf(err, "failed to read response body for %s", app) + } + str := string(resBody) + d.Raw = &str + return nil +} + +// BuildIndex reads the raw Document, discoverying the test name, and the URL +// fragments. The parser is based in the Kubernetes Conformance documentation: +// https://github.com/cncf/k8s-conformance/blob/master/docs/KubeConformance-1.27.md +func (d *TestDocumentation) BuildIndex() error { + lines := strings.Split(*d.Raw, "\n") + d.Tests = make(map[string]*TestDocumentationItem, len(lines)) + for number, line := range lines { + + // Build index for Kubernetes Conformance tests, parsing the page for version: + // https://github.com/cncf/k8s-conformance/blob/master/docs/KubeConformance-1.27.md + if strings.HasPrefix(line, "- Defined in code as: ") { + testArr := strings.Split(line, "Defined in code as: ") + if len(testArr) < 2 { + log.Debugf("Error BuildIndex(): unable to build documentation index for line: %s", line) + } + testName := testArr[1] + d.Tests[testName] = &TestDocumentationItem{ + Name: testName, + // The test reference/section are defined in the third line before the name definition. + Title: lines[number-3], + } + + // create url fragment for each test section + reDoc := regexp.MustCompile(`^## \[(.*)\]`) + match := reDoc.FindStringSubmatch(lines[number-3]) + if len(match) == 2 { + fragment := match[1] + // mount the fragment removing undesired symbols. + for _, c := range []string{":", "-", ".", ",", "="} { + fragment = strings.Replace(fragment, c, "", -1) + } + fragment = strings.Replace(fragment, " ", "-", -1) + fragment = strings.ToLower(fragment) + d.Tests[testName].URLFragment = fmt.Sprintf("%s#%s", *d.UserBaseURL, fragment) + } + } + } + return nil +} diff --git a/internal/opct/report/checks.go b/internal/opct/report/checks.go new file mode 100644 index 00000000..50537a95 --- /dev/null +++ b/internal/opct/report/checks.go @@ -0,0 +1,480 @@ +/* +Checks handles all acceptance criteria from data +collected and processed in summary package. + +Existing Checks: +- OPCT-001: "Plugin Conformance Kubernetes [10-openshift-kube-conformance] must pass (after filters)" +- OPCT-002: "Plugin Conformance Upgrade [05-openshift-cluster-upgrade] must pass" +- OPCT-003: "Plugin Collector [99-openshift-artifacts-collector] must pass" +- ...TBD +*/ +package report + +import ( + "fmt" + "os" + "strconv" + "strings" + + "github.com/redhat-openshift-ecosystem/provider-certification-tool/internal/opct/plugin" + log "github.com/sirupsen/logrus" +) + +const ( + docsRulesPath = "/review/rules" + defaultBaseURL = "https://redhat-openshift-ecosystem.github.io/provider-certification-tool" +) + +type CheckSummary struct { + baseURL string + Checks []*Check `json:"checks"` +} + +func NewCheckSummary(re *Report) *CheckSummary { + + baseURL := defaultBaseURL + msgDefaultNotMatch := "default value does not match the acceptance criteria" + // Developer environment: + // $ mkdocs serve + // $ export OPCT_DEV_BASE_URL_DOC="http://127.0.0.1:8000/provider-certification-tool" + localDevBaseURL := os.Getenv("OPCT_DEV_BASE_URL_DOC") + if localDevBaseURL != "" { + baseURL = localDevBaseURL + } + checkSum := &CheckSummary{ + Checks: []*Check{}, + baseURL: fmt.Sprintf("%s%s", baseURL, docsRulesPath), + } + + // OpenShift / Infrastructure Object Check + checkSum.Checks = append(checkSum.Checks, &Check{ + Name: "Platform Type should be None or External", + Test: func() CheckResult { + prefix := "Check OPCT-TBD Failed" + if re.Provider == nil || re.Provider.Infra == nil { + log.Debugf("%s: unable to read the infrastructure object", prefix) + return CheckResultFail + } + // Acceptance Criterias + if re.Provider.Infra.PlatformType == "None" { + return CheckResultPass + } + if re.Provider.Infra.PlatformType == "External" { + return CheckResultPass + } + log.Debugf("%s (Platform Type): %s: got=[%s]", prefix, msgDefaultNotMatch, re.Provider.Infra.PlatformType) + return CheckResultFail + }, + }) + checkSum.Checks = append(checkSum.Checks, &Check{ + Name: "Cluster Version Operator must be Available", + Test: func() CheckResult { + if re.Provider == nil || re.Provider.Version == nil || re.Provider.Version.OpenShift == nil { + return CheckResultFail + } + if re.Provider.Version.OpenShift.CondAvailable != "True" { + return CheckResultFail + } + return CheckResultPass + }, + }) + checkSum.Checks = append(checkSum.Checks, &Check{ + Name: "Cluster condition Failing must be False", + Test: func() CheckResult { + if re.Provider == nil || re.Provider.Version == nil || re.Provider.Version.OpenShift == nil { + return CheckResultFail + } + if re.Provider.Version.OpenShift.CondFailing != "False" { + return CheckResultFail + } + return CheckResultPass + }, + }) + checkSum.Checks = append(checkSum.Checks, &Check{ + Name: "Cluster upgrade must not be Progressing", + Test: func() CheckResult { + if re.Provider == nil || re.Provider.Version == nil || re.Provider.Version.OpenShift == nil { + return CheckResultFail + } + if re.Provider.Version.OpenShift.CondProgressing != "False" { + return CheckResultFail + } + return CheckResultPass + }, + }) + checkSum.Checks = append(checkSum.Checks, &Check{ + Name: "Cluster ReleaseAccepted must be True", + Test: func() CheckResult { + if re.Provider == nil || re.Provider.Version == nil || re.Provider.Version.OpenShift == nil { + return CheckResultFail + } + if re.Provider.Version.OpenShift.CondReleaseAccepted != "True" { + return CheckResultFail + } + return CheckResultPass + }, + }) + checkSum.Checks = append(checkSum.Checks, &Check{ + Name: "Infrastructure status must have Topology=HighlyAvailable", + Test: func() CheckResult { + if re.Provider == nil || re.Provider.Infra == nil { + return CheckResultFail + } + if re.Provider.Infra.Topology != "HighlyAvailable" { + return CheckResultFail + } + return CheckResultPass + }, + }) + checkSum.Checks = append(checkSum.Checks, &Check{ + Name: "Infrastructure status must have ControlPlaneTopology=HighlyAvailable", + Test: func() CheckResult { + if re.Provider == nil || re.Provider.Infra == nil { + return CheckResultFail + } + if re.Provider.Infra.ControlPlaneTopology != "HighlyAvailable" { + return CheckResultFail + } + return CheckResultPass + }, + }) + checkSum.Checks = append(checkSum.Checks, &Check{ + ID: "OPCT-008", + Name: "All nodes must be healthy", + Test: func() CheckResult { + if re.Provider == nil || re.Provider.ClusterHealth == nil { + log.Debugf("Check Failed: OPCT-008: unavailable results") + return CheckResultFail + } + if re.Provider.ClusterHealth.NodeHealthPerc != 100 { + log.Debugf("Check Failed: OPCT-008: want[!=100] got[%f]", re.Provider.ClusterHealth.NodeHealthPerc) + return CheckResultFail + } + return CheckResultPass + }, + }) + checkSum.Checks = append(checkSum.Checks, &Check{ + ID: "OPCT-009", + Name: "Pods Healthy must report higher than 98%", + Test: func() CheckResult { + if re.Provider == nil || re.Provider.ClusterHealth == nil { + return CheckResultFail + } + if re.Provider.ClusterHealth.PodHealthPerc < 98.0 { + return CheckResultFail + } + return CheckResultPass + }, + }) + // Plugins Checks + checkSum.Checks = append(checkSum.Checks, &Check{ + ID: "OPCT-001", + Name: "Plugin Conformance Kubernetes [10-openshift-kube-conformance] must pass (after filters)", + Test: func() CheckResult { + prefix := "Check OPCT-001 Failed" + if _, ok := re.Provider.Plugins[plugin.PluginNameKubernetesConformance]; !ok { + log.Debugf("%s Runtime: processed plugin data not found: %v", prefix, re.Provider.Plugins[plugin.PluginNameKubernetesConformance]) + return CheckResultFail + } + p := re.Provider.Plugins[plugin.PluginNameKubernetesConformance] + if p.Stat.Total == p.Stat.Failed { + log.Debugf("%s Runtime: Total and Failed counters are equals indicating execution failure", prefix) + return CheckResultFail + } + if len(p.TestsFailedPrio) > 0 { + log.Debugf("%s Acceptance criteria: TestsFailedPrio counter are greater than 0: %v", prefix, len(p.TestsFailedPrio)) + return CheckResultFail + } + return CheckResultPass + }, + }) + checkSum.Checks = append(checkSum.Checks, &Check{ + ID: "OPCT-004", + Name: "OpenShift Conformance [20-openshift-conformance-validated]: Failed tests must report less than 1.5%", + Test: func() CheckResult { + if _, ok := re.Provider.Plugins[plugin.PluginNameOpenShiftConformance]; !ok { + return CheckResultFail + } + // "Acceptance" are relative, the baselines is observed to set + // an "accepted" value considering a healthy cluster in known provider/installation. + plugin := re.Provider.Plugins[plugin.PluginNameOpenShiftConformance] + perc := (float64(plugin.Stat.Failed) / float64(plugin.Stat.Total)) * 100 + if perc > 1.5 { + return CheckResultFail + } + return CheckResultPass + }, + }) + checkSum.Checks = append(checkSum.Checks, &Check{ + ID: "OPCT-005", + Name: "OpenShift Conformance [20-openshift-conformance-validated]: Priority must report less than 0.5%", + Test: func() CheckResult { + if _, ok := re.Provider.Plugins[plugin.PluginNameOpenShiftConformance]; !ok { + return CheckResultFail + } + // "Acceptance" are relative, the baselines is observed to set + // an "accepted" value considering a healthy cluster in known provider/installation. + plugin := re.Provider.Plugins[plugin.PluginNameOpenShiftConformance] + perc := (float64(plugin.Stat.FilterFailedPrio) / float64(plugin.Stat.Total)) * 100 + if perc > 0.5 { + return CheckResultFail + } + return CheckResultPass + }, + }) + checkSum.Checks = append(checkSum.Checks, &Check{ + ID: "OPCT-006", + Name: "Suite Errors must report a lower number of log errors", + Test: func() CheckResult { + if re.Provider.ErrorCounters == nil { + return CheckResultFail + } + cnt := *re.Provider.ErrorCounters + if _, ok := cnt["total"]; !ok { + return CheckResultFail + } + // "Acceptance" are relative, the baselines is observed to set + // an "accepted" value considering a healthy cluster in known provider/installation. + total := cnt["total"] + if total > 150 { + return CheckResultFail + } + // 0? really? something went wrong! + if total == 0 { + return CheckResultFail + } + return CheckResultPass + }, + }) + checkSum.Checks = append(checkSum.Checks, &Check{ + ID: "OPCT-007", + Name: "Workloads must report a lower number of errors in the logs", + Test: func() CheckResult { + prefix := "Check OPCT-007 Failed" + if re.Provider.MustGatherInfo == nil { + log.Debugf("%s: MustGatherInfo is not defined", prefix) + return CheckResultFail + } + if _, ok := re.Provider.MustGatherInfo.ErrorCounters["total"]; !ok { + log.Debugf("%s: OPCT-007: ErrorCounters[\"total\"]", prefix) + return CheckResultFail + } + // "Acceptance" are relative, the baselines is observed to set + // an "accepted" value considering a healthy cluster in known provider/installation. + total := re.Provider.MustGatherInfo.ErrorCounters["total"] + wantLimit := 30000 + if total > wantLimit { + log.Debugf("%s acceptance criteria: want[<=%d] got[%d]", prefix, wantLimit, total) + return CheckResultFail + } + // 0? really? something went wrong! + if total == 0 { + log.Debugf("%s acceptance criteria: want[!=0] got[%d]", prefix, total) + return CheckResultFail + } + return CheckResultPass + }, + }) + checkSum.Checks = append(checkSum.Checks, &Check{ + ID: "OPCT-003", + Name: "Plugin Collector [99-openshift-artifacts-collector] must pass", + Test: func() CheckResult { + prefix := "Check OPCT-003 Failed" + if _, ok := re.Provider.Plugins[plugin.PluginNameArtifactsCollector]; !ok { + return CheckResultFail + } + p := re.Provider.Plugins[plugin.PluginNameArtifactsCollector] + if p.Stat.Total == p.Stat.Failed { + log.Debugf("%s Runtime: Total and Failed counters are equals indicating execution failure", prefix) + return CheckResultFail + } + // Acceptance check + if re.Provider.Plugins[plugin.PluginNameArtifactsCollector].Stat.Status == "passed" { + return CheckResultPass + } + log.Debugf("%s: %s", prefix, msgDefaultNotMatch) + return CheckResultFail + }, + }) + checkSum.Checks = append(checkSum.Checks, &Check{ + ID: "OPCT-002", + Name: "Plugin Conformance Upgrade [05-openshift-cluster-upgrade] must pass", + Test: func() CheckResult { + if _, ok := re.Provider.Plugins[plugin.PluginNameOpenShiftUpgrade]; !ok { + return CheckResultFail + } + if re.Provider.Plugins[plugin.PluginNameOpenShiftUpgrade].Stat.Status != "passed" { + return CheckResultFail + } + return CheckResultPass + }, + }) + // TODO(etcd) + /* + checkSum.Checks = append(checkSum.Checks, &Check{ + Name: "[TODO] etcd fio must accept the tests (TODO)", + Test: AcceptanceCheckFail, + }) + checkSum.Checks = append(checkSum.Checks, &Check{ + Name: "[TODO] etcd slow requests: p99 must be lower than 900ms", + Test: AcceptanceCheckFail, + }) + */ + checkSum.Checks = append(checkSum.Checks, &Check{ + ID: "OPCT-010", + Name: "etcd logs: slow requests: average should be under 500ms", + Test: func() CheckResult { + prefix := "Check OPCT-010 Failed" + wantLimit := 500.0 + if re.Provider.MustGatherInfo == nil { + log.Debugf("%s: unable to read must-gather information.", prefix) + return CheckResultFail + } + if re.Provider.MustGatherInfo.ErrorEtcdLogs.FilterRequestSlowAll["all"] == nil { + log.Debugf("%s: unable to read statistics from parsed etcd logs.", prefix) + return CheckResultFail + } + if re.Provider.MustGatherInfo.ErrorEtcdLogs.FilterRequestSlowAll["all"].StatMean == "" { + log.Debugf("%s: unable to get p50/mean statistics from parsed data: %v", prefix, re.Provider.MustGatherInfo.ErrorEtcdLogs.FilterRequestSlowAll["all"]) + return CheckResultFail + } + values := strings.Split(re.Provider.MustGatherInfo.ErrorEtcdLogs.FilterRequestSlowAll["all"].StatMean, " ") + if values[0] == "" { + log.Debugf("%s: unable to get parse p50/mean: %v", prefix, values) + return CheckResultFail + } + value, err := strconv.ParseFloat(values[0], 64) + if err != nil { + log.Debugf("%s: unable to convert p50/mean to float: %v", prefix, err) + return CheckResultFail + } + if value >= wantLimit { + log.Debugf("%s acceptance criteria: want=[<%.0f] got=[%v]", prefix, wantLimit, value) + return CheckResultFail + } + return CheckResultPass + }, + }) + checkSum.Checks = append(checkSum.Checks, &Check{ + ID: "OPCT-011", + Name: "etcd logs: slow requests: maximum should be under 1000ms", + Test: func() CheckResult { + prefix := "Check OPCT-011 Failed" + wantLimit := 1000.0 + if re.Provider.MustGatherInfo == nil { + log.Debugf("%s: unable to read must-gather information.", prefix) + return CheckResultFail + } + if re.Provider.MustGatherInfo.ErrorEtcdLogs.FilterRequestSlowAll["all"] == nil { + log.Debugf("%s: unable to read statistics from parsed etcd logs.", prefix) + return CheckResultFail + } + if re.Provider.MustGatherInfo.ErrorEtcdLogs.FilterRequestSlowAll["all"].StatMax == "" { + log.Debugf("%s: unable to get max statistics from parsed data: %v", prefix, re.Provider.MustGatherInfo.ErrorEtcdLogs.FilterRequestSlowAll["all"]) + return CheckResultFail + } + values := strings.Split(re.Provider.MustGatherInfo.ErrorEtcdLogs.FilterRequestSlowAll["all"].StatMax, " ") + if values[0] == "" { + log.Debugf("%s: unable to get parse max: %v", prefix, values) + return CheckResultFail + } + value, err := strconv.ParseFloat(values[0], 64) + if err != nil { + log.Debugf("%s: unable to convert max to float: %v", prefix, err) + return CheckResultFail + } + if value >= wantLimit { + log.Debugf("%s acceptance criteria: want=[<%.0f] got=[%v]", prefix, wantLimit, value) + return CheckResultFail + } + return CheckResultPass + }, + }) + // TODO(network): podConnectivityChecks must not have outages + + // Create docs reference when ID is set + for c := range checkSum.Checks { + if checkSum.Checks[c].ID != "" { + checkSum.Checks[c].Reference = fmt.Sprintf("%s/#%s", checkSum.baseURL, checkSum.Checks[c].ID) + } + } + return checkSum +} + +func (csum *CheckSummary) GetBaseURL() string { + return csum.baseURL +} + +func (csum *CheckSummary) GetChecksFailed() []*Check { + failures := []*Check{} + for _, check := range csum.Checks { + if check.Result == CheckResultFail { + failures = append(failures, check) + } + } + return failures +} + +func (csum *CheckSummary) GetChecksPassed() []*Check { + failures := []*Check{} + for _, check := range csum.Checks { + if check.Result == CheckResultPass { + failures = append(failures, check) + } + } + return failures +} + +func (csum *CheckSummary) Run() error { + for _, check := range csum.Checks { + check.Result = check.Test() + } + return nil +} + +type CheckResult string + +const CheckResultPass = "pass" +const CheckResultFail = "fail" + +type Check struct { + // ID is the unique identifier for the check. It is used + // to mount the documentation for each check. + ID string `json:"id"` + + // Name is the unique name for the check to be reported. + // It must have short and descriptive name identifying the + // failure item. + Name string `json:"name"` + + // Description describes shortly the check. + Description string `json:"description"` + + // Reference must point to documentation URL to review the + // item. + Reference string `json:"reference"` + + // Accepted must report acceptance criteria, when true + // the Check is accepted by the tool, otherwise it is + // failed and must be reviewede. + Result CheckResult `json:"result"` + + ResultMessage string `json:"resultMessage"` + + Test func() CheckResult `json:"-"` +} + +/* Checks implementation */ + +func ExampleAcceptanceCheckPass() CheckResult { + return CheckResultPass +} + +func AcceptanceCheckFail() CheckResult { + return CheckResultFail +} + +func CheckRespCustomFail(custom string) CheckResult { + resp := CheckResult(fmt.Sprintf("%s [%s]", CheckResultFail, custom)) + return resp +} diff --git a/internal/opct/report/checks_test.go b/internal/opct/report/checks_test.go new file mode 100644 index 00000000..d61e8c4b --- /dev/null +++ b/internal/opct/report/checks_test.go @@ -0,0 +1,7 @@ +package report + +// TODO: create validation for +// - name should not have more than X size +// - ID must be in the format OPCT-NNN +// - DOC reference must exists in docs/review/rules.md +// - returns should be pass or fail diff --git a/internal/opct/report/report.go b/internal/opct/report/report.go new file mode 100644 index 00000000..a27da409 --- /dev/null +++ b/internal/opct/report/report.go @@ -0,0 +1,559 @@ +package report + +import ( + "bytes" + "encoding/json" + "fmt" + "html/template" + "os" + "sort" + "strings" + + vfs "github.com/redhat-openshift-ecosystem/provider-certification-tool/internal/assets" + "github.com/redhat-openshift-ecosystem/provider-certification-tool/internal/opct/archive" + "github.com/redhat-openshift-ecosystem/provider-certification-tool/internal/opct/metrics" + "github.com/redhat-openshift-ecosystem/provider-certification-tool/internal/opct/plugin" + "github.com/redhat-openshift-ecosystem/provider-certification-tool/internal/opct/summary" + "github.com/redhat-openshift-ecosystem/provider-certification-tool/internal/openshift/mustgather" + log "github.com/sirupsen/logrus" + "github.com/vmware-tanzu/sonobuoy/pkg/discovery" +) + +const ( + ReportFileNameIndexJSON = "/opct-report.json" + ReportTemplateBasePath = "data/templates/report" +) + +type Report struct { + Summary *ReportSummary `json:"summary"` + Raw string `json:"-"` + Provider *ReportResult `json:"provider"` + Baseline *ReportResult `json:"baseline,omitempty"` + Checks *ReportChecks `json:"checks,omitempty"` + Setup *ReportSetup +} + +type ReportChecks struct { + BaseURL string `json:"baseURL"` + Fail []*Check `json:"failures"` + Pass []*Check `json:"successes"` +} + +type ReportResult struct { + Version *ReportVersion `json:"version"` + Infra *ReportInfra `json:"infra"` + ClusterOperators *ReportClusterOperators `json:"clusterOperators"` + ClusterHealth *ReportClusterHealth `json:"clusterHealth"` + Plugins map[string]*ReportPlugin `json:"plugins"` + HasValidBaseline bool `json:"hasValidBaseline"` + MustGatherInfo *mustgather.MustGather `json:"mustGatherInfo,omitempty"` + ErrorCounters *archive.ErrorCounter `json:"errorCounters,omitempty"` + Runtime *ReportRuntime `json:"runtime,omitempty"` +} + +type ReportSummary struct { + Tests *ReportSummaryTests `json:"tests"` + Alerts *ReportSummaryAlerts `json:"alerts"` + Runtime *ReportSummaryRuntime `json:"runtime,omitempty"` + Headline string `json:"headline"` +} + +type ReportSummaryRuntime struct { + Timers metrics.Timers `json:"timers,omitempty"` + Plugins map[string]string `json:"plugins,omitempty"` + ExecutionTime string `json:"executionTime,omitempty"` +} + +type ReportSummaryTests struct { + Archive string `json:"archive"` + ArchiveDiff string `json:"archiveDiff,omitempty"` +} + +type ReportSummaryAlerts struct { + PluginK8S string `json:"pluginK8S,omitempty"` + PluginK8SMessage string `json:"pluginK8SMessage,omitempty"` + PluginOCP string `json:"pluginOCP,omitempty"` + PluginOCPMessage string `json:"pluginOCPMessage,omitempty"` + SuiteErrors string `json:"suiteErrors,omitempty"` + SuiteErrorsMessage string `json:"suiteErrorsMessage,omitempty"` + WorkloadErrors string `json:"workloadErrors,omitempty"` + WorkloadErrorsMessage string `json:"workloadErrorsMessage,omitempty"` + Checks string `json:"checks,omitempty"` + ChecksMessage string `json:"checksMessage,omitempty"` +} + +type ReportVersion struct { + // OpenShift versions + OpenShift *summary.SummaryClusterVersionOutput `json:"openshift"` + + // Kubernetes Version + Kubernetes string `json:"kubernetes"` + + // OPCT Version + OPCTServer string `json:"opctServer,omitempty"` + OPCTClient string `json:"opctClient,omitempty"` +} + +type ReportInfra struct { + Name string `json:"name"` + PlatformType string `json:"platformType"` + PlatformName string `json:"platformName"` + Topology string `json:"topology,omitempty"` + ControlPlaneTopology string `json:"controlPlaneTopology,omitempty"` + APIServerURL string `json:"apiServerURL,omitempty"` + APIServerInternalURL string `json:"apiServerInternalURL,omitempty"` + NetworkType string `json:"networkType,omitempty"` +} + +type ReportClusterOperators struct { + CountAvailable uint64 `json:"countAvailable,omitempty"` + CountProgressing uint64 `json:"countProgressing,omitempty"` + CountDegraded uint64 `json:"countDegraded,omitempty"` +} + +type ReportClusterHealth struct { + NodeHealthTotal int `json:"nodeHealthTotal,omitempty"` + NodeHealthy int `json:"nodeHealthy,omitempty"` + NodeHealthPerc float64 `json:"nodeHealthPerc,omitempty"` + PodHealthTotal int `json:"podHealthTotal,omitempty"` + PodHealthy int `json:"podHealthy,omitempty"` + PodHealthPerc float64 `json:"podHealthPerc,omitempty"` + PodHealthDetails []discovery.HealthInfoDetails `json:"podHealthDetails,omitempty"` +} + +type ReportPlugin struct { + ID string `json:"id"` + Title string `json:"title"` + Name string `json:"name"` + Definition *plugin.PluginDefinition `json:"definition,omitempty"` + Stat *ReportPluginStat `json:"stat"` + ErrorCounters *archive.ErrorCounter `json:"errorCounters,omitempty"` + Suite *summary.OpenshiftTestsSuite `json:"suite"` + TagsFailedPrio string `json:"tagsFailuresPriority"` + TestsFailedPrio []*ReportTestFailure `json:"testsFailuresPriority"` + TagsFlakeCI string `json:"tagsFlakeCI"` + TestsFlakeCI []*ReportTestFailure `json:"testsFlakeCI"` + Tests map[string]*plugin.TestItem `json:"tests,omitempty"` +} + +type ReportPluginStat struct { + Completed string `json:"execution"` + Result string `json:"result"` + Status string `json:"status"` + Total int64 `json:"total"` + Passed int64 `json:"passed"` + Failed int64 `json:"failed"` + Timeout int64 `json:"timeout"` + Skipped int64 `json:"skipped"` + FilterSuite int64 `json:"filterSuite"` + FilterBaseline int64 `json:"filterBaseline"` + FilterFailedPrio int64 `json:"filterFailedPriority"` +} + +type ReportTestFailure struct { + ID string `json:"id"` + Name string `json:"name"` + Documentation string `json:"documentation"` + FlakePerc float64 `json:"flakePerc"` + FlakeCount int64 `json:"flakeCount"` + ErrorsCount int64 `json:"errorsTotal"` +} + +type ReportSetup struct { + Frontend *ReportSetupFrontend +} +type ReportSetupFrontend struct { + EmbedData bool +} + +type ReportRuntime struct { + ServerLogs []*archive.RuntimeInfoItem `json:"serverLogs,omitempty"` + ServerConfig []*archive.RuntimeInfoItem `json:"serverConfig,omitempty"` + OpctConfig []*archive.RuntimeInfoItem `json:"opctConfig,omitempty"` +} + +// Populate is a entrypoint to initialize, trigger the data source processors, +// and finalize the report data structure used by frontend (HTML or CLI). +func (re *Report) Populate(cs *summary.ConsolidatedSummary) error { + cs.Timers.Add("report-populate") + re.Summary = &ReportSummary{ + Tests: &ReportSummaryTests{ + Archive: cs.GetProvider().Archive, + }, + Runtime: &ReportSummaryRuntime{ + Plugins: make(map[string]string, 4), + }, + Alerts: &ReportSummaryAlerts{}, + } + if err := re.populateSource(cs.GetProvider()); err != nil { + return err + } + re.Provider.HasValidBaseline = cs.GetBaseline().HasValidResults() + if re.Provider.HasValidBaseline { + if err := re.populateSource(cs.GetBaseline()); err != nil { + return err + } + re.Summary.Tests.ArchiveDiff = cs.GetBaseline().Archive + re.Summary.Headline = fmt.Sprintf("%s (diff %s) | OCP %s | K8S %s", + re.Summary.Tests.Archive, + re.Summary.Tests.ArchiveDiff, + re.Provider.Version.OpenShift.Desired, + re.Provider.Version.Kubernetes, + ) + } + checks := NewCheckSummary(re) + err := checks.Run() + if err != nil { + log.Debugf("one or more errors found when running checks: %v", err) + } + re.Checks = &ReportChecks{ + BaseURL: checks.GetBaseURL(), + Pass: checks.GetChecksPassed(), + Fail: checks.GetChecksFailed(), + } + if len(re.Checks.Fail) > 0 { + re.Summary.Alerts.Checks = "danger" + re.Summary.Alerts.ChecksMessage = fmt.Sprintf("%d", len(re.Checks.Fail)) + } + + cs.Timers.Add("report-populate") + re.Summary.Runtime.Timers = cs.Timers + return nil +} + +// populateSource reads the loaded data, creating a report data for each result +// data source (provider and/or baseline). +func (re *Report) populateSource(rs *summary.ResultSummary) error { + + var reResult *ReportResult + if rs.Name == summary.ResultSourceNameBaseline { + re.Baseline = &ReportResult{} + reResult = re.Baseline + } else { + re.Provider = &ReportResult{} + reResult = re.Provider + reResult.MustGatherInfo = rs.MustGather + } + // Version + v, err := rs.GetOpenShift().GetClusterVersion() + if err != nil { + return err + } + reResult.Version = &ReportVersion{ + OpenShift: v, + Kubernetes: rs.GetSonobuoyCluster().APIVersion, + } + + // Infrastructure + infra, err := rs.GetOpenShift().GetInfrastructure() + if err != nil { + return err + } + platformName := "" + if string(infra.Status.PlatformStatus.Type) == "External" { + platformName = infra.Spec.PlatformSpec.External.PlatformName + } + sdn, err := rs.GetOpenShift().GetClusterNetwork() + if err != nil { + log.Errorf("unable to get clusterNetwork object: %v", err) + return err + } + reResult.Infra = &ReportInfra{ + PlatformType: string(infra.Status.PlatformStatus.Type), + PlatformName: platformName, + Name: string(infra.Status.InfrastructureName), + Topology: string(infra.Status.InfrastructureTopology), + ControlPlaneTopology: string(infra.Status.ControlPlaneTopology), + APIServerURL: string(infra.Status.APIServerURL), + APIServerInternalURL: string(infra.Status.APIServerInternalURL), + NetworkType: string(sdn.Spec.NetworkType), + } + + // Cluster Operators + co, err := rs.GetOpenShift().GetClusterOperator() + if err != nil { + return err + } + reResult.ClusterOperators = &ReportClusterOperators{ + CountAvailable: co.CountAvailable, + CountProgressing: co.CountProgressing, + CountDegraded: co.CountDegraded, + } + + // Node and Pod Status + sbCluster := rs.GetSonobuoyCluster() + reResult.ClusterHealth = &ReportClusterHealth{ + NodeHealthTotal: sbCluster.NodeHealth.Total, + NodeHealthy: sbCluster.NodeHealth.Healthy, + NodeHealthPerc: float64(100 * sbCluster.NodeHealth.Healthy / sbCluster.NodeHealth.Total), + PodHealthTotal: sbCluster.PodHealth.Total, + PodHealthy: sbCluster.PodHealth.Healthy, + PodHealthPerc: float64(100 * sbCluster.PodHealth.Healthy / sbCluster.PodHealth.Total), + } + for _, dt := range sbCluster.PodHealth.Details { + if !dt.Healthy { + reResult.ClusterHealth.PodHealthDetails = append(reResult.ClusterHealth.PodHealthDetails, dt) + } + } + + // Plugins + reResult.Plugins = make(map[string]*ReportPlugin, 4) + if err := re.populatePluginConformance(rs, reResult, plugin.PluginNameKubernetesConformance); err != nil { + return err + } + if err := re.populatePluginConformance(rs, reResult, plugin.PluginNameOpenShiftConformance); err != nil { + return err + } + if err := re.populatePluginConformance(rs, reResult, plugin.PluginNameOpenShiftUpgrade); err != nil { + return err + } + if err := re.populatePluginConformance(rs, reResult, plugin.PluginNameArtifactsCollector); err != nil { + return err + } + + // Aggregate Plugin errors + reResult.ErrorCounters = archive.MergeErrorCounters( + reResult.Plugins[plugin.PluginNameKubernetesConformance].ErrorCounters, + reResult.Plugins[plugin.PluginNameOpenShiftConformance].ErrorCounters, + ) + + // Runtime + if reResult.Runtime == nil { + reResult.Runtime = &ReportRuntime{} + } + if rs.Sonobuoy != nil && rs.Sonobuoy.MetaRuntime != nil { + reResult.Runtime.ServerLogs = rs.Sonobuoy.MetaRuntime + for _, e := range rs.Sonobuoy.MetaRuntime { + if strings.HasPrefix(e.Name, "plugin finished") { + arr := strings.Split(e.Name, "plugin finished ") + re.Summary.Runtime.Plugins[arr[len(arr)-1]] = e.Delta + } + if strings.HasPrefix(e.Name, "server finished") { + re.Summary.Runtime.ExecutionTime = e.Total + } + } + } + if rs.Sonobuoy != nil && rs.Sonobuoy.MetaConfig != nil { + reResult.Runtime.ServerConfig = rs.Sonobuoy.MetaConfig + } + if rs.Sonobuoy != nil && rs.Sonobuoy.MetaConfig != nil { + reResult.Runtime.OpctConfig = rs.Sonobuoy.OpctConfig + } + return nil +} + +// populatePluginConformance reads the plugin data, processing and creating the report data. +func (re *Report) populatePluginConformance(rs *summary.ResultSummary, reResult *ReportResult, pluginID string) error { + + var pluginSum *plugin.OPCTPluginSummary + var suite *summary.OpenshiftTestsSuite + var pluginTitle string + var pluginAlert string + var pluginAlertMessage string + + switch pluginID { + case plugin.PluginNameKubernetesConformance: + pluginSum = rs.GetOpenShift().GetResultK8SValidated() + pluginTitle = "Results for Kubernetes Conformance Suite" + suite = rs.GetSuites().KubernetesConformance + case plugin.PluginNameOpenShiftConformance: + pluginSum = rs.GetOpenShift().GetResultOCPValidated() + pluginTitle = "Results for OpenShift Conformance Suite" + suite = rs.GetSuites().OpenshiftConformance + case plugin.PluginNameOpenShiftUpgrade: + pluginSum = rs.GetOpenShift().GetResultConformanceUpgrade() + pluginTitle = "Results for OpenShift Conformance Upgrade Suite" + case plugin.PluginNameArtifactsCollector: + pluginSum = rs.GetOpenShift().GetResultArtifactsCollector() + pluginTitle = "Results for Plugin Collector" + } + + pluginRes := pluginSum.Status + reResult.Plugins[pluginID] = &ReportPlugin{ + ID: pluginID, + Title: pluginTitle, + Name: pluginSum.Name, + Stat: &ReportPluginStat{ + Completed: "TODO", + Status: pluginSum.Status, + Result: pluginRes, + Total: pluginSum.Total, + Passed: pluginSum.Passed, + Failed: pluginSum.Failed, + Timeout: pluginSum.Timeout, + Skipped: pluginSum.Skipped, + }, + Suite: suite, + Tests: pluginSum.Tests, + } + + // No more advanced fields to create for non-Conformance + switch pluginID { + case plugin.PluginNameOpenShiftUpgrade, plugin.PluginNameArtifactsCollector: + return nil + } + + // Fill stat for filters (non-standard in Sonobuoy) + reResult.Plugins[pluginID].Stat.FilterSuite = int64(len(pluginSum.FailedFilterSuite)) + reResult.Plugins[pluginID].Stat.FilterBaseline = int64(len(pluginSum.FailedFilterBaseline)) + reResult.Plugins[pluginID].Stat.FilterFailedPrio = int64(len(pluginSum.FailedFilterPrio)) + reResult.Plugins[pluginID].ErrorCounters = pluginSum.GetErrorCounters() + // Will consider passed when all conformance tests have passed (removing monitor) + if reResult.Plugins[pluginID].Stat.FilterSuite == 0 { + reResult.Plugins[pluginID].Stat.Result = "passed" + } + + if reResult.Plugins[pluginID].Stat.FilterFailedPrio != 0 { + pluginAlert = "danger" + pluginAlertMessage = fmt.Sprintf("%d", int64(len(pluginSum.FailedFilterPrio))) + } else if reResult.Plugins[pluginID].Stat.FilterSuite != 0 { + pluginAlert = "warning" + pluginAlertMessage = fmt.Sprintf("%d", int64(len(pluginSum.FailedFilterSuite))) + } + + if _, ok := rs.GetSonobuoy().PluginsDefinition[pluginID]; ok { + def := rs.GetSonobuoy().PluginsDefinition[pluginID] + reResult.Plugins[pluginID].Definition = &plugin.PluginDefinition{ + PluginImage: def.Definition.Spec.Image, + SonobuoyImage: def.SonobuoyImage, + Name: def.Definition.SonobuoyConfig.PluginName, + } + } + + // TODO move this filter to a dedicated function + noFlakes := make(map[string]struct{}) + testTagsFailedPrio := plugin.NewTestTagsEmpty(len(pluginSum.FailedFilterPrio)) + for _, test := range pluginSum.FailedFilterPrio { + noFlakes[test] = struct{}{} + testTagsFailedPrio.Add(&test) + testData := &ReportTestFailure{ + Name: test, + ID: pluginSum.Tests[test].ID, + Documentation: pluginSum.Tests[test].Documentation, + } + if _, ok := pluginSum.Tests[test].ErrorCounters["total"]; ok { + testData.ErrorsCount = int64(pluginSum.Tests[test].ErrorCounters["total"]) + } + reResult.Plugins[pluginID].TestsFailedPrio = append(reResult.Plugins[pluginID].TestsFailedPrio, testData) + } + reResult.Plugins[pluginID].TagsFailedPrio = testTagsFailedPrio.ShowSorted() + reResult.Plugins[pluginID].TestsFailedPrio = sortReportTestFailure(reResult.Plugins[pluginID].TestsFailedPrio) + + flakes := reResult.Plugins[pluginID].TestsFlakeCI + testTagsFlakeCI := plugin.NewTestTagsEmpty(len(pluginSum.FailedFilterBaseline)) + for _, test := range pluginSum.FailedFilterBaseline { + if _, ok := noFlakes[test]; ok { + continue + } + testData := &ReportTestFailure{Name: test, ID: pluginSum.Tests[test].ID} + if pluginSum.Tests[test].Flake != nil { + testData.FlakeCount = pluginSum.Tests[test].Flake.CurrentFlakes + testData.FlakePerc = pluginSum.Tests[test].Flake.CurrentFlakePerc + } + testTagsFlakeCI.Add(&test) + if _, ok := pluginSum.Tests[test].ErrorCounters["total"]; ok { + testData.ErrorsCount = int64(pluginSum.Tests[test].ErrorCounters["total"]) + } + flakes = append(flakes, testData) + } + reResult.Plugins[pluginID].TestsFlakeCI = sortReportTestFailure(flakes) + reResult.Plugins[pluginID].TagsFlakeCI = testTagsFlakeCI.ShowSorted() + + // update alerts + if rs.Name == summary.ResultSourceNameProvider && pluginAlert != "" { + switch pluginID { + case plugin.PluginNameKubernetesConformance: + re.Summary.Alerts.PluginK8S = pluginAlert + re.Summary.Alerts.PluginK8SMessage = pluginAlertMessage + case plugin.PluginNameOpenShiftConformance: + re.Summary.Alerts.PluginOCP = pluginAlert + re.Summary.Alerts.PluginOCPMessage = pluginAlertMessage + } + } + + return nil +} + +// SaveResults persist the processed data to the result directory. +func (re *Report) SaveResults(path string) error { + re.Summary.Runtime.Timers.Add("report-save/results") + + // opct-report.json (data source) + reportData, err := json.MarshalIndent(re, "", " ") + checkOrPanic(err) + // used when not using http file server + if re.Setup.Frontend.EmbedData { + re.Raw = string(reportData) + } + + err = os.WriteFile(fmt.Sprintf("%s/%s", path, ReportFileNameIndexJSON), reportData, 0644) + checkOrPanic(err) + + // render the template files from frontend report pages. + for _, file := range []string{"report.html", "filter.html"} { + srcTemplate := fmt.Sprintf("%s/%s", ReportTemplateBasePath, file) + destFile := fmt.Sprintf("%s/opct-%s", path, file) + + datS, err := vfs.GetData().ReadFile(srcTemplate) + checkOrPanic(err) + + // change go template delimiter to '[[]]' preventing conflict with + // javascript delimiter '{{}}'+ + tmplS, err := template.New("report").Delims("[[", "]]").Parse(string(datS)) + checkOrPanic(err) + + var fileBufferS bytes.Buffer + err = tmplS.Execute(&fileBufferS, re) + checkOrPanic(err) + + err = os.WriteFile(destFile, fileBufferS.Bytes(), 0644) + checkOrPanic(err) + } + + re.Summary.Runtime.Timers.Add("report-save/results") + return nil +} + +func checkOrPanic(e error) { + if e != nil { + panic(e) + } +} + +// ShowJSON print the raw json in stdout. +func (re *Report) ShowJSON() (string, error) { + val, err := json.MarshalIndent(re, "", " ") + if err != nil { + return "", err + } + return string(val), nil +} + +// SortedTestFailure stores the key/value to rank by Key. +type SortedTestFailure struct { + Key *ReportTestFailure + Value int +} + +func sortReportTestFailure(items []*ReportTestFailure) []*ReportTestFailure { + rank := make(SortedListTestFailure, len(items)) + i := 0 + for _, v := range items { + rank[i] = SortedTestFailure{v, int(v.ErrorsCount)} + i++ + } + sort.Sort(sort.Reverse(rank)) + newItems := make([]*ReportTestFailure, len(items)) + for i, data := range rank { + newItems[i] = data.Key + } + return newItems +} + +// SortedList stores the list of key/value map, implementing interfaces +// to sort/rank a map strings with integers as values. +type SortedListTestFailure []SortedTestFailure + +func (p SortedListTestFailure) Len() int { return len(p) } +func (p SortedListTestFailure) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p SortedListTestFailure) Less(i, j int) bool { return p[i].Value < p[j].Value } diff --git a/internal/opct/summary/consolidated.go b/internal/opct/summary/consolidated.go new file mode 100644 index 00000000..75dac730 --- /dev/null +++ b/internal/opct/summary/consolidated.go @@ -0,0 +1,604 @@ +package summary + +import ( + "bufio" + "fmt" + "os" + "regexp" + "sort" + + log "github.com/sirupsen/logrus" + + "github.com/pkg/errors" + + "github.com/redhat-openshift-ecosystem/provider-certification-tool/internal/opct/metrics" + "github.com/redhat-openshift-ecosystem/provider-certification-tool/internal/opct/plugin" + "github.com/redhat-openshift-ecosystem/provider-certification-tool/internal/openshift/ci/sippy" +) + +// ConsolidatedSummary Aggregate the results of provider and baseline +type ConsolidatedSummary struct { + Verbose bool + Timers metrics.Timers + Provider *ResultSummary + Baseline *ResultSummary +} + +// Process entrypoint to read and fill all summaries for each archive, plugin and suites +// applying any transformation it needs through filters. +func (cs *ConsolidatedSummary) Process() error { + cs.Timers.Add("cs-process") + + // Load Result Summary from Archives + log.Debug("Processing results/Populating Provider") + cs.Timers.Set("cs-process/populate-provider") + if err := cs.Provider.Populate(); err != nil { + fmt.Println("ERROR processing provider results...") + return err + } + + log.Debug("Processing results/Populating Baseline") + cs.Timers.Set("cs-process/populate-baseline") + if err := cs.Baseline.Populate(); err != nil { + fmt.Println("ERROR processing baseline results...") + return err + } + + // Filters + log.Debug("Processing results/Applying filters/Suite") + cs.Timers.Set("cs-process/filter-suite") + if err := cs.applyFilterSuite(); err != nil { + return err + } + + log.Debug("Processing results/Applying filters/Baseline") + cs.Timers.Set("cs-process/filter-baseline") + if err := cs.applyFilterBaseline(); err != nil { + return err + } + + log.Debug("Processing results/Applying filters/Flake") + cs.Timers.Set("cs-process/filter-flake") + if err := cs.applyFilterFlaky(); err != nil { + return err + } + + log.Debug("Processing results/Building tests documentation") + cs.Timers.Set("cs-process/build-docs") + if err := cs.buildDocumentation(); err != nil { + return err + } + + cs.Timers.Add("cs-process") + return nil +} + +// GetProvider get the provider results. +func (cs *ConsolidatedSummary) GetProvider() *ResultSummary { + return cs.Provider +} + +// GetBaseline get the baseline results. +func (cs *ConsolidatedSummary) GetBaseline() *ResultSummary { + return cs.Baseline +} + +// HasBaselineResults checks if the baseline results was set (--dif), +// and has valid data. +func (cs *ConsolidatedSummary) HasBaselineResults() bool { + if cs.Baseline == nil { + return false + } + return cs.Baseline.HasValidResults() +} + +// applyFilterSuite process the FailedList for each plugin, getting **intersection** tests +// for respective suite. +func (cs *ConsolidatedSummary) applyFilterSuite() error { + err := cs.applyFilterSuiteForPlugin(plugin.PluginNameKubernetesConformance) + if err != nil { + return err + } + + err = cs.applyFilterSuiteForPlugin(plugin.PluginNameOpenShiftConformance) + if err != nil { + return err + } + + return nil +} + +// applyFilterSuiteForPlugin calculates the intersection of Provider Failed AND suite +func (cs *ConsolidatedSummary) applyFilterSuiteForPlugin(pluginName string) error { + + var resultsProvider *plugin.OPCTPluginSummary + var pluginSuite *OpenshiftTestsSuite + + switch pluginName { + case plugin.PluginNameKubernetesConformance: + resultsProvider = cs.GetProvider().GetOpenShift().GetResultK8SValidated() + pluginSuite = cs.GetProvider().GetSuites().KubernetesConformance + case plugin.PluginNameOpenShiftConformance: + resultsProvider = cs.GetProvider().GetOpenShift().GetResultOCPValidated() + pluginSuite = cs.GetProvider().GetSuites().OpenshiftConformance + } + + e2eFailures := resultsProvider.FailedList + e2eSuite := pluginSuite.Tests + emptySuite := len(pluginSuite.Tests) == 0 + hashSuite := make(map[string]struct{}, len(e2eSuite)) + + for _, v := range e2eSuite { + hashSuite[v] = struct{}{} + } + + for _, v := range e2eFailures { + // move on the pipeline when the suite is empty. + resultsProvider.Tests[v].State = "filterSuiteOnly" + if emptySuite { + resultsProvider.FailedFilterSuite = append(resultsProvider.FailedFilterSuite, v) + } else { + if _, ok := hashSuite[v]; ok { + resultsProvider.FailedFilterSuite = append(resultsProvider.FailedFilterSuite, v) + } + } + } + sort.Strings(resultsProvider.FailedFilterSuite) + return nil +} + +// applyFilterBaseline process the FailedFilterSuite for each plugin, **excluding** failures from +// baseline test. +func (cs *ConsolidatedSummary) applyFilterBaseline() error { + err := cs.applyFilterBaselineForPlugin(plugin.PluginNameKubernetesConformance) + if err != nil { + return err + } + + err = cs.applyFilterBaselineForPlugin(plugin.PluginNameOpenShiftConformance) + if err != nil { + return err + } + + return nil +} + +// applyFilterBaselineForPlugin calculates the **exclusion** tests of +// Provider Failed included on suite and Baseline failed tests. +func (cs *ConsolidatedSummary) applyFilterBaselineForPlugin(pluginName string) error { + + var providerSummary *plugin.OPCTPluginSummary + var e2eFailuresBaseline []string + + switch pluginName { + case plugin.PluginNameKubernetesConformance: + providerSummary = cs.GetProvider().GetOpenShift().GetResultK8SValidated() + if cs.GetBaseline().HasValidResults() { + e2eFailuresBaseline = cs.GetBaseline().GetOpenShift().GetResultK8SValidated().FailedList + } + case plugin.PluginNameOpenShiftConformance: + providerSummary = cs.GetProvider().GetOpenShift().GetResultOCPValidated() + if cs.GetBaseline().HasValidResults() { + e2eFailuresBaseline = cs.GetBaseline().GetOpenShift().GetResultOCPValidated().FailedList + } + default: + return errors.New("Suite not found to apply filter: Flaky") + } + + e2eFailuresProvider := providerSummary.FailedFilterSuite + hashBaseline := make(map[string]struct{}, len(e2eFailuresBaseline)) + + for _, v := range e2eFailuresBaseline { + hashBaseline[v] = struct{}{} + } + + for _, v := range e2eFailuresProvider { + providerSummary.Tests[v].State = "filterBaseline" + if _, ok := hashBaseline[v]; !ok { + providerSummary.FailedFilterBaseline = append(providerSummary.FailedFilterBaseline, v) + } + } + sort.Strings(providerSummary.FailedFilterBaseline) + return nil +} + +// applyFilterFlaky process the FailedFilterSuite for each plugin, **excluding** failures from +// baseline test. +func (cs *ConsolidatedSummary) applyFilterFlaky() error { + err := cs.applyFilterFlakeForPlugin(plugin.PluginNameKubernetesConformance) + if err != nil { + return err + } + + err = cs.applyFilterFlakeForPlugin(plugin.PluginNameOpenShiftConformance) + if err != nil { + return err + } + + return nil +} + +// applyFilterFlakeForPlugin query the Sippy API looking for each failed test +// on each plugin/suite, saving the list on the ResultSummary. +func (cs *ConsolidatedSummary) applyFilterFlakeForPlugin(pluginName string) error { + + var ps *plugin.OPCTPluginSummary + + switch pluginName { + case plugin.PluginNameKubernetesConformance: + ps = cs.GetProvider().GetOpenShift().GetResultK8SValidated() + case plugin.PluginNameOpenShiftConformance: + ps = cs.GetProvider().GetOpenShift().GetResultOCPValidated() + default: + return errors.New("Suite not found to apply filter: Flaky") + } + + // TODO: define if we will check for flakes for all failures or only filtered + // Query Flaky only the FilteredBaseline to avoid many external queries. + ver, err := cs.GetProvider().GetOpenShift().GetClusterVersionXY() + if err != nil { + return errors.Errorf("Error getting cluster version: %v", err) + } + + api := sippy.NewSippyAPI(ver) + for _, name := range ps.FailedFilterBaseline { + ps.Tests[name].State = "filterFlake" + resp, err := api.QueryTests(&sippy.SippyTestsRequestInput{TestName: name}) + if err != nil { + log.Errorf("#> Error querying to Sippy API: %v", err) + continue + } + for _, r := range *resp { + if _, ok := ps.Tests[name]; ok { + ps.Tests[name].Flake = &r + } else { + ps.Tests[name] = &plugin.TestItem{ + Name: name, + Flake: &r, + } + } + + // Applying flake filter by moving only non-flakes to the pipeline. + // The tests reporing lower than 5% of CurrentFlakePerc by Sippy are selected as non-flake. + // TODO: Review flake severity + if ps.Tests[name].Flake.CurrentFlakePerc <= 5 { + ps.Tests[name].State = "filterPriority" + ps.FailedFilterPrio = append(ps.FailedFilterPrio, name) + } + } + } + + sort.Strings(ps.FailedFilterPrio) + return nil +} + +func (cs *ConsolidatedSummary) saveResultsPlugin(path, pluginName string) error { + + var resultsProvider *plugin.OPCTPluginSummary + var resultsBaseline *plugin.OPCTPluginSummary + var suite *OpenshiftTestsSuite + var prefix = "tests" + bProcessed := cs.GetBaseline().HasValidResults() + + switch pluginName { + case plugin.PluginNameKubernetesConformance: + resultsProvider = cs.GetProvider().GetOpenShift().GetResultK8SValidated() + if bProcessed { + resultsBaseline = cs.GetBaseline().GetOpenShift().GetResultK8SValidated() + } + suite = cs.GetProvider().GetSuites().KubernetesConformance + case plugin.PluginNameOpenShiftConformance: + resultsProvider = cs.GetProvider().GetOpenShift().GetResultOCPValidated() + if bProcessed { + resultsBaseline = cs.GetBaseline().GetOpenShift().GetResultOCPValidated() + } + suite = cs.GetProvider().GetSuites().OpenshiftConformance + } + + if cs.Verbose { + // Save Provider failures + filename := fmt.Sprintf("%s/%s_%s_provider_failures-1-ini.txt", path, prefix, pluginName) + if err := writeFileTestList(filename, resultsProvider.FailedList); err != nil { + return err + } + + // Save Provider failures with filter: Suite (only) + filename = fmt.Sprintf("%s/%s_%s_provider_failures-2-filter1_suite.txt", path, prefix, pluginName) + if err := writeFileTestList(filename, resultsProvider.FailedFilterSuite); err != nil { + return err + } + + // Save Provider failures with filter: Baseline exclusion + filename = fmt.Sprintf("%s/%s_%s_provider_failures-3-filter2_baseline.txt", path, prefix, pluginName) + if err := writeFileTestList(filename, resultsProvider.FailedFilterBaseline); err != nil { + return err + } + + // Save Provider failures with filter: Flaky + filename = fmt.Sprintf("%s/%s_%s_provider_failures-4-filter3_without_flakes.txt", path, prefix, pluginName) + if err := writeFileTestList(filename, resultsProvider.FailedFilterPrio); err != nil { + return err + } + + // Save the Providers failures for the latest filter to review (focus on this) + filename = fmt.Sprintf("%s/%s_%s_provider_failures.txt", path, prefix, pluginName) + if err := writeFileTestList(filename, resultsProvider.FailedFilterBaseline); err != nil { + return err + } + + // Save baseline failures + if bProcessed { + filename = fmt.Sprintf("%s/%s_%s_baseline_failures.txt", path, prefix, pluginName) + if err := writeFileTestList(filename, resultsBaseline.FailedList); err != nil { + return err + } + } + + // Save the openshift-tests suite use by this plugin: + filename = fmt.Sprintf("%s/%s_%s_suite_full.txt", path, prefix, pluginName) + if err := writeFileTestList(filename, suite.Tests); err != nil { + return err + } + } + + return nil +} + +func (cs *ConsolidatedSummary) extractFailuresDetailsByPlugin(path, pluginName string) error { + + var resultsProvider *plugin.OPCTPluginSummary + // var resultsBaseline *OPCTPluginSummary + // bProcessed := cs.GetBaseline().HasValidResults() + ignoreExistingDir := true + + switch pluginName { + case plugin.PluginNameKubernetesConformance: + resultsProvider = cs.GetProvider().GetOpenShift().GetResultK8SValidated() + // if bProcessed { + // resultsBaseline = cs.GetBaseline().GetOpenShift().GetResultK8SValidated() + // } + case plugin.PluginNameOpenShiftConformance: + resultsProvider = cs.GetProvider().GetOpenShift().GetResultOCPValidated() + // if bProcessed { + // resultsBaseline = cs.GetBaseline().GetOpenShift().GetResultOCPValidated() + // } + } + + // currentDirectory := "failures-provider-filtered" + // subdir := fmt.Sprintf("%s/%s", path, currentDirectory) + // if err := createDir(subdir, ignoreExistingDir); err != nil { + // return err + // } + + // subPrefix := fmt.Sprintf("%s/%s", subdir, plugin) + // errItems := resultsProvider.FailedItems + // errList := resultsProvider.FailedFilterBaseline + // if err := extractSaveTestErrors(subPrefix, errItems, errList); err != nil { + // return err + // } + + // currentDirectory = "failures-provider" + // subdir = fmt.Sprintf("%s/%s", path, currentDirectory) + // if err := createDir(subdir, ignoreExistingDir); err != nil { + // return err + // } + + // subPrefix = fmt.Sprintf("%s/%s", subdir, plugin) + // errItems = resultsProvider.FailedItems + // errList = resultsProvider.FailedList + // if err := extractSaveTestErrors(subPrefix, errItems, errList); err != nil { + // return err + // } + + // currentDirectory = "failures-baseline" + // subdir = fmt.Sprintf("%s/%s", path, currentDirectory) + // if err := createDir(subdir, ignoreExistingDir); err != nil { + // return err + // } + + // if bProcessed { + // subPrefix = fmt.Sprintf("%s/%s", subdir, plugin) + // errItems = resultsBaseline.FailedItems + // errList = resultsBaseline.FailedList + // if err := extractSaveTestErrors(subPrefix, errItems, errList); err != nil { + // return err + // } + // } + + // extract all failed by plugins + currentDirectory := fmt.Sprintf("failures-%s", pluginName) + subdir := fmt.Sprintf("%s/%s/", path, currentDirectory) + if err := createDir(subdir, ignoreExistingDir); err != nil { + return err + } + errFailures := make([]string, len(resultsProvider.Tests)) + for k := range resultsProvider.Tests { + errFailures = append(errFailures, k) + } + if err := extractSaveTestErrors(subdir, resultsProvider.Tests, errFailures); err != nil { + return err + } + + return nil +} + +// SaveResults dump all the results and processed to the disk to be used +// on the review process. +func (cs *ConsolidatedSummary) SaveResults(path string) error { + + cs.Timers.Add("cs-save/results") + if err := createDir(path, true); err != nil { + return err + } + + // Save the list of failures into individual files by Plugin + if err := cs.saveResultsPlugin(path, plugin.PluginNameKubernetesConformance); err != nil { + return err + } + if err := cs.saveResultsPlugin(path, plugin.PluginNameOpenShiftConformance); err != nil { + return err + } + + // Extract errors details to sub directories + if err := cs.extractFailuresDetailsByPlugin(path, plugin.PluginNameKubernetesConformance); err != nil { + return err + } + if err := cs.extractFailuresDetailsByPlugin(path, plugin.PluginNameOpenShiftConformance); err != nil { + return err + } + + fmt.Printf("\n Data Saved to directory '%s'\n", path) + cs.Timers.Add("cs-save/results") + return nil +} + +// writeFileTestList saves the list of test names to a new text file +func writeFileTestList(filename string, data []string) error { + fd, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + log.Fatalf("failed creating file: %s", err) + } + defer fd.Close() + + writer := bufio.NewWriter(fd) + defer writer.Flush() + + for _, line := range data { + _, err = writer.WriteString(line + "\n") + if err != nil { + return err + } + } + + return nil +} + +// extractTestErrors dumps the test error, summary and stdout, then saved +// to individual files. +func extractSaveTestErrors(prefix string, items plugin.Tests, failures []string) error { + + for _, line := range failures { + if _, ok := items[line]; ok { + file := fmt.Sprintf("%s%s-failure.txt", prefix, items[line].ID) + err := writeErrorToFile(file, items[line].Failure) + if err != nil { + log.Errorf("Error writing Failure for test: %s\n", line) + } + + file = fmt.Sprintf("%s%s-systemOut.txt", prefix, items[line].ID) + err = writeErrorToFile(file, items[line].SystemOut) + if err != nil { + log.Errorf("Error writing SystemOut for test: %s\n", line) + } + } + } + return nil +} + +// writeErrorToFile save the entire buffer to individual file. +func writeErrorToFile(file, data string) error { + fd, err := os.OpenFile(file, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + log.Fatalf("failed creating file: %s", err) + } + defer fd.Close() + + writer := bufio.NewWriter(fd) + defer writer.Flush() + + _, err = writer.WriteString(data) + if err != nil { + return err + } + + return nil +} + +// createDir checks if the directory exists, if not creates it, otherwise log and return error +func createDir(path string, ignoreexisting bool) error { + // Saved directory must be created by must-gather extractor. + // TODO check cases not covered by that flow. + if _, err := os.Stat(path); !os.IsNotExist(err) { + if ignoreexisting { + return nil + } + return errors.New(fmt.Sprintf("directory already exists: %s", path)) + } + + if err := os.Mkdir(path, os.ModePerm); err != nil { + log.Errorf("ERROR: Unable to create directory [%s]: %v", path, err) + return err + } + return nil +} + +// applyFilterFlaky process the FailedFilterSuite for each plugin, **excluding** failures from +// baseline test. +func (cs *ConsolidatedSummary) buildDocumentation() error { + err := cs.buildDocumentationForPlugin(plugin.PluginNameKubernetesConformance) + if err != nil { + return err + } + + err = cs.buildDocumentationForPlugin(plugin.PluginNameOpenShiftConformance) + if err != nil { + return err + } + + return nil +} + +// applyFilterFlakeForPlugin query the Sippy API looking for each failed test +// on each plugin/suite, saving the list on the ResultSummary. +func (cs *ConsolidatedSummary) buildDocumentationForPlugin(pluginName string) error { + + var ( + ps *plugin.OPCTPluginSummary + version string + docUserBaseURL string + docSourceBaseURL string + ) + + switch pluginName { + case plugin.PluginNameKubernetesConformance: + ps = cs.GetProvider().GetOpenShift().GetResultK8SValidated() + versionFull := cs.GetProvider().GetSonobuoyCluster().APIVersion + reVersion := regexp.MustCompile(`^v(\d+\.\d+)`) + matches := reVersion.FindStringSubmatch(versionFull) + if len(matches) != 2 { + log.Warnf("Unable to extract kubernetes version to build documentation: %v [%v]", versionFull, matches) + return nil + } + version = matches[1] + // fmt.Println(matches) + docUserBaseURL = fmt.Sprintf("https://github.com/cncf/k8s-conformance/blob/master/docs/KubeConformance-%s.md", version) + docSourceBaseURL = fmt.Sprintf("https://raw.githubusercontent.com/cncf/k8s-conformance/master/docs/KubeConformance-%s.md", version) + case plugin.PluginNameOpenShiftConformance: + ps = cs.GetProvider().GetOpenShift().GetResultOCPValidated() + // OCP tests does not have documentation (TODO: check what can be used) + // https://docs.openshift.com/container-platform/4.13/welcome/index.html + // https://access.redhat.com/search/ + docUserBaseURL = "https://github.com/openshift/origin/blob/master/test/extended/README.md" + docSourceBaseURL = docUserBaseURL + default: + return errors.New("Plugin not found to apply filter: Flaky") + } + + if ps.Documentation == nil { + ps.Documentation = plugin.NewTestDocumentation(docUserBaseURL, docSourceBaseURL) + err := ps.Documentation.Load() + if err != nil { + return err + } + err = ps.Documentation.BuildIndex() + if err != nil { + return err + } + } + + for _, test := range ps.Tests { + test.LookupDocumentation(ps.Documentation) + } + + return nil +} diff --git a/internal/opct/summary/openshift.go b/internal/opct/summary/openshift.go new file mode 100644 index 00000000..de92da4c --- /dev/null +++ b/internal/opct/summary/openshift.go @@ -0,0 +1,236 @@ +package summary + +import ( + "fmt" + "regexp" + + configv1 "github.com/openshift/api/config/v1" + "github.com/pkg/errors" + "github.com/redhat-openshift-ecosystem/provider-certification-tool/internal/opct/plugin" +) + +// OpenShiftSummary holds the data collected from artifacts related to OpenShift objects. +type OpenShiftSummary struct { + Infrastructure *configv1.Infrastructure + ClusterVersion *configv1.ClusterVersion + ClusterOperators *configv1.ClusterOperatorList + ClusterNetwork *configv1.Network + + // Plugin Results + PluginResultK8sConformance *plugin.OPCTPluginSummary + PluginResultOCPValidated *plugin.OPCTPluginSummary + PluginResultConformanceUpgrade *plugin.OPCTPluginSummary + PluginResultArtifactsCollector *plugin.OPCTPluginSummary + + // get from Sonobuoy metadata + VersionK8S string +} + +type SummaryClusterVersionOutput struct { + Desired string `json:"desired"` + Previous string `json:"previous"` + Channel string `json:"channel"` + ClusterID string `json:"clusterID"` + OverallStatus string `json:"overallStatus"` + OverallStatusReason string `json:"overallStatusReason,omitempty"` + OverallStatusMessage string `json:"overallStatusMessage,omitempty"` + CondAvailable string `json:"conditionAvailable,omitempty"` + CondFailing string `json:"conditionFailing,omitempty"` + CondProgressing string `json:"conditionProgressing,omitempty"` + CondProgressingMessage string `json:"conditionProgressingMessage,omitempty"` + CondRetrievedUpdates string `json:"conditionUpdates,omitempty"` + CondImplicitlyEnabledCapabilities string `json:"conditionImplicitlyEnabledCapabilities,omitempty"` + CondReleaseAccepted string `json:"conditionReleaseAccepted,omitempty"` +} + +type SummaryClusterOperatorOutput struct { + CountAvailable uint64 + CountProgressing uint64 + CountDegraded uint64 +} + +type SummaryOpenShiftInfrastructureV1 = configv1.Infrastructure +type SummaryOpenShiftClusterNetworkV1 = configv1.Network +type SummaryOpenShiftNetworkV1 = configv1.Network + +func NewOpenShiftSummary() *OpenShiftSummary { + return &OpenShiftSummary{} +} + +func (os *OpenShiftSummary) SetInfrastructure(cr *configv1.InfrastructureList) error { + if len(cr.Items) == 0 { + return errors.New("Unable to find result Items to set Infrastructures") + } + os.Infrastructure = &cr.Items[0] + return nil +} + +func (os *OpenShiftSummary) GetInfrastructure() (*SummaryOpenShiftInfrastructureV1, error) { + if os.Infrastructure == nil { + return &SummaryOpenShiftInfrastructureV1{}, nil + } + return os.Infrastructure, nil +} + +func (os *OpenShiftSummary) GetClusterNetwork() (*SummaryOpenShiftClusterNetworkV1, error) { + if os.Infrastructure == nil { + return &SummaryOpenShiftClusterNetworkV1{}, nil + } + return os.ClusterNetwork, nil +} + +func (os *OpenShiftSummary) SetClusterVersion(cr *configv1.ClusterVersionList) error { + if len(cr.Items) == 0 { + return errors.New("Unable to find result Items to set Infrastructures") + } + os.ClusterVersion = &cr.Items[0] + return nil +} + +func (os *OpenShiftSummary) GetClusterVersion() (*SummaryClusterVersionOutput, error) { + if os.ClusterVersion == nil { + return &SummaryClusterVersionOutput{}, nil + } + resp := SummaryClusterVersionOutput{ + Desired: os.ClusterVersion.Status.Desired.Version, + Channel: os.ClusterVersion.Spec.Channel, + ClusterID: string(os.ClusterVersion.Spec.ClusterID), + } + for _, condition := range os.ClusterVersion.Status.Conditions { + if condition.Type == configv1.OperatorProgressing { + resp.CondProgressing = string(condition.Status) + resp.CondProgressingMessage = condition.Message + if string(condition.Status) == "True" { + resp.OverallStatusReason = fmt.Sprintf("%sProgressing ", resp.OverallStatusReason) + } + continue + } + if string(condition.Type) == "ImplicitlyEnabledCapabilities" { + resp.CondImplicitlyEnabledCapabilities = string(condition.Status) + continue + } + if string(condition.Type) == "ReleaseAccepted" { + resp.CondReleaseAccepted = string(condition.Status) + continue + } + if string(condition.Type) == "Available" { + resp.CondAvailable = string(condition.Status) + if string(condition.Status) == "False" { + resp.OverallStatus = "Unavailable" + resp.OverallStatusReason = fmt.Sprintf("%sAvailable ", resp.OverallStatusReason) + resp.OverallStatusMessage = condition.Message + } else { + resp.OverallStatus = string(condition.Type) + } + continue + } + if string(condition.Type) == "Failing" { + resp.CondFailing = string(condition.Status) + if string(condition.Status) == "True" { + resp.OverallStatus = string(condition.Type) + resp.OverallStatusReason = fmt.Sprintf("%sFailing ", resp.OverallStatusReason) + resp.OverallStatusMessage = condition.Message + } + continue + } + if string(condition.Type) == "RetrievedUpdates" { + resp.CondRetrievedUpdates = string(condition.Status) + continue + } + } + // TODO navigate through history and fill Previous + resp.Previous = "TODO" + return &resp, nil +} + +func (os *OpenShiftSummary) GetClusterVersionXY() (string, error) { + out, err := os.GetClusterVersion() + if err != nil { + return "", err + } + re := regexp.MustCompile(`^(\d+.\d+)`) + match := re.FindStringSubmatch(out.Desired) + return match[1], nil +} + +func (os *OpenShiftSummary) SetClusterOperators(cr *configv1.ClusterOperatorList) error { + if len(cr.Items) == 0 { + return errors.New("Unable to find result Items to set ClusterOperators") + } + os.ClusterOperators = cr + return nil +} + +func (os *OpenShiftSummary) GetClusterOperator() (*SummaryClusterOperatorOutput, error) { + out := SummaryClusterOperatorOutput{} + for _, co := range os.ClusterOperators.Items { + for _, condition := range co.Status.Conditions { + switch condition.Type { + case configv1.OperatorAvailable: + if condition.Status == configv1.ConditionTrue { + out.CountAvailable += 1 + } + case configv1.OperatorProgressing: + if condition.Status == configv1.ConditionTrue { + out.CountProgressing += 1 + } + case configv1.OperatorDegraded: + if condition.Status == configv1.ConditionTrue { + out.CountDegraded += 1 + } + } + } + } + return &out, nil +} + +func (os *OpenShiftSummary) SetClusterNetwork(cn *configv1.NetworkList) error { + if len(cn.Items) == 0 { + return errors.New("Unable to find result Items to set ClusterNetwork") + } + os.ClusterNetwork = &cn.Items[0] + return nil +} + +func (os *OpenShiftSummary) SetPluginResult(in *plugin.OPCTPluginSummary) error { + switch in.Name { + case plugin.PluginNameKubernetesConformance: + os.PluginResultK8sConformance = in + case plugin.PluginOldNameKubernetesConformance: + in.NameAlias = in.Name + in.Name = plugin.PluginNameKubernetesConformance + os.PluginResultK8sConformance = in + + case plugin.PluginNameOpenShiftConformance: + os.PluginResultOCPValidated = in + case plugin.PluginOldNameOpenShiftConformance: + in.NameAlias = in.Name + in.Name = plugin.PluginOldNameOpenShiftConformance + os.PluginResultOCPValidated = in + + case plugin.PluginNameOpenShiftUpgrade: + os.PluginResultConformanceUpgrade = in + case plugin.PluginNameArtifactsCollector: + os.PluginResultArtifactsCollector = in + default: + // return fmt.Errorf("unable to Set Plugin results: Plugin not found: %s", in.Name) + return nil + } + return nil +} + +func (os *OpenShiftSummary) GetResultOCPValidated() *plugin.OPCTPluginSummary { + return os.PluginResultOCPValidated +} + +func (os *OpenShiftSummary) GetResultK8SValidated() *plugin.OPCTPluginSummary { + return os.PluginResultK8sConformance +} + +func (os *OpenShiftSummary) GetResultConformanceUpgrade() *plugin.OPCTPluginSummary { + return os.PluginResultConformanceUpgrade +} + +func (os *OpenShiftSummary) GetResultArtifactsCollector() *plugin.OPCTPluginSummary { + return os.PluginResultArtifactsCollector +} diff --git a/internal/pkg/summary/result.go b/internal/opct/summary/result.go similarity index 51% rename from internal/pkg/summary/result.go rename to internal/opct/summary/result.go index 2cd04a8f..87cf85fe 100644 --- a/internal/pkg/summary/result.go +++ b/internal/opct/summary/result.go @@ -7,7 +7,11 @@ import ( "os" "github.com/pkg/errors" + "github.com/redhat-openshift-ecosystem/provider-certification-tool/internal/opct/archive" + "github.com/redhat-openshift-ecosystem/provider-certification-tool/internal/opct/plugin" + "github.com/redhat-openshift-ecosystem/provider-certification-tool/internal/openshift/mustgather" log "github.com/sirupsen/logrus" + v1 "k8s.io/api/core/v1" configv1 "github.com/openshift/api/config/v1" "github.com/vmware-tanzu/sonobuoy/pkg/client/results" @@ -17,25 +21,27 @@ import ( const ( ResultSourceNameProvider = "provider" ResultSourceNameBaseline = "baseline" - - // OpenShift Custom Resources locations on archive file - pathResourceInfrastructures = "resources/cluster/config.openshift.io_v1_infrastructures.json" - pathResourceClusterVersions = "resources/cluster/config.openshift.io_v1_clusterversions.json" - pathResourceClusterOperators = "resources/cluster/config.openshift.io_v1_clusteroperators.json" - pathPluginArtifactTestsK8S = "plugins/99-openshift-artifacts-collector/results/global/artifacts_e2e-tests_kubernetes-conformance.txt" - pathPluginArtifactTestsOCP = "plugins/99-openshift-artifacts-collector/results/global/artifacts_e2e-tests_openshift-conformance.txt" - // TODO: the following file is used to keep compatibility with versions older than v0.3 - pathPluginArtifactTestsOCP2 = "plugins/99-openshift-artifacts-collector/results/global/artifacts_e2e-openshift-conformance.txt" ) -// ResultSummary persists the reference of resulta archive +// ResultSummary persists the reference of results archive. type ResultSummary struct { Name string Archive string Sonobuoy *SonobuoySummary OpenShift *OpenShiftSummary Suites *OpenshiftTestsSuites - reader *results.Reader + + // isConformance indicates if it is a conformance plugin when true. + isConformance bool + + // reader is a file description for the archive tarball. + reader *results.Reader + + // SavePath is the target path to save the extracted report. + SavePath string + + // MustGather stores the extracted items from must-gather. + MustGather *mustgather.MustGather } // HasValidResults checks if the result instance has valid archive to be processed, @@ -53,7 +59,7 @@ func (rs *ResultSummary) HasValidResults() bool { func (rs *ResultSummary) Populate() error { if !rs.HasValidResults() { - log.Warnf("Ignoring to populate source '%s'. Missing or invalid baseline artifact (-b): %s", rs.Name, rs.Archive) + // log.Warnf("Ignoring to populate source '%s'. Missing or invalid baseline artifact (-b): %s", rs.Name, rs.Archive) return nil } @@ -73,26 +79,27 @@ func (rs *ResultSummary) Populate() error { } var lastErr error - for _, plugin := range plugins { - log.Infof("Processing Plugin %s...\n", plugin) - switch plugin { - case PluginNameOpenShiftUpgrade, PluginNameArtifactsCollector: - log.Infof("Ignoring Plugin %s", plugin) - continue + for _, pluginName := range plugins { + log.Infof("Processing Plugin %s...\n", pluginName) + switch pluginName { + case plugin.PluginNameKubernetesConformance, plugin.PluginNameOpenShiftConformance: + rs.isConformance = true } - err := rs.processPlugin(plugin) + log.Debugf("Processing results/Populating/Processing Plugin/%s", pluginName) + err := rs.processPlugin(pluginName) if err != nil { lastErr = err } } - // TODO: review the fd usage for tarbal and file + log.Info("Processing results...") cleanup, err = rs.openReader() defer cleanup() if err != nil { return err } + log.Debugf("Processing results/Populating/Populating Summary") err = rs.populateSummary() if err != nil { lastErr = err @@ -172,7 +179,7 @@ func (rs *ResultSummary) openReader() (func(), error) { } // processPlugin receives the plugin name and load the result file to be processed. -func (rs *ResultSummary) processPlugin(plugin string) error { +func (rs *ResultSummary) processPlugin(pluginName string) error { // TODO: review the fd usage for tarbal and file cleanup, err := rs.openReader() @@ -181,7 +188,7 @@ func (rs *ResultSummary) processPlugin(plugin string) error { return err } - obj, err := rs.reader.PluginResultsItem(plugin) + obj, err := rs.reader.PluginResultsItem(pluginName) if err != nil { return err } @@ -196,43 +203,52 @@ func (rs *ResultSummary) processPlugin(plugin string) error { // processPluginResult receives the plugin results object and parse it to the summary. func (rs *ResultSummary) processPluginResult(obj *results.Item) error { statusCounts := map[string]int{} - var failures []results.Item - var failedList []string + var tests []results.Item + var failures []string - statusCounts, failures = walkForSummary(obj, statusCounts, failures) + statusCounts, tests = walkForSummary(obj, statusCounts, tests) total := 0 for _, v := range statusCounts { total += v } - failedItems := make(map[string]*PluginFailedItem, len(failures)) - for _, item := range failures { - failedItems[item.Name] = &PluginFailedItem{ - Name: item.Name, + testItems := make(map[string]*plugin.TestItem, len(tests)) + for idx, item := range tests { + testItems[item.Name] = &plugin.TestItem{ + Name: item.Name, + ID: fmt.Sprintf("%s-%d", obj.Name, idx), + State: "processed", } - if _, ok := item.Details["failure"]; ok { - failedItems[item.Name].Failure = item.Details["failure"].(string) + if item.Status != "" { + testItems[item.Name].Status = item.Status } - if _, ok := item.Details["system-out"]; ok { - failedItems[item.Name].SystemOut = item.Details["system-out"].(string) + switch item.Status { + case results.StatusFailed, results.StatusTimeout: + if _, ok := item.Details["failure"]; ok { + testItems[item.Name].Failure = item.Details["failure"].(string) + } + if _, ok := item.Details["system-out"]; ok { + testItems[item.Name].SystemOut = item.Details["system-out"].(string) + } + if _, ok := item.Details["offset"]; ok { + testItems[item.Name].Offset = item.Details["offset"].(int) + } + failures = append(failures, item.Name) + testItems[item.Name].UpdateErrorCounter() } - if _, ok := item.Details["offset"]; ok { - failedItems[item.Name].Offset = item.Details["offset"].(int) - } - failedList = append(failedList, item.Name) - } - - if err := rs.GetOpenShift().SetPluginResult(&OPCTPluginSummary{ - Name: obj.Name, - Status: obj.Status, - Total: int64(total), - Passed: int64(statusCounts[results.StatusPassed]), - Failed: int64(statusCounts[results.StatusFailed] + statusCounts[results.StatusTimeout]), - Timeout: int64(statusCounts[results.StatusTimeout]), - Skipped: int64(statusCounts[results.StatusSkipped]), - FailedList: failedList, - FailedItems: failedItems, + } + + if err := rs.GetOpenShift().SetPluginResult(&plugin.OPCTPluginSummary{ + Name: obj.Name, + Status: obj.Status, + Total: int64(total), + Passed: int64(statusCounts[results.StatusPassed]), + Failed: int64(statusCounts[results.StatusFailed] + statusCounts[results.StatusTimeout]), + Timeout: int64(statusCounts[results.StatusTimeout]), + Skipped: int64(statusCounts[results.StatusSkipped]), + FailedList: failures, + Tests: testItems, }); err != nil { return err } @@ -249,14 +265,44 @@ func (rs *ResultSummary) processPluginResult(obj *results.Item) error { // information to the ResultSummary. func (rs *ResultSummary) populateSummary() error { - var bugSuiteK8S bytes.Buffer - var bugSuiteOCP bytes.Buffer + const ( + // OpenShift Custom Resources locations on archive file + pathResourceInfrastructures = "resources/cluster/config.openshift.io_v1_infrastructures.json" + pathResourceClusterVersions = "resources/cluster/config.openshift.io_v1_clusterversions.json" + pathResourceClusterOperators = "resources/cluster/config.openshift.io_v1_clusteroperators.json" + pathResourceClusterNetwork = "resources/cluster/config.openshift.io_v1_networks.json" + pathPluginArtifactTestsK8S = "plugins/99-openshift-artifacts-collector/results/global/artifacts_e2e-tests_kubernetes-conformance.txt" + pathPluginArtifactTestsOCP = "plugins/99-openshift-artifacts-collector/results/global/artifacts_e2e-tests_openshift-conformance.txt" + pathPluginDefinition10 = "plugins/10-openshift-kube-conformance/definition.json" + pathPluginDefinition20 = "plugins/20-openshift-conformance-validated/definition.json" + // TODO: the following file is used to keep compatibility with versions older than v0.3 + pathPluginArtifactTestsOCP2 = "plugins/99-openshift-artifacts-collector/results/global/artifacts_e2e-openshift-conformance.txt" + pathMustGather = "plugins/99-openshift-artifacts-collector/results/global/artifacts_must-gather.tar.xz" + pathMetaRun = "meta/run.log" + pathMetaConfig = "meta/config.json" + pathResourceNSOpctConfigMap = "resources/ns/openshift-provider-certification/core_v1_configmaps.json" + ) + + var mustGather bytes.Buffer + saveMustGather := rs.SavePath != "" + testsSuiteK8S := bytes.Buffer{} + testsSuiteOCP := bytes.Buffer{} + + metaRunLogs := bytes.Buffer{} + metaConfig := archive.MetaConfigSonobuoy{} + opctConfigMapList := v1.ConfigMapList{} + sbCluster := discovery.ClusterSummary{} ocpInfra := configv1.InfrastructureList{} ocpCV := configv1.ClusterVersionList{} ocpCO := configv1.ClusterOperatorList{} + ocpCN := configv1.NetworkList{} + + pluginDef10 := SonobuoyPluginDefinition{} + pluginDef20 := SonobuoyPluginDefinition{} // Iterate over the archive to get the items as an object to build the Summary report. + log.Debugf("Processing results/Populating/Populating Summary/Extracting") err := rs.reader.WalkFiles(func(path string, info os.FileInfo, e error) error { if err := results.ExtractFileIntoStruct(results.ClusterHealthFilePath(), path, info, &sbCluster); err != nil { return errors.Wrap(err, fmt.Sprintf("extracting file '%s': %v", path, err)) @@ -270,24 +316,50 @@ func (rs *ResultSummary) populateSummary() error { if err := results.ExtractFileIntoStruct(pathResourceClusterOperators, path, info, &ocpCO); err != nil { return errors.Wrap(err, fmt.Sprintf("extracting file '%s': %v", path, err)) } - if warn := results.ExtractBytes(pathPluginArtifactTestsK8S, path, info, &bugSuiteK8S); warn != nil { + if err := results.ExtractFileIntoStruct(pathResourceClusterNetwork, path, info, &ocpCN); err != nil { + return errors.Wrap(err, fmt.Sprintf("extracting file '%s': %v", path, err)) + } + if err := results.ExtractFileIntoStruct(pathPluginDefinition10, path, info, &pluginDef10); err != nil { + return errors.Wrap(err, fmt.Sprintf("extracting file '%s': %v", path, err)) + } + if err := results.ExtractFileIntoStruct(pathPluginDefinition20, path, info, &pluginDef20); err != nil { + return errors.Wrap(err, fmt.Sprintf("extracting file '%s': %v", path, err)) + } + if warn := results.ExtractBytes(pathPluginArtifactTestsK8S, path, info, &testsSuiteK8S); warn != nil { log.Warnf("Unable to load file %s: %v\n", pathPluginArtifactTestsK8S, warn) return errors.Wrap(warn, fmt.Sprintf("extracting file '%s': %v", path, warn)) } - if warn := results.ExtractBytes(pathPluginArtifactTestsOCP, path, info, &bugSuiteOCP); warn != nil { + if warn := results.ExtractBytes(pathPluginArtifactTestsOCP, path, info, &testsSuiteOCP); warn != nil { log.Warnf("Unable to load file %s: %v\n", pathPluginArtifactTestsOCP, warn) return errors.Wrap(warn, fmt.Sprintf("extracting file '%s': %v", path, warn)) } - if warn := results.ExtractBytes(pathPluginArtifactTestsOCP2, path, info, &bugSuiteOCP); warn != nil { + if warn := results.ExtractBytes(pathPluginArtifactTestsOCP2, path, info, &testsSuiteOCP); warn != nil { log.Warnf("Unable to load file %s: %v\n", pathPluginArtifactTestsOCP2, warn) return errors.Wrap(warn, fmt.Sprintf("extracting file '%s': %v", path, warn)) } + if warn := results.ExtractBytes(pathMetaRun, path, info, &metaRunLogs); warn != nil { + log.Warnf("Unable to load file %s: %v\n", pathMetaRun, warn) + return errors.Wrap(warn, fmt.Sprintf("extracting file '%s': %v", path, warn)) + } + if err := results.ExtractFileIntoStruct(pathMetaConfig, path, info, &metaConfig); err != nil { + return errors.Wrap(err, fmt.Sprintf("extracting file '%s': %v", path, err)) + } + if err := results.ExtractFileIntoStruct(pathResourceNSOpctConfigMap, path, info, &opctConfigMapList); err != nil { + return errors.Wrap(err, fmt.Sprintf("extracting file '%s': %v", path, err)) + } + if saveMustGather { + if warn := results.ExtractBytes(pathMustGather, path, info, &mustGather); warn != nil { + log.Warnf("Unable to load file %s: %v\n", pathMustGather, warn) + return errors.Wrap(warn, fmt.Sprintf("extracting file '%s': %v", path, warn)) + } + } return e }) if err != nil { return err } + log.Debugf("Processing results/Populating/Populating Summary/Processing") if err := rs.GetSonobuoy().SetCluster(&sbCluster); err != nil { return err } @@ -300,13 +372,32 @@ func (rs *ResultSummary) populateSummary() error { if err := rs.GetOpenShift().SetClusterOperators(&ocpCO); err != nil { return err } - if err := rs.Suites.KubernetesConformance.Load(pathPluginArtifactTestsK8S, &bugSuiteK8S); err != nil { + if err := rs.GetOpenShift().SetClusterNetwork(&ocpCN); err != nil { return err } - if err := rs.Suites.OpenshiftConformance.Load(pathPluginArtifactTestsOCP, &bugSuiteOCP); err != nil { + if err := rs.Suites.KubernetesConformance.Load(pathPluginArtifactTestsK8S, &testsSuiteK8S); err != nil { return err } - + if err := rs.Suites.OpenshiftConformance.Load(pathPluginArtifactTestsOCP, &testsSuiteOCP); err != nil { + return err + } + rs.GetSonobuoy().SetPluginDefinition(plugin.PluginNameKubernetesConformance, &pluginDef10) + rs.GetSonobuoy().SetPluginDefinition(plugin.PluginNameOpenShiftConformance, &pluginDef20) + rs.GetSonobuoy().ParseMetaRunlogs(&metaRunLogs) + rs.GetSonobuoy().ParseMetaConfig(&metaConfig) + rs.GetSonobuoy().ParseOpctConfigMap(&opctConfigMapList) + + if saveMustGather { + log.Debugf("Processing results/Populating/Populating Summary/Processing/MustGather") + rs.MustGather = mustgather.NewMustGather(fmt.Sprintf("%s/must-gather", rs.SavePath)) + if err := rs.MustGather.Process(&mustGather); err != nil { + log.Errorf("Processing results/Populating/Populating Summary/Processing/MustGather: %v", err) + } else { + log.Debugf("Processing results/Populating/Populating Summary/Processing/MustGather/CalculatingErrors") + // Non blocking + rs.MustGather.AggregateCounters() + } + } return nil } @@ -324,8 +415,8 @@ func walkForSummary(result *results.Item, statusCounts map[string]int, failList if result.Status == results.StatusFailed || result.Status == results.StatusTimeout { result.Details["offset"] = statusCounts[result.Status] - failList = append(failList, *result) } + failList = append(failList, *result) return statusCounts, failList } diff --git a/internal/opct/summary/sonobuoy.go b/internal/opct/summary/sonobuoy.go new file mode 100644 index 00000000..83ba894a --- /dev/null +++ b/internal/opct/summary/sonobuoy.go @@ -0,0 +1,60 @@ +package summary + +import ( + "bytes" + "strings" + + "github.com/redhat-openshift-ecosystem/provider-certification-tool/internal/opct/archive" + "github.com/vmware-tanzu/sonobuoy/pkg/discovery" + "github.com/vmware-tanzu/sonobuoy/pkg/plugin/manifest" + v1 "k8s.io/api/core/v1" +) + +type SonobuoyPluginDefinitionManifest = manifest.Manifest + +// Plugin is the sonobuoy plugin definitoin. +type SonobuoyPluginDefinition struct { + Definition *SonobuoyPluginDefinitionManifest `json:"Definition"` + SonobuoyImage string `json:"SonobuoyImage"` +} + +type SonobuoySummary struct { + Cluster *discovery.ClusterSummary + MetaRuntime []*archive.RuntimeInfoItem + MetaConfig []*archive.RuntimeInfoItem + OpctConfig []*archive.RuntimeInfoItem + PluginsDefinition map[string]*SonobuoyPluginDefinition +} + +func NewSonobuoySummary() *SonobuoySummary { + return &SonobuoySummary{ + PluginsDefinition: make(map[string]*SonobuoyPluginDefinition, 4), + } +} + +func (s *SonobuoySummary) SetCluster(c *discovery.ClusterSummary) error { + s.Cluster = c + return nil +} + +func (s *SonobuoySummary) SetPluginsDefinition(p map[string]*SonobuoyPluginDefinition) error { + s.PluginsDefinition = make(map[string]*SonobuoyPluginDefinition, len(p)) + s.PluginsDefinition = p + return nil +} + +func (s *SonobuoySummary) SetPluginDefinition(name string, def *SonobuoyPluginDefinition) { + s.PluginsDefinition[name] = def +} + +func (s *SonobuoySummary) ParseMetaRunlogs(logLines *bytes.Buffer) { + s.MetaRuntime = archive.ParseMetaLogs(strings.Split(logLines.String(), "\n")) +} + +func (s *SonobuoySummary) ParseMetaConfig(metaConfig *archive.MetaConfigSonobuoy) { + s.MetaConfig = archive.ParseMetaConfig(metaConfig) +} + +func (s *SonobuoySummary) ParseOpctConfigMap(cm *v1.ConfigMapList) { + s.OpctConfig = archive.ParseOpctConfig(cm) +} diff --git a/internal/pkg/summary/suite.go b/internal/opct/summary/suite.go similarity index 96% rename from internal/pkg/summary/suite.go rename to internal/opct/summary/suite.go index 2f1d221c..82a73e52 100644 --- a/internal/pkg/summary/suite.go +++ b/internal/opct/summary/suite.go @@ -27,7 +27,7 @@ type OpenshiftTestsSuite struct { InputFile string Name string Count int - Tests []string + Tests []string `json:"-"` } func (s *OpenshiftTestsSuite) Load(ifile string, buf *bytes.Buffer) error { diff --git a/internal/pkg/sippy/sippy.go b/internal/openshift/ci/sippy/sippy.go similarity index 89% rename from internal/pkg/sippy/sippy.go rename to internal/openshift/ci/sippy/sippy.go index 75d94243..e81276f7 100644 --- a/internal/pkg/sippy/sippy.go +++ b/internal/openshift/ci/sippy/sippy.go @@ -55,17 +55,19 @@ type SippyTestsRequestOutput []SippyTestsResponse // SippyAPI is the Sippy API structure holding the API client type SippyAPI struct { - client *http.Client + client *http.Client + ocpVersion string } // NewSippyAPI creates a new API setting the http attributes to improve the connection reuse. -func NewSippyAPI() *SippyAPI { +func NewSippyAPI(ocpVersion string) *SippyAPI { t := http.DefaultTransport.(*http.Transport).Clone() t.MaxIdleConns = defaultMaxIdleConns t.MaxConnsPerHost = defaultMaxConnsPerHost t.MaxIdleConnsPerHost = defaultMaxIddleConnsPerHost return &SippyAPI{ + ocpVersion: ocpVersion, client: &http.Client{ Timeout: defaultConnTimeoutSec * time.Second, Transport: t, @@ -75,14 +77,14 @@ func NewSippyAPI() *SippyAPI { // QueryTests receive a input with attributes to query the results of a single test // by name on the CI, returning the list with result items. -func (a *SippyAPI) QueryTests(r *SippyTestsRequestInput) (*SippyTestsRequestOutput, error) { +func (a *SippyAPI) QueryTests(in *SippyTestsRequestInput) (*SippyTestsRequestOutput, error) { filter := SippyTestsRequestFilter{ Items: []SippyTestsRequestFilterItems{ { ColumnField: "name", OperatorValue: "equals", - Value: r.TestName, + Value: in.TestName, }, }, } @@ -98,7 +100,7 @@ func (a *SippyAPI) QueryTests(r *SippyTestsRequestInput) (*SippyTestsRequestOutp } params := url.Values{} - params.Add("release", "4.11") + params.Add("release", a.ocpVersion) params.Add("filter", string(b)) baseUrl.RawQuery = params.Encode() @@ -121,6 +123,10 @@ func (a *SippyAPI) QueryTests(r *SippyTestsRequestInput) (*SippyTestsRequestOutp } + if res.StatusCode < 200 || res.StatusCode > 299 { + return nil, fmt.Errorf("invalid status code: %d", res.StatusCode) + } + sippyResponse := SippyTestsRequestOutput{} if err := json.Unmarshal([]byte(body), &sippyResponse); err != nil { return nil, fmt.Errorf("couldn't unmarshal response body: %+v \nBody: %s", string(body), err) diff --git a/internal/openshift/ci/types.go b/internal/openshift/ci/types.go new file mode 100644 index 00000000..59a8fbf5 --- /dev/null +++ b/internal/openshift/ci/types.go @@ -0,0 +1,20 @@ +package ci + +// Source: https://github.com/openshift/release/blob/master/core-services/prow/02_config/_config.yaml#L84 +var CommonErrorPatterns = []string{ + `error:`, + `Failed to push image`, + `Failed`, + `timed out`, + `'ERROR:'`, + `ERRO\[`, + `^error:`, + `(^FAIL|FAIL: |Failure \[)\b`, + `panic(\.go)?:`, + `"level":"error"`, + `level=error`, + `level":"fatal"`, + `level=fatal`, + `│ Error:`, + `client connection lost`, +} diff --git a/internal/openshift/mustgather/etcd.go b/internal/openshift/mustgather/etcd.go new file mode 100644 index 00000000..13f26836 --- /dev/null +++ b/internal/openshift/mustgather/etcd.go @@ -0,0 +1,384 @@ +package mustgather + +import ( + "encoding/json" + "fmt" + "regexp" + "sort" + "strconv" + "strings" + + "github.com/montanaflynn/stats" + "github.com/redhat-openshift-ecosystem/provider-certification-tool/internal/opct/archive" + log "github.com/sirupsen/logrus" +) + +// ErrorEtcdLogs handle errors extracted/parsed from etcd pod logs. +type ErrorEtcdLogs struct { + ErrorCounters archive.ErrorCounter + FilterRequestSlowAll map[string]*BucketFilterStat + FilterRequestSlowHour map[string]*BucketFilterStat + Buffer []*string `json:"-"` +} + +// common errors to create counters +var commonTestErrorPatternEtcdLogs = []string{ + `rejected connection`, + `waiting for ReadIndex response took too long, retrying`, + `failed to find remote peer in cluster`, + `dropped Raft message since sending buffer is full (overloaded network)`, + `request stats`, + `apply request took too long`, + `failed to lock file`, +} + +func NewErrorEtcdLogs(buf *string) *ErrorEtcdLogs { + etcdLogs := &ErrorEtcdLogs{} + + // create counters + etcdLogs.ErrorCounters = archive.NewErrorCounter(buf, commonTestErrorPatternEtcdLogs) + + // filter Slow Requests (aggregate by hour) + filterATTL1 := NewFilterApplyTookTooLong("hour") + for _, line := range strings.Split(*buf, "\n") { + errLogLine := filterATTL1.ProcessLine(line) + if errLogLine != nil { + etcdLogs.Buffer = append(etcdLogs.Buffer, errLogLine) + } + } + // Check only the last 3 hours (average time of an opct execution) + etcdLogs.FilterRequestSlowHour = filterATTL1.GetStat(3) + + // filter Slow Requests (aggregate all) + filterATTL2 := NewFilterApplyTookTooLong("all") + for _, line := range strings.Split(*buf, "\n") { + filterATTL2.ProcessLine(line) + } + etcdLogs.FilterRequestSlowAll = filterATTL2.GetStat(1) + + return etcdLogs +} + +// LogPayloadETCD parses the etcd log file to extract insights +// {"level":"warn","ts":"2023-03-01T15:14:22.192Z", +// "caller":"etcdserver/util.go:166", +// "msg":"apply request took too long", +// "took":"231.023586ms","expected-duration":"200ms", +// "prefix":"read-only range ", +// "request":"key:\"/kubernetes.io/configmaps/kube-system/kube-controller-manager\" ", +// "response":"range_response_count:1 size:608"} +type LogPayloadETCD struct { + Took string `json:"took"` + Timestamp string `json:"ts"` +} + +type BucketGroup struct { + Bukets1s Buckets + Bukets500ms Buckets +} + +type FilterApplyTookTooLong struct { + Name string + GroupBy string + Group map[string]*BucketGroup + + // filter config + lineFilter string + reLineSplitter *regexp.Regexp + reMili *regexp.Regexp + reSec *regexp.Regexp + reTsMin *regexp.Regexp + reTsHour *regexp.Regexp + reTsDay *regexp.Regexp +} + +func NewFilterApplyTookTooLong(aggregator string) *FilterApplyTookTooLong { + var filter FilterApplyTookTooLong + + filter.Name = "ApplyTookTooLong" + filter.GroupBy = aggregator + filter.Group = make(map[string]*BucketGroup) + + filter.lineFilter = "apply request took too long" + filter.reLineSplitter, _ = regexp.Compile(`^\d+-\d+-\d+T\d+:\d+:\d+.\d+Z `) + filter.reMili, _ = regexp.Compile("([0-9]+.[0-9]+)ms") + filter.reSec, _ = regexp.Compile("([0-9]+.[0-9]+)s") + filter.reTsMin, _ = regexp.Compile(`^(\d+-\d+-\d+T\d+:\d+):\d+.\d+Z`) + filter.reTsHour, _ = regexp.Compile(`^(\d+-\d+-\d+T\d+):\d+:\d+.\d+Z`) + filter.reTsDay, _ = regexp.Compile(`^(\d+-\d+-\d+)T\d+:\d+:\d+.\d+Z`) + + return &filter +} + +func (f *FilterApplyTookTooLong) ProcessLine(line string) *string { + + // filter by required filter + if !strings.Contains(line, f.lineFilter) { + return nil + } + + // split timestamp + split := f.reLineSplitter.Split(line, -1) + if len(split) < 1 { + return nil + } + + // parse json + lineParsed := LogPayloadETCD{} + if err := json.Unmarshal([]byte(split[1]), &lineParsed); err != nil { + log.Errorf("couldn't parse json: %v", err) + } + + if match := f.reMili.MatchString(lineParsed.Took); match { // Extract milisseconds + matches := f.reMili.FindStringSubmatch(lineParsed.Took) + if len(matches) == 2 { + if v, err := strconv.ParseFloat(matches[1], 64); err == nil { + f.insertBucket(v, lineParsed.Timestamp) + } + } + } else if match := f.reSec.MatchString(lineParsed.Took); match { // Extract seconds + matches := f.reSec.FindStringSubmatch(lineParsed.Took) + if len(matches) == 2 { + if v, err := strconv.ParseFloat(matches[1], 64); err == nil { + v = v * 1000 + f.insertBucket(v, lineParsed.Timestamp) + } + } + } else { + fmt.Printf("No bucket for: %v\n", lineParsed.Took) + } + + return &line +} + +func (f *FilterApplyTookTooLong) insertBucket(v float64, ts string) { + var group *BucketGroup + var aggrKey string + + if f.GroupBy == "hour" { + aggrValue := "all" + if match := f.reTsHour.MatchString(ts); match { + matches := f.reTsHour.FindStringSubmatch(ts) + aggrValue = matches[1] + } + aggrKey = aggrValue + } else if f.GroupBy == "day" { + aggrValue := "all" + if match := f.reTsDay.MatchString(ts); match { + matches := f.reTsDay.FindStringSubmatch(ts) + aggrValue = matches[1] + } + aggrKey = aggrValue + } else if f.GroupBy == "minute" || f.GroupBy == "min" { + aggrValue := "all" + if match := f.reTsMin.MatchString(ts); match { + matches := f.reTsMin.FindStringSubmatch(ts) + aggrValue = matches[1] + } + aggrKey = aggrValue + } else { + aggrKey = f.GroupBy + } + + if _, ok := f.Group[aggrKey]; !ok { + f.Group[aggrKey] = &BucketGroup{} + group = f.Group[aggrKey] + group.Bukets1s = NewBuckets(buckets1s()) + group.Bukets500ms = NewBuckets(buckets500ms()) + } else { + group = f.Group[aggrKey] + } + + b1s := group.Bukets1s + b500ms := group.Bukets500ms + + switch { + case v < 200: + log.Debugf("etcd log parser - got request slower than 200 (should not happen): %v", v) + + case ((v >= 200) && (v < 300)): + k := "200-300" + b1s[k] = append(b1s[k], v) + b500ms[k] = append(b500ms[k], v) + + case ((v >= 300) && (v < 400)): + k := "300-400" + b1s[k] = append(b1s[k], v) + b500ms[k] = append(b500ms[k], v) + + case ((v >= 400) && (v < 500)): + k := "400-500" + + b1s[k] = append(b1s[k], v) + b500ms[k] = append(b500ms[k], v) + case ((v >= 500) && (v < 600)): + k := "500-600" + b1s[k] = append(b1s[k], v) + + k = "500-inf" + b500ms[k] = append(b500ms[k], v) + + case ((v >= 600) && (v < 700)): + k := "600-700" + b1s[k] = append(b1s[k], v) + + k = "500-inf" + b500ms[k] = append(b500ms[k], v) + case ((v >= 700) && (v < 800)): + k := "700-800" + b1s[k] = append(b1s[k], v) + + k = "500-inf" + b500ms[k] = append(b500ms[k], v) + + case ((v >= 800) && (v < 900)): + k := "800-900" + b1s[k] = append(b1s[k], v) + + k = "500-inf" + b500ms[k] = append(b500ms[k], v) + + case ((v >= 900) && (v < 1000)): + k := "900-999" + b1s[k] = append(b1s[k], v) + + k = "500-inf" + b500ms[k] = append(b500ms[k], v) + + case (v >= 1000): + k := "1000-inf" + b1s[k] = append(b1s[k], v) + + k = "500-inf" + b500ms[k] = append(b500ms[k], v) + + default: + k := "unkw" + b1s[k] = append(b1s[k], v) + b500ms[k] = append(b500ms[k], v) + } + k := "all" + b1s[k] = append(b1s[k], v) + b500ms[k] = append(b500ms[k], v) +} + +type BucketFilterStat struct { + RequestCount int64 + Higher500ms string + Buckets map[string]string + StatCount string + StatMin string + StatMedian string + StatMean string + StatMax string + StatSum string + StatStddev string + StatPerc90 string + StatPerc99 string + StatPerc999 string + StatOutliers string +} + +func (f *FilterApplyTookTooLong) GetStat(latest int) map[string]*BucketFilterStat { + + groups := make([]string, 0, len(f.Group)) + for k := range f.Group { + groups = append(groups, k) + } + sort.Strings(groups) + + // filter latest group + if latest > len(groups) { + latest = len(groups) + } + latestGroups := groups[len(groups)-latest:] + statGroups := make(map[string]*BucketFilterStat, latest) + for _, gk := range latestGroups { + group := f.Group[gk] + statGroups[gk] = &BucketFilterStat{} + + b1s := group.Bukets1s + b500ms := group.Bukets500ms + + getBucketStr := func(k string) string { + countB1ms := len(b1s[k]) + countB1all := len(b1s["all"]) + perc := fmt.Sprintf("(%.3f%%)", (float64(countB1ms)/float64(countB1all))*100) + if k == "all" { + perc = "" + } + // return fmt.Sprintf("%8.8s %6s %11.10s", k, fmt.Sprintf("%d", countB1ms), perc) + return fmt.Sprintf("%d %s", countB1ms, perc) + } + statGroups[gk].RequestCount = int64(len(b1s["all"])) + + v500 := len(b500ms["500-inf"]) + perc500inf := (float64(v500) / float64(len(b500ms["all"]))) * 100 + statGroups[gk].Higher500ms = fmt.Sprintf("%s (%.3f%%)", fmt.Sprintf("%d", v500), perc500inf) + + bukets := buckets1s() + statGroups[gk].Buckets = make(map[string]string, len(bukets)) + for _, bkt := range bukets { + statGroups[gk].Buckets[bkt] = getBucketStr(bkt) + } + + min, _ := stats.Min(b1s["all"]) + max, _ := stats.Max(b1s["all"]) + sum, _ := stats.Sum(b1s["all"]) + mean, _ := stats.Mean(b1s["all"]) + median, _ := stats.Median(b1s["all"]) + p90, _ := stats.Percentile(b1s["all"], 90) + p99, _ := stats.Percentile(b1s["all"], 99) + p999, _ := stats.Percentile(b1s["all"], 99.9) + stddev, _ := stats.StandardDeviationPopulation(b1s["all"]) + qoutliers, _ := stats.QuartileOutliers(b1s["all"]) + + statGroups[gk].StatCount = fmt.Sprintf("%d", len(b1s["all"])) + statGroups[gk].StatMin = fmt.Sprintf("%.3f (ms)", min) + statGroups[gk].StatMedian = fmt.Sprintf("%.3f (ms)", median) + statGroups[gk].StatMean = fmt.Sprintf("%.3f (ms)", mean) + statGroups[gk].StatMax = fmt.Sprintf("%.3f (ms)", max) + statGroups[gk].StatSum = fmt.Sprintf("%.3f (ms)", sum) + statGroups[gk].StatStddev = fmt.Sprintf("%.3f", stddev) + statGroups[gk].StatPerc90 = fmt.Sprintf("%.3f (ms)", p90) + statGroups[gk].StatPerc99 = fmt.Sprintf("%.3f (ms)", p99) + statGroups[gk].StatPerc999 = fmt.Sprintf("%.3f (ms)", p999) + statGroups[gk].StatOutliers = fmt.Sprintf("%v", qoutliers) + } + return statGroups +} + +func buckets1s() []string { + return []string{ + "200-300", + "300-400", + "400-500", + "500-600", + "600-700", + "700-800", + "800-900", + "900-999", + "1000-inf", + "all", + } +} + +func buckets500ms() []string { + return []string{ + "200-300", + "300-400", + "400-500", + "500-inf", + "all", + } +} + +type Buckets map[string][]float64 + +func NewBuckets(values []string) Buckets { + buckets := make(Buckets, len(values)) + for _, v := range values { + buckets[v] = []float64{} + } + return buckets +} diff --git a/internal/openshift/mustgather/mustgather.go b/internal/openshift/mustgather/mustgather.go new file mode 100644 index 00000000..75d9a605 --- /dev/null +++ b/internal/openshift/mustgather/mustgather.go @@ -0,0 +1,462 @@ +package mustgather + +import ( + "archive/tar" + "bytes" + "io" + "os" + "path/filepath" + "regexp" + "strings" + "sync" + "time" + + "github.com/pkg/errors" + "github.com/redhat-openshift-ecosystem/provider-certification-tool/internal/opct/archive" + "github.com/redhat-openshift-ecosystem/provider-certification-tool/internal/openshift/ci" + log "github.com/sirupsen/logrus" + "github.com/ulikunitz/xz" + "gopkg.in/yaml.v2" + "k8s.io/utils/pointer" +) + +/* MustGatehr raw files */ +type MustGatherFile struct { + Path string + PathAlias string `json:"PathAlias,omitempty"` + Data string `json:"Data,omitempty"` +} + +type MustGather struct { + // path to the directory must-gather will be saved. + path string + + // ErrorEtcdLogs summary of etcd errors parsed from must-gather. + ErrorEtcdLogs *ErrorEtcdLogs `json:"ErrorEtcdLogs,omitempty"` + ErrorEtcdLogsBuffer []*string `json:"-"` + + // ErrorCounters summary error counters parsed from must-gather. + ErrorCounters archive.ErrorCounter `json:"ErrorCounters,omitempty"` + + // NamespaceErrors hold pods reporting errors. + NamespaceErrors []*MustGatherLog `json:"NamespaceErrors,omitempty"` + namespaceCtrl sync.Mutex + + // FileData hold raw data from files must-gather. + RawFiles []*MustGatherFile `json:"RawFiles,omitempty"` + rawFilesCtrl sync.Mutex + + PodNetworkChecks MustGatherPodNetworkChecks +} + +func NewMustGather(file string) *MustGather { + return &MustGather{ + path: file, + } +} + +// InsertNamespaceErrors append the log data in safe way. +func (mg *MustGather) InsertNamespaceErrors(log *MustGatherLog) error { + mg.namespaceCtrl.Lock() + mg.NamespaceErrors = append(mg.NamespaceErrors, log) + mg.namespaceCtrl.Unlock() + return nil +} + +// InsertRawFiles append the file data in safe way. +func (mg *MustGather) InsertRawFiles(file *MustGatherFile) error { + mg.rawFilesCtrl.Lock() + mg.RawFiles = append(mg.RawFiles, file) + mg.rawFilesCtrl.Unlock() + return nil +} + +func (mg *MustGather) AggregateCounters() { + if mg.ErrorCounters == nil { + mg.ErrorCounters = make(archive.ErrorCounter, len(ci.CommonErrorPatterns)) + } + if mg.ErrorEtcdLogs == nil { + mg.ErrorEtcdLogs = &ErrorEtcdLogs{} + } + for nsi := range mg.NamespaceErrors { + // calculate + hasErrorCounters := false + hasEtcdCounters := false + if mg.NamespaceErrors[nsi].ErrorCounters != nil { + hasErrorCounters = true + } + if mg.NamespaceErrors[nsi].ErrorEtcdLogs != nil { + hasEtcdCounters = true + } + if mg.NamespaceErrors[nsi].ErrorEtcdLogs != nil { + if mg.NamespaceErrors[nsi].Namespace == "openshift-etcd" && + mg.NamespaceErrors[nsi].Container == "etcd" && + strings.HasSuffix(mg.NamespaceErrors[nsi].Path, "current.log") { + hasEtcdCounters = true + } + + } + // Error Counters + if hasErrorCounters { + for errName, errCounter := range mg.NamespaceErrors[nsi].ErrorCounters { + if _, ok := mg.ErrorCounters[errName]; !ok { + mg.ErrorCounters[errName] = errCounter + } else { + mg.ErrorCounters[errName] += errCounter + } + } + } + + // Aggregate logs for each etcd pod + if hasEtcdCounters { + // aggregate etcd request errors + log.Debugf("Processing results/Populating/Populating Summary/Processing/MustGather/CalculatingErrors/AggregatingPod{%s}", mg.NamespaceErrors[nsi].Pod) + if mg.NamespaceErrors[nsi].ErrorEtcdLogs.Buffer != nil { + mg.ErrorEtcdLogsBuffer = append(mg.ErrorEtcdLogsBuffer, mg.NamespaceErrors[nsi].ErrorEtcdLogs.Buffer...) + } + // aggregate etcd error counters + if mg.NamespaceErrors[nsi].ErrorEtcdLogs.ErrorCounters != nil { + if mg.ErrorEtcdLogs.ErrorCounters == nil { + mg.ErrorEtcdLogs.ErrorCounters = make(archive.ErrorCounter, len(commonTestErrorPatternEtcdLogs)) + } + for errName, errCounter := range mg.NamespaceErrors[nsi].ErrorEtcdLogs.ErrorCounters { + if _, ok := mg.ErrorEtcdLogs.ErrorCounters[errName]; !ok { + mg.ErrorEtcdLogs.ErrorCounters[errName] = errCounter + } else { + mg.ErrorEtcdLogs.ErrorCounters[errName] += errCounter + } + } + } + } + } + log.Debugf("Processing results/Populating/Populating Summary/Processing/MustGather/CalculatingErrors/CalculatingEtcdErrors") + mg.CalculateCountersEtcd() +} + +// CalculateCountersEtcd creates the aggregators, generating counters for each one. +func (mg *MustGather) CalculateCountersEtcd() { + + // filter Slow Requests (aggregate by hour) + filterATTL1 := NewFilterApplyTookTooLong("hour") + for _, line := range mg.ErrorEtcdLogsBuffer { + filterATTL1.ProcessLine(*line) + } + mg.ErrorEtcdLogs.FilterRequestSlowHour = filterATTL1.GetStat(4) + + // filter Slow Requests (aggregate all) + filterATTL2 := NewFilterApplyTookTooLong("all") + for _, line := range mg.ErrorEtcdLogsBuffer { + filterATTL2.ProcessLine(*line) + } + mg.ErrorEtcdLogs.FilterRequestSlowAll = filterATTL2.GetStat(1) +} + +// Process read the must-gather tarball. +func (mg *MustGather) Process(buf *bytes.Buffer) error { + log.Debugf("Processing results/Populating/Populating Summary/Processing/MustGather/Reading") + tar, err := mg.read(buf) + if err != nil { + return err + } + log.Debugf("Processing results/Populating/Populating Summary/Processing/MustGather/Processing") + err = mg.extract(tar) + if err != nil { + return err + } + return nil +} + +func (mg *MustGather) read(buf *bytes.Buffer) (*tar.Reader, error) { + file, err := xz.NewReader(buf) + if err != nil { + return nil, err + } + return tar.NewReader(file), nil +} + +// matchToExtract define patterns to continue the must-gather processor. +// the pattern must be defined if the must be extracted. It will return +// a boolean with match and the file group (pattern type). +func (mg *MustGather) matchToExtract(path string) (bool, string) { + patterns := make(map[string]string, 4) + patterns["logs"] = `(\/namespaces\/.*\/pods\/.*.log)` + patterns["events"] = `(\/event-filter.html)` + patterns["rawFile"] = `(\/etcd_info\/.*.json)` + patterns["podNetCheck"] = `(\/pod_network_connectivity_check\/podnetworkconnectivitychecks.yaml)` + // TODO /host_service_logs/.*.log + for typ, pattern := range patterns { + re := regexp.MustCompile(pattern) + if re.MatchString(path) { + return true, typ + } + } + return false, "" +} + +// extractRelativePath removes the prefix of must-gather path/image to save the +// relative file path when extracting the file or mapping in the counters. +// OPCT collects must-gather automatically saving in the directory must-gather-opct. +func (mg *MustGather) extractRelativePath(file string) string { + re := regexp.MustCompile(`must-gather-opct/([A-Za-z0-9]+(-[A-Za-z0-9]+)+\/)`) + + split := re.Split(file, -1) + if len(split) != 2 { + return file + } + return split[1] +} + +// extract dispatch to process must-gather items. +func (mg *MustGather) extract(tarball *tar.Reader) error { + + // Create must-gather directory + if _, err := os.Stat(mg.path); err != nil { + if err := os.MkdirAll(mg.path, 0755); err != nil { + return err + } + } + + // TODO()#1: create a queue package with a instance of MustGatherLog. + // TODO()#2: increase the parallelism targetting to decrease the total proc time. + // Leaky bucket implementation (queue limit) to parallel process must-gather items + // without exhausting resources. + // Benckmark info: this parallel processing decreased 3 times the total processing time. + // Samples: Serial=~100s, rate(100)=~30s, rate(150)=~25s. + keepReading := true + procQueueSize := 0 + var procQueueLocker sync.Mutex + // Creating queue monitor as Waiter group does not provide interface to check the + // queue size. + procQueueInc := func() { + procQueueLocker.Lock() + procQueueSize += 1 + procQueueLocker.Unlock() + } + procQueueDec := func() { + procQueueLocker.Lock() + procQueueSize -= 1 + procQueueLocker.Unlock() + } + go func() { + for keepReading { + log.Debugf("Must-gather processor - queue size monitor: %d", procQueueSize) + time.Sleep(10 * time.Second) + } + }() + + waiterProcNS := &sync.WaitGroup{} + chProcNSErrors := make(chan *MustGatherLog, 50) + semaphore := make(chan struct{}, 50) + // have a max rate of N/sec + rate := make(chan struct{}, 20) + for i := 0; i < cap(rate); i++ { + rate <- struct{}{} + } + // leaky bucket + go func() { + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + for range ticker.C { + _, ok := <-rate + // if this isn't going to run indefinitely, signal + // this to return by closing the rate channel. + if !ok { + return + } + } + }() + // consumer + go func() { + for mgLog := range chProcNSErrors { + mg.processNamespaceErrors(mgLog) + waiterProcNS.Done() + procQueueDec() + } + }() + + // Walk through files in must-gather tarball file. + for keepReading { + header, err := tarball.Next() + + switch { + // no more files + case err == io.EOF: + log.Debugf("Must-gather processor queued, queue size: %d", procQueueSize) + waiterProcNS.Wait() + keepReading = false + log.Debugf("Must-gather processor finished, queue size: %d", procQueueSize) + return nil + + // return on error + case err != nil: + return errors.Wrapf(err, "error reading tarball") + // return err + + // skip it when the headr isn't set (not sure how this happens) + case header == nil: + continue + } + + // the target location where the dir/file should be created. + target := filepath.Join(mg.path, header.Name) + ok, typ := mg.matchToExtract(target) + if !ok { + continue + } + targetAlias := mg.extractRelativePath(target) + + // the following switch could also be done using fi.Mode(), not sure if there + // a benefit of using one vs. the other. + // fi := header.FileInfo() + + switch header.Typeflag { + // directories in tarball. + case tar.TypeDir: + + // creating subdirectories structures will be ignored and need + // sub-directories under mg.path must be created previously if needed. + /* + targetDir := filepath.Join(mg.path, targetAlias) + if _, err := os.Stat(targetDir); err != nil { + if err := os.MkdirAll(targetDir, 0755); err != nil { + return err + } + } + */ + continue + + // files in tarball. Process only files classified by 'typ'. + case tar.TypeReg: + // Save/Process only files matching now types, it will prevent processing && saving + // all the files in must-gather, extracting only information needed by OPCT. + switch typ { + case "logs": + // parallel processing the logs + buf := bytes.Buffer{} + if _, err := io.Copy(&buf, tarball); err != nil { + return err + } + waiterProcNS.Add(1) + procQueueInc() + go func(filename string, buffer *bytes.Buffer) { + // wait for the rate limiter + rate <- struct{}{} + + // check the concurrency semaphore + semaphore <- struct{}{} + defer func() { + <-semaphore + }() + // log.Debugf("Producing log processor for file: %s", mgLog.Path) + chProcNSErrors <- &MustGatherLog{ + Path: filename, + buffer: buffer, + } + }(targetAlias, &buf) + + case "events": + // forcing file name for event filter + targetLocal := filepath.Join(mg.path, "event-filter.html") + f, err := os.OpenFile(targetLocal, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode)) + if err != nil { + return err + } + if _, err := io.Copy(f, tarball); err != nil { + return err + } + f.Close() + + case "rawFile": + log.Debugf("Must-gather extracting file %s", targetAlias) + raw := &MustGatherFile{} + raw.Path = targetAlias + buf := bytes.Buffer{} + if _, err := io.Copy(&buf, tarball); err != nil { + log.Errorf("error copying rawfile: %v", err) + break + } + raw.Data = buf.String() + err := mg.InsertRawFiles(raw) + if err != nil { + log.Errorf("error inserting rawfile: %v", err) + } + + case "podNetCheck": + log.Debugf("Must-gather extracting file %s", targetAlias) + raw := &MustGatherFile{} + raw.Path = targetAlias + buf := bytes.Buffer{} + if _, err := io.Copy(&buf, tarball); err != nil { + log.Errorf("error copying rawfile: %v", err) + break + } + var data map[string]interface{} + + err := yaml.Unmarshal(buf.Bytes(), &data) + if err != nil { + log.Errorf("error parsing yaml podNetCheck: %v", err) + break + } + + mg.PodNetworkChecks.Parse(data) + } + } + } + + return nil +} + +// processNamespaceErrors implements the consumer logic, creating the +// mustGather log item, processing it, appending to the data stored in +// NamespaceError. It must not stop on errors, but must log it. +func (mg *MustGather) processNamespaceErrors(mgLog *MustGatherLog) { + pathItems := strings.Split(mgLog.Path, "namespaces/") + + mgItems := strings.Split(pathItems[1], "/") + mgLog.Namespace = mgItems[0] + mgLog.Pod = mgItems[2] + mgLog.Container = mgItems[3] + // TODO: log errors + mgLog.ErrorCounters = archive.NewErrorCounter(pointer.String(mgLog.buffer.String()), ci.CommonErrorPatterns) + // additional parsers + if mgLog.Namespace == "openshift-etcd" && + mgLog.Container == "etcd" && + strings.HasSuffix(mgLog.Path, "current.log") { + log.Debugf("Must-gather processor - Processing pods logs: %s/%s/%s", mgLog.Namespace, mgLog.Pod, mgLog.Container) + // TODO: collect errors + mgLog.ErrorEtcdLogs = NewErrorEtcdLogs(pointer.String(mgLog.buffer.String())) + log.Debugf("Must-gather processor - Done logs processing: %s/%s/%s", mgLog.Namespace, mgLog.Pod, mgLog.Container) + } + + // Insert only if there are logs parsed + if mgLog.Processed() { + if err := mg.InsertNamespaceErrors(mgLog); err != nil { + log.Errorf("one or more errors found when inserting errors: %v", err) + } + } +} + +/* MustGatehr log items */ + +type MustGatherLog struct { + Path string + PathAlias string + Namespace string + Pod string + Container string + ErrorCounters archive.ErrorCounter `json:"ErrorCounters,omitempty"` + ErrorEtcdLogs *ErrorEtcdLogs `json:"ErrorEtcdLogs,omitempty"` + buffer *bytes.Buffer `json:"-"` +} + +// Processed check if there are items processed, otherwise will save +// storage preventing items without relevant information. +func (mge *MustGatherLog) Processed() bool { + if len(mge.ErrorCounters) > 0 { + return true + } + if mge.ErrorEtcdLogs != nil { + return true + } + return false +} diff --git a/internal/openshift/mustgather/podnetconcheck.go b/internal/openshift/mustgather/podnetconcheck.go new file mode 100644 index 00000000..9afce70b --- /dev/null +++ b/internal/openshift/mustgather/podnetconcheck.go @@ -0,0 +1,135 @@ +package mustgather + +import log "github.com/sirupsen/logrus" + +/* MustGather PodNetworkChecks handle connectivity monitor */ + +type MustGatherPodNetworkCheck struct { + Name string + SpecSource string + SpecTarget string + TotalFailures int64 + TotalOutages int64 + TotalSuccess int64 +} + +type MustGatherPodNetworkChecks struct { + TotalFailures int64 + TotalOutages int64 + TotalSuccess int64 + Checks []*MustGatherPodNetworkCheck + Outages []*NetworkOutage + Failures []*NetworkCheckFailure +} + +func (p *MustGatherPodNetworkChecks) InsertCheck( + check *MustGatherPodNetworkCheck, + failures []*NetworkCheckFailure, + outages []*NetworkOutage, +) { + p.Checks = append(p.Checks, check) + p.Outages = append(p.Outages, outages...) + p.Failures = append(p.Failures, failures...) + p.TotalFailures += check.TotalFailures + p.TotalOutages += check.TotalOutages + p.TotalSuccess += check.TotalSuccess +} + +func (p *MustGatherPodNetworkChecks) Parse(data map[string]interface{}) { + + // TODO#1 use CRD PodNetworkConnectivityCheck and api controlplane.operator.openshift.io/v1alpha1 to parse + // TODO#2 use reflection to read data + for _, d := range data["items"].([]interface{}) { + item := d.(map[interface{}]interface{}) + + if item["metadata"] == nil { + log.Errorf("unable to retrieve pod network check metadata: %v", item["metadata"]) + continue + } + metadata := item["metadata"].(map[interface{}]interface{}) + + if item["spec"] == nil { + log.Errorf("unable to retrieve pod network check spec: %v", item["spec"]) + continue + } + spec := item["spec"].(map[interface{}]interface{}) + + if item["status"] == nil { + log.Errorf("unable to retrieve pod network check status: %v", item["status"]) + continue + } + status := item["status"].(map[interface{}]interface{}) + + name := metadata["name"].(string) + check := &MustGatherPodNetworkCheck{ + Name: name, + SpecSource: spec["sourcePod"].(string), + SpecTarget: spec["targetEndpoint"].(string), + } + if status["successes"] != nil { + check.TotalSuccess = int64(len(status["successes"].([]interface{}))) + } + + netFailures := []*NetworkCheckFailure{} + if status["failures"] != nil { + failures := status["failures"].([]interface{}) + check.TotalFailures = int64(len(failures)) + for _, f := range failures { + if f.(map[interface{}]interface{})["time"] == nil { + continue + } + nf := &NetworkCheckFailure{ + Name: name, + Time: f.(map[interface{}]interface{})["time"].(string), + } + if f.(map[interface{}]interface{})["latency"] != nil { + nf.Latency = f.(map[interface{}]interface{})["latency"].(string) + } + if f.(map[interface{}]interface{})["reason"] != nil { + nf.Reason = f.(map[interface{}]interface{})["reason"].(string) + } + if f.(map[interface{}]interface{})["message"] != nil { + nf.Message = f.(map[interface{}]interface{})["message"].(string) + } + netFailures = append(netFailures, nf) + } + } + + netOutages := []*NetworkOutage{} + if status["outages"] != nil { + outages := status["outages"].([]interface{}) + check.TotalOutages = int64(len(outages)) + for _, o := range outages { + no := &NetworkOutage{Name: name} + if o.(map[interface{}]interface{})["start"] == nil { + continue + } + no.Start = o.(map[interface{}]interface{})["start"].(string) + if o.(map[interface{}]interface{})["end"] != nil { + no.End = o.(map[interface{}]interface{})["end"].(string) + } + if o.(map[interface{}]interface{})["message"] != nil { + no.Message = o.(map[interface{}]interface{})["message"].(string) + } + netOutages = append(netOutages, no) + } + } + p.InsertCheck(check, netFailures, netOutages) + } + +} + +type NetworkOutage struct { + Start string + End string + Name string + Message string +} + +type NetworkCheckFailure struct { + Time string + Reason string + Latency string + Name string + Message string +} diff --git a/internal/pkg/summary/consolidated.go b/internal/pkg/summary/consolidated.go deleted file mode 100644 index 454ee4e2..00000000 --- a/internal/pkg/summary/consolidated.go +++ /dev/null @@ -1,557 +0,0 @@ -package summary - -import ( - "bufio" - "fmt" - "os" - "sort" - - log "github.com/sirupsen/logrus" - - "github.com/pkg/errors" - - "github.com/redhat-openshift-ecosystem/provider-certification-tool/internal/pkg/sippy" - "github.com/xuri/excelize/v2" -) - -// ConsolidatedSummary Aggregate the results of provider and baseline -type ConsolidatedSummary struct { - Provider *ResultSummary - Baseline *ResultSummary -} - -// Process entrypoint to read and fill all summaries for each archive, plugin and suites -// applying any transformation it needs through filters. -func (cs *ConsolidatedSummary) Process() error { - - // Load Result Summary from Archives - if err := cs.Provider.Populate(); err != nil { - fmt.Println("ERROR processing provider results...") - return err - } - - if err := cs.Baseline.Populate(); err != nil { - fmt.Println("ERROR processing baseline results...") - return err - } - - // Filters - if err := cs.applyFilterSuite(); err != nil { - return err - } - - if err := cs.applyFilterBaseline(); err != nil { - return err - } - - if err := cs.applyFilterFlaky(); err != nil { - return err - } - - return nil -} - -func (cs *ConsolidatedSummary) GetProvider() *ResultSummary { - return cs.Provider -} - -func (cs *ConsolidatedSummary) GetBaseline() *ResultSummary { - return cs.Baseline -} - -// applyFilterSuite process the FailedList for each plugin, getting **intersection** tests -// for respective suite. -func (cs *ConsolidatedSummary) applyFilterSuite() error { - err := cs.applyFilterSuiteForPlugin(PluginNameKubernetesConformance) - if err != nil { - return err - } - - err = cs.applyFilterSuiteForPlugin(PluginNameOpenShiftConformance) - if err != nil { - return err - } - - return nil -} - -// applyFilterSuiteForPlugin calculates the intersection of Provider Failed AND suite -func (cs *ConsolidatedSummary) applyFilterSuiteForPlugin(plugin string) error { - - var resultsProvider *OPCTPluginSummary - var pluginSuite *OpenshiftTestsSuite - - switch plugin { - case PluginNameKubernetesConformance: - resultsProvider = cs.GetProvider().GetOpenShift().GetResultK8SValidated() - pluginSuite = cs.GetProvider().GetSuites().KubernetesConformance - case PluginNameOpenShiftConformance: - resultsProvider = cs.GetProvider().GetOpenShift().GetResultOCPValidated() - pluginSuite = cs.GetProvider().GetSuites().OpenshiftConformance - } - - e2eFailures := resultsProvider.FailedList - e2eSuite := pluginSuite.Tests - hashSuite := make(map[string]struct{}, len(e2eSuite)) - - for _, v := range e2eSuite { - hashSuite[v] = struct{}{} - } - - for _, v := range e2eFailures { - if _, ok := hashSuite[v]; ok { - resultsProvider.FailedFilterSuite = append(resultsProvider.FailedFilterSuite, v) - } - } - sort.Strings(resultsProvider.FailedFilterSuite) - return nil -} - -// applyFilterBaseline process the FailedFilterSuite for each plugin, **excluding** failures from -// baseline test. -func (cs *ConsolidatedSummary) applyFilterBaseline() error { - err := cs.applyFilterBaselineForPlugin(PluginNameKubernetesConformance) - if err != nil { - return err - } - - err = cs.applyFilterBaselineForPlugin(PluginNameOpenShiftConformance) - if err != nil { - return err - } - - return nil -} - -// applyFilterBaselineForPlugin calculates the **exclusion** tests of -// Provider Failed included on suite and Baseline failed tests. -func (cs *ConsolidatedSummary) applyFilterBaselineForPlugin(plugin string) error { - - var providerSummary *OPCTPluginSummary - var e2eFailuresBaseline []string - - switch plugin { - case PluginNameKubernetesConformance: - providerSummary = cs.GetProvider().GetOpenShift().GetResultK8SValidated() - if cs.GetBaseline().HasValidResults() { - e2eFailuresBaseline = cs.GetBaseline().GetOpenShift().GetResultK8SValidated().FailedList - } - case PluginNameOpenShiftConformance: - providerSummary = cs.GetProvider().GetOpenShift().GetResultOCPValidated() - if cs.GetBaseline().HasValidResults() { - e2eFailuresBaseline = cs.GetBaseline().GetOpenShift().GetResultOCPValidated().FailedList - } - default: - return errors.New("Suite not found to apply filter: Flaky") - } - - e2eFailuresProvider := providerSummary.FailedFilterSuite - hashBaseline := make(map[string]struct{}, len(e2eFailuresBaseline)) - - for _, v := range e2eFailuresBaseline { - hashBaseline[v] = struct{}{} - } - - for _, v := range e2eFailuresProvider { - if _, ok := hashBaseline[v]; !ok { - providerSummary.FailedFilterBaseline = append(providerSummary.FailedFilterBaseline, v) - } - } - sort.Strings(providerSummary.FailedFilterBaseline) - return nil -} - -// applyFilterFlaky process the FailedFilterSuite for each plugin, **excluding** failures from -// baseline test. -func (cs *ConsolidatedSummary) applyFilterFlaky() error { - err := cs.applyFilterFlakyForPlugin(PluginNameKubernetesConformance) - if err != nil { - return err - } - - err = cs.applyFilterFlakyForPlugin(PluginNameOpenShiftConformance) - if err != nil { - return err - } - - return nil -} - -// applyFilterFlakyForPlugin query the Sippy API looking for each failed test -// on each plugin/suite, saving the list on the ResultSummary. -func (cs *ConsolidatedSummary) applyFilterFlakyForPlugin(plugin string) error { - - var ps *OPCTPluginSummary - - switch plugin { - case PluginNameKubernetesConformance: - ps = cs.GetProvider().GetOpenShift().GetResultK8SValidated() - case PluginNameOpenShiftConformance: - ps = cs.GetProvider().GetOpenShift().GetResultOCPValidated() - default: - return errors.New("Suite not found to apply filter: Flaky") - } - - // TODO: define if we will check for flakes for all failures or only filtered - // Query Flaky only the FilteredBaseline to avoid many external queries. - api := sippy.NewSippyAPI() - for _, name := range ps.FailedFilterBaseline { - - resp, err := api.QueryTests(&sippy.SippyTestsRequestInput{TestName: name}) - if err != nil { - log.Errorf("#> Error querying to Sippy API: %v", err) - continue - } - for _, r := range *resp { - if _, ok := ps.FailedItems[name]; ok { - ps.FailedItems[name].Flaky = &r - } else { - ps.FailedItems[name] = &PluginFailedItem{ - Name: name, - Flaky: &r, - } - } - - // Remove all flakes, regardless the percentage. - // TODO: Review checking flaky severity - if ps.FailedItems[name].Flaky.CurrentFlakes == 0 { - ps.FailedFilterFlaky = append(ps.FailedFilterFlaky, name) - } - } - } - - sort.Strings(ps.FailedFilterFlaky) - return nil -} - -func (cs *ConsolidatedSummary) saveResultsPlugin(path, plugin string) error { - - var resultsProvider *OPCTPluginSummary - var resultsBaseline *OPCTPluginSummary - var suite *OpenshiftTestsSuite - var prefix = "tests" - bProcessed := cs.GetBaseline().HasValidResults() - - switch plugin { - case PluginNameKubernetesConformance: - resultsProvider = cs.GetProvider().GetOpenShift().GetResultK8SValidated() - if bProcessed { - resultsBaseline = cs.GetBaseline().GetOpenShift().GetResultK8SValidated() - } - suite = cs.GetProvider().GetSuites().KubernetesConformance - case PluginNameOpenShiftConformance: - resultsProvider = cs.GetProvider().GetOpenShift().GetResultOCPValidated() - if bProcessed { - resultsBaseline = cs.GetBaseline().GetOpenShift().GetResultOCPValidated() - } - suite = cs.GetProvider().GetSuites().OpenshiftConformance - } - - // Save Provider failures - filename := fmt.Sprintf("%s/%s_%s_provider_failures-1-ini.txt", path, prefix, plugin) - if err := writeFileTestList(filename, resultsProvider.FailedList); err != nil { - return err - } - - // Save Provider failures with filter: Suite (only) - filename = fmt.Sprintf("%s/%s_%s_provider_failures-2-filter1_suite.txt", path, prefix, plugin) - if err := writeFileTestList(filename, resultsProvider.FailedFilterSuite); err != nil { - return err - } - - // Save Provider failures with filter: Baseline exclusion - filename = fmt.Sprintf("%s/%s_%s_provider_failures-3-filter2_baseline.txt", path, prefix, plugin) - if err := writeFileTestList(filename, resultsProvider.FailedFilterBaseline); err != nil { - return err - } - - // Save Provider failures with filter: Flaky - filename = fmt.Sprintf("%s/%s_%s_provider_failures-4-filter3_without_flakes.txt", path, prefix, plugin) - if err := writeFileTestList(filename, resultsProvider.FailedFilterFlaky); err != nil { - return err - } - - // Save the Providers failures for the latest filter to review (focus on this) - filename = fmt.Sprintf("%s/%s_%s_provider_failures.txt", path, prefix, plugin) - if err := writeFileTestList(filename, resultsProvider.FailedFilterBaseline); err != nil { - return err - } - - // Save baseline failures - if bProcessed { - filename = fmt.Sprintf("%s/%s_%s_baseline_failures.txt", path, prefix, plugin) - if err := writeFileTestList(filename, resultsBaseline.FailedList); err != nil { - return err - } - } - - // Save the openshift-tests suite use by this plugin: - filename = fmt.Sprintf("%s/%s_%s_suite_full.txt", path, prefix, plugin) - if err := writeFileTestList(filename, suite.Tests); err != nil { - return err - } - - return nil -} - -func (cs *ConsolidatedSummary) extractFailuresDetailsByPlugin(path, plugin string) error { - - var resultsProvider *OPCTPluginSummary - var resultsBaseline *OPCTPluginSummary - bProcessed := cs.GetBaseline().HasValidResults() - ignoreExistingDir := true - - switch plugin { - case PluginNameKubernetesConformance: - resultsProvider = cs.GetProvider().GetOpenShift().GetResultK8SValidated() - if bProcessed { - resultsBaseline = cs.GetBaseline().GetOpenShift().GetResultK8SValidated() - } - case PluginNameOpenShiftConformance: - resultsProvider = cs.GetProvider().GetOpenShift().GetResultOCPValidated() - if bProcessed { - resultsBaseline = cs.GetBaseline().GetOpenShift().GetResultOCPValidated() - } - } - - currentDirectory := "failures-provider-filtered" - subdir := fmt.Sprintf("%s/%s", path, currentDirectory) - if err := createDir(subdir, ignoreExistingDir); err != nil { - return err - } - - subPrefix := fmt.Sprintf("%s/%s", subdir, plugin) - errItems := resultsProvider.FailedItems - errList := resultsProvider.FailedFilterBaseline - if err := extractTestErrors(subPrefix, errItems, errList); err != nil { - return err - } - - currentDirectory = "failures-provider" - subdir = fmt.Sprintf("%s/%s", path, currentDirectory) - if err := createDir(subdir, ignoreExistingDir); err != nil { - return err - } - - subPrefix = fmt.Sprintf("%s/%s", subdir, plugin) - errItems = resultsProvider.FailedItems - errList = resultsProvider.FailedList - if err := extractTestErrors(subPrefix, errItems, errList); err != nil { - return err - } - - currentDirectory = "failures-baseline" - subdir = fmt.Sprintf("%s/%s", path, currentDirectory) - if err := createDir(subdir, ignoreExistingDir); err != nil { - return err - } - - if bProcessed { - subPrefix = fmt.Sprintf("%s/%s", subdir, plugin) - errItems = resultsBaseline.FailedItems - errList = resultsBaseline.FailedList - if err := extractTestErrors(subPrefix, errItems, errList); err != nil { - return err - } - } - - return nil -} - -func (cs *ConsolidatedSummary) saveFailuresIndexToSheet(path string) error { - - var rowN int64 - var errList []string - bProcessed := cs.GetBaseline().HasValidResults() - sheet := excelize.NewFile() - sheetFile := fmt.Sprintf("%s/failures-index.xlsx", path) - defer saveSheet(sheet, sheetFile) - - sheetName := "failures-provider-filtered" - sheet.SetActiveSheet(sheet.NewSheet(sheetName)) - if err := createSheet(sheet, sheetName); err != nil { - log.Error(err) - } else { - errList = cs.GetProvider().GetOpenShift().GetResultK8SValidated().FailedFilterBaseline - rowN = 2 - populateSheet(sheet, sheetName, PluginNameKubernetesConformance, errList, &rowN) - - errList = cs.GetProvider().GetOpenShift().GetResultOCPValidated().FailedFilterBaseline - populateSheet(sheet, sheetName, PluginNameOpenShiftConformance, errList, &rowN) - } - - sheetName = "failures-provider" - sheet.SetActiveSheet(sheet.NewSheet(sheetName)) - if err := createSheet(sheet, sheetName); err != nil { - log.Error(err) - } else { - errList = cs.GetProvider().GetOpenShift().GetResultK8SValidated().FailedList - rowN = 2 - populateSheet(sheet, sheetName, PluginNameKubernetesConformance, errList, &rowN) - - errList = cs.GetProvider().GetOpenShift().GetResultOCPValidated().FailedList - populateSheet(sheet, sheetName, PluginNameOpenShiftConformance, errList, &rowN) - } - - if bProcessed { - sheetName = "failures-baseline" - sheet.SetActiveSheet(sheet.NewSheet(sheetName)) - if err := createSheet(sheet, sheetName); err != nil { - log.Error(err) - } else { - errList = cs.GetBaseline().GetOpenShift().GetResultK8SValidated().FailedList - rowN = 2 - populateSheet(sheet, sheetName, PluginNameKubernetesConformance, errList, &rowN) - - errList = cs.GetBaseline().GetOpenShift().GetResultOCPValidated().FailedList - populateSheet(sheet, sheetName, PluginNameOpenShiftConformance, errList, &rowN) - } - } - - return nil -} - -// SaveResults dump all the results and processed to the disk to be used -// on the review process. -func (cs *ConsolidatedSummary) SaveResults(path string) error { - - if err := createDir(path, false); err != nil { - return err - } - - // Save the list of failures into individual files by Plugin - if err := cs.saveResultsPlugin(path, PluginNameKubernetesConformance); err != nil { - return err - } - if err := cs.saveResultsPlugin(path, PluginNameOpenShiftConformance); err != nil { - return err - } - - // Extract errors details to sub directories - if err := cs.extractFailuresDetailsByPlugin(path, PluginNameKubernetesConformance); err != nil { - return err - } - if err := cs.extractFailuresDetailsByPlugin(path, PluginNameOpenShiftConformance); err != nil { - return err - } - - // Save one Sheet file with Failures to be used on the review process - if err := cs.saveFailuresIndexToSheet(path); err != nil { - return err - } - - fmt.Printf("\n Data Saved to directory '%s/'\n", path) - return nil -} - -// writeFileTestList saves the list of test names to a new text file -func writeFileTestList(filename string, data []string) error { - fd, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - log.Fatalf("failed creating file: %s", err) - } - defer fd.Close() - - writer := bufio.NewWriter(fd) - defer writer.Flush() - - for _, line := range data { - _, err = writer.WriteString(line + "\n") - if err != nil { - return err - } - } - - return nil -} - -// extractTestErrors dumps the test error, summary and stdout, to be saved -// to individual files. -func extractTestErrors(prefix string, items map[string]*PluginFailedItem, failures []string) error { - for idx, line := range failures { - if _, ok := items[line]; ok { - file := fmt.Sprintf("%s_%d-failure.txt", prefix, idx+1) - err := writeErrorToFile(file, items[line].Failure) - if err != nil { - log.Errorf("Error writing Failure for test: %s\n", line) - } - - file = fmt.Sprintf("%s_%d-systemOut.txt", prefix, idx+1) - err = writeErrorToFile(file, items[line].SystemOut) - if err != nil { - log.Errorf("Error writing SystemOut for test: %s\n", line) - } - } - } - return nil -} - -// writeErrorToFile save the entire buffer to individual file. -func writeErrorToFile(file, data string) error { - fd, err := os.OpenFile(file, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - log.Fatalf("failed creating file: %s", err) - } - defer fd.Close() - - writer := bufio.NewWriter(fd) - defer writer.Flush() - - _, err = writer.WriteString(data) - if err != nil { - return err - } - - return nil -} - -// createDir checks if the directory exists, if not creates it, otherwise log and return error -func createDir(path string, ignoreexisting bool) error { - if _, err := os.Stat(path); !os.IsNotExist(err) { - if ignoreexisting { - return nil - } - log.Errorf("ERROR: Directory already exists [%s]: %v", path, err) - return err - } - - if err := os.Mkdir(path, os.ModePerm); err != nil { - log.Errorf("ERROR: Unable to create directory [%s]: %v", path, err) - return err - } - return nil -} - -// createSheet creates the excel spreadsheet headers -func createSheet(sheet *excelize.File, sheeName string) error { - header := map[string]string{ - "A1": "Plugin", "B1": "Index", "C1": "Error_Directory", - "D1": "Test_Name", "E1": "Notes_Review", "F1": "References"} - - // create header - for k, v := range header { - _ = sheet.SetCellValue(sheeName, k, v) - } - - return nil -} - -// populateGsheet fill each row per error item. -func populateSheet(sheet *excelize.File, sheeName, suite string, list []string, rowN *int64) { - for idx, v := range list { - _ = sheet.SetCellValue(sheeName, fmt.Sprintf("A%d", *rowN), suite) - _ = sheet.SetCellValue(sheeName, fmt.Sprintf("B%d", *rowN), idx+1) - _ = sheet.SetCellValue(sheeName, fmt.Sprintf("C%d", *rowN), sheeName) - _ = sheet.SetCellValue(sheeName, fmt.Sprintf("D%d", *rowN), v) - _ = sheet.SetCellValue(sheeName, fmt.Sprintf("E%d", *rowN), "TODO Review") - _ = sheet.SetCellValue(sheeName, fmt.Sprintf("F%d", *rowN), "") - *(rowN) += 1 - } -} - -// save the excel sheet to the disk. -func saveSheet(sheet *excelize.File, sheetFileName string) { - if err := sheet.SaveAs(sheetFileName); err != nil { - log.Error(err) - } -} diff --git a/internal/pkg/summary/opct.go b/internal/pkg/summary/opct.go deleted file mode 100644 index edc45016..00000000 --- a/internal/pkg/summary/opct.go +++ /dev/null @@ -1,52 +0,0 @@ -package summary - -import ( - "github.com/redhat-openshift-ecosystem/provider-certification-tool/internal/pkg/sippy" -) - -const ( - PluginNameOpenShiftUpgrade = "05-openshift-cluster-upgrade" - PluginNameKubernetesConformance = "10-openshift-kube-conformance" - PluginNameOpenShiftConformance = "20-openshift-conformance-validated" - PluginNameArtifactsCollector = "99-openshift-artifacts-collector" - - // Old Plugin names (prior v0.2). It's used to keep compatibility - PluginOldNameKubernetesConformance = "openshift-kube-conformance" - PluginOldNameOpenShiftConformance = "openshift-conformance-validated" -) - -// OPCT -type OPCTPluginSummary struct { - Name string - NameAlias string - Status string - Total int64 - Passed int64 - Failed int64 - Timeout int64 - Skipped int64 - - // FailedItems is the map with details for each failure - FailedItems map[string]*PluginFailedItem - // FailedList is the list of tests failures on the original execution - FailedList []string - // FailedFilterSuite is the list of failures (A) included only in the original suite (B): A INTERSECTION B - FailedFilterSuite []string - // FailedFilterBaseline is the list of failures (A) excluding the baseline(B): A EXCLUDE B - FailedFilterBaseline []string - // FailedFilteFlaky is the list of failures with no Flakes on OpenShift CI - FailedFilterFlaky []string -} - -type PluginFailedItem struct { - // Name is the name of the e2e test - Name string - // Failure contains the failure reason extracted from JUnit field 'item.detials.failure' - Failure string - // SystemOut contains the entire test stdout extracted from JUnit field 'item.detials.system-out' - SystemOut string - // Offset is the offset of failure from the plugin result file - Offset int - // Flaky contains the flaky information from OpenShift CI - scraped from Sippy API - Flaky *sippy.SippyTestsResponse -} diff --git a/internal/pkg/summary/openshift.go b/internal/pkg/summary/openshift.go deleted file mode 100644 index 29bdf071..00000000 --- a/internal/pkg/summary/openshift.go +++ /dev/null @@ -1,138 +0,0 @@ -package summary - -import ( - "fmt" - - configv1 "github.com/openshift/api/config/v1" - "github.com/pkg/errors" -) - -type OpenShiftSummary struct { - Infrastructure *configv1.Infrastructure - ClusterVersion *configv1.ClusterVersion - ClusterOperators *configv1.ClusterOperatorList - - // Plugin Results - PluginResultK8sConformance *OPCTPluginSummary - PluginResultOCPValidated *OPCTPluginSummary - - // get from Sonobuoy metadata - VersionK8S string -} - -type SummaryClusterVersionOutput struct { - DesiredVersion string - Progressing string - ProgressingMessage string -} - -type SummaryClusterOperatorOutput struct { - CountAvailable uint64 - CountProgressing uint64 - CountDegraded uint64 -} - -type SummaryOpenShiftInfrastructureV1 = configv1.Infrastructure - -func NewOpenShiftSummary() *OpenShiftSummary { - return &OpenShiftSummary{} -} - -func (os *OpenShiftSummary) SetInfrastructure(cr *configv1.InfrastructureList) error { - if len(cr.Items) == 0 { - return errors.New("Unable to find result Items to set Infrastructures") - } - os.Infrastructure = &cr.Items[0] - return nil -} - -func (os *OpenShiftSummary) GetInfrastructure() (*SummaryOpenShiftInfrastructureV1, error) { - if os.Infrastructure == nil { - return &SummaryOpenShiftInfrastructureV1{}, nil - } - return os.Infrastructure, nil -} - -func (os *OpenShiftSummary) SetClusterVersion(cr *configv1.ClusterVersionList) error { - if len(cr.Items) == 0 { - return errors.New("Unable to find result Items to set Infrastructures") - } - os.ClusterVersion = &cr.Items[0] - return nil -} - -func (os *OpenShiftSummary) GetClusterVersion() (*SummaryClusterVersionOutput, error) { - if os.ClusterVersion == nil { - return &SummaryClusterVersionOutput{}, nil - } - resp := SummaryClusterVersionOutput{ - DesiredVersion: os.ClusterVersion.Status.Desired.Version, - } - for _, condition := range os.ClusterVersion.Status.Conditions { - if condition.Type == configv1.OperatorProgressing { - resp.Progressing = string(condition.Status) - resp.ProgressingMessage = condition.Message - } - } - return &resp, nil -} - -func (os *OpenShiftSummary) SetClusterOperators(cr *configv1.ClusterOperatorList) error { - if len(cr.Items) == 0 { - return errors.New("Unable to find result Items to set ClusterOperators") - } - os.ClusterOperators = cr - return nil -} - -func (os *OpenShiftSummary) GetClusterOperator() (*SummaryClusterOperatorOutput, error) { - out := SummaryClusterOperatorOutput{} - for _, co := range os.ClusterOperators.Items { - for _, condition := range co.Status.Conditions { - switch condition.Type { - case configv1.OperatorAvailable: - if condition.Status == configv1.ConditionTrue { - out.CountAvailable += 1 - } - case configv1.OperatorProgressing: - if condition.Status == configv1.ConditionTrue { - out.CountProgressing += 1 - } - case configv1.OperatorDegraded: - if condition.Status == configv1.ConditionTrue { - out.CountDegraded += 1 - } - } - } - } - return &out, nil -} - -func (os *OpenShiftSummary) SetPluginResult(in *OPCTPluginSummary) error { - switch in.Name { - case PluginNameKubernetesConformance: - os.PluginResultK8sConformance = in - case PluginOldNameKubernetesConformance: - in.NameAlias = in.Name - in.Name = PluginNameKubernetesConformance - os.PluginResultK8sConformance = in - - case PluginNameOpenShiftConformance: - os.PluginResultOCPValidated = in - case PluginOldNameOpenShiftConformance: - in.NameAlias = in.Name - in.Name = PluginOldNameOpenShiftConformance - os.PluginResultOCPValidated = in - default: - return fmt.Errorf("unable to Set Plugin results: Plugin not found: %s", in.Name) - } - return nil -} - -func (os *OpenShiftSummary) GetResultOCPValidated() *OPCTPluginSummary { - return os.PluginResultOCPValidated -} - -func (os *OpenShiftSummary) GetResultK8SValidated() *OPCTPluginSummary { - return os.PluginResultK8sConformance -} diff --git a/internal/pkg/summary/sonobuoy.go b/internal/pkg/summary/sonobuoy.go deleted file mode 100644 index 669c61d9..00000000 --- a/internal/pkg/summary/sonobuoy.go +++ /dev/null @@ -1,14 +0,0 @@ -package summary - -import ( - "github.com/vmware-tanzu/sonobuoy/pkg/discovery" -) - -type SonobuoySummary struct { - Cluster *discovery.ClusterSummary -} - -func (s *SonobuoySummary) SetCluster(c *discovery.ClusterSummary) error { - s.Cluster = c - return nil -} diff --git a/main.go b/main.go index 8ce65897..d092ac2a 100644 --- a/main.go +++ b/main.go @@ -3,7 +3,7 @@ package main import ( "embed" - cmd "github.com/redhat-openshift-ecosystem/provider-certification-tool/cmd" + cmd "github.com/redhat-openshift-ecosystem/provider-certification-tool/cmd/opct" "github.com/redhat-openshift-ecosystem/provider-certification-tool/internal/assets" ) diff --git a/mkdocs.yml b/mkdocs.yml index ce721ff0..9eb8a9e9 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -13,6 +13,8 @@ markdown_extensions: - name: mermaid class: mermaid format: !!python/name:pymdownx.superfences.fence_code_format + # - pymdownx.tabbed: + # alternate_style: true plugins: - diagrams: @@ -28,23 +30,27 @@ theme: features: #- toc.integrate - navigation.top + - navigation.tabs + - navigation.top nav: - - User Guides: - - "User Guide": "user.md" - - "Installation Checklist": user-installation-checklist.md - - "Installation Review": user-installation-review.md - - "Disconnected Installations": user-installation-disconnected.md - - "Troubleshooting": troubleshooting-guide.md - - Support Guides: - - Support Guide: support-guide.md + - Home: + - README.md - Review: - - review/index.md + - Home: review/index.md - OPCT Rules: review/rules.md - - Developer Guides: + - "Installation Review": user-installation-review.md + - "Troubleshooting": troubleshooting-guide.md + - "Installation Checklist": user-installation-checklist.md + - Review Guide: support-guide.md + - Validation Guides: + - "User Guide": "user.md" + - "Disconnected Installations": user-installation-disconnected.md + - Contribute: TODO.md + - Developement: - Development Guide: dev.md - Diagrams: - diagrams/index.md - diagrams/opct-sequence.md - "Reference Architecture": diagrams/ocp-architecture-reference.md - - CHANGELOG: CHANGELOG.md \ No newline at end of file + - CHANGELOG: CHANGELOG.md diff --git a/pkg/cmd/report/report.go b/pkg/cmd/report/report.go new file mode 100644 index 00000000..494f4098 --- /dev/null +++ b/pkg/cmd/report/report.go @@ -0,0 +1,593 @@ +package report + +import ( + "fmt" + "net/http" + "os" + "strings" + + "github.com/pkg/errors" + "github.com/spf13/cobra" + + "text/tabwriter" + + "github.com/redhat-openshift-ecosystem/provider-certification-tool/internal/opct/metrics" + "github.com/redhat-openshift-ecosystem/provider-certification-tool/internal/opct/plugin" + "github.com/redhat-openshift-ecosystem/provider-certification-tool/internal/opct/report" + "github.com/redhat-openshift-ecosystem/provider-certification-tool/internal/opct/summary" + log "github.com/sirupsen/logrus" + "github.com/vmware-tanzu/sonobuoy/pkg/errlog" +) + +type Input struct { + archive string + archiveBase string + saveTo string + serverAddress string + serverSkip bool + embedData bool + saveOnly bool + verbose bool + json bool +} + +func NewCmdReport() *cobra.Command { + data := Input{} + cmd := &cobra.Command{ + Use: "report archive.tar.gz", + Short: "Create a report from results.", + Run: func(cmd *cobra.Command, args []string) { + data.archive = args[0] + checkFlags(&data) + if err := processResult(&data); err != nil { + errlog.LogError(errors.Wrapf(err, "could not process archive: %v", args[0])) + os.Exit(1) + } + }, + Args: cobra.ExactArgs(1), + } + + cmd.Flags().StringVarP( + &data.archiveBase, "baseline", "b", "", + "[DEPRECATED] Baseline result archive file. Example: -b file.tar.gz", + ) + cmd.Flags().StringVarP( + &data.archiveBase, "diff", "d", "", + "Diff results from a baseline archive file. Example: --diff file.tar.gz", + ) + cmd.Flags().StringVarP( + &data.saveTo, "save-to", "s", "", + "Extract and Save Results to disk. Example: -s ./results", + ) + cmd.Flags().StringVarP( + &data.serverAddress, "server-address", "", "0.0.0.0:9090", + "HTTP server address to serve files when --save-to is used. Example: --server-address 0.0.0.0:9090", + ) + cmd.Flags().BoolVarP( + &data.serverSkip, "server-skip", "", false, + "HTTP server address to serve files when --save-to is used. Example: --server-address 0.0.0.0:9090", + ) + cmd.Flags().BoolVarP( + &data.embedData, "embed-data", "", false, + "Force to embed the data into HTML report, allwoing the use of file protocol/CORS in the browser.", + ) + cmd.Flags().BoolVarP( + &data.saveOnly, "save-only", "", false, + "Save data and exit. Requires --save-to. Example: -s ./results --save-only", + ) + cmd.Flags().BoolVarP( + &data.verbose, "verbose", "v", false, + "Show test details of test failures", + ) + cmd.Flags().BoolVarP( + &data.json, "json", "", false, + "Show report in json format", + ) + + return cmd +} + +// checkFlags +func checkFlags(input *Input) { + if input.embedData { + log.Warnf("--embed-data is set to true, forcing --server-skip to true.") + input.serverSkip = true + } +} + +// processResult reads the artifacts and show it as an report format. +func processResult(input *Input) error { + + log.Println("Creating report...") + timers := metrics.NewTimers() + timers.Add("report-total") + + report := &report.Report{ + Setup: &report.ReportSetup{ + Frontend: &report.ReportSetupFrontend{ + EmbedData: input.embedData, + }, + }, + } + cs := summary.ConsolidatedSummary{ + Verbose: input.verbose, + Timers: timers, + Provider: &summary.ResultSummary{ + Name: summary.ResultSourceNameProvider, + Archive: input.archive, + OpenShift: &summary.OpenShiftSummary{}, + Sonobuoy: summary.NewSonobuoySummary(), + Suites: &summary.OpenshiftTestsSuites{ + OpenshiftConformance: &summary.OpenshiftTestsSuite{Name: "openshiftConformance"}, + KubernetesConformance: &summary.OpenshiftTestsSuite{Name: "kubernetesConformance"}, + }, + SavePath: input.saveTo, + }, + Baseline: &summary.ResultSummary{ + Name: summary.ResultSourceNameBaseline, + Archive: input.archiveBase, + OpenShift: &summary.OpenShiftSummary{}, + Sonobuoy: summary.NewSonobuoySummary(), + Suites: &summary.OpenshiftTestsSuites{ + OpenshiftConformance: &summary.OpenshiftTestsSuite{Name: "openshiftConformance"}, + KubernetesConformance: &summary.OpenshiftTestsSuite{Name: "kubernetesConformance"}, + }, + }, + } + + log.Debug("Processing results") + if err := cs.Process(); err != nil { + return err + } + + log.Debug("Processing report") + if err := report.Populate(&cs); err != nil { + return err + } + + if input.json { + timers.Add("report-total") + resReport, err := report.ShowJSON() + if err != nil { + return err + } + fmt.Println(resReport) + os.Exit(0) + } + + if input.saveTo != "" { + // TODO: ConsolidatedSummary should be migrated to SaveResults + if err := cs.SaveResults(input.saveTo); err != nil { + return err + } + timers.Add("report-total") + if err := report.SaveResults(input.saveTo); err != nil { + return err + } + if input.saveOnly { + os.Exit(0) + } + } + + if err := showReportAggregatedSummary(report); err != nil { + return err + } + + if err := showProcessedSummary(report); err != nil { + return err + } + + if err := showErrorDetails(report, input.verbose); err != nil { + return err + } + + if err := showChecks(report); err != nil { + return err + } + + // run http server to serve static report + if input.saveTo != "" && !input.serverSkip { + fs := http.FileServer(http.Dir(input.saveTo)) + // TODO: redirect home to the opct-reporet.html (or rename to index.html) without + // affecting the fileserver. + http.Handle("/", fs) + + log.Debugf("Listening on %s...", input.serverAddress) + log.Infof("The report server is available in http://%s, open your browser and navigate to results.", input.serverAddress) + log.Infof("To get started open the report http://%s/opct-report.html.", input.serverAddress) + err := http.ListenAndServe(input.serverAddress, nil) + if err != nil { + log.Fatalf("Unable to start the report server at address %s: %v", input.serverAddress, err) + } + } + if input.saveTo != "" && input.serverSkip { + log.Infof("The report server is not enabled (--server-skip=true)., you'll need to navigate it locallly") + log.Infof("To read the report open your browser and navigate to the path file://%s", input.saveTo) + log.Infof("To get started open the report file://%s/opct-report.html.", input.saveTo) + } + + return nil +} + +func showReportAggregatedSummary(re *report.Report) error { + fmt.Printf("\n> OPCT Summary <\n\n") + + baselineProcessed := re.Baseline != nil + + newLineWithTab := "\t\t\n" + tbWriter := tabwriter.NewWriter(os.Stdout, 0, 8, 1, '\t', tabwriter.AlignRight) + + if baselineProcessed { + fmt.Fprintf(tbWriter, " Cluster Version:\n") + fmt.Fprintf(tbWriter, " - Kubernetes\t: %s\t: %s\n", re.Provider.Version.Kubernetes, re.Baseline.Version.Kubernetes) + fmt.Fprintf(tbWriter, " - OpenShift\t: %s\t: %s\n", re.Provider.Version.OpenShift.Desired, re.Baseline.Version.OpenShift.Desired) + // fmt.Fprintf(tbWriter, " - Cluster Update Progressing\t: %s\t: %s\n", re.Provider.Version.OpenShiftUpdProg, re.Baseline.Version.OpenShiftUpdProg) + fmt.Fprintf(tbWriter, " - OpenShift (Previous)\t: %s\t: %s\n", re.Provider.Version.OpenShift.Previous, re.Baseline.Version.OpenShift.Previous) + fmt.Fprintf(tbWriter, " - Channel\t: %s\t: %s\n", re.Provider.Version.OpenShift.Channel, re.Baseline.Version.OpenShift.Channel) + } else { + fmt.Fprintf(tbWriter, " Cluster Version:\n") + fmt.Fprintf(tbWriter, " - Kubernetes\t: %s\n", re.Provider.Version.Kubernetes) + fmt.Fprintf(tbWriter, " - OpenShift\t: %s\n", re.Provider.Version.OpenShift.Desired) + + // fmt.Fprintf(tbWriter, " - OpenShift Previous\t: %s\n", re.Provider.Version.OpenShift.Previous) + fmt.Fprintf(tbWriter, " - Channel\t: %s\n", re.Provider.Version.OpenShift.Channel) + fmt.Fprintf(tbWriter, " Cluster Status\t: %s\n", re.Provider.Version.OpenShift.OverallStatus) + if re.Provider.Version.OpenShift.OverallStatus != "Available" { + fmt.Fprintf(tbWriter, " - Reason\t: %s\n", re.Provider.Version.OpenShift.OverallStatusReason) + fmt.Fprintf(tbWriter, " - Message\t: %s\n", re.Provider.Version.OpenShift.OverallStatusMessage) + } + fmt.Fprintf(tbWriter, " Cluster Status/Conditions:\n") + fmt.Fprintf(tbWriter, " - Available\t: %s\n", re.Provider.Version.OpenShift.CondAvailable) + fmt.Fprintf(tbWriter, " - Failing\t: %s\n", re.Provider.Version.OpenShift.CondFailing) + fmt.Fprintf(tbWriter, " - Progressing (Update)\t: %s\n", re.Provider.Version.OpenShift.CondProgressing) + fmt.Fprintf(tbWriter, " - RetrievedUpdates\t: %s\n", re.Provider.Version.OpenShift.CondRetrievedUpdates) + fmt.Fprintf(tbWriter, " - EnabledCapabilities\t: %s\n", re.Provider.Version.OpenShift.CondImplicitlyEnabledCapabilities) + fmt.Fprintf(tbWriter, " - ReleaseAccepted\t: %s\n", re.Provider.Version.OpenShift.CondReleaseAccepted) + } + + fmt.Fprint(tbWriter, newLineWithTab) + joinPlatformType := func(infra *report.ReportInfra) string { + tp := infra.PlatformType + if tp == "External" { + tp = fmt.Sprintf("%s (%s)", tp, infra.PlatformName) + } + return tp + } + if baselineProcessed { + fmt.Fprintf(tbWriter, " Infrastructure:\t\t\n") + fmt.Fprintf(tbWriter, " - PlatformType\t: %s\t: %s\n", joinPlatformType(re.Provider.Infra), joinPlatformType(re.Baseline.Infra)) + fmt.Fprintf(tbWriter, " - Name\t: %s\t: %s\n", re.Provider.Infra.Name, re.Baseline.Infra.Name) + fmt.Fprintf(tbWriter, " - Topology\t: %s\t: %s\n", re.Provider.Infra.Topology, re.Baseline.Infra.Topology) + fmt.Fprintf(tbWriter, " - ControlPlaneTopology\t: %s\t: %s\n", re.Provider.Infra.ControlPlaneTopology, re.Baseline.Infra.ControlPlaneTopology) + fmt.Fprintf(tbWriter, " - API Server URL\t: %s\t: %s\n", re.Provider.Infra.APIServerURL, re.Baseline.Infra.APIServerURL) + fmt.Fprintf(tbWriter, " - API Server URL (internal)\t: %s\t: %s\n", re.Provider.Infra.APIServerInternalURL, re.Baseline.Infra.APIServerInternalURL) + } else { + fmt.Fprintf(tbWriter, " Infrastructure:\t\n") + fmt.Fprintf(tbWriter, " - PlatformType\t: %s\n", joinPlatformType(re.Provider.Infra)) + fmt.Fprintf(tbWriter, " - Name\t: %s\n", re.Provider.Infra.Name) + fmt.Fprintf(tbWriter, " - ClusterID\t: %s\n", re.Provider.Version.OpenShift.ClusterID) + fmt.Fprintf(tbWriter, " - Topology\t: %s\n", re.Provider.Infra.Topology) + fmt.Fprintf(tbWriter, " - ControlPlaneTopology\t: %s\n", re.Provider.Infra.ControlPlaneTopology) + fmt.Fprintf(tbWriter, " - API Server URL\t: %s\n", re.Provider.Infra.APIServerURL) + fmt.Fprintf(tbWriter, " - API Server URL (internal)\t: %s\n", re.Provider.Infra.APIServerInternalURL) + // fmt.Fprintf(tbWriter, " - Install Type\t: %s\n", "TODO (IPI or UPI?)") + fmt.Fprintf(tbWriter, " - NetworkType\t: %s\n", re.Provider.Infra.NetworkType) + // fmt.Fprintf(tbWriter, " - Proxy Configured\t: %s\n", "TODO (HTTP and/or HTTPS)") + } + + fmt.Fprint(tbWriter, newLineWithTab) + fmt.Fprintf(tbWriter, " Plugins summary by name:\t Status [Total/Passed/Failed/Skipped] (timeout)\n") + + pluginName := plugin.PluginNameKubernetesConformance + if _, ok := re.Provider.Plugins[pluginName]; !ok { + errlog.LogError(errors.New(fmt.Sprintf("Unable to load plugin %s", pluginName))) + } + plK8S := re.Provider.Plugins[pluginName] + name := plK8S.Name + stat := plK8S.Stat + pOCPPluginRes := fmt.Sprintf("%s [%d/%d/%d/%d] (%d)", stat.Status, stat.Total, stat.Passed, stat.Failed, stat.Skipped, stat.Timeout) + if baselineProcessed { + plK8S = re.Baseline.Plugins[pluginName] + stat := plK8S.Stat + bOCPPluginRes := fmt.Sprintf("%s [%d/%d/%d/%d] (%d)", stat.Status, stat.Total, stat.Passed, stat.Failed, stat.Skipped, stat.Timeout) + fmt.Fprintf(tbWriter, " - %s\t: %s\t: %s\n", name, pOCPPluginRes, bOCPPluginRes) + } else { + fmt.Fprintf(tbWriter, " - %s\t: %s\n", name, pOCPPluginRes) + } + + pluginName = plugin.PluginNameKubernetesConformance + if _, ok := re.Provider.Plugins[pluginName]; !ok { + errlog.LogError(errors.New(fmt.Sprintf("Unable to load plugin %s", pluginName))) + } + plOCP := re.Provider.Plugins[pluginName] + name = plOCP.Name + stat = plOCP.Stat + pOCPPluginRes = fmt.Sprintf("%s [%d/%d/%d/%d] (%d)", stat.Status, stat.Total, stat.Passed, stat.Failed, stat.Skipped, stat.Timeout) + if baselineProcessed { + plOCP = re.Baseline.Plugins[pluginName] + stat = plOCP.Stat + bOCPPluginRes := fmt.Sprintf("%s [%d/%d/%d/%d] (%d)", stat.Status, stat.Total, stat.Passed, stat.Failed, stat.Skipped, stat.Timeout) + fmt.Fprintf(tbWriter, " - %s\t: %s\t: %s\n", name, pOCPPluginRes, bOCPPluginRes) + } else { + fmt.Fprintf(tbWriter, " - %s\t: %s\n", name, pOCPPluginRes) + } + + fmt.Fprint(tbWriter, newLineWithTab) + fmt.Fprintf(tbWriter, " Health summary:\t [A=True/P=True/D=True]\t\n") + + pOCPCO := re.Provider.ClusterOperators + if baselineProcessed { + bOCPCO := re.Baseline.ClusterOperators + fmt.Fprintf(tbWriter, " - Cluster Operators\t: [%d/%d/%d]\t: [%d/%d/%d]\n", + pOCPCO.CountAvailable, pOCPCO.CountProgressing, pOCPCO.CountDegraded, + bOCPCO.CountAvailable, bOCPCO.CountProgressing, bOCPCO.CountDegraded, + ) + } else { + fmt.Fprintf(tbWriter, " - Cluster Operators\t: [%d/%d/%d]\n", + pOCPCO.CountAvailable, pOCPCO.CountProgressing, pOCPCO.CountDegraded, + ) + } + + // Show Nodes Health info collected by Sonobuoy + pNhMessage := fmt.Sprintf("%d/%d %s", re.Provider.ClusterHealth.NodeHealthy, re.Provider.ClusterHealth.NodeHealthTotal, "") + if re.Provider.ClusterHealth.NodeHealthTotal != 0 { + pNhMessage = fmt.Sprintf("%s (%.2f%%)", pNhMessage, re.Provider.ClusterHealth.NodeHealthPerc) + } + + if baselineProcessed { + bNhMessage := fmt.Sprintf("%d/%d %s", re.Baseline.ClusterHealth.NodeHealthy, re.Baseline.ClusterHealth.NodeHealthTotal, "") + if re.Baseline.ClusterHealth.NodeHealthTotal != 0 { + bNhMessage = fmt.Sprintf("%s (%.2f%%)", bNhMessage, re.Baseline.ClusterHealth.NodeHealthPerc) + } + fmt.Fprintf(tbWriter, " - Node health\t: %s\t: %s\n", pNhMessage, bNhMessage) + } else { + fmt.Fprintf(tbWriter, " - Node health\t: %s\n", pNhMessage) + } + + // Show Pods Health info collected by Sonobuoy + pPodsHealthMsg := "" + bPodsHealthMsg := "" + phTotal := "" + + if re.Provider.ClusterHealth.PodHealthTotal != 0 { + phTotal = fmt.Sprintf(" (%.2f%%)", re.Provider.ClusterHealth.PodHealthPerc) + } + pPodsHealthMsg = fmt.Sprintf("%d/%d %s", re.Provider.ClusterHealth.PodHealthy, re.Provider.ClusterHealth.PodHealthTotal, phTotal) + + if baselineProcessed { + phTotal := "" + if re.Baseline.ClusterHealth.PodHealthTotal != 0 { + phTotal = fmt.Sprintf(" (%.2f%%)", re.Baseline.ClusterHealth.PodHealthPerc) + } + bPodsHealthMsg = fmt.Sprintf("%d/%d %s", re.Baseline.ClusterHealth.PodHealthy, re.Baseline.ClusterHealth.PodHealthTotal, phTotal) + fmt.Fprintf(tbWriter, " - Pods health\t: %s\t: %s\n", pPodsHealthMsg, bPodsHealthMsg) + } else { + fmt.Fprintf(tbWriter, " - Pods health\t: %s\n", pPodsHealthMsg) + } + + fmt.Fprint(tbWriter, newLineWithTab) + + if len(re.Provider.ClusterHealth.PodHealthDetails) > 0 { + fmt.Fprintf(tbWriter, " Failed pods:\n") + + fmt.Fprintf(tbWriter, " %s/%s\t%s\t%s\t%s\t%s\n", "Namespace", "PodName", "Healthy", "Ready", "Reason", "Message") + for _, podDetails := range re.Provider.ClusterHealth.PodHealthDetails { + fmt.Fprintf(tbWriter, " %s/%s\t%t\t%s\t%s\t%s\n", podDetails.Namespace, podDetails.Name, podDetails.Healthy, podDetails.Ready, podDetails.Reason, podDetails.Message) + } + } + + tbWriter.Flush() + return nil +} + +func showProcessedSummary(re *report.Report) error { + fmt.Printf("\n> Processed Summary <\n") + + fmt.Printf("\n Total tests by conformance suites:\n") + checkEmpty := func(counter int) string { + if counter == 0 { + return "(FAIL)" + } + return "" + } + total := re.Provider.Plugins[plugin.PluginNameKubernetesConformance].Suite.Count + fmt.Printf(" - %s: %d %s\n", summary.SuiteNameKubernetesConformance, total, checkEmpty(total)) + total = re.Provider.Plugins[plugin.PluginNameOpenShiftConformance].Suite.Count + fmt.Printf(" - %s: %d %s\n", summary.SuiteNameOpenshiftConformance, total, checkEmpty(total)) + + fmt.Printf("\n Result Summary by conformance plugins:\n") + bProcessed := re.Provider.HasValidBaseline + showSummaryPlugin(re.Provider, plugin.PluginNameKubernetesConformance, bProcessed) + showSummaryPlugin(re.Provider, plugin.PluginNameOpenShiftConformance, bProcessed) + showSummaryPlugin(re.Provider, plugin.PluginNameOpenShiftUpgrade, bProcessed) + showSummaryPlugin(re.Provider, plugin.PluginNameArtifactsCollector, bProcessed) + + return nil +} + +func showSummaryPlugin(re *report.ReportResult, pluginName string, bProcessed bool) { + if re.Plugins[pluginName] == nil { + log.Errorf("unable to get plugin %s", pluginName) + return + } + p := re.Plugins[pluginName] + if p.Stat == nil { + log.Errorf("unable to get stat for plugin %s", pluginName) + return + } + stat := p.Stat + fmt.Printf(" - %s:\n", p.Name) + fmt.Printf(" - Status: %s\n", stat.Status) + fmt.Printf(" - Total: %d\n", stat.Total) + fmt.Printf(" - Passed: %s\n", plugin.UtilsCalcPercStr(stat.Passed, stat.Total)) + fmt.Printf(" - Failed: %s\n", plugin.UtilsCalcPercStr(stat.Failed, stat.Total)) + fmt.Printf(" - Timeout: %s\n", plugin.UtilsCalcPercStr(stat.Timeout, stat.Total)) + fmt.Printf(" - Skipped: %s\n", plugin.UtilsCalcPercStr(stat.Skipped, stat.Total)) + if p.Name == plugin.PluginNameOpenShiftUpgrade || p.Name == plugin.PluginNameArtifactsCollector { + return + } + // fmt.Printf(" - Failed (without filters) : %s\n", calcPercStr(int64(len(p.FailedList)), stat.Total)) + fmt.Printf(" - Failed (Filter SuiteOnly): %s\n", plugin.UtilsCalcPercStr(stat.FilterSuite, stat.Total)) + if bProcessed { + fmt.Printf(" - Failed (Filter Baseline) : %s\n", plugin.UtilsCalcPercStr(stat.FilterBaseline, stat.Total)) + } + fmt.Printf(" - Failed (Priority): %s\n", plugin.UtilsCalcPercStr(stat.FilterFailedPrio, stat.Total)) + + // TODO: review suites provides better signal. + // The final results for non-kubernetes conformance will be hidden (pass|fail) will be hiden for a while for those reasons: + // - OPCT was created to provide feeaback of conformance results, not a passing binary value. The numbers should be interpreted + // - Conformance results could have flakes or runtime failures which need to be investigated by executor + // - Force user/executor to review the results, and not only the summary. + // That behavior is aligned with BU: we expect kubernetes conformance passes in all providers, the reviewer + // must set this as a target in the review process. + if p.Name != plugin.PluginNameKubernetesConformance { + return + } + // checking for runtime failures + runtimeFailed := false + if stat.Total == stat.Failed { + runtimeFailed = true + } + + // rewrite the original status when pass on all filters and not failed on runtime + status := stat.Status + if (stat.FilterFailedPrio == 0) && !runtimeFailed { + status = "passed" + } + + fmt.Printf(" - Status After Filters : %s\n", status) +} + +// showErrorDetails show details of failres for each plugin. +func showErrorDetails(re *report.Report, verbose bool) error { + fmt.Printf("\n Result details by conformance plugins: \n") + + bProcessed := re.Provider.HasValidBaseline + showErrorDetailPlugin(re.Provider.Plugins[plugin.PluginNameKubernetesConformance], verbose, bProcessed) + showErrorDetailPlugin(re.Provider.Plugins[plugin.PluginNameOpenShiftConformance], verbose, bProcessed) + + return nil +} + +// showErrorDetailPlugin Show failed e2e tests by filter, when verbose each filter will be shown. +func showErrorDetailPlugin(p *report.ReportPlugin, verbose bool, bProcessed bool) { + flakeCount := p.Stat.FilterBaseline - p.Stat.FilterFailedPrio + + if verbose { + fmt.Printf("\n\n => %s: (%d failures, %d failures filtered, %d flakes)\n", p.Name, p.Stat.Failed, p.Stat.FilterBaseline, flakeCount) + + fmt.Printf("\n --> [verbose] Failed tests detected on archive (without filters):\n") + if p.Stat.Failed == 0 { + fmt.Println("") + } + for _, test := range p.Tests { + if test.State == "failed" { + fmt.Println(test.Name) + } + } + + fmt.Printf("\n --> [verbose] Failed tests detected on suite (Filter SuiteOnly):\n") + if p.Stat.FilterSuite == 0 { + fmt.Println("") + } + for _, test := range p.Tests { + if test.State == "filterSuiteOnly" { + fmt.Println(test.Name) + } + } + if bProcessed { + fmt.Printf("\n --> [verbose] Failed tests removing baseline (Filter Baseline):\n") + if p.Stat.FilterBaseline == 0 { + fmt.Println("") + } + for _, test := range p.Tests { + if test.State == "filterBaseline" { + fmt.Println(test.Name) + } + } + } + } else { + fmt.Printf("\n\n => %s: (%d failures, %d flakes)\n", p.Name, p.Stat.FilterBaseline, flakeCount) + } + + fmt.Printf("\n --> Failed tests to Review (without flakes) - Immediate action:\n") + noFlakes := make(map[string]struct{}) + if p.Stat.FilterBaseline == flakeCount { + fmt.Println("") + } else { // TODO move to small functions + testTags := plugin.NewTestTagsEmpty(int(p.Stat.FilterFailedPrio)) + var testsWErrCnt []string + for _, test := range p.TestsFailedPrio { + noFlakes[test.Name] = struct{}{} + testTags.Add(&test.Name) + errCount := 0 + if _, ok := p.Tests[test.Name].ErrorCounters["total"]; ok { + errCount = p.Tests[test.Name].ErrorCounters["total"] + } + testsWErrCnt = append(testsWErrCnt, fmt.Sprintf("%d\t%s", errCount, test.Name)) + } + // Failed tests grouped by tag (first value between '[]') + fmt.Printf("%s\n\n", testTags.ShowSorted()) + fmt.Println(strings.Join(testsWErrCnt[:], "\n")) + } + + fmt.Printf("\n --> Failed flake tests - Statistic from OpenShift CI\n") + tbWriter := tabwriter.NewWriter(os.Stdout, 0, 8, 1, '\t', tabwriter.AlignRight) + + if p.Stat.FilterBaseline == 0 { + fmt.Fprintf(tbWriter, "\n") + } else { + testTags := plugin.NewTestTagsEmpty(int(p.Stat.FilterBaseline)) + fmt.Fprintf(tbWriter, "Flakes\tPerc\tErrCount\t TestName\n") + for _, test := range p.TestsFlakeCI { + // preventing duplication when flake tests was already listed. + if _, ok := noFlakes[test.Name]; ok { + continue + } + // TODO: fix issues when retrieving flakes from Sippy API. + // Fallback to '--' when has issues. + if p.Tests[test.Name].Flake == nil { + fmt.Fprintf(tbWriter, "--\t--\t%s\n", test.Name) + } else if p.Tests[test.Name].Flake.CurrentFlakes != 0 { + errCount := 0 + if _, ok := p.Tests[test.Name].ErrorCounters["total"]; ok { + errCount = p.Tests[test.Name].ErrorCounters["total"] + } + fmt.Fprintf(tbWriter, "%d\t%.3f%%\t%d\t%s\n", + p.Tests[test.Name].Flake.CurrentFlakes, + p.Tests[test.Name].Flake.CurrentFlakePerc, + errCount, test.Name) + } + testTags.Add(&test.Name) + } + fmt.Printf("%s\n\n", testTags.ShowSorted()) + } + tbWriter.Flush() +} + +func showChecks(re *report.Report) error { + + tbWriter := tabwriter.NewWriter(os.Stdout, 0, 8, 1, '\t', tabwriter.AlignRight) + fmt.Fprintf(tbWriter, "\n> Presubmit Validation Checks\t\n") + fmt.Fprintf(tbWriter, "\n>> Failed checks (must be reviewed before submitting the results):\t\n") + for _, check := range re.Checks.Fail { + name := check.Name + if check.ID != "" { + name = fmt.Sprintf("[%s] %s", check.ID, check.Name) + } + fmt.Fprintf(tbWriter, " - %s\t: %s\n", name, check.Result) + } + + fmt.Fprintf(tbWriter, "\t\n>> Passed checks:\t\n") + for _, check := range re.Checks.Pass { + name := check.Name + if check.ID != "" { + name = fmt.Sprintf("[%s] %s", check.ID, check.Name) + } + fmt.Fprintf(tbWriter, " - %s\t: %s\n", name, check.Result) + } + + fmt.Fprintf(tbWriter, "\n> Check the docs for each rule at %s\n", re.Checks.BaseURL) + tbWriter.Flush() + return nil +} diff --git a/pkg/report/cmd.go b/pkg/report/cmd.go deleted file mode 100644 index d7d99a14..00000000 --- a/pkg/report/cmd.go +++ /dev/null @@ -1,372 +0,0 @@ -package report - -import ( - "fmt" - "os" - - "github.com/pkg/errors" - "github.com/spf13/cobra" - - "text/tabwriter" - - "github.com/redhat-openshift-ecosystem/provider-certification-tool/internal/pkg/summary" - "github.com/vmware-tanzu/sonobuoy/pkg/errlog" -) - -type Input struct { - archive string - archiveBase string - saveTo string - verbose bool -} - -func NewCmdReport() *cobra.Command { - data := Input{} - cmd := &cobra.Command{ - Use: "report archive.tar.gz", - Short: "Create a report from results.", - Run: func(cmd *cobra.Command, args []string) { - data.archive = args[0] - if err := processResult(&data); err != nil { - errlog.LogError(errors.Wrapf(err, "could not process archive: %v", args[0])) - os.Exit(1) - } - }, - Args: cobra.ExactArgs(1), - } - - cmd.Flags().StringVarP( - &data.archiveBase, "baseline", "b", "", - "Baseline result archive file. Example: -b file.tar.gz", - ) - _ = cmd.MarkFlagRequired("base") - - cmd.Flags().StringVarP( - &data.saveTo, "save-to", "s", "", - "Extract and Save Results to disk. Example: -s ./results", - ) - cmd.Flags().BoolVarP( - &data.verbose, "verbose", "v", false, - "Show test details of test failures", - ) - return cmd -} - -func processResult(input *Input) error { - - cs := summary.ConsolidatedSummary{ - Provider: &summary.ResultSummary{ - Name: summary.ResultSourceNameProvider, - Archive: input.archive, - OpenShift: &summary.OpenShiftSummary{}, - Sonobuoy: &summary.SonobuoySummary{}, - Suites: &summary.OpenshiftTestsSuites{ - OpenshiftConformance: &summary.OpenshiftTestsSuite{Name: "openshiftConformance"}, - KubernetesConformance: &summary.OpenshiftTestsSuite{Name: "kubernetesConformance"}, - }, - }, - Baseline: &summary.ResultSummary{ - Name: summary.ResultSourceNameBaseline, - Archive: input.archiveBase, - OpenShift: &summary.OpenShiftSummary{}, - Sonobuoy: &summary.SonobuoySummary{}, - Suites: &summary.OpenshiftTestsSuites{ - OpenshiftConformance: &summary.OpenshiftTestsSuite{Name: "openshiftConformance"}, - KubernetesConformance: &summary.OpenshiftTestsSuite{Name: "kubernetesConformance"}, - }, - }, - } - - if err := cs.Process(); err != nil { - return err - } - - if err := showAggregatedSummary(&cs); err != nil { - return err - } - - if err := showProcessedSummary(&cs); err != nil { - return err - } - - if err := showErrorDetails(&cs, input.verbose); err != nil { - return err - } - - if input.saveTo != "" { - if err := cs.SaveResults(input.saveTo); err != nil { - return err - } - } - - return nil -} - -func showAggregatedSummary(cs *summary.ConsolidatedSummary) error { - fmt.Printf("\n> OPCT Summary <\n\n") - - // vars starting with p* represents the 'partner' artifact - // vars starting with b* represents 'baseline' artifact - pOCP := cs.GetProvider().GetOpenShift() - pOCPCV, _ := pOCP.GetClusterVersion() - pOCPInfra, _ := pOCP.GetInfrastructure() - - var bOCP *summary.OpenShiftSummary - var bOCPCV *summary.SummaryClusterVersionOutput - var bOCPInfra *summary.SummaryOpenShiftInfrastructureV1 - baselineProcessed := cs.GetBaseline().HasValidResults() - if baselineProcessed { - bOCP = cs.GetBaseline().GetOpenShift() - bOCPCV, _ = bOCP.GetClusterVersion() - bOCPInfra, _ = bOCP.GetInfrastructure() - } - - // Provider and Baseline Cluster (archive) - pCL := cs.GetProvider().GetSonobuoyCluster() - bCL := cs.GetBaseline().GetSonobuoyCluster() - - newLineWithTab := "\t\t\n" - tbWriter := tabwriter.NewWriter(os.Stdout, 0, 8, 1, '\t', tabwriter.AlignRight) - - if baselineProcessed { - fmt.Fprintf(tbWriter, " Kubernetes API Server version\t: %s\t: %s\n", pCL.APIVersion, bCL.APIVersion) - fmt.Fprintf(tbWriter, " OpenShift Container Platform version\t: %s\t: %s\n", pOCPCV.DesiredVersion, bOCPCV.DesiredVersion) - fmt.Fprintf(tbWriter, " - Cluster Update Progressing\t: %s\t: %s\n", pOCPCV.Progressing, bOCPCV.Progressing) - fmt.Fprintf(tbWriter, " - Cluster Target Version\t: %s\t: %s\n", pOCPCV.ProgressingMessage, bOCPCV.ProgressingMessage) - } else { - fmt.Fprintf(tbWriter, " Kubernetes API Server version\t: %s\n", pCL.APIVersion) - fmt.Fprintf(tbWriter, " OpenShift Container Platform version\t: %s\n", pOCPCV.DesiredVersion) - fmt.Fprintf(tbWriter, " - Cluster Update Progressing\t: %s\n", pOCPCV.Progressing) - fmt.Fprintf(tbWriter, " - Cluster Target Version\t: %s\n", pOCPCV.ProgressingMessage) - } - - fmt.Fprint(tbWriter, newLineWithTab) - partnerPlatformName := string(pOCPInfra.Status.PlatformStatus.Type) - if pOCPInfra.Status.PlatformStatus.Type == "External" { - partnerPlatformName = fmt.Sprintf("%s (%s)", partnerPlatformName, pOCPInfra.Spec.PlatformSpec.External.PlatformName) - } - if baselineProcessed { - baselinePlatformName := string(bOCPInfra.Status.PlatformStatus.Type) - if bOCPInfra.Status.PlatformStatus.Type == "External" { - baselinePlatformName = fmt.Sprintf("%s (%s)", baselinePlatformName, bOCPInfra.Spec.PlatformSpec.External.PlatformName) - } - fmt.Fprintf(tbWriter, " OCP Infrastructure:\t\t\n") - fmt.Fprintf(tbWriter, " - PlatformType\t: %s\t: %s\n", partnerPlatformName, baselinePlatformName) - fmt.Fprintf(tbWriter, " - Name\t: %s\t: %s\n", pOCPInfra.Status.InfrastructureName, bOCPInfra.Status.InfrastructureName) - fmt.Fprintf(tbWriter, " - Topology\t: %s\t: %s\n", pOCPInfra.Status.InfrastructureTopology, bOCPInfra.Status.InfrastructureTopology) - fmt.Fprintf(tbWriter, " - ControlPlaneTopology\t: %s\t: %s\n", pOCPInfra.Status.ControlPlaneTopology, bOCPInfra.Status.ControlPlaneTopology) - fmt.Fprintf(tbWriter, " - API Server URL\t: %s\t: %s\n", pOCPInfra.Status.APIServerURL, bOCPInfra.Status.APIServerURL) - fmt.Fprintf(tbWriter, " - API Server URL (internal)\t: %s\t: %s\n", pOCPInfra.Status.APIServerInternalURL, bOCPInfra.Status.APIServerInternalURL) - } else { - fmt.Fprintf(tbWriter, " OCP Infrastructure:\t\n") - fmt.Fprintf(tbWriter, " - PlatformType\t: %s\n", partnerPlatformName) - fmt.Fprintf(tbWriter, " - Name\t: %s\n", pOCPInfra.Status.InfrastructureName) - fmt.Fprintf(tbWriter, " - Topology\t: %s\n", pOCPInfra.Status.InfrastructureTopology) - fmt.Fprintf(tbWriter, " - ControlPlaneTopology\t: %s\n", pOCPInfra.Status.ControlPlaneTopology) - fmt.Fprintf(tbWriter, " - API Server URL\t: %s\n", pOCPInfra.Status.APIServerURL) - fmt.Fprintf(tbWriter, " - API Server URL (internal)\t: %s\n", pOCPInfra.Status.APIServerInternalURL) - } - - fmt.Fprint(tbWriter, newLineWithTab) - fmt.Fprintf(tbWriter, " Plugins summary by name:\t Status [Total/Passed/Failed/Skipped] (timeout)\n") - - plK8S := pOCP.GetResultK8SValidated() - name := plK8S.Name - pOCPPluginRes := fmt.Sprintf("%s [%d/%d/%d/%d] (%d)", plK8S.Status, plK8S.Total, plK8S.Passed, plK8S.Failed, plK8S.Skipped, plK8S.Timeout) - if baselineProcessed { - plK8S = bOCP.GetResultK8SValidated() - bOCPPluginRes := fmt.Sprintf("%s [%d/%d/%d/%d] (%d)", plK8S.Status, plK8S.Total, plK8S.Passed, plK8S.Failed, plK8S.Skipped, plK8S.Timeout) - fmt.Fprintf(tbWriter, " - %s\t: %s\t: %s\n", name, pOCPPluginRes, bOCPPluginRes) - } else { - fmt.Fprintf(tbWriter, " - %s\t: %s\n", name, pOCPPluginRes) - } - - plOCP := pOCP.GetResultOCPValidated() - name = plOCP.Name - pOCPPluginRes = fmt.Sprintf("%s [%d/%d/%d/%d] (%d)", plOCP.Status, plOCP.Total, plOCP.Passed, plOCP.Failed, plOCP.Skipped, plOCP.Timeout) - - if baselineProcessed { - plOCP = bOCP.GetResultOCPValidated() - bOCPPluginRes := fmt.Sprintf("%s [%d/%d/%d/%d] (%d)", plOCP.Status, plOCP.Total, plOCP.Passed, plOCP.Failed, plOCP.Skipped, plOCP.Timeout) - fmt.Fprintf(tbWriter, " - %s\t: %s\t: %s\n", name, pOCPPluginRes, bOCPPluginRes) - } else { - fmt.Fprintf(tbWriter, " - %s\t: %s\n", name, pOCPPluginRes) - } - - fmt.Fprint(tbWriter, newLineWithTab) - fmt.Fprintf(tbWriter, " Health summary:\t [A=True/P=True/D=True]\t\n") - pOCPCO, _ := pOCP.GetClusterOperator() - - if baselineProcessed { - bOCPCO, _ := bOCP.GetClusterOperator() - fmt.Fprintf(tbWriter, " - Cluster Operators\t: [%d/%d/%d]\t: [%d/%d/%d]\n", - pOCPCO.CountAvailable, pOCPCO.CountProgressing, pOCPCO.CountDegraded, - bOCPCO.CountAvailable, bOCPCO.CountProgressing, bOCPCO.CountDegraded, - ) - } else { - fmt.Fprintf(tbWriter, " - Cluster Operators\t: [%d/%d/%d]\n", - pOCPCO.CountAvailable, pOCPCO.CountProgressing, pOCPCO.CountDegraded, - ) - } - - pNhMessage := fmt.Sprintf("%d/%d %s", pCL.NodeHealth.Total, pCL.NodeHealth.Total, "") - if pCL.NodeHealth.Total != 0 { - pNhMessage = fmt.Sprintf("%s (%d%%)", pNhMessage, 100*pCL.NodeHealth.Healthy/pCL.NodeHealth.Total) - } - - bNhMessage := fmt.Sprintf("%d/%d %s", bCL.NodeHealth.Total, bCL.NodeHealth.Total, "") - if bCL.NodeHealth.Total != 0 { - bNhMessage = fmt.Sprintf("%s (%d%%)", bNhMessage, 100*bCL.NodeHealth.Healthy/bCL.NodeHealth.Total) - } - if baselineProcessed { - fmt.Fprintf(tbWriter, " - Node health\t: %s\t: %s\n", pNhMessage, bNhMessage) - } else { - fmt.Fprintf(tbWriter, " - Node health\t: %s\n", pNhMessage) - } - - pPodsHealthMsg := "" - bPodsHealthMsg := "" - if len(pCL.PodHealth.Details) > 0 { - phTotal := "" - if pCL.PodHealth.Total != 0 { - phTotal = fmt.Sprintf(" (%d%%)", 100*pCL.PodHealth.Healthy/pCL.PodHealth.Total) - } - pPodsHealthMsg = fmt.Sprintf("%d/%d %s", pCL.PodHealth.Healthy, pCL.PodHealth.Total, phTotal) - } - if baselineProcessed { - if len(bCL.PodHealth.Details) > 0 { - phTotal := "" - if bCL.PodHealth.Total != 0 { - phTotal = fmt.Sprintf(" (%d%%)", 100*bCL.PodHealth.Healthy/bCL.PodHealth.Total) - } - bPodsHealthMsg = fmt.Sprintf("%d/%d %s", bCL.PodHealth.Healthy, bCL.PodHealth.Total, phTotal) - } - fmt.Fprintf(tbWriter, " - Pods health\t: %s\t: %s\n", pPodsHealthMsg, bPodsHealthMsg) - } else { - fmt.Fprintf(tbWriter, " - Pods health\t: %s\n", pPodsHealthMsg) - } - - tbWriter.Flush() - return nil -} - -func showProcessedSummary(cs *summary.ConsolidatedSummary) error { - - fmt.Printf("\n> Processed Summary <\n") - - fmt.Printf("\n Total tests by conformance suites:\n") - fmt.Printf(" - %s: %d \n", summary.SuiteNameKubernetesConformance, cs.GetProvider().GetSuites().GetTotalK8S()) - fmt.Printf(" - %s: %d \n", summary.SuiteNameOpenshiftConformance, cs.GetProvider().GetSuites().GetTotalOCP()) - - fmt.Printf("\n Result Summary by conformance plugins:\n") - bProcessed := cs.GetBaseline().HasValidResults() - showSummaryPlugin(cs.GetProvider().GetOpenShift().GetResultK8SValidated(), bProcessed) - showSummaryPlugin(cs.GetProvider().GetOpenShift().GetResultOCPValidated(), bProcessed) - - return nil -} - -func showSummaryPlugin(p *summary.OPCTPluginSummary, bProcessed bool) { - fmt.Printf(" - %s:\n", p.Name) - fmt.Printf(" - Status: %s\n", p.Status) - fmt.Printf(" - Total: %d\n", p.Total) - fmt.Printf(" - Passed: %d\n", p.Passed) - fmt.Printf(" - Failed: %d\n", p.Failed) - fmt.Printf(" - Timeout: %d\n", p.Timeout) - fmt.Printf(" - Skipped: %d\n", p.Skipped) - fmt.Printf(" - Failed (without filters) : %d\n", len(p.FailedList)) - fmt.Printf(" - Failed (Filter SuiteOnly): %d\n", len(p.FailedFilterSuite)) - if bProcessed { - fmt.Printf(" - Failed (Filter Baseline) : %d\n", len(p.FailedFilterBaseline)) - } - fmt.Printf(" - Failed (Filter CI Flakes): %d\n", len(p.FailedFilterFlaky)) - - // checking for runtime failure - runtimeFailed := false - if p.Total == p.Failed { - runtimeFailed = true - } - - // rewrite the original status when pass on all filters and not failed on runtime - status := p.Status - if (len(p.FailedFilterFlaky) == 0) && !runtimeFailed { - status = "pass" - } - - fmt.Printf(" - Status After Filters : %s\n", status) -} - -// showErrorDetails show details of failres for each plugin. -func showErrorDetails(cs *summary.ConsolidatedSummary, verbose bool) error { - - fmt.Printf("\n Result details by conformance plugins: \n") - bProcessed := cs.GetBaseline().HasValidResults() - showErrorDetailPlugin(cs.GetProvider().GetOpenShift().GetResultK8SValidated(), verbose, bProcessed) - showErrorDetailPlugin(cs.GetProvider().GetOpenShift().GetResultOCPValidated(), verbose, bProcessed) - - return nil -} - -// showErrorDetailPlugin Show failed e2e tests by filter, when verbose each filter will be shown. -func showErrorDetailPlugin(p *summary.OPCTPluginSummary, verbose bool, bProcessed bool) { - - flakeCount := len(p.FailedFilterBaseline) - len(p.FailedFilterFlaky) - - if verbose { - fmt.Printf("\n\n => %s: (%d failures, %d failures filtered, %d flakes)\n", p.Name, len(p.FailedList), len(p.FailedFilterBaseline), flakeCount) - - fmt.Printf("\n --> [verbose] Failed tests detected on archive (without filters):\n") - if len(p.FailedList) == 0 { - fmt.Println("") - } - for _, test := range p.FailedList { - fmt.Println(test) - } - - fmt.Printf("\n --> [verbose] Failed tests detected on suite (Filter SuiteOnly):\n") - if len(p.FailedFilterSuite) == 0 { - fmt.Println("") - } - for _, test := range p.FailedFilterSuite { - fmt.Println(test) - } - if bProcessed { - fmt.Printf("\n --> [verbose] Failed tests removing baseline (Filter Baseline):\n") - if len(p.FailedFilterBaseline) == 0 { - fmt.Println("") - } - for _, test := range p.FailedFilterBaseline { - fmt.Println(test) - } - } - } else { - fmt.Printf("\n\n => %s: (%d failures, %d flakes)\n", p.Name, len(p.FailedFilterBaseline), flakeCount) - } - - fmt.Printf("\n --> Failed tests to Review (without flakes) - Immediate action:\n") - if len(p.FailedFilterBaseline) == flakeCount { - fmt.Println("") - } - for _, test := range p.FailedFilterFlaky { - fmt.Println(test) - } - - fmt.Printf("\n --> Failed flake tests - Statistic from OpenShift CI\n") - tbWriter := tabwriter.NewWriter(os.Stdout, 0, 8, 1, '\t', tabwriter.AlignRight) - - if len(p.FailedFilterBaseline) == 0 { - fmt.Fprintf(tbWriter, "\n") - } else { - fmt.Fprintf(tbWriter, "Flakes\tPerc\t TestName\n") - for _, test := range p.FailedFilterBaseline { - // When the was issues to create the flaky item (network connectivity with Sippy API), - // fallback to '--' values. - if p.FailedItems[test].Flaky == nil { - fmt.Fprintf(tbWriter, "--\t--\t%s\n", test) - } else if p.FailedItems[test].Flaky.CurrentFlakes != 0 { - fmt.Fprintf(tbWriter, "%d\t%.3f%%\t%s\n", p.FailedItems[test].Flaky.CurrentFlakes, p.FailedItems[test].Flaky.CurrentFlakePerc, test) - } - } - } - tbWriter.Flush() -} diff --git a/pkg/retrieve/retrieve.go b/pkg/retrieve/retrieve.go index dfae8f32..4e6592d6 100644 --- a/pkg/retrieve/retrieve.go +++ b/pkg/retrieve/retrieve.go @@ -48,7 +48,7 @@ func NewCmdRetrieve() *cobra.Command { return } - s := status.NewStatusOptions(false) + s := status.NewStatusOptions(&status.StatusInput{Watch: false}) err = s.PreRunCheck(kclient) if err != nil { log.Error(err) diff --git a/pkg/run/run.go b/pkg/run/run.go index bcf27d43..f58f5ae6 100644 --- a/pkg/run/run.go +++ b/pkg/run/run.go @@ -40,12 +40,13 @@ type RunOptions struct { // PluginsImage // defines the image containing plugins associated with the provider-certification-tool. // this variable is referenced by plugin manifest templates to dynamically reference the plugins image. - PluginsImage string - timeout int - watch bool - devCount string - mode string - upgradeImage string + PluginsImage string + timeout int + watch bool + watchInterval int + devCount string + mode string + upgradeImage string } const ( @@ -110,9 +111,9 @@ func NewCmdRun() *cobra.Command { } // Sleep to give status time to appear - time.Sleep(status.StatusInterval) + s := status.NewStatusOptions(&status.StatusInput{Watch: o.watch, IntervalSeconds: o.watchInterval}) + time.Sleep(s.GetIntervalSeconds()) - s := status.NewStatusOptions(o.watch) err = s.WaitForStatusReport(cmd.Context(), sclient) if err != nil { log.WithError(err).Fatal("error retrieving aggregator status") @@ -144,6 +145,7 @@ func NewCmdRun() *cobra.Command { cmd.Flags().StringVar(&o.imageRepository, "image-repository", "", "Image repository containing required images test environment. Example: openshift-provider-cert-tool --mirror-repository mirror.repository.net/ocp-cert") cmd.Flags().IntVar(&o.timeout, "timeout", defaultRunTimeoutSeconds, "Execution timeout in seconds") cmd.Flags().BoolVarP(&o.watch, "watch", "w", defaultRunWatchFlag, "Keep watch status after running") + cmd.Flags().IntVarP(&o.watchInterval, "watch-interval", "", status.DefaultStatusIntervalSeconds, "Interval to watch the status and print in the stdout") // Hide optional flags hideOptionalFlags(cmd, "dedicated") diff --git a/pkg/status/printer.go b/pkg/status/printer.go index b3cc832c..f0b7e9ef 100644 --- a/pkg/status/printer.go +++ b/pkg/status/printer.go @@ -60,6 +60,9 @@ func getPrintableRunningStatus(s *aggregation.Status) PrintableStatus { } } else if pl.ResultStatus == "" { message = "waiting for post-processor..." + if pl.Status != "" { + message = pl.Status + } } else { passCount := pl.ResultStatusCounts["passed"] failedCount := pl.ResultStatusCounts["failed"] diff --git a/pkg/status/status.go b/pkg/status/status.go index fa68b49a..b99642fa 100644 --- a/pkg/status/status.go +++ b/pkg/status/status.go @@ -20,7 +20,8 @@ import ( ) const ( - StatusInterval = time.Second * 10 + DefaultStatusIntervalSeconds = 10 + // StatusInterval = time.Second * 10 StatusRetryLimit = 10 ) @@ -28,16 +29,31 @@ type StatusOptions struct { Latest *aggregation.Status watch bool shownPostProcessMsg bool + waitInterval time.Duration } -func NewStatusOptions(watch bool) *StatusOptions { - return &StatusOptions{ - watch: watch, +type StatusInput struct { + Watch bool + IntervalSeconds int +} + +func NewStatusOptions(in *StatusInput) *StatusOptions { + s := &StatusOptions{ + watch: in.Watch, + waitInterval: time.Second * DefaultStatusIntervalSeconds, + } + if in.IntervalSeconds != 0 { + s.waitInterval = time.Duration(in.IntervalSeconds) * time.Second } + return s +} + +func (s *StatusOptions) GetIntervalSeconds() time.Duration { + return s.waitInterval } func NewCmdStatus() *cobra.Command { - o := NewStatusOptions(false) + o := NewStatusOptions(&StatusInput{Watch: false}) cmd := &cobra.Command{ Use: "status", @@ -139,7 +155,7 @@ func (s *StatusOptions) GetStatus() string { // An error will not result in immediate failure and will be retried. func (s *StatusOptions) WaitForStatusReport(ctx context.Context, sclient sonobuoyclient.Interface) error { tries := 1 - err := wait2.PollImmediateUntilWithContext(ctx, StatusInterval, func(ctx context.Context) (done bool, err error) { + err := wait2.PollImmediateUntilWithContext(ctx, s.waitInterval, func(ctx context.Context) (done bool, err error) { if tries == StatusRetryLimit { return false, errors.New("retry limit reached checking for aggregator status") } @@ -152,7 +168,7 @@ func (s *StatusOptions) WaitForStatusReport(ctx context.Context, sclient sonobuo } tries++ - log.Warnf("waiting %ds to retry", int(StatusInterval.Seconds())) + log.Warnf("waiting %ds to retry", int(s.waitInterval.Seconds())) return false, nil }) return err @@ -165,7 +181,7 @@ func (s *StatusOptions) Print(cmd *cobra.Command, sclient sonobuoyclient.Interfa } tries := 1 - return wait2.PollImmediateInfiniteWithContext(cmd.Context(), StatusInterval, func(ctx context.Context) (done bool, err error) { + return wait2.PollImmediateInfiniteWithContext(cmd.Context(), s.waitInterval, func(ctx context.Context) (done bool, err error) { if tries == StatusRetryLimit { // we hit back-to-back errors too many times. return true, errors.New("retry limit reached checking status") diff --git a/pkg/types.go b/pkg/types.go index 69374305..a769f4f2 100644 --- a/pkg/types.go +++ b/pkg/types.go @@ -13,7 +13,7 @@ const ( SonobuoyLabelComponentName = "component" SonobuoyLabelComponentValue = "sonobuoy" DefaultToolsRepository = "quay.io/ocp-cert" - PluginsImage = "openshift-tests-provider-cert:v0.4.0" + PluginsImage = "openshift-tests-provider-cert:v0.5.0-beta0" ) var (