Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

metrics: support pyroscope for continous profile #8958

Open
wants to merge 19 commits into
base: master
Choose a base branch
from
+47 −0
Open

metrics: support pyroscope for continous profile #8958

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
55c3422
support pyroscope
bufferflies Dec 30, 2024
2a6220a
go mod tidy
bufferflies Dec 30, 2024
3aa96f1
format
bufferflies Dec 30, 2024
4ee9551
Merge branch 'master' into support_pyroscope
ti-chi-bot[bot] Feb 8, 2025
1df85e5
Merge branch 'master' into support_pyroscope
ti-chi-bot[bot] Feb 10, 2025
130d6b1
Merge branch 'master' into support_pyroscope
ti-chi-bot[bot] Feb 10, 2025
660c43f
Merge branch 'master' into support_pyroscope
ti-chi-bot[bot] Feb 11, 2025
796b5ea
Merge branch 'master' into support_pyroscope
ti-chi-bot[bot] Feb 11, 2025
a87dec1
Merge branch 'master' into support_pyroscope
ti-chi-bot[bot] Feb 11, 2025
17a0d7a
Merge branch 'master' into support_pyroscope
ti-chi-bot[bot] Feb 11, 2025
047e215
Merge branch 'master' into support_pyroscope
ti-chi-bot[bot] Feb 11, 2025
19984d2
Merge branch 'master' into support_pyroscope
ti-chi-bot[bot] Feb 13, 2025
02b3267
Merge branch 'master' into support_pyroscope
ti-chi-bot[bot] Feb 14, 2025
4f8b68d
Merge branch 'master' into support_pyroscope
ti-chi-bot[bot] Feb 17, 2025
73d6413
Merge branch 'master' into support_pyroscope
ti-chi-bot[bot] Feb 17, 2025
f76006b
Merge branch 'master' into support_pyroscope
ti-chi-bot[bot] Feb 18, 2025
c626b83
Merge branch 'master' into support_pyroscope
ti-chi-bot[bot] Feb 19, 2025
1b7b39a
Merge branch 'master' into support_pyroscope
ti-chi-bot[bot] Feb 19, 2025
275597d
Merge branch 'master' into support_pyroscope
ti-chi-bot[bot] Feb 19, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions cmd/pd-server/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,7 @@ func start(cmd *cobra.Command, args []string, services ...string) {
grpcprometheus.EnableHandlingTimeHistogram()

metricutil.Push(&cfg.Metric)
metricutil.EnablePyroscope()

err = join.PrepareJoinCluster(cfg)
if err != nil {
Expand Down
2 changes: 2 additions & 0 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ require (
github.com/gogo/protobuf v1.3.2
github.com/google/btree v1.1.2
github.com/gorilla/mux v1.7.4
github.com/grafana/pyroscope-go v1.2.0
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
github.com/joho/godotenv v1.4.0
github.com/mailru/easyjson v0.7.6
Expand Down Expand Up @@ -126,6 +127,7 @@ require (
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/gorilla/websocket v1.5.1 // indirect
github.com/grafana/pyroscope-go/godeltaprof v0.1.8 // indirect
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect
github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect
Expand Down
4 changes: 4 additions & 0 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,10 @@ github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB7
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY=
github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY=
github.com/grafana/pyroscope-go v1.2.0 h1:aILLKjTj8CS8f/24OPMGPewQSYlhmdQMBmol1d3KGj8=
github.com/grafana/pyroscope-go v1.2.0/go.mod h1:2GHr28Nr05bg2pElS+dDsc98f3JTUh2f6Fz1hWXrqwk=
github.com/grafana/pyroscope-go/godeltaprof v0.1.8 h1:iwOtYXeeVSAeYefJNaxDytgjKtUuKQbJqgAIjlnicKg=
github.com/grafana/pyroscope-go/godeltaprof v0.1.8/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU=
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI=
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8=
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho=
Expand Down
28 changes: 28 additions & 0 deletions pkg/utils/metricutil/metricutil.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,14 @@

import (
"os"
"runtime"
"time"
"unicode"

"github.com/grafana/pyroscope-go"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/push"
"go.uber.org/zap"

"github.com/pingcap/log"

Expand Down Expand Up @@ -102,3 +105,28 @@
}
return hostname
}

// EnablePyroscope enables pyroscope if pyroscope is enabled.
func EnablePyroscope() {
if os.Getenv("PYROSCOPE_SERVER_ADDRESS") != "" {
runtime.SetMutexProfileFraction(5)
runtime.SetBlockProfileRate(5)
_, err := pyroscope.Start(pyroscope.Config{
ApplicationName: "pd",
ServerAddress: os.Getenv("PYROSCOPE_SERVER_ADDRESS"),
Logger: pyroscope.StandardLogger,
AuthToken: os.Getenv("PYROSCOPE_AUTH_TOKEN"),
TenantID: os.Getenv("PYROSCOPE_TENANT_ID"),
BasicAuthUser: os.Getenv("PYROSCOPE_BASIC_AUTH_USER"),
BasicAuthPassword: os.Getenv("PYROSCOPE_BASIC_AUTH_PASSWORD"),
ProfileTypes: []pyroscope.ProfileType{
pyroscope.ProfileCPU,
pyroscope.ProfileAllocSpace,
},
UploadRate: 30 * time.Second,
})
if err != nil {
log.Fatal("fail to start pyroscope", zap.Error(err))
}

Check warning on line 130 in pkg/utils/metricutil/metricutil.go

View check run for this annotation

Codecov / codecov/patch

pkg/utils/metricutil/metricutil.go#L110-L130 

Added lines #L110 - L130 were not covered by tests
}
}
2 changes: 2 additions & 0 deletions tests/integrations/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,8 @@ require (
github.com/google/uuid v1.6.0 // indirect
github.com/gorilla/mux v1.7.4 // indirect
github.com/gorilla/websocket v1.5.1 // indirect
github.com/grafana/pyroscope-go v1.2.0 // indirect
github.com/grafana/pyroscope-go/godeltaprof v0.1.8 // indirect
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
Expand Down
4 changes: 4 additions & 0 deletions tests/integrations/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -237,6 +237,10 @@ github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB7
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY=
github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY=
github.com/grafana/pyroscope-go v1.2.0 h1:aILLKjTj8CS8f/24OPMGPewQSYlhmdQMBmol1d3KGj8=
github.com/grafana/pyroscope-go v1.2.0/go.mod h1:2GHr28Nr05bg2pElS+dDsc98f3JTUh2f6Fz1hWXrqwk=
github.com/grafana/pyroscope-go/godeltaprof v0.1.8 h1:iwOtYXeeVSAeYefJNaxDytgjKtUuKQbJqgAIjlnicKg=
github.com/grafana/pyroscope-go/godeltaprof v0.1.8/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU=
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI=
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8=
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho=
Expand Down
2 changes: 2 additions & 0 deletions tools/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,8 @@ require (
github.com/google/uuid v1.6.0 // indirect
github.com/gorilla/mux v1.7.4 // indirect
github.com/gorilla/websocket v1.5.1 // indirect
github.com/grafana/pyroscope-go v1.2.0 // indirect
github.com/grafana/pyroscope-go/godeltaprof v0.1.8 // indirect
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
Expand Down
4 changes: 4 additions & 0 deletions tools/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -236,6 +236,10 @@ github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB7
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY=
github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY=
github.com/grafana/pyroscope-go v1.2.0 h1:aILLKjTj8CS8f/24OPMGPewQSYlhmdQMBmol1d3KGj8=
github.com/grafana/pyroscope-go v1.2.0/go.mod h1:2GHr28Nr05bg2pElS+dDsc98f3JTUh2f6Fz1hWXrqwk=
github.com/grafana/pyroscope-go/godeltaprof v0.1.8 h1:iwOtYXeeVSAeYefJNaxDytgjKtUuKQbJqgAIjlnicKg=
github.com/grafana/pyroscope-go/godeltaprof v0.1.8/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU=
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI=
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8=
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho=
Expand Down