Change pprof labels to be prometheus compatible (#32865)

Enables scrapping pprof endpoint for continuous profiling

Closes: #32854

modules/graceful/manager.go

@@ -136,7 +136,7 @@ func (g *Manager) doShutdown() {
 	}
 	g.lock.Lock()
 	g.shutdownCtxCancel()
-	atShutdownCtx := pprof.WithLabels(g.hammerCtx, pprof.Labels("graceful-lifecycle", "post-shutdown"))
+	atShutdownCtx := pprof.WithLabels(g.hammerCtx, pprof.Labels(LifecyclePProfLabel, "post-shutdown"))
 	pprof.SetGoroutineLabels(atShutdownCtx)
 	for _, fn := range g.toRunAtShutdown {
 		go fn()
@@ -167,7 +167,7 @@ func (g *Manager) doHammerTime(d time.Duration) {
 	default:
 		log.Warn("Setting Hammer condition")
 		g.hammerCtxCancel()
-		atHammerCtx := pprof.WithLabels(g.terminateCtx, pprof.Labels("graceful-lifecycle", "post-hammer"))
+		atHammerCtx := pprof.WithLabels(g.terminateCtx, pprof.Labels(LifecyclePProfLabel, "post-hammer"))
 		pprof.SetGoroutineLabels(atHammerCtx)
 	}
 	g.lock.Unlock()
@@ -183,7 +183,7 @@ func (g *Manager) doTerminate() {
 	default:
 		log.Warn("Terminating")
 		g.terminateCtxCancel()
-		atTerminateCtx := pprof.WithLabels(g.managerCtx, pprof.Labels("graceful-lifecycle", "post-terminate"))
+		atTerminateCtx := pprof.WithLabels(g.managerCtx, pprof.Labels(LifecyclePProfLabel, "post-terminate"))
 		pprof.SetGoroutineLabels(atTerminateCtx)
 
 		for _, fn := range g.toRunAtTerminate {

modules/graceful/manager_common.go

@@ -22,6 +22,12 @@ const (
 	watchdogMsg  systemdNotifyMsg = "WATCHDOG=1"
 )
 
+// LifecyclePProfLabel is a label marking manager lifecycle phase
+// Making it compliant with prometheus key regex https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels
+// would enable someone interested to be able to to continuously gather profiles into pyroscope.
+// Other labels for pprof (in "modules/process" package) should also follow this rule.
+const LifecyclePProfLabel = "graceful_lifecycle"
+
 func statusMsg(msg string) systemdNotifyMsg {
 	return systemdNotifyMsg("STATUS=" + msg)
 }
@@ -65,10 +71,10 @@ func (g *Manager) prepare(ctx context.Context) {
 	g.hammerCtx, g.hammerCtxCancel = context.WithCancel(ctx)
 	g.managerCtx, g.managerCtxCancel = context.WithCancel(ctx)
 
-	g.terminateCtx = pprof.WithLabels(g.terminateCtx, pprof.Labels("graceful-lifecycle", "with-terminate"))
-	g.shutdownCtx = pprof.WithLabels(g.shutdownCtx, pprof.Labels("graceful-lifecycle", "with-shutdown"))
-	g.hammerCtx = pprof.WithLabels(g.hammerCtx, pprof.Labels("graceful-lifecycle", "with-hammer"))
-	g.managerCtx = pprof.WithLabels(g.managerCtx, pprof.Labels("graceful-lifecycle", "with-manager"))
+	g.terminateCtx = pprof.WithLabels(g.terminateCtx, pprof.Labels(LifecyclePProfLabel, "with-terminate"))
+	g.shutdownCtx = pprof.WithLabels(g.shutdownCtx, pprof.Labels(LifecyclePProfLabel, "with-shutdown"))
+	g.hammerCtx = pprof.WithLabels(g.hammerCtx, pprof.Labels(LifecyclePProfLabel, "with-hammer"))
+	g.managerCtx = pprof.WithLabels(g.managerCtx, pprof.Labels(LifecyclePProfLabel, "with-manager"))
 
 	if !g.setStateTransition(stateInit, stateRunning) {
 		panic("invalid graceful manager state: transition from init to running failed")

modules/process/context.go

@@ -32,7 +32,7 @@ func (c *Context) Value(key any) any {
 }
 
 // ProcessContextKey is the key under which process contexts are stored
-var ProcessContextKey any = "process-context"
+var ProcessContextKey any = "process_context"
 
 // GetContext will return a process context if one exists
 func GetContext(ctx context.Context) *Context {

modules/process/manager.go

@@ -26,7 +26,7 @@ var (
 )
 
 // DescriptionPProfLabel is a label set on goroutines that have a process attached
-const DescriptionPProfLabel = "process-description"
+const DescriptionPProfLabel = "process_description"
 
 // PIDPProfLabel is a label set on goroutines that have a process attached
 const PIDPProfLabel = "pid"
@@ -35,7 +35,7 @@ const PIDPProfLabel = "pid"
 const PPIDPProfLabel = "ppid"
 
 // ProcessTypePProfLabel is a label set on goroutines that have a process attached
-const ProcessTypePProfLabel = "process-type"
+const ProcessTypePProfLabel = "process_type"
 
 // IDType is a pid type
 type IDType string