CASSANDRA-20854: Support low-overhead async profiling

.build/cassandra-deps-maven-pom.xml

@@ -161,6 +161,10 @@
       <groupId>com.github.jbellis</groupId>
       <artifactId>jamm</artifactId>
     </dependency>
+    <dependency>
+      <groupId>tools.profiler</groupId>
+      <artifactId>async-profiler</artifactId>
+    </dependency>
     <dependency>
       <groupId>io.netty</groupId>
       <artifactId>netty-all</artifactId>

.build/parent-maven-pom.xml

@@ -40,6 +40,7 @@
     <byteman.version>4.0.20</byteman.version>
     <netty.version>4.1.125.Final</netty.version>
     <ohc.version>0.5.1</ohc.version>
+    <async-profiler.version>4.2</async-profiler.version>
 
     <!-- These are referenced in build.xml, so need to be propagated from there -->
     <asm.version>@asm.version@</asm.version>
@@ -465,6 +466,11 @@
         <artifactId>jamm</artifactId>
         <version>${jamm.version}</version>
       </dependency>
+      <dependency>
+        <groupId>tools.profiler</groupId>
+        <artifactId>async-profiler</artifactId>
+        <version>${async-profiler.version}</version>
+      </dependency>
       <dependency>
         <!-- Test scoped jackson-dataformat-yaml also depends on snakeyaml. For now, these versions are aligned
         but if you happen to bump it here then exclude it in jackson-dataformat-yaml.

CHANGES.txt

@@ -1,4 +1,5 @@
 5.1
+ * Support low-overhead async profiling (CASSANDRA-20854)
  * Introducing comments and security labels for schema elements (CASSANDRA-20943)
  * Extend nodetool tablestats for dictionary memory usage (CASSANDRA-20940)
  * Introduce separate GCInspector thresholds for concurrent GC events (CASSANDRA-20980)

conf/jvm-server.options

@@ -156,6 +156,11 @@
 # Disable chronicle analytics. See CASSANDRA-19656
 -Dchronicle.analytics.disable=true
 
+# AsyncProfiler Flags
+#-Dcassandra.async_profiler.enabled=true|false
+#-Dcassandra.async_profiler.advanced_mode=true|false
+#-Dcassandra.async_profiler.output_directory=/tmp/cassandra-profiling
+
 ### Debug options
 
 # uncomment to enable flight recorder

src/java/org/apache/cassandra/config/CassandraRelevantProperties.java

@@ -60,6 +60,9 @@ public enum CassandraRelevantProperties
     ALLOW_UNSAFE_REPLACE("cassandra.allow_unsafe_replace"),
     ALLOW_UNSAFE_TRANSIENT_CHANGES("cassandra.allow_unsafe_transient_changes"),
     APPROXIMATE_TIME_PRECISION_MS("cassandra.approximate_time_precision_ms", "2"),
+    ASYNC_PROFILER_ADVANCED_MODE("cassandra.async_profiler.advanced_mode", "false"),
+    ASYNC_PROFILER_ENABLED("cassandra.async_profiler.enabled", "true"),
+    ASYNC_PROFILER_OUTPUT_DIRECTORY("cassandra.async_profiler.output_directory", "/var/log/cassandra-profiling"),
     /** 2 ** GENSALT_LOG2_ROUNDS rounds of hashing will be performed. */
     AUTH_BCRYPT_GENSALT_LOG2_ROUNDS("cassandra.auth_bcrypt_gensalt_log2_rounds", "4"),
     /** We expect default values on cache retries and interval to be sufficient for everyone but have this escape hatch just in case. */

src/java/org/apache/cassandra/profiler/AsyncProfiler.java

@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.profiler;
+import org.apache.cassandra.tools.profiler.AsyncProfilerService;
+
+public abstract class AsyncProfiler implements AsyncProfilerMBean
+{
+    public static final String MBEAN_NAME = "org.apache.cassandra.profiler:type=AsyncProfiler";
+    private final AsyncProfilerService service = new AsyncProfilerService();
+
+    public void start(String event, String outputFormat, int timeout, String outputFileName)
+    {
+        getService().start(event, outputFormat, timeout, outputFileName);
+    }
+
+    public void stop(String outputFileName)
+    {
+        getService().stop(outputFileName);
+    }
+
+    public boolean isAvailable()
+    {
+        return getService().isAvailable();
+    }
+
+    public AsyncProfilerService getService()
+    {
+        return service;
+    }
+}

src/java/org/apache/cassandra/profiler/AsyncProfilerMBean.java

@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.profiler;
+
+public interface AsyncProfilerMBean
+{
+    void start(String event, String outputFormat, int timeout, String outputFileName);
+
+    void stop(String outputFileName);
+
+    void execute(String command);
+
+    boolean isAvailable();
+}

src/java/org/apache/cassandra/profiler/AsyncProfilerSafe.java

@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.profiler;
+
+public class AsyncProfilerSafe extends AsyncProfiler
+{
+    public void execute(String command)
+    {
+        throw new SecurityException(String.format("Execute commands are not permitted " +
+                                                  "with this MBean. Please use unsafe MBean" +
+                                                  "if they are needed. Command: %s", command));
+    }
+}

src/java/org/apache/cassandra/profiler/AsyncProfilerUnsafe.java

@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.profiler;
+
+public class AsyncProfilerUnsafe extends AsyncProfiler
+{
+    public void execute(String command)
+    {
+        getService().execute(command);
+    }
+}

src/java/org/apache/cassandra/service/CassandraDaemon.java

@@ -39,6 +39,8 @@
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableList;
+import org.apache.cassandra.profiler.AsyncProfiler;
+import org.apache.cassandra.profiler.AsyncProfilerMBean;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -74,6 +76,8 @@
 import org.apache.cassandra.io.util.FileUtils;
 import org.apache.cassandra.locator.InetAddressAndPort;
 import org.apache.cassandra.locator.Locator;
+import org.apache.cassandra.profiler.AsyncProfilerSafe;
+import org.apache.cassandra.profiler.AsyncProfilerUnsafe;
 import org.apache.cassandra.tcm.CMSOperations;
 import org.apache.cassandra.tcm.ClusterMetadataService;
 import org.apache.cassandra.tcm.RegistrationStatus;
@@ -104,6 +108,7 @@
 import org.apache.cassandra.utils.logging.VirtualTableAppender;
 
 import static java.util.concurrent.TimeUnit.NANOSECONDS;
+import static org.apache.cassandra.config.CassandraRelevantProperties.ASYNC_PROFILER_ADVANCED_MODE;
 import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_FOREGROUND;
 import static org.apache.cassandra.config.CassandraRelevantProperties.CASSANDRA_PID_FILE;
 import static org.apache.cassandra.config.CassandraRelevantProperties.COM_SUN_MANAGEMENT_JMXREMOTE_PORT;
@@ -751,6 +756,7 @@ public void activate(boolean closeStdOutErr)
             applyConfig();
 
             registerNativeAccess();
+            registerAsyncProfiler();
 
             setup();
 
@@ -803,6 +809,21 @@ public static void registerNativeAccess() throws javax.management.NotCompliantMB
         MBeanWrapper.instance.registerMBean(new StandardMBean(new NativeAccess(), NativeAccessMBean.class), MBEAN_NAME, MBeanWrapper.OnException.LOG);
     }
 
+    @VisibleForTesting
+    public static void registerAsyncProfiler() throws javax.management.NotCompliantMBeanException
+    {
+        AsyncProfiler asyncProfiler;
+        if (!ASYNC_PROFILER_ADVANCED_MODE.getBoolean())
+        {
+            asyncProfiler = new AsyncProfilerSafe();
+        }
+        else
+        {
+            asyncProfiler = new AsyncProfilerUnsafe();
+        }
+        MBeanWrapper.instance.registerMBean(new StandardMBean(asyncProfiler, AsyncProfilerMBean.class), AsyncProfiler.MBEAN_NAME, MBeanWrapper.OnException.LOG);
+    }
+
     public void applyConfig()
     {
         DatabaseDescriptor.daemonInitialization();

src/java/org/apache/cassandra/tools/NodeProbe.java

@@ -117,6 +117,8 @@
 import org.apache.cassandra.metrics.ThreadPoolMetrics;
 import org.apache.cassandra.net.MessagingService;
 import org.apache.cassandra.net.MessagingServiceMBean;
+import org.apache.cassandra.profiler.AsyncProfiler;
+import org.apache.cassandra.profiler.AsyncProfilerMBean;
 import org.apache.cassandra.service.ActiveRepairServiceMBean;
 import org.apache.cassandra.service.AutoRepairService;
 import org.apache.cassandra.service.AutoRepairServiceMBean;
@@ -187,6 +189,7 @@ public class NodeProbe implements AutoCloseable
     protected PermissionsCacheMBean pcProxy;
     protected RolesCacheMBean rcProxy;
     protected AutoRepairServiceMBean autoRepairProxy;
+    protected AsyncProfilerMBean asyncProfilerProxy;
     protected GuardrailsMBean grProxy;
     protected volatile Output output;
 
@@ -335,6 +338,9 @@ protected void connect() throws IOException
             name = new ObjectName(AutoRepairService.MBEAN_NAME);
             autoRepairProxy = JMX.newMBeanProxy(mbeanServerConn, name, AutoRepairServiceMBean.class);
 
+            name = new ObjectName(AsyncProfiler.MBEAN_NAME);
+            asyncProfilerProxy = JMX.newMBeanProxy(mbeanServerConn, name, AsyncProfilerMBean.class);
+
             name = new ObjectName(Guardrails.MBEAN_NAME);
             grProxy = JMX.newMBeanProxy(mbeanServerConn, name, GuardrailsMBean.class);
         }
@@ -1329,6 +1335,11 @@ public AccordOperationsMBean getAccordOperationsProxy()
         return accordProxy;
     }
 
+    public AsyncProfilerMBean getAsyncProfilerProxy()
+    {
+        return asyncProfilerProxy;
+    }
+
     public GossiperMBean getGossProxy()
     {
         return gossProxy;

src/java/org/apache/cassandra/tools/nodetool/NodetoolCommand.java

@@ -149,6 +149,7 @@
                          NetStats.class,
                          PauseHandoff.class,
                          ProfileLoad.class,
+                         Profile.class,
                          ProxyHistograms.class,
                          RangeKeySample.class,
                          Rebuild.class,