#65 '-t' option to split profile by threads

README.md

@@ -210,6 +210,10 @@ method ids that should fit in the buffer. If you receive messages about an
 insufficient frame buffer size, increase this value from the default.  
 Example: `./profiler.sh -b 5000000 8983`
 
+* `-t` - profile threads separately. Each stack trace will end with a frame
+that denotes a single thread.
+Example: `./profiler.sh -t 8983`
+
 * `-o fmt[,fmt...]` - specifies what information to dump when profiling ends.
 This is a comma-separated list of the following options:
   - `summary` - dump basic profiling statistics;

profiler.sh

@@ -15,6 +15,7 @@ usage() {
     echo "  -f filename       dump output to <filename>"
     echo "  -i interval       sampling interval in nanoseconds"
     echo "  -b bufsize        frame buffer size"
+    echo "  -t                profile different threads separately"
     echo "  -o fmt[,fmt...]   output format: summary|traces|flat|collapsed"
     echo ""
     echo "<pid> is a numeric process ID of the target JVM"
@@ -74,6 +75,7 @@ FILE=""
 USE_TMP="true"
 INTERVAL=""
 FRAMEBUF=""
+THREADS=""
 OUTPUT="summary,traces=200,flat=200"
 
 while [[ $# -gt 0 ]]; do
@@ -105,6 +107,9 @@ while [[ $# -gt 0 ]]; do
             FRAMEBUF=",framebuf=$2"
             shift
             ;;
+        -t)
+            THREADS=",threads"
+            ;;
         -o)
             OUTPUT="$2"
             shift
@@ -134,7 +139,7 @@ fi
 
 case $ACTION in
     start)
-        jattach start,event=$EVENT,file=$FILE$INTERVAL$FRAMEBUF
+        jattach start,event=$EVENT,file=$FILE$INTERVAL$FRAMEBUF$THREADS
         ;;
     stop)
         jattach stop,file=$FILE,$OUTPUT
@@ -146,7 +151,7 @@ case $ACTION in
         jattach list,file=$FILE
         ;;
     collect)
-        jattach start,event=$EVENT,file=$FILE$INTERVAL$FRAMEBUF
+        jattach start,event=$EVENT,file=$FILE$INTERVAL$FRAMEBUF$THREADS
         sleep $DURATION
         jattach stop,file=$FILE,$OUTPUT
         ;;

src/arguments.cpp

@@ -41,6 +41,7 @@ const Error Error::OK(NULL);
 //     flat[=N]      - dump top N methods (aka flat profile)
 //     interval=N    - sampling interval in ns (default: 1'000'000, i.e. 1 ms)
 //     framebuf=N    - size of the buffer for stack frames (default: 1'000'000)
+//     threads       - profile different threads separately
 //     file=FILENAME - output file name for dumping
 //
 // It is possible to specify multiple dump options at the same time
@@ -89,6 +90,8 @@ Error Arguments::parse(char* args) {
             if (value == NULL || (_framebuf = atoi(value)) <= 0) {
                 return Error("framebuf must be > 0");
             }
+        } else if (strcmp(arg, "threads") == 0) {
+            _threads = true;
         } else if (strcmp(arg, "file") == 0) {
             if (value == NULL || value[0] == 0) {
                 return Error("file must not be empty");

src/arguments.h

@@ -78,6 +78,7 @@ class Arguments {
     const char* _event;
     long _interval;
     int _framebuf;
+    bool _threads;
     char* _file;
     bool _dump_collapsed;
     bool _dump_summary;
@@ -90,6 +91,7 @@ class Arguments {
         _event(EVENT_CPU),
         _interval(0),
         _framebuf(DEFAULT_FRAMEBUF),
+        _threads(false),
         _file(NULL),
         _dump_collapsed(false),
         _dump_summary(false),

src/frameName.cpp

@@ -94,6 +94,10 @@ FrameName::FrameName(ASGCT_CallFrame& frame, bool dotted) {
         VMKlass* alloc_class = (VMKlass*)((uintptr_t)frame.method_id ^ 1);
         _str = strcat(javaClassName(alloc_class), dotted ? " (out)" : "_[k]");
 
+    } else if (frame.bci == BCI_THREAD_ID) {
+        snprintf(_buf, sizeof(_buf), "[thread %d]", (int)(uintptr_t)frame.method_id);
+        _str = _buf;
+
     } else {
         jclass method_class;
         char* class_name = NULL;

src/javaApi.cpp

@@ -30,7 +30,7 @@ static void throw_illegal_state(JNIEnv* env, const char* message) {
 extern "C" JNIEXPORT void JNICALL
 Java_one_profiler_AsyncProfiler_start0(JNIEnv* env, jobject unused, jstring event, jlong interval) {
     const char* event_str = env->GetStringUTFChars(event, NULL);
-    Error error = Profiler::_instance.start(event_str, interval, DEFAULT_FRAMEBUF);
+    Error error = Profiler::_instance.start(event_str, interval, DEFAULT_FRAMEBUF, false);
     env->ReleaseStringUTFChars(event, event_str);
 
     if (error) {

src/perfEvents.h

@@ -32,7 +32,6 @@ class PerfEvents : public Engine {
     static PerfEventType* _event_type;
     static long _interval;
 
-    static int tid();
     static void createForThread(int tid);
     static void createForAllThreads();
     static void destroyForThread(int tid);
@@ -49,8 +48,9 @@ class PerfEvents : public Engine {
     void stop();
 
     static void init();
+    static int tid();
     static const char** getAvailableEvents();
-    static int getCallChain(const void** callchain, int max_depth);
+    static int getCallChain(int tid, const void** callchain, int max_depth);
 
     static void JNICALL ThreadStart(jvmtiEnv* jvmti, JNIEnv* jni, jthread thread) {
         createForThread(tid());

src/perfEvents_linux.cpp

@@ -318,8 +318,8 @@ const char** PerfEvents::getAvailableEvents() {
     return available_events;
 }
 
-int PerfEvents::getCallChain(const void** callchain, int max_depth) {
-    PerfEvent* event = &_events[tid()];
+int PerfEvents::getCallChain(int tid, const void** callchain, int max_depth) {
+    PerfEvent* event = &_events[tid];
     if (!event->tryLock()) {
         return 0;  // the event is being destroyed
     }

src/perfEvents_macos.cpp

@@ -18,6 +18,7 @@
 
 #include <string.h>
 #include <sys/time.h>
+#include <pthread.h>
 #include "perfEvents.h"
 #include "profiler.h"
 
@@ -30,7 +31,9 @@ long PerfEvents::_interval;
 
 void PerfEvents::init() {}
 
-int PerfEvents::tid() { return 0; }
+int PerfEvents::tid() {
+    return pthread_mach_thread_np(pthread_self());
+}
 
 void PerfEvents::createForThread(int tid)  {}
 void PerfEvents::createForAllThreads()     {}
@@ -84,7 +87,7 @@ const char** PerfEvents::getAvailableEvents() {
     return available_events;
 }
 
-int PerfEvents::getCallChain(const void** callchain, int max_depth) {
+int PerfEvents::getCallChain(int tid, const void** callchain, int max_depth) {
     return 0;
 }
 

src/profiler.cpp

@@ -181,9 +181,9 @@ const char* Profiler::findNativeMethod(const void* address) {
     return NULL;
 }
 
-int Profiler::getNativeTrace(void* ucontext, ASGCT_CallFrame* frames) {
+int Profiler::getNativeTrace(int tid, ASGCT_CallFrame* frames) {
     const void* native_callchain[MAX_NATIVE_FRAMES];
-    int native_frames = PerfEvents::getCallChain(native_callchain, MAX_NATIVE_FRAMES);
+    int native_frames = PerfEvents::getCallChain(tid, native_callchain, MAX_NATIVE_FRAMES);
 
     for (int i = 0; i < native_frames; i++) {
         const void* address = native_callchain[i];
@@ -300,15 +300,20 @@ void Profiler::recordSample(void* ucontext, u64 counter, jint event_type, jmetho
 
     ASGCT_CallFrame* frames = _calltrace_buffer[lock_index]._asgct_frames;
     int num_frames;
+    int tid = PerfEvents::tid();
 
     if (event == NULL) {
-        num_frames = getNativeTrace(ucontext, frames);
-        num_frames += getJavaTraceAsync(ucontext, frames + num_frames, MAX_STACK_FRAMES - num_frames);
+        num_frames = getNativeTrace(tid, frames);
+        num_frames += getJavaTraceAsync(ucontext, frames + num_frames, MAX_STACK_FRAMES - 1 - num_frames);
     } else {
         // Events like object allocation happen at known places where it is safe to call JVM TI
         jvmtiFrameInfo* jvmti_frames = _calltrace_buffer[lock_index]._jvmti_frames;
         num_frames = makeEventFrame(frames, event_type, event);
-        num_frames += getJavaTraceJVMTI(jvmti_frames + num_frames, frames + num_frames, MAX_STACK_FRAMES - num_frames);
+        num_frames += getJavaTraceJVMTI(jvmti_frames + num_frames, frames + num_frames, MAX_STACK_FRAMES - 1 - num_frames);
+    }
+
+    if (_threads) {
+        num_frames += makeEventFrame(frames + num_frames, BCI_THREAD_ID, (jmethodID)(uintptr_t)tid);
     }
 
     if (num_frames > 0) {
@@ -326,7 +331,7 @@ void Profiler::resetSymbols() {
     _native_lib_count = Symbols::parseMaps(_native_libs, MAX_NATIVE_LIBS);
 }
 
-Error Profiler::start(const char* event, long interval, int frame_buffer_size) {
+Error Profiler::start(const char* event, long interval, int frame_buffer_size, bool threads) {
     MutexLocker ml(_state_lock);
     if (_state != IDLE) {
         return Error("Profiler already started");
@@ -349,6 +354,7 @@ Error Profiler::start(const char* event, long interval, int frame_buffer_size) {
     _frame_buffer = (ASGCT_CallFrame*)malloc(_frame_buffer_size * sizeof(ASGCT_CallFrame));
     _frame_buffer_index = 0;
     _frame_buffer_overflow = false;
+    _threads = threads;
 
     resetSymbols();
 
@@ -512,7 +518,7 @@ void Profiler::dumpFlat(std::ostream& out, int max_methods) {
 void Profiler::runInternal(Arguments& args, std::ostream& out) {
     switch (args._action) {
         case ACTION_START: {
-            Error error = start(args._event, args._interval, args._framebuf);
+            Error error = start(args._event, args._interval, args._framebuf, args._threads);
             if (error) {
                 out << error.message() << std::endl;
             } else {

src/profiler.h

@@ -135,6 +135,7 @@ class Profiler {
     int _frame_buffer_size;
     volatile int _frame_buffer_index;
     bool _frame_buffer_overflow;
+    bool _threads;
 
     SpinLock _jit_lock;
     const void* _jit_min_address;
@@ -150,7 +151,7 @@ class Profiler {
     void updateJitRange(const void* min_address, const void* max_address);
 
     const char* findNativeMethod(const void* address);
-    int getNativeTrace(void* ucontext, ASGCT_CallFrame* frames);
+    int getNativeTrace(int tid, ASGCT_CallFrame* frames);
     int getJavaTraceAsync(void* ucontext, ASGCT_CallFrame* frames, int max_depth);
     int getJavaTraceJVMTI(jvmtiFrameInfo* jvmti_frames, ASGCT_CallFrame* frames, int max_depth);
     int makeEventFrame(ASGCT_CallFrame* frames, jint event_type, jmethodID event);
@@ -184,7 +185,7 @@ class Profiler {
     time_t uptime()     { return time(NULL) - _start_time; }
 
     void run(Arguments& args);
-    Error start(const char* event, long interval, int frame_buffer_size);
+    Error start(const char* event, long interval, int frame_buffer_size, bool threads);
     Error stop();
     void dumpSummary(std::ostream& out);
     void dumpCollapsed(std::ostream& out, Counter counter);

src/vmEntry.h

@@ -25,6 +25,7 @@ enum ASGCT_CallFrameType {
     BCI_NATIVE_FRAME       = -10,  // method_id is native function name (char*)
     BCI_KLASS              = -11,  // method_id is Klass descriptor (VMKlass*)
     BCI_KLASS_OUTSIDE_TLAB = -12,  // VMKlass* specifically for allocations outside TLAB
+    BCI_THREAD_ID          = -13,  // method_id designates a thread
 };
 
 typedef struct {