Differential Flame Graphs (#1553)

.assets/html/flamegraph.html

@@ -75,9 +75,11 @@
 	// SPDX-License-Identifier: Apache-2.0
 	'use strict';
 	let root, px, pattern;
-	let level0 = 0, left0 = 0, width0 = 0;
+	let level0 = 0, left0 = 0, width0 = 0, d = 0;
 	let nav = [], navIndex, matchval;
 	let inverted = false;
+	const U = undefined;
+	const maxdiff = -1;
 	const levels = Array(36);
 	for (let h = 0; h < levels.length; h++) {
 		levels[h] = [];
@@ -111,10 +113,18 @@
 		return '#' + (p[0] + ((p[1] * v) << 16 | (p[2] * v) << 8 | (p[3] * v))).toString(16);
 	}
 
+	function getDiffColor(diff) {
+		if (diff === U) return '#ffdd33';
+		if (diff === 0) return '#e0e0e0';
+		const v = Math.round(128 * (maxdiff - Math.abs(diff)) / maxdiff) + 96;
+		return diff > 0 ? 'rgb(255,' + v + ',' + v + ')' : 'rgb(' + v + ',' + v + ',255)';
+	}
+
 	function f(key, level, left, width, inln, c1, int) {
 		levels[level0 = level].push({level, left: left0 += left, width: width0 = width || width0,
-			color: getColor(palette[key & 7]), title: cpool[key >>> 3],
+			color: maxdiff >= 0 ? getDiffColor(d) : getColor(palette[key & 7]),
-			details: (int ? ', int=' + int : '') + (c1 ? ', c1=' + c1 : '') + (inln ? ', inln=' + inln : '')
+			title: cpool[key >>> 3],
+			details: (d ? (d > 0 ? ', +' : ', ') + d : '') + (int ? ', int=' + int : '') + (c1 ? ', c1=' + c1 : '') + (inln ? ', inln=' + inln : '')
 		});
 	}
 

.licenserc.yaml

@@ -15,6 +15,7 @@ header:
       - 'src/jattach'
       - 'src/res'
       - '**/MANIFEST.MF'
+      - 'test/**/*.collapsed'
     license:
       content: |
         Copyright The async-profiler authors

docs/ConverterUsage.md

@@ -43,6 +43,8 @@ Conversion options:
 
   # otlp: OpenTelemetry profile format.
 
+Differential Flame Graph:
+  --diff <base-profile> <new-profile>
 
 JFR options:
     --cpu              Generate only CPU profile during conversion
@@ -120,7 +122,7 @@ jfrconv --cpu foo.jfr
 
 for HTML output as HTML is the default format for conversion from JFR.
 
-#### Flame Graph options
+### Flame Graph options
 
 To add a custom title to the generated Flame Graph, use `--title`, which has the default value `Flame Graph`:
 
@@ -128,9 +130,37 @@ To add a custom title to the generated Flame Graph, use `--title`, which has the
 jfrconv --cpu foo.jfr foo.html -r --title "Custom Title"
 ```
 
-### Other formats
+### Differential Flame Graph
 
-`jfrconv` supports converting a JFR file to `collapsed`, `pprof`, `pb.gz` and `heatmap` formats as well.
+To find performance regressions, it may be useful to compare current profile
+to a previous one that serves as a baseline. Differential Flame Graph
+visualizes such a comparsion with a special color scheme:
+
+- Red color denotes frames with more samples comparing to the baseline (i.e. regression);
+- Blue is for frames with less samples;
+- Yellow are new frames that were absent in the baseline.
+
+The more intense the color, the larger the delta.
+For each different frame, the delta value is displayed in a tooltip.
+
+![](/.assets/images/flamegraph_diff.png)
+
+Differential Flame Graph takes the shape of the current profile:
+all frames have exactly the same size as in the normal Flame Graph.
+This means, frames that exist only in the base profile will not be visible.
+To see such frames, create another differential Flame Graph,
+swapping the base and the current input file.
+
+To create differential Flame Graph, run `jfrconv --diff` with two input files:
+basline profile and new profile. Both files can be in JFR, HTML, or collapsed format.
+Other converter options work as usual.
+
+```
+jfrconv --cpu --diff baseline.jfr new.jfr diff.html
+```
+
+Output file name is optional. If omitted, `jfrconv` takes the name
+of the second input file, replacing its extension with `.diff.html`.
 
 ## Standalone converter examples
 

src/converter/one/convert/Arguments.java

@@ -24,6 +24,7 @@ public class Arguments {
     public boolean help;
     public boolean reverse;
     public boolean inverted;
+    public boolean diff;
     public boolean cpu;
     public boolean cpuTime;
     public boolean wall;

src/converter/one/convert/FlameGraph.java

@@ -20,6 +20,7 @@ public class FlameGraph implements Comparator<Frame> {
     private static final String[] FRAME_SUFFIX = {"_[0]", "_[j]", "_[i]", "", "", "_[k]", "_[1]"};
     private static final byte HAS_SUFFIX = (byte) 0x80;
     private static final int FLUSH_THRESHOLD = 15000;
+    private static final long NEW_FRAME_DIFF = Long.MIN_VALUE;
     private static final Pattern TID_FRAME_PATTERN = Pattern.compile("\\[(.* )?tid=\\d+]");
 
     private final Arguments args;
@@ -29,11 +30,14 @@ public class FlameGraph implements Comparator<Frame> {
 
     private String title = "Flame Graph";
     private int[] order;
+    private int[] cpoolMap;
     private int depth;
     private int lastLevel;
     private long lastX;
     private long lastTotal;
+    private long lastDiff;
     private long mintotal;
+    private long maxdiff = -1;
 
     public FlameGraph(Arguments args) {
         this.args = args;
@@ -90,6 +94,8 @@ public class FlameGraph implements Comparator<Frame> {
             while (!br.readLine().isEmpty()) ;
 
             for (String line; !(line = br.readLine()).isEmpty(); ) {
+                if (line.startsWith("d=")) continue;  // artifact of a differential flame graph
+
                 StringTokenizer st = new StringTokenizer(line.substring(2, line.length() - 1), ",");
                 int nameAndType = Integer.parseInt(st.nextToken());
 
@@ -109,12 +115,10 @@ public class FlameGraph implements Comparator<Frame> {
 
                 int titleIndex = nameAndType >>> 3;
                 byte type = (byte) (nameAndType & 7);
-                if (st.hasMoreTokens() && (type <= TYPE_INLINED || type >= TYPE_C1_COMPILED)) {
+                byte normalizedType = type <= TYPE_INLINED || type >= TYPE_C1_COMPILED ? TYPE_JIT_COMPILED : type;
-                    type = TYPE_JIT_COMPILED;
-                }
 
-                Frame f = level > 0 || needRebuild ? new Frame(titleIndex, type) : root;
+                Frame f = level > 0 || needRebuild ? new Frame(titleIndex, normalizedType) : root;
-                f.self = f.total = total;
+                fillFrameCounters(f, type, total);
                 if (st.hasMoreTokens()) f.inlined = Long.parseLong(st.nextToken());
                 if (st.hasMoreTokens()) f.c1 = Long.parseLong(st.nextToken());
                 if (st.hasMoreTokens()) f.interpreted = Long.parseLong(st.nextToken());
@@ -177,6 +181,26 @@ public class FlameGraph implements Comparator<Frame> {
         depth = Math.max(depth, stack.size);
     }
 
+    public void diff(FlameGraph base) {
+        // Build a map that translates this cpool keys to the base flamegraph's cpool keys
+        cpoolMap = Arrays.stream(cpool.keys()).mapToInt(title -> base.cpool.getOrDefault(title, -1)).toArray();
+        diff(base.root, root);
+    }
+
+    private void diff(Frame base, Frame current) {
+        current.diff = base == null ? NEW_FRAME_DIFF : current.self - base.self;
+        maxdiff = Math.max(maxdiff, Math.abs(current.diff));
+
+        for (Frame child : current.values()) {
+            Frame baseChild = base == null ? null : base.get(translateKey(child.key));
+            diff(baseChild, child);
+        }
+    }
+
+    private int translateKey(int key) {
+        return cpoolMap[key & TITLE_MASK] | (key & ~TITLE_MASK);
+    }
+
     public void dump(OutputStream out) throws IOException {
         try (PrintStream ps = new PrintStream(out, false, "UTF-8")) {
             dump(ps);
@@ -205,6 +229,9 @@ public class FlameGraph implements Comparator<Frame> {
         tail = printTill(out, tail, "/*inverted:*/false");
         out.print(args.reverse ^ args.inverted);
 
+        tail = printTill(out, tail, "/*maxdiff:*/-1");
+        out.print(maxdiff);
+
         tail = printTill(out, tail, "/*depth:*/0");
         out.print(depth);
 
@@ -239,6 +266,15 @@ public class FlameGraph implements Comparator<Frame> {
     }
 
     private void printFrame(PrintStream out, Frame frame, int level, long x) {
+        StringBuilder sb = outbuf;
+        if (frame.diff != lastDiff) {
+            if (frame.diff == NEW_FRAME_DIFF) {
+                sb.append("d=U\n");
+            } else {
+                sb.append("d=").append(frame.diff).append('\n');
+            }
+        }
+
         int nameAndType = order[frame.getTitleIndex()] << 3 | frame.getType();
         boolean hasExtraTypes = (frame.inlined | frame.c1 | frame.interpreted) != 0 &&
                 frame.inlined < frame.total && frame.interpreted < frame.total;
@@ -250,7 +286,7 @@ public class FlameGraph implements Comparator<Frame> {
             func = 'n';
         }
 
-        StringBuilder sb = outbuf.append(func).append('(').append(nameAndType);
+        sb.append(func).append('(').append(nameAndType);
         if (func == 'f') {
             sb.append(',').append(level).append(',').append(x - lastX);
         }
@@ -270,6 +306,7 @@ public class FlameGraph implements Comparator<Frame> {
         lastLevel = level;
         lastX = x;
         lastTotal = frame.total;
+        lastDiff = frame.diff;
 
         Frame[] children = frame.values().toArray(EMPTY_FRAME_ARRAY);
         Arrays.sort(children, this);
@@ -291,6 +328,9 @@ public class FlameGraph implements Comparator<Frame> {
             sb.append(strings[frame.getTitleIndex()]).append(FRAME_SUFFIX[frame.getType()]);
             if (frame.self > 0) {
                 int tmpLength = sb.length();
+                if (maxdiff >= 0) {
+                    sb.append(' ').append(frame.diff == NEW_FRAME_DIFF ? 0 : frame.self - frame.diff);
+                }
                 out.print(sb.append(' ').append(frame.self).append('\n'));
                 sb.setLength(tmpLength);
             }
@@ -328,6 +368,21 @@ public class FlameGraph implements Comparator<Frame> {
         return include != null;
     }
 
+    private static void fillFrameCounters(Frame frame, byte type, long ticks) {
+        frame.self = frame.total = ticks;
+        switch (type) {
+            case TYPE_INTERPRETED:
+                frame.interpreted = ticks;
+                break;
+            case TYPE_INLINED:
+                frame.inlined = ticks;
+                break;
+            case TYPE_C1_COMPILED:
+                frame.c1 = ticks;
+                break;
+        }
+    }
+
     private Frame addChild(Frame frame, String title, byte type, long ticks) {
         frame.total += ticks;
 

src/converter/one/convert/Frame.java

@@ -16,11 +16,13 @@ public class Frame extends HashMap<Integer, Frame> {
     public static final byte TYPE_KERNEL = 5;
     public static final byte TYPE_C1_COMPILED = 6;
 
-    private static final int TYPE_SHIFT = 28;
+    static final int TYPE_SHIFT = 28;
+    static final int TITLE_MASK = (1 << TYPE_SHIFT) - 1;
 
     final int key;
     long total;
     long self;
+    long diff;
     long inlined, c1, interpreted;
 
     private Frame(int key) {
@@ -36,7 +38,7 @@ public class Frame extends HashMap<Integer, Frame> {
     }
 
     int getTitleIndex() {
-        return key & ((1 << TYPE_SHIFT) - 1);
+        return key & TITLE_MASK;
     }
 
     byte getType() {

src/converter/one/convert/Main.java

@@ -7,6 +7,7 @@ package one.convert;
 
 import java.io.File;
 import java.io.FileInputStream;
+import java.io.FileOutputStream;
 import java.io.IOException;
 
 public class Main {
@@ -18,7 +19,7 @@ public class Main {
             return;
         }
 
-        if (args.files.size() == 1) {
+        if (args.files.size() == (args.diff ? 2 : 1)) {
             args.files.add(".");
         }
 
@@ -35,6 +36,34 @@ public class Main {
             }
         }
 
+        if (args.diff) {
+            if (fileCount != 2) {
+                throw new IllegalArgumentException("--diff option requires two input files");
+            }
+            if (!"html".equals(args.output) && !"collapsed".equals(args.output)) {
+                throw new IllegalArgumentException("--diff option requires html or collapsed output format");
+            }
+
+            args.norm = true;  // don't let random IDs in class names spoil comparison
+
+            String input1 = args.files.get(0);
+            String input2 = args.files.get(1);
+            String output = isDirectory ? new File(lastFile, replaceExt(input2, "diff." + args.output)).getPath() : lastFile;
+
+            System.out.print("Converting " + getFileName(input2) + " vs " + getFileName(input1) + " -> " + getFileName(output) + " ");
+            System.out.flush();
+
+            long startTime = System.nanoTime();
+            FlameGraph base = parseFlameGraph(input1, args);
+            FlameGraph current = parseFlameGraph(input2, args);
+            current.diff(base);
+            current.dump(new FileOutputStream(output));
+            long endTime = System.nanoTime();
+
+            System.out.print("# " + (endTime - startTime) / 1000000 / 1000.0 + " s\n");
+            return;
+        }
+
         for (int i = 0; i < fileCount; i++) {
             String input = args.files.get(i);
             String output = isDirectory ? new File(lastFile, replaceExt(input, args.output)).getPath() : lastFile;
@@ -106,6 +135,7 @@ public class Main {
                 "  -o --output FORMAT    Output format: html, collapsed, pprof, pb.gz, heatmap, otlp\n" +
                 "  -I --include REGEX    Include only stacks with the specified frames\n" +
                 "  -X --exclude REGEX    Exclude stacks with the specified frames\n" +
+                "     --diff             Create differential Flame Graph from two input files\n" +
                 "\n" +
                 "JFR options:\n" +
                 "     --cpu              CPU profile (ExecutionSample)\n" +

src/res/flame.html

@@ -75,9 +75,11 @@
 	// SPDX-License-Identifier: Apache-2.0
 	'use strict';
 	let root, px, pattern;
-	let level0 = 0, left0 = 0, width0 = 0;
+	let level0 = 0, left0 = 0, width0 = 0, d = 0;
 	let nav = [], navIndex, matchval;
 	let inverted = /*inverted:*/false;
+	const U = undefined;
+	const maxdiff = /*maxdiff:*/-1;
 	const levels = Array(/*depth:*/0);
 	for (let h = 0; h < levels.length; h++) {
 		levels[h] = [];
@@ -111,10 +113,18 @@
 		return '#' + (p[0] + ((p[1] * v) << 16 | (p[2] * v) << 8 | (p[3] * v))).toString(16);
 	}
 
+	function getDiffColor(diff) {
+		if (diff === U) return '#ffdd33';
+		if (diff === 0) return '#e0e0e0';
+		const v = Math.round(128 * (maxdiff - Math.abs(diff)) / maxdiff) + 96;
+		return diff > 0 ? 'rgb(255,' + v + ',' + v + ')' : 'rgb(' + v + ',' + v + ',255)';
+	}
+
 	function f(key, level, left, width, inln, c1, int) {
 		levels[level0 = level].push({level, left: left0 += left, width: width0 = width || width0,
-			color: getColor(palette[key & 7]), title: cpool[key >>> 3],
+			color: maxdiff >= 0 ? getDiffColor(d) : getColor(palette[key & 7]),
-			details: (int ? ', int=' + int : '') + (c1 ? ', c1=' + c1 : '') + (inln ? ', inln=' + inln : '')
+			title: cpool[key >>> 3],
+			details: (d ? (d > 0 ? ', +' : ', ') + d : '') + (int ? ', int=' + int : '') + (c1 ? ', c1=' + c1 : '') + (inln ? ', inln=' + inln : '')
 		});
 	}
 

test/test/jfrconverter/JfrconverterTests.java

@@ -5,13 +5,19 @@
 
 package test.jfrconverter;
 
-import test.otlp.CpuBurner;
 import one.convert.*;
 import one.jfr.JfrReader;
+import one.jfr.StackTrace;
 import one.jfr.event.Event;
 import one.jfr.event.EventCollector;
-import one.jfr.StackTrace;
+import one.profiler.test.Output;
-import one.profiler.test.*;
+import one.profiler.test.Test;
+import one.profiler.test.TestProcess;
+import test.otlp.CpuBurner;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.Arrays;
 
 // Simple smoke tests for JFR converter. The output is not inspected for errors,
 // we only verify that the conversion completes successfully.
@@ -72,4 +78,33 @@ public class JfrconverterTests {
             assert !found[3];
         }
     }
+
+    @Test(mainClass = Main.class, args = "--diff test/test/jfrconverter/sample1.collapsed test/test/jfrconverter/sample2.collapsed %diff.collapsed")
+    public void diffCollapsed(TestProcess p) throws Exception {
+        Output out = p.waitForExit("%diff");
+        assert out.containsExact("BusyClient.run_[j] 4 1");
+        assert out.containsExact("BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j] 2 2");
+        assert out.containsExact("ByteBuffer.get_[i];ByteBuffer.getArray_[i] 0 1");
+        assert out.samples("ByteBuffer.get") == 2;
+    }
+
+    @Test(mainClass = Main.class, args = "--diff test/test/jfrconverter/sample1.collapsed test/test/jfrconverter/sample2.collapsed %diff.html")
+    public void diffHtml(TestProcess p) throws Exception {
+        Output out = p.waitForExit("%diff");
+        assert out.containsExact("d=-3");
+        assert out.containsExact("d=0");
+        assert out.containsExact("d=U");
+
+        // It should be possible to reconstruct original FlameGraph from the differential one
+        byte[] original = buildFlameGraph("test/test/jfrconverter/sample2.collapsed");
+        byte[] reconstructed = buildFlameGraph(p.getFilePath("%diff"));
+        assert Arrays.equals(original, reconstructed);
+    }
+
+    private static byte[] buildFlameGraph(String input) throws IOException {
+        FlameGraph fg = FlameGraph.parse(input, new Arguments());
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        fg.dump(baos);
+        return baos.toByteArray();
+    }
 }

test/test/jfrconverter/sample1.collapsed

@@ -0,0 +1,17 @@
+BusyClient.run_[j] 4
+BusyClient.run_[j];InputStream.read_[j] 1
+BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j] 2
+BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j] 3
+BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j] 1
+BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j];NioSocketImpl.implRead_[j];NioSocketImpl.beginRead_[i] 1
+BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j];NioSocketImpl.implRead_[j];NioSocketImpl.beginRead_[i];NativeThread.current_[i];NativeThread.current0_[j] 3
+BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j];NioSocketImpl.implRead_[j];NioSocketImpl.endRead_[i] 3
+BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j];NioSocketImpl.implRead_[j];NioSocketImpl.tryRead_[j] 1
+BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j];NioSocketImpl.implRead_[j];NioSocketImpl.tryRead_[j];Util.getTemporaryDirectBuffer_[i];CarrierThreadLocal.get_[i];System$2.getCarrierThreadLocal_[i];ThreadLocal.getCarrierThreadLocal_[i];jlong_disjoint_arraycopy 15
+BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j];NioSocketImpl.implRead_[j];NioSocketImpl.tryRead_[j];Util.getTemporaryDirectBuffer_[i];Util$BufferCache.get_[i];Buffer.capacity_[i] 1
+BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j];NioSocketImpl.implRead_[j];NioSocketImpl.tryRead_[j];SocketDispatcher.read_[i] 1
+BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j];NioSocketImpl.implRead_[j];NioSocketImpl.tryRead_[j];SocketDispatcher.read_[i];SocketDispatcher.read0_[j] 3
+BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j];NioSocketImpl.implRead_[j];NioSocketImpl.tryRead_[j];SocketDispatcher.read_[i];SocketDispatcher.read0_[j];Java_sun_nio_ch_SocketDispatcher_read0;read 143
+BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j];ReentrantLock.lock_[i];ReentrantLock$Sync.lock_[i];ReentrantLock$NonfairSync.initialTryLock_[i] 1
+BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j];ReentrantLock.lock_[i];ReentrantLock$Sync.lock_[i];ReentrantLock$NonfairSync.initialTryLock_[i];AbstractQueuedSynchronizer.compareAndSetState_[i] 1
+BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j];ReentrantLock.unlock_[i];AbstractQueuedSynchronizer.release_[i];ReentrantLock$Sync.tryRelease_[i];AbstractQueuedSynchronizer.setState_[i] 1

test/test/jfrconverter/sample2.collapsed

@@ -0,0 +1,17 @@
+BusyClient.run_[j] 1
+BusyClient.run_[j];InputStream.read_[j] 1
+BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j] 2
+BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j] 1
+BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j] 1
+BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j];NioSocketImpl.implRead_[j] 3
+BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j];NioSocketImpl.implRead_[j];NioSocketImpl.beginRead_[i] 3
+BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j];NioSocketImpl.implRead_[j];NioSocketImpl.beginRead_[i];NativeThread.current_[i];NativeThread.current0_[j] 4
+BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j];NioSocketImpl.implRead_[j];NioSocketImpl.endRead_[i] 4
+BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j];NioSocketImpl.implRead_[j];NioSocketImpl.tryRead_[j];ByteBuffer.get_[i];ByteBuffer.getArray_[i] 1
+BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j];NioSocketImpl.implRead_[j];NioSocketImpl.tryRead_[j];ByteBuffer.get_[i];Buffer.position_[i] 1
+BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j];NioSocketImpl.implRead_[j];NioSocketImpl.tryRead_[j];Util.getTemporaryDirectBuffer_[i];CarrierThreadLocal.get_[i];System$2.getCarrierThreadLocal_[i];ThreadLocal.getCarrierThreadLocal_[i];jlong_disjoint_arraycopy 6
+BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j];NioSocketImpl.implRead_[j];NioSocketImpl.tryRead_[j];SocketDispatcher.read_[i];SocketDispatcher.read0_[j] 4
+BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j];NioSocketImpl.implRead_[j];NioSocketImpl.tryRead_[j];SocketDispatcher.read_[i];SocketDispatcher.read0_[j];Java_sun_nio_ch_SocketDispatcher_read0 1
+BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j];NioSocketImpl.implRead_[j];NioSocketImpl.tryRead_[j];SocketDispatcher.read_[i];SocketDispatcher.read0_[j];Java_sun_nio_ch_SocketDispatcher_read0;read 151
+BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j];ReentrantLock.lock_[i] 1
+BusyClient.run_[j];InputStream.read_[j];Socket$SocketInputStream.read_[j];NioSocketImpl$1.read_[j];NioSocketImpl.read_[j];ReentrantLock.unlock_[i];AbstractQueuedSynchronizer.release_[i];ReentrantLock$Sync.tryRelease_[i];AbstractQueuedSynchronizer.setState_[i] 3