diff --git a/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp b/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp index 127ac9f6951..f3ef12d4b95 100644 --- a/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp +++ b/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp @@ -888,50 +888,127 @@ class StubGenerator: public StubCodeGenerator { bool is_backward = step < 0; int granularity = g_uabs(step); - const Register src = x30, dst = x31, vl = x14, cnt = x15, tmp1 = x16, tmp2 = x17; - assert_different_registers(s, d, cnt, vl, tmp1, tmp2); + const Register src = x30; + const Register dst = x31; + const Register cnt = x15; + const Register saved_vl = x16; + const Register bytes_per_iter = x17; + const Register tmp0 = x5; + + assert_different_registers(s, d, cnt, saved_vl, bytes_per_iter, tmp0); Assembler::SEW sew = Assembler::elembytes_to_sew(granularity); - Label loop_forward, loop_backward, done; + + Label small_copy, forward_main, backward_main, loop_fwd, loop_bwd, + tail_fwd, tail_bwd, done; __ mv(dst, d); __ mv(src, s); __ mv(cnt, count); - __ bind(loop_forward); - __ vsetvli(vl, cnt, sew, Assembler::m8); - if (is_backward) { - __ bne(vl, cnt, loop_backward); + /* small copy path (≤ 32 bytes) */ + const int max_elems_small = 32 / granularity; + if (max_elems_small > 0) { + __ li(tmp0, max_elems_small); + __ bleu(cnt, tmp0, small_copy); } - __ vlex_v(v0, src, sew); - __ sub(cnt, cnt, vl); - if (sew != Assembler::e8) { - // when sew == e8 (e.g., elem size is 1 byte), slli R, R, 0 is a nop and unnecessary - __ slli(vl, vl, sew); - } - __ add(src, src, vl); - - __ vsex_v(v0, dst, sew); - __ add(dst, dst, vl); - __ bnez(cnt, loop_forward); - if (is_backward) { + __ j(backward_main); + } else { + __ j(forward_main); + } + + /* small copy */ + if (max_elems_small > 0) { + __ bind(small_copy); + __ vsetvli(tmp0, cnt, sew, Assembler::m8); + __ vlex_v(v0, src, sew); + __ vsex_v(v0, dst, sew); __ j(done); - - __ bind(loop_backward); - __ sub(t0, cnt, vl); - if (sew != Assembler::e8) { - // when sew == e8 (e.g., elem size is 1 byte), slli R, R, 0 is a nop and unnecessary - __ slli(t0, t0, sew); - } - __ add(tmp1, s, t0); - __ vlex_v(v0, tmp1, sew); - __ add(tmp2, d, t0); - __ vsex_v(v0, tmp2, sew); - __ sub(cnt, cnt, vl); - __ bnez(cnt, loop_forward); - __ bind(done); } + + /* forward copy path */ + __ bind(forward_main); + __ vsetvli(saved_vl, cnt, sew, Assembler::m8); + + if (granularity > 1) { + __ slli(bytes_per_iter, saved_vl, exact_log2(granularity)); + } else { + __ mv(bytes_per_iter, saved_vl); + } + + /* forward copy */ + __ bind(loop_fwd); + { + __ blt(cnt, saved_vl, tail_fwd); + __ vlex_v(v0, src, sew); + __ vsex_v(v0, dst, sew); + __ add(src, src, bytes_per_iter); + __ add(dst, dst, bytes_per_iter); + __ sub(cnt, cnt, saved_vl); + __ bnez(cnt, loop_fwd); + } + + /* forward tail */ + __ bind(tail_fwd); + { + __ beqz(cnt, done); + __ vsetvli(tmp0, cnt, sew, Assembler::m8); + __ vlex_v(v0, src, sew); + __ vsex_v(v0, dst, sew); + __ j(done); + } + + /* backward copy path */ + __ bind(backward_main); + + if (granularity > 1) { + __ slli(tmp0, count, exact_log2(granularity)); + } else { + __ mv(tmp0, count); + } + __ add(src, src, tmp0); + __ add(dst, dst, tmp0); + + __ vsetvli(saved_vl, cnt, sew, Assembler::m8); + + if (granularity > 1) { + __ slli(bytes_per_iter, saved_vl, exact_log2(granularity)); + } else { + __ mv(bytes_per_iter, saved_vl); + } + + /* backward copy */ + __ bind(loop_bwd); + { + __ blt(cnt, saved_vl, tail_bwd); + __ vlex_v(v0, src, sew); + __ vsex_v(v0, dst, sew); + __ sub(src, src, bytes_per_iter); + __ sub(dst, dst, bytes_per_iter); + __ sub(cnt, cnt, saved_vl); + __ bnez(cnt, loop_bwd); + } + + /* backward tail */ + __ bind(tail_bwd); + { + __ beqz(cnt, done); + if (granularity > 1) { + __ slli(bytes_per_iter, cnt, exact_log2(granularity)); + } else { + __ mv(bytes_per_iter, cnt); + } + + __ vsetvli(tmp0, cnt, sew, Assembler::m8); + __ sub(src, src, bytes_per_iter); + __ sub(dst, dst, bytes_per_iter); + __ vlex_v(v0, src, sew); + __ vsex_v(v0, dst, sew); + } + + /* end */ + __ bind(done); } // All-singing all-dancing memory copy.