Adjust format

Reviewed-by: kevinw

doc/testing.html

@@ -450,7 +450,7 @@ itself (<code>-timeoutFactor</code>). Also, some test cases that
 programmatically wait a certain amount of time will apply this factor.
 If we run in forced compilation mode (<code>-Xcomp</code>), the build
 system will automatically adjust this factor to compensate for less
-performance. Defaults to 1.</p>
+performance. Defaults to 4.</p>
 <h4 id="failure_handler_timeout">FAILURE_HANDLER_TIMEOUT</h4>
 <p>Sets the argument <code>-timeoutHandlerTimeout</code> for JTReg. The
 default value is 0. This is only valid if the failure handler is

doc/testing.md

@@ -387,7 +387,7 @@ The `TIMEOUT_FACTOR` is forwarded to JTReg framework itself
 (`-timeoutFactor`). Also, some test cases that programmatically wait a
 certain amount of time will apply this factor. If we run in forced
 compilation mode (`-Xcomp`), the build system will automatically
-adjust this factor to compensate for less performance. Defaults to 1.
+adjust this factor to compensate for less performance. Defaults to 4.
 
 #### FAILURE_HANDLER_TIMEOUT
 

make/RunTests.gmk

@@ -946,8 +946,8 @@ define SetupRunJtregTestBody
   JTREG_ALL_OPTIONS := $$(JTREG_JAVA_OPTIONS) $$(JTREG_VM_OPTIONS)
 
   JTREG_AUTO_PROBLEM_LISTS :=
-  # Please reach consensus before changing this. It was not easy changing it to a `1`.
-  JTREG_AUTO_TIMEOUT_FACTOR := 1
+  # Please reach consensus before changing this.
+  JTREG_AUTO_TIMEOUT_FACTOR := 4
 
   ifneq ($$(findstring -Xcomp, $$(JTREG_ALL_OPTIONS)), )
     JTREG_AUTO_PROBLEM_LISTS += ProblemList-Xcomp.txt

make/conf/github-actions.conf

@@ -29,21 +29,21 @@ GTEST_VERSION=1.14.0
 JTREG_VERSION=8.1+1
 
 LINUX_X64_BOOT_JDK_EXT=tar.gz
-LINUX_X64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk24/1f9ff9062db4449d8ca828c504ffae90/36/GPL/openjdk-24_linux-x64_bin.tar.gz
-LINUX_X64_BOOT_JDK_SHA256=88b090fa80c6c1d084ec9a755233967458788e2c0777ae2e172230c5c692d7ef
+LINUX_X64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk25/bd75d5f9689641da8e1daabeccb5528b/36/GPL/openjdk-25_linux-x64_bin.tar.gz
+LINUX_X64_BOOT_JDK_SHA256=59cdcaf255add4721de38eb411d4ecfe779356b61fb671aee63c7dec78054c2b
 
 ALPINE_LINUX_X64_BOOT_JDK_EXT=tar.gz
-ALPINE_LINUX_X64_BOOT_JDK_URL=https://github.com/adoptium/temurin24-binaries/releases/download/jdk-24%2B36/OpenJDK24U-jdk_x64_alpine-linux_hotspot_24_36.tar.gz
-ALPINE_LINUX_X64_BOOT_JDK_SHA256=a642608f0da78344ee6812fb1490b8bc1d7ad5a18064c70994d6f330568c51cb
+ALPINE_LINUX_X64_BOOT_JDK_URL=https://github.com/adoptium/temurin25-binaries/releases/download/jdk-25%2B36/OpenJDK25U-jdk_x64_alpine-linux_hotspot_25_36.tar.gz
+ALPINE_LINUX_X64_BOOT_JDK_SHA256=637e47474d411ed86134f413af7d5fef4180ddb0bf556347b7e74a88cf8904c8
 
 MACOS_AARCH64_BOOT_JDK_EXT=tar.gz
-MACOS_AARCH64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk24/1f9ff9062db4449d8ca828c504ffae90/36/GPL/openjdk-24_macos-aarch64_bin.tar.gz
-MACOS_AARCH64_BOOT_JDK_SHA256=f7133238a12714a62c5ad2bd4da6741130be1a82512065da9ca23dee26b2d3d3
+MACOS_AARCH64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk25/bd75d5f9689641da8e1daabeccb5528b/36/GPL/openjdk-25_macos-aarch64_bin.tar.gz
+MACOS_AARCH64_BOOT_JDK_SHA256=2006337bf326fdfdf6117081751ba38c1c8706d63419ecac7ff102ff7c776876
 
 MACOS_X64_BOOT_JDK_EXT=tar.gz
-MACOS_X64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk24/1f9ff9062db4449d8ca828c504ffae90/36/GPL/openjdk-24_macos-x64_bin.tar.gz
-MACOS_X64_BOOT_JDK_SHA256=6bbfb1d01741cbe55ab90299cb91464b695de9a3ace85c15131aa2f50292f321
+MACOS_X64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk25/bd75d5f9689641da8e1daabeccb5528b/36/GPL/openjdk-25_macos-x64_bin.tar.gz
+MACOS_X64_BOOT_JDK_SHA256=47482ad9888991ecac9b2bcc131e2b53ff78aff275104cef85f66252308e8a09
 
 WINDOWS_X64_BOOT_JDK_EXT=zip
-WINDOWS_X64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk24/1f9ff9062db4449d8ca828c504ffae90/36/GPL/openjdk-24_windows-x64_bin.zip
-WINDOWS_X64_BOOT_JDK_SHA256=11d1d9f6ac272d5361c8a0bef01894364081c7fb1a6914c2ad2fc312ae83d63b
+WINDOWS_X64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk25/bd75d5f9689641da8e1daabeccb5528b/36/GPL/openjdk-25_windows-x64_bin.zip
+WINDOWS_X64_BOOT_JDK_SHA256=85bcc178461e2cb3c549ab9ca9dfa73afd54c09a175d6510d0884071867137d3

make/conf/jib-profiles.js

@@ -387,8 +387,8 @@ var getJibProfilesCommon = function (input, data) {
         };
     };
 
-    common.boot_jdk_version = "24";
-    common.boot_jdk_build_number = "36";
+    common.boot_jdk_version = "25";
+    common.boot_jdk_build_number = "37";
     common.boot_jdk_home = input.get("boot_jdk", "install_path") + "/jdk-"
         + common.boot_jdk_version
         + (input.build_os == "macosx" ? ".jdk/Contents/Home" : "");

make/conf/version-numbers.conf

@@ -37,6 +37,6 @@ DEFAULT_VERSION_DATE=2026-03-17
 DEFAULT_VERSION_CLASSFILE_MAJOR=70  # "`$EXPR $DEFAULT_VERSION_FEATURE + 44`"
 DEFAULT_VERSION_CLASSFILE_MINOR=0
 DEFAULT_VERSION_DOCS_API_SINCE=11
-DEFAULT_ACCEPTABLE_BOOT_VERSIONS="24 25 26"
+DEFAULT_ACCEPTABLE_BOOT_VERSIONS="25 26"
 DEFAULT_JDK_SOURCE_TARGET_VERSION=26
 DEFAULT_PROMOTED_VERSION_PRE=ea

make/hotspot/lib/CompileJvm.gmk

@@ -189,7 +189,6 @@ $(eval $(call SetupJdkLibrary, BUILD_LIBJVM, \
     DISABLED_WARNINGS_gcc_c1_Runtime1_aarch64.cpp := unused-const-variable, \
     DISABLED_WARNINGS_gcc_cgroupV1Subsystem_linux.cpp := address, \
     DISABLED_WARNINGS_gcc_cgroupV2Subsystem_linux.cpp := address, \
-    DISABLED_WARNINGS_gcc_g1FreeIdSet.cpp := unused-const-variable, \
     DISABLED_WARNINGS_gcc_handshake.cpp := stringop-overflow, \
     DISABLED_WARNINGS_gcc_interp_masm_x86.cpp := uninitialized, \
     DISABLED_WARNINGS_gcc_javaClasses.cpp := unused-const-variable, \

src/hotspot/cpu/aarch64/aarch64.ad

@@ -1267,38 +1267,38 @@ source %{
     // registers conditionally reserved.
 
     _ANY_REG32_mask = _ALL_REG32_mask;
-    _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
+    _ANY_REG32_mask.remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
 
     _ANY_REG_mask = _ALL_REG_mask;
 
     _PTR_REG_mask = _ALL_REG_mask;
 
     _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
-    _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
+    _NO_SPECIAL_REG32_mask.subtract(_NON_ALLOCATABLE_REG32_mask);
 
     _NO_SPECIAL_REG_mask = _ALL_REG_mask;
-    _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
+    _NO_SPECIAL_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
 
     _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
-    _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
+    _NO_SPECIAL_PTR_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
 
     // r27 is not allocatable when compressed oops is on and heapbase is not
     // zero, compressed klass pointers doesn't use r27 after JDK-8234794
     if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
-      _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
-      _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
-      _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
+      _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
+      _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
+      _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
     }
 
     // r29 is not allocatable when PreserveFramePointer is on
     if (PreserveFramePointer) {
-      _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
-      _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
-      _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
+      _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
+      _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
+      _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
     }
 
     _NO_SPECIAL_NO_RFP_PTR_REG_mask = _NO_SPECIAL_PTR_REG_mask;
-    _NO_SPECIAL_NO_RFP_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
+    _NO_SPECIAL_NO_RFP_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
   }
 
   // Optimizaton of volatile gets and puts
@@ -1734,7 +1734,7 @@ uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
   }
 
 //=============================================================================
-const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
+const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::EMPTY;
 
 int ConstantTable::calculate_table_base_offset() const {
   return 0;  // absolute addressing, no offset
@@ -2520,10 +2520,10 @@ uint Matcher::int_pressure_limit()
   // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
   // derived pointers and lastly fail to spill after reaching maximum
   // number of iterations. Lowering the default pressure threshold to
-  // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
+  // (_NO_SPECIAL_REG32_mask.size() minus 1) forces CallNode to become
   // a high register pressure area of the code so that split_DEF can
   // generate DefinitionSpillCopy for the derived pointer.
-  uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
+  uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.size() - 1;
   if (!PreserveFramePointer) {
     // When PreserveFramePointer is off, frame pointer is allocatable,
     // but different from other SOC registers, it is excluded from
@@ -2538,7 +2538,7 @@ uint Matcher::int_pressure_limit()
 uint Matcher::float_pressure_limit()
 {
   // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
-  return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
+  return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.size() : FLOATPRESSURE;
 }
 
 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {

src/hotspot/cpu/aarch64/aarch64_vector.ad

@@ -7081,29 +7081,31 @@ instruct vcompress(vReg dst, vReg src, pRegGov pg) %{
 %}
 
 instruct vcompressB(vReg dst, vReg src, pReg pg, vReg tmp1, vReg tmp2,
-                    vReg tmp3, vReg tmp4, pReg ptmp, pRegGov pgtmp) %{
+                    vReg tmp3, pReg ptmp, pRegGov pgtmp) %{
   predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_BYTE);
-  effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP ptmp, TEMP pgtmp);
+  effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP ptmp, TEMP pgtmp);
   match(Set dst (CompressV src pg));
-  format %{ "vcompressB $dst, $src, $pg\t# KILL $tmp1, $tmp2, $tmp3, tmp4, $ptmp, $pgtmp" %}
+  format %{ "vcompressB $dst, $src, $pg\t# KILL $tmp1, $tmp2, $tmp3, $ptmp, $pgtmp" %}
   ins_encode %{
+    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
     __ sve_compress_byte($dst$$FloatRegister, $src$$FloatRegister, $pg$$PRegister,
-                         $tmp1$$FloatRegister,$tmp2$$FloatRegister,
-                         $tmp3$$FloatRegister,$tmp4$$FloatRegister,
-                         $ptmp$$PRegister, $pgtmp$$PRegister);
+                         $tmp1$$FloatRegister, $tmp2$$FloatRegister, $tmp3$$FloatRegister,
+                         $ptmp$$PRegister, $pgtmp$$PRegister, length_in_bytes);
   %}
   ins_pipe(pipe_slow);
 %}
 
-instruct vcompressS(vReg dst, vReg src, pReg pg,
-                    vReg tmp1, vReg tmp2, pRegGov pgtmp) %{
+instruct vcompressS(vReg dst, vReg src, pReg pg, vReg tmp1, vReg tmp2, pRegGov pgtmp) %{
   predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_SHORT);
   effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2, TEMP pgtmp);
   match(Set dst (CompressV src pg));
   format %{ "vcompressS $dst, $src, $pg\t# KILL $tmp1, $tmp2, $pgtmp" %}
   ins_encode %{
+    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
+    __ sve_dup($tmp1$$FloatRegister, __ H, 0);
     __ sve_compress_short($dst$$FloatRegister, $src$$FloatRegister, $pg$$PRegister,
-                          $tmp1$$FloatRegister,$tmp2$$FloatRegister, $pgtmp$$PRegister);
+                          $tmp1$$FloatRegister, $tmp2$$FloatRegister, $pgtmp$$PRegister,
+                          length_in_bytes);
   %}
   ins_pipe(pipe_slow);
 %}

src/hotspot/cpu/aarch64/aarch64_vector_ad.m4

@@ -5069,29 +5069,31 @@ instruct vcompress(vReg dst, vReg src, pRegGov pg) %{
 %}
 
 instruct vcompressB(vReg dst, vReg src, pReg pg, vReg tmp1, vReg tmp2,
-                    vReg tmp3, vReg tmp4, pReg ptmp, pRegGov pgtmp) %{
+                    vReg tmp3, pReg ptmp, pRegGov pgtmp) %{
   predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_BYTE);
-  effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP ptmp, TEMP pgtmp);
+  effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP ptmp, TEMP pgtmp);
   match(Set dst (CompressV src pg));
-  format %{ "vcompressB $dst, $src, $pg\t# KILL $tmp1, $tmp2, $tmp3, tmp4, $ptmp, $pgtmp" %}
+  format %{ "vcompressB $dst, $src, $pg\t# KILL $tmp1, $tmp2, $tmp3, $ptmp, $pgtmp" %}
   ins_encode %{
+    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
     __ sve_compress_byte($dst$$FloatRegister, $src$$FloatRegister, $pg$$PRegister,
-                         $tmp1$$FloatRegister,$tmp2$$FloatRegister,
-                         $tmp3$$FloatRegister,$tmp4$$FloatRegister,
-                         $ptmp$$PRegister, $pgtmp$$PRegister);
+                         $tmp1$$FloatRegister, $tmp2$$FloatRegister, $tmp3$$FloatRegister,
+                         $ptmp$$PRegister, $pgtmp$$PRegister, length_in_bytes);
   %}
   ins_pipe(pipe_slow);
 %}
 
-instruct vcompressS(vReg dst, vReg src, pReg pg,
-                    vReg tmp1, vReg tmp2, pRegGov pgtmp) %{
+instruct vcompressS(vReg dst, vReg src, pReg pg, vReg tmp1, vReg tmp2, pRegGov pgtmp) %{
   predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_SHORT);
   effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2, TEMP pgtmp);
   match(Set dst (CompressV src pg));
   format %{ "vcompressS $dst, $src, $pg\t# KILL $tmp1, $tmp2, $pgtmp" %}
   ins_encode %{
+    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
+    __ sve_dup($tmp1$$FloatRegister, __ H, 0);
     __ sve_compress_short($dst$$FloatRegister, $src$$FloatRegister, $pg$$PRegister,
-                          $tmp1$$FloatRegister,$tmp2$$FloatRegister, $pgtmp$$PRegister);
+                          $tmp1$$FloatRegister, $tmp2$$FloatRegister, $pgtmp$$PRegister,
+                          length_in_bytes);
   %}
   ins_pipe(pipe_slow);
 %}

src/hotspot/cpu/aarch64/assembler_aarch64.hpp

@@ -3486,6 +3486,7 @@ public:
   INSN(sve_smaxv, 0b00000100, 0b001000001); // signed maximum reduction to scalar
   INSN(sve_smin,  0b00000100, 0b001010000); // signed minimum vectors
   INSN(sve_sminv, 0b00000100, 0b001010001); // signed minimum reduction to scalar
+  INSN(sve_splice,0b00000101, 0b101100100); // splice two vectors under predicate control, destructive
   INSN(sve_sub,   0b00000100, 0b000001000); // vector sub
   INSN(sve_uaddv, 0b00000100, 0b000001001); // unsigned add reduction to scalar
   INSN(sve_umax,  0b00000100, 0b001001000); // unsigned maximum vectors

src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp

@@ -2203,114 +2203,117 @@ void C2_MacroAssembler::sve_gen_mask_imm(PRegister dst, BasicType bt, uint32_t l
 // Pack active elements of src, under the control of mask, into the lowest-numbered elements of dst.
 // Any remaining elements of dst will be filled with zero.
 // Clobbers: rscratch1
-// Preserves: src, mask
+// Preserves: mask, vzr
 void C2_MacroAssembler::sve_compress_short(FloatRegister dst, FloatRegister src, PRegister mask,
-                                           FloatRegister vtmp1, FloatRegister vtmp2,
-                                           PRegister pgtmp) {
+                                           FloatRegister vzr, FloatRegister vtmp,
+                                           PRegister pgtmp, unsigned vector_length_in_bytes) {
   assert(pgtmp->is_governing(), "This register has to be a governing predicate register");
-  assert_different_registers(dst, src, vtmp1, vtmp2);
+  // When called by sve_compress_byte, src and vtmp may be the same register.
+  assert_different_registers(dst, src, vzr);
+  assert_different_registers(dst, vtmp, vzr);
   assert_different_registers(mask, pgtmp);
-
-  // Example input:   src   = 8888 7777 6666 5555 4444 3333 2222 1111
-  //                  mask  = 0001 0000 0000 0001 0001 0000 0001 0001
-  // Expected result: dst   = 0000 0000 0000 8888 5555 4444 2222 1111
-  sve_dup(vtmp2, H, 0);
+  // high <-- low
+  // Example input:   src   = hh gg ff ee dd cc bb aa, one character is 8 bits.
+  //                  mask  = 01 00 00 01 01 00 01 01, one character is 1 bit.
+  // Expected result: dst   = 00 00 00 hh ee dd bb aa
 
   // Extend lowest half to type INT.
-  // dst = 00004444 00003333 00002222 00001111
+  // dst   =  00dd  00cc  00bb  00aa
   sve_uunpklo(dst, S, src);
-  // pgtmp = 00000001 00000000 00000001 00000001
+  // pgtmp =  0001  0000  0001  0001
   sve_punpklo(pgtmp, mask);
   // Pack the active elements in size of type INT to the right,
   // and fill the remainings with zero.
-  // dst = 00000000 00004444 00002222 00001111
+  // dst   =  0000  00dd  00bb  00aa
   sve_compact(dst, S, dst, pgtmp);
   // Narrow the result back to type SHORT.
-  // dst = 0000 0000 0000 0000 0000 4444 2222 1111
-  sve_uzp1(dst, H, dst, vtmp2);
+  // dst   = 00 00 00 00 00 dd bb aa
+  sve_uzp1(dst, H, dst, vzr);
+
+  // Return if the vector length is no more than MaxVectorSize/2, since the
+  // highest half is invalid.
+  if (vector_length_in_bytes <= (MaxVectorSize >> 1)) {
+    return;
+  }
+
   // Count the active elements of lowest half.
   // rscratch1 = 3
   sve_cntp(rscratch1, S, ptrue, pgtmp);
 
   // Repeat to the highest half.
-  // pgtmp = 00000001 00000000 00000000 00000001
+  // pgtmp =  0001  0000  0000  0001
   sve_punpkhi(pgtmp, mask);
-  // vtmp1 = 00008888 00007777 00006666 00005555
-  sve_uunpkhi(vtmp1, S, src);
-  // vtmp1 = 00000000 00000000 00008888 00005555
-  sve_compact(vtmp1, S, vtmp1, pgtmp);
-  // vtmp1 = 0000 0000 0000 0000 0000 0000 8888 5555
-  sve_uzp1(vtmp1, H, vtmp1, vtmp2);
+  // vtmp  =  00hh  00gg  00ff  00ee
+  sve_uunpkhi(vtmp, S, src);
+  // vtmp  =  0000  0000  00hh  00ee
+  sve_compact(vtmp, S, vtmp, pgtmp);
+  // vtmp  = 00 00 00 00 00 00 hh ee
+  sve_uzp1(vtmp, H, vtmp, vzr);
 
-  // Compressed low:   dst   = 0000 0000 0000 0000 0000 4444 2222 1111
-  // Compressed high:  vtmp1 = 0000 0000 0000 0000 0000 0000 8888  5555
-  // Left shift(cross lane) compressed high with TRUE_CNT lanes,
-  // TRUE_CNT is the number of active elements in the compressed low.
-  neg(rscratch1, rscratch1);
-  // vtmp2 = {4 3 2 1 0 -1 -2 -3}
-  sve_index(vtmp2, H, rscratch1, 1);
-  // vtmp1 = 0000 0000 0000 8888 5555 0000 0000 0000
-  sve_tbl(vtmp1, H, vtmp1, vtmp2);
-
-  // Combine the compressed high(after shifted) with the compressed low.
-  // dst = 0000 0000 0000 8888 5555 4444 2222 1111
-  sve_orr(dst, dst, vtmp1);
+  // pgtmp = 00 00 00 00 00 01 01 01
+  sve_whilelt(pgtmp, H, zr, rscratch1);
+  // Compressed low:  dst  = 00 00 00 00 00 dd bb aa
+  // Compressed high: vtmp = 00 00 00 00 00 00 hh ee
+  // Combine the compressed low with the compressed high:
+  //                  dst  = 00 00 00 hh ee dd bb aa
+  sve_splice(dst, H, pgtmp, vtmp);
 }
 
 // Clobbers: rscratch1, rscratch2
 // Preserves: src, mask
 void C2_MacroAssembler::sve_compress_byte(FloatRegister dst, FloatRegister src, PRegister mask,
-                                          FloatRegister vtmp1, FloatRegister vtmp2,
-                                          FloatRegister vtmp3, FloatRegister vtmp4,
-                                          PRegister ptmp, PRegister pgtmp) {
+                                          FloatRegister vtmp1, FloatRegister vtmp2, FloatRegister vtmp3,
+                                          PRegister ptmp, PRegister pgtmp, unsigned vector_length_in_bytes) {
   assert(pgtmp->is_governing(), "This register has to be a governing predicate register");
-  assert_different_registers(dst, src, vtmp1, vtmp2, vtmp3, vtmp4);
+  assert_different_registers(dst, src, vtmp1, vtmp2, vtmp3);
   assert_different_registers(mask, ptmp, pgtmp);
-  // Example input:   src   = 88 77 66 55 44 33 22 11
-  //                  mask  = 01 00 00 01 01 00 01 01
-  // Expected result: dst   = 00 00 00 88 55 44 22 11
+  // high <-- low
+  // Example input:   src   = q p n m l k j i h g f e d c b a, one character is 8 bits.
+  //                  mask  = 0 1 0 0 0 0 0 1 0 1 0 0 0 1 0 1, one character is 1 bit.
+  // Expected result: dst   = 0 0 0 0 0 0 0 0 0 0 0 p i g c a
+  FloatRegister vzr = vtmp3;
+  sve_dup(vzr, B, 0);
 
-  sve_dup(vtmp4, B, 0);
   // Extend lowest half to type SHORT.
-  // vtmp1 = 0044 0033 0022 0011
+  // vtmp1 =  0h  0g  0f  0e  0d  0c  0b  0a
   sve_uunpklo(vtmp1, H, src);
-  // ptmp = 0001 0000 0001 0001
+  // ptmp  =  00  01  00  00  00  01  00  01
   sve_punpklo(ptmp, mask);
+  // Pack the active elements in size of type SHORT to the right,
+  // and fill the remainings with zero.
+  // dst   =  00  00  00  00  00  0g  0c  0a
+  unsigned extended_size = vector_length_in_bytes << 1;
+  sve_compress_short(dst, vtmp1, ptmp, vzr, vtmp2, pgtmp, extended_size > MaxVectorSize ? MaxVectorSize : extended_size);
+  // Narrow the result back to type BYTE.
+  // dst   = 0 0 0 0 0 0 0 0 0 0 0 0 0 g c a
+  sve_uzp1(dst, B, dst, vzr);
+
+  // Return if the vector length is no more than MaxVectorSize/2, since the
+  // highest half is invalid.
+  if (vector_length_in_bytes <= (MaxVectorSize >> 1)) {
+    return;
+  }
   // Count the active elements of lowest half.
   // rscratch2 = 3
   sve_cntp(rscratch2, H, ptrue, ptmp);
-  // Pack the active elements in size of type SHORT to the right,
-  // and fill the remainings with zero.
-  // dst = 0000 0044 0022 0011
-  sve_compress_short(dst, vtmp1, ptmp, vtmp2, vtmp3, pgtmp);
-  // Narrow the result back to type BYTE.
-  // dst = 00 00 00 00 00 44 22 11
-  sve_uzp1(dst, B, dst, vtmp4);
 
   // Repeat to the highest half.
-  // ptmp = 0001 0000 0000 0001
+  // ptmp  =  00  01  00  00  00  00  00  01
   sve_punpkhi(ptmp, mask);
-  // vtmp1 = 0088 0077 0066 0055
+  // vtmp2 =  0q  0p  0n  0m  0l  0k  0j  0i
   sve_uunpkhi(vtmp2, H, src);
-  // vtmp1 = 0000 0000 0088 0055
-  sve_compress_short(vtmp1, vtmp2, ptmp, vtmp3, vtmp4, pgtmp);
+  // vtmp1 =  00  00  00  00  00  00  0p  0i
+  sve_compress_short(vtmp1, vtmp2, ptmp, vzr, vtmp2, pgtmp, extended_size - MaxVectorSize);
+  // vtmp1 = 0 0 0 0 0 0 0 0 0 0 0 0 0 0 p i
+  sve_uzp1(vtmp1, B, vtmp1, vzr);
 
-  sve_dup(vtmp4, B, 0);
-  // vtmp1 = 00 00 00 00 00 00 88 55
-  sve_uzp1(vtmp1, B, vtmp1, vtmp4);
-
-  // Compressed low:   dst   = 00 00 00 00 00 44 22 11
-  // Compressed high:  vtmp1 = 00 00 00 00 00 00 88 55
-  // Left shift(cross lane) compressed high with TRUE_CNT lanes,
-  // TRUE_CNT is the number of active elements in the compressed low.
-  neg(rscratch2, rscratch2);
-  // vtmp2 = {4 3 2 1 0 -1 -2 -3}
-  sve_index(vtmp2, B, rscratch2, 1);
-  // vtmp1 = 00 00 00 88 55 00 00 00
-  sve_tbl(vtmp1, B, vtmp1, vtmp2);
-  // Combine the compressed high(after shifted) with the compressed low.
-  // dst = 00 00 00 88 55 44 22 11
-  sve_orr(dst, dst, vtmp1);
+  // ptmp  = 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1
+  sve_whilelt(ptmp, B, zr, rscratch2);
+  // Compressed low:  dst   = 0 0 0 0 0 0 0 0 0 0 0 0 0 g c a
+  // Compressed high: vtmp1 = 0 0 0 0 0 0 0 0 0 0 0 0 0 0 p i
+  // Combine the compressed low with the compressed high:
+  //                  dst   = 0 0 0 0 0 0 0 0 0 0 0 p i g c a
+  sve_splice(dst, B, ptmp, vtmp1);
 }
 
 void C2_MacroAssembler::neon_reverse_bits(FloatRegister dst, FloatRegister src, BasicType bt, bool isQ) {

src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp

@@ -173,13 +173,12 @@
   // lowest-numbered elements of dst. Any remaining elements of dst will
   // be filled with zero.
   void sve_compress_byte(FloatRegister dst, FloatRegister src, PRegister mask,
-                         FloatRegister vtmp1, FloatRegister vtmp2,
-                         FloatRegister vtmp3, FloatRegister vtmp4,
-                         PRegister ptmp, PRegister pgtmp);
+                         FloatRegister vtmp1, FloatRegister vtmp2, FloatRegister vtmp3,
+                         PRegister ptmp, PRegister pgtmp, unsigned vector_length_in_bytes);
 
   void sve_compress_short(FloatRegister dst, FloatRegister src, PRegister mask,
-                          FloatRegister vtmp1, FloatRegister vtmp2,
-                          PRegister pgtmp);
+                          FloatRegister vzr, FloatRegister vtmp,
+                          PRegister pgtmp, unsigned vector_length_in_bytes);
 
   void neon_reverse_bits(FloatRegister dst, FloatRegister src, BasicType bt, bool isQ);
 

src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp

@@ -1704,3 +1704,14 @@ void InterpreterMacroAssembler::load_method_entry(Register cache, Register index
   add(cache, cache, Array<ResolvedMethodEntry>::base_offset_in_bytes());
   lea(cache, Address(cache, index));
 }
+
+#ifdef ASSERT
+void InterpreterMacroAssembler::verify_field_offset(Register reg) {
+  // Verify the field offset is not in the header, implicitly checks for 0
+  Label L;
+  subs(zr, reg, oopDesc::base_offset_in_bytes());
+  br(Assembler::GE, L);
+  stop("bad field offset");
+  bind(L);
+}
+#endif

src/hotspot/cpu/aarch64/interp_masm_aarch64.hpp

@@ -319,6 +319,8 @@ class InterpreterMacroAssembler: public MacroAssembler {
   void load_resolved_indy_entry(Register cache, Register index);
   void load_field_entry(Register cache, Register index, int bcp_offset = 1);
   void load_method_entry(Register cache, Register index, int bcp_offset = 1);
+
+  void verify_field_offset(Register reg) NOT_DEBUG_RETURN;
 };
 
 #endif // CPU_AARCH64_INTERP_MASM_AARCH64_HPP

src/hotspot/cpu/aarch64/templateTable_aarch64.cpp

@@ -168,6 +168,7 @@ void TemplateTable::patch_bytecode(Bytecodes::Code bc, Register bc_reg,
                                    Register temp_reg, bool load_bc_into_bc_reg/*=true*/,
                                    int byte_no)
 {
+  assert_different_registers(bc_reg, temp_reg);
   if (!RewriteBytecodes)  return;
   Label L_patch_done;
 
@@ -231,9 +232,12 @@ void TemplateTable::patch_bytecode(Bytecodes::Code bc, Register bc_reg,
   __ stop("patching the wrong bytecode");
   __ bind(L_okay);
 #endif
-
-  // patch bytecode
-  __ strb(bc_reg, at_bcp(0));
+  // Patch bytecode with release store to coordinate with ResolvedFieldEntry loads
+  // in fast bytecode codelets. load_field_entry has a memory barrier that gains
+  // the needed ordering, together with control dependency on entering the fast codelet
+  // itself.
+  __ lea(temp_reg, at_bcp(0));
+  __ stlrb(bc_reg, temp_reg);
   __ bind(L_patch_done);
 }
 
@@ -2269,7 +2273,7 @@ void TemplateTable::resolve_cache_and_index_for_method(int byte_no,
   assert_different_registers(Rcache, index, temp);
   assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
 
-  Label resolved, clinit_barrier_slow;
+  Label L_clinit_barrier_slow, L_done;
 
   Bytecodes::Code code = bytecode();
   __ load_method_entry(Rcache, index);
@@ -2284,11 +2288,20 @@ void TemplateTable::resolve_cache_and_index_for_method(int byte_no,
   // Load-acquire the bytecode to match store-release in InterpreterRuntime
   __ ldarb(temp, temp);
   __ subs(zr, temp, (int) code);  // have we resolved this bytecode?
-  __ br(Assembler::EQ, resolved);
+
+  // Class initialization barrier for static methods
+  if (VM_Version::supports_fast_class_init_checks() && bytecode() == Bytecodes::_invokestatic) {
+    __ br(Assembler::NE, L_clinit_barrier_slow);
+    __ ldr(temp, Address(Rcache, in_bytes(ResolvedMethodEntry::method_offset())));
+    __ load_method_holder(temp, temp);
+    __ clinit_barrier(temp, rscratch1, &L_done, /*L_slow_path*/ nullptr);
+    __ bind(L_clinit_barrier_slow);
+  } else {
+    __ br(Assembler::EQ, L_done);
+  }
 
   // resolve first time through
   // Class initialization barrier slow path lands here as well.
-  __ bind(clinit_barrier_slow);
   address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache);
   __ mov(temp, (int) code);
   __ call_VM(noreg, entry, temp);
@@ -2297,14 +2310,7 @@ void TemplateTable::resolve_cache_and_index_for_method(int byte_no,
   __ load_method_entry(Rcache, index);
   // n.b. unlike x86 Rcache is now rcpool plus the indexed offset
   // so all clients ofthis method must be modified accordingly
-  __ bind(resolved);
-
-  // Class initialization barrier for static methods
-  if (VM_Version::supports_fast_class_init_checks() && bytecode() == Bytecodes::_invokestatic) {
-    __ ldr(temp, Address(Rcache, in_bytes(ResolvedMethodEntry::method_offset())));
-    __ load_method_holder(temp, temp);
-    __ clinit_barrier(temp, rscratch1, nullptr, &clinit_barrier_slow);
-  }
+  __ bind(L_done);
 }
 
 void TemplateTable::resolve_cache_and_index_for_field(int byte_no,
@@ -2313,7 +2319,7 @@ void TemplateTable::resolve_cache_and_index_for_field(int byte_no,
   const Register temp = r19;
   assert_different_registers(Rcache, index, temp);
 
-  Label resolved;
+  Label L_clinit_barrier_slow, L_done;
 
   Bytecodes::Code code = bytecode();
   switch (code) {
@@ -2332,16 +2338,29 @@ void TemplateTable::resolve_cache_and_index_for_field(int byte_no,
   // Load-acquire the bytecode to match store-release in ResolvedFieldEntry::fill_in()
   __ ldarb(temp, temp);
   __ subs(zr, temp, (int) code);  // have we resolved this bytecode?
-  __ br(Assembler::EQ, resolved);
+
+  // Class initialization barrier for static fields
+  if (VM_Version::supports_fast_class_init_checks() &&
+      (bytecode() == Bytecodes::_getstatic || bytecode() == Bytecodes::_putstatic)) {
+    const Register field_holder = temp;
+
+    __ br(Assembler::NE, L_clinit_barrier_slow);
+    __ ldr(field_holder, Address(Rcache, in_bytes(ResolvedFieldEntry::field_holder_offset())));
+    __ clinit_barrier(field_holder, rscratch1, &L_done, /*L_slow_path*/ nullptr);
+    __ bind(L_clinit_barrier_slow);
+  } else {
+    __ br(Assembler::EQ, L_done);
+  }
 
   // resolve first time through
+  // Class initialization barrier slow path lands here as well.
   address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache);
   __ mov(temp, (int) code);
   __ call_VM(noreg, entry, temp);
 
   // Update registers with resolved info
   __ load_field_entry(Rcache, index);
-  __ bind(resolved);
+  __ bind(L_done);
 }
 
 void TemplateTable::load_resolved_field_entry(Register obj,
@@ -3079,6 +3098,7 @@ void TemplateTable::fast_storefield(TosState state)
 
   // R1: field offset, R2: field holder, R5: flags
   load_resolved_field_entry(r2, r2, noreg, r1, r5);
+  __ verify_field_offset(r1);
 
   {
     Label notVolatile;
@@ -3168,6 +3188,8 @@ void TemplateTable::fast_accessfield(TosState state)
   __ load_field_entry(r2, r1);
 
   __ load_sized_value(r1, Address(r2, in_bytes(ResolvedFieldEntry::field_offset_offset())), sizeof(int), true /*is_signed*/);
+  __ verify_field_offset(r1);
+
   __ load_unsigned_byte(r3, Address(r2, in_bytes(ResolvedFieldEntry::flags_offset())));
 
   // r0: object
@@ -3234,7 +3256,9 @@ void TemplateTable::fast_xaccess(TosState state)
   __ ldr(r0, aaddress(0));
   // access constant pool cache
   __ load_field_entry(r2, r3, 2);
+
   __ load_sized_value(r1, Address(r2, in_bytes(ResolvedFieldEntry::field_offset_offset())), sizeof(int), true /*is_signed*/);
+  __ verify_field_offset(r1);
 
   // 8179954: We need to make sure that the code generated for
   // volatile accesses forms a sequentially-consistent set of

src/hotspot/cpu/aarch64/vm_version_aarch64.hpp

@@ -199,6 +199,8 @@ enum Ampere_CPU_Model {
 
   constexpr static bool supports_secondary_supers_table() { return true; }
 
+  constexpr static bool supports_misaligned_vector_accesses() { return true; }
+
   static void get_compatible_board(char *buf, int buflen);
 
   static const SpinWait& spin_wait_desc() { return _spin_wait; }

src/hotspot/cpu/ppc/assembler_ppc.hpp

@@ -2059,12 +2059,12 @@ class Assembler : public AbstractAssembler {
  protected:
   inline void tdi_unchecked(int tobits, Register a, int si16);
   inline void twi_unchecked(int tobits, Register a, int si16);
+ public:
   inline void tdi(          int tobits, Register a, int si16);   // asserts UseSIGTRAP
   inline void twi(          int tobits, Register a, int si16);   // asserts UseSIGTRAP
   inline void td(           int tobits, Register a, Register b); // asserts UseSIGTRAP
   inline void tw(           int tobits, Register a, Register b); // asserts UseSIGTRAP
 
- public:
   static bool is_tdi(int x, int tobits, int ra, int si16) {
      return (TDI_OPCODE == (x & TDI_OPCODE_MASK))
          && (tobits == inv_to_field(x))

src/hotspot/cpu/ppc/atomicAccess_ppc.hpp

@@ -0,0 +1,649 @@
+/*
+ * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2025 SAP SE. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef CPU_PPC_ATOMICACCESS_PPC_HPP
+#define CPU_PPC_ATOMICACCESS_PPC_HPP
+
+#ifndef PPC64
+#error "Atomic currently only implemented for PPC64"
+#endif
+
+#include "orderAccess_ppc.hpp"
+#include "utilities/debug.hpp"
+
+// Implementation of class AtomicAccess
+
+//
+// machine barrier instructions:
+//
+// - sync            two-way memory barrier, aka fence
+// - lwsync          orders  Store|Store,
+//                            Load|Store,
+//                            Load|Load,
+//                   but not Store|Load
+// - eieio           orders memory accesses for device memory (only)
+// - isync           invalidates speculatively executed instructions
+//                   From the POWER ISA 2.06 documentation:
+//                    "[...] an isync instruction prevents the execution of
+//                   instructions following the isync until instructions
+//                   preceding the isync have completed, [...]"
+//                   From IBM's AIX assembler reference:
+//                    "The isync [...] instructions causes the processor to
+//                   refetch any instructions that might have been fetched
+//                   prior to the isync instruction. The instruction isync
+//                   causes the processor to wait for all previous instructions
+//                   to complete. Then any instructions already fetched are
+//                   discarded and instruction processing continues in the
+//                   environment established by the previous instructions."
+//
+// semantic barrier instructions:
+// (as defined in orderAccess.hpp)
+//
+// - release         orders Store|Store,       (maps to lwsync)
+//                           Load|Store
+// - acquire         orders  Load|Store,       (maps to lwsync)
+//                           Load|Load
+// - fence           orders Store|Store,       (maps to sync)
+//                           Load|Store,
+//                           Load|Load,
+//                          Store|Load
+//
+
+inline void pre_membar(atomic_memory_order order) {
+  switch (order) {
+    case memory_order_relaxed:
+    case memory_order_acquire: break;
+    case memory_order_release:
+    case memory_order_acq_rel: __asm__ __volatile__ ("lwsync" : : : "memory"); break;
+    default /*conservative*/ : __asm__ __volatile__ ("sync"   : : : "memory"); break;
+  }
+}
+
+inline void post_membar(atomic_memory_order order) {
+  switch (order) {
+    case memory_order_relaxed:
+    case memory_order_release: break;
+    case memory_order_acquire:
+    case memory_order_acq_rel: __asm__ __volatile__ ("isync"  : : : "memory"); break;
+    default /*conservative*/ : __asm__ __volatile__ ("sync"   : : : "memory"); break;
+  }
+}
+
+
+
+template<size_t byte_size>
+struct AtomicAccess::PlatformAdd {
+  template<typename D, typename I>
+  D add_then_fetch(D volatile* dest, I add_value, atomic_memory_order order) const;
+
+  template<typename D, typename I>
+  D fetch_then_add(D volatile* dest, I add_value, atomic_memory_order order) const {
+    return add_then_fetch(dest, add_value, order) - add_value;
+  }
+};
+
+template<>
+template<typename D, typename I>
+inline D AtomicAccess::PlatformAdd<4>::add_then_fetch(D volatile* dest, I add_value,
+                                                      atomic_memory_order order) const {
+  STATIC_ASSERT(4 == sizeof(I));
+  STATIC_ASSERT(4 == sizeof(D));
+
+  D result;
+
+  pre_membar(order);
+
+  __asm__ __volatile__ (
+    "1: lwarx   %[result], 0, %[dest]                 \n"
+    "   add     %[result], %[result], %[add_value]    \n"
+    "   stwcx.  %[result], 0, %[dest]                 \n"
+    "   bne-    1b                                    \n"
+    : [result]     "=&r"  (result)
+    : [add_value]  "r"    (add_value),
+      [dest]       "b"    (dest)
+    : "cc", "memory" );
+
+  post_membar(order);
+
+  return result;
+}
+
+
+template<>
+template<typename D, typename I>
+inline D AtomicAccess::PlatformAdd<8>::add_then_fetch(D volatile* dest, I add_value,
+                                                      atomic_memory_order order) const {
+  STATIC_ASSERT(8 == sizeof(I));
+  STATIC_ASSERT(8 == sizeof(D));
+
+  D result;
+
+  pre_membar(order);
+
+  __asm__ __volatile__ (
+    "1: ldarx   %[result], 0, %[dest]                 \n"
+    "   add     %[result], %[result], %[add_value]    \n"
+    "   stdcx.  %[result], 0, %[dest]                 \n"
+    "   bne-    1b                                    \n"
+    : [result]     "=&r"  (result)
+    : [add_value]  "r"    (add_value),
+      [dest]       "b"    (dest)
+    : "cc", "memory" );
+
+  post_membar(order);
+
+  return result;
+}
+
+template<>
+template<typename T>
+inline T AtomicAccess::PlatformXchg<4>::operator()(T volatile* dest,
+                                                   T exchange_value,
+                                                   atomic_memory_order order) const {
+  STATIC_ASSERT(4 == sizeof(T));
+  // Note that xchg doesn't necessarily do an acquire
+  // (see synchronizer.cpp).
+
+  T old_value;
+
+  pre_membar(order);
+
+  __asm__ __volatile__ (
+    /* atomic loop */
+    "1:                                                 \n"
+    "   lwarx   %[old_value], 0, %[dest]                \n"
+    "   stwcx.  %[exchange_value], 0, %[dest]           \n"
+    "   bne-    1b                                      \n"
+    /* exit */
+    "2:                                                 \n"
+    /* out */
+    : [old_value]       "=&r"   (old_value)
+    /* in */
+    : [dest]            "b"     (dest),
+      [exchange_value]  "r"     (exchange_value)
+    /* clobber */
+    : "cc",
+      "memory"
+    );
+
+  post_membar(order);
+
+  return old_value;
+}
+
+template<>
+template<typename T>
+inline T AtomicAccess::PlatformXchg<8>::operator()(T volatile* dest,
+                                                   T exchange_value,
+                                                   atomic_memory_order order) const {
+  STATIC_ASSERT(8 == sizeof(T));
+  // Note that xchg doesn't necessarily do an acquire
+  // (see synchronizer.cpp).
+
+  T old_value;
+
+  pre_membar(order);
+
+  __asm__ __volatile__ (
+    /* atomic loop */
+    "1:                                                 \n"
+    "   ldarx   %[old_value], 0, %[dest]                \n"
+    "   stdcx.  %[exchange_value], 0, %[dest]           \n"
+    "   bne-    1b                                      \n"
+    /* exit */
+    "2:                                                 \n"
+    /* out */
+    : [old_value]       "=&r"   (old_value)
+    /* in */
+    : [dest]            "b"     (dest),
+      [exchange_value]  "r"     (exchange_value)
+    /* clobber */
+    : "cc",
+      "memory"
+    );
+
+  post_membar(order);
+
+  return old_value;
+}
+
+template<>
+template<typename T>
+inline T AtomicAccess::PlatformCmpxchg<1>::operator()(T volatile* dest,
+                                                      T compare_value,
+                                                      T exchange_value,
+                                                      atomic_memory_order order) const {
+  STATIC_ASSERT(1 == sizeof(T));
+
+  // Note that cmpxchg guarantees a two-way memory barrier across
+  // the cmpxchg, so it's really a 'fence_cmpxchg_fence' if not
+  // specified otherwise (see atomicAccess.hpp).
+
+  const unsigned int masked_compare_val = (unsigned int)(unsigned char)compare_value;
+
+  unsigned int old_value;
+
+  pre_membar(order);
+
+  __asm__ __volatile__ (
+    /* simple guard */
+    "   lbz     %[old_value], 0(%[dest])                  \n"
+    "   cmpw    %[masked_compare_val], %[old_value]       \n"
+    "   bne-    2f                                        \n"
+    /* atomic loop */
+    "1:                                                   \n"
+    "   lbarx   %[old_value], 0, %[dest]                  \n"
+    "   cmpw    %[masked_compare_val], %[old_value]       \n"
+    "   bne-    2f                                        \n"
+    "   stbcx.  %[exchange_value], 0, %[dest]             \n"
+    "   bne-    1b                                        \n"
+    /* exit */
+    "2:                                                   \n"
+    /* out */
+    : [old_value]       "=&r"   (old_value)
+    /* in */
+    : [dest]                   "b"     (dest),
+      [masked_compare_val]     "r"     (masked_compare_val),
+      [exchange_value]         "r"     (exchange_value)
+    /* clobber */
+    : "cc",
+      "memory"
+    );
+
+  post_membar(order);
+
+  return PrimitiveConversions::cast<T>((unsigned char)old_value);
+}
+
+template<>
+template<typename T>
+inline T AtomicAccess::PlatformCmpxchg<4>::operator()(T volatile* dest,
+                                                      T compare_value,
+                                                      T exchange_value,
+                                                      atomic_memory_order order) const {
+  STATIC_ASSERT(4 == sizeof(T));
+
+  // Note that cmpxchg guarantees a two-way memory barrier across
+  // the cmpxchg, so it's really a 'fence_cmpxchg_fence' if not
+  // specified otherwise (see atomicAccess.hpp).
+
+  T old_value;
+
+  pre_membar(order);
+
+  __asm__ __volatile__ (
+    /* simple guard */
+    "   lwz     %[old_value], 0(%[dest])                \n"
+    "   cmpw    %[compare_value], %[old_value]          \n"
+    "   bne-    2f                                      \n"
+    /* atomic loop */
+    "1:                                                 \n"
+    "   lwarx   %[old_value], 0, %[dest]                \n"
+    "   cmpw    %[compare_value], %[old_value]          \n"
+    "   bne-    2f                                      \n"
+    "   stwcx.  %[exchange_value], 0, %[dest]           \n"
+    "   bne-    1b                                      \n"
+    /* exit */
+    "2:                                                 \n"
+    /* out */
+    : [old_value]       "=&r"   (old_value)
+    /* in */
+    : [dest]            "b"     (dest),
+      [compare_value]   "r"     (compare_value),
+      [exchange_value]  "r"     (exchange_value)
+    /* clobber */
+    : "cc",
+      "memory"
+    );
+
+  post_membar(order);
+
+  return old_value;
+}
+
+template<>
+template<typename T>
+inline T AtomicAccess::PlatformCmpxchg<8>::operator()(T volatile* dest,
+                                                      T compare_value,
+                                                      T exchange_value,
+                                                      atomic_memory_order order) const {
+  STATIC_ASSERT(8 == sizeof(T));
+
+  // Note that cmpxchg guarantees a two-way memory barrier across
+  // the cmpxchg, so it's really a 'fence_cmpxchg_fence' if not
+  // specified otherwise (see atomicAccess.hpp).
+
+  T old_value;
+
+  pre_membar(order);
+
+  __asm__ __volatile__ (
+    /* simple guard */
+    "   ld      %[old_value], 0(%[dest])                \n"
+    "   cmpd    %[compare_value], %[old_value]          \n"
+    "   bne-    2f                                      \n"
+    /* atomic loop */
+    "1:                                                 \n"
+    "   ldarx   %[old_value], 0, %[dest]                \n"
+    "   cmpd    %[compare_value], %[old_value]          \n"
+    "   bne-    2f                                      \n"
+    "   stdcx.  %[exchange_value], 0, %[dest]           \n"
+    "   bne-    1b                                      \n"
+    /* exit */
+    "2:                                                 \n"
+    /* out */
+    : [old_value]       "=&r"   (old_value)
+    /* in */
+    : [dest]            "b"     (dest),
+      [compare_value]   "r"     (compare_value),
+      [exchange_value]  "r"     (exchange_value)
+    /* clobber */
+    : "cc",
+      "memory"
+    );
+
+  post_membar(order);
+
+  return old_value;
+}
+
+template<size_t byte_size>
+struct AtomicAccess::PlatformOrderedLoad<byte_size, X_ACQUIRE>
+{
+  template <typename T>
+  T operator()(const volatile T* p) const {
+    T t = AtomicAccess::load(p);
+    // Use twi-isync for load_acquire (faster than lwsync).
+    __asm__ __volatile__ ("twi 0,%0,0\n isync\n" : : "r" (t) : "memory");
+    return t;
+  }
+};
+
+template<>
+class AtomicAccess::PlatformBitops<4, true> {
+public:
+  template<typename T>
+  T fetch_then_and(T volatile* dest, T bits, atomic_memory_order order) const {
+    STATIC_ASSERT(4 == sizeof(T));
+    T old_value, result;
+
+    pre_membar(order);
+
+    __asm__ __volatile__ (
+      "1: lwarx   %[old_value], 0, %[dest]            \n"
+      "   and     %[result], %[old_value], %[bits]    \n"
+      "   stwcx.  %[result], 0, %[dest]               \n"
+      "   bne-    1b                                  \n"
+      : [old_value]  "=&r"  (old_value),
+        [result]     "=&r"  (result)
+      : [dest]       "b"    (dest),
+        [bits]       "r"    (bits)
+      : "cc", "memory" );
+
+    post_membar(order);
+    return old_value;
+  }
+
+  template<typename T>
+  T fetch_then_or(T volatile* dest, T bits, atomic_memory_order order) const {
+    STATIC_ASSERT(4 == sizeof(T));
+    T old_value, result;
+
+    pre_membar(order);
+
+    __asm__ __volatile__ (
+      "1: lwarx   %[old_value], 0, %[dest]            \n"
+      "   or      %[result], %[old_value], %[bits]    \n"
+      "   stwcx.  %[result], 0, %[dest]               \n"
+      "   bne-    1b                                  \n"
+      : [old_value]  "=&r"  (old_value),
+        [result]     "=&r"  (result)
+      : [dest]       "b"    (dest),
+        [bits]       "r"    (bits)
+      : "cc", "memory" );
+
+    post_membar(order);
+    return old_value;
+  }
+
+  template<typename T>
+  T fetch_then_xor(T volatile* dest, T bits, atomic_memory_order order) const {
+    STATIC_ASSERT(4 == sizeof(T));
+    T old_value, result;
+
+    pre_membar(order);
+
+    __asm__ __volatile__ (
+      "1: lwarx   %[old_value], 0, %[dest]            \n"
+      "   xor     %[result], %[old_value], %[bits]    \n"
+      "   stwcx.  %[result], 0, %[dest]               \n"
+      "   bne-    1b                                  \n"
+      : [old_value]  "=&r"  (old_value),
+        [result]     "=&r"  (result)
+      : [dest]       "b"    (dest),
+        [bits]       "r"    (bits)
+      : "cc", "memory" );
+
+    post_membar(order);
+    return old_value;
+  }
+
+  template<typename T>
+  T and_then_fetch(T volatile* dest, T bits, atomic_memory_order order) const {
+    STATIC_ASSERT(4 == sizeof(T));
+    T result;
+
+    pre_membar(order);
+
+    __asm__ __volatile__ (
+      "1: lwarx   %[result], 0, %[dest]            \n"
+      "   and     %[result], %[result], %[bits]    \n"
+      "   stwcx.  %[result], 0, %[dest]            \n"
+      "   bne-    1b                               \n"
+      : [result]  "=&r"  (result)
+      : [dest]    "b"    (dest),
+        [bits]    "r"    (bits)
+      : "cc", "memory" );
+
+    post_membar(order);
+    return result;
+  }
+
+  template<typename T>
+  T or_then_fetch(T volatile* dest, T bits, atomic_memory_order order) const {
+    STATIC_ASSERT(4 == sizeof(T));
+    T result;
+
+    pre_membar(order);
+
+    __asm__ __volatile__ (
+      "1: lwarx   %[result], 0, %[dest]            \n"
+      "   or      %[result], %[result], %[bits]    \n"
+      "   stwcx.  %[result], 0, %[dest]            \n"
+      "   bne-    1b                               \n"
+      : [result]  "=&r"  (result)
+      : [dest]    "b"    (dest),
+        [bits]    "r"    (bits)
+      : "cc", "memory" );
+
+    post_membar(order);
+    return result;
+  }
+
+  template<typename T>
+  T xor_then_fetch(T volatile* dest, T bits, atomic_memory_order order) const {
+    STATIC_ASSERT(4 == sizeof(T));
+    T result;
+
+    pre_membar(order);
+
+    __asm__ __volatile__ (
+      "1: lwarx   %[result], 0, %[dest]            \n"
+      "   xor     %[result], %[result], %[bits]    \n"
+      "   stwcx.  %[result], 0, %[dest]            \n"
+      "   bne-    1b                               \n"
+      : [result]  "=&r"  (result)
+      : [dest]    "b"    (dest),
+        [bits]    "r"    (bits)
+      : "cc", "memory" );
+
+    post_membar(order);
+    return result;
+  }
+};
+
+template<>
+class AtomicAccess::PlatformBitops<8, true> {
+public:
+  template<typename T>
+  T fetch_then_and(T volatile* dest, T bits, atomic_memory_order order) const {
+    STATIC_ASSERT(8 == sizeof(T));
+    T old_value, result;
+
+    pre_membar(order);
+
+    __asm__ __volatile__ (
+      "1: ldarx   %[old_value], 0, %[dest]            \n"
+      "   and     %[result], %[old_value], %[bits]    \n"
+      "   stdcx.  %[result], 0, %[dest]               \n"
+      "   bne-    1b                                  \n"
+      : [old_value]  "=&r"  (old_value),
+        [result]     "=&r"  (result)
+      : [dest]       "b"    (dest),
+        [bits]       "r"    (bits)
+      : "cc", "memory" );
+
+    post_membar(order);
+    return old_value;
+  }
+
+  template<typename T>
+  T fetch_then_or(T volatile* dest, T bits, atomic_memory_order order) const {
+    STATIC_ASSERT(8 == sizeof(T));
+    T old_value, result;
+
+    pre_membar(order);
+
+    __asm__ __volatile__ (
+      "1: ldarx   %[old_value], 0, %[dest]            \n"
+      "   or      %[result], %[old_value], %[bits]    \n"
+      "   stdcx.  %[result], 0, %[dest]               \n"
+      "   bne-    1b                                  \n"
+      : [old_value]  "=&r"  (old_value),
+        [result]     "=&r"  (result)
+      : [dest]       "b"    (dest),
+        [bits]       "r"    (bits)
+      : "cc", "memory" );
+
+    post_membar(order);
+    return old_value;
+  }
+
+  template<typename T>
+  T fetch_then_xor(T volatile* dest, T bits, atomic_memory_order order) const {
+    STATIC_ASSERT(8 == sizeof(T));
+    T old_value, result;
+
+    pre_membar(order);
+
+    __asm__ __volatile__ (
+      "1: ldarx   %[old_value], 0, %[dest]            \n"
+      "   xor     %[result], %[old_value], %[bits]    \n"
+      "   stdcx.  %[result], 0, %[dest]               \n"
+      "   bne-    1b                                  \n"
+      : [old_value]  "=&r"  (old_value),
+        [result]     "=&r"  (result)
+      : [dest]       "b"    (dest),
+        [bits]       "r"    (bits)
+      : "cc", "memory" );
+
+    post_membar(order);
+    return old_value;
+  }
+
+  template<typename T>
+  T and_then_fetch(T volatile* dest, T bits, atomic_memory_order order) const {
+    STATIC_ASSERT(8 == sizeof(T));
+    T result;
+
+    pre_membar(order);
+
+    __asm__ __volatile__ (
+      "1: ldarx   %[result], 0, %[dest]            \n"
+      "   and     %[result], %[result], %[bits]    \n"
+      "   stdcx.  %[result], 0, %[dest]            \n"
+      "   bne-    1b                               \n"
+      : [result]     "=&r"  (result)
+      : [dest]       "b"    (dest),
+        [bits]       "r"    (bits)
+      : "cc", "memory" );
+
+    post_membar(order);
+    return result;
+  }
+
+  template<typename T>
+  T or_then_fetch(T volatile* dest, T bits, atomic_memory_order order) const {
+    STATIC_ASSERT(8 == sizeof(T));
+    T result;
+
+    pre_membar(order);
+
+    __asm__ __volatile__ (
+      "1: ldarx   %[result], 0, %[dest]            \n"
+      "   or      %[result], %[result], %[bits]    \n"
+      "   stdcx.  %[result], 0, %[dest]            \n"
+      "   bne-    1b                               \n"
+      : [result]     "=&r"  (result)
+      : [dest]       "b"    (dest),
+        [bits]       "r"    (bits)
+      : "cc", "memory" );
+
+    post_membar(order);
+    return result;
+  }
+
+  template<typename T>
+  T xor_then_fetch(T volatile* dest, T bits, atomic_memory_order order) const {
+    STATIC_ASSERT(8 == sizeof(T));
+    T result;
+
+    pre_membar(order);
+
+    __asm__ __volatile__ (
+      "1: ldarx   %[result], 0, %[dest]            \n"
+      "   xor     %[result], %[result], %[bits]    \n"
+      "   stdcx.  %[result], 0, %[dest]            \n"
+      "   bne-    1b                               \n"
+      : [result]     "=&r"  (result)
+      : [dest]       "b"    (dest),
+        [bits]       "r"    (bits)
+      : "cc", "memory" );
+
+    post_membar(order);
+    return result;
+  }
+};
+#endif // CPU_PPC_ATOMICACCESS_PPC_HPP

src/hotspot/cpu/ppc/gc/shared/barrierSetAssembler_ppc.cpp

@@ -193,14 +193,19 @@ void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm, Register t
 
   // Low order half of 64 bit value is currently used.
   __ ld(R0, in_bytes(bs_nm->thread_disarmed_guard_value_offset()), R16_thread);
-  __ cmpw(CR0, R0, tmp);
 
-  // Load stub address using toc (fixed instruction size, unlike load_const_optimized)
-  __ calculate_address_from_global_toc(tmp, StubRoutines::method_entry_barrier(),
-                                       true, true, false); // 2 instructions
-  __ mtctr(tmp);
+  if (TrapBasedNMethodEntryBarriers) {
+    __ tw(Assembler::traptoLessThanUnsigned | Assembler::traptoGreaterThanUnsigned, R0, tmp);
+  } else {
+    __ cmpw(CR0, R0, tmp);
 
-  __ bnectrl(CR0);
+    // Load stub address using toc (fixed instruction size, unlike load_const_optimized)
+    __ calculate_address_from_global_toc(tmp, StubRoutines::method_entry_barrier(),
+                                         true, true, false); // 2 instructions
+    __ mtctr(tmp);
+
+    __ bnectrl(CR0);
+  }
 
   // Oops may have been changed. Make those updates observable.
   // "isync" can serve both, data and instruction patching.

src/hotspot/cpu/ppc/gc/shared/barrierSetNMethod_ppc.cpp

@@ -100,17 +100,21 @@ public:
 
     verify_op_code(current_instruction, Assembler::LD_OPCODE);
 
-    // cmpw (mnemonic)
-    verify_op_code(current_instruction, Assembler::CMP_OPCODE);
+    if (TrapBasedNMethodEntryBarriers) {
+      verify_op_code(current_instruction, Assembler::TW_OPCODE);
+    } else {
+      // cmpw (mnemonic)
+      verify_op_code(current_instruction, Assembler::CMP_OPCODE);
 
-    // calculate_address_from_global_toc (compound instruction)
-    verify_op_code_manually(current_instruction, MacroAssembler::is_addis(*current_instruction));
-    verify_op_code_manually(current_instruction, MacroAssembler::is_addi(*current_instruction));
+      // calculate_address_from_global_toc (compound instruction)
+      verify_op_code_manually(current_instruction, MacroAssembler::is_addis(*current_instruction));
+      verify_op_code_manually(current_instruction, MacroAssembler::is_addi(*current_instruction));
 
-    verify_op_code_manually(current_instruction, MacroAssembler::is_mtctr(*current_instruction));
+      verify_op_code_manually(current_instruction, MacroAssembler::is_mtctr(*current_instruction));
 
-    // bnectrl (mnemonic) (weak check; not checking the exact type)
-    verify_op_code(current_instruction, Assembler::BCCTR_OPCODE);
+      // bnectrl (mnemonic) (weak check; not checking the exact type)
+      verify_op_code(current_instruction, Assembler::BCCTR_OPCODE);
+    }
 
     // isync is optional
   }
@@ -131,9 +135,10 @@ private:
 
 static NativeNMethodBarrier* get_nmethod_barrier(nmethod* nm) {
   BarrierSetAssembler* bs_asm = BarrierSet::barrier_set()->barrier_set_assembler();
-  address barrier_address = nm->code_begin() + nm->frame_complete_offset() + (-8 * 4);
+  address barrier_address = nm->code_begin() + nm->frame_complete_offset() -
+                            (TrapBasedNMethodEntryBarriers ? 4 : 8) * BytesPerInstWord;
   if (bs_asm->nmethod_patching_type() != NMethodPatchingType::stw_instruction_and_data_patch) {
-    barrier_address -= 4; // isync (see nmethod_entry_barrier)
+    barrier_address -= BytesPerInstWord; // isync (see nmethod_entry_barrier)
   }
 
   auto barrier = reinterpret_cast<NativeNMethodBarrier*>(barrier_address);

src/hotspot/cpu/ppc/globals_ppc.hpp

@@ -145,6 +145,8 @@ define_pd_global(intx, InitArrayShortSize, 9*BytesPerLong);
           "switch off all optimizations requiring SIGTRAP.")                \
   product(bool, TrapBasedICMissChecks, true, DIAGNOSTIC,                    \
           "Raise and handle SIGTRAP if inline cache miss detected.")        \
+  product(bool, TrapBasedNMethodEntryBarriers, true, DIAGNOSTIC,            \
+          "Raise and handle SIGTRAP if nmethod entry barrier armed.")       \
                                                                             \
   product(bool, TraceTraps, false, DIAGNOSTIC,                              \
           "Trace all traps the signal handler handles.")                    \

src/hotspot/cpu/ppc/javaFrameAnchor_ppc.hpp

@@ -33,17 +33,17 @@ public:
   //  2 - saving a current state (javaCalls)
   //  3 - restoring an old state (javaCalls)
 
+  // No hardware barriers are necessary. All members are volatile and the profiler
+  // is run from a signal handler and only observers the thread its running on.
+
   inline void clear(void) {
     // clearing _last_Java_sp must be first
     _last_Java_sp = nullptr;
-    // fence?
-    OrderAccess::release();
     _last_Java_pc = nullptr;
   }
 
   inline void set(intptr_t* sp, address pc) {
     _last_Java_pc = pc;
-    OrderAccess::release();
     _last_Java_sp = sp;
   }
 
@@ -56,11 +56,9 @@ public:
     // unless the value is changing.
     if (_last_Java_sp != src->_last_Java_sp) {
       _last_Java_sp = nullptr;
-      OrderAccess::release();
     }
     _last_Java_pc = src->_last_Java_pc;
     // Must be last so profiler will always see valid frame if has_last_frame() is true.
-    OrderAccess::release();
     _last_Java_sp = src->_last_Java_sp;
   }
 
@@ -75,6 +73,6 @@ public:
 
   intptr_t* last_Java_fp() const      { return *(intptr_t**)_last_Java_sp; }
 
-  void set_last_Java_sp(intptr_t* sp) { OrderAccess::release(); _last_Java_sp = sp; }
+  void set_last_Java_sp(intptr_t* sp) { _last_Java_sp = sp; }
 
 #endif // CPU_PPC_JAVAFRAMEANCHOR_PPC_HPP

src/hotspot/cpu/ppc/nativeInst_ppc.hpp

@@ -77,6 +77,12 @@ class NativeInstruction {
   }
 #endif
 
+  bool is_sigtrap_nmethod_entry_barrier() {
+    assert(UseSIGTRAP && TrapBasedNMethodEntryBarriers, "precondition");
+    return Assembler::is_tw(long_at(0), Assembler::traptoLessThanUnsigned | Assembler::traptoGreaterThanUnsigned,
+                            0, -1);
+  }
+
   bool is_safepoint_poll() {
     // The current arguments of the instruction are not checked!
     if (USE_POLL_BIT_ONLY) {

src/hotspot/cpu/ppc/orderAccess_ppc.hpp

@@ -0,0 +1,86 @@
+/*
+ * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2025 SAP SE. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef CPU_PPC_ORDERACCESS_PPC_HPP
+#define CPU_PPC_ORDERACCESS_PPC_HPP
+
+#ifndef PPC64
+#error "OrderAccess currently only implemented for PPC64"
+#endif
+
+// Compiler version last used for testing: gcc 4.1.2
+// Please update this information when this file changes
+
+// Implementation of class OrderAccess.
+
+//
+// Machine barrier instructions:
+//
+// - sync            Two-way memory barrier, aka fence.
+// - lwsync          orders  Store|Store,
+//                            Load|Store,
+//                            Load|Load,
+//                   but not Store|Load
+// - eieio           orders  Store|Store
+// - isync           Invalidates speculatively executed instructions,
+//                   but isync may complete before storage accesses
+//                   associated with instructions preceding isync have
+//                   been performed.
+//
+// Semantic barrier instructions:
+// (as defined in orderAccess.hpp)
+//
+// - release         orders Store|Store,       (maps to lwsync)
+//                           Load|Store
+// - acquire         orders  Load|Store,       (maps to lwsync)
+//                           Load|Load
+// - fence           orders Store|Store,       (maps to sync)
+//                           Load|Store,
+//                           Load|Load,
+//                          Store|Load
+//
+
+#define inlasm_sync()     __asm__ __volatile__ ("sync"   : : : "memory");
+#define inlasm_lwsync()   __asm__ __volatile__ ("lwsync" : : : "memory");
+#define inlasm_eieio()    __asm__ __volatile__ ("eieio"  : : : "memory");
+#define inlasm_isync()    __asm__ __volatile__ ("isync"  : : : "memory");
+
+inline void   OrderAccess::loadload()   { inlasm_lwsync(); }
+inline void   OrderAccess::storestore() { inlasm_lwsync(); }
+inline void   OrderAccess::loadstore()  { inlasm_lwsync(); }
+inline void   OrderAccess::storeload()  { inlasm_sync();   }
+
+inline void   OrderAccess::acquire()    { inlasm_lwsync(); }
+inline void   OrderAccess::release()    { inlasm_lwsync(); }
+inline void   OrderAccess::fence()      { inlasm_sync();   }
+inline void   OrderAccess::cross_modify_fence_impl()
+                                        { inlasm_isync();  }
+
+#undef inlasm_sync
+#undef inlasm_lwsync
+#undef inlasm_eieio
+#undef inlasm_isync
+
+#endif // CPU_PPC_ORDERACCESS_PPC_HPP

src/hotspot/cpu/ppc/templateTable_ppc_64.cpp

@@ -2179,17 +2179,11 @@ void TemplateTable::_return(TosState state) {
 //   - Rscratch
 void TemplateTable::resolve_cache_and_index_for_method(int byte_no, Register Rcache, Register Rscratch) {
   assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
-  Label Lresolved, Ldone, L_clinit_barrier_slow;
+
+  Label L_clinit_barrier_slow, L_done;
   Register Rindex = Rscratch;
 
   Bytecodes::Code code = bytecode();
-  switch (code) {
-    case Bytecodes::_nofast_getfield: code = Bytecodes::_getfield; break;
-    case Bytecodes::_nofast_putfield: code = Bytecodes::_putfield; break;
-    default:
-      break;
-  }
-
   const int bytecode_offset = (byte_no == f1_byte) ? in_bytes(ResolvedMethodEntry::bytecode1_offset())
                                                    : in_bytes(ResolvedMethodEntry::bytecode2_offset());
   __ load_method_entry(Rcache, Rindex);
@@ -2197,20 +2191,8 @@ void TemplateTable::resolve_cache_and_index_for_method(int byte_no, Register Rca
   __ lbz(Rscratch, bytecode_offset, Rcache);
   // Acquire by cmp-br-isync (see below).
   __ cmpdi(CR0, Rscratch, (int)code);
-  __ beq(CR0, Lresolved);
+  __ bne(CR0, L_clinit_barrier_slow);
 
-  // Class initialization barrier slow path lands here as well.
-  __ bind(L_clinit_barrier_slow);
-
-  address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache);
-  __ li(R4_ARG2, code);
-  __ call_VM(noreg, entry, R4_ARG2, true);
-
-  // Update registers with resolved info.
-  __ load_method_entry(Rcache, Rindex);
-  __ b(Ldone);
-
-  __ bind(Lresolved);
   __ isync(); // Order load wrt. succeeding loads.
 
   // Class initialization barrier for static methods
@@ -2220,18 +2202,26 @@ void TemplateTable::resolve_cache_and_index_for_method(int byte_no, Register Rca
 
     __ ld(method, in_bytes(ResolvedMethodEntry::method_offset()), Rcache);
     __ load_method_holder(klass, method);
-    __ clinit_barrier(klass, R16_thread, nullptr /*L_fast_path*/, &L_clinit_barrier_slow);
+    __ clinit_barrier(klass, R16_thread, &L_done, /*L_slow_path*/ nullptr);
+  } else {
+    __ b(L_done);
   }
 
-  __ bind(Ldone);
+  // Class initialization barrier slow path lands here as well.
+  __ bind(L_clinit_barrier_slow);
+  address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache);
+  __ li(R4_ARG2, code);
+  __ call_VM(noreg, entry, R4_ARG2);
+
+  // Update registers with resolved info.
+  __ load_method_entry(Rcache, Rindex);
+  __ bind(L_done);
 }
 
-void TemplateTable::resolve_cache_and_index_for_field(int byte_no,
-                                            Register Rcache,
-                                            Register index) {
+void TemplateTable::resolve_cache_and_index_for_field(int byte_no, Register Rcache, Register index) {
   assert_different_registers(Rcache, index);
 
-  Label resolved;
+  Label L_clinit_barrier_slow, L_done;
 
   Bytecodes::Code code = bytecode();
   switch (code) {
@@ -2246,19 +2236,34 @@ void TemplateTable::resolve_cache_and_index_for_field(int byte_no,
                                          : in_bytes(ResolvedFieldEntry::put_code_offset());
   __ lbz(R0, code_offset, Rcache);
   __ cmpwi(CR0, R0, (int)code); // have we resolved this bytecode?
-  __ beq(CR0, resolved);
+  __ bne(CR0, L_clinit_barrier_slow);
+
+  __ isync(); // Order load wrt. succeeding loads.
+
+  // Class initialization barrier for static fields
+  if (VM_Version::supports_fast_class_init_checks() &&
+      (bytecode() == Bytecodes::_getstatic || bytecode() == Bytecodes::_putstatic)) {
+    const Register field_holder = R4_ARG2;
+
+    // InterpreterRuntime::resolve_get_put sets field_holder and finally release-stores put_code.
+    // We have seen the released put_code above and will read the corresponding field_holder and init_state
+    // (ordered by compare-branch-isync).
+    __ ld(field_holder, ResolvedFieldEntry::field_holder_offset(), Rcache);
+    __ clinit_barrier(field_holder, R16_thread, &L_done, /*L_slow_path*/ nullptr);
+  } else {
+    __ b(L_done);
+  }
 
   // resolve first time through
+  // Class initialization barrier slow path lands here as well.
+  __ bind(L_clinit_barrier_slow);
   address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache);
-  __ li(R4_ARG2, (int)code);
+  __ li(R4_ARG2, code);
   __ call_VM(noreg, entry, R4_ARG2);
 
   // Update registers with resolved info
   __ load_field_entry(Rcache, index);
-  __ bind(resolved);
-
-  // Use acquire semantics for the bytecode (see ResolvedFieldEntry::fill_in()).
-  __ isync(); // Order load wrt. succeeding loads.
+  __ bind(L_done);
 }
 
 void TemplateTable::load_resolved_field_entry(Register obj,

src/hotspot/cpu/ppc/vm_version_ppc.cpp

@@ -87,8 +87,10 @@ void VM_Version::initialize() {
   if (!UseSIGTRAP) {
     MSG(TrapBasedICMissChecks);
     MSG(TrapBasedNullChecks);
-    FLAG_SET_ERGO(TrapBasedNullChecks,       false);
-    FLAG_SET_ERGO(TrapBasedICMissChecks,     false);
+    MSG(TrapBasedNMethodEntryBarriers);
+    FLAG_SET_ERGO(TrapBasedNullChecks,           false);
+    FLAG_SET_ERGO(TrapBasedICMissChecks,         false);
+    FLAG_SET_ERGO(TrapBasedNMethodEntryBarriers, false);
   }
 
 #ifdef COMPILER2
@@ -97,6 +99,10 @@ void VM_Version::initialize() {
     FLAG_SET_ERGO(TrapBasedRangeChecks, false);
   }
 
+  if (FLAG_IS_DEFAULT(UsePopCountInstruction)) {
+    FLAG_SET_ERGO(UsePopCountInstruction, true);
+  }
+
   if (PowerArchitecturePPC64 >= 9) {
     // Performance is good since Power9.
     if (FLAG_IS_DEFAULT(SuperwordUseVSX)) {

src/hotspot/cpu/ppc/vm_version_ppc.hpp

@@ -64,6 +64,7 @@ public:
   constexpr static bool supports_stack_watermark_barrier() { return true; }
   constexpr static bool supports_recursive_lightweight_locking() { return true; }
   constexpr static bool supports_secondary_supers_table() { return true; }
+  constexpr static bool supports_misaligned_vector_accesses() { return true; }
 
   static bool supports_float16() { return PowerArchitecturePPC64 >= 9; }
 

src/hotspot/cpu/riscv/interp_masm_riscv.cpp

@@ -1841,6 +1841,15 @@ void InterpreterMacroAssembler::load_method_entry(Register cache, Register index
 }
 
 #ifdef ASSERT
+void InterpreterMacroAssembler::verify_field_offset(Register reg) {
+  // Verify the field offset is not in the header, implicitly checks for 0
+  Label L;
+  mv(t0, oopDesc::base_offset_in_bytes());
+  bge(reg, t0, L);
+  stop("bad field offset");
+  bind(L);
+}
+
 void InterpreterMacroAssembler::verify_access_flags(Register access_flags, uint32_t flag,
                                                     const char* msg, bool stop_by_hit) {
   Label L;

src/hotspot/cpu/riscv/interp_masm_riscv.hpp

@@ -300,6 +300,8 @@ class InterpreterMacroAssembler: public MacroAssembler {
   void load_field_entry(Register cache, Register index, int bcp_offset = 1);
   void load_method_entry(Register cache, Register index, int bcp_offset = 1);
 
+  void verify_field_offset(Register reg) NOT_DEBUG_RETURN;
+
 #ifdef ASSERT
   void verify_access_flags(Register access_flags, uint32_t flag,
                            const char* msg, bool stop_by_hit = true);

src/hotspot/cpu/riscv/riscv.ad

@@ -1093,39 +1093,39 @@ RegMask _NO_SPECIAL_NO_FP_PTR_REG_mask;
 void reg_mask_init() {
 
   _ANY_REG32_mask = _ALL_REG32_mask;
-  _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(x0->as_VMReg()));
+  _ANY_REG32_mask.remove(OptoReg::as_OptoReg(x0->as_VMReg()));
 
   _ANY_REG_mask = _ALL_REG_mask;
-  _ANY_REG_mask.SUBTRACT(_ZR_REG_mask);
+  _ANY_REG_mask.subtract(_ZR_REG_mask);
 
   _PTR_REG_mask = _ALL_REG_mask;
-  _PTR_REG_mask.SUBTRACT(_ZR_REG_mask);
+  _PTR_REG_mask.subtract(_ZR_REG_mask);
 
   _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
-  _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
+  _NO_SPECIAL_REG32_mask.subtract(_NON_ALLOCATABLE_REG32_mask);
 
   _NO_SPECIAL_REG_mask = _ALL_REG_mask;
-  _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
+  _NO_SPECIAL_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
 
   _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
-  _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
+  _NO_SPECIAL_PTR_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
 
   // x27 is not allocatable when compressed oops is on
   if (UseCompressedOops) {
-    _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(x27->as_VMReg()));
-    _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(x27->as_VMReg()));
-    _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(x27->as_VMReg()));
+    _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(x27->as_VMReg()));
+    _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(x27->as_VMReg()));
+    _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(x27->as_VMReg()));
   }
 
   // x8 is not allocatable when PreserveFramePointer is on
   if (PreserveFramePointer) {
-    _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(x8->as_VMReg()));
-    _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(x8->as_VMReg()));
-    _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(x8->as_VMReg()));
+    _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(x8->as_VMReg()));
+    _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(x8->as_VMReg()));
+    _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(x8->as_VMReg()));
   }
 
   _NO_SPECIAL_NO_FP_PTR_REG_mask = _NO_SPECIAL_PTR_REG_mask;
-  _NO_SPECIAL_NO_FP_PTR_REG_mask.Remove(OptoReg::as_OptoReg(x8->as_VMReg()));
+  _NO_SPECIAL_NO_FP_PTR_REG_mask.remove(OptoReg::as_OptoReg(x8->as_VMReg()));
 }
 
 void PhaseOutput::pd_perform_mach_node_analysis() {
@@ -1326,7 +1326,7 @@ uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
   }
 
 //=============================================================================
-const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
+const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::EMPTY;
 
 int ConstantTable::calculate_table_base_offset() const {
   return 0;  // absolute addressing, no offset
@@ -2104,10 +2104,10 @@ uint Matcher::int_pressure_limit()
   // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
   // derived pointers and lastly fail to spill after reaching maximum
   // number of iterations. Lowering the default pressure threshold to
-  // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
+  // (_NO_SPECIAL_REG32_mask.size() minus 1) forces CallNode to become
   // a high register pressure area of the code so that split_DEF can
   // generate DefinitionSpillCopy for the derived pointer.
-  uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
+  uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.size() - 1;
   if (!PreserveFramePointer) {
     // When PreserveFramePointer is off, frame pointer is allocatable,
     // but different from other SOC registers, it is excluded from
@@ -2122,7 +2122,7 @@ uint Matcher::int_pressure_limit()
 uint Matcher::float_pressure_limit()
 {
   // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
-  return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
+  return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.size() : FLOATPRESSURE;
 }
 
 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {

src/hotspot/cpu/riscv/templateTable_riscv.cpp

@@ -133,6 +133,7 @@ Address TemplateTable::at_bcp(int offset) {
 void TemplateTable::patch_bytecode(Bytecodes::Code bc, Register bc_reg,
                                    Register temp_reg, bool load_bc_into_bc_reg /*=true*/,
                                    int byte_no) {
+  assert_different_registers(bc_reg, temp_reg);
   if (!RewriteBytecodes) { return; }
   Label L_patch_done;
 
@@ -196,7 +197,11 @@ void TemplateTable::patch_bytecode(Bytecodes::Code bc, Register bc_reg,
   __ bind(L_okay);
 #endif
 
-  // patch bytecode
+  // Patch bytecode with release store to coordinate with ResolvedFieldEntry loads
+  // in fast bytecode codelets. load_field_entry has a memory barrier that gains
+  // the needed ordering, together with control dependency on entering the fast codelet
+  // itself.
+  __ membar(MacroAssembler::LoadStore | MacroAssembler::StoreStore);
   __ sb(bc_reg, at_bcp(0));
   __ bind(L_patch_done);
 }
@@ -2168,7 +2173,7 @@ void TemplateTable::resolve_cache_and_index_for_method(int byte_no,
   assert_different_registers(Rcache, index, temp);
   assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
 
-  Label resolved, clinit_barrier_slow;
+  Label L_clinit_barrier_slow, L_done;
 
   Bytecodes::Code code = bytecode();
   __ load_method_entry(Rcache, index);
@@ -2185,12 +2190,20 @@ void TemplateTable::resolve_cache_and_index_for_method(int byte_no,
   __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
 
   __ mv(t0, (int) code);
-  __ beq(temp, t0, resolved);  // have we resolved this bytecode?
+
+  // Class initialization barrier for static methods
+  if (VM_Version::supports_fast_class_init_checks() && bytecode() == Bytecodes::_invokestatic) {
+    __ bne(temp, t0, L_clinit_barrier_slow);  // have we resolved this bytecode?
+    __ ld(temp, Address(Rcache, in_bytes(ResolvedMethodEntry::method_offset())));
+    __ load_method_holder(temp, temp);
+    __ clinit_barrier(temp, t0, &L_done, /*L_slow_path*/ nullptr);
+    __ bind(L_clinit_barrier_slow);
+  } else {
+    __ beq(temp, t0, L_done);  // have we resolved this bytecode?
+  }
 
   // resolve first time through
   // Class initialization barrier slow path lands here as well.
-  __ bind(clinit_barrier_slow);
-
   address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache);
   __ mv(temp, (int) code);
   __ call_VM(noreg, entry, temp);
@@ -2199,14 +2212,7 @@ void TemplateTable::resolve_cache_and_index_for_method(int byte_no,
   __ load_method_entry(Rcache, index);
   // n.b. unlike x86 Rcache is now rcpool plus the indexed offset
   // so all clients ofthis method must be modified accordingly
-  __ bind(resolved);
-
-  // Class initialization barrier for static methods
-  if (VM_Version::supports_fast_class_init_checks() && bytecode() == Bytecodes::_invokestatic) {
-    __ ld(temp, Address(Rcache, in_bytes(ResolvedMethodEntry::method_offset())));
-    __ load_method_holder(temp, temp);
-    __ clinit_barrier(temp, t0, nullptr, &clinit_barrier_slow);
-  }
+  __ bind(L_done);
 }
 
 void TemplateTable::resolve_cache_and_index_for_field(int byte_no,
@@ -2215,13 +2221,13 @@ void TemplateTable::resolve_cache_and_index_for_field(int byte_no,
   const Register temp = x9;
   assert_different_registers(Rcache, index, temp);
 
-  Label resolved;
+  Label L_clinit_barrier_slow, L_done;
 
   Bytecodes::Code code = bytecode();
   switch (code) {
-  case Bytecodes::_nofast_getfield: code = Bytecodes::_getfield; break;
-  case Bytecodes::_nofast_putfield: code = Bytecodes::_putfield; break;
-  default: break;
+    case Bytecodes::_nofast_getfield: code = Bytecodes::_getfield; break;
+    case Bytecodes::_nofast_putfield: code = Bytecodes::_putfield; break;
+    default: break;
   }
 
   assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
@@ -2235,16 +2241,29 @@ void TemplateTable::resolve_cache_and_index_for_field(int byte_no,
   __ lbu(temp, Address(temp, 0));
   __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
   __ mv(t0, (int) code);  // have we resolved this bytecode?
-  __ beq(temp, t0, resolved);
+
+  // Class initialization barrier for static fields
+  if (VM_Version::supports_fast_class_init_checks() &&
+      (bytecode() == Bytecodes::_getstatic || bytecode() == Bytecodes::_putstatic)) {
+    const Register field_holder = temp;
+
+    __ bne(temp, t0, L_clinit_barrier_slow);
+    __ ld(field_holder, Address(Rcache, in_bytes(ResolvedFieldEntry::field_holder_offset())));
+    __ clinit_barrier(field_holder, t0, &L_done, /*L_slow_path*/ nullptr);
+    __ bind(L_clinit_barrier_slow);
+  } else {
+    __ beq(temp, t0, L_done);
+  }
 
   // resolve first time through
+  // Class initialization barrier slow path lands here as well.
   address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache);
   __ mv(temp, (int) code);
   __ call_VM(noreg, entry, temp);
 
   // Update registers with resolved info
   __ load_field_entry(Rcache, index);
-  __ bind(resolved);
+  __ bind(L_done);
 }
 
 void TemplateTable::load_resolved_field_entry(Register obj,
@@ -3014,6 +3033,7 @@ void TemplateTable::fast_storefield(TosState state) {
 
   // X11: field offset, X12: field holder, X13: flags
   load_resolved_field_entry(x12, x12, noreg, x11, x13);
+  __ verify_field_offset(x11);
 
   {
     Label notVolatile;
@@ -3101,6 +3121,8 @@ void TemplateTable::fast_accessfield(TosState state) {
   __ load_field_entry(x12, x11);
 
   __ load_sized_value(x11, Address(x12, in_bytes(ResolvedFieldEntry::field_offset_offset())), sizeof(int), true /*is_signed*/);
+  __ verify_field_offset(x11);
+
   __ load_unsigned_byte(x13, Address(x12, in_bytes(ResolvedFieldEntry::flags_offset())));
 
   // x10: object
@@ -3156,7 +3178,9 @@ void TemplateTable::fast_xaccess(TosState state) {
   __ ld(x10, aaddress(0));
   // access constant pool cache
   __ load_field_entry(x12, x13, 2);
+
   __ load_sized_value(x11, Address(x12, in_bytes(ResolvedFieldEntry::field_offset_offset())), sizeof(int), true /*is_signed*/);
+  __ verify_field_offset(x11);
 
   // make sure exception is reported in correct bcp range (getfield is
   // next instruction)

src/hotspot/cpu/riscv/vm_version_riscv.cpp

@@ -24,9 +24,14 @@
  *
  */
 
+#include "asm/macroAssembler.hpp"
+#include "asm/macroAssembler.inline.hpp"
 #include "classfile/vmIntrinsics.hpp"
+#include "code/codeBlob.hpp"
+#include "memory/resourceArea.hpp"
 #include "runtime/java.hpp"
 #include "runtime/os.inline.hpp"
+#include "runtime/stubCodeGenerator.hpp"
 #include "runtime/vm_version.hpp"
 #include "utilities/formatBuffer.hpp"
 #include "utilities/macros.hpp"
@@ -34,6 +39,57 @@
 #include <ctype.h>
 
 uint32_t VM_Version::_initial_vector_length = 0;
+address VM_Version::_misaligned_vector_fault_pc1 = nullptr;
+address VM_Version::_misaligned_vector_fault_pc2 = nullptr;
+address VM_Version::_misaligned_vector_continuation_pc = nullptr;
+short short_array[4] = { 0, 0, 0, 0 };
+
+static BufferBlob* stub_blob;
+static const int stub_size = 256;
+
+extern "C" {
+  typedef int (*detect_misaligned_vector_stub_t)();
+}
+
+static detect_misaligned_vector_stub_t detect_misaligned_vector_stub = nullptr;
+
+
+class VM_Version_StubGenerator: public StubCodeGenerator {
+ public:
+
+  VM_Version_StubGenerator(CodeBuffer *c) : StubCodeGenerator(c) {}
+  ~VM_Version_StubGenerator() {}
+
+  address generate_detect_misaligned_vector(address* fault_pc1, address* fault_pc2, address* continuation_pc) {
+    StubCodeMark mark(this, "VM_Version", "detect_misaligned_vector_stub");
+#   define __ _masm->
+    address start = __ pc();
+
+    __ enter();
+    __ mv(x10, zr);
+    __ la(t1, ExternalAddress((address) short_array));
+    __ addi(t1, t1, 1);   // Misaligned address
+    __ vsetivli(x0, 1, Assembler::e16);
+    __ vmv_s_x(v2, zr);
+
+    __ addi(t2, zr, 1);
+    __ vmv_s_x(v1, t2);
+    *fault_pc1 = __ pc();
+    __ vse16_v(v1, t1);   // Misaligned vector store
+
+    *fault_pc2 = __ pc();
+    __ vle16_v(v2, t1);   // Misaligned vector load
+
+    *continuation_pc = __ pc();
+    __ vmv_x_s(x10, v2);
+    __ leave();
+    __ ret();
+
+#   undef __
+
+    return start;
+  }
+};
 
 #define DEF_RV_EXT_FEATURE(PRETTY, LINUX_BIT, FSTRING, FLAGF) \
 VM_Version::ext_##PRETTY##RVExtFeatureValue VM_Version::ext_##PRETTY;
@@ -103,17 +159,6 @@ void VM_Version::common_initialize() {
     useRVA23U64Profile();
   }
 
-  // Enable vendor specific features
-
-  if (mvendorid.enabled()) {
-    // Rivos
-    if (mvendorid.value() == RIVOS) {
-      if (FLAG_IS_DEFAULT(UseConservativeFence)) {
-        FLAG_SET_DEFAULT(UseConservativeFence, false);
-      }
-    }
-  }
-
   if (UseZic64b) {
     if (CacheLineSize != 64) {
       assert(!FLAG_IS_DEFAULT(CacheLineSize), "default cache line size should be 64 bytes");
@@ -178,9 +223,25 @@ void VM_Version::common_initialize() {
       (unaligned_scalar.value() == MISALIGNED_SCALAR_FAST));
   }
 
-  if (FLAG_IS_DEFAULT(AlignVector)) {
-    FLAG_SET_DEFAULT(AlignVector,
-      unaligned_vector.value() != MISALIGNED_VECTOR_FAST);
+  if (UseRVV) {
+    // The hwprobe syscall won't be able to detect support for misaligned vector accesses on old kernels.
+    // Resort to detect_misaligned_vector_support() to see if misaligned vector accesses may trap or not.
+    if (!unaligned_vector.enabled()) {
+      if (AlignVector == false && !VM_Version::detect_misaligned_vector_support()) {
+        warning("Misaligned vector accesses are not supported on this CPU");
+        FLAG_SET_DEFAULT(AlignVector, true);
+      }
+    } else {
+      if (FLAG_IS_DEFAULT(AlignVector)) {
+        FLAG_SET_DEFAULT(AlignVector,
+          unaligned_vector.value() != MISALIGNED_VECTOR_FAST);
+      } else if (AlignVector == false) {
+        if (unaligned_vector.value() != MISALIGNED_VECTOR_FAST) {
+          warning("Misaligned vector accesses are not supported on this CPU");
+          FLAG_SET_DEFAULT(AlignVector, true);
+        }
+      }
+    }
   }
 
 #ifdef __riscv_ztso
@@ -199,7 +260,7 @@ void VM_Version::common_initialize() {
     FLAG_SET_DEFAULT(UsePopCountInstruction, false);
   }
 
-  if (UseZicboz && zicboz_block_size.enabled() && zicboz_block_size.value() > 0) {
+  if (UseZicboz && zicboz_block_size.value() > 0) {
     assert(is_power_of_2(zicboz_block_size.value()), "Sanity");
     if (FLAG_IS_DEFAULT(UseBlockZeroing)) {
       FLAG_SET_DEFAULT(UseBlockZeroing, true);
@@ -492,3 +553,22 @@ bool VM_Version::is_intrinsic_supported(vmIntrinsicID id) {
   }
   return true;
 }
+
+bool VM_Version::detect_misaligned_vector_support() {
+  ResourceMark rm;
+
+  stub_blob = BufferBlob::create("detect_misaligned_vector_stub", stub_size);
+    if (stub_blob == nullptr) {
+      vm_exit_during_initialization("Unable to allocate detect_misaligned_vector_stub");
+    }
+
+    CodeBuffer c(stub_blob);
+    VM_Version_StubGenerator g(&c);
+    detect_misaligned_vector_stub = CAST_TO_FN_PTR(detect_misaligned_vector_stub_t,
+                                                   g.generate_detect_misaligned_vector(
+                                                   &VM_Version::_misaligned_vector_fault_pc1,
+                                                   &VM_Version::_misaligned_vector_fault_pc2,
+                                                   &VM_Version::_misaligned_vector_continuation_pc));
+
+  return detect_misaligned_vector_stub() == 1;
+}

src/hotspot/cpu/riscv/vm_version_riscv.hpp

@@ -52,53 +52,19 @@ class VM_Version : public Abstract_VM_Version {
     const char* const _pretty;
     const bool        _feature_string;
     const uint64_t    _linux_feature_bit;
-    int64_t           _value;
+
    public:
     RVFeatureValue(const char* pretty, int linux_bit_num, bool fstring) :
-      _pretty(pretty), _feature_string(fstring), _linux_feature_bit(nth_bit(linux_bit_num)),
-      _value(-1) {
-    }
-    virtual void enable_feature(int64_t value = 0) {
-      _value = value;
-    }
-    virtual void disable_feature() {
-      _value = -1;
+      _pretty(pretty), _feature_string(fstring), _linux_feature_bit(nth_bit(linux_bit_num)) {
     }
+    virtual void enable_feature(int64_t value = 0) = 0;
+    virtual void disable_feature() = 0;
     const char* pretty()         { return _pretty; }
     uint64_t feature_bit()       { return _linux_feature_bit; }
     bool feature_string()        { return _feature_string; }
-    int64_t value()              { return _value; }
     virtual bool enabled() = 0;
     virtual void update_flag() = 0;
-
-   protected:
-    bool deps_all_enabled(RVFeatureValue* dep0, ...) {
-      assert(dep0 != nullptr, "must not");
-
-      va_list va;
-      va_start(va, dep0);
-      RVFeatureValue* next = dep0;
-      bool enabled = true;
-      while (next != nullptr && enabled) {
-        enabled = next->enabled();
-        next = va_arg(va, RVFeatureValue*);
-      }
-      va_end(va);
-      return enabled;
-    }
-
-    void deps_string(stringStream& ss, RVFeatureValue* dep0, ...) {
-      assert(dep0 != nullptr, "must not");
-      ss.print("%s (%s)", dep0->pretty(), dep0->enabled() ? "enabled" : "disabled");
-
-      va_list va;
-      va_start(va, dep0);
-      RVFeatureValue* next = nullptr;
-      while ((next = va_arg(va, RVFeatureValue*)) != nullptr) {
-        ss.print(", %s (%s)", next->pretty(), next->enabled() ? "enabled" : "disabled");
-      }
-      va_end(va);
-    }
+    virtual void log_enabled() = 0;
   };
 
   #define UPDATE_DEFAULT(flag)           \
@@ -117,8 +83,9 @@ class VM_Version : public Abstract_VM_Version {
   #define UPDATE_DEFAULT_DEP(flag, dep0, ...)                                                               \
   void update_flag() {                                                                                      \
       assert(enabled(), "Must be.");                                                                        \
+      DEBUG_ONLY(verify_deps(dep0, ##__VA_ARGS__));                                                         \
       if (FLAG_IS_DEFAULT(flag)) {                                                                          \
-        if (this->deps_all_enabled(dep0, ##__VA_ARGS__)) {                                                  \
+        if (deps_all_enabled(dep0, ##__VA_ARGS__)) {                                                        \
           FLAG_SET_DEFAULT(flag, true);                                                                     \
         } else {                                                                                            \
           FLAG_SET_DEFAULT(flag, false);                                                                    \
@@ -149,40 +116,96 @@ class VM_Version : public Abstract_VM_Version {
 
   class RVExtFeatureValue : public RVFeatureValue {
     const uint32_t _cpu_feature_index;
+
    public:
     RVExtFeatureValue(const char* pretty, int linux_bit_num, uint32_t cpu_feature_index, bool fstring) :
       RVFeatureValue(pretty, linux_bit_num, fstring),
       _cpu_feature_index(cpu_feature_index) {
     }
+    int cpu_feature_index() {
+      // Can be used to check, for example, v is declared before Zvfh in RV_EXT_FEATURE_FLAGS.
+      return _cpu_feature_index;
+    }
     bool enabled() {
       return RVExtFeatures::current()->support_feature(_cpu_feature_index);
     }
     void enable_feature(int64_t value = 0) {
-      RVFeatureValue::enable_feature(value);
       RVExtFeatures::current()->set_feature(_cpu_feature_index);
     }
     void disable_feature() {
-      RVFeatureValue::disable_feature();
       RVExtFeatures::current()->clear_feature(_cpu_feature_index);
     }
+    void log_enabled();
+
+   protected:
+    bool deps_all_enabled(RVExtFeatureValue* dep0, ...) {
+      assert(dep0 != nullptr, "must not");
+
+      va_list va;
+      va_start(va, dep0);
+      RVExtFeatureValue* next = dep0;
+      bool enabled = true;
+      while (next != nullptr && enabled) {
+        enabled = next->enabled();
+        next = va_arg(va, RVExtFeatureValue*);
+      }
+      va_end(va);
+      return enabled;
+    }
+
+    void deps_string(stringStream& ss, RVExtFeatureValue* dep0, ...) {
+      assert(dep0 != nullptr, "must not");
+      ss.print("%s (%s)", dep0->pretty(), dep0->enabled() ? "enabled" : "disabled");
+
+      va_list va;
+      va_start(va, dep0);
+      RVExtFeatureValue* next = nullptr;
+      while ((next = va_arg(va, RVExtFeatureValue*)) != nullptr) {
+        ss.print(", %s (%s)", next->pretty(), next->enabled() ? "enabled" : "disabled");
+      }
+      va_end(va);
+    }
+
+#ifdef ASSERT
+    void verify_deps(RVExtFeatureValue* dep0, ...) {
+      assert(dep0 != nullptr, "must not");
+      assert(cpu_feature_index() >= 0, "must");
+
+      va_list va;
+      va_start(va, dep0);
+      RVExtFeatureValue* next = dep0;
+      while (next != nullptr) {
+        assert(next->cpu_feature_index() >= 0, "must");
+        // We only need to check depenency relationship for extension flags.
+        // The dependant ones must be declared before this, for example, v must be declared
+        // before Zvfh in RV_EXT_FEATURE_FLAGS. The reason is in setup_cpu_available_features
+        // we need to make sure v is `update_flag`ed before Zvfh, so Zvfh is `update_flag`ed
+        // based on v.
+        assert(cpu_feature_index() > next->cpu_feature_index(), "Invalid");
+        next = va_arg(va, RVExtFeatureValue*);
+      }
+      va_end(va);
+    }
+#endif // ASSERT
   };
 
   class RVNonExtFeatureValue : public RVFeatureValue {
-    bool _enabled;
+    static const int64_t DEFAULT_VALUE = -1;
+    int64_t _value;
+
    public:
     RVNonExtFeatureValue(const char* pretty, int linux_bit_num, bool fstring) :
       RVFeatureValue(pretty, linux_bit_num, fstring),
-      _enabled(false) {
+      _value(DEFAULT_VALUE) {
     }
-    bool enabled()               { return _enabled; }
-    void enable_feature(int64_t value = 0) {
-      RVFeatureValue::enable_feature(value);
-      _enabled = true;
-    }
-    void disable_feature() {
-      RVFeatureValue::disable_feature();
-      _enabled = false;
+    bool enabled() { return _value != DEFAULT_VALUE; }
+    void enable_feature(int64_t value) {
+      assert(value != DEFAULT_VALUE, "Sanity");
+      _value = value;
     }
+    void disable_feature() { _value = DEFAULT_VALUE; }
+    int64_t value() { return _value; }
+    void log_enabled();
   };
 
  public:
@@ -282,14 +305,14 @@ class VM_Version : public Abstract_VM_Version {
   decl(marchid           ,  RV_NO_FLAG_BIT,  false,  NO_UPDATE_DEFAULT)                        \
   /* A unique encoding of the version of the processor implementation. */                      \
   decl(mimpid            ,  RV_NO_FLAG_BIT,  false,  NO_UPDATE_DEFAULT)                        \
+  /* Manufactory JEDEC id encoded, ISA vol 2 3.1.2.. */                                        \
+  decl(mvendorid         ,  RV_NO_FLAG_BIT,  false,  NO_UPDATE_DEFAULT)                        \
   /* SATP bits (number of virtual addr bits) mbare, sv39, sv48, sv57, sv64 */                  \
   decl(satp_mode         ,  RV_NO_FLAG_BIT,  false,  NO_UPDATE_DEFAULT)                        \
   /* Performance of misaligned scalar accesses (unknown, emulated, slow, fast, unsupported) */ \
   decl(unaligned_scalar  ,  RV_NO_FLAG_BIT,  false,  NO_UPDATE_DEFAULT)                        \
   /* Performance of misaligned vector accesses (unknown, unspported, slow, fast) */            \
   decl(unaligned_vector  ,  RV_NO_FLAG_BIT,  false,  NO_UPDATE_DEFAULT)                        \
-  /* Manufactory JEDEC id encoded, ISA vol 2 3.1.2.. */                                        \
-  decl(mvendorid         ,  RV_NO_FLAG_BIT,  false,  NO_UPDATE_DEFAULT)                        \
   decl(zicboz_block_size ,  RV_NO_FLAG_BIT,  false,  NO_UPDATE_DEFAULT)                        \
 
   #define DECLARE_RV_NON_EXT_FEATURE(PRETTY, LINUX_BIT, FSTRING, FLAGF)            \
@@ -479,6 +502,21 @@ private:
 
   constexpr static bool supports_secondary_supers_table() { return true; }
 
+  static bool supports_misaligned_vector_accesses() { return !AlignVector; }
+
+  static bool is_misaligned_vector_fault(address pc) {
+    return pc != nullptr && (pc == _misaligned_vector_fault_pc1 || pc == _misaligned_vector_fault_pc2);
+  }
+
+  static address continuation_for_misaligned_vector_fault(address pc) {
+    assert(_misaligned_vector_continuation_pc != nullptr , "not initialized");
+    return _misaligned_vector_continuation_pc;
+  }
+
+  static address _misaligned_vector_fault_pc1;
+  static address _misaligned_vector_fault_pc2;
+  static address _misaligned_vector_continuation_pc;
+
   static bool supports_on_spin_wait() { return UseZihintpause; }
 
   // RISCV64 supports fast class initialization checks
@@ -491,6 +529,9 @@ private:
 
   // Check intrinsic support
   static bool is_intrinsic_supported(vmIntrinsicID id);
+
+  // Detect misaligned vector support
+  static bool detect_misaligned_vector_support();
 };
 
 #endif // CPU_RISCV_VM_VERSION_RISCV_HPP

src/hotspot/cpu/s390/gc/g1/g1_s390.ad

@@ -356,7 +356,7 @@ instruct g1CompareAndExchangeP(iRegP mem_ptr, rarg5RegP oldval, iRegP_N2P newval
 
     __ z_lgr($res$$Register, $oldval$$Register); // previous content
 
-    __ z_csg($oldval$$Register, $newval$$Register, 0, $mem_ptr$$reg);
+    __ z_csg($res$$Register, $newval$$Register, 0, $mem_ptr$$reg);
 
     write_barrier_post(masm, this,
                        $mem_ptr$$Register /* store_addr */,

src/hotspot/cpu/s390/templateTable_s390.cpp

@@ -2360,7 +2360,7 @@ void TemplateTable::resolve_cache_and_index_for_method(int byte_no,
   assert_different_registers(Rcache, index);
   assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
 
-  Label resolved, clinit_barrier_slow;
+  Label L_clinit_barrier_slow, L_done;
 
   Bytecodes::Code code = bytecode();
   switch (code) {
@@ -2375,27 +2375,30 @@ void TemplateTable::resolve_cache_and_index_for_method(int byte_no,
 
   __ load_method_entry(Rcache, index);
   __ z_cli(Address(Rcache, bc_offset), code);
-  __ z_bre(resolved);
+
+  // Class initialization barrier for static methods
+  if (VM_Version::supports_fast_class_init_checks() && bytecode() == Bytecodes::_invokestatic) {
+    const Register method = Z_R1_scratch;
+    const Register klass  = Z_R1_scratch;
+    __ z_brne(L_clinit_barrier_slow);
+    __ z_lg(method, Address(Rcache, in_bytes(ResolvedMethodEntry::method_offset())));
+    __ load_method_holder(klass, method);
+    __ clinit_barrier(klass, Z_thread, &L_done, /*L_slow_path*/ nullptr);
+    __ bind(L_clinit_barrier_slow);
+  } else {
+    __ z_bre(L_done);
+  }
 
   // Resolve first time through
   // Class initialization barrier slow path lands here as well.
-  __ bind(clinit_barrier_slow);
   address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache);
   __ load_const_optimized(Z_ARG2, (int)code);
   __ call_VM(noreg, entry, Z_ARG2);
 
   // Update registers with resolved info.
   __ load_method_entry(Rcache, index);
-  __ bind(resolved);
 
-  // Class initialization barrier for static methods
-  if (VM_Version::supports_fast_class_init_checks() && bytecode() == Bytecodes::_invokestatic) {
-    const Register method = Z_R1_scratch;
-    const Register klass  = Z_R1_scratch;
-    __ z_lg(method, Address(Rcache, in_bytes(ResolvedMethodEntry::method_offset())));
-    __ load_method_holder(klass, method);
-    __ clinit_barrier(klass, Z_thread, nullptr /*L_fast_path*/, &clinit_barrier_slow);
-  }
+  __ bind(L_done);
 
   BLOCK_COMMENT("} resolve_cache_and_index_for_method");
 }
@@ -2408,7 +2411,7 @@ void TemplateTable::resolve_cache_and_index_for_field(int byte_no,
   assert_different_registers(cache, index);
   assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
 
-  NearLabel resolved;
+  Label L_clinit_barrier_slow, L_done;
 
   Bytecodes::Code code = bytecode();
   switch (code) {
@@ -2422,9 +2425,22 @@ void TemplateTable::resolve_cache_and_index_for_field(int byte_no,
                                                  in_bytes(ResolvedFieldEntry::put_code_offset()) ;
 
   __ z_cli(Address(cache, code_offset), code);
-  __ z_bre(resolved);
+
+  // Class initialization barrier for static fields
+  if (VM_Version::supports_fast_class_init_checks() &&
+      (bytecode() == Bytecodes::_getstatic || bytecode() == Bytecodes::_putstatic)) {
+    const Register field_holder = index;
+
+    __ z_brne(L_clinit_barrier_slow);
+    __ load_sized_value(field_holder, Address(cache, ResolvedFieldEntry::field_holder_offset()), sizeof(void*), false);
+    __ clinit_barrier(field_holder, Z_thread, &L_done, /*L_slow_path*/ nullptr);
+    __ bind(L_clinit_barrier_slow);
+  } else {
+    __ z_bre(L_done);
+  }
 
   // resolve first time through
+  // Class initialization barrier slow path lands here as well.
   address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache);
   __ load_const_optimized(Z_ARG2, (int)code);
   __ call_VM(noreg, entry, Z_ARG2);
@@ -2432,7 +2448,7 @@ void TemplateTable::resolve_cache_and_index_for_field(int byte_no,
   // Update registers with resolved info.
   __ load_field_entry(cache, index);
 
-  __ bind(resolved);
+  __ bind(L_done);
 
   BLOCK_COMMENT("} resolve_cache_and_index_for_field");
 }

src/hotspot/cpu/s390/vm_version_s390.hpp

@@ -425,6 +425,8 @@ class VM_Version: public Abstract_VM_Version {
 
   constexpr static bool supports_secondary_supers_table() { return true; }
 
+  constexpr static bool supports_misaligned_vector_accesses() { return true; }
+
   constexpr static bool supports_recursive_lightweight_locking() { return true; }
 
   // CPU feature query functions

src/hotspot/cpu/x86/gc/shared/barrierSetAssembler_x86.cpp

@@ -471,33 +471,33 @@ void SaveLiveRegisters::initialize(BarrierStubC2* stub) {
   // Create mask of caller saved registers that need to
   // be saved/restored if live
   RegMask caller_saved;
-  caller_saved.Insert(OptoReg::as_OptoReg(rax->as_VMReg()));
-  caller_saved.Insert(OptoReg::as_OptoReg(rcx->as_VMReg()));
-  caller_saved.Insert(OptoReg::as_OptoReg(rdx->as_VMReg()));
-  caller_saved.Insert(OptoReg::as_OptoReg(rsi->as_VMReg()));
-  caller_saved.Insert(OptoReg::as_OptoReg(rdi->as_VMReg()));
-  caller_saved.Insert(OptoReg::as_OptoReg(r8->as_VMReg()));
-  caller_saved.Insert(OptoReg::as_OptoReg(r9->as_VMReg()));
-  caller_saved.Insert(OptoReg::as_OptoReg(r10->as_VMReg()));
-  caller_saved.Insert(OptoReg::as_OptoReg(r11->as_VMReg()));
+  caller_saved.insert(OptoReg::as_OptoReg(rax->as_VMReg()));
+  caller_saved.insert(OptoReg::as_OptoReg(rcx->as_VMReg()));
+  caller_saved.insert(OptoReg::as_OptoReg(rdx->as_VMReg()));
+  caller_saved.insert(OptoReg::as_OptoReg(rsi->as_VMReg()));
+  caller_saved.insert(OptoReg::as_OptoReg(rdi->as_VMReg()));
+  caller_saved.insert(OptoReg::as_OptoReg(r8->as_VMReg()));
+  caller_saved.insert(OptoReg::as_OptoReg(r9->as_VMReg()));
+  caller_saved.insert(OptoReg::as_OptoReg(r10->as_VMReg()));
+  caller_saved.insert(OptoReg::as_OptoReg(r11->as_VMReg()));
 
   if (UseAPX) {
-    caller_saved.Insert(OptoReg::as_OptoReg(r16->as_VMReg()));
-    caller_saved.Insert(OptoReg::as_OptoReg(r17->as_VMReg()));
-    caller_saved.Insert(OptoReg::as_OptoReg(r18->as_VMReg()));
-    caller_saved.Insert(OptoReg::as_OptoReg(r19->as_VMReg()));
-    caller_saved.Insert(OptoReg::as_OptoReg(r20->as_VMReg()));
-    caller_saved.Insert(OptoReg::as_OptoReg(r21->as_VMReg()));
-    caller_saved.Insert(OptoReg::as_OptoReg(r22->as_VMReg()));
-    caller_saved.Insert(OptoReg::as_OptoReg(r23->as_VMReg()));
-    caller_saved.Insert(OptoReg::as_OptoReg(r24->as_VMReg()));
-    caller_saved.Insert(OptoReg::as_OptoReg(r25->as_VMReg()));
-    caller_saved.Insert(OptoReg::as_OptoReg(r26->as_VMReg()));
-    caller_saved.Insert(OptoReg::as_OptoReg(r27->as_VMReg()));
-    caller_saved.Insert(OptoReg::as_OptoReg(r28->as_VMReg()));
-    caller_saved.Insert(OptoReg::as_OptoReg(r29->as_VMReg()));
-    caller_saved.Insert(OptoReg::as_OptoReg(r30->as_VMReg()));
-    caller_saved.Insert(OptoReg::as_OptoReg(r31->as_VMReg()));
+    caller_saved.insert(OptoReg::as_OptoReg(r16->as_VMReg()));
+    caller_saved.insert(OptoReg::as_OptoReg(r17->as_VMReg()));
+    caller_saved.insert(OptoReg::as_OptoReg(r18->as_VMReg()));
+    caller_saved.insert(OptoReg::as_OptoReg(r19->as_VMReg()));
+    caller_saved.insert(OptoReg::as_OptoReg(r20->as_VMReg()));
+    caller_saved.insert(OptoReg::as_OptoReg(r21->as_VMReg()));
+    caller_saved.insert(OptoReg::as_OptoReg(r22->as_VMReg()));
+    caller_saved.insert(OptoReg::as_OptoReg(r23->as_VMReg()));
+    caller_saved.insert(OptoReg::as_OptoReg(r24->as_VMReg()));
+    caller_saved.insert(OptoReg::as_OptoReg(r25->as_VMReg()));
+    caller_saved.insert(OptoReg::as_OptoReg(r26->as_VMReg()));
+    caller_saved.insert(OptoReg::as_OptoReg(r27->as_VMReg()));
+    caller_saved.insert(OptoReg::as_OptoReg(r28->as_VMReg()));
+    caller_saved.insert(OptoReg::as_OptoReg(r29->as_VMReg()));
+    caller_saved.insert(OptoReg::as_OptoReg(r30->as_VMReg()));
+    caller_saved.insert(OptoReg::as_OptoReg(r31->as_VMReg()));
   }
 
   int gp_spill_size = 0;
@@ -511,7 +511,7 @@ void SaveLiveRegisters::initialize(BarrierStubC2* stub) {
     const VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
 
     if (vm_reg->is_Register()) {
-      if (caller_saved.Member(opto_reg)) {
+      if (caller_saved.member(opto_reg)) {
         _gp_registers.append(vm_reg->as_Register());
         gp_spill_size += 8;
       }

src/hotspot/cpu/x86/templateTable_x86.cpp

@@ -2196,8 +2196,7 @@ void TemplateTable::resolve_cache_and_index_for_method(int byte_no,
   const Register temp = rbx;
   assert_different_registers(cache, index, temp);
 
-  Label L_clinit_barrier_slow;
-  Label resolved;
+  Label L_clinit_barrier_slow, L_done;
 
   Bytecodes::Code code = bytecode();
 
@@ -2215,37 +2214,38 @@ void TemplateTable::resolve_cache_and_index_for_method(int byte_no,
       ShouldNotReachHere();
   }
   __ cmpl(temp, code);  // have we resolved this bytecode?
-  __ jcc(Assembler::equal, resolved);
-
-  // resolve first time through
-  // Class initialization barrier slow path lands here as well.
-  __ bind(L_clinit_barrier_slow);
-  address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache);
-  __ movl(temp, code);
-  __ call_VM(noreg, entry, temp);
-  // Update registers with resolved info
-  __ load_method_entry(cache, index);
-
-  __ bind(resolved);
 
   // Class initialization barrier for static methods
   if (VM_Version::supports_fast_class_init_checks() && bytecode() == Bytecodes::_invokestatic) {
     const Register method = temp;
     const Register klass  = temp;
 
+    __ jcc(Assembler::notEqual, L_clinit_barrier_slow);
     __ movptr(method, Address(cache, in_bytes(ResolvedMethodEntry::method_offset())));
     __ load_method_holder(klass, method);
-    __ clinit_barrier(klass, nullptr /*L_fast_path*/, &L_clinit_barrier_slow);
+    __ clinit_barrier(klass, &L_done, /*L_slow_path*/ nullptr);
+    __ bind(L_clinit_barrier_slow);
+  } else {
+    __ jcc(Assembler::equal, L_done);
   }
+
+  // resolve first time through
+  // Class initialization barrier slow path lands here as well.
+  address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache);
+  __ movl(temp, code);
+  __ call_VM(noreg, entry, temp);
+  // Update registers with resolved info
+  __ load_method_entry(cache, index);
+  __ bind(L_done);
 }
 
 void TemplateTable::resolve_cache_and_index_for_field(int byte_no,
-                                            Register cache,
-                                            Register index) {
+                                                      Register cache,
+                                                      Register index) {
   const Register temp = rbx;
   assert_different_registers(cache, index, temp);
 
-  Label resolved;
+  Label L_clinit_barrier_slow, L_done;
 
   Bytecodes::Code code = bytecode();
   switch (code) {
@@ -2262,16 +2262,28 @@ void TemplateTable::resolve_cache_and_index_for_field(int byte_no,
     __ load_unsigned_byte(temp, Address(cache, in_bytes(ResolvedFieldEntry::put_code_offset())));
   }
   __ cmpl(temp, code);  // have we resolved this bytecode?
-  __ jcc(Assembler::equal, resolved);
+
+  // Class initialization barrier for static fields
+  if (VM_Version::supports_fast_class_init_checks() &&
+      (bytecode() == Bytecodes::_getstatic || bytecode() == Bytecodes::_putstatic)) {
+    const Register field_holder = temp;
+
+    __ jcc(Assembler::notEqual, L_clinit_barrier_slow);
+    __ movptr(field_holder, Address(cache, in_bytes(ResolvedFieldEntry::field_holder_offset())));
+    __ clinit_barrier(field_holder, &L_done, /*L_slow_path*/ nullptr);
+    __ bind(L_clinit_barrier_slow);
+  } else {
+    __ jcc(Assembler::equal, L_done);
+  }
 
   // resolve first time through
+  // Class initialization barrier slow path lands here as well.
   address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache);
   __ movl(temp, code);
   __ call_VM(noreg, entry, temp);
   // Update registers with resolved info
   __ load_field_entry(cache, index);
-
-  __ bind(resolved);
+  __ bind(L_done);
 }
 
 void TemplateTable::load_resolved_field_entry(Register obj,

src/hotspot/cpu/x86/vm_version_x86.hpp

@@ -991,6 +991,10 @@ public:
     return true;
   }
 
+  constexpr static bool supports_misaligned_vector_accesses() {
+    return true;
+  }
+
   constexpr static bool supports_stack_watermark_barrier() {
     return true;
   }

src/hotspot/cpu/x86/x86_64.ad

@@ -500,89 +500,89 @@ void reg_mask_init() {
   _ANY_REG_mask = _ALL_REG_mask;
 
   if (PreserveFramePointer) {
-    _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
-    _ANY_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
+    _ANY_REG_mask.remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
+    _ANY_REG_mask.remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
   }
   if (need_r12_heapbase()) {
-    _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
-    _ANY_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()->next()));
+    _ANY_REG_mask.remove(OptoReg::as_OptoReg(r12->as_VMReg()));
+    _ANY_REG_mask.remove(OptoReg::as_OptoReg(r12->as_VMReg()->next()));
   }
 
   _PTR_REG_mask = _ANY_REG_mask;
-  _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()));
-  _PTR_REG_mask.Remove(OptoReg::as_OptoReg(rsp->as_VMReg()->next()));
-  _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()));
-  _PTR_REG_mask.Remove(OptoReg::as_OptoReg(r15->as_VMReg()->next()));
+  _PTR_REG_mask.remove(OptoReg::as_OptoReg(rsp->as_VMReg()));
+  _PTR_REG_mask.remove(OptoReg::as_OptoReg(rsp->as_VMReg()->next()));
+  _PTR_REG_mask.remove(OptoReg::as_OptoReg(r15->as_VMReg()));
+  _PTR_REG_mask.remove(OptoReg::as_OptoReg(r15->as_VMReg()->next()));
   if (!UseAPX) {
     for (uint i = 0; i < sizeof(egprs)/sizeof(Register); i++) {
-      _PTR_REG_mask.Remove(OptoReg::as_OptoReg(egprs[i]->as_VMReg()));
-      _PTR_REG_mask.Remove(OptoReg::as_OptoReg(egprs[i]->as_VMReg()->next()));
+      _PTR_REG_mask.remove(OptoReg::as_OptoReg(egprs[i]->as_VMReg()));
+      _PTR_REG_mask.remove(OptoReg::as_OptoReg(egprs[i]->as_VMReg()->next()));
     }
   }
 
   _STACK_OR_PTR_REG_mask = _PTR_REG_mask;
-  _STACK_OR_PTR_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
+  _STACK_OR_PTR_REG_mask.or_with(STACK_OR_STACK_SLOTS_mask());
 
   _PTR_REG_NO_RBP_mask = _PTR_REG_mask;
-  _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
-  _PTR_REG_NO_RBP_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
+  _PTR_REG_NO_RBP_mask.remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
+  _PTR_REG_NO_RBP_mask.remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
 
   _PTR_NO_RAX_REG_mask = _PTR_REG_mask;
-  _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
-  _PTR_NO_RAX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
+  _PTR_NO_RAX_REG_mask.remove(OptoReg::as_OptoReg(rax->as_VMReg()));
+  _PTR_NO_RAX_REG_mask.remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
 
   _PTR_NO_RAX_RBX_REG_mask = _PTR_NO_RAX_REG_mask;
-  _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()));
-  _PTR_NO_RAX_RBX_REG_mask.Remove(OptoReg::as_OptoReg(rbx->as_VMReg()->next()));
+  _PTR_NO_RAX_RBX_REG_mask.remove(OptoReg::as_OptoReg(rbx->as_VMReg()));
+  _PTR_NO_RAX_RBX_REG_mask.remove(OptoReg::as_OptoReg(rbx->as_VMReg()->next()));
 
 
   _LONG_REG_mask = _PTR_REG_mask;
   _STACK_OR_LONG_REG_mask = _LONG_REG_mask;
-  _STACK_OR_LONG_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
+  _STACK_OR_LONG_REG_mask.or_with(STACK_OR_STACK_SLOTS_mask());
 
   _LONG_NO_RAX_RDX_REG_mask = _LONG_REG_mask;
-  _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
-  _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
-  _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
-  _LONG_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()->next()));
+  _LONG_NO_RAX_RDX_REG_mask.remove(OptoReg::as_OptoReg(rax->as_VMReg()));
+  _LONG_NO_RAX_RDX_REG_mask.remove(OptoReg::as_OptoReg(rax->as_VMReg()->next()));
+  _LONG_NO_RAX_RDX_REG_mask.remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
+  _LONG_NO_RAX_RDX_REG_mask.remove(OptoReg::as_OptoReg(rdx->as_VMReg()->next()));
 
   _LONG_NO_RCX_REG_mask = _LONG_REG_mask;
-  _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
-  _LONG_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()->next()));
+  _LONG_NO_RCX_REG_mask.remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
+  _LONG_NO_RCX_REG_mask.remove(OptoReg::as_OptoReg(rcx->as_VMReg()->next()));
 
   _LONG_NO_RBP_R13_REG_mask = _LONG_REG_mask;
-  _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
-  _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
-  _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
-  _LONG_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()->next()));
+  _LONG_NO_RBP_R13_REG_mask.remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
+  _LONG_NO_RBP_R13_REG_mask.remove(OptoReg::as_OptoReg(rbp->as_VMReg()->next()));
+  _LONG_NO_RBP_R13_REG_mask.remove(OptoReg::as_OptoReg(r13->as_VMReg()));
+  _LONG_NO_RBP_R13_REG_mask.remove(OptoReg::as_OptoReg(r13->as_VMReg()->next()));
 
   _INT_REG_mask = _ALL_INT_REG_mask;
   if (!UseAPX) {
     for (uint i = 0; i < sizeof(egprs)/sizeof(Register); i++) {
-      _INT_REG_mask.Remove(OptoReg::as_OptoReg(egprs[i]->as_VMReg()));
+      _INT_REG_mask.remove(OptoReg::as_OptoReg(egprs[i]->as_VMReg()));
     }
   }
 
   if (PreserveFramePointer) {
-    _INT_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
+    _INT_REG_mask.remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
   }
   if (need_r12_heapbase()) {
-    _INT_REG_mask.Remove(OptoReg::as_OptoReg(r12->as_VMReg()));
+    _INT_REG_mask.remove(OptoReg::as_OptoReg(r12->as_VMReg()));
   }
 
   _STACK_OR_INT_REG_mask = _INT_REG_mask;
-  _STACK_OR_INT_REG_mask.OR(STACK_OR_STACK_SLOTS_mask());
+  _STACK_OR_INT_REG_mask.or_with(STACK_OR_STACK_SLOTS_mask());
 
   _INT_NO_RAX_RDX_REG_mask = _INT_REG_mask;
-  _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rax->as_VMReg()));
-  _INT_NO_RAX_RDX_REG_mask.Remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
+  _INT_NO_RAX_RDX_REG_mask.remove(OptoReg::as_OptoReg(rax->as_VMReg()));
+  _INT_NO_RAX_RDX_REG_mask.remove(OptoReg::as_OptoReg(rdx->as_VMReg()));
 
   _INT_NO_RCX_REG_mask = _INT_REG_mask;
-  _INT_NO_RCX_REG_mask.Remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
+  _INT_NO_RCX_REG_mask.remove(OptoReg::as_OptoReg(rcx->as_VMReg()));
 
   _INT_NO_RBP_R13_REG_mask = _INT_REG_mask;
-  _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
-  _INT_NO_RBP_R13_REG_mask.Remove(OptoReg::as_OptoReg(r13->as_VMReg()));
+  _INT_NO_RBP_R13_REG_mask.remove(OptoReg::as_OptoReg(rbp->as_VMReg()));
+  _INT_NO_RBP_R13_REG_mask.remove(OptoReg::as_OptoReg(r13->as_VMReg()));
 
   // _FLOAT_REG_LEGACY_mask/_FLOAT_REG_EVEX_mask is generated by adlc
   // from the float_reg_legacy/float_reg_evex register class.
@@ -756,7 +756,7 @@ static void emit_fp_min_max(MacroAssembler* masm, XMMRegister dst,
 }
 
 //=============================================================================
-const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
+const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::EMPTY;
 
 int ConstantTable::calculate_table_base_offset() const {
   return 0;  // absolute addressing, no offset
@@ -1658,7 +1658,7 @@ bool Matcher::is_spillable_arg(int reg)
 
 uint Matcher::int_pressure_limit()
 {
-  return (INTPRESSURE == -1) ? _INT_REG_mask.Size() : INTPRESSURE;
+  return (INTPRESSURE == -1) ? _INT_REG_mask.size() : INTPRESSURE;
 }
 
 uint Matcher::float_pressure_limit()
@@ -1666,7 +1666,7 @@ uint Matcher::float_pressure_limit()
   // After experiment around with different values, the following default threshold
   // works best for LCM's register pressure scheduling on x64.
   uint dec_count  = VM_Version::supports_evex() ? 4 : 2;
-  uint default_float_pressure_threshold = _FLOAT_REG_mask.Size() - dec_count;
+  uint default_float_pressure_threshold = _FLOAT_REG_mask.size() - dec_count;
   return (FLOATPRESSURE == -1) ? default_float_pressure_threshold : FLOATPRESSURE;
 }
 

src/hotspot/os/bsd/gc/z/zNUMA_bsd.cpp

@@ -46,3 +46,7 @@ uint32_t ZNUMA::memory_id(uintptr_t addr) {
   // NUMA support not enabled, assume everything belongs to node zero
   return 0;
 }
+
+int ZNUMA::numa_id_to_node(uint32_t numa_id) {
+  ShouldNotCallThis();
+}

src/hotspot/os/linux/gc/z/zNUMA_linux.cpp

@@ -32,12 +32,35 @@
 #include "runtime/os.hpp"
 #include "utilities/debug.hpp"
 
+static uint* z_numa_id_to_node = nullptr;
+static uint32_t* z_node_to_numa_id = nullptr;
+
 void ZNUMA::pd_initialize() {
   _enabled = UseNUMA;
 
+  size_t configured_nodes = 0;
+
+  if (UseNUMA) {
+    const size_t max_nodes = os::Linux::numa_num_configured_nodes();
+    z_numa_id_to_node = NEW_C_HEAP_ARRAY(uint, max_nodes, mtGC);
+    configured_nodes = os::numa_get_leaf_groups(z_numa_id_to_node, 0);
+
+    z_node_to_numa_id = NEW_C_HEAP_ARRAY(uint32_t, max_nodes, mtGC);
+
+    // Fill the array with invalid NUMA ids
+    for (uint32_t i = 0; i < max_nodes; i++) {
+      z_node_to_numa_id[i] = (uint32_t)-1;
+    }
+
+    // Fill the reverse mappings
+    for (uint32_t i = 0; i < configured_nodes; i++) {
+      z_node_to_numa_id[z_numa_id_to_node[i]] = i;
+    }
+  }
+
   // UseNUMA and is_faked() are mutually excluded in zArguments.cpp.
   _count = UseNUMA
-      ? os::Linux::numa_max_node() + 1
+      ? configured_nodes
       : !FLAG_IS_DEFAULT(ZFakeNUMA)
             ? ZFakeNUMA
             : 1;  // No NUMA nodes
@@ -54,7 +77,7 @@ uint32_t ZNUMA::id() {
     return 0;
   }
 
-  return os::Linux::get_node_by_cpu(ZCPU::id());
+  return z_node_to_numa_id[os::Linux::get_node_by_cpu(ZCPU::id())];
 }
 
 uint32_t ZNUMA::memory_id(uintptr_t addr) {
@@ -63,14 +86,21 @@ uint32_t ZNUMA::memory_id(uintptr_t addr) {
     return 0;
   }
 
-  uint32_t id = (uint32_t)-1;
+  int node = -1;
 
-  if (ZSyscall::get_mempolicy((int*)&id, nullptr, 0, (void*)addr, MPOL_F_NODE | MPOL_F_ADDR) == -1) {
+  if (ZSyscall::get_mempolicy(&node, nullptr, 0, (void*)addr, MPOL_F_NODE | MPOL_F_ADDR) == -1) {
     ZErrno err;
     fatal("Failed to get NUMA id for memory at " PTR_FORMAT " (%s)", addr, err.to_string());
   }
 
-  assert(id < _count, "Invalid NUMA id");
+  DEBUG_ONLY(const int max_nodes = os::Linux::numa_num_configured_nodes();)
+  assert(node < max_nodes, "NUMA node is out of bounds node=%d, max=%d", node, max_nodes);
 
-  return id;
+  return z_node_to_numa_id[node];
+}
+
+int ZNUMA::numa_id_to_node(uint32_t numa_id) {
+  assert(numa_id < _count, "NUMA id out of range 0 <= %ud <= %ud", numa_id, _count);
+
+  return (int)z_numa_id_to_node[numa_id];
 }

src/hotspot/os/linux/gc/z/zPhysicalMemoryBacking_linux.cpp

@@ -629,7 +629,7 @@ retry:
 
 size_t ZPhysicalMemoryBacking::commit_numa_preferred(zbacking_offset offset, size_t length, uint32_t numa_id) const {
   // Setup NUMA policy to allocate memory from a preferred node
-  os::Linux::numa_set_preferred((int)numa_id);
+  os::Linux::numa_set_preferred(ZNUMA::numa_id_to_node(numa_id));
 
   const size_t committed = commit_default(offset, length);
 

src/hotspot/os/posix/signals_posix.cpp

@@ -42,6 +42,7 @@
 #include "signals_posix.hpp"
 #include "suspendResume_posix.hpp"
 #include "utilities/checkedCast.hpp"
+#include "utilities/deferredStatic.hpp"
 #include "utilities/events.hpp"
 #include "utilities/ostream.hpp"
 #include "utilities/parseInteger.hpp"
@@ -167,9 +168,9 @@ static get_signal_t get_signal_action = nullptr;
 
 // suspend/resume support
 #if defined(__APPLE__)
-  static OSXSemaphore sr_semaphore;
+static DeferredStatic<OSXSemaphore> sr_semaphore;
 #else
-  static PosixSemaphore sr_semaphore;
+static DeferredStatic<PosixSemaphore> sr_semaphore;
 #endif
 
 // Signal number used to suspend/resume a thread
@@ -177,7 +178,7 @@ static get_signal_t get_signal_action = nullptr;
 int PosixSignals::SR_signum = SIGUSR2;
 
 // sun.misc.Signal support
-static Semaphore* sig_semaphore = nullptr;
+static DeferredStatic<Semaphore> sig_semaphore;
 // a counter for each possible signal value
 static volatile jint pending_signals[NSIG+1] = { 0 };
 
@@ -351,18 +352,17 @@ static void jdk_misc_signal_init() {
   ::memset((void*)pending_signals, 0, sizeof(pending_signals));
 
   // Initialize signal semaphore
-  sig_semaphore = new Semaphore();
+  int sem_count = 0;
+  sig_semaphore.initialize(sem_count);
 }
 
 void os::signal_notify(int sig) {
-  if (sig_semaphore != nullptr) {
+    // Signal thread is not created with ReduceSignalUsage and jdk_misc_signal_init
+    // initialization isn't called. This code is also never called.
+    assert(!ReduceSignalUsage, "Should not reach here if ReduceSignalUsage is set");
+
     AtomicAccess::inc(&pending_signals[sig]);
     sig_semaphore->signal();
-  } else {
-    // Signal thread is not created with ReduceSignalUsage and jdk_misc_signal_init
-    // initialization isn't called.
-    assert(ReduceSignalUsage, "signal semaphore should be created");
-  }
 }
 
 static int check_pending_signals() {
@@ -1696,7 +1696,7 @@ static void SR_handler(int sig, siginfo_t* siginfo, void* context) {
       pthread_sigmask(SIG_BLOCK, nullptr, &suspend_set);
       sigdelset(&suspend_set, PosixSignals::SR_signum);
 
-      sr_semaphore.signal();
+      sr_semaphore->signal();
 
       // wait here until we are resumed
       while (1) {
@@ -1705,7 +1705,7 @@ static void SR_handler(int sig, siginfo_t* siginfo, void* context) {
         SuspendResume::State result = osthread->sr.running();
         if (result == SuspendResume::SR_RUNNING) {
           // double check AIX doesn't need this!
-          sr_semaphore.signal();
+          sr_semaphore->signal();
           break;
         } else if (result != SuspendResume::SR_SUSPENDED) {
           ShouldNotReachHere();
@@ -1731,6 +1731,9 @@ static void SR_handler(int sig, siginfo_t* siginfo, void* context) {
 }
 
 static int SR_initialize() {
+  int sem_count = 0;
+  sr_semaphore.initialize(sem_count);
+
   struct sigaction act;
   char *s;
   // Get signal number to use for suspend/resume
@@ -1778,7 +1781,7 @@ static int sr_notify(OSThread* osthread) {
 // but this seems the normal response to library errors
 bool PosixSignals::do_suspend(OSThread* osthread) {
   assert(osthread->sr.is_running(), "thread should be running");
-  assert(!sr_semaphore.trywait(), "semaphore has invalid state");
+  assert(!sr_semaphore->trywait(), "semaphore has invalid state");
 
   // mark as suspended and send signal
   if (osthread->sr.request_suspend() != SuspendResume::SR_SUSPEND_REQUEST) {
@@ -1793,7 +1796,7 @@ bool PosixSignals::do_suspend(OSThread* osthread) {
 
   // managed to send the signal and switch to SUSPEND_REQUEST, now wait for SUSPENDED
   while (true) {
-    if (sr_semaphore.timedwait(2)) {
+    if (sr_semaphore->timedwait(2)) {
       break;
     } else {
       // timeout
@@ -1802,7 +1805,7 @@ bool PosixSignals::do_suspend(OSThread* osthread) {
         return false;
       } else if (cancelled == SuspendResume::SR_SUSPENDED) {
         // make sure that we consume the signal on the semaphore as well
-        sr_semaphore.wait();
+        sr_semaphore->wait();
         break;
       } else {
         ShouldNotReachHere();
@@ -1817,7 +1820,7 @@ bool PosixSignals::do_suspend(OSThread* osthread) {
 
 void PosixSignals::do_resume(OSThread* osthread) {
   assert(osthread->sr.is_suspended(), "thread should be suspended");
-  assert(!sr_semaphore.trywait(), "invalid semaphore state");
+  assert(!sr_semaphore->trywait(), "invalid semaphore state");
 
   if (osthread->sr.request_wakeup() != SuspendResume::SR_WAKEUP_REQUEST) {
     // failed to switch to WAKEUP_REQUEST
@@ -1827,7 +1830,7 @@ void PosixSignals::do_resume(OSThread* osthread) {
 
   while (true) {
     if (sr_notify(osthread) == 0) {
-      if (sr_semaphore.timedwait(2)) {
+      if (sr_semaphore->timedwait(2)) {
         if (osthread->sr.is_running()) {
           return;
         }

src/hotspot/os/windows/gc/z/zNUMA_windows.cpp

@@ -46,3 +46,7 @@ uint32_t ZNUMA::memory_id(uintptr_t addr) {
   // NUMA support not enabled, assume everything belongs to node zero
   return 0;
 }
+
+int ZNUMA::numa_id_to_node(uint32_t numa_id) {
+  ShouldNotCallThis();
+}

src/hotspot/os_cpu/aix_ppc/atomicAccess_aix_ppc.hpp

@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2019 SAP SE. All rights reserved.
+ * Copyright (c) 2012, 2025 SAP SE. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -23,395 +23,5 @@
  *
  */
 
-#ifndef OS_CPU_AIX_PPC_ATOMICACCESS_AIX_PPC_HPP
-#define OS_CPU_AIX_PPC_ATOMICACCESS_AIX_PPC_HPP
-
-#ifndef PPC64
-#error "Atomic currently only implemented for PPC64"
-#endif
-
-#include "orderAccess_aix_ppc.hpp"
-#include "utilities/debug.hpp"
-
-// Implementation of class AtomicAccess
-
-//
-// machine barrier instructions:
-//
-// - sync            two-way memory barrier, aka fence
-// - lwsync          orders  Store|Store,
-//                            Load|Store,
-//                            Load|Load,
-//                   but not Store|Load
-// - eieio           orders memory accesses for device memory (only)
-// - isync           invalidates speculatively executed instructions
-//                   From the POWER ISA 2.06 documentation:
-//                    "[...] an isync instruction prevents the execution of
-//                   instructions following the isync until instructions
-//                   preceding the isync have completed, [...]"
-//                   From IBM's AIX assembler reference:
-//                    "The isync [...] instructions causes the processor to
-//                   refetch any instructions that might have been fetched
-//                   prior to the isync instruction. The instruction isync
-//                   causes the processor to wait for all previous instructions
-//                   to complete. Then any instructions already fetched are
-//                   discarded and instruction processing continues in the
-//                   environment established by the previous instructions."
-//
-// semantic barrier instructions:
-// (as defined in orderAccess.hpp)
-//
-// - release         orders Store|Store,       (maps to lwsync)
-//                           Load|Store
-// - acquire         orders  Load|Store,       (maps to lwsync)
-//                           Load|Load
-// - fence           orders Store|Store,       (maps to sync)
-//                           Load|Store,
-//                           Load|Load,
-//                          Store|Load
-//
-
-inline void pre_membar(atomic_memory_order order) {
-  switch (order) {
-    case memory_order_relaxed:
-    case memory_order_acquire: break;
-    case memory_order_release:
-    case memory_order_acq_rel: __asm__ __volatile__ ("lwsync" : : : "memory"); break;
-    default /*conservative*/ : __asm__ __volatile__ ("sync"   : : : "memory"); break;
-  }
-}
-
-inline void post_membar(atomic_memory_order order) {
-  switch (order) {
-    case memory_order_relaxed:
-    case memory_order_release: break;
-    case memory_order_acquire:
-    case memory_order_acq_rel: __asm__ __volatile__ ("isync"  : : : "memory"); break;
-    default /*conservative*/ : __asm__ __volatile__ ("sync"   : : : "memory"); break;
-  }
-}
-
-
-template<size_t byte_size>
-struct AtomicAccess::PlatformAdd {
-  template<typename D, typename I>
-  D add_then_fetch(D volatile* dest, I add_value, atomic_memory_order order) const;
-
-  template<typename D, typename I>
-  D fetch_then_add(D volatile* dest, I add_value, atomic_memory_order order) const {
-    return add_then_fetch(dest, add_value, order) - add_value;
-  }
-};
-
-template<>
-template<typename D, typename I>
-inline D AtomicAccess::PlatformAdd<4>::add_then_fetch(D volatile* dest, I add_value,
-                                                      atomic_memory_order order) const {
-  STATIC_ASSERT(4 == sizeof(I));
-  STATIC_ASSERT(4 == sizeof(D));
-
-  D result;
-
-  pre_membar(order);
-
-  __asm__ __volatile__ (
-    "1: lwarx   %0,  0, %2    \n"
-    "   add     %0, %0, %1    \n"
-    "   stwcx.  %0,  0, %2    \n"
-    "   bne-    1b            \n"
-    : /*%0*/"=&r" (result)
-    : /*%1*/"r" (add_value), /*%2*/"r" (dest)
-    : "cc", "memory" );
-
-  post_membar(order);
-
-  return result;
-}
-
-
-template<>
-template<typename D, typename I>
-inline D AtomicAccess::PlatformAdd<8>::add_then_fetch(D volatile* dest, I add_value,
-                                                      atomic_memory_order order) const {
-  STATIC_ASSERT(8 == sizeof(I));
-  STATIC_ASSERT(8 == sizeof(D));
-
-  D result;
-
-  pre_membar(order);
-
-  __asm__ __volatile__ (
-    "1: ldarx   %0,  0, %2    \n"
-    "   add     %0, %0, %1    \n"
-    "   stdcx.  %0,  0, %2    \n"
-    "   bne-    1b            \n"
-    : /*%0*/"=&r" (result)
-    : /*%1*/"r" (add_value), /*%2*/"r" (dest)
-    : "cc", "memory" );
-
-  post_membar(order);
-
-  return result;
-}
-
-template<>
-template<typename T>
-inline T AtomicAccess::PlatformXchg<4>::operator()(T volatile* dest,
-                                                   T exchange_value,
-                                                   atomic_memory_order order) const {
-  // Note that xchg doesn't necessarily do an acquire
-  // (see synchronizer.cpp).
-
-  T old_value;
-  const uint64_t zero = 0;
-
-  pre_membar(order);
-
-  __asm__ __volatile__ (
-    /* atomic loop */
-    "1:                                                 \n"
-    "   lwarx   %[old_value], %[dest], %[zero]          \n"
-    "   stwcx.  %[exchange_value], %[dest], %[zero]     \n"
-    "   bne-    1b                                      \n"
-    /* exit */
-    "2:                                                 \n"
-    /* out */
-    : [old_value]       "=&r"   (old_value),
-                        "=m"    (*dest)
-    /* in */
-    : [dest]            "b"     (dest),
-      [zero]            "r"     (zero),
-      [exchange_value]  "r"     (exchange_value),
-                        "m"     (*dest)
-    /* clobber */
-    : "cc",
-      "memory"
-    );
-
-  post_membar(order);
-
-  return old_value;
-}
-
-template<>
-template<typename T>
-inline T AtomicAccess::PlatformXchg<8>::operator()(T volatile* dest,
-                                                   T exchange_value,
-                                                   atomic_memory_order order) const {
-  STATIC_ASSERT(8 == sizeof(T));
-  // Note that xchg doesn't necessarily do an acquire
-  // (see synchronizer.cpp).
-
-  T old_value;
-  const uint64_t zero = 0;
-
-  pre_membar(order);
-
-  __asm__ __volatile__ (
-    /* atomic loop */
-    "1:                                                 \n"
-    "   ldarx   %[old_value], %[dest], %[zero]          \n"
-    "   stdcx.  %[exchange_value], %[dest], %[zero]     \n"
-    "   bne-    1b                                      \n"
-    /* exit */
-    "2:                                                 \n"
-    /* out */
-    : [old_value]       "=&r"   (old_value),
-                        "=m"    (*dest)
-    /* in */
-    : [dest]            "b"     (dest),
-      [zero]            "r"     (zero),
-      [exchange_value]  "r"     (exchange_value),
-                        "m"     (*dest)
-    /* clobber */
-    : "cc",
-      "memory"
-    );
-
-  post_membar(order);
-
-  return old_value;
-}
-
-template<>
-template<typename T>
-inline T AtomicAccess::PlatformCmpxchg<1>::operator()(T volatile* dest,
-                                                      T compare_value,
-                                                      T exchange_value,
-                                                      atomic_memory_order order) const {
-  STATIC_ASSERT(1 == sizeof(T));
-
-  // Note that cmpxchg guarantees a two-way memory barrier across
-  // the cmpxchg, so it's really a 'fence_cmpxchg_fence' if not
-  // specified otherwise (see atomicAccess.hpp).
-
-  // Using 32 bit internally.
-  volatile int *dest_base = (volatile int*)((uintptr_t)dest & ~3);
-
-#ifdef VM_LITTLE_ENDIAN
-  const unsigned int shift_amount        = ((uintptr_t)dest & 3) * 8;
-#else
-  const unsigned int shift_amount        = ((~(uintptr_t)dest) & 3) * 8;
-#endif
-  const unsigned int masked_compare_val  = ((unsigned int)(unsigned char)compare_value),
-                     masked_exchange_val = ((unsigned int)(unsigned char)exchange_value),
-                     xor_value           = (masked_compare_val ^ masked_exchange_val) << shift_amount;
-
-  unsigned int old_value, value32;
-
-  pre_membar(order);
-
-  __asm__ __volatile__ (
-    /* simple guard */
-    "   lbz     %[old_value], 0(%[dest])                  \n"
-    "   cmpw    %[masked_compare_val], %[old_value]       \n"
-    "   bne-    2f                                        \n"
-    /* atomic loop */
-    "1:                                                   \n"
-    "   lwarx   %[value32], 0, %[dest_base]               \n"
-    /* extract byte and compare */
-    "   srd     %[old_value], %[value32], %[shift_amount] \n"
-    "   clrldi  %[old_value], %[old_value], 56            \n"
-    "   cmpw    %[masked_compare_val], %[old_value]       \n"
-    "   bne-    2f                                        \n"
-    /* replace byte and try to store */
-    "   xor     %[value32], %[xor_value], %[value32]      \n"
-    "   stwcx.  %[value32], 0, %[dest_base]               \n"
-    "   bne-    1b                                        \n"
-    /* exit */
-    "2:                                                   \n"
-    /* out */
-    : [old_value]           "=&r"   (old_value),
-      [value32]             "=&r"   (value32),
-                            "=m"    (*dest),
-                            "=m"    (*dest_base)
-    /* in */
-    : [dest]                "b"     (dest),
-      [dest_base]           "b"     (dest_base),
-      [shift_amount]        "r"     (shift_amount),
-      [masked_compare_val]  "r"     (masked_compare_val),
-      [xor_value]           "r"     (xor_value),
-                            "m"     (*dest),
-                            "m"     (*dest_base)
-    /* clobber */
-    : "cc",
-      "memory"
-    );
-
-  post_membar(order);
-
-  return PrimitiveConversions::cast<T>((unsigned char)old_value);
-}
-
-template<>
-template<typename T>
-inline T AtomicAccess::PlatformCmpxchg<4>::operator()(T volatile* dest,
-                                                      T compare_value,
-                                                      T exchange_value,
-                                                      atomic_memory_order order) const {
-  STATIC_ASSERT(4 == sizeof(T));
-
-  // Note that cmpxchg guarantees a two-way memory barrier across
-  // the cmpxchg, so it's really a 'fence_cmpxchg_fence' if not
-  // specified otherwise (see atomicAccess.hpp).
-
-  T old_value;
-  const uint64_t zero = 0;
-
-  pre_membar(order);
-
-  __asm__ __volatile__ (
-    /* simple guard */
-    "   lwz     %[old_value], 0(%[dest])                \n"
-    "   cmpw    %[compare_value], %[old_value]          \n"
-    "   bne-    2f                                      \n"
-    /* atomic loop */
-    "1:                                                 \n"
-    "   lwarx   %[old_value], %[dest], %[zero]          \n"
-    "   cmpw    %[compare_value], %[old_value]          \n"
-    "   bne-    2f                                      \n"
-    "   stwcx.  %[exchange_value], %[dest], %[zero]     \n"
-    "   bne-    1b                                      \n"
-    /* exit */
-    "2:                                                 \n"
-    /* out */
-    : [old_value]       "=&r"   (old_value),
-                        "=m"    (*dest)
-    /* in */
-    : [dest]            "b"     (dest),
-      [zero]            "r"     (zero),
-      [compare_value]   "r"     (compare_value),
-      [exchange_value]  "r"     (exchange_value),
-                        "m"     (*dest)
-    /* clobber */
-    : "cc",
-      "memory"
-    );
-
-  post_membar(order);
-
-  return old_value;
-}
-
-template<>
-template<typename T>
-inline T AtomicAccess::PlatformCmpxchg<8>::operator()(T volatile* dest,
-                                                      T compare_value,
-                                                      T exchange_value,
-                                                      atomic_memory_order order) const {
-  STATIC_ASSERT(8 == sizeof(T));
-
-  // Note that cmpxchg guarantees a two-way memory barrier across
-  // the cmpxchg, so it's really a 'fence_cmpxchg_fence' if not
-  // specified otherwise (see atomicAccess.hpp).
-
-  T old_value;
-  const uint64_t zero = 0;
-
-  pre_membar(order);
-
-  __asm__ __volatile__ (
-    /* simple guard */
-    "   ld      %[old_value], 0(%[dest])                \n"
-    "   cmpd    %[compare_value], %[old_value]          \n"
-    "   bne-    2f                                      \n"
-    /* atomic loop */
-    "1:                                                 \n"
-    "   ldarx   %[old_value], %[dest], %[zero]          \n"
-    "   cmpd    %[compare_value], %[old_value]          \n"
-    "   bne-    2f                                      \n"
-    "   stdcx.  %[exchange_value], %[dest], %[zero]     \n"
-    "   bne-    1b                                      \n"
-    /* exit */
-    "2:                                                 \n"
-    /* out */
-    : [old_value]       "=&r"   (old_value),
-                        "=m"    (*dest)
-    /* in */
-    : [dest]            "b"     (dest),
-      [zero]            "r"     (zero),
-      [compare_value]   "r"     (compare_value),
-      [exchange_value]  "r"     (exchange_value),
-                        "m"     (*dest)
-    /* clobber */
-    : "cc",
-      "memory"
-    );
-
-  post_membar(order);
-
-  return old_value;
-}
-
-template<size_t byte_size>
-struct AtomicAccess::PlatformOrderedLoad<byte_size, X_ACQUIRE> {
-  template <typename T>
-  T operator()(const volatile T* p) const {
-    T t = AtomicAccess::load(p);
-    // Use twi-isync for load_acquire (faster than lwsync).
-    __asm__ __volatile__ ("twi 0,%0,0\n isync\n" : : "r" (t) : "memory");
-    return t;
-  }
-};
-
-#endif // OS_CPU_AIX_PPC_ATOMICACCESS_AIX_PPC_HPP
+// Including inline assembler functions that are shared between multiple PPC64 platforms.
+#include "atomicAccess_ppc.hpp"

src/hotspot/os_cpu/aix_ppc/orderAccess_aix_ppc.hpp

@@ -1,6 +1,6 @@
 /*
- * Copyright (c) 1997, 2020, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2019 SAP SE. All rights reserved.
+ * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2025 SAP SE. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -23,62 +23,5 @@
  *
  */
 
-#ifndef OS_CPU_AIX_PPC_ORDERACCESS_AIX_PPC_HPP
-#define OS_CPU_AIX_PPC_ORDERACCESS_AIX_PPC_HPP
-
-// Included in orderAccess.hpp header file.
-
-// Compiler version last used for testing: xlc 12
-// Please update this information when this file changes
-
-// Implementation of class OrderAccess.
-
-//
-// Machine barrier instructions:
-//
-// - sync            Two-way memory barrier, aka fence.
-// - lwsync          orders  Store|Store,
-//                            Load|Store,
-//                            Load|Load,
-//                   but not Store|Load
-// - eieio           orders  Store|Store
-// - isync           Invalidates speculatively executed instructions,
-//                   but isync may complete before storage accesses
-//                   associated with instructions preceding isync have
-//                   been performed.
-//
-// Semantic barrier instructions:
-// (as defined in orderAccess.hpp)
-//
-// - release         orders Store|Store,       (maps to lwsync)
-//                           Load|Store
-// - acquire         orders  Load|Store,       (maps to lwsync)
-//                           Load|Load
-// - fence           orders Store|Store,       (maps to sync)
-//                           Load|Store,
-//                           Load|Load,
-//                          Store|Load
-//
-
-#define inlasm_sync()     __asm__ __volatile__ ("sync"   : : : "memory");
-#define inlasm_lwsync()   __asm__ __volatile__ ("lwsync" : : : "memory");
-#define inlasm_eieio()    __asm__ __volatile__ ("eieio"  : : : "memory");
-#define inlasm_isync()    __asm__ __volatile__ ("isync"  : : : "memory");
-
-inline void OrderAccess::loadload()   { inlasm_lwsync(); }
-inline void OrderAccess::storestore() { inlasm_lwsync(); }
-inline void OrderAccess::loadstore()  { inlasm_lwsync(); }
-inline void OrderAccess::storeload()  { inlasm_sync();   }
-
-inline void OrderAccess::acquire()    { inlasm_lwsync(); }
-inline void OrderAccess::release()    { inlasm_lwsync(); }
-inline void OrderAccess::fence()      { inlasm_sync();   }
-inline void OrderAccess::cross_modify_fence_impl()
-                                      { inlasm_isync();  }
-
-#undef inlasm_sync
-#undef inlasm_lwsync
-#undef inlasm_eieio
-#undef inlasm_isync
-
-#endif // OS_CPU_AIX_PPC_ORDERACCESS_AIX_PPC_HPP
+// Including inline assembler functions that are shared between multiple PPC64 platforms.
+#include "orderAccess_ppc.hpp"

src/hotspot/os_cpu/aix_ppc/os_aix_ppc.cpp

@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2024 SAP SE. All rights reserved.
+ * Copyright (c) 2012, 2025 SAP SE. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -254,6 +254,18 @@ bool PosixSignals::pd_hotspot_signal_handler(int sig, siginfo_t* info,
         goto run_stub;
       }
 
+      // SIGTRAP-based nmethod entry barriers.
+      else if (sig == SIGTRAP && TrapBasedNMethodEntryBarriers &&
+               nativeInstruction_at(pc)->is_sigtrap_nmethod_entry_barrier() &&
+               CodeCache::contains((void*) pc)) {
+        if (TraceTraps) {
+          tty->print_cr("trap: nmethod entry barrier at " INTPTR_FORMAT " (SIGTRAP)", p2i(pc));
+        }
+        stub = StubRoutines::method_entry_barrier();
+        uc->uc_mcontext.jmp_context.lr = (uintptr_t)(pc + BytesPerInstWord); // emulate call by setting LR
+        goto run_stub;
+      }
+
       // SIGTRAP-based ic miss check in compiled code.
       else if (sig == SIGTRAP && TrapBasedICMissChecks &&
                nativeInstruction_at(pc)->is_sigtrap_ic_miss_check()) {
@@ -282,6 +294,7 @@ bool PosixSignals::pd_hotspot_signal_handler(int sig, siginfo_t* info,
           tty->print_cr("trap: null_check at " INTPTR_FORMAT " (SIGSEGV)", p2i(pc));
         }
         stub = SharedRuntime::continuation_for_implicit_exception(thread, pc, SharedRuntime::IMPLICIT_NULL);
+        goto run_stub;
       }
 
 #ifdef COMPILER2

src/hotspot/os_cpu/linux_ppc/atomicAccess_linux_ppc.hpp

@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2019 SAP SE. All rights reserved.
+ * Copyright (c) 2012, 2025 SAP SE. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -23,373 +23,5 @@
  *
  */
 
-#ifndef OS_CPU_LINUX_PPC_ATOMICACCESS_LINUX_PPC_HPP
-#define OS_CPU_LINUX_PPC_ATOMICACCESS_LINUX_PPC_HPP
-
-#ifndef PPC64
-#error "Atomic currently only implemented for PPC64"
-#endif
-
-#include "orderAccess_linux_ppc.hpp"
-#include "utilities/debug.hpp"
-
-// Implementation of class AtomicAccess
-
-//
-// machine barrier instructions:
-//
-// - sync            two-way memory barrier, aka fence
-// - lwsync          orders  Store|Store,
-//                            Load|Store,
-//                            Load|Load,
-//                   but not Store|Load
-// - eieio           orders memory accesses for device memory (only)
-// - isync           invalidates speculatively executed instructions
-//                   From the POWER ISA 2.06 documentation:
-//                    "[...] an isync instruction prevents the execution of
-//                   instructions following the isync until instructions
-//                   preceding the isync have completed, [...]"
-//                   From IBM's AIX assembler reference:
-//                    "The isync [...] instructions causes the processor to
-//                   refetch any instructions that might have been fetched
-//                   prior to the isync instruction. The instruction isync
-//                   causes the processor to wait for all previous instructions
-//                   to complete. Then any instructions already fetched are
-//                   discarded and instruction processing continues in the
-//                   environment established by the previous instructions."
-//
-// semantic barrier instructions:
-// (as defined in orderAccess.hpp)
-//
-// - release         orders Store|Store,       (maps to lwsync)
-//                           Load|Store
-// - acquire         orders  Load|Store,       (maps to lwsync)
-//                           Load|Load
-// - fence           orders Store|Store,       (maps to sync)
-//                           Load|Store,
-//                           Load|Load,
-//                          Store|Load
-//
-
-inline void pre_membar(atomic_memory_order order) {
-  switch (order) {
-    case memory_order_relaxed:
-    case memory_order_acquire: break;
-    case memory_order_release:
-    case memory_order_acq_rel: __asm__ __volatile__ ("lwsync" : : : "memory"); break;
-    default /*conservative*/ : __asm__ __volatile__ ("sync"   : : : "memory"); break;
-  }
-}
-
-inline void post_membar(atomic_memory_order order) {
-  switch (order) {
-    case memory_order_relaxed:
-    case memory_order_release: break;
-    case memory_order_acquire:
-    case memory_order_acq_rel: __asm__ __volatile__ ("isync"  : : : "memory"); break;
-    default /*conservative*/ : __asm__ __volatile__ ("sync"   : : : "memory"); break;
-  }
-}
-
-
-template<size_t byte_size>
-struct AtomicAccess::PlatformAdd {
-  template<typename D, typename I>
-  D add_then_fetch(D volatile* dest, I add_value, atomic_memory_order order) const;
-
-  template<typename D, typename I>
-  D fetch_then_add(D volatile* dest, I add_value, atomic_memory_order order) const {
-    return add_then_fetch(dest, add_value, order) - add_value;
-  }
-};
-
-template<>
-template<typename D, typename I>
-inline D AtomicAccess::PlatformAdd<4>::add_then_fetch(D volatile* dest, I add_value,
-                                                      atomic_memory_order order) const {
-  STATIC_ASSERT(4 == sizeof(I));
-  STATIC_ASSERT(4 == sizeof(D));
-
-  D result;
-
-  pre_membar(order);
-
-  __asm__ __volatile__ (
-    "1: lwarx   %0,  0, %2    \n"
-    "   add     %0, %0, %1    \n"
-    "   stwcx.  %0,  0, %2    \n"
-    "   bne-    1b            \n"
-    : /*%0*/"=&r" (result)
-    : /*%1*/"r" (add_value), /*%2*/"r" (dest)
-    : "cc", "memory" );
-
-  post_membar(order);
-
-  return result;
-}
-
-
-template<>
-template<typename D, typename I>
-inline D AtomicAccess::PlatformAdd<8>::add_then_fetch(D volatile* dest, I add_value,
-                                                      atomic_memory_order order) const {
-  STATIC_ASSERT(8 == sizeof(I));
-  STATIC_ASSERT(8 == sizeof(D));
-
-  D result;
-
-  pre_membar(order);
-
-  __asm__ __volatile__ (
-    "1: ldarx   %0,  0, %2    \n"
-    "   add     %0, %0, %1    \n"
-    "   stdcx.  %0,  0, %2    \n"
-    "   bne-    1b            \n"
-    : /*%0*/"=&r" (result)
-    : /*%1*/"r" (add_value), /*%2*/"r" (dest)
-    : "cc", "memory" );
-
-  post_membar(order);
-
-  return result;
-}
-
-template<>
-template<typename T>
-inline T AtomicAccess::PlatformXchg<4>::operator()(T volatile* dest,
-                                                   T exchange_value,
-                                                   atomic_memory_order order) const {
-  // Note that xchg doesn't necessarily do an acquire
-  // (see synchronizer.cpp).
-
-  T old_value;
-  const uint64_t zero = 0;
-
-  pre_membar(order);
-
-  __asm__ __volatile__ (
-    /* atomic loop */
-    "1:                                                 \n"
-    "   lwarx   %[old_value], %[dest], %[zero]          \n"
-    "   stwcx.  %[exchange_value], %[dest], %[zero]     \n"
-    "   bne-    1b                                      \n"
-    /* exit */
-    "2:                                                 \n"
-    /* out */
-    : [old_value]       "=&r"   (old_value),
-                        "=m"    (*dest)
-    /* in */
-    : [dest]            "b"     (dest),
-      [zero]            "r"     (zero),
-      [exchange_value]  "r"     (exchange_value),
-                        "m"     (*dest)
-    /* clobber */
-    : "cc",
-      "memory"
-    );
-
-  post_membar(order);
-
-  return old_value;
-}
-
-template<>
-template<typename T>
-inline T AtomicAccess::PlatformXchg<8>::operator()(T volatile* dest,
-                                                   T exchange_value,
-                                                   atomic_memory_order order) const {
-  STATIC_ASSERT(8 == sizeof(T));
-  // Note that xchg doesn't necessarily do an acquire
-  // (see synchronizer.cpp).
-
-  T old_value;
-  const uint64_t zero = 0;
-
-  pre_membar(order);
-
-  __asm__ __volatile__ (
-    /* atomic loop */
-    "1:                                                 \n"
-    "   ldarx   %[old_value], %[dest], %[zero]          \n"
-    "   stdcx.  %[exchange_value], %[dest], %[zero]     \n"
-    "   bne-    1b                                      \n"
-    /* exit */
-    "2:                                                 \n"
-    /* out */
-    : [old_value]       "=&r"   (old_value),
-                        "=m"    (*dest)
-    /* in */
-    : [dest]            "b"     (dest),
-      [zero]            "r"     (zero),
-      [exchange_value]  "r"     (exchange_value),
-                        "m"     (*dest)
-    /* clobber */
-    : "cc",
-      "memory"
-    );
-
-  post_membar(order);
-
-  return old_value;
-}
-
-template<>
-template<typename T>
-inline T AtomicAccess::PlatformCmpxchg<1>::operator()(T volatile* dest,
-                                                      T compare_value,
-                                                      T exchange_value,
-                                                      atomic_memory_order order) const {
-  STATIC_ASSERT(1 == sizeof(T));
-
-  // Note that cmpxchg guarantees a two-way memory barrier across
-  // the cmpxchg, so it's really a 'fence_cmpxchg_fence' if not
-  // specified otherwise (see atomicAccess.hpp).
-
-  // Using 32 bit internally.
-  unsigned int old_value, loaded_value;
-  pre_membar(order);
-
-  __asm__ __volatile__ (
-    /* atomic loop */
-    "1:                                                   \n"
-    "   lbarx   %[old_value], 0, %[dest]                  \n"
-    /* extract byte and compare */
-    "   cmpw    %[compare_value], %[old_value]            \n"
-    "   bne-    2f                                        \n"
-    /* replace byte and try to store */
-    "   stbcx.  %[exchange_value], 0, %[dest]             \n"
-    "   bne-    1b                                        \n"
-    /* exit */
-    "2:                                                   \n"
-    /* out */
-    : [old_value]           "=&r"   (old_value),
-      [loaded_value]        "=&r"   (loaded_value),
-                            "=m"    (*dest)
-    /* in */
-    : [dest]            "b"     (dest),
-      [compare_value]   "r"     (compare_value),
-      [exchange_value]  "r"     (exchange_value),
-                        "m"     (*dest)
-    /* clobber */
-    : "cc",
-      "memory"
-    );
-
-  post_membar(order);
-
-  return PrimitiveConversions::cast<T>((unsigned char)old_value);
-}
-
-template<>
-template<typename T>
-inline T AtomicAccess::PlatformCmpxchg<4>::operator()(T volatile* dest,
-                                                      T compare_value,
-                                                      T exchange_value,
-                                                      atomic_memory_order order) const {
-  STATIC_ASSERT(4 == sizeof(T));
-
-  // Note that cmpxchg guarantees a two-way memory barrier across
-  // the cmpxchg, so it's really a 'fence_cmpxchg_fence' if not
-  // specified otherwise (see atomicAccess.hpp).
-
-  T old_value;
-  const uint64_t zero = 0;
-
-  pre_membar(order);
-
-  __asm__ __volatile__ (
-    /* simple guard */
-    "   lwz     %[old_value], 0(%[dest])                \n"
-    "   cmpw    %[compare_value], %[old_value]          \n"
-    "   bne-    2f                                      \n"
-    /* atomic loop */
-    "1:                                                 \n"
-    "   lwarx   %[old_value], %[dest], %[zero]          \n"
-    "   cmpw    %[compare_value], %[old_value]          \n"
-    "   bne-    2f                                      \n"
-    "   stwcx.  %[exchange_value], %[dest], %[zero]     \n"
-    "   bne-    1b                                      \n"
-    /* exit */
-    "2:                                                 \n"
-    /* out */
-    : [old_value]       "=&r"   (old_value),
-                        "=m"    (*dest)
-    /* in */
-    : [dest]            "b"     (dest),
-      [zero]            "r"     (zero),
-      [compare_value]   "r"     (compare_value),
-      [exchange_value]  "r"     (exchange_value),
-                        "m"     (*dest)
-    /* clobber */
-    : "cc",
-      "memory"
-    );
-
-  post_membar(order);
-
-  return old_value;
-}
-
-template<>
-template<typename T>
-inline T AtomicAccess::PlatformCmpxchg<8>::operator()(T volatile* dest,
-                                                      T compare_value,
-                                                      T exchange_value,
-                                                      atomic_memory_order order) const {
-  STATIC_ASSERT(8 == sizeof(T));
-
-  // Note that cmpxchg guarantees a two-way memory barrier across
-  // the cmpxchg, so it's really a 'fence_cmpxchg_fence' if not
-  // specified otherwise (see atomicAccess.hpp).
-
-  T old_value;
-  const uint64_t zero = 0;
-
-  pre_membar(order);
-
-  __asm__ __volatile__ (
-    /* simple guard */
-    "   ld      %[old_value], 0(%[dest])                \n"
-    "   cmpd    %[compare_value], %[old_value]          \n"
-    "   bne-    2f                                      \n"
-    /* atomic loop */
-    "1:                                                 \n"
-    "   ldarx   %[old_value], %[dest], %[zero]          \n"
-    "   cmpd    %[compare_value], %[old_value]          \n"
-    "   bne-    2f                                      \n"
-    "   stdcx.  %[exchange_value], %[dest], %[zero]     \n"
-    "   bne-    1b                                      \n"
-    /* exit */
-    "2:                                                 \n"
-    /* out */
-    : [old_value]       "=&r"   (old_value),
-                        "=m"    (*dest)
-    /* in */
-    : [dest]            "b"     (dest),
-      [zero]            "r"     (zero),
-      [compare_value]   "r"     (compare_value),
-      [exchange_value]  "r"     (exchange_value),
-                        "m"     (*dest)
-    /* clobber */
-    : "cc",
-      "memory"
-    );
-
-  post_membar(order);
-
-  return old_value;
-}
-
-template<size_t byte_size>
-struct AtomicAccess::PlatformOrderedLoad<byte_size, X_ACQUIRE>
-{
-  template <typename T>
-  T operator()(const volatile T* p) const {
-    T t = AtomicAccess::load(p);
-    // Use twi-isync for load_acquire (faster than lwsync).
-    __asm__ __volatile__ ("twi 0,%0,0\n isync\n" : : "r" (t) : "memory");
-    return t;
-  }
-};
-
-#endif // OS_CPU_LINUX_PPC_ATOMICACCESS_LINUX_PPC_HPP
+// Including inline assembler functions that are shared between multiple PPC64 platforms.
+#include "atomicAccess_ppc.hpp"

src/hotspot/os_cpu/linux_ppc/orderAccess_linux_ppc.hpp

@@ -1,6 +1,6 @@
 /*
- * Copyright (c) 1997, 2020, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2014 SAP SE. All rights reserved.
+ * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2025 SAP SE. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -23,66 +23,5 @@
  *
  */
 
-#ifndef OS_CPU_LINUX_PPC_ORDERACCESS_LINUX_PPC_HPP
-#define OS_CPU_LINUX_PPC_ORDERACCESS_LINUX_PPC_HPP
-
-// Included in orderAccess.hpp header file.
-
-#ifndef PPC64
-#error "OrderAccess currently only implemented for PPC64"
-#endif
-
-// Compiler version last used for testing: gcc 4.1.2
-// Please update this information when this file changes
-
-// Implementation of class OrderAccess.
-
-//
-// Machine barrier instructions:
-//
-// - sync            Two-way memory barrier, aka fence.
-// - lwsync          orders  Store|Store,
-//                            Load|Store,
-//                            Load|Load,
-//                   but not Store|Load
-// - eieio           orders  Store|Store
-// - isync           Invalidates speculatively executed instructions,
-//                   but isync may complete before storage accesses
-//                   associated with instructions preceding isync have
-//                   been performed.
-//
-// Semantic barrier instructions:
-// (as defined in orderAccess.hpp)
-//
-// - release         orders Store|Store,       (maps to lwsync)
-//                           Load|Store
-// - acquire         orders  Load|Store,       (maps to lwsync)
-//                           Load|Load
-// - fence           orders Store|Store,       (maps to sync)
-//                           Load|Store,
-//                           Load|Load,
-//                          Store|Load
-//
-
-#define inlasm_sync()     __asm__ __volatile__ ("sync"   : : : "memory");
-#define inlasm_lwsync()   __asm__ __volatile__ ("lwsync" : : : "memory");
-#define inlasm_eieio()    __asm__ __volatile__ ("eieio"  : : : "memory");
-#define inlasm_isync()    __asm__ __volatile__ ("isync"  : : : "memory");
-
-inline void   OrderAccess::loadload()   { inlasm_lwsync(); }
-inline void   OrderAccess::storestore() { inlasm_lwsync(); }
-inline void   OrderAccess::loadstore()  { inlasm_lwsync(); }
-inline void   OrderAccess::storeload()  { inlasm_sync();   }
-
-inline void   OrderAccess::acquire()    { inlasm_lwsync(); }
-inline void   OrderAccess::release()    { inlasm_lwsync(); }
-inline void   OrderAccess::fence()      { inlasm_sync();   }
-inline void   OrderAccess::cross_modify_fence_impl()
-                                        { inlasm_isync();  }
-
-#undef inlasm_sync
-#undef inlasm_lwsync
-#undef inlasm_eieio
-#undef inlasm_isync
-
-#endif // OS_CPU_LINUX_PPC_ORDERACCESS_LINUX_PPC_HPP
+// Including inline assembler functions that are shared between multiple PPC64 platforms.
+#include "orderAccess_ppc.hpp"

src/hotspot/os_cpu/linux_ppc/os_linux_ppc.cpp

@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2024 SAP SE. All rights reserved.
+ * Copyright (c) 2012, 2025 SAP SE. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -286,6 +286,17 @@ bool PosixSignals::pd_hotspot_signal_handler(int sig, siginfo_t* info,
         stub = SharedRuntime::polling_page_return_handler_blob()->entry_point();
       }
 
+      // SIGTRAP-based nmethod entry barriers.
+      else if (sig == SIGTRAP && TrapBasedNMethodEntryBarriers &&
+               nativeInstruction_at(pc)->is_sigtrap_nmethod_entry_barrier() &&
+               CodeCache::contains((void*) pc)) {
+        if (TraceTraps) {
+          tty->print_cr("trap: nmethod entry barrier at " INTPTR_FORMAT " (SIGTRAP)", p2i(pc));
+        }
+        stub = StubRoutines::method_entry_barrier();
+        uc->uc_mcontext.regs->link = (uintptr_t)(pc + BytesPerInstWord); // emulate call by setting LR
+      }
+
       // SIGTRAP-based ic miss check in compiled code.
       else if (sig == SIGTRAP && TrapBasedICMissChecks &&
                nativeInstruction_at(pc)->is_sigtrap_ic_miss_check()) {

src/hotspot/os_cpu/linux_riscv/os_linux_riscv.cpp

@@ -287,6 +287,11 @@ bool PosixSignals::pd_hotspot_signal_handler(int sig, siginfo_t* info,
         stub = addr_slow;
       }
     }
+
+    if (sig == SIGBUS && VM_Version::is_misaligned_vector_fault(pc)) {
+      os::Posix::ucontext_set_pc(uc, VM_Version::continuation_for_misaligned_vector_fault(pc));
+      return true;
+    }
   }
 
   if (stub != nullptr) {

src/hotspot/os_cpu/linux_riscv/riscv_hwprobe.cpp

@@ -167,27 +167,20 @@ static bool is_set(int64_t key, uint64_t value_mask) {
 void RiscvHwprobe::add_features_from_query_result() {
   assert(rw_hwprobe_completed, "hwprobe not init yet.");
 
-  if (is_valid(RISCV_HWPROBE_KEY_MVENDORID)) {
-    VM_Version::mvendorid.enable_feature(query[RISCV_HWPROBE_KEY_MVENDORID].value);
-  }
-  if (is_valid(RISCV_HWPROBE_KEY_MARCHID)) {
-    VM_Version::marchid.enable_feature(query[RISCV_HWPROBE_KEY_MARCHID].value);
-  }
-  if (is_valid(RISCV_HWPROBE_KEY_MIMPID)) {
-    VM_Version::mimpid.enable_feature(query[RISCV_HWPROBE_KEY_MIMPID].value);
-  }
+  // ====== extensions ======
+  //
   if (is_set(RISCV_HWPROBE_KEY_BASE_BEHAVIOR, RISCV_HWPROBE_BASE_BEHAVIOR_IMA)) {
+    VM_Version::ext_a.enable_feature();
     VM_Version::ext_i.enable_feature();
     VM_Version::ext_m.enable_feature();
-    VM_Version::ext_a.enable_feature();
-  }
-  if (is_set(RISCV_HWPROBE_KEY_IMA_EXT_0, RISCV_HWPROBE_IMA_FD)) {
-    VM_Version::ext_f.enable_feature();
-    VM_Version::ext_d.enable_feature();
   }
   if (is_set(RISCV_HWPROBE_KEY_IMA_EXT_0, RISCV_HWPROBE_IMA_C)) {
     VM_Version::ext_c.enable_feature();
   }
+  if (is_set(RISCV_HWPROBE_KEY_IMA_EXT_0, RISCV_HWPROBE_IMA_FD)) {
+    VM_Version::ext_d.enable_feature();
+    VM_Version::ext_f.enable_feature();
+  }
   if (is_set(RISCV_HWPROBE_KEY_IMA_EXT_0, RISCV_HWPROBE_IMA_V)) {
     // Linux signal return bug when using vector with vlen > 128b in pre 6.8.5.
     long major, minor, patch;
@@ -202,21 +195,29 @@ void RiscvHwprobe::add_features_from_query_result() {
       VM_Version::ext_v.enable_feature();
     }
   }
+
+#ifndef PRODUCT
+  if (is_set(RISCV_HWPROBE_KEY_IMA_EXT_0, RISCV_HWPROBE_EXT_ZACAS)) {
+    VM_Version::ext_Zacas.enable_feature();
+  }
+#endif
   if (is_set(RISCV_HWPROBE_KEY_IMA_EXT_0, RISCV_HWPROBE_EXT_ZBA)) {
     VM_Version::ext_Zba.enable_feature();
   }
   if (is_set(RISCV_HWPROBE_KEY_IMA_EXT_0, RISCV_HWPROBE_EXT_ZBB)) {
     VM_Version::ext_Zbb.enable_feature();
   }
+#ifndef PRODUCT
+  if (is_set(RISCV_HWPROBE_KEY_IMA_EXT_0, RISCV_HWPROBE_EXT_ZBKB)) {
+    VM_Version::ext_Zbkb.enable_feature();
+  }
+#endif
   if (is_set(RISCV_HWPROBE_KEY_IMA_EXT_0, RISCV_HWPROBE_EXT_ZBS)) {
     VM_Version::ext_Zbs.enable_feature();
   }
 #ifndef PRODUCT
-  if (is_set(RISCV_HWPROBE_KEY_IMA_EXT_0, RISCV_HWPROBE_EXT_ZICBOZ)) {
-    VM_Version::ext_Zicboz.enable_feature();
-  }
-  if (is_set(RISCV_HWPROBE_KEY_IMA_EXT_0, RISCV_HWPROBE_EXT_ZBKB)) {
-    VM_Version::ext_Zbkb.enable_feature();
+  if (is_set(RISCV_HWPROBE_KEY_IMA_EXT_0, RISCV_HWPROBE_EXT_ZFA)) {
+    VM_Version::ext_Zfa.enable_feature();
   }
 #endif
   if (is_set(RISCV_HWPROBE_KEY_IMA_EXT_0, RISCV_HWPROBE_EXT_ZFH)) {
@@ -226,15 +227,28 @@ void RiscvHwprobe::add_features_from_query_result() {
     VM_Version::ext_Zfhmin.enable_feature();
   }
 #ifndef PRODUCT
+  if (is_set(RISCV_HWPROBE_KEY_IMA_EXT_0, RISCV_HWPROBE_EXT_ZICBOZ)) {
+    VM_Version::ext_Zicboz.enable_feature();
+  }
+  // Currently tests shows that cmove using Zicond instructions will bring
+  // performance regression, but to get a test coverage all the time, will
+  // still prefer to enabling it in debug version.
+  if (is_set(RISCV_HWPROBE_KEY_IMA_EXT_0, RISCV_HWPROBE_EXT_ZICOND)) {
+    VM_Version::ext_Zicond.enable_feature();
+  }
+  if (is_set(RISCV_HWPROBE_KEY_IMA_EXT_0, RISCV_HWPROBE_EXT_ZTSO)) {
+    VM_Version::ext_Ztso.enable_feature();
+  }
   if (is_set(RISCV_HWPROBE_KEY_IMA_EXT_0, RISCV_HWPROBE_EXT_ZVBB)) {
     VM_Version::ext_Zvbb.enable_feature();
   }
-#endif
-#ifndef PRODUCT
   if (is_set(RISCV_HWPROBE_KEY_IMA_EXT_0, RISCV_HWPROBE_EXT_ZVBC)) {
     VM_Version::ext_Zvbc.enable_feature();
   }
 #endif
+  if (is_set(RISCV_HWPROBE_KEY_IMA_EXT_0, RISCV_HWPROBE_EXT_ZVFH)) {
+    VM_Version::ext_Zvfh.enable_feature();
+  }
 #ifndef PRODUCT
   if (is_set(RISCV_HWPROBE_KEY_IMA_EXT_0, RISCV_HWPROBE_EXT_ZVKNED) &&
       is_set(RISCV_HWPROBE_KEY_IMA_EXT_0, RISCV_HWPROBE_EXT_ZVKNHB) &&
@@ -243,30 +257,18 @@ void RiscvHwprobe::add_features_from_query_result() {
     VM_Version::ext_Zvkn.enable_feature();
   }
 #endif
-  if (is_set(RISCV_HWPROBE_KEY_IMA_EXT_0, RISCV_HWPROBE_EXT_ZVFH)) {
-    VM_Version::ext_Zvfh.enable_feature();
+
+  // ====== non-extensions ======
+  //
+  if (is_valid(RISCV_HWPROBE_KEY_MARCHID)) {
+    VM_Version::marchid.enable_feature(query[RISCV_HWPROBE_KEY_MARCHID].value);
   }
-#ifndef PRODUCT
-  if (is_set(RISCV_HWPROBE_KEY_IMA_EXT_0, RISCV_HWPROBE_EXT_ZFA)) {
-    VM_Version::ext_Zfa.enable_feature();
+  if (is_valid(RISCV_HWPROBE_KEY_MIMPID)) {
+    VM_Version::mimpid.enable_feature(query[RISCV_HWPROBE_KEY_MIMPID].value);
   }
-#endif
-#ifndef PRODUCT
-  if (is_set(RISCV_HWPROBE_KEY_IMA_EXT_0, RISCV_HWPROBE_EXT_ZTSO)) {
-    VM_Version::ext_Ztso.enable_feature();
+  if (is_valid(RISCV_HWPROBE_KEY_MVENDORID)) {
+    VM_Version::mvendorid.enable_feature(query[RISCV_HWPROBE_KEY_MVENDORID].value);
   }
-#endif
-#ifndef PRODUCT
-  if (is_set(RISCV_HWPROBE_KEY_IMA_EXT_0, RISCV_HWPROBE_EXT_ZACAS)) {
-    VM_Version::ext_Zacas.enable_feature();
-  }
-  // Currently tests shows that cmove using Zicond instructions will bring
-  // performance regression, but to get a test coverage all the time, will
-  // still prefer to enabling it in debug version.
-  if (is_set(RISCV_HWPROBE_KEY_IMA_EXT_0, RISCV_HWPROBE_EXT_ZICOND)) {
-    VM_Version::ext_Zicond.enable_feature();
-  }
-#endif
   // RISCV_HWPROBE_KEY_CPUPERF_0 is deprecated and returns similar values
   // to RISCV_HWPROBE_KEY_MISALIGNED_SCALAR_PERF. Keep it there for backward
   // compatibility with old kernels.
@@ -277,7 +279,6 @@ void RiscvHwprobe::add_features_from_query_result() {
     VM_Version::unaligned_scalar.enable_feature(
        query[RISCV_HWPROBE_KEY_MISALIGNED_SCALAR_PERF].value);
   }
-
   if (is_valid(RISCV_HWPROBE_KEY_MISALIGNED_VECTOR_PERF)) {
     VM_Version::unaligned_vector.enable_feature(
        query[RISCV_HWPROBE_KEY_MISALIGNED_VECTOR_PERF].value);

src/hotspot/os_cpu/linux_riscv/vm_version_linux_riscv.cpp

@@ -103,6 +103,14 @@ uint32_t VM_Version::cpu_vector_length() {
   return (uint32_t)read_csr(CSR_VLENB);
 }
 
+void VM_Version::RVExtFeatureValue::log_enabled() {
+  log_debug(os, cpu)("Enabled RV64 feature \"%s\"", pretty());
+}
+
+void VM_Version::RVNonExtFeatureValue::log_enabled() {
+  log_debug(os, cpu)("Enabled RV64 feature \"%s\" (%ld)", pretty(), value());
+}
+
 void VM_Version::setup_cpu_available_features() {
 
   assert(ext_i.feature_bit() == HWCAP_ISA_I, "Bit for I must follow Linux HWCAP");
@@ -144,9 +152,8 @@ void VM_Version::setup_cpu_available_features() {
         continue;
       }
 
-      log_debug(os, cpu)("Enabled RV64 feature \"%s\" (%ld)",
-             _feature_list[i]->pretty(),
-             _feature_list[i]->value());
+      _feature_list[i]->log_enabled();
+
       // The feature string
       if (_feature_list[i]->feature_string()) {
         const char* tmp = _feature_list[i]->pretty();

src/hotspot/share/adlc/archDesc.cpp

@@ -899,10 +899,12 @@ int ArchDesc::emit_msg(int quiet, int flag, int line, const char *fmt,
 
 // Construct the name of the register mask.
 static const char *getRegMask(const char *reg_class_name) {
-  if( reg_class_name == nullptr ) return "RegMask::Empty";
+  if (reg_class_name == nullptr) {
+    return "RegMask::EMPTY";
+  }
 
   if (strcmp(reg_class_name,"Universe")==0) {
-    return "RegMask::Empty";
+    return "RegMask::EMPTY";
   } else if (strcmp(reg_class_name,"stack_slots")==0) {
     return "(Compile::current()->FIRST_STACK_mask())";
   } else if (strcmp(reg_class_name, "dynamic")==0) {
@@ -920,7 +922,7 @@ static const char *getRegMask(const char *reg_class_name) {
 
 // Convert a register class name to its register mask.
 const char *ArchDesc::reg_class_to_reg_mask(const char *rc_name) {
-  const char *reg_mask = "RegMask::Empty";
+  const char* reg_mask = "RegMask::EMPTY";
 
   if( _register ) {
     RegClass *reg_class  = _register->getRegClass(rc_name);
@@ -939,7 +941,7 @@ const char *ArchDesc::reg_class_to_reg_mask(const char *rc_name) {
 
 // Obtain the name of the RegMask for an OperandForm
 const char *ArchDesc::reg_mask(OperandForm  &opForm) {
-  const char *regMask      = "RegMask::Empty";
+  const char* regMask = "RegMask::EMPTY";
 
   // Check constraints on result's register class
   const char *result_class = opForm.constrained_reg_class();
@@ -968,9 +970,9 @@ const char *ArchDesc::reg_mask(InstructForm &inForm) {
     abort();
   }
 
-  // Instructions producing 'Universe' use RegMask::Empty
+  // Instructions producing 'Universe' use RegMask::EMPTY
   if (strcmp(result,"Universe") == 0) {
-    return "RegMask::Empty";
+    return "RegMask::EMPTY";
   }
 
   // Lookup this result operand and get its register class

src/hotspot/share/adlc/formssel.cpp

@@ -2422,7 +2422,7 @@ const char *OperandForm::constrained_reg_class() const {
 
 // Return the register class associated with 'leaf'.
 const char *OperandForm::in_reg_class(uint leaf, FormDict &globals) {
-  const char *reg_class = nullptr; // "RegMask::Empty";
+  const char* reg_class = nullptr; // "RegMask::EMPTY";
 
   if((_matrule == nullptr) || (_matrule->is_chain_rule(globals))) {
     reg_class = constrained_reg_class();

src/hotspot/share/adlc/output_c.cpp

@@ -2837,7 +2837,7 @@ static void defineIn_RegMask(FILE *fp, FormDict &globals, OperandForm &oper) {
       if (strcmp(first_reg_class, "stack_slots") == 0) {
         fprintf(fp,"  return &(Compile::current()->FIRST_STACK_mask());\n");
       } else if (strcmp(first_reg_class, "dynamic") == 0) {
-        fprintf(fp,"  return &RegMask::Empty;\n");
+        fprintf(fp, "  return &RegMask::EMPTY;\n");
       } else {
         const char* first_reg_class_to_upper = toUpper(first_reg_class);
         fprintf(fp,"  return &%s_mask();\n", first_reg_class_to_upper);

src/hotspot/share/cds/aotLinkedClassBulkLoader.cpp

@@ -42,6 +42,8 @@
 #include "oops/trainingData.hpp"
 #include "runtime/handles.inline.hpp"
 #include "runtime/java.hpp"
+#include "runtime/serviceThread.hpp"
+#include "utilities/growableArray.hpp"
 
 void AOTLinkedClassBulkLoader::serialize(SerializeClosure* soc) {
   AOTLinkedClassTable::get()->serialize(soc);
@@ -53,6 +55,8 @@ void AOTLinkedClassBulkLoader::serialize(SerializeClosure* soc) {
 // step in restoring the JVM's state from the snapshot recorded in the AOT cache: other AOT optimizations
 // such as AOT compiled methods can make direct references to the preloaded classes, knowing that
 // these classes are guaranteed to be in at least the "loaded" state.
+//
+// Note: we can't link the classes yet because SharedRuntime is not yet ready to generate adapters.
 void AOTLinkedClassBulkLoader::preload_classes(JavaThread* current) {
   preload_classes_impl(current);
   if (current->has_pending_exception()) {
@@ -112,6 +116,44 @@ void AOTLinkedClassBulkLoader::preload_classes_in_table(Array<InstanceKlass*>* c
   }
 }
 
+// Some cached heap objects may hold references to methods in aot-linked
+// classes (via MemberName). We need to make sure all classes are
+// linked before executing any bytecode.
+void AOTLinkedClassBulkLoader::link_classes(JavaThread* current) {
+  link_classes_impl(current);
+  if (current->has_pending_exception()) {
+    exit_on_exception(current);
+  }
+}
+
+void AOTLinkedClassBulkLoader::link_classes_impl(TRAPS) {
+  precond(CDSConfig::is_using_aot_linked_classes());
+
+  AOTLinkedClassTable* table = AOTLinkedClassTable::get();
+
+  link_classes_in_table(table->boot1(), CHECK);
+  link_classes_in_table(table->boot2(), CHECK);
+  link_classes_in_table(table->platform(), CHECK);
+  link_classes_in_table(table->app(), CHECK);
+}
+
+void AOTLinkedClassBulkLoader::link_classes_in_table(Array<InstanceKlass*>* classes, TRAPS) {
+  if (classes != nullptr) {
+    for (int i = 0; i < classes->length(); i++) {
+      // NOTE: CDSConfig::is_preserving_verification_constraints() is required
+      // when storing ik in the AOT cache. This means we don't have to verify
+      // ik at all.
+      //
+      // Without is_preserving_verification_constraints(), ik->link_class() may cause
+      // class loading, which may result in invocation of ClassLoader::loadClass() calls,
+      // which CANNOT happen because we are not ready to execute any Java byecodes yet
+      // at this point.
+      InstanceKlass* ik = classes->at(i);
+      ik->link_class(CHECK);
+    }
+  }
+}
+
 #ifdef ASSERT
 void AOTLinkedClassBulkLoader::validate_module_of_preloaded_classes() {
   oop javabase_module_oop = ModuleEntryTable::javabase_moduleEntry()->module_oop();
@@ -173,25 +215,21 @@ void AOTLinkedClassBulkLoader::validate_module(Klass* k, const char* category_na
 }
 #endif
 
-// Link all java.base classes in the AOTLinkedClassTable. Of those classes,
-// move the ones that have been AOT-initialized to the "initialized" state.
-void AOTLinkedClassBulkLoader::link_or_init_javabase_classes(JavaThread* current) {
-  link_or_init_classes_for_loader(Handle(), AOTLinkedClassTable::get()->boot1(), current);
+void AOTLinkedClassBulkLoader::init_javabase_classes(JavaThread* current) {
+  init_classes_for_loader(Handle(), AOTLinkedClassTable::get()->boot1(), current);
   if (current->has_pending_exception()) {
     exit_on_exception(current);
   }
 }
 
-// Do the same thing as link_or_init_javabase_classes(), but for the classes that are not
-// in the java.base module.
-void AOTLinkedClassBulkLoader::link_or_init_non_javabase_classes(JavaThread* current) {
-  link_or_init_non_javabase_classes_impl(current);
+void AOTLinkedClassBulkLoader::init_non_javabase_classes(JavaThread* current) {
+  init_non_javabase_classes_impl(current);
   if (current->has_pending_exception()) {
     exit_on_exception(current);
   }
 }
 
-void AOTLinkedClassBulkLoader::link_or_init_non_javabase_classes_impl(TRAPS) {
+void AOTLinkedClassBulkLoader::init_non_javabase_classes_impl(TRAPS) {
   assert(CDSConfig::is_using_aot_linked_classes(), "sanity");
 
   DEBUG_ONLY(validate_module_of_preloaded_classes());
@@ -208,9 +246,9 @@ void AOTLinkedClassBulkLoader::link_or_init_non_javabase_classes_impl(TRAPS) {
   assert(h_system_loader() != nullptr,   "must be");
 
   AOTLinkedClassTable* table = AOTLinkedClassTable::get();
-  link_or_init_classes_for_loader(Handle(), table->boot2(), CHECK);
-  link_or_init_classes_for_loader(h_platform_loader, table->platform(), CHECK);
-  link_or_init_classes_for_loader(h_system_loader, table->app(), CHECK);
+  init_classes_for_loader(Handle(), table->boot2(), CHECK);
+  init_classes_for_loader(h_platform_loader, table->platform(), CHECK);
+  init_classes_for_loader(h_system_loader, table->app(), CHECK);
 
   if (Universe::is_fully_initialized() && VerifyDuringStartup) {
     // Make sure we're still in a clean state.
@@ -242,8 +280,9 @@ void AOTLinkedClassBulkLoader::exit_on_exception(JavaThread* current) {
     log_error(aot)("Out of memory. Please run with a larger Java heap, current MaxHeapSize = "
                    "%zuM", MaxHeapSize/M);
   } else {
+    oop message = java_lang_Throwable::message(current->pending_exception());
     log_error(aot)("%s: %s", current->pending_exception()->klass()->external_name(),
-                   java_lang_String::as_utf8_string(java_lang_Throwable::message(current->pending_exception())));
+                   message == nullptr ? "(no message)" : java_lang_String::as_utf8_string(message));
   }
   vm_exit_during_initialization("Unexpected exception when loading aot-linked classes.");
 }
@@ -289,23 +328,13 @@ void AOTLinkedClassBulkLoader::initiate_loading(JavaThread* current, const char*
 // - classes that were AOT-initialized by AOTClassInitializer
 // - the classes of all objects that are reachable from the archived mirrors of
 //   the AOT-linked classes for <class_loader>.
-void AOTLinkedClassBulkLoader::link_or_init_classes_for_loader(Handle class_loader, Array<InstanceKlass*>* classes, TRAPS) {
+void AOTLinkedClassBulkLoader::init_classes_for_loader(Handle class_loader, Array<InstanceKlass*>* classes, TRAPS) {
   if (classes != nullptr) {
     for (int i = 0; i < classes->length(); i++) {
       InstanceKlass* ik = classes->at(i);
-      if (ik->class_loader_data() == nullptr) {
-        // This class is not yet loaded. We will initialize it in a later phase.
-        // For example, we have loaded only AOTLinkedClassCategory::BOOT1 classes
-        // but k is part of AOTLinkedClassCategory::BOOT2.
-        continue;
-      }
+      assert(ik->class_loader_data() != nullptr, "must be");
       if (ik->has_aot_initialized_mirror()) {
         ik->initialize_with_aot_initialized_mirror(CHECK);
-      } else {
-        // Some cached heap objects may hold references to methods in aot-linked
-        // classes (via MemberName). We need to make sure all classes are
-        // linked to allow such MemberNames to be invoked.
-        ik->link_class(CHECK);
       }
     }
   }

src/hotspot/share/cds/aotLinkedClassBulkLoader.hpp

@@ -52,10 +52,11 @@ class AOTLinkedClassBulkLoader :  AllStatic {
   static void preload_classes_impl(TRAPS);
   static void preload_classes_in_table(Array<InstanceKlass*>* classes,
                                        const char* category_name, Handle loader, TRAPS);
-  static void initiate_loading(JavaThread* current, const char* category, Handle initiating_loader,
-                               Array<InstanceKlass*>* classes);
-  static void link_or_init_non_javabase_classes_impl(TRAPS);
-  static void link_or_init_classes_for_loader(Handle class_loader, Array<InstanceKlass*>* classes, TRAPS);
+  static void initiate_loading(JavaThread* current, const char* category, Handle initiating_loader, Array<InstanceKlass*>* classes);
+  static void link_classes_impl(TRAPS);
+  static void link_classes_in_table(Array<InstanceKlass*>* classes, TRAPS);
+  static void init_non_javabase_classes_impl(TRAPS);
+  static void init_classes_for_loader(Handle class_loader, Array<InstanceKlass*>* classes, TRAPS);
   static void replay_training_at_init(Array<InstanceKlass*>* classes, TRAPS) NOT_CDS_RETURN;
 
 #ifdef ASSERT
@@ -67,9 +68,10 @@ class AOTLinkedClassBulkLoader :  AllStatic {
 
 public:
   static void serialize(SerializeClosure* soc) NOT_CDS_RETURN;
-  static void preload_classes(JavaThread* current);
-  static void link_or_init_javabase_classes(JavaThread* current) NOT_CDS_RETURN;
-  static void link_or_init_non_javabase_classes(JavaThread* current) NOT_CDS_RETURN;
+  static void preload_classes(JavaThread* current) NOT_CDS_RETURN;
+  static void link_classes(JavaThread* current) NOT_CDS_RETURN;
+  static void init_javabase_classes(JavaThread* current) NOT_CDS_RETURN;
+  static void init_non_javabase_classes(JavaThread* current) NOT_CDS_RETURN;
   static void exit_on_exception(JavaThread* current);
 
   static void replay_training_at_init_for_preloaded_classes(TRAPS) NOT_CDS_RETURN;

src/hotspot/share/cds/aotMapLogger.cpp

@@ -363,6 +363,12 @@ void AOTMapLogger::log_metaspace_objects_impl(address region_base, address regio
     case MetaspaceObj::KlassTrainingDataType:
       log_klass_training_data((KlassTrainingData*)src, requested_addr, type_name, bytes, current);
       break;
+    case MetaspaceObj::MethodTrainingDataType:
+      log_method_training_data((MethodTrainingData*)src, requested_addr, type_name, bytes, current);
+      break;
+    case MetaspaceObj::CompileTrainingDataType:
+      log_compile_training_data((CompileTrainingData*)src, requested_addr, type_name, bytes, current);
+      break;
     default:
       log_debug(aot, map)(_LOG_PREFIX, p2i(requested_addr), type_name, bytes);
       break;
@@ -431,6 +437,7 @@ void AOTMapLogger::log_symbol(Symbol* s, address requested_addr, const char* typ
   log_debug(aot, map)(_LOG_PREFIX " %s", p2i(requested_addr), type_name, bytes,
                       s->as_quoted_ascii());
 }
+
 void AOTMapLogger::log_klass_training_data(KlassTrainingData* ktd, address requested_addr, const char* type_name,
                                            int bytes, Thread* current) {
   ResourceMark rm(current);
@@ -442,6 +449,27 @@ void AOTMapLogger::log_klass_training_data(KlassTrainingData* ktd, address reque
   }
 }
 
+void AOTMapLogger::log_method_training_data(MethodTrainingData* mtd, address requested_addr, const char* type_name,
+                                            int bytes, Thread* current) {
+  ResourceMark rm(current);
+  if (mtd->has_holder()) {
+    log_debug(aot, map)(_LOG_PREFIX " %s", p2i(requested_addr), type_name, bytes,
+                        mtd->holder()->external_name());
+  } else {
+    log_debug(aot, map)(_LOG_PREFIX, p2i(requested_addr), type_name, bytes);
+  }
+}
+
+void AOTMapLogger::log_compile_training_data(CompileTrainingData* ctd, address requested_addr, const char* type_name,
+                                             int bytes, Thread* current) {
+  ResourceMark rm(current);
+  if (ctd->method() != nullptr && ctd->method()->has_holder()) {
+    log_debug(aot, map)(_LOG_PREFIX " %d %s", p2i(requested_addr), type_name, bytes,
+                         ctd->level(), ctd->method()->holder()->external_name());
+  } else {
+    log_debug(aot, map)(_LOG_PREFIX, p2i(requested_addr), type_name, bytes);
+  }
+}
 #undef _LOG_PREFIX
 
 // Log all the data [base...top). Pretend that the base address

src/hotspot/share/cds/aotMapLogger.hpp

@@ -33,9 +33,11 @@
 #include "utilities/growableArray.hpp"
 
 class ArchiveHeapInfo;
+class CompileTrainingData;
 class DumpRegion;
 class FileMapInfo;
 class KlassTrainingData;
+class MethodTrainingData;
 class outputStream;
 
 // Write detailed info to a mapfile to analyze contents of the AOT cache/CDS archive.
@@ -107,6 +109,8 @@ class AOTMapLogger : AllStatic {
   static void log_method(Method* m, address requested_addr, const char* type_name, int bytes, Thread* current);
   static void log_symbol(Symbol* s, address requested_addr, const char* type_name, int bytes, Thread* current);
   static void log_klass_training_data(KlassTrainingData* ktd, address requested_addr, const char* type_name, int bytes, Thread* current);
+  static void log_method_training_data(MethodTrainingData* mtd, address requested_addr, const char* type_name, int bytes, Thread* current);
+  static void log_compile_training_data(CompileTrainingData* ctd, address requested_addr, const char* type_name, int bytes, Thread* current);
 
 
 #if INCLUDE_CDS_JAVA_HEAP

src/hotspot/share/cds/archiveBuilder.cpp

@@ -655,12 +655,10 @@ void ArchiveBuilder::make_shallow_copies(DumpRegion *dump_region,
 
 void ArchiveBuilder::make_shallow_copy(DumpRegion *dump_region, SourceObjInfo* src_info) {
   address src = src_info->source_addr();
-  int bytes = src_info->size_in_bytes();
-  char* dest;
-  char* oldtop;
-  char* newtop;
+  int bytes = src_info->size_in_bytes(); // word-aligned
+  size_t alignment = SharedSpaceObjectAlignment; // alignment for the dest pointer
 
-  oldtop = dump_region->top();
+  char* oldtop = dump_region->top();
   if (src_info->msotype() == MetaspaceObj::ClassType) {
     // Allocate space for a pointer directly in front of the future InstanceKlass, so
     // we can do a quick lookup from InstanceKlass* -> RunTimeClassInfo*
@@ -671,21 +669,19 @@ void ArchiveBuilder::make_shallow_copy(DumpRegion *dump_region, SourceObjInfo* s
       SystemDictionaryShared::validate_before_archiving(InstanceKlass::cast(klass));
       dump_region->allocate(sizeof(address));
     }
-    // Allocate space for the future InstanceKlass with proper alignment
-    const size_t alignment =
 #ifdef _LP64
-      UseCompressedClassPointers ?
-        nth_bit(ArchiveBuilder::precomputed_narrow_klass_shift()) :
-        SharedSpaceObjectAlignment;
-#else
-      SharedSpaceObjectAlignment;
+    // More strict alignments needed for UseCompressedClassPointers
+    if (UseCompressedClassPointers) {
+      alignment = nth_bit(ArchiveBuilder::precomputed_narrow_klass_shift());
+    }
 #endif
-    dest = dump_region->allocate(bytes, alignment);
-  } else {
-    dest = dump_region->allocate(bytes);
+  } else if (src_info->msotype() == MetaspaceObj::SymbolType) {
+    // Symbols may be allocated by using AllocateHeap, so their sizes
+    // may be less than size_in_bytes() indicates.
+    bytes = ((Symbol*)src)->byte_size();
   }
-  newtop = dump_region->top();
 
+  char* dest = dump_region->allocate(bytes, alignment);
   memcpy(dest, src, bytes);
 
   // Update the hash of buffered sorted symbols for static dump so that the symbols have deterministic contents
@@ -713,6 +709,7 @@ void ArchiveBuilder::make_shallow_copy(DumpRegion *dump_region, SourceObjInfo* s
   log_trace(aot)("Copy: " PTR_FORMAT " ==> " PTR_FORMAT " %d", p2i(src), p2i(dest), bytes);
   src_info->set_buffered_addr((address)dest);
 
+  char* newtop = dump_region->top();
   _alloc_stats.record(src_info->msotype(), int(newtop - oldtop), src_info->read_only());
 
   DEBUG_ONLY(_alloc_stats.verify((int)dump_region->used(), src_info->read_only()));

src/hotspot/share/cds/cdsConfig.cpp

@@ -943,8 +943,9 @@ bool CDSConfig::is_preserving_verification_constraints() {
     return AOTClassLinking;
   } else if (is_dumping_final_static_archive()) { // writing AOT cache
     return is_dumping_aot_linked_classes();
+  } else if (is_dumping_classic_static_archive()) {
+    return is_dumping_aot_linked_classes();
   } else {
-    // For simplicity, we don't support this optimization with the old CDS workflow.
     return false;
   }
 }

src/hotspot/share/cds/runTimeClassInfo.cpp

@@ -41,7 +41,7 @@ void RunTimeClassInfo::init(DumpTimeClassInfo& info) {
   _num_loader_constraints   = info.num_loader_constraints();
   int i;
 
-  if (CDSConfig::is_preserving_verification_constraints() && CDSConfig::is_dumping_final_static_archive()) {
+  if (CDSConfig::is_preserving_verification_constraints()) {
     // The production run doesn't need the verifier constraints, as we can guarantee that all classes checked by
     // the verifier during AOT training/assembly phases cannot be replaced in the production run.
     _num_verifier_constraints = 0;

src/hotspot/share/classfile/modules.cpp

@@ -456,7 +456,7 @@ void Modules::define_module(Handle module, jboolean is_open, jstring version,
 #if COMPILER2_OR_JVMCI
   // Special handling of jdk.incubator.vector
   if (strcmp(module_name, "jdk.incubator.vector") == 0) {
-    if (FLAG_IS_DEFAULT(EnableVectorSupport)) {
+    if (FLAG_IS_DEFAULT(EnableVectorSupport) && VM_Version::supports_misaligned_vector_accesses()) {
       FLAG_SET_DEFAULT(EnableVectorSupport, true);
     }
     if (EnableVectorSupport && FLAG_IS_DEFAULT(EnableVectorReboxing)) {

src/hotspot/share/classfile/systemDictionaryShared.cpp

@@ -855,6 +855,28 @@ public:
   }
 };
 
+void SystemDictionaryShared::link_all_exclusion_check_candidates(InstanceKlass* ik) {
+  bool need_to_link = false;
+  {
+    MutexLocker ml(DumpTimeTable_lock, Mutex::_no_safepoint_check_flag);
+    ExclusionCheckCandidates candidates(ik);
+
+    candidates.iterate_all([&] (InstanceKlass* k, DumpTimeClassInfo* info) {
+      if (!k->is_linked()) {
+        need_to_link = true;
+      }
+    });
+  }
+  if (need_to_link) {
+    JavaThread* THREAD = JavaThread::current();
+    if (log_is_enabled(Info, aot, link)) {
+      ResourceMark rm(THREAD);
+      log_info(aot, link)("Link all loaded classes for %s", ik->external_name());
+    }
+    AOTMetaspace::link_all_loaded_classes(THREAD);
+  }
+}
+
 // Returns true if the class should be excluded. This can be called by
 // AOTConstantPoolResolver before or after we enter the CDS safepoint.
 // When called before the safepoint, we need to link the class so that
@@ -878,27 +900,19 @@ bool SystemDictionaryShared::should_be_excluded(Klass* k) {
     InstanceKlass* ik = InstanceKlass::cast(k);
 
     if (!SafepointSynchronize::is_at_safepoint()) {
-      if (!ik->is_linked()) {
-        // should_be_excluded_impl() below doesn't link unlinked classes. We come
-        // here only when we are trying to aot-link constant pool entries, so
-        // we'd better link the class.
-        JavaThread* THREAD = JavaThread::current();
-        ik->link_class(THREAD);
-        if (HAS_PENDING_EXCEPTION) {
-          CLEAR_PENDING_EXCEPTION;
-          return true; // linking failed -- let's exclude it
+      {
+        // fast path
+        MutexLocker ml(DumpTimeTable_lock, Mutex::_no_safepoint_check_flag);
+        DumpTimeClassInfo* p = get_info_locked(ik);
+        if (p->has_checked_exclusion()) {
+          return p->is_excluded();
         }
-
-        // Also link any classes that were loaded for the verification of ik or its supertypes.
-        // Otherwise we might miss the verification constraints of those classes.
-        AOTMetaspace::link_all_loaded_classes(THREAD);
       }
 
+      link_all_exclusion_check_candidates(ik);
+
       MutexLocker ml(DumpTimeTable_lock, Mutex::_no_safepoint_check_flag);
       DumpTimeClassInfo* p = get_info_locked(ik);
-      if (p->is_excluded()) {
-        return true;
-      }
       return should_be_excluded_impl(ik, p);
     } else {
       // When called within the CDS safepoint, the correctness of this function
@@ -912,7 +926,7 @@ bool SystemDictionaryShared::should_be_excluded(Klass* k) {
 
       // No need to check for is_linked() as all eligible classes should have
       // already been linked in AOTMetaspace::link_class_for_cds().
-      // Can't take the lock as we are in safepoint.
+      // Don't take DumpTimeTable_lock as we are in safepoint.
       DumpTimeClassInfo* p = _dumptime_table->get(ik);
       if (p->is_excluded()) {
         return true;
@@ -1420,6 +1434,10 @@ void SystemDictionaryShared::get_all_archived_classes(bool is_static_archive, Gr
   get_archive(is_static_archive)->_builtin_dictionary.iterate([&] (const RunTimeClassInfo* record) {
       classes->append(record->klass());
     });
+
+  get_archive(is_static_archive)->_unregistered_dictionary.iterate([&] (const RunTimeClassInfo* record) {
+      classes->append(record->klass());
+    });
 }
 
 class SharedDictionaryPrinter : StackObj {

src/hotspot/share/classfile/systemDictionaryShared.hpp

@@ -175,6 +175,7 @@ private:
   static void write_dictionary(RunTimeSharedDictionary* dictionary,
                                bool is_builtin);
   static bool is_jfr_event_class(InstanceKlass *k);
+  static void link_all_exclusion_check_candidates(InstanceKlass* ik);
   static bool should_be_excluded_impl(InstanceKlass* k, DumpTimeClassInfo* info);
 
   // exclusion checks

src/hotspot/share/classfile/verifier.cpp

@@ -2442,209 +2442,6 @@ void ClassVerifier::verify_field_instructions(RawBytecodeStream* bcs,
   }
 }
 
-// Look at the method's handlers.  If the bci is in the handler's try block
-// then check if the handler_pc is already on the stack.  If not, push it
-// unless the handler has already been scanned.
-void ClassVerifier::push_handlers(ExceptionTable* exhandlers,
-                                  GrowableArray<u4>* handler_list,
-                                  GrowableArray<u4>* handler_stack,
-                                  u4 bci) {
-  int exlength = exhandlers->length();
-  for(int x = 0; x < exlength; x++) {
-    if (bci >= exhandlers->start_pc(x) && bci < exhandlers->end_pc(x)) {
-      u4 exhandler_pc = exhandlers->handler_pc(x);
-      if (!handler_list->contains(exhandler_pc)) {
-        handler_stack->append_if_missing(exhandler_pc);
-        handler_list->append(exhandler_pc);
-      }
-    }
-  }
-}
-
-// Return TRUE if all code paths starting with start_bc_offset end in
-// bytecode athrow or loop.
-bool ClassVerifier::ends_in_athrow(u4 start_bc_offset) {
-  ResourceMark rm;
-  // Create bytecode stream.
-  RawBytecodeStream bcs(method());
-  int code_length = method()->code_size();
-  bcs.set_start(start_bc_offset);
-
-  // Create stack for storing bytecode start offsets for if* and *switch.
-  GrowableArray<u4>* bci_stack = new GrowableArray<u4>(30);
-  // Create stack for handlers for try blocks containing this handler.
-  GrowableArray<u4>* handler_stack = new GrowableArray<u4>(30);
-  // Create list of handlers that have been pushed onto the handler_stack
-  // so that handlers embedded inside of their own TRY blocks only get
-  // scanned once.
-  GrowableArray<u4>* handler_list = new GrowableArray<u4>(30);
-  // Create list of visited branch opcodes (goto* and if*).
-  GrowableArray<u4>* visited_branches = new GrowableArray<u4>(30);
-  ExceptionTable exhandlers(_method());
-
-  while (true) {
-    if (bcs.is_last_bytecode()) {
-      // if no more starting offsets to parse or if at the end of the
-      // method then return false.
-      if ((bci_stack->is_empty()) || (bcs.end_bci() == code_length))
-        return false;
-      // Pop a bytecode starting offset and scan from there.
-      bcs.set_start(bci_stack->pop());
-    }
-    Bytecodes::Code opcode = bcs.raw_next();
-    int bci = bcs.bci();
-
-    // If the bytecode is in a TRY block, push its handlers so they
-    // will get parsed.
-    push_handlers(&exhandlers, handler_list, handler_stack, bci);
-
-    switch (opcode) {
-      case Bytecodes::_if_icmpeq:
-      case Bytecodes::_if_icmpne:
-      case Bytecodes::_if_icmplt:
-      case Bytecodes::_if_icmpge:
-      case Bytecodes::_if_icmpgt:
-      case Bytecodes::_if_icmple:
-      case Bytecodes::_ifeq:
-      case Bytecodes::_ifne:
-      case Bytecodes::_iflt:
-      case Bytecodes::_ifge:
-      case Bytecodes::_ifgt:
-      case Bytecodes::_ifle:
-      case Bytecodes::_if_acmpeq:
-      case Bytecodes::_if_acmpne:
-      case Bytecodes::_ifnull:
-      case Bytecodes::_ifnonnull: {
-        int target = bcs.dest();
-        if (visited_branches->contains(bci)) {
-          if (bci_stack->is_empty()) {
-            if (handler_stack->is_empty()) {
-              return true;
-            } else {
-              // Parse the catch handlers for try blocks containing athrow.
-              bcs.set_start(handler_stack->pop());
-            }
-          } else {
-            // Pop a bytecode starting offset and scan from there.
-            bcs.set_start(bci_stack->pop());
-          }
-        } else {
-          if (target > bci) { // forward branch
-            if (target >= code_length) return false;
-            // Push the branch target onto the stack.
-            bci_stack->push(target);
-            // then, scan bytecodes starting with next.
-            bcs.set_start(bcs.next_bci());
-          } else { // backward branch
-            // Push bytecode offset following backward branch onto the stack.
-            bci_stack->push(bcs.next_bci());
-            // Check bytecodes starting with branch target.
-            bcs.set_start(target);
-          }
-          // Record target so we don't branch here again.
-          visited_branches->append(bci);
-        }
-        break;
-        }
-
-      case Bytecodes::_goto:
-      case Bytecodes::_goto_w: {
-        int target = (opcode == Bytecodes::_goto ? bcs.dest() : bcs.dest_w());
-        if (visited_branches->contains(bci)) {
-          if (bci_stack->is_empty()) {
-            if (handler_stack->is_empty()) {
-              return true;
-            } else {
-              // Parse the catch handlers for try blocks containing athrow.
-              bcs.set_start(handler_stack->pop());
-            }
-          } else {
-            // Been here before, pop new starting offset from stack.
-            bcs.set_start(bci_stack->pop());
-          }
-        } else {
-          if (target >= code_length) return false;
-          // Continue scanning from the target onward.
-          bcs.set_start(target);
-          // Record target so we don't branch here again.
-          visited_branches->append(bci);
-        }
-        break;
-        }
-
-      // Check that all switch alternatives end in 'athrow' bytecodes. Since it
-      // is  difficult to determine where each switch alternative ends, parse
-      // each switch alternative until either hit a 'return', 'athrow', or reach
-      // the end of the method's bytecodes.  This is gross but should be okay
-      // because:
-      // 1. tableswitch and lookupswitch byte codes in handlers for ctor explicit
-      //    constructor invocations should be rare.
-      // 2. if each switch alternative ends in an athrow then the parsing should be
-      //    short.  If there is no athrow then it is bogus code, anyway.
-      case Bytecodes::_lookupswitch:
-      case Bytecodes::_tableswitch:
-        {
-          address aligned_bcp = align_up(bcs.bcp() + 1, jintSize);
-          int default_offset = Bytes::get_Java_u4(aligned_bcp) + bci;
-          int keys, delta;
-          if (opcode == Bytecodes::_tableswitch) {
-            jint low = (jint)Bytes::get_Java_u4(aligned_bcp + jintSize);
-            jint high = (jint)Bytes::get_Java_u4(aligned_bcp + 2*jintSize);
-            // This is invalid, but let the regular bytecode verifier
-            // report this because the user will get a better error message.
-            if (low > high) return true;
-            keys = high - low + 1;
-            delta = 1;
-          } else {
-            keys = (int)Bytes::get_Java_u4(aligned_bcp + jintSize);
-            delta = 2;
-          }
-          // Invalid, let the regular bytecode verifier deal with it.
-          if (keys < 0) return true;
-
-          // Push the offset of the next bytecode onto the stack.
-          bci_stack->push(bcs.next_bci());
-
-          // Push the switch alternatives onto the stack.
-          for (int i = 0; i < keys; i++) {
-            int target = bci + (jint)Bytes::get_Java_u4(aligned_bcp+(3+i*delta)*jintSize);
-            if (target > code_length) return false;
-            bci_stack->push(target);
-          }
-
-          // Start bytecode parsing for the switch at the default alternative.
-          if (default_offset > code_length) return false;
-          bcs.set_start(default_offset);
-          break;
-        }
-
-      case Bytecodes::_return:
-        return false;
-
-      case Bytecodes::_athrow:
-        {
-          if (bci_stack->is_empty()) {
-            if (handler_stack->is_empty()) {
-              return true;
-            } else {
-              // Parse the catch handlers for try blocks containing athrow.
-              bcs.set_start(handler_stack->pop());
-            }
-          } else {
-            // Pop a bytecode offset and starting scanning from there.
-            bcs.set_start(bci_stack->pop());
-          }
-        }
-        break;
-
-      default:
-        ;
-    } // end switch
-  } // end while loop
-
-  return false;
-}
-
 void ClassVerifier::verify_invoke_init(
     RawBytecodeStream* bcs, u2 ref_class_index, VerificationType ref_class_type,
     StackMapFrame* current_frame, u4 code_length, bool in_try_block,
@@ -2669,25 +2466,6 @@ void ClassVerifier::verify_invoke_init(
     // sure that all catch clause paths end in a throw.  Otherwise, this can
     // result in returning an incomplete object.
     if (in_try_block) {
-      ExceptionTable exhandlers(_method());
-      int exlength = exhandlers.length();
-      for(int i = 0; i < exlength; i++) {
-        u2 start_pc = exhandlers.start_pc(i);
-        u2 end_pc = exhandlers.end_pc(i);
-
-        if (bci >= start_pc && bci < end_pc) {
-          if (!ends_in_athrow(exhandlers.handler_pc(i))) {
-            verify_error(ErrorContext::bad_code(bci),
-              "Bad <init> method call from after the start of a try block");
-            return;
-          } else if (log_is_enabled(Debug, verification)) {
-            ResourceMark rm(THREAD);
-            log_debug(verification)("Survived call to ends_in_athrow(): %s",
-                                          current_class()->name()->as_C_string());
-          }
-        }
-      }
-
       // Check the exception handler target stackmaps with the locals from the
       // incoming stackmap (before initialize_object() changes them to outgoing
       // state).

src/hotspot/share/classfile/verifier.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1998, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2025, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -334,17 +334,6 @@ class ClassVerifier : public StackObj {
     bool* this_uninit, const constantPoolHandle& cp, StackMapTable* stackmap_table,
     TRAPS);
 
-  // Used by ends_in_athrow() to push all handlers that contain bci onto the
-  // handler_stack, if the handler has not already been pushed on the stack.
-  void push_handlers(ExceptionTable* exhandlers,
-                     GrowableArray<u4>* handler_list,
-                     GrowableArray<u4>* handler_stack,
-                     u4 bci);
-
-  // Returns true if all paths starting with start_bc_offset end in athrow
-  // bytecode or loop.
-  bool ends_in_athrow(u4 start_bc_offset);
-
   void verify_invoke_instructions(
     RawBytecodeStream* bcs, u4 code_length, StackMapFrame* current_frame,
     bool in_try_block, bool* this_uninit, VerificationType return_type,

src/hotspot/share/code/nmethod.cpp

@@ -23,6 +23,7 @@
  */
 
 #include "asm/assembler.inline.hpp"
+#include "cds/cdsConfig.hpp"
 #include "code/codeCache.hpp"
 #include "code/compiledIC.hpp"
 #include "code/dependencies.hpp"
@@ -1147,7 +1148,7 @@ nmethod* nmethod::new_nmethod(const methodHandle& method,
     + align_up(speculations_len                  , oopSize)
 #endif
     + align_up(debug_info->data_size()           , oopSize)
-    + align_up(ImmutableDataReferencesCounterSize, oopSize);
+    + ImmutableDataReferencesCounterSize;
 
   // First, allocate space for immutable data in C heap.
   address immutable_data = nullptr;
@@ -1322,6 +1323,7 @@ nmethod::nmethod(
 #if INCLUDE_JVMCI
     _speculations_offset     = 0;
 #endif
+    _immutable_data_reference_counter_offset = 0;
 
     code_buffer->copy_code_and_locs_to(this);
     code_buffer->copy_values_to(this);
@@ -1420,15 +1422,6 @@ nmethod::nmethod(const nmethod &nm) : CodeBlob(nm._name, nm._kind, nm._size, nm.
   _method                       = nm._method;
   _osr_link                     = nullptr;
 
-  // Increment number of references to immutable data to share it between nmethods
-  _immutable_data_size          = nm._immutable_data_size;
-  if (_immutable_data_size > 0) {
-    _immutable_data             = nm._immutable_data;
-    set_immutable_data_references_counter(get_immutable_data_references_counter() + 1);
-  } else {
-    _immutable_data             = blob_end();
-  }
-
   _exception_cache              = nullptr;
   _gc_data                      = nullptr;
   _oops_do_mark_nmethods        = nullptr;
@@ -1444,6 +1437,7 @@ nmethod::nmethod(const nmethod &nm) : CodeBlob(nm._name, nm._kind, nm._size, nm.
   _entry_offset                 = nm._entry_offset;
   _verified_entry_offset        = nm._verified_entry_offset;
   _entry_bci                    = nm._entry_bci;
+  _immutable_data_size          = nm._immutable_data_size;
 
   _skipped_instructions_size    = nm._skipped_instructions_size;
   _stub_offset                  = nm._stub_offset;
@@ -1462,6 +1456,15 @@ nmethod::nmethod(const nmethod &nm) : CodeBlob(nm._name, nm._kind, nm._size, nm.
 #if INCLUDE_JVMCI
   _speculations_offset          = nm._speculations_offset;
 #endif
+  _immutable_data_reference_counter_offset = nm._immutable_data_reference_counter_offset;
+
+  // Increment number of references to immutable data to share it between nmethods
+  if (_immutable_data_size > 0) {
+    _immutable_data             = nm._immutable_data;
+    set_immutable_data_references_counter(get_immutable_data_references_counter() + 1);
+  } else {
+    _immutable_data             = blob_end();
+  }
 
   _orig_pc_offset               = nm._orig_pc_offset;
   _compile_id                   = nm._compile_id;
@@ -1751,9 +1754,11 @@ nmethod::nmethod(
 
 #if INCLUDE_JVMCI
     _speculations_offset  = _scopes_data_offset   + align_up(debug_info->data_size(), oopSize);
-    DEBUG_ONLY( int immutable_data_end_offset = _speculations_offset + align_up(speculations_len, oopSize) + align_up(ImmutableDataReferencesCounterSize, oopSize); )
+    _immutable_data_reference_counter_offset = _speculations_offset + align_up(speculations_len, oopSize);
+    DEBUG_ONLY( int immutable_data_end_offset = _immutable_data_reference_counter_offset + ImmutableDataReferencesCounterSize; )
 #else
-    DEBUG_ONLY( int immutable_data_end_offset = _scopes_data_offset + align_up(debug_info->data_size(), oopSize) + align_up(ImmutableDataReferencesCounterSize, oopSize); )
+    _immutable_data_reference_counter_offset =  _scopes_data_offset + align_up(debug_info->data_size(), oopSize);
+    DEBUG_ONLY( int immutable_data_end_offset = _immutable_data_reference_counter_offset + ImmutableDataReferencesCounterSize; )
 #endif
     assert(immutable_data_end_offset <= immutable_data_size, "wrong read-only data size: %d > %d",
            immutable_data_end_offset, immutable_data_size);
@@ -2500,11 +2505,48 @@ void nmethod::post_compiled_method(CompileTask* task) {
   maybe_print_nmethod(directive);
 }
 
+#if INCLUDE_CDS
+static GrowableArrayCHeap<nmethod*, mtClassShared>* _delayed_compiled_method_load_events = nullptr;
+
+void nmethod::add_delayed_compiled_method_load_event(nmethod* nm) {
+  precond(CDSConfig::is_using_aot_linked_classes());
+  precond(!ServiceThread::has_started());
+
+  // We are still in single threaded stage of VM bootstrap. No need to lock.
+  if (_delayed_compiled_method_load_events == nullptr) {
+    _delayed_compiled_method_load_events = new GrowableArrayCHeap<nmethod*, mtClassShared>();
+  }
+  _delayed_compiled_method_load_events->append(nm);
+}
+
+void nmethod::post_delayed_compiled_method_load_events() {
+  precond(ServiceThread::has_started());
+  if (_delayed_compiled_method_load_events != nullptr) {
+    for (int i = 0; i < _delayed_compiled_method_load_events->length(); i++) {
+      nmethod* nm = _delayed_compiled_method_load_events->at(i);
+      nm->post_compiled_method_load_event();
+    }
+    delete _delayed_compiled_method_load_events;
+    _delayed_compiled_method_load_events = nullptr;
+  }
+}
+#endif
+
 // ------------------------------------------------------------------
 // post_compiled_method_load_event
 // new method for install_code() path
 // Transfer information from compilation to jvmti
 void nmethod::post_compiled_method_load_event(JvmtiThreadState* state) {
+#if INCLUDE_CDS
+  if (!ServiceThread::has_started()) {
+    // With AOT-linked classes, we could compile wrappers for native methods before the
+    // ServiceThread has been started, so we must delay the events to be posted later.
+    assert(state == nullptr, "must be");
+    add_delayed_compiled_method_load_event(this);
+    return;
+  }
+#endif
+
   // This is a bad time for a safepoint.  We don't want
   // this nmethod to get unloaded while we're queueing the event.
   NoSafepointVerifier nsv;

src/hotspot/share/code/nmethod.hpp

@@ -250,6 +250,7 @@ class nmethod : public CodeBlob {
 #if INCLUDE_JVMCI
   int      _speculations_offset;
 #endif
+  int      _immutable_data_reference_counter_offset;
 
   // location in frame (offset for sp) that deopt can store the original
   // pc during a deopt.
@@ -646,12 +647,11 @@ public:
 #if INCLUDE_JVMCI
   address scopes_data_end       () const { return           _immutable_data + _speculations_offset ; }
   address speculations_begin    () const { return           _immutable_data + _speculations_offset ; }
-  address speculations_end      () const { return           immutable_data_end() - ImmutableDataReferencesCounterSize ; }
+  address speculations_end      () const { return           _immutable_data + _immutable_data_reference_counter_offset ; }
 #else
-  address scopes_data_end       () const { return           immutable_data_end() - ImmutableDataReferencesCounterSize ; }
+  address scopes_data_end       () const { return           _immutable_data + _immutable_data_reference_counter_offset ; }
 #endif
-
-  address immutable_data_references_counter_begin () const { return immutable_data_end() - ImmutableDataReferencesCounterSize ; }
+  address immutable_data_references_counter_begin () const { return _immutable_data + _immutable_data_reference_counter_offset ; }
 
   // Sizes
   int immutable_data_size() const { return _immutable_data_size; }
@@ -965,6 +965,8 @@ public:
   inline int  get_immutable_data_references_counter()           { return *((int*)immutable_data_references_counter_begin());  }
   inline void set_immutable_data_references_counter(int count)  { *((int*)immutable_data_references_counter_begin()) = count; }
 
+  static void add_delayed_compiled_method_load_event(nmethod* nm) NOT_CDS_RETURN;
+
  public:
   // ScopeDesc retrieval operation
   PcDesc* pc_desc_at(address pc)   { return find_pc_desc(pc, false); }
@@ -999,6 +1001,9 @@ public:
   // Avoid hiding of parent's 'decode(outputStream*)' method.
   void decode(outputStream* st) const { decode2(st); } // just delegate here.
 
+  // AOT cache support
+  static void post_delayed_compiled_method_load_events() NOT_CDS_RETURN;
+
   // printing support
   void print_on_impl(outputStream* st) const;
   void print_code();

src/hotspot/share/compiler/compilerOracle.cpp

@@ -617,18 +617,44 @@ static void usage() {
   tty->cr();
   print_commands();
   tty->cr();
-    tty->print_cr("Method patterns has the format:");
-  tty->print_cr("  package/Class.method()");
+  tty->print_cr("The <method pattern> has the format '<class>.<method><descriptor>'.");
+  tty->cr();
+  tty->print_cr("For example, the <method pattern>");
+  tty->cr();
+  tty->print_cr("  package/Class.method(Lpackage/Parameter;)Lpackage/Return;");
+  tty->cr();
+  tty->print_cr("matches the <method> 'method' in <class> 'package/Class' with <descriptor>");
+  tty->print_cr("'(Lpackage/Parameter;)Lpackage/Return;'");
   tty->cr();
   tty->print_cr("For backward compatibility this form is also allowed:");
-  tty->print_cr("  package.Class::method()");
   tty->cr();
-  tty->print_cr("The signature can be separated by an optional whitespace or comma:");
-  tty->print_cr("  package/Class.method ()");
+  tty->print_cr("  package.Class::method(Lpackage.Parameter;)Lpackage.Return;");
   tty->cr();
-  tty->print_cr("The class and method identifier can be used together with leading or");
-  tty->print_cr("trailing *'s for wildcard matching:");
-  tty->print_cr("  *ackage/Clas*.*etho*()");
+  tty->print_cr("A whitespace or comma can optionally separate the <descriptor> from the");
+  tty->print_cr("<method>:");
+  tty->cr();
+  tty->print_cr("  package/Class.method (Lpackage/Parameter;)Lpackage/Return;");
+  tty->print_cr("  package/Class.method,(Lpackage/Parameter;)Lpackage/Return;");
+  tty->cr();
+  tty->print_cr("The <class> and <method> accept leading and trailing '*' wildcards");
+  tty->print_cr("matching:");
+  tty->cr();
+  tty->print_cr("  *ackage/Clas*.*etho*(Lpackage/Parameter;)Lpackage/Return;");
+  tty->cr();
+  tty->print_cr("The <descriptor> does not support explicit wildcards and");
+  tty->print_cr("always has an implicit trailing wildcard. Therefore,");
+  tty->cr();
+  tty->print_cr("  package/Class.method(Lpackage/Parameter;)Lpackage/Return;");
+  tty->cr();
+  tty->print_cr("matches a subset of");
+  tty->cr();
+  tty->print_cr("  package/Class.method(Lpackage/Parameter;)");
+  tty->cr();
+  tty->print_cr("which matches a subset of");
+  tty->cr();
+  tty->print_cr("  package/Class.method");
+  tty->cr();
+  tty->print_cr("which matches all possible descriptors.");
   tty->cr();
   tty->print_cr("It is possible to use more than one CompileCommand on the command line:");
   tty->print_cr("  -XX:CompileCommand=exclude,java/*.* -XX:CompileCommand=log,java*.*");

src/hotspot/share/gc/epsilon/epsilonHeap.cpp

@@ -52,7 +52,7 @@ jint EpsilonHeap::initialize() {
   initialize_reserved_region(heap_rs);
 
   _space = new ContiguousSpace();
-  _space->initialize(committed_region, /* clear_space = */ true, /* mangle_space = */ true);
+  _space->initialize(committed_region, /* clear_space = */ true);
 
   // Precompute hot fields
   _max_tlab_size = MIN2(CollectedHeap::max_tlab_size(), align_object_size(EpsilonMaxTLABSize / HeapWordSize));

src/hotspot/share/gc/g1/g1BarrierSet.hpp

@@ -84,10 +84,6 @@ class G1BarrierSet: public CardTableBarrierSet {
   // Update the given thread's card table (byte map) base to the current card table's.
   void update_card_table_base(Thread* thread);
 
-  virtual bool card_mark_must_follow_store() const {
-    return true;
-  }
-
   // Add "pre_val" to a set of objects that may have been disconnected from the
   // pre-marking object graph. Prefer the version that takes location, as it
   // can avoid touching the heap unnecessarily.

src/hotspot/share/gc/g1/g1CollectedHeap.cpp

@@ -1391,7 +1391,6 @@ jint G1CollectedHeap::initialize() {
   G1CardTable* refinement_table = new G1CardTable(_reserved);
 
   G1BarrierSet* bs = new G1BarrierSet(card_table, refinement_table);
-  bs->initialize();
   assert(bs->is_a(BarrierSet::G1BarrierSet), "sanity");
 
   // Create space mappers.
@@ -2269,6 +2268,10 @@ void G1CollectedHeap::print_heap_regions() const {
   }
 }
 
+static void print_region_type(outputStream* st, const char* type, uint count, bool last = false) {
+  st->print("%u %s (%zuM)%s", count, type, count * G1HeapRegion::GrainBytes / M, last ? "\n" : ", ");
+}
+
 void G1CollectedHeap::print_heap_on(outputStream* st) const {
   size_t heap_used = Heap_lock->owned_by_self() ? used() : used_unlocked();
   st->print("%-20s", "garbage-first heap");
@@ -2280,14 +2283,13 @@ void G1CollectedHeap::print_heap_on(outputStream* st) const {
   st->cr();
 
   StreamIndentor si(st, 1);
-  st->print("region size %zuK, ", G1HeapRegion::GrainBytes / K);
-  uint young_regions = young_regions_count();
-  st->print("%u young (%zuK), ", young_regions,
-            (size_t) young_regions * G1HeapRegion::GrainBytes / K);
-  uint survivor_regions = survivor_regions_count();
-  st->print("%u survivors (%zuK)", survivor_regions,
-            (size_t) survivor_regions * G1HeapRegion::GrainBytes / K);
-  st->cr();
+  st->print("region size %zuM, ", G1HeapRegion::GrainBytes / M);
+  print_region_type(st, "eden", eden_regions_count());
+  print_region_type(st, "survivor", survivor_regions_count());
+  print_region_type(st, "old", old_regions_count());
+  print_region_type(st, "humongous", humongous_regions_count());
+  print_region_type(st, "free", num_free_regions(), true /* last */);
+
   if (_numa->is_enabled()) {
     uint num_nodes = _numa->num_active_nodes();
     st->print("remaining free region(s) on each NUMA node: ");
@@ -2564,13 +2566,6 @@ void G1CollectedHeap::verify_after_young_collection(G1HeapVerifier::G1VerifyType
   phase_times()->record_verify_after_time_ms((Ticks::now() - start).seconds() * MILLIUNITS);
 }
 
-void G1CollectedHeap::do_collection_pause_at_safepoint(size_t allocation_word_size) {
-  assert_at_safepoint_on_vm_thread();
-  guarantee(!is_stw_gc_active(), "collection is not reentrant");
-
-  do_collection_pause_at_safepoint_helper(allocation_word_size);
-}
-
 G1HeapPrinterMark::G1HeapPrinterMark(G1CollectedHeap* g1h) : _g1h(g1h), _heap_transition(g1h) {
   // This summary needs to be printed before incrementing total collections.
   _g1h->rem_set()->print_periodic_summary_info("Before GC RS summary",
@@ -2632,7 +2627,10 @@ void G1CollectedHeap::flush_region_pin_cache() {
   }
 }
 
-void G1CollectedHeap::do_collection_pause_at_safepoint_helper(size_t allocation_word_size) {
+void G1CollectedHeap::do_collection_pause_at_safepoint(size_t allocation_word_size) {
+  assert_at_safepoint_on_vm_thread();
+  assert(!is_stw_gc_active(), "collection is not reentrant");
+
   ResourceMark rm;
 
   IsSTWGCActiveMark active_gc_mark;

src/hotspot/share/gc/g1/g1CollectedHeap.hpp

@@ -769,10 +769,6 @@ private:
   // precondition: !is_stw_gc_active()
   void do_collection_pause_at_safepoint(size_t allocation_word_size = 0);
 
-  // Helper for do_collection_pause_at_safepoint, containing the guts
-  // of the incremental collection pause, executed by the vm thread.
-  void do_collection_pause_at_safepoint_helper(size_t allocation_word_size);
-
   void verify_before_young_collection(G1HeapVerifier::G1VerifyType type);
   void verify_after_young_collection(G1HeapVerifier::G1VerifyType type);
 

src/hotspot/share/gc/g1/g1FreeIdSet.cpp

@@ -1,105 +0,0 @@
-/*
- * Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#include "gc/g1/g1FreeIdSet.hpp"
-#include "memory/allocation.inline.hpp"
-#include "runtime/atomicAccess.hpp"
-#include "utilities/checkedCast.hpp"
-#include "utilities/debug.hpp"
-#include "utilities/globalDefinitions.hpp"
-#include "utilities/macros.hpp"
-#include "utilities/powerOfTwo.hpp"
-
-G1FreeIdSet::G1FreeIdSet(uint start, uint size) :
-  _sem(size),          // counting semaphore for available ids
-  _next(nullptr),      // array of "next" indices
-  _start(start),       // first id value
-  _size(size),         // number of available ids
-  _head_index_mask(0), // mask for extracting index from a _head value.
-  _head(0)             // low part: index; high part: update counter
-{
-  assert(size != 0, "precondition");
-  assert(start <= (UINT_MAX - size),
-         "start (%u) + size (%u) overflow: ", start, size);
-  // 2^shift must be greater than size. Equal is not permitted, because
-  // size is the "end of list" value, and can be the index part of _head.
-  uint shift = log2i(size) + 1;
-  assert(shift <= (BitsPerWord / 2), "excessive size %u", size);
-  _head_index_mask = (uintx(1) << shift) - 1;
-  assert(size <= _head_index_mask, "invariant");
-  _next = NEW_C_HEAP_ARRAY(uint, size, mtGC);
-  for (uint i = 0; i < size; ++i) {
-    _next[i] = i + 1;
-  }
-}
-
-G1FreeIdSet::~G1FreeIdSet() {
-  FREE_C_HEAP_ARRAY(uint, _next);
-}
-
-uint G1FreeIdSet::head_index(uintx head) const {
-  return checked_cast<uint>(head & _head_index_mask);
-}
-
-uintx G1FreeIdSet::make_head(uint index, uintx old_head) const {
-  // Include incremented old update counter to avoid ABA problem.
-  return index | ((old_head & ~_head_index_mask) + 1 + _head_index_mask);
-}
-
-const uint Claimed = UINT_MAX;
-
-uint G1FreeIdSet::claim_par_id() {
-  _sem.wait();
-  // Semaphore gate permits passage by no more than the number of
-  // available ids, so there must be one that we can claim.  But there
-  // may be multiple threads trying to claim ids at the same time.
-  uintx old_head = AtomicAccess::load(&_head);
-  uint index;
-  while (true) {
-    index = head_index(old_head);
-    assert(index < _size, "invariant");
-    uintx new_head = make_head(_next[index], old_head);
-    new_head = AtomicAccess::cmpxchg(&_head, old_head, new_head);
-    if (new_head == old_head) break;
-    old_head = new_head;
-  }
-  DEBUG_ONLY(_next[index] = Claimed;)
-  return _start + index;
-}
-
-void G1FreeIdSet::release_par_id(uint id) {
-  uint index = id - _start;
-  assert(index < _size, "invalid id %u", id);
-  assert(_next[index] == Claimed, "precondition");
-  uintx old_head = AtomicAccess::load(&_head);
-  while (true) {
-    _next[index] = head_index(old_head);
-    uintx new_head = make_head(index, old_head);
-    new_head = AtomicAccess::cmpxchg(&_head, old_head, new_head);
-    if (new_head == old_head) break;
-    old_head = new_head;
-  }
-  // Now that id has been released, permit another thread through the gate.
-  _sem.signal();
-}

src/hotspot/share/gc/g1/g1FreeIdSet.hpp

@@ -1,62 +0,0 @@
-/*
- * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#ifndef SHARE_GC_G1_G1FREEIDSET_HPP
-#define SHARE_GC_G1_G1FREEIDSET_HPP
-
-#include "runtime/semaphore.hpp"
-#include "utilities/globalDefinitions.hpp"
-
-// Represents a set of small integer ids, from which elements can be
-// temporarily allocated for exclusive use.  The ids are in a
-// contiguous range from 'start' to 'start + size'.  Used to obtain a
-// distinct worker_id value for a mutator thread that doesn't normally
-// have such an id.
-class G1FreeIdSet {
-  Semaphore _sem;
-  uint* _next;
-  uint _start;
-  uint _size;
-  uintx _head_index_mask;
-  volatile uintx _head;
-
-  uint head_index(uintx head) const;
-  uintx make_head(uint index, uintx old_head) const;
-
-  NONCOPYABLE(G1FreeIdSet);
-
-public:
-  G1FreeIdSet(uint start, uint size);
-  ~G1FreeIdSet();
-
-  // Returns an unclaimed parallel id (waiting for one to be released if
-  // necessary).  Must not safepoint while holding a claimed id.
-  uint claim_par_id();
-
-  void release_par_id(uint id);
-
-  struct TestSupport;           // For unit test access.
-};
-
-#endif // SHARE_GC_G1_G1FREEIDSET_HPP

src/hotspot/share/gc/parallel/mutableNUMASpace.cpp

@@ -37,21 +37,11 @@
 #include "runtime/threadSMR.hpp"
 #include "utilities/align.hpp"
 
-MutableNUMASpace::MutableNUMASpace(size_t alignment) : MutableSpace(alignment), _must_use_large_pages(false) {
+MutableNUMASpace::MutableNUMASpace(size_t page_size) : MutableSpace(page_size) {
   _lgrp_spaces = new (mtGC) GrowableArray<LGRPSpace*>(0, mtGC);
-  _page_size = os::vm_page_size();
   _adaptation_cycles = 0;
   _samples_count = 0;
 
-#ifdef LINUX
-  // Changing the page size can lead to freeing of memory. When using large pages
-  // and the memory has been both reserved and committed, Linux does not support
-  // freeing parts of it.
-    if (UseLargePages && !os::can_commit_large_page_memory()) {
-      _must_use_large_pages = true;
-    }
-#endif // LINUX
-
   size_t lgrp_limit = os::numa_get_groups_num();
   uint *lgrp_ids = NEW_C_HEAP_ARRAY(uint, lgrp_limit, mtGC);
   size_t lgrp_num = os::numa_get_leaf_groups(lgrp_ids, lgrp_limit);
@@ -60,7 +50,7 @@ MutableNUMASpace::MutableNUMASpace(size_t alignment) : MutableSpace(alignment),
   lgrp_spaces()->reserve(checked_cast<int>(lgrp_num));
   // Add new spaces for the new nodes
   for (size_t i = 0; i < lgrp_num; i++) {
-    lgrp_spaces()->append(new LGRPSpace(lgrp_ids[i], alignment));
+    lgrp_spaces()->append(new LGRPSpace(lgrp_ids[i], page_size));
   }
 
   FREE_C_HEAP_ARRAY(uint, lgrp_ids);
@@ -128,7 +118,10 @@ MutableNUMASpace::LGRPSpace *MutableNUMASpace::lgrp_space_for_thread(Thread* thr
     return space->lgrp_id() == (uint)lgrp_id;
   });
 
-  assert(lgrp_spaces_index != -1, "must have created spaces for all lgrp_ids");
+  if (lgrp_spaces_index == -1) {
+    // Running on a CPU with no memory; pick another CPU based on %.
+    lgrp_spaces_index = lgrp_id % lgrp_spaces()->length();
+  }
   return lgrp_spaces()->at(lgrp_spaces_index);
 }
 
@@ -146,22 +139,19 @@ size_t MutableNUMASpace::unsafe_max_tlab_alloc(Thread *thr) const {
 
 // Bias region towards the first-touching lgrp. Set the right page sizes.
 void MutableNUMASpace::bias_region(MemRegion mr, uint lgrp_id) {
-  HeapWord *start = align_up(mr.start(), page_size());
-  HeapWord *end = align_down(mr.end(), page_size());
-  if (end > start) {
-    MemRegion aligned_region(start, end);
-    assert((intptr_t)aligned_region.start()     % page_size() == 0 &&
-           (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment");
-    assert(region().contains(aligned_region), "Sanity");
-    // First we tell the OS which page size we want in the given range. The underlying
-    // large page can be broken down if we require small pages.
-    const size_t os_align = UseLargePages ? page_size() : os::vm_page_size();
-    os::realign_memory((char*)aligned_region.start(), aligned_region.byte_size(), os_align);
-    // Then we uncommit the pages in the range.
-    os::disclaim_memory((char*)aligned_region.start(), aligned_region.byte_size());
-    // And make them local/first-touch biased.
-    os::numa_make_local((char*)aligned_region.start(), aligned_region.byte_size(), checked_cast<int>(lgrp_id));
+  assert(is_aligned(mr.start(), page_size()), "precondition");
+  assert(is_aligned(mr.end(), page_size()), "precondition");
+
+  if (mr.is_empty()) {
+    return;
   }
+  // First we tell the OS which page size we want in the given range. The underlying
+  // large page can be broken down if we require small pages.
+  os::realign_memory((char*) mr.start(), mr.byte_size(), page_size());
+  // Then we uncommit the pages in the range.
+  os::disclaim_memory((char*) mr.start(), mr.byte_size());
+  // And make them local/first-touch biased.
+  os::numa_make_local((char*)mr.start(), mr.byte_size(), checked_cast<int>(lgrp_id));
 }
 
 // Update space layout. Perform adaptation.
@@ -210,14 +200,15 @@ size_t MutableNUMASpace::current_chunk_size(int i) {
 // Return the default chunk size by equally diving the space.
 // page_size() aligned.
 size_t MutableNUMASpace::default_chunk_size() {
-  return base_space_size() / lgrp_spaces()->length() * page_size();
+  // The number of pages may not be evenly divided.
+  return align_down(capacity_in_bytes() / lgrp_spaces()->length(), page_size());
 }
 
 // Produce a new chunk size. page_size() aligned.
 // This function is expected to be called on sequence of i's from 0 to
 // lgrp_spaces()->length().
 size_t MutableNUMASpace::adaptive_chunk_size(int i, size_t limit) {
-  size_t pages_available = base_space_size();
+  size_t pages_available = capacity_in_bytes() / page_size();
   for (int j = 0; j < i; j++) {
     pages_available -= align_down(current_chunk_size(j), page_size()) / page_size();
   }
@@ -263,20 +254,13 @@ size_t MutableNUMASpace::adaptive_chunk_size(int i, size_t limit) {
 // |----bottom_region--|---intersection---|------top_region------|
 void MutableNUMASpace::select_tails(MemRegion new_region, MemRegion intersection,
                                     MemRegion* bottom_region, MemRegion *top_region) {
+  assert(is_aligned(new_region.start(), page_size()), "precondition");
+  assert(is_aligned(new_region.end(), page_size()), "precondition");
+  assert(is_aligned(intersection.start(), page_size()), "precondition");
+  assert(is_aligned(intersection.end(), page_size()), "precondition");
+
   // Is there bottom?
   if (new_region.start() < intersection.start()) { // Yes
-    // Try to coalesce small pages into a large one.
-    if (UseLargePages && page_size() >= alignment()) {
-      HeapWord* p = align_up(intersection.start(), alignment());
-      if (new_region.contains(p)
-          && pointer_delta(p, new_region.start(), sizeof(char)) >= alignment()) {
-        if (intersection.contains(p)) {
-          intersection = MemRegion(p, intersection.end());
-        } else {
-          intersection = MemRegion(p, p);
-        }
-      }
-    }
     *bottom_region = MemRegion(new_region.start(), intersection.start());
   } else {
     *bottom_region = MemRegion();
@@ -284,18 +268,6 @@ void MutableNUMASpace::select_tails(MemRegion new_region, MemRegion intersection
 
   // Is there top?
   if (intersection.end() < new_region.end()) { // Yes
-    // Try to coalesce small pages into a large one.
-    if (UseLargePages && page_size() >= alignment()) {
-      HeapWord* p = align_down(intersection.end(), alignment());
-      if (new_region.contains(p)
-          && pointer_delta(new_region.end(), p, sizeof(char)) >= alignment()) {
-        if (intersection.contains(p)) {
-          intersection = MemRegion(intersection.start(), p);
-        } else {
-          intersection = MemRegion(p, p);
-        }
-      }
-    }
     *top_region = MemRegion(intersection.end(), new_region.end());
   } else {
     *top_region = MemRegion();
@@ -309,6 +281,8 @@ void MutableNUMASpace::initialize(MemRegion mr,
                                   WorkerThreads* pretouch_workers) {
   assert(clear_space, "Reallocation will destroy data!");
   assert(lgrp_spaces()->length() > 0, "There should be at least one space");
+  assert(is_aligned(mr.start(), page_size()), "precondition");
+  assert(is_aligned(mr.end(), page_size()), "precondition");
 
   MemRegion old_region = region(), new_region;
   set_bottom(mr.start());
@@ -316,37 +290,22 @@ void MutableNUMASpace::initialize(MemRegion mr,
   // Must always clear the space
   clear(SpaceDecorator::DontMangle);
 
-  // Compute chunk sizes
-  size_t prev_page_size = page_size();
-  set_page_size(alignment());
-  HeapWord* rounded_bottom = align_up(bottom(), page_size());
-  HeapWord* rounded_end = align_down(end(), page_size());
-  size_t base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size();
+  size_t num_pages = mr.byte_size() / page_size();
 
-  // Try small pages if the chunk size is too small
-  if (base_space_size_pages / lgrp_spaces()->length() == 0
-      && page_size() > os::vm_page_size()) {
-    // Changing the page size below can lead to freeing of memory. So we fail initialization.
-    if (_must_use_large_pages) {
-      vm_exit_during_initialization("Failed initializing NUMA with large pages. Too small heap size");
-    }
-    set_page_size(os::vm_page_size());
-    rounded_bottom = align_up(bottom(), page_size());
-    rounded_end = align_down(end(), page_size());
-    base_space_size_pages = pointer_delta(rounded_end, rounded_bottom, sizeof(char)) / page_size();
+  if (num_pages < (size_t)lgrp_spaces()->length()) {
+    log_warning(gc)("Degraded NUMA config: #os-pages (%zu) < #CPU (%d); space-size: %zu, page-size: %zu",
+      num_pages, lgrp_spaces()->length(), mr.byte_size(), page_size());
+
+    // Keep only the first few CPUs.
+    lgrp_spaces()->trunc_to((int)num_pages);
   }
-  guarantee(base_space_size_pages / lgrp_spaces()->length() > 0, "Space too small");
-  set_base_space_size(base_space_size_pages);
 
   // Handle space resize
   MemRegion top_region, bottom_region;
   if (!old_region.equals(region())) {
-    new_region = MemRegion(rounded_bottom, rounded_end);
+    new_region = mr;
     MemRegion intersection = new_region.intersection(old_region);
-    if (intersection.start() == nullptr ||
-        intersection.end() == nullptr   ||
-        prev_page_size > page_size()) { // If the page size got smaller we have to change
-                                        // the page size preference for the whole space.
+    if (intersection.is_empty()) {
       intersection = MemRegion(new_region.start(), new_region.start());
     }
     select_tails(new_region, intersection, &bottom_region, &top_region);
@@ -393,19 +352,18 @@ void MutableNUMASpace::initialize(MemRegion mr,
 
     if (i == 0) { // Bottom chunk
       if (i != lgrp_spaces()->length() - 1) {
-        new_region = MemRegion(bottom(), rounded_bottom + (chunk_byte_size >> LogHeapWordSize));
+        new_region = MemRegion(bottom(), chunk_byte_size >> LogHeapWordSize);
       } else {
         new_region = MemRegion(bottom(), end());
       }
-    } else
-      if (i < lgrp_spaces()->length() - 1) { // Middle chunks
-        MutableSpace *ps = lgrp_spaces()->at(i - 1)->space();
-        new_region = MemRegion(ps->end(),
-                               ps->end() + (chunk_byte_size >> LogHeapWordSize));
-      } else { // Top chunk
-        MutableSpace *ps = lgrp_spaces()->at(i - 1)->space();
-        new_region = MemRegion(ps->end(), end());
-      }
+    } else if (i < lgrp_spaces()->length() - 1) { // Middle chunks
+      MutableSpace* ps = lgrp_spaces()->at(i - 1)->space();
+      new_region = MemRegion(ps->end(),
+                             chunk_byte_size >> LogHeapWordSize);
+    } else { // Top chunk
+      MutableSpace* ps = lgrp_spaces()->at(i - 1)->space();
+      new_region = MemRegion(ps->end(), end());
+    }
     guarantee(region().contains(new_region), "Region invariant");
 
 
@@ -432,9 +390,8 @@ void MutableNUMASpace::initialize(MemRegion mr,
 
     // Clear space (set top = bottom) but never mangle.
     s->initialize(new_region, SpaceDecorator::Clear, SpaceDecorator::DontMangle, MutableSpace::DontSetupPages);
-
-    set_adaptation_cycles(samples_count());
   }
+  set_adaptation_cycles(samples_count());
 }
 
 // Set the top of the whole space.

src/hotspot/share/gc/parallel/mutableNUMASpace.hpp

@@ -80,8 +80,8 @@ class MutableNUMASpace : public MutableSpace {
     SpaceStats _space_stats;
 
    public:
-    LGRPSpace(uint l, size_t alignment) : _lgrp_id(l), _allocation_failed(false) {
-      _space = new MutableSpace(alignment);
+    LGRPSpace(uint l, size_t page_size) : _lgrp_id(l), _allocation_failed(false) {
+      _space = new MutableSpace(page_size);
       _alloc_rate = new AdaptiveWeightedAverage(NUMAChunkResizeWeight);
     }
     ~LGRPSpace() {
@@ -117,24 +117,14 @@ class MutableNUMASpace : public MutableSpace {
   };
 
   GrowableArray<LGRPSpace*>* _lgrp_spaces;
-  size_t _page_size;
   unsigned _adaptation_cycles, _samples_count;
 
-  bool _must_use_large_pages;
-
-  void set_page_size(size_t psz)                     { _page_size = psz;          }
-  size_t page_size() const                           { return _page_size;         }
-
   unsigned adaptation_cycles()                       { return _adaptation_cycles; }
   void set_adaptation_cycles(int v)                  { _adaptation_cycles = v;    }
 
   unsigned samples_count()                           { return _samples_count;     }
   void increment_samples_count()                     { ++_samples_count;          }
 
-  size_t _base_space_size;
-  void set_base_space_size(size_t v)                 { _base_space_size = v;      }
-  size_t base_space_size() const                     { return _base_space_size;   }
-
   // Bias region towards the lgrp.
   void bias_region(MemRegion mr, uint lgrp_id);
 
@@ -154,7 +144,7 @@ class MutableNUMASpace : public MutableSpace {
 
 public:
   GrowableArray<LGRPSpace*>* lgrp_spaces() const     { return _lgrp_spaces;       }
-  MutableNUMASpace(size_t alignment);
+  MutableNUMASpace(size_t page_size);
   virtual ~MutableNUMASpace();
   // Space initialization.
   virtual void initialize(MemRegion mr,

src/hotspot/share/gc/parallel/mutableSpace.cpp

@@ -34,30 +34,26 @@
 #include "utilities/align.hpp"
 #include "utilities/macros.hpp"
 
-MutableSpace::MutableSpace(size_t alignment) :
+MutableSpace::MutableSpace(size_t page_size) :
   _last_setup_region(),
-  _alignment(alignment),
+  _page_size(page_size),
   _bottom(nullptr),
   _top(nullptr),
-  _end(nullptr)
-{
-  assert(MutableSpace::alignment() % os::vm_page_size() == 0,
-         "Space should be aligned");
-}
+  _end(nullptr) {}
 
-void MutableSpace::numa_setup_pages(MemRegion mr, size_t page_size, bool clear_space) {
-  if (!mr.is_empty()) {
-    HeapWord *start = align_up(mr.start(), page_size);
-    HeapWord *end =   align_down(mr.end(), page_size);
-    if (end > start) {
-      size_t size = pointer_delta(end, start, sizeof(char));
-      if (clear_space) {
-        // Prefer page reallocation to migration.
-        os::disclaim_memory((char*)start, size);
-      }
-      os::numa_make_global((char*)start, size);
-    }
+void MutableSpace::numa_setup_pages(MemRegion mr, bool clear_space) {
+  assert(is_aligned(mr.start(), page_size()), "precondition");
+  assert(is_aligned(mr.end(), page_size()), "precondition");
+
+  if (mr.is_empty()) {
+    return;
   }
+
+  if (clear_space) {
+    // Prefer page reallocation to migration.
+    os::disclaim_memory((char*) mr.start(), mr.byte_size());
+  }
+  os::numa_make_global((char*) mr.start(), mr.byte_size());
 }
 
 void MutableSpace::initialize(MemRegion mr,
@@ -105,20 +101,17 @@ void MutableSpace::initialize(MemRegion mr,
     }
     assert(mr.contains(head) && mr.contains(tail), "Sanity");
 
-    size_t page_size = alignment();
-
     if (UseNUMA) {
-      numa_setup_pages(head, page_size, clear_space);
-      numa_setup_pages(tail, page_size, clear_space);
+      numa_setup_pages(head, clear_space);
+      numa_setup_pages(tail, clear_space);
     }
 
     if (AlwaysPreTouch) {
-      size_t pretouch_page_size = UseLargePages ? page_size : os::vm_page_size();
       PretouchTask::pretouch("ParallelGC PreTouch head", (char*)head.start(), (char*)head.end(),
-                             pretouch_page_size, pretouch_workers);
+                             page_size(), pretouch_workers);
 
       PretouchTask::pretouch("ParallelGC PreTouch tail", (char*)tail.start(), (char*)tail.end(),
-                             pretouch_page_size, pretouch_workers);
+                             page_size(), pretouch_workers);
     }
 
     // Remember where we stopped so that we can continue later.

src/hotspot/share/gc/parallel/mutableSpace.hpp

@@ -51,17 +51,20 @@ class MutableSpace: public CHeapObj<mtGC> {
 
   // The last region which page had been setup to be interleaved.
   MemRegion _last_setup_region;
-  size_t _alignment;
+  size_t _page_size;
   HeapWord* _bottom;
   HeapWord* volatile _top;
   HeapWord* _end;
 
-  void numa_setup_pages(MemRegion mr, size_t page_size, bool clear_space);
+  void numa_setup_pages(MemRegion mr, bool clear_space);
 
   void set_last_setup_region(MemRegion mr) { _last_setup_region = mr;   }
   MemRegion last_setup_region() const      { return _last_setup_region; }
 
- public:
+protected:
+  size_t page_size() const                 { return _page_size;         }
+
+public:
   virtual ~MutableSpace() = default;
   MutableSpace(size_t page_size);
 
@@ -77,8 +80,6 @@ class MutableSpace: public CHeapObj<mtGC> {
   HeapWord* volatile* top_addr()           { return &_top; }
   HeapWord** end_addr()                    { return &_end; }
 
-  size_t alignment()                       { return _alignment; }
-
   MemRegion region() const { return MemRegion(bottom(), end()); }
 
   size_t capacity_in_bytes() const { return capacity_in_words() * HeapWordSize; }

src/hotspot/share/gc/parallel/objectStartArray.cpp

@@ -47,7 +47,10 @@ ObjectStartArray::ObjectStartArray(MemRegion covered_region)
 
   // Do not use large-pages for the backing store. The one large page region
   // will be used for the heap proper.
-  ReservedSpace backing_store = MemoryReserver::reserve(bytes_to_reserve, mtGC);
+  ReservedSpace backing_store = MemoryReserver::reserve(bytes_to_reserve,
+                                                        os::vm_allocation_granularity(),
+                                                        os::vm_page_size(),
+                                                        mtGC);
   if (!backing_store.is_reserved()) {
     vm_exit_during_initialization("Could not reserve space for ObjectStartArray");
   }

src/hotspot/share/gc/parallel/parallelArguments.cpp

@@ -103,15 +103,10 @@ void ParallelArguments::initialize() {
   FullGCForwarding::initialize_flags(heap_reserved_size_bytes());
 }
 
-// The alignment used for spaces in young gen and old gen
-static size_t default_space_alignment() {
-  return 64 * K * HeapWordSize;
-}
-
 void ParallelArguments::initialize_alignments() {
   // Initialize card size before initializing alignments
   CardTable::initialize_card_size();
-  SpaceAlignment = default_space_alignment();
+  SpaceAlignment = ParallelScavengeHeap::default_space_alignment();
   HeapAlignment = compute_heap_alignment();
 }
 
@@ -123,12 +118,23 @@ void ParallelArguments::initialize_heap_flags_and_sizes_one_pass() {
 void ParallelArguments::initialize_heap_flags_and_sizes() {
   initialize_heap_flags_and_sizes_one_pass();
 
+  if (!UseLargePages) {
+    ParallelScavengeHeap::set_desired_page_size(os::vm_page_size());
+    return;
+  }
+
+  // If using large-page, need to update SpaceAlignment so that spaces are page-size aligned.
   const size_t min_pages = 4; // 1 for eden + 1 for each survivor + 1 for old
   const size_t page_sz = os::page_size_for_region_aligned(MinHeapSize, min_pages);
+  ParallelScavengeHeap::set_desired_page_size(page_sz);
 
-  // Can a page size be something else than a power of two?
-  assert(is_power_of_2((intptr_t)page_sz), "must be a power of 2");
-  size_t new_alignment = align_up(page_sz, SpaceAlignment);
+  if (page_sz == os::vm_page_size()) {
+    log_warning(gc, heap)("MinHeapSize (%zu) must be large enough for 4 * page-size; Disabling UseLargePages for heap", MinHeapSize);
+    return;
+  }
+
+  // Space is largepage-aligned.
+  size_t new_alignment = page_sz;
   if (new_alignment != SpaceAlignment) {
     SpaceAlignment = new_alignment;
     // Redo everything from the start

src/hotspot/share/gc/parallel/parallelScavengeHeap.cpp

@@ -61,11 +61,18 @@ PSYoungGen*  ParallelScavengeHeap::_young_gen = nullptr;
 PSOldGen*    ParallelScavengeHeap::_old_gen = nullptr;
 PSAdaptiveSizePolicy* ParallelScavengeHeap::_size_policy = nullptr;
 GCPolicyCounters* ParallelScavengeHeap::_gc_policy_counters = nullptr;
+size_t ParallelScavengeHeap::_desired_page_size = 0;
 
 jint ParallelScavengeHeap::initialize() {
   const size_t reserved_heap_size = ParallelArguments::heap_reserved_size_bytes();
 
-  ReservedHeapSpace heap_rs = Universe::reserve_heap(reserved_heap_size, HeapAlignment);
+  assert(_desired_page_size != 0, "Should be initialized");
+  ReservedHeapSpace heap_rs = Universe::reserve_heap(reserved_heap_size, HeapAlignment, _desired_page_size);
+  // Adjust SpaceAlignment based on actually used large page size.
+  if (UseLargePages) {
+    SpaceAlignment = MAX2(heap_rs.page_size(), default_space_alignment());
+  }
+  assert(is_aligned(SpaceAlignment, heap_rs.page_size()), "inv");
 
   trace_actual_reserved_page_size(reserved_heap_size, heap_rs);
 
@@ -79,7 +86,6 @@ jint ParallelScavengeHeap::initialize() {
   card_table->initialize(old_rs.base(), young_rs.base());
 
   CardTableBarrierSet* const barrier_set = new CardTableBarrierSet(card_table);
-  barrier_set->initialize();
   BarrierSet::set_barrier_set(barrier_set);
 
   // Set up WorkerThreads
@@ -702,17 +708,14 @@ void ParallelScavengeHeap::print_heap_change(const PreGenGCValues& pre_gc_values
 }
 
 void ParallelScavengeHeap::verify(VerifyOption option /* ignored */) {
-  // Why do we need the total_collections()-filter below?
-  if (total_collections() > 0) {
-    log_debug(gc, verify)("Tenured");
-    old_gen()->verify();
+  log_debug(gc, verify)("Tenured");
+  old_gen()->verify();
 
-    log_debug(gc, verify)("Eden");
-    young_gen()->verify();
+  log_debug(gc, verify)("Eden");
+  young_gen()->verify();
 
-    log_debug(gc, verify)("CardTable");
-    card_table()->verify_all_young_refs_imprecise();
-  }
+  log_debug(gc, verify)("CardTable");
+  card_table()->verify_all_young_refs_imprecise();
 }
 
 void ParallelScavengeHeap::trace_actual_reserved_page_size(const size_t reserved_heap_size, const ReservedSpace rs) {

src/hotspot/share/gc/parallel/parallelScavengeHeap.hpp

@@ -76,6 +76,9 @@ class ParallelScavengeHeap : public CollectedHeap {
   static PSAdaptiveSizePolicy*       _size_policy;
   static GCPolicyCounters*           _gc_policy_counters;
 
+  // At startup, calculate the desired OS page-size based on heap size and large-page flags.
+  static size_t _desired_page_size;
+
   GCMemoryManager* _young_manager;
   GCMemoryManager* _old_manager;
 
@@ -128,6 +131,18 @@ public:
     _gc_overhead_counter(0),
     _is_heap_almost_full(false) {}
 
+  // The alignment used for spaces in young gen and old gen
+  constexpr static size_t default_space_alignment() {
+    constexpr size_t alignment = 64 * K * HeapWordSize;
+    static_assert(is_power_of_2(alignment), "inv");
+    return alignment;
+  }
+
+  static void set_desired_page_size(size_t page_size) {
+    assert(is_power_of_2(page_size), "precondition");
+    _desired_page_size = page_size;
+  }
+
   Name kind() const override {
     return CollectedHeap::Parallel;
   }

src/hotspot/share/gc/parallel/psOldGen.cpp

@@ -96,7 +96,7 @@ void PSOldGen::initialize_work() {
   // ObjectSpace stuff
   //
 
-  _object_space = new MutableSpace(virtual_space()->alignment());
+  _object_space = new MutableSpace(virtual_space()->page_size());
   object_space()->initialize(committed_mr,
                              SpaceDecorator::Clear,
                              SpaceDecorator::Mangle,

src/hotspot/share/gc/parallel/psVirtualspace.cpp

@@ -29,8 +29,8 @@
 #include "utilities/align.hpp"
 
 PSVirtualSpace::PSVirtualSpace(ReservedSpace rs, size_t alignment) :
-  _alignment(alignment)
-{
+  _alignment(alignment),
+  _page_size(rs.page_size()) {
   set_reserved(rs);
   set_committed(reserved_low_addr(), reserved_low_addr());
   DEBUG_ONLY(verify());
@@ -88,7 +88,8 @@ bool PSVirtualSpace::shrink_by(size_t bytes) {
 
 #ifndef PRODUCT
 void PSVirtualSpace::verify() const {
-  assert(is_aligned(_alignment, os::vm_page_size()), "bad alignment");
+  assert(is_aligned(_page_size, os::vm_page_size()), "bad alignment");
+  assert(is_aligned(_alignment, _page_size), "inv");
   assert(is_aligned(reserved_low_addr(), _alignment), "bad reserved_low_addr");
   assert(is_aligned(reserved_high_addr(), _alignment), "bad reserved_high_addr");
   assert(is_aligned(committed_low_addr(), _alignment), "bad committed_low_addr");

src/hotspot/share/gc/parallel/psVirtualspace.hpp

@@ -41,6 +41,9 @@ class PSVirtualSpace : public CHeapObj<mtGC> {
   // ReservedSpace passed to initialize() must be aligned to this value.
   const size_t _alignment;
 
+  // OS page size used. If using Transparent Huge Pages, it's the desired large page-size.
+  const size_t _page_size;
+
   // Reserved area
   char* _reserved_low_addr;
   char* _reserved_high_addr;
@@ -68,6 +71,7 @@ class PSVirtualSpace : public CHeapObj<mtGC> {
 
   // Accessors (all sizes are bytes).
   size_t alignment()          const { return _alignment; }
+  size_t page_size()          const { return _page_size; }
   char* reserved_low_addr()   const { return _reserved_low_addr; }
   char* reserved_high_addr()  const { return _reserved_high_addr; }
   char* committed_low_addr()  const { return _committed_low_addr; }

src/hotspot/share/gc/parallel/psYoungGen.cpp

@@ -83,12 +83,12 @@ void PSYoungGen::initialize_work() {
   }
 
   if (UseNUMA) {
-    _eden_space = new MutableNUMASpace(virtual_space()->alignment());
+    _eden_space = new MutableNUMASpace(virtual_space()->page_size());
   } else {
-    _eden_space = new MutableSpace(virtual_space()->alignment());
+    _eden_space = new MutableSpace(virtual_space()->page_size());
   }
-  _from_space = new MutableSpace(virtual_space()->alignment());
-  _to_space   = new MutableSpace(virtual_space()->alignment());
+  _from_space = new MutableSpace(virtual_space()->page_size());
+  _to_space   = new MutableSpace(virtual_space()->page_size());
 
   // Generation Counters - generation 0, 3 subspaces
   _gen_counters = new GenerationCounters("new", 0, 3, min_gen_size(),

src/hotspot/share/gc/parallel/vmStructs_parallelgc.hpp

@@ -40,6 +40,7 @@
   /* Parallel GC fields */                                                                                                           \
   /**********************/                                                                                                           \
   nonstatic_field(PSVirtualSpace,              _alignment,                                    const size_t)                          \
+  nonstatic_field(PSVirtualSpace,              _page_size,                                    const size_t)                          \
   nonstatic_field(PSVirtualSpace,              _reserved_low_addr,                            char*)                                 \
   nonstatic_field(PSVirtualSpace,              _reserved_high_addr,                           char*)                                 \
   nonstatic_field(PSVirtualSpace,              _committed_low_addr,                           char*)                                 \

src/hotspot/share/gc/serial/defNewGeneration.cpp

@@ -225,16 +225,12 @@ DefNewGeneration::DefNewGeneration(ReservedSpace rs,
     _promo_failure_drain_in_progress(false),
     _string_dedup_requests()
 {
-  MemRegion cmr((HeapWord*)_virtual_space.low(),
-                (HeapWord*)_virtual_space.high());
-  SerialHeap* gch = SerialHeap::heap();
-
-  gch->rem_set()->resize_covered_region(cmr);
-
   _eden_space = new ContiguousSpace();
   _from_space = new ContiguousSpace();
   _to_space   = new ContiguousSpace();
 
+  init_spaces();
+
   // Compute the maximum eden and survivor space sizes. These sizes
   // are computed assuming the entire reserved space is committed.
   // These values are exported as performance counters.
@@ -256,7 +252,6 @@ DefNewGeneration::DefNewGeneration(ReservedSpace rs,
   _to_counters = new CSpaceCounters("s1", 2, _max_survivor_size, _to_space,
                                     _gen_counters);
 
-  compute_space_boundaries(0, SpaceDecorator::Clear, SpaceDecorator::Mangle);
   update_counters();
   _old_gen = nullptr;
   _tenuring_threshold = MaxTenuringThreshold;
@@ -268,74 +263,51 @@ DefNewGeneration::DefNewGeneration(ReservedSpace rs,
   _gc_tracer = new DefNewTracer();
 }
 
-void DefNewGeneration::compute_space_boundaries(uintx minimum_eden_size,
-                                                bool clear_space,
-                                                bool mangle_space) {
-  // If the spaces are being cleared (only done at heap initialization
-  // currently), the survivor spaces need not be empty.
-  // Otherwise, no care is taken for used areas in the survivor spaces
-  // so check.
-  assert(clear_space || (to()->is_empty() && from()->is_empty()),
-    "Initialization of the survivor spaces assumes these are empty");
+void DefNewGeneration::init_spaces() {
+  // Using layout: from, to, eden, so only from can be non-empty.
+  assert(eden()->is_empty(), "precondition");
+  assert(to()->is_empty(), "precondition");
+
+  if (!from()->is_empty()) {
+    assert((char*) from()->bottom() == _virtual_space.low(), "inv");
+  }
 
   // Compute sizes
-  uintx size = _virtual_space.committed_size();
-  uintx survivor_size = compute_survivor_size(size, SpaceAlignment);
-  uintx eden_size = size - (2*survivor_size);
-  if (eden_size > max_eden_size()) {
-    // Need to reduce eden_size to satisfy the max constraint. The delta needs
-    // to be 2*SpaceAlignment aligned so that both survivors are properly
-    // aligned.
-    uintx eden_delta = align_up(eden_size - max_eden_size(), 2*SpaceAlignment);
-    eden_size     -= eden_delta;
-    survivor_size += eden_delta/2;
-  }
+  size_t size = _virtual_space.committed_size();
+  size_t survivor_size = compute_survivor_size(size, SpaceAlignment);
+  assert(survivor_size >= from()->used(), "inv");
+  assert(size > 2 * survivor_size, "inv");
+  size_t eden_size = size - (2 * survivor_size);
   assert(eden_size > 0 && survivor_size <= eden_size, "just checking");
 
-  if (eden_size < minimum_eden_size) {
-    // May happen due to 64Kb rounding, if so adjust eden size back up
-    minimum_eden_size = align_up(minimum_eden_size, SpaceAlignment);
-    uintx maximum_survivor_size = (size - minimum_eden_size) / 2;
-    uintx unaligned_survivor_size =
-      align_down(maximum_survivor_size, SpaceAlignment);
-    survivor_size = MAX2(unaligned_survivor_size, SpaceAlignment);
-    eden_size = size - (2*survivor_size);
-    assert(eden_size > 0 && survivor_size <= eden_size, "just checking");
-    assert(eden_size >= minimum_eden_size, "just checking");
-  }
+  // layout: from, to, eden
+  char* from_start = _virtual_space.low();
+  char* to_start = from_start + survivor_size;
+  char* eden_start = to_start + survivor_size;
+  char* eden_end = eden_start + eden_size;
 
-  char *eden_start = _virtual_space.low();
-  char *from_start = eden_start + eden_size;
-  char *to_start   = from_start + survivor_size;
-  char *to_end     = to_start   + survivor_size;
-
-  assert(to_end == _virtual_space.high(), "just checking");
-  assert(is_aligned(eden_start, SpaceAlignment), "checking alignment");
+  assert(eden_end == _virtual_space.high(), "just checking");
   assert(is_aligned(from_start, SpaceAlignment), "checking alignment");
   assert(is_aligned(to_start, SpaceAlignment),   "checking alignment");
+  assert(is_aligned(eden_start, SpaceAlignment), "checking alignment");
+  assert(is_aligned(eden_end, SpaceAlignment), "checking alignment");
 
-  MemRegion edenMR((HeapWord*)eden_start, (HeapWord*)from_start);
   MemRegion fromMR((HeapWord*)from_start, (HeapWord*)to_start);
-  MemRegion toMR  ((HeapWord*)to_start, (HeapWord*)to_end);
-
-  // A minimum eden size implies that there is a part of eden that
-  // is being used and that affects the initialization of any
-  // newly formed eden.
-  bool live_in_eden = minimum_eden_size > 0;
+  MemRegion toMR  ((HeapWord*)to_start, (HeapWord*)eden_start);
+  MemRegion edenMR((HeapWord*)eden_start, (HeapWord*)eden_end);
 
   // Reset the spaces for their new regions.
-  eden()->initialize(edenMR,
-                     clear_space && !live_in_eden,
-                     SpaceDecorator::Mangle);
-  // If clear_space and live_in_eden, we will not have cleared any
-  // portion of eden above its top. This can cause newly
-  // expanded space not to be mangled if using ZapUnusedHeapArea.
-  // We explicitly do such mangling here.
-  if (ZapUnusedHeapArea && clear_space && live_in_eden && mangle_space) {
-    eden()->mangle_unused_area();
-  }
-  from()->initialize(fromMR, clear_space, mangle_space);
-  to()->initialize(toMR, clear_space, mangle_space);
+  from()->initialize(fromMR, from()->is_empty());
+  to()->initialize(toMR, true);
+  eden()->initialize(edenMR, true);
+
+  post_resize();
+}
+
+void DefNewGeneration::post_resize() {
+  MemRegion cmr((HeapWord*)_virtual_space.low(),
+                (HeapWord*)_virtual_space.high());
+  SerialHeap::heap()->rem_set()->resize_covered_region(cmr);
 }
 
 void DefNewGeneration::swap_spaces() {
@@ -351,20 +323,28 @@ void DefNewGeneration::swap_spaces() {
 }
 
 bool DefNewGeneration::expand(size_t bytes) {
-  HeapWord* prev_high = (HeapWord*) _virtual_space.high();
+  assert(bytes != 0, "precondition");
+  assert(is_aligned(bytes, SpaceAlignment), "precondition");
+
   bool success = _virtual_space.expand_by(bytes);
-  if (success && ZapUnusedHeapArea) {
-    // Mangle newly committed space immediately because it
-    // can be done here more simply that after the new
-    // spaces have been computed.
-    HeapWord* new_high = (HeapWord*) _virtual_space.high();
-    MemRegion mangle_region(prev_high, new_high);
-    SpaceMangler::mangle_region(mangle_region);
+  if (!success) {
+    log_info(gc)("Failed to expand young-gen by %zu bytes", bytes);
   }
 
   return success;
 }
 
+void DefNewGeneration::expand_eden_by(size_t delta_bytes) {
+  if (!expand(delta_bytes)) {
+    return;
+  }
+
+  MemRegion eden_mr{eden()->bottom(), (HeapWord*)_virtual_space.high()};
+  eden()->initialize(eden_mr, eden()->is_empty());
+
+  post_resize();
+}
+
 size_t DefNewGeneration::calculate_thread_increase_size(int threads_count) const {
     size_t thread_increase_size = 0;
     // Check an overflow at 'threads_count * NewSizeThreadIncrease'.
@@ -397,18 +377,8 @@ size_t DefNewGeneration::adjust_for_thread_increase(size_t new_size_candidate,
   return desired_new_size;
 }
 
-void DefNewGeneration::compute_new_size() {
-  // This is called after a GC that includes the old generation, so from-space
-  // will normally be empty.
-  // Note that we check both spaces, since if scavenge failed they revert roles.
-  // If not we bail out (otherwise we would have to relocate the objects).
-  if (!from()->is_empty() || !to()->is_empty()) {
-    return;
-  }
-
-  SerialHeap* gch = SerialHeap::heap();
-
-  size_t old_size = gch->old_gen()->capacity();
+size_t DefNewGeneration::calculate_desired_young_gen_bytes() const {
+  size_t old_size = SerialHeap::heap()->old_gen()->capacity();
   size_t new_size_before = _virtual_space.committed_size();
   size_t min_new_size = NewSize;
   size_t max_new_size = reserved().byte_size();
@@ -429,46 +399,82 @@ void DefNewGeneration::compute_new_size() {
 
   // Adjust new generation size
   desired_new_size = clamp(desired_new_size, min_new_size, max_new_size);
-  assert(desired_new_size <= max_new_size, "just checking");
+  if (!from()->is_empty()) {
+    // Mininum constraint to hold all live objs inside from-space.
+    size_t min_survivor_size = align_up(from()->used(), alignment);
 
-  bool changed = false;
-  if (desired_new_size > new_size_before) {
-    size_t change = desired_new_size - new_size_before;
-    assert(change % alignment == 0, "just checking");
-    if (expand(change)) {
-       changed = true;
+    // SurvivorRatio := eden_size / survivor_size
+    // young-gen-size = eden_size                     + 2 * survivor_size
+    //                = SurvivorRatio * survivor_size + 2 * survivor_size
+    //                = (SurvivorRatio + 2) * survivor_size
+    size_t min_young_gen_size = min_survivor_size * (SurvivorRatio + 2);
+
+    desired_new_size = MAX2(min_young_gen_size, desired_new_size);
+  }
+  assert(is_aligned(desired_new_size, alignment), "postcondition");
+
+  return desired_new_size;
+}
+
+void DefNewGeneration::resize_inner() {
+  assert(eden()->is_empty(), "precondition");
+  assert(to()->is_empty(), "precondition");
+
+  size_t current_young_gen_size_bytes = _virtual_space.committed_size();
+  size_t desired_young_gen_size_bytes = calculate_desired_young_gen_bytes();
+  if (current_young_gen_size_bytes == desired_young_gen_size_bytes) {
+    return;
+  }
+
+  // Commit/uncommit
+  if (desired_young_gen_size_bytes > current_young_gen_size_bytes) {
+    size_t delta_bytes = desired_young_gen_size_bytes - current_young_gen_size_bytes;
+    if (!expand(delta_bytes)) {
+      return;
     }
-    // If the heap failed to expand to the desired size,
-    // "changed" will be false.  If the expansion failed
-    // (and at this point it was expected to succeed),
-    // ignore the failure (leaving "changed" as false).
+  } else {
+    size_t delta_bytes = current_young_gen_size_bytes - desired_young_gen_size_bytes;
+    _virtual_space.shrink_by(delta_bytes);
   }
-  if (desired_new_size < new_size_before && eden()->is_empty()) {
-    // bail out of shrinking if objects in eden
-    size_t change = new_size_before - desired_new_size;
-    assert(change % alignment == 0, "just checking");
-    _virtual_space.shrink_by(change);
-    changed = true;
-  }
-  if (changed) {
-    // The spaces have already been mangled at this point but
-    // may not have been cleared (set top = bottom) and should be.
-    // Mangling was done when the heap was being expanded.
-    compute_space_boundaries(eden()->used(),
-                             SpaceDecorator::Clear,
-                             SpaceDecorator::DontMangle);
-    MemRegion cmr((HeapWord*)_virtual_space.low(),
-                  (HeapWord*)_virtual_space.high());
-    gch->rem_set()->resize_covered_region(cmr);
 
-    log_debug(gc, ergo, heap)(
-        "New generation size %zuK->%zuK [eden=%zuK,survivor=%zuK]",
-        new_size_before/K, _virtual_space.committed_size()/K,
-        eden()->capacity()/K, from()->capacity()/K);
-    log_trace(gc, ergo, heap)(
-        "  [allowed %zuK extra for %d threads]",
-          thread_increase_size/K, threads_count);
-      }
+  assert(desired_young_gen_size_bytes == _virtual_space.committed_size(), "inv");
+
+  init_spaces();
+
+  log_debug(gc, ergo, heap)("New generation size %zuK->%zuK [eden=%zuK,survivor=%zuK]",
+    current_young_gen_size_bytes/K, _virtual_space.committed_size()/K,
+    eden()->capacity()/K, from()->capacity()/K);
+}
+
+void DefNewGeneration::resize_after_young_gc() {
+  // Called only after successful young-gc.
+  assert(eden()->is_empty(), "precondition");
+  assert(to()->is_empty(), "precondition");
+
+  if ((char*)to()->bottom() == _virtual_space.low()) {
+    // layout: to, from, eden; can't resize.
+    return;
+  }
+
+  assert((char*)from()->bottom() == _virtual_space.low(), "inv");
+  resize_inner();
+}
+
+void DefNewGeneration::resize_after_full_gc() {
+  if (eden()->is_empty() && from()->is_empty() && to()->is_empty()) {
+    resize_inner();
+    return;
+  }
+
+  // Usually the young-gen is empty after full-gc.
+  // This is the extreme case; expand young-gen to its max size.
+  if (_virtual_space.uncommitted_size() == 0) {
+    // Already at its max size.
+    return;
+  }
+
+  // Keep from/to and expand eden.
+  expand_eden_by(_virtual_space.uncommitted_size());
 }
 
 void DefNewGeneration::ref_processor_init() {
@@ -483,13 +489,11 @@ size_t DefNewGeneration::capacity() const {
        + from()->capacity();  // to() is only used during scavenge
 }
 
-
 size_t DefNewGeneration::used() const {
   return eden()->used()
        + from()->used();      // to() is only used during scavenge
 }
 
-
 size_t DefNewGeneration::free() const {
   return eden()->free()
        + from()->free();      // to() is only used during scavenge
@@ -497,7 +501,8 @@ size_t DefNewGeneration::free() const {
 
 size_t DefNewGeneration::max_capacity() const {
   const size_t reserved_bytes = reserved().byte_size();
-  return reserved_bytes - compute_survivor_size(reserved_bytes, SpaceAlignment);
+  const size_t min_survivor_bytes = SpaceAlignment;
+  return reserved_bytes - min_survivor_bytes;
 }
 
 bool DefNewGeneration::is_in(const void* p) const {
@@ -589,7 +594,6 @@ bool DefNewGeneration::collect(bool clear_all_soft_refs) {
   IsAliveClosure is_alive(this);
 
   age_table()->clear();
-  to()->clear(SpaceDecorator::Mangle);
 
   YoungGenScanClosure young_gen_cl(this);
   OldGenScanClosure   old_gen_cl(this);
@@ -839,13 +843,18 @@ void DefNewGeneration::print_on(outputStream* st) const {
   to()->print_on(st, "to   ");
 }
 
-HeapWord* DefNewGeneration::allocate(size_t word_size) {
-  // This is the slow-path allocation for the DefNewGeneration.
-  // Most allocations are fast-path in compiled code.
-  // We try to allocate from the eden.  If that works, we are happy.
-  // Note that since DefNewGeneration supports lock-free allocation, we
-  // have to use it here, as well.
-  HeapWord* result = eden()->par_allocate(word_size);
+HeapWord* DefNewGeneration::expand_and_allocate(size_t word_size) {
+  assert(SafepointSynchronize::is_at_safepoint(), "precondition");
+  assert(Thread::current()->is_VM_thread(), "precondition");
+
+  size_t eden_free_bytes = eden()->free();
+  size_t requested_bytes = word_size * HeapWordSize;
+  if (eden_free_bytes < requested_bytes) {
+    size_t expand_bytes = requested_bytes - eden_free_bytes;
+    expand_eden_by(align_up(expand_bytes, SpaceAlignment));
+  }
+
+  HeapWord* result = eden()->allocate(word_size);
   return result;
 }
 

src/hotspot/share/gc/serial/defNewGeneration.hpp

@@ -131,6 +131,13 @@ class DefNewGeneration: public Generation {
     return n > alignment ? align_down(n, alignment) : alignment;
   }
 
+  size_t calculate_desired_young_gen_bytes() const;
+
+  void expand_eden_by(size_t delta_bytes);
+
+  void resize_inner();
+  void post_resize();
+
  public:
   DefNewGeneration(ReservedSpace rs,
                    size_t initial_byte_size,
@@ -183,9 +190,8 @@ class DefNewGeneration: public Generation {
 
   HeapWord* block_start(const void* p) const;
 
-  // Allocate requested size or return null; single-threaded and lock-free versions.
-  HeapWord* allocate(size_t word_size);
   HeapWord* par_allocate(size_t word_size);
+  HeapWord* expand_and_allocate(size_t word_size);
 
   void gc_epilogue();
 
@@ -196,8 +202,8 @@ class DefNewGeneration: public Generation {
   // Reset for contribution of "to-space".
   void reset_scratch();
 
-  // GC support
-  void compute_new_size();
+  void resize_after_young_gc();
+  void resize_after_full_gc();
 
   bool collect(bool clear_all_soft_refs);
 
@@ -220,13 +226,9 @@ class DefNewGeneration: public Generation {
 
   DefNewTracer* gc_tracer() const { return _gc_tracer; }
 
- protected:
-  // If clear_space is true, clear the survivor spaces.  Eden is
-  // cleared if the minimum size of eden is 0.  If mangle_space
-  // is true, also mangle the space in debug mode.
-  void compute_space_boundaries(uintx minimum_eden_size,
-                                bool clear_space,
-                                bool mangle_space);
+ private:
+  // Initialize eden/from/to spaces.
+  void init_spaces();
 
   // Return adjusted new size for NewSizeThreadIncrease.
   // If any overflow happens, revert to previous new size.

src/hotspot/share/gc/serial/serialHeap.cpp

@@ -182,7 +182,6 @@ jint SerialHeap::initialize() {
   _rem_set->initialize(young_rs.base(), old_rs.base());
 
   CardTableBarrierSet *bs = new CardTableBarrierSet(_rem_set);
-  bs->initialize();
   BarrierSet::set_barrier_set(bs);
 
   _young_gen = new DefNewGeneration(young_rs, NewSize, MinNewSize, MaxNewSize);
@@ -269,9 +268,9 @@ size_t SerialHeap::max_capacity() const {
 }
 
 HeapWord* SerialHeap::expand_heap_and_allocate(size_t size, bool is_tlab) {
-  HeapWord* result = _young_gen->allocate(size);
+  HeapWord* result = _young_gen->expand_and_allocate(size);
 
-  if (result == nullptr) {
+  if (result == nullptr && !is_tlab) {
     result = _old_gen->expand_and_allocate(size);
   }
 
@@ -388,14 +387,13 @@ bool SerialHeap::do_young_collection(bool clear_soft_refs) {
   // Only update stats for successful young-gc
   if (result) {
     _old_gen->update_promote_stats();
+    _young_gen->resize_after_young_gc();
   }
 
   if (should_verify && VerifyAfterGC) {
     Universe::verify("After GC");
   }
 
-  _young_gen->compute_new_size();
-
   print_heap_change(pre_gc_values);
 
   // Track memory usage and detect low memory after GC finishes
@@ -581,7 +579,7 @@ void SerialHeap::do_full_collection(bool clear_all_soft_refs) {
 
   // Adjust generation sizes.
   _old_gen->compute_new_size();
-  _young_gen->compute_new_size();
+  _young_gen->resize_after_full_gc();
 
   _old_gen->update_promote_stats();
 

src/hotspot/share/gc/serial/serialHeap.hpp

@@ -55,10 +55,10 @@ class TenuredGeneration;
 //                                              +-- generation boundary (fixed after startup)
 //                                              |
 // |<-    young gen (reserved MaxNewSize)     ->|<- old gen (reserved MaxOldSize) ->|
-// +-----------------+--------+--------+--------+---------------+-------------------+
-// |       eden      |  from  |   to   |        |      old      |                   |
-// |                 |  (to)  | (from) |        |               |                   |
-// +-----------------+--------+--------+--------+---------------+-------------------+
+// +--------+--------+-----------------+--------+---------------+-------------------+
+// |  from  |   to   |       eden      |        |      old      |                   |
+// |  (to)  | (from) |                 |        |               |                   |
+// +--------+--------+-----------------+--------+---------------+-------------------+
 // |<-          committed            ->|        |<- committed ->|
 //
 class SerialHeap : public CollectedHeap {

src/hotspot/share/gc/serial/tenuredGeneration.cpp

@@ -314,7 +314,7 @@ TenuredGeneration::TenuredGeneration(ReservedSpace rs,
   HeapWord* bottom = (HeapWord*) _virtual_space.low();
   HeapWord* end    = (HeapWord*) _virtual_space.high();
   _the_space  = new ContiguousSpace();
-  _the_space->initialize(MemRegion(bottom, end), SpaceDecorator::Clear, SpaceDecorator::Mangle);
+  _the_space->initialize(MemRegion(bottom, end), SpaceDecorator::Clear);
   // If we don't shrink the heap in steps, '_shrink_factor' is always 100%.
   _shrink_factor = ShrinkHeapInSteps ? 0 : 100;
   _capacity_at_prologue = 0;

src/hotspot/share/gc/shared/c2/barrierSetC2.cpp

@@ -115,7 +115,7 @@ uint8_t BarrierStubC2::barrier_data() const {
 void BarrierStubC2::preserve(Register r) {
   const VMReg vm_reg = r->as_VMReg();
   assert(vm_reg->is_Register(), "r must be a general-purpose register");
-  _preserve.Insert(OptoReg::as_OptoReg(vm_reg));
+  _preserve.insert(OptoReg::as_OptoReg(vm_reg));
 }
 
 void BarrierStubC2::dont_preserve(Register r) {
@@ -124,7 +124,7 @@ void BarrierStubC2::dont_preserve(Register r) {
   // Subtract the given register and all its sub-registers (e.g. {R11, R11_H}
   // for r11 in aarch64).
   do {
-    _preserve.Remove(OptoReg::as_OptoReg(vm_reg));
+    _preserve.remove(OptoReg::as_OptoReg(vm_reg));
     vm_reg = vm_reg->next();
   } while (vm_reg->is_Register() && !vm_reg->is_concrete());
 }
@@ -1171,7 +1171,7 @@ void BarrierSetC2::compute_liveness_at_stubs() const {
     // Initialize to union of successors
     for (uint i = 0; i < block->_num_succs; i++) {
       const uint succ_id = block->_succs[i]->_pre_order;
-      new_live.OR(live[succ_id]);
+      new_live.or_with(live[succ_id]);
     }
 
     // Walk block backwards, computing liveness
@@ -1182,7 +1182,7 @@ void BarrierSetC2::compute_liveness_at_stubs() const {
       if (!bs_state->needs_livein_data()) {
         RegMask* const regs = bs_state->live(node);
         if (regs != nullptr) {
-          regs->OR(new_live);
+          regs->or_with(new_live);
         }
       }
 
@@ -1190,10 +1190,10 @@ void BarrierSetC2::compute_liveness_at_stubs() const {
       const OptoReg::Name first = bs->refine_register(node, regalloc->get_reg_first(node));
       const OptoReg::Name second = bs->refine_register(node, regalloc->get_reg_second(node));
       if (first != OptoReg::Bad) {
-        new_live.Remove(first);
+        new_live.remove(first);
       }
       if (second != OptoReg::Bad) {
-        new_live.Remove(second);
+        new_live.remove(second);
       }
 
       // Add use bits
@@ -1202,10 +1202,10 @@ void BarrierSetC2::compute_liveness_at_stubs() const {
         const OptoReg::Name first = bs->refine_register(use, regalloc->get_reg_first(use));
         const OptoReg::Name second = bs->refine_register(use, regalloc->get_reg_second(use));
         if (first != OptoReg::Bad) {
-          new_live.Insert(first);
+          new_live.insert(first);
         }
         if (second != OptoReg::Bad) {
-          new_live.Insert(second);
+          new_live.insert(second);
         }
       }
 
@@ -1213,16 +1213,16 @@ void BarrierSetC2::compute_liveness_at_stubs() const {
       if (bs_state->needs_livein_data()) {
         RegMask* const regs = bs_state->live(node);
         if (regs != nullptr) {
-          regs->OR(new_live);
+          regs->or_with(new_live);
         }
       }
     }
 
     // Now at block top, see if we have any changes
-    new_live.SUBTRACT(old_live);
-    if (!new_live.is_Empty()) {
+    new_live.subtract(old_live);
+    if (!new_live.is_empty()) {
       // Liveness has refined, update and propagate to prior blocks
-      old_live.OR(new_live);
+      old_live.or_with(new_live);
       for (uint i = 1; i < block->num_preds(); ++i) {
         Block* const pred = cfg->get_block_for_node(block->pred(i));
         worklist.push(pred);

src/hotspot/share/gc/shared/cardTableBarrierSet.cpp

@@ -57,7 +57,6 @@ CardTableBarrierSet::CardTableBarrierSet(BarrierSetAssembler* barrier_set_assemb
                    barrier_set_c1,
                    barrier_set_c2,
                    fake_rtti.add_tag(BarrierSet::CardTableBarrierSet)),
-  _defer_initial_card_mark(false),
   _card_table(card_table)
 {}
 
@@ -66,14 +65,9 @@ CardTableBarrierSet::CardTableBarrierSet(CardTable* card_table) :
                    make_barrier_set_c1<CardTableBarrierSetC1>(),
                    make_barrier_set_c2<CardTableBarrierSetC2>(),
                    BarrierSet::FakeRtti(BarrierSet::CardTableBarrierSet)),
-  _defer_initial_card_mark(false),
   _card_table(card_table)
 {}
 
-void CardTableBarrierSet::initialize() {
-  initialize_deferred_card_mark_barriers();
-}
-
 CardTableBarrierSet::~CardTableBarrierSet() {
   delete _card_table;
 }
@@ -108,9 +102,7 @@ void CardTableBarrierSet::print_on(outputStream* st) const {
 //     to the post-barrier, we note that G1 needs a RS update barrier
 //     which simply enqueues a (sequence of) dirty cards which may
 //     optionally be refined by the concurrent update threads. Note
-//     that this barrier need only be applied to a non-young write,
-//     but, because of the presence of concurrent refinement,
-//     must strictly follow the oop-store.
+//     that this barrier need only be applied to a non-young write.
 //
 // For any future collector, this code should be reexamined with
 // that specific collector in mind, and the documentation above suitably
@@ -120,72 +112,13 @@ void CardTableBarrierSet::on_slowpath_allocation_exit(JavaThread* thread, oop ne
   if (!ReduceInitialCardMarks) {
     return;
   }
-  // If a previous card-mark was deferred, flush it now.
-  flush_deferred_card_mark_barrier(thread);
   if (new_obj->is_typeArray() || _card_table->is_in_young(new_obj)) {
     // Arrays of non-references don't need a post-barrier.
-    // The deferred_card_mark region should be empty
-    // following the flush above.
-    assert(thread->deferred_card_mark().is_empty(), "Error");
   } else {
     MemRegion mr(cast_from_oop<HeapWord*>(new_obj), new_obj->size());
     assert(!mr.is_empty(), "Error");
-    if (_defer_initial_card_mark) {
-      // Defer the card mark
-      thread->set_deferred_card_mark(mr);
-    } else {
-      // Do the card mark
-      write_region(mr);
-    }
+    // Do the card mark
+    write_region(mr);
   }
 #endif // COMPILER2_OR_JVMCI
 }
-
-void CardTableBarrierSet::initialize_deferred_card_mark_barriers() {
-  // Used for ReduceInitialCardMarks (when COMPILER2 or JVMCI is used);
-  // otherwise remains unused.
-#if COMPILER2_OR_JVMCI
-  _defer_initial_card_mark = CompilerConfig::is_c2_or_jvmci_compiler_enabled() && ReduceInitialCardMarks
-                             && (DeferInitialCardMark || card_mark_must_follow_store());
-#else
-  assert(_defer_initial_card_mark == false, "Who would set it?");
-#endif
-}
-
-void CardTableBarrierSet::flush_deferred_card_mark_barrier(JavaThread* thread) {
-#if COMPILER2_OR_JVMCI
-  MemRegion deferred = thread->deferred_card_mark();
-  if (!deferred.is_empty()) {
-    assert(_defer_initial_card_mark, "Otherwise should be empty");
-    {
-      // Verify that the storage points to a parsable object in heap
-      DEBUG_ONLY(oop old_obj = cast_to_oop(deferred.start());)
-      assert(!_card_table->is_in_young(old_obj),
-             "Else should have been filtered in on_slowpath_allocation_exit()");
-      assert(oopDesc::is_oop(old_obj), "Not an oop");
-      assert(deferred.word_size() == old_obj->size(),
-             "Mismatch: multiple objects?");
-    }
-    write_region(thread, deferred);
-    // "Clear" the deferred_card_mark field
-    thread->set_deferred_card_mark(MemRegion());
-  }
-  assert(thread->deferred_card_mark().is_empty(), "invariant");
-#else
-  assert(!_defer_initial_card_mark, "Should be false");
-  assert(thread->deferred_card_mark().is_empty(), "Should be empty");
-#endif
-}
-
-void CardTableBarrierSet::on_thread_detach(Thread* thread) {
-  // The deferred store barriers must all have been flushed to the
-  // card-table (or other remembered set structure) before GC starts
-  // processing the card-table (or other remembered set).
-  if (thread->is_Java_thread()) { // Only relevant for Java threads.
-    flush_deferred_card_mark_barrier(JavaThread::cast(thread));
-  }
-}
-
-bool CardTableBarrierSet::card_mark_must_follow_store() const {
-  return false;
-}

src/hotspot/share/gc/shared/cardTableBarrierSet.hpp

@@ -47,9 +47,6 @@ class CardTableBarrierSet: public ModRefBarrierSet {
 
 protected:
   typedef CardTable::CardValue CardValue;
-  // Used in support of ReduceInitialCardMarks; only consulted if COMPILER2
-  // or INCLUDE_JVMCI is being used
-  bool       _defer_initial_card_mark;
   CardTable* _card_table;
 
   CardTableBarrierSet(BarrierSetAssembler* barrier_set_assembler,
@@ -64,13 +61,10 @@ public:
 
   CardTable* card_table() const { return _card_table; }
 
-  void initialize();
-
   void write_region(JavaThread* thread, MemRegion mr) {
     write_region(mr);
   }
 
- public:
   // Record a reference update. Note that these versions are precise!
   // The scanning code has to handle the fact that the write barrier may be
   // either precise or imprecise. We make non-virtual inline variants of
@@ -80,29 +74,7 @@ public:
 
   virtual void write_region(MemRegion mr);
 
-  // ReduceInitialCardMarks
-  void initialize_deferred_card_mark_barriers();
-
-  // If the CollectedHeap was asked to defer a store barrier above,
-  // this informs it to flush such a deferred store barrier to the
-  // remembered set.
-  void flush_deferred_card_mark_barrier(JavaThread* thread);
-
-  // If a compiler is eliding store barriers for TLAB-allocated objects,
-  // we will be informed of a slow-path allocation by a call
-  // to on_slowpath_allocation_exit() below. Such a call precedes the
-  // initialization of the object itself, and no post-store-barriers will
-  // be issued. Some heap types require that the barrier strictly follows
-  // the initializing stores. (This is currently implemented by deferring the
-  // barrier until the next slow-path allocation or gc-related safepoint.)
-  // This interface answers whether a particular barrier type needs the card
-  // mark to be thus strictly sequenced after the stores.
-  virtual bool card_mark_must_follow_store() const;
-
   virtual void on_slowpath_allocation_exit(JavaThread* thread, oop new_obj);
-  virtual void on_thread_detach(Thread* thread);
-
-  virtual void make_parsable(JavaThread* thread) { flush_deferred_card_mark_barrier(thread); }
 
   virtual void print_on(outputStream* st) const;