8391075: 'jfr assemble' doesn't handle unfinished files

Reviewed-by: mgronlun

.github/workflows/main.yml

@@ -60,7 +60,7 @@ jobs:
     runs-on: ubuntu-24.04
     env:
       # List of platforms to exclude by default
-      EXCLUDED_PLATFORMS: 'alpine-linux-x64'
+      EXCLUDED_PLATFORMS: 'alpine-linux-x64,macos-x64'
     outputs:
       linux-x64: ${{ steps.include.outputs.linux-x64 }}
       linux-x64-variants: ${{ steps.include.outputs.linux-x64-variants }}

doc/testing.html

@@ -424,11 +424,29 @@ use the <code>--with-jcov-modules</code> arguments to
 <p>For more fine-grained control, you can pass arbitrary filters to JCov
 using <code>--with-jcov-filters</code>, and you can specify a specific
 JDK to instrument using <code>--with-jcov-input-jdk</code>.</p>
+<p>The resulting coverage is written into
+<code>build/$BUILD/test-results/jcov-output/result.xml</code>.</p>
 <p>The JCov report is stored in
 <code>build/$BUILD/test-results/jcov-output/report</code>.</p>
 <p>Please note that running with JCov reporting can be very memory
 intensive.</p>
-<h4 id="jcov_diff_changeset">JCOV_DIFF_CHANGESET</h4>
+<h5 id="jcov-scales">JCov scales</h5>
+<p>JCov scales make it possible to record which tests cover each part of
+the instrumented code. To collect coverage with scales, set
+<code>JCOV_SCALES=true</code>, for example:</p>
+<pre><code>$ make jcov-test TEST=jdk_lang TEST_OPTS=&quot;JCOV_SCALES=true&quot;</code></pre>
+<p>The resulting coverage data contains the association between covered
+code and the tests that covered it. A corresponding
+<code>testlist.txt</code> file, which contains the test names, is
+generated in the same directory.</p>
+<p>The JCov report displays the names of the tests that cover each
+class.</p>
+<p>Collecting coverage scales forces jtreg tests to be run in
+<code>othervm</code> mode, which takes longer than ordinary JCov
+collection. The coverage data is also larger because it includes scale
+information, and the generated report is larger because it includes test
+names.</p>
+<h5 id="jcov_diff_changeset">JCOV_DIFF_CHANGESET</h5>
 <p>While collecting code coverage with JCov, it is also possible to find
 coverage for only recently changed code. JCOV_DIFF_CHANGESET specifies a
 source revision. A textual report will be generated showing coverage of

doc/testing.md

@@ -353,11 +353,33 @@ For more fine-grained control, you can pass arbitrary filters to JCov using
 `--with-jcov-filters`, and you can specify a specific JDK to instrument
 using `--with-jcov-input-jdk`.
 
+The resulting coverage is written into
+`build/$BUILD/test-results/jcov-output/result.xml`.
+
 The JCov report is stored in `build/$BUILD/test-results/jcov-output/report`.
 
 Please note that running with JCov reporting can be very memory intensive.
 
-#### JCOV_DIFF_CHANGESET
+##### JCov scales
+
+JCov scales make it possible to record which tests cover each part of the
+instrumented code. To collect coverage with scales, set `JCOV_SCALES=true`,
+for example:
+
+    $ make jcov-test TEST=jdk_lang TEST_OPTS="JCOV_SCALES=true"
+
+The resulting coverage data contains the association between covered code and
+the tests that covered it. A corresponding `testlist.txt` file, which contains
+the test names, is generated in the same directory.
+
+The JCov report displays the names of the tests that cover each class.
+
+Collecting coverage scales forces jtreg tests to be run in `othervm` mode,
+which takes longer than ordinary JCov collection. The coverage data is also
+larger because it includes scale information, and the generated report is
+larger because it includes test names.
+
+##### JCOV_DIFF_CHANGESET
 
 While collecting code coverage with JCov, it is also possible to find coverage
 for only recently changed code. JCOV_DIFF_CHANGESET specifies a source

make/Main.gmk

@@ -751,6 +751,18 @@ $(eval $(call SetupTarget, test-image-lib, \
     DEPS := build-test-lib, \
 ))
 
+$(eval $(call SetupTarget, build-hotspot-compiler-testlibrary, \
+    MAKEFILE := hotspot/test/BuildCompilerTestlibrary, \
+    TARGET := build-hotspot-compiler-testlibrary, \
+    DEPS := exploded-image build-test-lib, \
+))
+
+$(eval $(call SetupTarget, test-image-hotspot-compiler-testlibrary, \
+    MAKEFILE := hotspot/test/BuildCompilerTestlibrary, \
+    TARGET := test-image-hotspot-compiler-testlibrary, \
+    DEPS := build-hotspot-compiler-testlibrary, \
+))
+
 $(eval $(call SetupTarget, build-test-setup-aot, \
     MAKEFILE := test/BuildTestSetupAOT, \
     DEPS := interim-langtools exploded-image, \
@@ -1303,7 +1315,7 @@ all-docs-bundles: docs-jdk-bundles docs-javase-bundles docs-reference-bundles
 test-image: prepare-test-image test-image-jdk-jtreg-native \
     test-image-demos-jdk test-image-libtest-jtreg-native \
     test-image-lib test-image-lib-native \
-    test-image-setup-aot
+    test-image-setup-aot test-image-hotspot-compiler-testlibrary
 
 ifneq ($(JVM_TEST_IMAGE_TARGETS), )
   # If JVM_TEST_IMAGE_TARGETS is externally defined, use it instead of the

make/RunTests.gmk

@@ -45,7 +45,7 @@ ifneq ($(TEST_VM_OPTS), )
 endif
 
 $(eval $(call ParseKeywordVariable, TEST_OPTS, \
-    SINGLE_KEYWORDS := JOBS TIMEOUT_FACTOR JCOV JCOV_DIFF_CHANGESET AOT_JDK, \
+    SINGLE_KEYWORDS := JOBS TIMEOUT_FACTOR JCOV JCOV_DIFF_CHANGESET JCOV_SCALES AOT_JDK, \
     STRING_KEYWORDS := VM_OPTIONS JAVA_OPTIONS, \
 ))
 
@@ -121,9 +121,21 @@ ifeq ($(TEST_OPTS_JCOV), true)
   JCOV_SUPPORT_DIR := $(TEST_SUPPORT_DIR)/jcov-support
   JCOV_GRABBER_LOG := $(JCOV_OUTPUT_DIR)/grabber.log
   JCOV_RESULT_FILE := $(JCOV_OUTPUT_DIR)/result.xml
+  JCOV_TESTLIST := $(JCOV_OUTPUT_DIR)/testlist.txt
   JCOV_REPORT := $(JCOV_OUTPUT_DIR)/report
+  JCOV_GRABBER_OPTIONS ?=
+  JCOV_REPGEN_OPTIONS ?=
+  TEST_OPTS_JCOV_SCALES ?= false
   JCOV_MEM_OPTIONS := -Xms64m -Xmx4g
 
+  ifeq ($(TEST_OPTS_JCOV_SCALES), true)
+    JCOV_GRABBER_OPTIONS += -scale -mergebyname -outTestList $(JCOV_TESTLIST)
+    TEST_JOBS := 1
+    JTREG_TEST_MODE := othervm
+    JTREG_VM_OPTIONS += -Djcov.extension=com.sun.tdk.jcov.runtime.TestNameDecorator
+    JCOV_REPGEN_OPTIONS += -tests $(JCOV_TESTLIST)
+  endif
+
   # Replace our normal test JDK with the JCov image.
   JDK_UNDER_TEST := $(JCOV_IMAGE_DIR)
 
@@ -1054,9 +1066,11 @@ define SetupRunJtregTestBody
   $1_JTREG_BASIC_OPTIONS += -timeoutFactor:$$(JTREG_TIMEOUT_FACTOR)
 
   clean-outputdirs-$1:
-	$$(call LogWarn, Clean up dirs for $1)
-	$$(RM) -r $$($1_TEST_SUPPORT_DIR)
-	$$(RM) -r $$($1_TEST_RESULTS_DIR)
+        ifeq ($(JTREG_STATUS),)
+	  $$(call LogWarn, Clean up dirs for $1)
+	  $$(RM) -r $$($1_TEST_SUPPORT_DIR)
+	  $$(RM) -r $$($1_TEST_RESULTS_DIR)
+        endif
 
   $1_COMMAND_LINE := \
       $$(JTREG_JAVA) $$($1_JTREG_LAUNCHER_OPTIONS) \
@@ -1412,6 +1426,7 @@ ifeq ($(TEST_OPTS_JCOV), true)
 	fi
 	$(JAVA) $(JCOV_VM_OPTS) -jar $(JCOV_HOME)/lib/jcov.jar Grabber -v -t \
 	    $(JCOV_IMAGE_DIR)/template.xml -o $(JCOV_RESULT_FILE) \
+	    $(JCOV_GRABBER_OPTIONS) \
 	    1>$(JCOV_GRABBER_LOG) 2>&1 &
 
   jcov-start-grabber: jcov-do-start-grabber
@@ -1439,6 +1454,7 @@ ifeq ($(TEST_OPTS_JCOV), true)
 	    `$(ECHO) $(TOPDIR)/src/*/share/classes/ | $(TR) ' ' ':'` -fmt html \
 	    $(JCOV_MODULES_FILTER) $(JCOV_FILTERS) \
 	    -mainReportTitle "$(JCOV_REPORT_TITLE)" \
+	    $(JCOV_REPGEN_OPTIONS) \
 	    -o $(JCOV_REPORT) $(JCOV_RESULT_FILE))
 
   TARGETS += jcov-do-start-grabber jcov-start-grabber jcov-stop-grabber \

make/autoconf/platform.m4

@@ -660,7 +660,19 @@ AC_DEFUN([PLATFORM_CHECK_DEPRECATION],
 [
   AC_ARG_ENABLE(deprecated-ports, [AS_HELP_STRING([--enable-deprecated-ports@<:@=yes/no@:>@],
       [Suppress the error when configuring for a deprecated port @<:@no@:>@])])
-  # There are no deprecated ports. Implement the deprecation warnings here.
+  if test "x$OPENJDK_TARGET_OS" = xmacosx && test "x$OPENJDK_TARGET_CPU" = xx86_64; then
+    # Unfortunately, variants have not been parsed yet, so we have to check the configure option
+    # directly. Allow only the directly specified Zero variant, treat any other mix as containing
+    # something non-Zero.
+    if test "x$with_jvm_variants" != xzero; then
+      if test "x$enable_deprecated_ports" = "xyes"; then
+        AC_MSG_WARN([The macOS/x64 port is deprecated and may be removed in a future release.])
+      else
+        AC_MSG_ERROR(m4_normalize([The macOS/x64 port is deprecated and may be removed in a future release.
+          Use --enable-deprecated-ports to suppress this error.]))
+      fi
+    fi
+  fi
 ])
 
 AC_DEFUN_ONCE([PLATFORM_SETUP_OPENJDK_BUILD_OS_VERSION],

make/conf/jib-profiles.js

@@ -416,6 +416,7 @@ var getJibProfilesProfiles = function (input, common, data) {
                 "--with-zlib=system",
                 "--with-macosx-version-max=11.00.00",
                 "--enable-compatible-cds-alignment",
+                "--enable-deprecated-ports",
                 // Use system SetFile instead of the one in the devkit as the
                 // devkit one may not work on Catalina.
                 "SETFILE=/usr/bin/SetFile"
@@ -1192,8 +1193,8 @@ var getJibProfilesDependencies = function (input, common) {
             server: "jpg",
             product: "jcov",
             version: "3.0",
-            build_number: "6",
-            file: "bundles/jcov-3.0+6.zip",
+            build_number: "9",
+            file: "bundles/jcov-3.0+9.zip",
             environment_name: "JCOV_HOME",
         },
 

make/hotspot/lib/CompileJvm.gmk

@@ -285,13 +285,6 @@ ifeq ($(call isTargetOs, windows), true)
   $(BUILD_LIBJVM_TARGET): $(WIN_EXPORT_FILE)
 endif
 
-# Always recompile abstract_vm_version.cpp if libjvm needs to be relinked. This ensures
-# that the internal vm version is updated as it relies on __DATE__ and __TIME__
-# macros.
-ABSTRACT_VM_VERSION_OBJ := $(JVM_OUTPUTDIR)/objs/abstract_vm_version$(OBJ_SUFFIX)
-$(ABSTRACT_VM_VERSION_OBJ): $(filter-out $(ABSTRACT_VM_VERSION_OBJ), \
-    $(BUILD_LIBJVM_TARGET_DEPS))
-
 ifneq ($(GENERATE_COMPILE_COMMANDS_ONLY), true)
   ifeq ($(call isTargetOs, windows), true)
     # It doesn't matter which jvm.lib file gets exported, but we need

make/hotspot/test/BuildCompilerTestlibrary.gmk

@@ -0,0 +1,76 @@
+#
+# Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# This code is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 2 only, as
+# published by the Free Software Foundation.  Oracle designates this
+# particular file as subject to the "Classpath" exception as provided
+# by Oracle in the LICENSE file that accompanied this code.
+#
+# This code is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# version 2 for more details (a copy is included in the LICENSE file that
+# accompanied this code).
+#
+# You should have received a copy of the GNU General Public License version
+# 2 along with this work; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+# or visit www.oracle.com if you need additional information or have any
+# questions.
+#
+
+include MakeFileStart.gmk
+
+################################################################################
+# This file builds the Hotspot compiler testlibrary.
+################################################################################
+
+include CopyFiles.gmk
+include JavaCompilation.gmk
+
+###############################################################################
+
+COMPILER_TESTLIBRARY_BASEDIR := $(TOPDIR)/test/hotspot/jtreg/compiler/lib
+COMPILER_TESTLIBRARY_SUPPORT := $(SUPPORT_OUTPUTDIR)/test/compiler-testlibrary
+COMPILER_TESTLIBRARY_JAR := $(COMPILER_TESTLIBRARY_SUPPORT)/compiler-testlibrary.jar
+TEST_LIB_SUPPORT := $(SUPPORT_OUTPUTDIR)/test/lib
+WB_CP := $(TEST_LIB_SUPPORT)/wb_classes
+TEST_LIB_CP := $(TEST_LIB_SUPPORT)/test-lib_classes
+
+$(eval $(call SetupJavaCompilation, BUILD_COMPILER_TESTLIBRARY, \
+    TARGET_RELEASE := $(TARGET_RELEASE_NEWJDK_UPGRADED), \
+    SRC := $(COMPILER_TESTLIBRARY_BASEDIR), \
+    BIN := $(COMPILER_TESTLIBRARY_SUPPORT)/classes, \
+    JAR := $(COMPILER_TESTLIBRARY_JAR), \
+    JAVAC_FLAGS := -cp $(WB_CP) \
+        -cp $(TEST_LIB_CP) \
+        --add-exports java.base/jdk.internal.math=ALL-UNNAMED, \
+))
+
+TARGETS += $(BUILD_COMPILER_TESTLIBRARY)
+
+build-hotspot-compiler-testlibrary: $(TARGETS)
+
+################################################################################
+# Targets for building test-image.
+################################################################################
+
+# Copy to hotspot jtreg test image
+$(eval $(call SetupCopyFiles, COPY_COMPILER_TESTLIBRARY, \
+    DEST := $(TEST_IMAGE_DIR)/compiler-testlibrary, \
+    FILES := $(COMPILER_TESTLIBRARY_JAR), \
+))
+
+IMAGE_TARGETS += $(COPY_COMPILER_TESTLIBRARY)
+
+test-image-hotspot-compiler-testlibrary: $(IMAGE_TARGETS)
+
+.PHONY: build-hotspot-compiler-testlibrary test-image-hotspot-compiler-testlibrary
+
+################################################################################
+
+include MakeFileEnd.gmk

src/hotspot/cpu/aarch64/bytes_aarch64.hpp

@@ -1,57 +0,0 @@
-/*
- * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2014, Red Hat Inc. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#ifndef CPU_AARCH64_BYTES_AARCH64_HPP
-#define CPU_AARCH64_BYTES_AARCH64_HPP
-
-#include "memory/allStatic.hpp"
-#include "utilities/byteswap.hpp"
-
-class Bytes: AllStatic {
- public:
-  // Efficient reading and writing of unaligned unsigned data in platform-specific byte ordering
-  // (no special code is needed since x86 CPUs can access unaligned data)
-  static inline u2   get_native_u2(address p)         { return *(u2*)p; }
-  static inline u4   get_native_u4(address p)         { return *(u4*)p; }
-  static inline u8   get_native_u8(address p)         { return *(u8*)p; }
-
-  static inline void put_native_u2(address p, u2 x)   { *(u2*)p = x; }
-  static inline void put_native_u4(address p, u4 x)   { *(u4*)p = x; }
-  static inline void put_native_u8(address p, u8 x)   { *(u8*)p = x; }
-
-
-  // Efficient reading and writing of unaligned unsigned data in Java
-  // byte ordering (i.e. big-endian ordering). Byte-order reversal is
-  // needed since x86 CPUs use little-endian format.
-  static inline u2   get_Java_u2(address p)           { return byteswap(get_native_u2(p)); }
-  static inline u4   get_Java_u4(address p)           { return byteswap(get_native_u4(p)); }
-  static inline u8   get_Java_u8(address p)           { return byteswap(get_native_u8(p)); }
-
-  static inline void put_Java_u2(address p, u2 x)     { put_native_u2(p, byteswap(x)); }
-  static inline void put_Java_u4(address p, u4 x)     { put_native_u4(p, byteswap(x)); }
-  static inline void put_Java_u8(address p, u8 x)     { put_native_u8(p, byteswap(x)); }
-};
-
-#endif // CPU_AARCH64_BYTES_AARCH64_HPP

src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp

@@ -2436,7 +2436,8 @@ void C2_MacroAssembler::neon_reverse_bytes(FloatRegister dst, FloatRegister src,
 void C2_MacroAssembler::neon_rearrange_hsd(FloatRegister dst, FloatRegister src,
                                            FloatRegister shuffle, FloatRegister tmp,
                                            BasicType bt, bool isQ) {
-  assert_different_registers(dst, src, shuffle, tmp);
+  assert_different_registers(dst, src, tmp);
+  assert_different_registers(shuffle, tmp);
   SIMD_Arrangement size1 = isQ ? T16B : T8B;
   SIMD_Arrangement size2 = esize2arrangement((uint)type2aelembytes(bt), isQ);
 

src/hotspot/cpu/aarch64/downcallLinker_aarch64.cpp

@@ -308,7 +308,7 @@ void DowncallLinker::StubGenerator::generate() {
     // Restore cpu control state after JNI call
     __ restore_cpu_control_state_after_jni(rscratch1, tmp1);
 
-    __ mov(tmp1, _thread_in_native_trans);
+    __ mov(tmp1, _thread_in_vm);
     __ strw(tmp1, Address(rthread, JavaThread::thread_state_offset()));
 
     // Force this write out before the read below
@@ -388,5 +388,5 @@ void DowncallLinker::StubGenerator::generate() {
 
   //////////////////////////////////////////////////////////////////////////////
 
-  __ flush();
+  // Code will be copied. No ICache sync required.
 }

src/hotspot/cpu/aarch64/gc/z/z_aarch64.ad

@@ -33,14 +33,14 @@ source %{
 
 #include "gc/z/zBarrierSetAssembler.hpp"
 
-static void z_color(MacroAssembler* masm, const MachNode* node, Register dst, Register src) {
+static void z_color(MacroAssembler* masm, Register dst, Register src) {
   assert_different_registers(src, dst);
   __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatStoreGoodBeforeMov);
   __ movzw(dst, barrier_Relocation::unpatched);
   __ orr(dst, dst, src, Assembler::LSL, ZPointerLoadShift);
 }
 
-static void z_uncolor(MacroAssembler* masm, const MachNode* node, Register ref) {
+static void z_uncolor(MacroAssembler* masm, Register ref) {
   __ lsr(ref, ref, ZPointerLoadShift);
 }
 
@@ -50,7 +50,7 @@ static void z_keep_alive_load_barrier(MacroAssembler* masm, const MachNode* node
   __ tst(ref, tmp);
   ZLoadBarrierStubC2Aarch64* const stub = ZLoadBarrierStubC2Aarch64::create(node, ref_addr, ref);
   __ br(Assembler::NE, *stub->entry());
-  z_uncolor(masm, node, ref);
+  z_uncolor(masm, ref);
   __ bind(*stub->continuation());
 }
 
@@ -66,7 +66,7 @@ static void z_load_barrier(MacroAssembler* masm, const MachNode* node, Address r
   }
 
   if (node->barrier_data() == ZBarrierElided) {
-    z_uncolor(masm, node, ref);
+    z_uncolor(masm, ref);
     return;
   }
 
@@ -81,14 +81,14 @@ static void z_load_barrier(MacroAssembler* masm, const MachNode* node, Address r
     __ b(*stub->entry());
     __ bind(good);
   }
-  z_uncolor(masm, node, ref);
+  z_uncolor(masm, ref);
   __ bind(*stub->continuation());
 }
 
 static void z_store_barrier(MacroAssembler* masm, const MachNode* node, Address ref_addr, Register rnew_zaddress, Register rnew_zpointer, Register tmp, bool is_atomic) {
   Assembler::InlineSkippedInstructionsCounter skipped_counter(masm);
   if (node->barrier_data() == ZBarrierElided) {
-    z_color(masm, node, rnew_zpointer, rnew_zaddress);
+    z_color(masm, rnew_zpointer, rnew_zaddress);
   } else {
     bool is_native = (node->barrier_data() & ZBarrierNative) != 0;
     bool is_nokeepalive = (node->barrier_data() & ZBarrierNoKeepalive) != 0;
@@ -206,7 +206,7 @@ instruct zCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newva
     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
     Address ref_addr($mem$$Register);
     z_store_barrier(masm, this, ref_addr, $newval$$Register, $newval_tmp$$Register, rscratch2, true /* is_atomic */);
-    z_color(masm, this, $oldval_tmp$$Register, $oldval$$Register);
+    z_color(masm, $oldval_tmp$$Register, $oldval$$Register);
     __ cmpxchg($mem$$Register, $oldval_tmp$$Register, $newval_tmp$$Register, Assembler::xword, memory_order_release);
     __ cset($res$$Register, Assembler::EQ);
   %}
@@ -229,7 +229,7 @@ instruct zCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP ne
     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
     Address ref_addr($mem$$Register);
     z_store_barrier(masm, this, ref_addr, $newval$$Register, $newval_tmp$$Register, rscratch2, true /* is_atomic */);
-    z_color(masm, this, $oldval_tmp$$Register, $oldval$$Register);
+    z_color(masm, $oldval_tmp$$Register, $oldval$$Register);
     __ cmpxchg($mem$$Register, $oldval_tmp$$Register, $newval_tmp$$Register, Assembler::xword, memory_order_seq_cst);
     __ cset($res$$Register, Assembler::EQ);
   %}
@@ -251,10 +251,10 @@ instruct zCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP n
     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
     Address ref_addr($mem$$Register);
     z_store_barrier(masm, this, ref_addr, $newval$$Register, $newval_tmp$$Register, rscratch2, true /* is_atomic */);
-    z_color(masm, this, $oldval_tmp$$Register, $oldval$$Register);
+    z_color(masm, $oldval_tmp$$Register, $oldval$$Register);
     __ cmpxchg($mem$$Register, $oldval_tmp$$Register, $newval_tmp$$Register, Assembler::xword,
                memory_order_release, $res$$Register);
-    z_uncolor(masm, this, $res$$Register);
+    z_uncolor(masm, $res$$Register);
   %}
 
   ins_pipe(pipe_slow);
@@ -274,10 +274,10 @@ instruct zCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iReg
     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
     Address ref_addr($mem$$Register);
     z_store_barrier(masm, this, ref_addr, $newval$$Register, $newval_tmp$$Register, rscratch2, true /* is_atomic */);
-    z_color(masm, this, $oldval_tmp$$Register, $oldval$$Register);
+    z_color(masm, $oldval_tmp$$Register, $oldval$$Register);
     __ cmpxchg($mem$$Register, $oldval_tmp$$Register, $newval_tmp$$Register, Assembler::xword,
                memory_order_seq_cst, $res$$Register);
-    z_uncolor(masm, this, $res$$Register);
+    z_uncolor(masm, $res$$Register);
   %}
 
   ins_pipe(pipe_slow);
@@ -295,7 +295,7 @@ instruct zGetAndSetP(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{
   ins_encode %{
     z_store_barrier(masm, this, Address($mem$$Register), $newv$$Register, $prev$$Register, rscratch2, true /* is_atomic */);
     __ atomic_xchg($prev$$Register, $prev$$Register, $mem$$Register);
-    z_uncolor(masm, this, $prev$$Register);
+    z_uncolor(masm, $prev$$Register);
   %}
 
   ins_pipe(pipe_serial);
@@ -313,7 +313,7 @@ instruct zGetAndSetPAcq(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr)
   ins_encode %{
     z_store_barrier(masm, this, Address($mem$$Register), $newv$$Register, $prev$$Register, rscratch2, true /* is_atomic */);
     __ atomic_xchgal($prev$$Register, $prev$$Register, $mem$$Register);
-    z_uncolor(masm, this, $prev$$Register);
+    z_uncolor(masm, $prev$$Register);
   %}
 
   ins_pipe(pipe_serial);

src/hotspot/cpu/aarch64/interpreterRT_aarch64.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
  * Copyright (c) 2021, Azul Systems, Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -181,7 +181,7 @@ void InterpreterRuntime::SignatureHandlerGenerator::generate(uint64_t fingerprin
   __ lea(r0, ExternalAddress(Interpreter::result_handler(method()->result_type())));
   __ ret(lr);
 
-  __ flush();
+  __ invalidate_icache();
 }
 
 

src/hotspot/cpu/aarch64/jniFastGetField_aarch64.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2004, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2004, 2026, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -206,7 +206,7 @@ address JNI_FastGetField::generate_fast_get_int_field0(BasicType type) {
     __ leave();
     __ ret(lr);
   }
-  __ flush ();
+  __ invalidate_icache();
 
   return fast_entry;
 }

src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp

@@ -1972,7 +1972,9 @@ void MacroAssembler::verify_secondary_supers_table(Register r_sub_klass,
     mov(r1, r_sub_klass);           // r1 <- r4
     mov(r2, /*expected*/rscratch1); // r2 <- r8
     mov(r3, result);                // r3 <- r5
-    mov(r4, (address)("mismatch")); // r4 <- const
+    const char* msg = "mismatch";
+    const char* str = (code_section()->scratch_emit()) ? msg : AOTCodeCache::add_C_string(msg);
+    lea(r4, ExternalAddress((address)str)); // r4 <- const
     rt_call(CAST_FROM_FN_PTR(address, Klass::on_secondary_supers_verification_failure), rscratch2);
     should_not_reach_here();
   }
@@ -2043,7 +2045,7 @@ void MacroAssembler::_verify_oop(Register reg, const char* s, const char* file,
   stp(rscratch2, lr, Address(pre(sp, -2 * wordSize)));
 
   mov(r0, reg);
-  movptr(rscratch1, (uintptr_t)(address)b);
+  lea(rscratch1, ExternalAddress((address)b));
 
   // call indirectly to solve generation ordering problem
   lea(rscratch2, RuntimeAddress(StubRoutines::verify_oop_subroutine_entry_address()));
@@ -2094,7 +2096,7 @@ void MacroAssembler::_verify_oop_addr(Address addr, const char* s, const char* f
   } else {
     ldr(r0, addr);
   }
-  movptr(rscratch1, (uintptr_t)(address)b);
+  lea(rscratch1, ExternalAddress((address)b));
 
   // call indirectly to solve generation ordering problem
   lea(rscratch2, RuntimeAddress(StubRoutines::verify_oop_subroutine_entry_address()));

src/hotspot/cpu/aarch64/runtime_aarch64.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2014, Red Hat Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -249,8 +249,7 @@ UncommonTrapBlob* OptoRuntime::generate_uncommon_trap_blob() {
   // Jump to interpreter
   __ ret(lr);
 
-  // Make sure all code is generated
-  masm->flush();
+  // Code will be copied. No ICache sync required.
 
   UncommonTrapBlob *ut_blob = UncommonTrapBlob::create(&buffer, oop_maps,
                                                        SimpleRuntimeFrame::framesize >> 1);
@@ -391,8 +390,7 @@ ExceptionBlob* OptoRuntime::generate_exception_blob() {
 
   __ br(r8);
 
-  // Make sure all code is generated
-  masm->flush();
+  // Code will be copied. No ICache sync required.
 
   // Set exception blob
   ExceptionBlob* ex_blob = ExceptionBlob::create(&buffer, oop_maps, SimpleRuntimeFrame::framesize >> 1);
@@ -400,5 +398,3 @@ ExceptionBlob* OptoRuntime::generate_exception_blob() {
   return ex_blob;
 }
 #endif // COMPILER2
-
-

src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp

@@ -1612,7 +1612,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
     assert(vep_offset != -1,        "Must be set");
 #endif
 
-    __ flush();
+    // Code will be copied. No ICache sync required.
     nmethod* nm = nmethod::new_native_nmethod(method,
                                               compile_id,
                                               masm->code(),
@@ -1646,7 +1646,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
                          in_sig_bt,
                          in_regs);
     int frame_complete = ((intptr_t)__ pc()) - start;  // not complete, period
-    __ flush();
+    // Code will be copied. No ICache sync required.
     int stack_slots = SharedRuntime::out_preserve_stack_slots();  // no out slots at all, actually
     return nmethod::new_native_nmethod(method,
                                        compile_id,
@@ -2047,14 +2047,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
 
   Label safepoint_in_progress, safepoint_in_progress_done;
 
-  // Switch thread to "native transition" state before reading the synchronization state.
-  // This additional state is necessary because reading and testing the synchronization
-  // state is not atomic w.r.t. GC, as this scenario demonstrates:
-  //     Java thread A, in _thread_in_native state, loads _not_synchronized and is preempted.
-  //     VM thread changes sync state to synchronizing and suspends threads for GC.
-  //     Thread A is resumed to finish this native method, but doesn't block here since it
-  //     didn't see any synchronization is progress, and escapes.
-  __ mov(rscratch1, _thread_in_native_trans);
+  __ mov(rscratch1, _thread_in_vm);
 
   __ strw(rscratch1, Address(rthread, JavaThread::thread_state_offset()));
 
@@ -2323,7 +2316,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
     }
   }
 
-  __ flush();
+  // Code will be copied. No ICache sync required.
 
   nmethod *nm = nmethod::new_native_nmethod(method,
                                             compile_id,
@@ -2664,8 +2657,7 @@ void SharedRuntime::generate_deopt_blob() {
   // Jump to interpreter
   __ ret(lr);
 
-  // Make sure all code is generated
-  masm->flush();
+  // Code will be copied. No ICache sync required.
 
   _deopt_blob = DeoptimizationBlob::create(&buffer, oop_maps, 0, exception_offset, reexecute_offset, frame_size_in_words);
   _deopt_blob->set_unpack_with_exception_in_tls_offset(exception_in_tls_offset);
@@ -2813,8 +2805,7 @@ SafepointBlob* SharedRuntime::generate_handler_blob(StubId id, address call_ptr)
   __ stop("Attempting to adjust pc to skip safepoint poll but the return point is not what we expected");
 #endif
 
-  // Make sure all code is generated
-  masm->flush();
+  // Code will be copied. No ICache sync required.
 
   // Fill-out other meta info
   SafepointBlob* sp_blob = SafepointBlob::create(&buffer, oop_maps, frame_size_in_words);
@@ -2909,9 +2900,7 @@ RuntimeStub* SharedRuntime::generate_resolve_blob(StubId id, address destination
   __ ldr(r0, Address(rthread, Thread::pending_exception_offset()));
   __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
 
-  // -------------
-  // make sure all code is generated
-  masm->flush();
+  // Code will be copied. No ICache sync required.
 
   // return the  blob
   // frame_size_words or bytes??
@@ -3065,7 +3054,7 @@ BufferedInlineTypeBlob* SharedRuntime::generate_buffered_inline_type_adapter(con
 
   __ ret(lr);
 
-  __ flush();
+  // Code will be copied. No ICache sync required.
 
   return BufferedInlineTypeBlob::create(&buffer, pack_fields_off, pack_fields_jobject_off, unpack_fields_off);
 }
@@ -3316,9 +3305,7 @@ RuntimeStub* SharedRuntime::generate_return_value_stub(address destination) {
   __ leave();
   __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
 
-  // -------------
-  // make sure all code is generated
-  masm->flush();
+  // Code will be copied. No ICache sync required.
 
   RuntimeStub* stub = RuntimeStub::new_runtime_stub(name, &code, frame_complete, frame_size_in_words, oop_maps, false);
   AOTCodeCache::store_code_blob(*stub, AOTCodeEntry::SharedBlob, StubInfo::blob(id));

src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp

@@ -827,7 +827,7 @@ class StubGenerator: public StubCodeGenerator {
     assert(frame::arg_reg_save_area_bytes == 0, "not expecting frame reg save area");
 #endif
     BLOCK_COMMENT("call MacroAssembler::debug");
-    __ mov(rscratch1, CAST_FROM_FN_PTR(address, MacroAssembler::debug64));
+    __ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64)));
     __ blr(rscratch1);
     __ hlt(0);
 
@@ -12826,7 +12826,7 @@ class StubGenerator: public StubCodeGenerator {
     // Native caller has no idea how to handle exceptions,
     // so we just crash here. Up to callee to catch exceptions.
     __ verify_oop(r0);
-    __ movptr(rscratch1, CAST_FROM_FN_PTR(uint64_t, UpcallLinker::handle_uncaught_exception));
+    __ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, UpcallLinker::handle_uncaught_exception)));
     __ blr(rscratch1);
     __ should_not_reach_here();
 

src/hotspot/cpu/aarch64/templateInterpreterGenerator_aarch64.cpp

@@ -1422,7 +1422,7 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) {
   __ verify_sve_vector_length();
 
   // change thread state
-  __ mov(rscratch1, _thread_in_native_trans);
+  __ mov(rscratch1, _thread_in_vm);
   __ lea(rscratch2, Address(rthread, JavaThread::thread_state_offset()));
   __ stlrw(rscratch1, rscratch2);
 

src/hotspot/cpu/aarch64/upcallLinker_aarch64.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2019, 2022, Arm Limited. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -310,7 +310,7 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Symbol* signature,
 
   //////////////////////////////////////////////////////////////////////////////
 
-  _masm->flush();
+  // Code will be copied. No ICache sync required.
 
 #ifndef PRODUCT
   stringStream ss;

src/hotspot/cpu/aarch64/vtableStubs_aarch64.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2014, Red Hat Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -132,7 +132,7 @@ VtableStub* VtableStubs::create_vtable_stub(int vtable_index, bool caller_is_c1)
   __ ldr(rscratch1, Address(rmethod, entry_offset));
   __ br(rscratch1);
 
-  masm->flush();
+  masm->invalidate_icache();
   bookkeeping(masm, tty, s, npe_addr, ame_addr, true, vtable_index, slop_bytes, 0);
 
   return s;
@@ -233,7 +233,7 @@ VtableStub* VtableStubs::create_itable_stub(int itable_index, bool caller_is_c1)
   assert(SharedRuntime::get_handle_wrong_method_stub() != nullptr, "check initialization order");
   __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 
-  masm->flush();
+  masm->invalidate_icache();
   bookkeeping(masm, tty, s, npe_addr, ame_addr, false, itable_index, slop_bytes, 0);
 
   return s;

src/hotspot/cpu/arm/bytes_arm.hpp

@@ -1,180 +0,0 @@
-/*
- * Copyright (c) 2008, 2022, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#ifndef CPU_ARM_BYTES_ARM_HPP
-#define CPU_ARM_BYTES_ARM_HPP
-
-#include "memory/allStatic.hpp"
-#include "utilities/macros.hpp"
-
-#ifndef VM_LITTLE_ENDIAN
-#define VM_LITTLE_ENDIAN  1
-#endif
-
-class Bytes: AllStatic {
-
- public:
-  static inline u2 get_Java_u2(address p) {
-    return (u2(p[0]) << 8) | u2(p[1]);
-  }
-
-  static inline u4 get_Java_u4(address p) {
-    return u4(p[0]) << 24 |
-           u4(p[1]) << 16 |
-           u4(p[2]) <<  8 |
-           u4(p[3]);
-  }
-
-  static inline u8 get_Java_u8(address p) {
-    return u8(p[0]) << 56 |
-           u8(p[1]) << 48 |
-           u8(p[2]) << 40 |
-           u8(p[3]) << 32 |
-           u8(p[4]) << 24 |
-           u8(p[5]) << 16 |
-           u8(p[6]) <<  8 |
-           u8(p[7]);
-  }
-
-  static inline void put_Java_u2(address p, u2 x) {
-    p[0] = x >> 8;
-    p[1] = x;
-  }
-
-  static inline void put_Java_u4(address p, u4 x) {
-    ((u1*)p)[0] = x >> 24;
-    ((u1*)p)[1] = x >> 16;
-    ((u1*)p)[2] = x >>  8;
-    ((u1*)p)[3] = x;
-  }
-
-  static inline void put_Java_u8(address p, u8 x) {
-    ((u1*)p)[0] = x >> 56;
-    ((u1*)p)[1] = x >> 48;
-    ((u1*)p)[2] = x >> 40;
-    ((u1*)p)[3] = x >> 32;
-    ((u1*)p)[4] = x >> 24;
-    ((u1*)p)[5] = x >> 16;
-    ((u1*)p)[6] = x >>  8;
-    ((u1*)p)[7] = x;
-  }
-
-#ifdef VM_LITTLE_ENDIAN
-
-  static inline u2 get_native_u2(address p) {
-    return (intptr_t(p) & 1) == 0 ? *(u2*)p : u2(p[0]) | (u2(p[1]) << 8);
-  }
-
-  static inline u4 get_native_u4(address p) {
-    switch (intptr_t(p) & 3) {
-      case 0:  return *(u4*)p;
-      case 2:  return u4(((u2*)p)[0]) |
-                      u4(((u2*)p)[1]) << 16;
-      default: return u4(p[0])       |
-                      u4(p[1]) <<  8 |
-                      u4(p[2]) << 16 |
-                      u4(p[3]) << 24;
-    }
-  }
-
-  static inline u8 get_native_u8(address p) {
-    switch (intptr_t(p) & 7) {
-      case 0:  return *(u8*)p;
-      case 4:  return u8(((u4*)p)[0]) |
-                      u8(((u4*)p)[1]) << 32;
-      case 2:  return u8(((u2*)p)[0])       |
-                      u8(((u2*)p)[1]) << 16 |
-                      u8(((u2*)p)[2]) << 32 |
-                      u8(((u2*)p)[3]) << 48;
-      default: return u8(p[0])       |
-                      u8(p[1]) <<  8 |
-                      u8(p[2]) << 16 |
-                      u8(p[3]) << 24 |
-                      u8(p[4]) << 32 |
-                      u8(p[5]) << 40 |
-                      u8(p[6]) << 48 |
-                      u8(p[7]) << 56;
-    }
-  }
-
-  static inline void put_native_u2(address p, u2 x) {
-    if ((intptr_t(p) & 1) == 0) {
-      *(u2*)p = x;
-    } else {
-      p[0] = x;
-      p[1] = x >> 8;
-    }
-  }
-
-  static inline void put_native_u4(address p, u4 x) {
-    switch (intptr_t(p) & 3) {
-      case 0:  *(u4*)p = x;
-               break;
-      case 2:  ((u2*)p)[0] = x;
-               ((u2*)p)[1] = x >> 16;
-               break;
-      default: ((u1*)p)[0] = x;
-               ((u1*)p)[1] = x >>  8;
-               ((u1*)p)[2] = x >> 16;
-               ((u1*)p)[3] = x >> 24;
-               break;
-    }
-  }
-
-  static inline void put_native_u8(address p, u8 x) {
-    switch (intptr_t(p) & 7) {
-      case 0:  *(u8*)p = x;
-               break;
-      case 4:  ((u4*)p)[0] = x;
-               ((u4*)p)[1] = x >> 32;
-               break;
-      case 2:  ((u2*)p)[0] = x;
-               ((u2*)p)[1] = x >> 16;
-               ((u2*)p)[2] = x >> 32;
-               ((u2*)p)[3] = x >> 48;
-               break;
-      default: ((u1*)p)[0] = x;
-               ((u1*)p)[1] = x >>  8;
-               ((u1*)p)[2] = x >> 16;
-               ((u1*)p)[3] = x >> 24;
-               ((u1*)p)[4] = x >> 32;
-               ((u1*)p)[5] = x >> 40;
-               ((u1*)p)[6] = x >> 48;
-               ((u1*)p)[7] = x >> 56;
-    }
-  }
-
-#else
-
-  static inline u2 get_native_u2(address p) { return get_Java_u2(p); }
-  static inline u4 get_native_u4(address p) { return get_Java_u4(p); }
-  static inline u8 get_native_u8(address p) { return get_Java_u8(p); }
-  static inline void put_native_u2(address p, u2 x) { put_Java_u2(p, x); }
-  static inline void put_native_u4(address p, u4 x) { put_Java_u4(p, x); }
-  static inline void put_native_u8(address p, u8 x) { put_Java_u8(p, x); }
-
-#endif // VM_LITTLE_ENDIAN
-};
-
-#endif // CPU_ARM_BYTES_ARM_HPP

src/hotspot/cpu/arm/jniFastGetField_arm.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2008, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2008, 2026, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -210,7 +210,7 @@ address JNI_FastGetField::generate_fast_get_int_field0(BasicType type) {
 
   __ bind_literal(safepoint_counter_addr);
 
-  __ flush();
+  __ invalidate_icache();
 
   guarantee((__ pc() - fast_entry) <= BUFFER_SIZE, "BUFFER_SIZE too small");
 

src/hotspot/cpu/arm/macroAssembler_arm.hpp

@@ -449,7 +449,7 @@ public:
   int should_not_call_this() {
     raw_push(FP, LR);
     should_not_reach_here();
-    flush();
+    invalidate_icache();
     return 2; // frame_size_in_words (FP+LR)
   }
 

src/hotspot/cpu/arm/runtime_arm.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2008, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2008, 2026, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -176,7 +176,7 @@ UncommonTrapBlob* OptoRuntime::generate_uncommon_trap_blob() {
   __ mov(SP, FP);
   __ pop(RegisterSet(FP) | RegisterSet(PC));
 
-  masm->flush();
+  masm->invalidate_icache();
   return UncommonTrapBlob::create(&buffer, nullptr, 2 /* LR+FP */);
 }
 
@@ -280,7 +280,7 @@ ExceptionBlob* OptoRuntime::generate_exception_blob() {
 
   // -------------
   // make sure all code is generated
-  masm->flush();
+  masm->invalidate_icache();
 
   return ExceptionBlob::create(&buffer, oop_maps, framesize_in_words);
 }

src/hotspot/cpu/arm/sharedRuntime_arm.cpp

@@ -850,7 +850,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
                          in_sig_bt,
                          in_regs);
     int frame_complete = ((intptr_t)__ pc()) - start;  // not complete, period
-    __ flush();
+    __ invalidate_icache();
     int stack_slots = SharedRuntime::out_preserve_stack_slots();  // no out slots at all, actually
     return nmethod::new_native_nmethod(method,
                                        compile_id,
@@ -1263,9 +1263,9 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
     __ c2bool(R0);
   }
 
-  // Do a safepoint check while thread is in transition state
+  // Do a safepoint check
   Label call_safepoint_runtime, return_to_java;
-  __ mov(Rtemp, _thread_in_native_trans);
+  __ mov(Rtemp, _thread_in_vm);
   __ str_32(Rtemp, Address(Rthread, JavaThread::thread_state_offset()));
 
   // make sure the store is observed before reading the SafepointSynchronize state and further mem refs
@@ -1292,6 +1292,9 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
 
   Label slow_unlock, unlock_done;
   if (method->is_synchronized()) {
+    // Get locked oop from the handle we passed to jni
+    __ ldr(sync_obj, Address(sync_handle));
+
     log_trace(fastlock)("SharedRuntime unlock fast");
     __ fast_unlock(sync_obj, R2 /* t1 */, tmp /* t2 */, Rtemp /* t3 */,
                    7 /* savemask */, slow_unlock);
@@ -1382,7 +1385,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
     __ b(unlock_done);
   }
 
-  __ flush();
+  __ invalidate_icache();
   return nmethod::new_native_nmethod(method,
                                      compile_id,
                                      masm->code(),
@@ -1651,7 +1654,7 @@ void SharedRuntime::generate_deopt_blob() {
 
   __ pop(RegisterSet(FP) | RegisterSet(PC));
 
-  __ flush();
+  __ invalidate_icache();
 
   _deopt_blob = DeoptimizationBlob::create(&buffer, oop_maps, 0, exception_offset,
                                            reexecute_offset, frame_size_in_words);
@@ -1731,7 +1734,7 @@ SafepointBlob* SharedRuntime::generate_handler_blob(StubId id, address call_ptr)
 
   __ jump(StubRoutines::forward_exception_entry(), relocInfo::runtime_call_type, Rtemp);
 
-  __ flush();
+  __ invalidate_icache();
 
   return SafepointBlob::create(&buffer, oop_maps, frame_size_words);
 }
@@ -1791,7 +1794,7 @@ RuntimeStub* SharedRuntime::generate_resolve_blob(StubId id, address destination
   __ mov(Rexception_pc, LR);
   __ jump(StubRoutines::forward_exception_entry(), relocInfo::runtime_call_type, Rtemp);
 
-  __ flush();
+  __ invalidate_icache();
 
   return RuntimeStub::new_runtime_stub(name, &buffer, frame_complete, frame_size_words, oop_maps, true);
 }

src/hotspot/cpu/arm/templateInterpreterGenerator_arm.cpp

@@ -1014,7 +1014,7 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) {
   }
 
   // Do safepoint check
-  __ mov(Rtemp, _thread_in_native_trans);
+  __ mov(Rtemp, _thread_in_vm);
   __ str_32(Rtemp, Address(Rthread, JavaThread::thread_state_offset()));
 
   // Force this write out before the read below

src/hotspot/cpu/arm/vtableStubs_arm.cpp

@@ -110,7 +110,7 @@ VtableStub* VtableStubs::create_vtable_stub(int vtable_index, bool caller_is_c1)
   address ame_addr = __ pc();
   __ ldr(PC, Address(Rmethod, Method::from_compiled_offset()));
 
-  masm->flush();
+  masm->invalidate_icache();
   bookkeeping(masm, tty, s, npe_addr, ame_addr, true, vtable_index, slop_bytes, 0);
 
   return s;
@@ -205,7 +205,7 @@ VtableStub* VtableStubs::create_itable_stub(int itable_index, bool caller_is_c1)
   assert(SharedRuntime::get_handle_wrong_method_stub() != nullptr, "check initialization order");
   __ jump(SharedRuntime::get_handle_wrong_method_stub(), relocInfo::runtime_call_type, Rtemp);
 
-  masm->flush();
+  masm->invalidate_icache();
   bookkeeping(masm, tty, s, npe_addr, ame_addr, false, itable_index, slop_bytes, 0);
 
   return s;

src/hotspot/cpu/ppc/assembler_ppc.hpp

@@ -539,6 +539,10 @@ class Assembler : public AbstractAssembler {
     STXVL_OPCODE   = (31u << OPCODE_SHIFT |  397u << 1),
     LXVD2X_OPCODE  = (31u << OPCODE_SHIFT |  844u << 1),
     STXVD2X_OPCODE = (31u << OPCODE_SHIFT |  972u << 1),
+    LXVW4X_OPCODE  = (31u << OPCODE_SHIFT |  780u << 1),
+    STXVW4X_OPCODE = (31u << OPCODE_SHIFT |  908u << 1),
+    LXVB16X_OPCODE = (31u << OPCODE_SHIFT |  876u << 1),
+    STXVB16X_OPCODE= (31u << OPCODE_SHIFT | 1004u << 1),
     MTVSRD_OPCODE  = (31u << OPCODE_SHIFT |  179u << 1),
     MTVSRDD_OPCODE = (31u << OPCODE_SHIFT |  435u << 1),
     MTVSRWZ_OPCODE = (31u << OPCODE_SHIFT |  243u << 1),
@@ -1365,10 +1369,6 @@ class Assembler : public AbstractAssembler {
     return (0 == addr % a);
   }
 
-  void flush() {
-    AbstractAssembler::flush();
-  }
-
   inline void emit_int32(int);  // shadows AbstractAssembler::emit_int32
   inline void emit_data(int);
   inline void emit_data(int, RelocationHolder const&);
@@ -2386,8 +2386,17 @@ class Assembler : public AbstractAssembler {
   inline void lxvd2x(   VectorSRegister d, Register a, Register b);
   inline void stxvd2x(  VectorSRegister d, Register a);
   inline void stxvd2x(  VectorSRegister d, Register a, Register b);
+  inline void lxvw4x(   VectorSRegister d, Register a);
+  inline void lxvw4x(   VectorSRegister d, Register a, Register b);
+  inline void stxvw4x(  VectorSRegister d, Register a);
+  inline void stxvw4x(  VectorSRegister d, Register a, Register b);
 
   // Power9
+  inline void lxvb16x(  VectorSRegister d, Register a);
+  inline void lxvb16x(  VectorSRegister d, Register a, Register b);
+  inline void stxvb16x( VectorSRegister d, Register a);
+  inline void stxvb16x( VectorSRegister d, Register a, Register b);
+
   inline void lxv(      VectorSRegister d, int si16, Register a);
   inline void stxv(     VectorSRegister d, int si16, Register a);
   inline void lxvx(     VectorSRegister d, Register a, Register b);
@@ -2590,6 +2599,15 @@ class Assembler : public AbstractAssembler {
   inline void vec_perm(VectorRegister first_dest, VectorRegister second, VectorRegister perm);
   inline void vec_perm(VectorRegister dest, VectorRegister first, VectorRegister second, VectorRegister perm);
 
+  // Load/Store unaligned vectors with offs (multiple of 16). Byte versions require vp for Power8 LE.
+  inline void load_byte_vector_unaligned(VectorRegister dest, int offs, Register base, Register tmp,
+                                         VectorRegister vp); // vp should be pre-computed (see generator below)
+  inline void store_byte_vector_unaligned(VectorRegister val, int offs, Register base, Register tmp,
+                                          VectorRegister vp, VectorRegister vtmp = vnoreg); // clobbers val if no vtmp provided
+  inline void compute_vp_for_byte_vector_unaligned(VectorRegister dest, VectorRegister vtmp);
+  inline void load_word_vector_unaligned(VectorRegister dest, int offs, Register base, Register tmp);
+  inline void store_word_vector_unaligned(VectorRegister val, int offs, Register base, Register tmp);
+
   // RegisterOrConstant versions.
   // These emitters choose between the versions using two registers and
   // those with register and immediate, depending on the content of roc.

src/hotspot/cpu/ppc/assembler_ppc.inline.hpp

@@ -856,6 +856,14 @@ inline void Assembler::lxvd2x(  VectorSRegister d, Register s1)              { e
 inline void Assembler::lxvd2x(  VectorSRegister d, Register s1, Register s2) { emit_int32( LXVD2X_OPCODE  | vsrt(d) | ra0mem(s1) | rb(s2)); }
 inline void Assembler::stxvd2x( VectorSRegister d, Register s1)              { emit_int32( STXVD2X_OPCODE | vsrs(d) | ra(0) | rb(s1)); }
 inline void Assembler::stxvd2x( VectorSRegister d, Register s1, Register s2) { emit_int32( STXVD2X_OPCODE | vsrs(d) | ra0mem(s1) | rb(s2)); }
+inline void Assembler::lxvw4x(  VectorSRegister d, Register s1)              { emit_int32( LXVW4X_OPCODE  | vsrt(d) | ra(0) | rb(s1)); }
+inline void Assembler::lxvw4x(  VectorSRegister d, Register s1, Register s2) { emit_int32( LXVW4X_OPCODE  | vsrt(d) | ra0mem(s1) | rb(s2)); }
+inline void Assembler::stxvw4x( VectorSRegister d, Register s1)              { emit_int32( STXVW4X_OPCODE | vsrs(d) | ra(0) | rb(s1)); }
+inline void Assembler::stxvw4x( VectorSRegister d, Register s1, Register s2) { emit_int32( STXVW4X_OPCODE | vsrs(d) | ra0mem(s1) | rb(s2)); }
+inline void Assembler::lxvb16x( VectorSRegister d, Register s1)              { emit_int32( LXVB16X_OPCODE | vsrt(d) | ra(0) | rb(s1)); }
+inline void Assembler::lxvb16x( VectorSRegister d, Register s1, Register s2) { emit_int32( LXVB16X_OPCODE | vsrt(d) | ra0mem(s1) | rb(s2)); }
+inline void Assembler::stxvb16x(VectorSRegister d, Register s1)              { emit_int32( STXVB16X_OPCODE| vsrs(d) | ra(0) | rb(s1)); }
+inline void Assembler::stxvb16x(VectorSRegister d, Register s1, Register s2) { emit_int32( STXVB16X_OPCODE| vsrs(d) | ra0mem(s1) | rb(s2)); }
 inline void Assembler::mtvsrd(  VectorSRegister d, Register a)               { emit_int32( MTVSRD_OPCODE  | vsrt(d)  | ra(a)); }
 inline void Assembler::mtvsrdd( VectorSRegister d, Register a, Register b)   { emit_int32( MTVSRDD_OPCODE | vsrt(d)  | ra(a) | rb(b)); }
 inline void Assembler::mfvsrd(  Register d, VectorSRegister a)               { emit_int32( MFVSRD_OPCODE  | vsrs(a)  | ra(d)); }
@@ -1232,6 +1240,108 @@ inline void Assembler::vec_perm(VectorRegister dest, VectorRegister first, Vecto
 #endif
 }
 
+inline void Assembler::load_byte_vector_unaligned(VectorRegister dest, int offs, Register base, Register tmp,
+                                                  VectorRegister vp) {
+  VectorSRegister vsr = dest->to_vsr();
+  if (PowerArchitecturePPC64 >= 9) {
+#if !defined(VM_LITTLE_ENDIAN)
+    lxv(vsr, offs, base); // all vector load/store instructions use the same byte order on BE
+#else
+    if (offs == 0) {
+      lxvb16x(vsr, base);
+    } else {
+      li(tmp, offs);
+      lxvb16x(vsr, base, tmp);
+    }
+#endif
+  } else { // Power8 only supports very limited instructions
+    if (offs == 0) {
+      lxvd2x(vsr, base);
+    } else {
+      li(tmp, offs);
+      lxvd2x(vsr, base, tmp);
+    }
+#if defined(VM_LITTLE_ENDIAN)
+    // need to swap bytes in both double-words
+    vperm(dest, dest, dest, vp);
+#endif
+  }
+}
+
+inline void Assembler::store_byte_vector_unaligned(VectorRegister val, int offs, Register base, Register tmp,
+                                                   VectorRegister vp, VectorRegister vtmp) {
+  VectorSRegister vsr = val->to_vsr();
+  if (PowerArchitecturePPC64 >= 9) {
+#if !defined(VM_LITTLE_ENDIAN)
+    stxv(vsr, offs, base); // all vector load/store instructions use the same byte order on BE
+#else
+    if (offs == 0) {
+      stxvb16x(vsr, base);
+    } else {
+      li(tmp, offs);
+      stxvb16x(vsr, base, tmp);
+    }
+#endif
+  } else { // Power8 only supports very limited instructions
+#if defined(VM_LITTLE_ENDIAN)
+    // need to swap bytes in both double-words
+    if (vtmp != vnoreg) {
+      vperm(vtmp, val, val, vp);
+      vsr = vtmp->to_vsr();
+    } else {
+      vperm(val, val, val, vp); // clobbers val!
+    }
+#endif
+    if (offs == 0) {
+      stxvd2x(vsr, base);
+    } else {
+      li(tmp, offs);
+      stxvd2x(vsr, base, tmp);
+    }
+  }
+}
+
+inline void Assembler::compute_vp_for_byte_vector_unaligned(VectorRegister dest, VectorRegister vtmp) {
+#if defined(VM_LITTLE_ENDIAN)
+  if (PowerArchitecturePPC64 < 9) {
+    li(R0, 0);
+    vspltisb(vtmp, 7);      // vtmp = [7, ..., 7]
+    lvsl(dest, R0);         // dest = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]
+    vxor(dest, dest, vtmp); // dest = [7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8]
+  }
+#endif
+}
+
+inline void Assembler::load_word_vector_unaligned(VectorRegister dest, int offs, Register base, Register tmp) {
+  VectorSRegister vsr = dest->to_vsr();
+#if !defined(VM_LITTLE_ENDIAN)
+  if (PowerArchitecturePPC64 >= 9) {
+    lxv(vsr, offs, base); // all vector load/store instructions use the same byte order on BE
+  } else
+#endif
+  if (offs == 0) {
+    lxvw4x(vsr, base);
+  } else {
+    li(tmp, offs);
+    lxvw4x(vsr, base, tmp);
+  }
+}
+
+inline void Assembler::store_word_vector_unaligned(VectorRegister val, int offs, Register base, Register tmp) {
+  VectorSRegister vsr = val->to_vsr();
+#if !defined(VM_LITTLE_ENDIAN)
+  if (PowerArchitecturePPC64 >= 9) {
+    stxv(vsr, offs, base); // all vector load/store instructions use the same byte order on BE
+  } else
+#endif
+  if (offs == 0) {
+    stxvw4x(vsr, base);
+  } else {
+    li(tmp, offs);
+    stxvw4x(vsr, base, tmp);
+  }
+}
+
 inline void Assembler::load_const(Register d, void* x, Register tmp) {
    load_const(d, (long)x, tmp);
 }

src/hotspot/cpu/ppc/bytes_ppc.hpp

@@ -1,260 +0,0 @@
-/*
- * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2022 SAP SE. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#ifndef CPU_PPC_BYTES_PPC_HPP
-#define CPU_PPC_BYTES_PPC_HPP
-
-#include "memory/allStatic.hpp"
-#include "utilities/byteswap.hpp"
-
-class Bytes: AllStatic {
- public:
-  // Efficient reading and writing of unaligned unsigned data in platform-specific byte ordering
-  // PowerPC needs to check for alignment.
-
-  // Can I count on address always being a pointer to an unsigned char? Yes.
-
-#if defined(VM_LITTLE_ENDIAN)
-
-  static inline u2   get_native_u2(address p) {
-    return (intptr_t(p) & 1) == 0
-             ?   *(u2*)p
-             :   ( u2(p[1]) << 8 )
-               | ( u2(p[0])      );
-  }
-
-  static inline u4   get_native_u4(address p) {
-    switch (intptr_t(p) & 3) {
-     case 0:  return *(u4*)p;
-
-     case 2:  return (  u4( ((u2*)p)[1] ) << 16  )
-                   | (  u4( ((u2*)p)[0] )        );
-
-    default:  return ( u4(p[3]) << 24 )
-                   | ( u4(p[2]) << 16 )
-                   | ( u4(p[1]) <<  8 )
-                   |   u4(p[0]);
-    }
-  }
-
-  static inline u8   get_native_u8(address p) {
-    switch (intptr_t(p) & 7) {
-      case 0:  return *(u8*)p;
-
-      case 4:  return (  u8( ((u4*)p)[1] ) << 32  )
-                    | (  u8( ((u4*)p)[0] )        );
-
-      case 2:  return (  u8( ((u2*)p)[3] ) << 48  )
-                    | (  u8( ((u2*)p)[2] ) << 32  )
-                    | (  u8( ((u2*)p)[1] ) << 16  )
-                    | (  u8( ((u2*)p)[0] )        );
-
-     default:  return ( u8(p[7]) << 56 )
-                    | ( u8(p[6]) << 48 )
-                    | ( u8(p[5]) << 40 )
-                    | ( u8(p[4]) << 32 )
-                    | ( u8(p[3]) << 24 )
-                    | ( u8(p[2]) << 16 )
-                    | ( u8(p[1]) <<  8 )
-                    |   u8(p[0]);
-    }
-  }
-
-
-
-  static inline void put_native_u2(address p, u2 x) {
-    if ( (intptr_t(p) & 1) == 0 )  *(u2*)p = x;
-    else {
-      p[1] = x >> 8;
-      p[0] = x;
-    }
-  }
-
-  static inline void put_native_u4(address p, u4 x) {
-    switch ( intptr_t(p) & 3 ) {
-    case 0:  *(u4*)p = x;
-              break;
-
-    case 2:  ((u2*)p)[1] = x >> 16;
-             ((u2*)p)[0] = x;
-             break;
-
-    default: ((u1*)p)[3] = x >> 24;
-             ((u1*)p)[2] = x >> 16;
-             ((u1*)p)[1] = x >>  8;
-             ((u1*)p)[0] = x;
-             break;
-    }
-  }
-
-  static inline void put_native_u8(address p, u8 x) {
-    switch ( intptr_t(p) & 7 ) {
-    case 0:  *(u8*)p = x;
-             break;
-
-    case 4:  ((u4*)p)[1] = x >> 32;
-             ((u4*)p)[0] = x;
-             break;
-
-    case 2:  ((u2*)p)[3] = x >> 48;
-             ((u2*)p)[2] = x >> 32;
-             ((u2*)p)[1] = x >> 16;
-             ((u2*)p)[0] = x;
-             break;
-
-    default: ((u1*)p)[7] = x >> 56;
-             ((u1*)p)[6] = x >> 48;
-             ((u1*)p)[5] = x >> 40;
-             ((u1*)p)[4] = x >> 32;
-             ((u1*)p)[3] = x >> 24;
-             ((u1*)p)[2] = x >> 16;
-             ((u1*)p)[1] = x >>  8;
-             ((u1*)p)[0] = x;
-    }
-  }
-
-  // Efficient reading and writing of unaligned unsigned data in Java byte ordering (i.e. big-endian ordering)
-  // (no byte-order reversal is needed since Power CPUs are big-endian oriented).
-  static inline u2   get_Java_u2(address p) { return byteswap(get_native_u2(p)); }
-  static inline u4   get_Java_u4(address p) { return byteswap(get_native_u4(p)); }
-  static inline u8   get_Java_u8(address p) { return byteswap(get_native_u8(p)); }
-
-  static inline void put_Java_u2(address p, u2 x)     { put_native_u2(p, byteswap(x)); }
-  static inline void put_Java_u4(address p, u4 x)     { put_native_u4(p, byteswap(x)); }
-  static inline void put_Java_u8(address p, u8 x)     { put_native_u8(p, byteswap(x)); }
-
-#else // !defined(VM_LITTLE_ENDIAN)
-
-  static inline u2   get_native_u2(address p) {
-    return (intptr_t(p) & 1) == 0
-             ?   *(u2*)p
-             :   ( u2(p[0]) << 8 )
-               | ( u2(p[1])      );
-  }
-
-  static inline u4   get_native_u4(address p) {
-    switch (intptr_t(p) & 3) {
-     case 0:  return *(u4*)p;
-
-     case 2:  return (  u4( ((u2*)p)[0] ) << 16  )
-                   | (  u4( ((u2*)p)[1] )        );
-
-    default:  return ( u4(p[0]) << 24 )
-                   | ( u4(p[1]) << 16 )
-                   | ( u4(p[2]) <<  8 )
-                   |   u4(p[3]);
-    }
-  }
-
-  static inline u8   get_native_u8(address p) {
-    switch (intptr_t(p) & 7) {
-      case 0:  return *(u8*)p;
-
-      case 4:  return (  u8( ((u4*)p)[0] ) << 32  )
-                    | (  u8( ((u4*)p)[1] )        );
-
-      case 2:  return (  u8( ((u2*)p)[0] ) << 48  )
-                    | (  u8( ((u2*)p)[1] ) << 32  )
-                    | (  u8( ((u2*)p)[2] ) << 16  )
-                    | (  u8( ((u2*)p)[3] )        );
-
-     default:  return ( u8(p[0]) << 56 )
-                    | ( u8(p[1]) << 48 )
-                    | ( u8(p[2]) << 40 )
-                    | ( u8(p[3]) << 32 )
-                    | ( u8(p[4]) << 24 )
-                    | ( u8(p[5]) << 16 )
-                    | ( u8(p[6]) <<  8 )
-                    |   u8(p[7]);
-    }
-  }
-
-
-
-  static inline void put_native_u2(address p, u2 x) {
-    if ( (intptr_t(p) & 1) == 0 ) { *(u2*)p = x; }
-    else {
-      p[0] = x >> 8;
-      p[1] = x;
-    }
-  }
-
-  static inline void put_native_u4(address p, u4 x) {
-    switch ( intptr_t(p) & 3 ) {
-    case 0:  *(u4*)p = x;
-              break;
-
-    case 2:  ((u2*)p)[0] = x >> 16;
-             ((u2*)p)[1] = x;
-             break;
-
-    default: ((u1*)p)[0] = x >> 24;
-             ((u1*)p)[1] = x >> 16;
-             ((u1*)p)[2] = x >>  8;
-             ((u1*)p)[3] = x;
-             break;
-    }
-  }
-
-  static inline void put_native_u8(address p, u8 x) {
-    switch ( intptr_t(p) & 7 ) {
-    case 0:  *(u8*)p = x;
-             break;
-
-    case 4:  ((u4*)p)[0] = x >> 32;
-             ((u4*)p)[1] = x;
-             break;
-
-    case 2:  ((u2*)p)[0] = x >> 48;
-             ((u2*)p)[1] = x >> 32;
-             ((u2*)p)[2] = x >> 16;
-             ((u2*)p)[3] = x;
-             break;
-
-    default: ((u1*)p)[0] = x >> 56;
-             ((u1*)p)[1] = x >> 48;
-             ((u1*)p)[2] = x >> 40;
-             ((u1*)p)[3] = x >> 32;
-             ((u1*)p)[4] = x >> 24;
-             ((u1*)p)[5] = x >> 16;
-             ((u1*)p)[6] = x >>  8;
-             ((u1*)p)[7] = x;
-    }
-  }
-
-  // Efficient reading and writing of unaligned unsigned data in Java byte ordering (i.e. big-endian ordering)
-  // (no byte-order reversal is needed since Power CPUs are big-endian oriented).
-  static inline u2   get_Java_u2(address p) { return get_native_u2(p); }
-  static inline u4   get_Java_u4(address p) { return get_native_u4(p); }
-  static inline u8   get_Java_u8(address p) { return get_native_u8(p); }
-
-  static inline void put_Java_u2(address p, u2 x)     { put_native_u2(p, x); }
-  static inline void put_Java_u4(address p, u4 x)     { put_native_u4(p, x); }
-  static inline void put_Java_u8(address p, u8 x)     { put_native_u8(p, x); }
-
-#endif // VM_LITTLE_ENDIAN
-};
-
-#endif // CPU_PPC_BYTES_PPC_HPP

src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp

@@ -2250,7 +2250,7 @@ void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
 
 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
   LP64_ONLY( __ extsw(op->len()->as_register(), op->len()->as_register()); )
-  if (UseSlowPath ||
+  if (UseSlowPath || op->always_slow_path() ||
       (!UseFastNewObjectArray && (is_reference_type(op->type()))) ||
       (!UseFastNewTypeArray   && (!is_reference_type(op->type())))) {
     __ b(*op->stub()->entry());

src/hotspot/cpu/ppc/downcallLinker_ppc.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, 2025 SAP SE. All rights reserved.
+ * Copyright (c) 2020, 2026 SAP SE. All rights reserved.
  * Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -297,7 +297,7 @@ void DowncallLinker::StubGenerator::generate() {
   Label L_after_reguard;
 
   if (_needs_transition) {
-    __ li(tmp, _thread_in_native_trans);
+    __ li(tmp, _thread_in_vm);
     __ release();
     __ stw(tmp, in_bytes(JavaThread::thread_state_offset()), R16_thread);
     if (!UseSystemMemoryBarrier) {
@@ -374,5 +374,5 @@ void DowncallLinker::StubGenerator::generate() {
 
   //////////////////////////////////////////////////////////////////////////////
 
-  __ flush();
+  // Code will be copied. No ICache sync required.
 }

src/hotspot/cpu/ppc/interpreterRT_ppc.cpp

@@ -1,6 +1,6 @@
 /*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2025 SAP SE. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2026 SAP SE. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -127,7 +127,7 @@ void InterpreterRuntime::SignatureHandlerGenerator::generate(uint64_t fingerprin
   __ load_const(R3_RET, AbstractInterpreter::result_handler(method()->result_type()));
   __ blr();
 
-  __ flush();
+  __ invalidate_icache();
 }
 
 #undef __

src/hotspot/cpu/ppc/jniFastGetField_ppc.cpp

@@ -154,7 +154,7 @@ address JNI_FastGetField::generate_fast_get_int_field0(BasicType type) {
   __ load_const_optimized(R12, slow_case_addr, R0);
   __ call_c_and_return_to_caller(R12); // tail call
 
-  __ flush();
+  __ invalidate_icache();
 
   return fast_entry;
 }

src/hotspot/cpu/ppc/macroAssembler_ppc_sha.cpp

@@ -613,7 +613,6 @@ void MacroAssembler::sha512_update_sha_state(const Register state,
   VectorRegister ini_e = VR14;
   VectorRegister ini_g = VR16;
   static const VectorRegister inis[] = {ini_a, ini_c, ini_e, ini_g};
-  static const int total_inis = sizeof(inis)/sizeof(VectorRegister);
 
   Label state_save_aligned, after_state_save_aligned;
 

src/hotspot/cpu/ppc/runtime_ppc.cpp

@@ -1,6 +1,6 @@
 /*
- * Copyright (c) 1998, 2025, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2025 SAP SE. All rights reserved.
+ * Copyright (c) 1998, 2026, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2026 SAP SE. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -141,8 +141,7 @@ ExceptionBlob* OptoRuntime::generate_exception_blob() {
   __ mtlr(R4_ARG2);
   __ bctr();
 
-  // Make sure all code is generated.
-  masm->flush();
+  // Code will be copied. No ICache sync required.
 
   // Set exception blob.
   return ExceptionBlob::create(&buffer, oop_maps,

src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp

@@ -2169,7 +2169,7 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm,
     assert(vep_offset != -1,        "Must be set");
 #endif
 
-    __ flush();
+    // Code will be copied. No ICache sync required.
     nmethod* nm = nmethod::new_native_nmethod(method,
                                               compile_id,
                                               masm->code(),
@@ -2198,7 +2198,7 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm,
                          in_sig_bt,
                          in_regs);
     int frame_complete = ((intptr_t)__ pc()) - start;  // not complete, period
-    __ flush();
+    // Code will be copied. No ICache sync required.
     int stack_slots = SharedRuntime::out_preserve_stack_slots();  // no out slots at all, actually
     return nmethod::new_native_nmethod(method,
                                        compile_id,
@@ -2617,21 +2617,8 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm,
   }
 
   // Publish thread state
-  // --------------------------------------------------------------------------
-
-  // Switch thread to "native transition" state before reading the
-  // synchronization state. This additional state is necessary because reading
-  // and testing the synchronization state is not atomic w.r.t. GC, as this
-  // scenario demonstrates:
-  //   - Java thread A, in _thread_in_native state, loads _not_synchronized
-  //     and is preempted.
-  //   - VM thread changes sync state to synchronizing and suspends threads
-  //     for GC.
-  //   - Thread A is resumed to finish this native method, but doesn't block
-  //     here since it didn't see any synchronization in progress, and escapes.
-
-  // Transition from _thread_in_native to _thread_in_native_trans.
-  __ li(R0, _thread_in_native_trans);
+  // Transition from _thread_in_native to _thread_in_vm.
+  __ li(R0, _thread_in_vm);
   __ release();
   // TODO: PPC port assert(4 == JavaThread::sz_thread_state(), "unexpected field size");
   __ stw(R0, thread_(thread_state));
@@ -2682,10 +2669,10 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm,
     // Publish thread state.
     // --------------------------------------------------------------------------
 
-    // Thread state is thread_in_native_trans. Any safepoint blocking has
+    // Thread state is _thread_in_vm. Any safepoint blocking has
     // already happened so we can now change state to _thread_in_Java.
 
-    // Transition from _thread_in_native_trans to _thread_in_Java.
+    // Transition from _thread_in_vm to _thread_in_Java.
     __ li(R0, _thread_in_Java);
     __ lwsync(); // Acquire safepoint and suspend state, release thread state.
     // TODO: PPC port assert(4 == JavaThread::sz_thread_state(), "unexpected field size");
@@ -2849,7 +2836,7 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm,
   // Done.
   // --------------------------------------------------------------------------
 
-  __ flush();
+  // Code will be copied. No ICache sync required.
 
   nmethod *nm = nmethod::new_native_nmethod(method,
                                             compile_id,
@@ -3216,8 +3203,7 @@ void SharedRuntime::generate_deopt_blob() {
   __ unimplemented("deopt blob needed only with compiler");
 #endif
 
-  // Make sure all code is generated
-  __ flush();
+  // Code will be copied. No ICache sync required.
 
   _deopt_blob = DeoptimizationBlob::create(&buffer, oop_maps, 0, exception_offset,
                                            reexecute_offset, first_frame_size_in_bytes / wordSize);
@@ -3354,7 +3340,7 @@ UncommonTrapBlob* OptoRuntime::generate_uncommon_trap_blob() {
   // Return to the interpreter entry point.
   __ blr();
 
-  masm->flush();
+  // Code will be copied. No ICache sync required.
 
   return UncommonTrapBlob::create(&buffer, oop_maps, frame_size_in_bytes/wordSize);
 }
@@ -3460,8 +3446,7 @@ SafepointBlob* SharedRuntime::generate_handler_blob(StubId id, address call_ptr)
 
   __ blr();
 
-  // Make sure all code is generated
-  masm->flush();
+  // Code will be copied. No ICache sync required.
 
   // Fill-out other meta info
   // CodeBlob frame size is in words.
@@ -3547,9 +3532,7 @@ RuntimeStub* SharedRuntime::generate_resolve_blob(StubId id, address destination
   __ std(R11_scratch1, in_bytes(JavaThread::vm_result_oop_offset()), R16_thread);
   __ b64_patchable(StubRoutines::forward_exception_entry(), relocInfo::runtime_call_type);
 
-  // -------------
-  // Make sure all code is generated.
-  masm->flush();
+  // Code will be copied. No ICache sync required.
 
   // return the blob
   // frame_size_words or bytes??

src/hotspot/cpu/ppc/stubGenerator_ppc.cpp

@@ -2781,10 +2781,8 @@ class StubGenerator: public StubCodeGenerator {
     Register to             = R4_ARG2;  // destination array address
     Register key            = R5_ARG3;  // round key array
 
-    Register keylen         = R8;
-    Register temp           = R9;
-    Register keypos         = R10;
-    Register fifteen        = R12;
+    Register keylen         = R6;
+    Register tmp            = R7;
 
     VectorRegister vRet     = VR0;
 
@@ -2793,68 +2791,27 @@ class StubGenerator: public StubCodeGenerator {
     VectorRegister vKey3    = VR3;
     VectorRegister vKey4    = VR4;
 
-    VectorRegister fromPerm = VR5;
-    VectorRegister keyPerm  = VR6;
-    VectorRegister toPerm   = VR7;
-    VectorRegister fSplt    = VR8;
+    VectorRegister vp       = VR6;  // permute vector for byte vector accesses on P8 LE
 
-    VectorRegister vTmp1    = VR9;
-    VectorRegister vTmp2    = VR10;
-    VectorRegister vTmp3    = VR11;
-    VectorRegister vTmp4    = VR12;
-
-    __ li              (fifteen, 15);
+    __ compute_vp_for_byte_vector_unaligned(vp, /*temp*/ vRet);
 
     // load unaligned from[0-15] to vRet
-    __ lvx             (vRet, from);
-    __ lvx             (vTmp1, fifteen, from);
-    __ lvsl            (fromPerm, from);
-#ifdef VM_LITTLE_ENDIAN
-    __ vspltisb        (fSplt, 0x0f);
-    __ vxor            (fromPerm, fromPerm, fSplt);
-#endif
-    __ vperm           (vRet, vRet, vTmp1, fromPerm);
+    __ load_byte_vector_unaligned(vRet, 0, from, tmp, vp);
+
+    // load the 1st round key to vKey1
+    __ load_word_vector_unaligned(vKey1, 0, key, tmp);
 
     // load keylen (44 or 52 or 60)
     __ lwz             (keylen, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT), key);
 
-    // to load keys
-    __ load_perm       (keyPerm, key);
-#ifdef VM_LITTLE_ENDIAN
-    __ vspltisb        (vTmp2, -16);
-    __ vrld            (keyPerm, keyPerm, vTmp2);
-    __ vrld            (keyPerm, keyPerm, vTmp2);
-    __ vsldoi          (keyPerm, keyPerm, keyPerm, 8);
-#endif
-
-    // load the 1st round key to vTmp1
-    __ lvx             (vTmp1, key);
-    __ li              (keypos, 16);
-    __ lvx             (vKey1, keypos, key);
-    __ vec_perm        (vTmp1, vKey1, keyPerm);
-
     // 1st round
-    __ vxor            (vRet, vRet, vTmp1);
+    __ vxor            (vRet, vRet, vKey1);
 
-    // load the 2nd round key to vKey1
-    __ li              (keypos, 32);
-    __ lvx             (vKey2, keypos, key);
-    __ vec_perm        (vKey1, vKey2, keyPerm);
-
-    // load the 3rd round key to vKey2
-    __ li              (keypos, 48);
-    __ lvx             (vKey3, keypos, key);
-    __ vec_perm        (vKey2, vKey3, keyPerm);
-
-    // load the 4th round key to vKey3
-    __ li              (keypos, 64);
-    __ lvx             (vKey4, keypos, key);
-    __ vec_perm        (vKey3, vKey4, keyPerm);
-
-    // load the 5th round key to vKey4
-    __ li              (keypos, 80);
-    __ lvx             (vTmp1, keypos, key);
-    __ vec_perm        (vKey4, vTmp1, keyPerm);
+    // load the 2nd - 5th round key to vKey1 - vKey4
+    __ load_word_vector_unaligned(vKey1, 16, key, tmp);
+    __ load_word_vector_unaligned(vKey2, 32, key, tmp);
+    __ load_word_vector_unaligned(vKey3, 48, key, tmp);
+    __ load_word_vector_unaligned(vKey4, 64, key, tmp);
 
     // 2nd - 5th rounds
     __ vcipher         (vRet, vRet, vKey1);
@@ -2862,25 +2819,11 @@ class StubGenerator: public StubCodeGenerator {
     __ vcipher         (vRet, vRet, vKey3);
     __ vcipher         (vRet, vRet, vKey4);
 
-    // load the 6th round key to vKey1
-    __ li              (keypos, 96);
-    __ lvx             (vKey2, keypos, key);
-    __ vec_perm        (vKey1, vTmp1, vKey2, keyPerm);
-
-    // load the 7th round key to vKey2
-    __ li              (keypos, 112);
-    __ lvx             (vKey3, keypos, key);
-    __ vec_perm        (vKey2, vKey3, keyPerm);
-
-    // load the 8th round key to vKey3
-    __ li              (keypos, 128);
-    __ lvx             (vKey4, keypos, key);
-    __ vec_perm        (vKey3, vKey4, keyPerm);
-
-    // load the 9th round key to vKey4
-    __ li              (keypos, 144);
-    __ lvx             (vTmp1, keypos, key);
-    __ vec_perm        (vKey4, vTmp1, keyPerm);
+    // load the 6th - 9th round key to vKey1 - vKey4
+    __ load_word_vector_unaligned(vKey1, 80, key, tmp);
+    __ load_word_vector_unaligned(vKey2, 96, key, tmp);
+    __ load_word_vector_unaligned(vKey3, 112, key, tmp);
+    __ load_word_vector_unaligned(vKey4, 128, key, tmp);
 
     // 6th - 9th rounds
     __ vcipher         (vRet, vRet, vKey1);
@@ -2888,15 +2831,9 @@ class StubGenerator: public StubCodeGenerator {
     __ vcipher         (vRet, vRet, vKey3);
     __ vcipher         (vRet, vRet, vKey4);
 
-    // load the 10th round key to vKey1
-    __ li              (keypos, 160);
-    __ lvx             (vKey2, keypos, key);
-    __ vec_perm        (vKey1, vTmp1, vKey2, keyPerm);
-
-    // load the 11th round key to vKey2
-    __ li              (keypos, 176);
-    __ lvx             (vTmp1, keypos, key);
-    __ vec_perm        (vKey2, vTmp1, keyPerm);
+    // load the 10th - 11th round key to vKey1 - vKey2
+    __ load_word_vector_unaligned(vKey1, 144, key, tmp);
+    __ load_word_vector_unaligned(vKey2, 160, key, tmp);
 
     // if all round keys are loaded, skip next 4 rounds
     __ cmpwi           (CR0, keylen, 44);
@@ -2906,15 +2843,9 @@ class StubGenerator: public StubCodeGenerator {
     __ vcipher         (vRet, vRet, vKey1);
     __ vcipher         (vRet, vRet, vKey2);
 
-    // load the 12th round key to vKey1
-    __ li              (keypos, 192);
-    __ lvx             (vKey2, keypos, key);
-    __ vec_perm        (vKey1, vTmp1, vKey2, keyPerm);
-
-    // load the 13th round key to vKey2
-    __ li              (keypos, 208);
-    __ lvx             (vTmp1, keypos, key);
-    __ vec_perm        (vKey2, vTmp1, keyPerm);
+    // load the 12th - 13th round key to vKey1 - vKey2
+    __ load_word_vector_unaligned(vKey1, 176, key, tmp);
+    __ load_word_vector_unaligned(vKey2, 192, key, tmp);
 
     // if all round keys are loaded, skip next 2 rounds
     __ cmpwi           (CR0, keylen, 52);
@@ -2929,15 +2860,9 @@ class StubGenerator: public StubCodeGenerator {
     __ vcipher         (vRet, vRet, vKey1);
     __ vcipher         (vRet, vRet, vKey2);
 
-    // load the 14th round key to vKey1
-    __ li              (keypos, 224);
-    __ lvx             (vKey2, keypos, key);
-    __ vec_perm        (vKey1, vTmp1, vKey2, keyPerm);
-
-    // load the 15th round key to vKey2
-    __ li              (keypos, 240);
-    __ lvx             (vTmp1, keypos, key);
-    __ vec_perm        (vKey2, vTmp1, keyPerm);
+    // load the 14th - 15th round key to vKey1 - vKey2
+    __ load_word_vector_unaligned(vKey1, 208, key, tmp);
+    __ load_word_vector_unaligned(vKey2, 224, key, tmp);
 
     __ bind(L_doLast);
 
@@ -2945,23 +2870,8 @@ class StubGenerator: public StubCodeGenerator {
     __ vcipher         (vRet, vRet, vKey1);
     __ vcipherlast     (vRet, vRet, vKey2);
 
-#ifdef VM_LITTLE_ENDIAN
-    // toPerm = 0x0F0E0D0C0B0A09080706050403020100
-    __ lvsl            (toPerm, keypos); // keypos is a multiple of 16
-    __ vxor            (toPerm, toPerm, fSplt);
-
-    // Swap Bytes
-    __ vperm           (vRet, vRet, vRet, toPerm);
-#endif
-
     // store result (unaligned)
-    // Note: We can't use a read-modify-write sequence which touches additional Bytes.
-    Register lo = temp, hi = fifteen; // Reuse
-    __ vsldoi          (vTmp1, vRet, vRet, 8);
-    __ mfvrd           (hi, vRet);
-    __ mfvrd           (lo, vTmp1);
-    __ std             (hi, 0 LITTLE_ENDIAN_ONLY(+ 8), to);
-    __ std             (lo, 0 BIG_ENDIAN_ONLY(+ 8), to);
+    __ store_byte_vector_unaligned(vRet, 0, to, tmp, vp);
 
     __ blr();
 
@@ -2989,10 +2899,8 @@ class StubGenerator: public StubCodeGenerator {
     Register to             = R4_ARG2;  // destination array address
     Register key            = R5_ARG3;  // round key array
 
-    Register keylen         = R8;
-    Register temp           = R9;
-    Register keypos         = R10;
-    Register fifteen        = R12;
+    Register keylen         = R6;
+    Register tmp            = R7;
 
     VectorRegister vRet     = VR0;
 
@@ -3002,41 +2910,16 @@ class StubGenerator: public StubCodeGenerator {
     VectorRegister vKey4    = VR4;
     VectorRegister vKey5    = VR5;
 
-    VectorRegister fromPerm = VR6;
-    VectorRegister keyPerm  = VR7;
-    VectorRegister toPerm   = VR8;
-    VectorRegister fSplt    = VR9;
+    VectorRegister vp       = VR6;  // permute vector for byte vector accesses on P8 LE
 
-    VectorRegister vTmp1    = VR10;
-    VectorRegister vTmp2    = VR11;
-    VectorRegister vTmp3    = VR12;
-    VectorRegister vTmp4    = VR13;
-
-    __ li              (fifteen, 15);
+    __ compute_vp_for_byte_vector_unaligned(vp, /*temp*/ vRet);
 
     // load unaligned from[0-15] to vRet
-    __ lvx             (vRet, from);
-    __ lvx             (vTmp1, fifteen, from);
-    __ lvsl            (fromPerm, from);
-#ifdef VM_LITTLE_ENDIAN
-    __ vspltisb        (fSplt, 0x0f);
-    __ vxor            (fromPerm, fromPerm, fSplt);
-#endif
-    __ vperm           (vRet, vRet, vTmp1, fromPerm); // align [and byte swap in LE]
+    __ load_byte_vector_unaligned(vRet, 0, from, tmp, vp);
 
     // load keylen (44 or 52 or 60)
     __ lwz             (keylen, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT), key);
 
-    // to load keys
-    __ load_perm       (keyPerm, key);
-#ifdef VM_LITTLE_ENDIAN
-    __ vxor            (vTmp2, vTmp2, vTmp2);
-    __ vspltisb        (vTmp2, -16);
-    __ vrld            (keyPerm, keyPerm, vTmp2);
-    __ vrld            (keyPerm, keyPerm, vTmp2);
-    __ vsldoi          (keyPerm, keyPerm, keyPerm, 8);
-#endif
-
     __ cmpwi           (CR0, keylen, 44);
     __ beq             (CR0, L_do44);
 
@@ -3048,32 +2931,12 @@ class StubGenerator: public StubCodeGenerator {
     __ bne             (CR0, L_error);
 #endif
 
-    // load the 15th round key to vKey1
-    __ li              (keypos, 240);
-    __ lvx             (vKey1, keypos, key);
-    __ li              (keypos, 224);
-    __ lvx             (vKey2, keypos, key);
-    __ vec_perm        (vKey1, vKey2, vKey1, keyPerm);
-
-    // load the 14th round key to vKey2
-    __ li              (keypos, 208);
-    __ lvx             (vKey3, keypos, key);
-    __ vec_perm        (vKey2, vKey3, vKey2, keyPerm);
-
-    // load the 13th round key to vKey3
-    __ li              (keypos, 192);
-    __ lvx             (vKey4, keypos, key);
-    __ vec_perm        (vKey3, vKey4, vKey3, keyPerm);
-
-    // load the 12th round key to vKey4
-    __ li              (keypos, 176);
-    __ lvx             (vKey5, keypos, key);
-    __ vec_perm        (vKey4, vKey5, vKey4, keyPerm);
-
-    // load the 11th round key to vKey5
-    __ li              (keypos, 160);
-    __ lvx             (vTmp1, keypos, key);
-    __ vec_perm        (vKey5, vTmp1, vKey5, keyPerm);
+    // load the 15th - 11th round key to vKey1 - vKey5
+    __ load_word_vector_unaligned(vKey1, 224, key, tmp);
+    __ load_word_vector_unaligned(vKey2, 208, key, tmp);
+    __ load_word_vector_unaligned(vKey3, 192, key, tmp);
+    __ load_word_vector_unaligned(vKey4, 176, key, tmp);
+    __ load_word_vector_unaligned(vKey5, 160, key, tmp);
 
     // 1st - 5th rounds
     __ vxor            (vRet, vRet, vKey1);
@@ -3087,22 +2950,10 @@ class StubGenerator: public StubCodeGenerator {
     __ align(32);
     __ bind            (L_do52);
 
-    // load the 13th round key to vKey1
-    __ li              (keypos, 208);
-    __ lvx             (vKey1, keypos, key);
-    __ li              (keypos, 192);
-    __ lvx             (vKey2, keypos, key);
-    __ vec_perm        (vKey1, vKey2, vKey1, keyPerm);
-
-    // load the 12th round key to vKey2
-    __ li              (keypos, 176);
-    __ lvx             (vKey3, keypos, key);
-    __ vec_perm        (vKey2, vKey3, vKey2, keyPerm);
-
-    // load the 11th round key to vKey3
-    __ li              (keypos, 160);
-    __ lvx             (vTmp1, keypos, key);
-    __ vec_perm        (vKey3, vTmp1, vKey3, keyPerm);
+    // load the 13th - 11th round key to vKey1 - vKey3
+    __ load_word_vector_unaligned(vKey1, 192, key, tmp);
+    __ load_word_vector_unaligned(vKey2, 176, key, tmp);
+    __ load_word_vector_unaligned(vKey3, 160, key, tmp);
 
     // 1st - 3rd rounds
     __ vxor            (vRet, vRet, vKey1);
@@ -3115,41 +2966,19 @@ class StubGenerator: public StubCodeGenerator {
     __ bind            (L_do44);
 
     // load the 11th round key to vKey1
-    __ li              (keypos, 176);
-    __ lvx             (vKey1, keypos, key);
-    __ li              (keypos, 160);
-    __ lvx             (vTmp1, keypos, key);
-    __ vec_perm        (vKey1, vTmp1, vKey1, keyPerm);
+    __ load_word_vector_unaligned(vKey1, 160, key, tmp);
 
     // 1st round
     __ vxor            (vRet, vRet, vKey1);
 
     __ bind            (L_doLast);
 
-    // load the 10th round key to vKey1
-    __ li              (keypos, 144);
-    __ lvx             (vKey2, keypos, key);
-    __ vec_perm        (vKey1, vKey2, vTmp1, keyPerm);
-
-    // load the 9th round key to vKey2
-    __ li              (keypos, 128);
-    __ lvx             (vKey3, keypos, key);
-    __ vec_perm        (vKey2, vKey3, vKey2, keyPerm);
-
-    // load the 8th round key to vKey3
-    __ li              (keypos, 112);
-    __ lvx             (vKey4, keypos, key);
-    __ vec_perm        (vKey3, vKey4, vKey3, keyPerm);
-
-    // load the 7th round key to vKey4
-    __ li              (keypos, 96);
-    __ lvx             (vKey5, keypos, key);
-    __ vec_perm        (vKey4, vKey5, vKey4, keyPerm);
-
-    // load the 6th round key to vKey5
-    __ li              (keypos, 80);
-    __ lvx             (vTmp1, keypos, key);
-    __ vec_perm        (vKey5, vTmp1, vKey5, keyPerm);
+    // load the 10th - 6th round key to vKey1 - vKey5
+    __ load_word_vector_unaligned(vKey1, 144, key, tmp);
+    __ load_word_vector_unaligned(vKey2, 128, key, tmp);
+    __ load_word_vector_unaligned(vKey3, 112, key, tmp);
+    __ load_word_vector_unaligned(vKey4, 96, key, tmp);
+    __ load_word_vector_unaligned(vKey5, 80, key, tmp);
 
     // last 10th - 6th rounds
     __ vncipher        (vRet, vRet, vKey1);
@@ -3158,29 +2987,12 @@ class StubGenerator: public StubCodeGenerator {
     __ vncipher        (vRet, vRet, vKey4);
     __ vncipher        (vRet, vRet, vKey5);
 
-    // load the 5th round key to vKey1
-    __ li              (keypos, 64);
-    __ lvx             (vKey2, keypos, key);
-    __ vec_perm        (vKey1, vKey2, vTmp1, keyPerm);
-
-    // load the 4th round key to vKey2
-    __ li              (keypos, 48);
-    __ lvx             (vKey3, keypos, key);
-    __ vec_perm        (vKey2, vKey3, vKey2, keyPerm);
-
-    // load the 3rd round key to vKey3
-    __ li              (keypos, 32);
-    __ lvx             (vKey4, keypos, key);
-    __ vec_perm        (vKey3, vKey4, vKey3, keyPerm);
-
-    // load the 2nd round key to vKey4
-    __ li              (keypos, 16);
-    __ lvx             (vKey5, keypos, key);
-    __ vec_perm        (vKey4, vKey5, vKey4, keyPerm);
-
-    // load the 1st round key to vKey5
-    __ lvx             (vTmp1, key);
-    __ vec_perm        (vKey5, vTmp1, vKey5, keyPerm);
+    // load the 5th - 1st round key to vKey1 - vKey5
+    __ load_word_vector_unaligned(vKey1, 64, key, tmp);
+    __ load_word_vector_unaligned(vKey2, 48, key, tmp);
+    __ load_word_vector_unaligned(vKey3, 32, key, tmp);
+    __ load_word_vector_unaligned(vKey4, 16, key, tmp);
+    __ load_word_vector_unaligned(vKey5, 0, key, tmp);
 
     // last 5th - 1th rounds
     __ vncipher        (vRet, vRet, vKey1);
@@ -3189,23 +3001,8 @@ class StubGenerator: public StubCodeGenerator {
     __ vncipher        (vRet, vRet, vKey4);
     __ vncipherlast    (vRet, vRet, vKey5);
 
-#ifdef VM_LITTLE_ENDIAN
-    // toPerm = 0x0F0E0D0C0B0A09080706050403020100
-    __ lvsl            (toPerm, keypos); // keypos is a multiple of 16
-    __ vxor            (toPerm, toPerm, fSplt);
-
-    // Swap Bytes
-    __ vperm           (vRet, vRet, vRet, toPerm);
-#endif
-
     // store result (unaligned)
-    // Note: We can't use a read-modify-write sequence which touches additional Bytes.
-    Register lo = temp, hi = fifteen; // Reuse
-    __ vsldoi          (vTmp1, vRet, vRet, 8);
-    __ mfvrd           (hi, vRet);
-    __ mfvrd           (lo, vTmp1);
-    __ std             (hi, 0 LITTLE_ENDIAN_ONLY(+ 8), to);
-    __ std             (lo, 0 BIG_ENDIAN_ONLY(+ 8), to);
+    __ store_byte_vector_unaligned(vRet, 0, to, tmp, vp);
 
     __ blr();
 
@@ -3216,6 +3013,306 @@ class StubGenerator: public StubCodeGenerator {
      return start;
   }
 
+  // ==========================================================================
+  // AES helper functions for PPC64
+  //
+  // These emit the AES round instructions.
+  // Each call to these helpers emits a sequence of vcipher/vncipher
+  // instructions.
+  //
+  // ==========================================================================
+  // Emits the AES encrypt round instructions.
+  //
+  // vRet:    in/out — the AES state (plaintext in, ciphertext out)
+  // key:     register holding pointer to expanded key array
+  // keylen:  register holding key length (44/52/60)
+  //
+  void aes_encrypt_rounds(VectorRegister vRet,
+                          Register key, Register keylen, Register tmp,
+                          VectorRegister vKey1, VectorRegister vKey2,
+                          VectorRegister vKey3, VectorRegister vKey4) {
+    Label L_doLast;
+
+    // round 0: AddRoundKey
+    __ load_word_vector_unaligned(vKey1, 0, key, tmp);
+    __ vxor            (vRet, vRet, vKey1);
+
+    // rounds 2-5
+    __ load_word_vector_unaligned(vKey1, 16, key, tmp);
+    __ load_word_vector_unaligned(vKey2, 32, key, tmp);
+    __ load_word_vector_unaligned(vKey3, 48, key, tmp);
+    __ load_word_vector_unaligned(vKey4, 64, key, tmp);
+    __ vcipher         (vRet, vRet, vKey1);
+    __ vcipher         (vRet, vRet, vKey2);
+    __ vcipher         (vRet, vRet, vKey3);
+    __ vcipher         (vRet, vRet, vKey4);
+
+    // rounds 6-9
+    __ load_word_vector_unaligned(vKey1, 80, key, tmp);
+    __ load_word_vector_unaligned(vKey2, 96, key, tmp);
+    __ load_word_vector_unaligned(vKey3, 112, key, tmp);
+    __ load_word_vector_unaligned(vKey4, 128, key, tmp);
+    __ vcipher         (vRet, vRet, vKey1);
+    __ vcipher         (vRet, vRet, vKey2);
+    __ vcipher         (vRet, vRet, vKey3);
+    __ vcipher         (vRet, vRet, vKey4);
+
+    // rounds 10-11
+    __ load_word_vector_unaligned(vKey1, 144, key, tmp);
+    __ load_word_vector_unaligned(vKey2, 160, key, tmp);
+
+    __ cmpwi           (CR0, keylen, 44);   // AES-128 -> final rounds
+    __ beq             (CR0, L_doLast);
+
+    __ vcipher         (vRet, vRet, vKey1);
+    __ vcipher         (vRet, vRet, vKey2);
+
+    // rounds 12-13
+    __ load_word_vector_unaligned(vKey1, 176, key, tmp);
+    __ load_word_vector_unaligned(vKey2, 192, key, tmp);
+
+    __ cmpwi           (CR0, keylen, 52);   // AES-192 -> final rounds
+    __ beq             (CR0, L_doLast);
+#ifdef ASSERT
+      __ cmpwi         (CR0, keylen, 60);
+      __ asm_assert_eq(FILE_AND_LINE ": aes_encrypt_rounds - invalid key length");
+#endif
+
+    __ vcipher         (vRet, vRet, vKey1);
+    __ vcipher         (vRet, vRet, vKey2);
+
+    // rounds 14-15
+    __ load_word_vector_unaligned(vKey1, 208, key, tmp);
+    __ load_word_vector_unaligned(vKey2, 224, key, tmp);
+
+    __ bind(L_doLast);
+    __ vcipher         (vRet, vRet, vKey1);
+    __ vcipherlast     (vRet, vRet, vKey2);
+  }
+
+
+  // ==========================================================================
+  // Emits the AES decrypt round instructions.
+  //
+  // vRet:    in/out — the AES state (ciphertext in, plaintext out)
+  // key:     register holding pointer to expanded key array
+  // keylen:  register holding key length (44/52/60)
+  //
+  void aes_decrypt_rounds(VectorRegister vRet,
+                          Register key, Register keylen, Register tmp,
+                          VectorRegister vKey1, VectorRegister vKey2,
+                          VectorRegister vKey3, VectorRegister vKey4,
+                          VectorRegister vKey5) {
+    Label L_doLast, L_do44, L_do52;
+
+    __ cmpwi           (CR0, keylen, 44);
+    __ beq             (CR0, L_do44);
+
+    __ cmpwi           (CR0, keylen, 52);
+    __ beq             (CR0, L_do52);
+
+#ifdef ASSERT
+      __ cmpwi         (CR0, keylen, 60);
+      __ asm_assert_eq(FILE_AND_LINE ": aes_decrypt_rounds - invalid key length");
+#endif
+    // ---- AES-256: round keys 15-11 ----
+    __ load_word_vector_unaligned(vKey1, 224, key, tmp);
+    __ load_word_vector_unaligned(vKey2, 208, key, tmp);
+    __ load_word_vector_unaligned(vKey3, 192, key, tmp);
+    __ load_word_vector_unaligned(vKey4, 176, key, tmp);
+    __ load_word_vector_unaligned(vKey5, 160, key, tmp);
+
+    __ vxor            (vRet, vRet, vKey1);
+    __ vncipher        (vRet, vRet, vKey2);
+    __ vncipher        (vRet, vRet, vKey3);
+    __ vncipher        (vRet, vRet, vKey4);
+    __ vncipher        (vRet, vRet, vKey5);
+    __ b               (L_doLast);
+
+    __ align(32);
+    // ---- AES-192: round keys 13-11 ----
+    __ bind            (L_do52);
+    __ load_word_vector_unaligned(vKey1, 192, key, tmp);
+    __ load_word_vector_unaligned(vKey2, 176, key, tmp);
+    __ load_word_vector_unaligned(vKey3, 160, key, tmp);
+
+    __ vxor            (vRet, vRet, vKey1);
+    __ vncipher        (vRet, vRet, vKey2);
+    __ vncipher        (vRet, vRet, vKey3);
+    __ b               (L_doLast);
+
+    __ align(32);
+    // ---- AES-128: round key 11 ----
+    __ bind            (L_do44);
+    __ load_word_vector_unaligned(vKey1, 160, key, tmp);
+    __ vxor            (vRet, vRet, vKey1);
+
+    // ---- Common rounds 10-1 ----
+    __ bind            (L_doLast);
+    __ load_word_vector_unaligned(vKey1, 144, key, tmp);
+    __ load_word_vector_unaligned(vKey2, 128, key, tmp);
+    __ load_word_vector_unaligned(vKey3, 112, key, tmp);
+    __ load_word_vector_unaligned(vKey4, 96,  key, tmp);
+    __ load_word_vector_unaligned(vKey5, 80,  key, tmp);
+
+    __ vncipher        (vRet, vRet, vKey1);
+    __ vncipher        (vRet, vRet, vKey2);
+    __ vncipher        (vRet, vRet, vKey3);
+    __ vncipher        (vRet, vRet, vKey4);
+    __ vncipher        (vRet, vRet, vKey5);
+    __ load_word_vector_unaligned(vKey1, 64, key, tmp);
+    __ load_word_vector_unaligned(vKey2, 48, key, tmp);
+    __ load_word_vector_unaligned(vKey3, 32, key, tmp);
+    __ load_word_vector_unaligned(vKey4, 16, key, tmp);
+    __ load_word_vector_unaligned(vKey5, 0,  key, tmp);
+    __ vncipher        (vRet, vRet, vKey1);
+    __ vncipher        (vRet, vRet, vKey2);
+    __ vncipher        (vRet, vRet, vKey3);
+    __ vncipher        (vRet, vRet, vKey4);
+    __ vncipherlast    (vRet, vRet, vKey5);
+  }
+
+  // ==========================================================================
+  // CBC Encrypt stub — using helper functions
+  //   from:      R3_ARG1          - source byte array address (plaintext)
+  //   to:        R4_ARG2          - destination byte array address (ciphertext)
+  //   key:       R5_ARG3          - round key array
+  //   rvec:      R6_ARG4          - r vector byte array address (initialization vector)
+  //   input_len: R7_ARG5          - length of input in bytes
+  //
+  // Returns:
+  //   R3_RET - number of bytes processed
+  //
+  address generate_cipherBlockChaining_encryptAESCrypt() {
+    assert(UseAESIntrinsics, "need AES instructions support");
+    StubId stub_id = StubId::stubgen_cipherBlockChaining_encryptAESCrypt_id;
+    StubCodeMark mark(this, stub_id);
+
+    address start = __ function_entry();
+
+    Label L_enc_loop;
+
+    Register from       = R3_ARG1;
+    Register to         = R4_ARG2;
+    Register key        = R5_ARG3;
+    Register rvec       = R6_ARG4;
+    Register input_len  = R7_ARG5;
+
+    Register keylen     = R8;
+    Register tmp        = R9;
+    Register len        = R10;
+
+    VectorRegister vRet  = VR0;
+    VectorRegister vKey1 = VR1;
+    VectorRegister vKey2 = VR2;
+    VectorRegister vKey3 = VR3;
+    VectorRegister vKey4 = VR4;
+    VectorRegister vIn   = VR5;
+    VectorRegister vp    = VR6;   // permute vector for P8 LE byte accesses
+    VectorRegister vTmp  = VR7;
+
+    __ mr              (len, input_len);
+
+    // vp must be computed once, before any byte vector access. Clobbers R0.
+    __ compute_vp_for_byte_vector_unaligned(vp, /*temp*/ vRet);
+
+    __ load_byte_vector_unaligned(vRet, 0, rvec, tmp, vp);
+
+    __ lwz             (keylen, arrayOopDesc::length_offset_in_bytes() -
+                                arrayOopDesc::base_offset_in_bytes(T_INT), key);
+
+    __ align(32);
+    __ bind(L_enc_loop);
+    __ load_byte_vector_unaligned(vIn, 0, from, tmp, vp);
+    __ addi            (from, from, 16);
+    __ vxor            (vRet, vRet, vIn);                      // CBC XOR
+    aes_encrypt_rounds(vRet, key, keylen, tmp, vKey1, vKey2, vKey3, vKey4);
+    __ store_byte_vector_unaligned(vRet, 0, to, tmp, vp, vTmp);
+    __ addi            (to, to, 16);
+    __ addic_          (len, len, -16);
+    __ bne             (CR0, L_enc_loop);
+
+    // save the last ciphertext block in rvec; it is the IV for the next call
+    __ store_byte_vector_unaligned(vRet, 0, rvec, tmp, vp, vTmp);
+    __ mr              (R3_RET, input_len);
+    __ blr();
+
+    return start;
+  }
+
+  // ==========================================================================
+  //  CBC Decrypt stub
+  //  Arguments:
+  //    R3_ARG1 - from:      source byte array address (ciphertext)
+  //    R4_ARG2 - to:        destination byte array address (plaintext)
+  //    R5_ARG3 - key:       round key array
+  //    R6_ARG4 - rvec:      r vector byte array address (in/out), holds the
+  //                         initialization vector on entry and is updated with
+  //                         the last ciphertext block on exit
+  //    R7_ARG5 - input_len: length of input in bytes, a multiple of 16
+  //
+  //  Returns:
+  //    R3_RET  - number of bytes processed
+  // ==========================================================================
+
+  address generate_cipherBlockChaining_decryptAESCrypt() {
+    assert(UseAESIntrinsics, "need AES instructions support");
+    StubId stub_id = StubId::stubgen_cipherBlockChaining_decryptAESCrypt_id;
+    StubCodeMark mark(this, stub_id);
+
+    address start = __ function_entry();
+
+    Label L_dec_loop;
+
+    Register from       = R3_ARG1;
+    Register to         = R4_ARG2;
+    Register key        = R5_ARG3;
+    Register rvec       = R6_ARG4;
+    Register input_len  = R7_ARG5;
+
+    Register keylen     = R8;
+    Register tmp        = R9;
+    Register len        = R10;
+
+    VectorRegister vRet     = VR0;
+    VectorRegister vKey1    = VR1;
+    VectorRegister vKey2    = VR2;
+    VectorRegister vKey3    = VR3;
+    VectorRegister vKey4    = VR4;
+    VectorRegister vKey5    = VR5;
+    VectorRegister vIV      = VR6;
+    VectorRegister vSavedCT = VR7;
+    VectorRegister vp       = VR8;   // permute vector for P8 LE byte accesses
+    VectorRegister vTmp     = VR9;
+    __ mr              (len, input_len);
+    // vp must be computed before any byte vector access. Clobbers R0.
+    __ compute_vp_for_byte_vector_unaligned(vp, /*temp*/ vRet);
+
+    __ load_byte_vector_unaligned(vIV, 0, rvec, tmp, vp);
+
+    __ lwz             (keylen, arrayOopDesc::length_offset_in_bytes() -
+                                arrayOopDesc::base_offset_in_bytes(T_INT), key);
+
+    __ align(32);
+    __ bind(L_dec_loop);
+    __ load_byte_vector_unaligned(vRet, 0, from, tmp, vp);
+    __ addi            (from, from, 16);
+    __ vor             (vSavedCT, vRet, vRet);      // AES will destroy vRet
+    aes_decrypt_rounds(vRet, key, keylen, tmp, vKey1, vKey2, vKey3, vKey4, vKey5);
+    __ vxor            (vRet, vRet, vIV);           // CBC XOR (after decrypt)
+    __ vor             (vIV, vSavedCT, vSavedCT);   // IV = previous ciphertext
+    __ store_byte_vector_unaligned(vRet, 0, to, tmp, vp, vTmp);
+    __ addi            (to, to, 16);
+    __ addic_          (len, len, -16);
+    __ bne             (CR0, L_dec_loop);
+
+    __ store_byte_vector_unaligned(vIV, 0, rvec, tmp, vp, vTmp);
+    __ mr              (R3_RET, input_len);
+    __ blr();
+
+    return start;
+  }
+
   address generate_sha256_implCompress(StubId stub_id) {
     assert(UseSHA, "need SHA instructions");
     bool multi_block;
@@ -5092,6 +5189,8 @@ void generate_lookup_secondary_supers_table_stub() {
     if (UseAESIntrinsics) {
       StubRoutines::_aescrypt_encryptBlock = generate_aescrypt_encryptBlock();
       StubRoutines::_aescrypt_decryptBlock = generate_aescrypt_decryptBlock();
+      StubRoutines::_cipherBlockChaining_encryptAESCrypt = generate_cipherBlockChaining_encryptAESCrypt();
+      StubRoutines::_cipherBlockChaining_decryptAESCrypt = generate_cipherBlockChaining_decryptAESCrypt();
     }
 
     if (UseSHA256Intrinsics) {

src/hotspot/cpu/ppc/templateInterpreterGenerator_ppc.cpp

@@ -1162,7 +1162,7 @@ address TemplateInterpreterGenerator::generate_math_entry(AbstractInterpreter::M
   __ resize_frame_absolute(R21_sender_SP, R11_scratch1, R0);
   __ blr();
 
-  __ flush();
+  __ invalidate_icache();
 
   return entry;
 }
@@ -1179,7 +1179,7 @@ address TemplateInterpreterGenerator::generate_Float_floatToFloat16_entry() {
   __ resize_frame_absolute(R21_sender_SP, R11_scratch1, R0);
   __ blr();
 
-  __ flush();
+  __ invalidate_icache();
 
   return entry;
 }
@@ -1200,7 +1200,7 @@ address TemplateInterpreterGenerator::generate_Float_float16ToFloat_entry() {
   __ resize_frame_absolute(R21_sender_SP, R11_scratch1, R0);
   __ blr();
 
-  __ flush();
+  __ invalidate_icache();
 
   return entry;
 }
@@ -1484,21 +1484,10 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) {
   // In order for GC to work, don't clear the last_Java_sp until after
   // blocking.
 
-  //=============================================================================
-  // Switch thread to "native transition" state before reading the
-  // synchronization state. This additional state is necessary
-  // because reading and testing the synchronization state is not
-  // atomic w.r.t. GC, as this scenario demonstrates: Java thread A,
-  // in _thread_in_native state, loads _not_synchronized and is
-  // preempted. VM thread changes sync state to synchronizing and
-  // suspends threads for GC. Thread A is resumed to finish this
-  // native method, but doesn't block here since it didn't see any
-  // synchronization in progress, and escapes.
-
   // We use release_store_fence to update values like the thread state, where
   // we don't want the current thread to continue until all our prior memory
   // accesses (including the new thread state) are visible to other threads.
-  __ li(R0/*thread_state*/, _thread_in_native_trans);
+  __ li(R0/*thread_state*/, _thread_in_vm);
   __ release();
   __ stw(R0/*thread_state*/, thread_(thread_state));
   if (!UseSystemMemoryBarrier) {
@@ -1506,9 +1495,8 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) {
   }
 
   // Now before we return to java we must look for a current safepoint
-  // (a new safepoint can not start since we entered native_trans).
-  // We must check here because a current safepoint could be modifying
-  // the callers registers right this moment.
+  // (a new safepoint can not start since we entered _thread_in_vm).
+  // We must check here because a current safepoint could be in progress.
 
   // Acquire isn't strictly necessary here because of the fence, but
   // sync_state is declared to be volatile, so we do it anyway
@@ -1538,7 +1526,7 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) {
   //=============================================================================
   // <<<<<< Back in Interpreter Frame >>>>>
 
-  // We are in thread_in_native_trans here and back in the normal
+  // We are in _thread_in_vm here and back in the normal
   // interpreter frame. We don't have to do anything special about
   // safepoints and we can switch to Java mode anytime we are ready.
 

src/hotspot/cpu/ppc/templateTable_ppc_64.cpp

@@ -2647,10 +2647,12 @@ void TemplateTable::getfield_or_static(int byte_no, bool is_static, RewriteContr
                  Rscratch      = R11_scratch1; // used by load_field_cp_cache_entry
                  // R12_scratch2 used by load_field_cp_cache_entry
 
-  static address field_branch_table[number_of_states],
+  static address field_rw_branch_table[number_of_states],
+                 field_norw_branch_table[number_of_states],
                  static_branch_table[number_of_states];
 
-  address* branch_table = (is_static || rc == may_not_rewrite) ? static_branch_table : field_branch_table;
+  address* branch_table = is_static ? static_branch_table :
+    (rc == may_rewrite ? field_rw_branch_table : field_norw_branch_table);
 
   // Get field offset.
   resolve_cache_and_index_for_field(byte_no, Rcache, Rscratch);
@@ -2698,14 +2700,7 @@ void TemplateTable::getfield_or_static(int byte_no, bool is_static, RewriteContr
 #ifdef ASSERT
   __ bind(LFlagInvalid);
   __ stop("got invalid flag");
-#endif
 
-  if (!is_static && rc == may_not_rewrite) {
-    // We reuse the code from is_static.  It's jumped to via the table above.
-    return;
-  }
-
-#ifdef ASSERT
   // __ bind(Lvtos);
   address pc_before_fence = __ pc();
   __ fence(); // Volatile entry point (one instruction before non-volatile_entry point).

src/hotspot/cpu/ppc/upcallLinker_ppc.cpp

@@ -1,6 +1,6 @@
 /*
- * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2023, 2025 SAP SE. All rights reserved.
+ * Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2023, 2026 SAP SE. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -243,7 +243,7 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Symbol* signature,
 
   //////////////////////////////////////////////////////////////////////////////
 
-  _masm->flush();
+  // Code will be copied. No ICache sync required.
 
 #ifndef PRODUCT
   stringStream ss;

src/hotspot/cpu/ppc/vm_version_ppc.cpp

@@ -514,7 +514,7 @@ void VM_Version::determine_features() {
   a->blr();
 
   uint32_t *code_end = (uint32_t *)a->pc();
-  a->flush();
+  a->invalidate_icache();
   _features = VM_Version::unknown_m;
 
   // Print the detection code.
@@ -570,7 +570,7 @@ void VM_Version::config_dscr() {
   a->blr();
 
   uint32_t *code_end = (uint32_t *)a->pc();
-  a->flush();
+  a->invalidate_icache();
 
   // Print the detection code.
   if (PrintAssembly) {

src/hotspot/cpu/ppc/vtableStubs_ppc_64.cpp

@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2025 SAP SE. All rights reserved.
+ * Copyright (c) 2012, 2026 SAP SE. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -124,7 +124,7 @@ VtableStub* VtableStubs::create_vtable_stub(int vtable_index, bool caller_is_c1)
   __ mtctr(R12_scratch2);
   __ bctr();
 
-  masm->flush();
+  masm->invalidate_icache();
   bookkeeping(masm, tty, s, npe_addr, ame_addr, true, vtable_index, slop_bytes, 0);
 
   return s;
@@ -224,7 +224,7 @@ VtableStub* VtableStubs::create_itable_stub(int itable_index, bool caller_is_c1)
   __ mtctr(R11_scratch1);
   __ bctr();
 
-  masm->flush();
+  masm->invalidate_icache();
   bookkeeping(masm, tty, s, npe_addr, ame_addr, false, itable_index, slop_bytes, 0);
 
   return s;

src/hotspot/cpu/riscv/assembler_riscv.hpp

@@ -514,20 +514,138 @@ protected:
     rdy = 0b111,     // in instruction's rm field, selects dynamic rounding mode.In Rounding Mode register, Invalid.
   };
 
+  // Efficient reading and writing of unaligned data in platform-specific byte ordering
+  // RISC-V needs to check for alignment.
+
+  static inline u2 get_native_u2(address p) {
+    if ((intptr_t(p) & 1) == 0) {
+      return *(u2*)p;
+    } else {
+      return ((u2)(p[1]) << 8) |
+             ((u2)(p[0]));
+    }
+  }
+
+  static inline u4 get_native_u4(address p) {
+    switch (intptr_t(p) & 3) {
+      case 0:
+        return *(u4*)p;
+
+      case 2:
+        return ((u4)(((u2*)p)[1]) << 16) |
+               ((u4)(((u2*)p)[0]));
+
+      default:
+        return ((u4)(p[3]) << 24) |
+               ((u4)(p[2]) << 16) |
+               ((u4)(p[1]) <<  8) |
+               ((u4)(p[0]));
+    }
+  }
+
+  static inline u8 get_native_u8(address p) {
+    switch (intptr_t(p) & 7) {
+      case 0:
+        return *(u8*)p;
+
+      case 4:
+        return ((u8)(((u4*)p)[1]) << 32) |
+               ((u8)(((u4*)p)[0]));
+
+      case 2:
+      case 6:
+        return ((u8)(((u2*)p)[3]) << 48) |
+               ((u8)(((u2*)p)[2]) << 32) |
+               ((u8)(((u2*)p)[1]) << 16) |
+               ((u8)(((u2*)p)[0]));
+
+      default:
+        return ((u8)(p[7]) << 56) |
+               ((u8)(p[6]) << 48) |
+               ((u8)(p[5]) << 40) |
+               ((u8)(p[4]) << 32) |
+               ((u8)(p[3]) << 24) |
+               ((u8)(p[2]) << 16) |
+               ((u8)(p[1]) <<  8) |
+               ((u8)(p[0]));
+    }
+  }
+
+  static inline void put_native_u2(address p, u2 x) {
+    if ((intptr_t(p) & 1) == 0) {
+      *(u2*)p = x;
+    } else {
+      p[1] = x >> 8;
+      p[0] = x;
+    }
+  }
+
+  static inline void put_native_u4(address p, u4 x) {
+    switch (intptr_t(p) & 3) {
+      case 0:
+        *(u4*)p = x;
+        break;
+
+      case 2:
+        ((u2*)p)[1] = x >> 16;
+        ((u2*)p)[0] = x;
+        break;
+
+      default:
+        ((u1*)p)[3] = x >> 24;
+        ((u1*)p)[2] = x >> 16;
+        ((u1*)p)[1] = x >>  8;
+        ((u1*)p)[0] = x;
+        break;
+    }
+  }
+
+  static inline void put_native_u8(address p, u8 x) {
+    switch (intptr_t(p) & 7) {
+      case 0:
+        *(u8*)p = x;
+        break;
+
+      case 4:
+        ((u4*)p)[1] = x >> 32;
+        ((u4*)p)[0] = x;
+        break;
+
+      case 2:
+      case 6:
+        ((u2*)p)[3] = x >> 48;
+        ((u2*)p)[2] = x >> 32;
+        ((u2*)p)[1] = x >> 16;
+        ((u2*)p)[0] = x;
+        break;
+
+      default:
+        ((u1*)p)[7] = x >> 56;
+        ((u1*)p)[6] = x >> 48;
+        ((u1*)p)[5] = x >> 40;
+        ((u1*)p)[4] = x >> 32;
+        ((u1*)p)[3] = x >> 24;
+        ((u1*)p)[2] = x >> 16;
+        ((u1*)p)[1] = x >>  8;
+        ((u1*)p)[0] = x;
+        break;
+    }
+  }
+
   // handle unaligned access
   static inline uint16_t ld_c_instr(address addr) {
-    return Bytes::get_native_u2(addr);
+    return get_native_u2(addr);
   }
   static inline void sd_c_instr(address addr, uint16_t c_instr) {
-    Bytes::put_native_u2(addr, c_instr);
+    put_native_u2(addr, c_instr);
   }
 
   // handle unaligned access
   static inline uint32_t ld_instr(address addr) {
-    return Bytes::get_native_u4(addr);
+    return get_native_u4(addr);
   }
   static inline void sd_instr(address addr, uint32_t instr) {
-    Bytes::put_native_u4(addr, instr);
+    put_native_u4(addr, instr);
   }
 
   static inline uint32_t extract(uint32_t val, unsigned msb, unsigned lsb) {

src/hotspot/cpu/riscv/bytes_riscv.hpp

@@ -1,167 +0,0 @@
-/*
- * Copyright (c) 1997, 2019, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2016 SAP SE. All rights reserved.
- * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#ifndef CPU_RISCV_BYTES_RISCV_HPP
-#define CPU_RISCV_BYTES_RISCV_HPP
-
-#include "memory/allStatic.hpp"
-#include "utilities/byteswap.hpp"
-
-class Bytes: AllStatic {
- public:
-  // Efficient reading and writing of unaligned unsigned data in platform-specific byte ordering
-  // RISCV needs to check for alignment.
-
-  static inline u2 get_native_u2(address p) {
-    if ((intptr_t(p) & 1) == 0) {
-      return *(u2*)p;
-    } else {
-      return ((u2)(p[1]) << 8) |
-             ((u2)(p[0]));
-    }
-  }
-
-  static inline u4 get_native_u4(address p) {
-    switch (intptr_t(p) & 3) {
-      case 0:
-        return *(u4*)p;
-
-      case 2:
-        return ((u4)(((u2*)p)[1]) << 16) |
-               ((u4)(((u2*)p)[0]));
-
-      default:
-        return ((u4)(p[3]) << 24) |
-               ((u4)(p[2]) << 16) |
-               ((u4)(p[1]) <<  8) |
-               ((u4)(p[0]));
-    }
-  }
-
-  static inline u8 get_native_u8(address p) {
-    switch (intptr_t(p) & 7) {
-      case 0:
-        return *(u8*)p;
-
-      case 4:
-        return ((u8)(((u4*)p)[1]) << 32) |
-               ((u8)(((u4*)p)[0]));
-
-      case 2:
-      case 6:
-        return ((u8)(((u2*)p)[3]) << 48) |
-               ((u8)(((u2*)p)[2]) << 32) |
-               ((u8)(((u2*)p)[1]) << 16) |
-               ((u8)(((u2*)p)[0]));
-
-      default:
-        return ((u8)(p[7]) << 56) |
-               ((u8)(p[6]) << 48) |
-               ((u8)(p[5]) << 40) |
-               ((u8)(p[4]) << 32) |
-               ((u8)(p[3]) << 24) |
-               ((u8)(p[2]) << 16) |
-               ((u8)(p[1]) <<  8) |
-               ((u8)(p[0]));
-    }
-  }
-
-  static inline void put_native_u2(address p, u2 x) {
-    if ((intptr_t(p) & 1) == 0) {
-      *(u2*)p = x;
-    } else {
-      p[1] = x >> 8;
-      p[0] = x;
-    }
-  }
-
-  static inline void put_native_u4(address p, u4 x) {
-    switch (intptr_t(p) & 3) {
-      case 0:
-        *(u4*)p = x;
-        break;
-
-      case 2:
-        ((u2*)p)[1] = x >> 16;
-        ((u2*)p)[0] = x;
-        break;
-
-      default:
-        ((u1*)p)[3] = x >> 24;
-        ((u1*)p)[2] = x >> 16;
-        ((u1*)p)[1] = x >>  8;
-        ((u1*)p)[0] = x;
-        break;
-    }
-  }
-
-  static inline void put_native_u8(address p, u8 x) {
-    switch (intptr_t(p) & 7) {
-      case 0:
-        *(u8*)p = x;
-        break;
-
-      case 4:
-        ((u4*)p)[1] = x >> 32;
-        ((u4*)p)[0] = x;
-        break;
-
-      case 2:
-      case 6:
-        ((u2*)p)[3] = x >> 48;
-        ((u2*)p)[2] = x >> 32;
-        ((u2*)p)[1] = x >> 16;
-        ((u2*)p)[0] = x;
-        break;
-
-      default:
-        ((u1*)p)[7] = x >> 56;
-        ((u1*)p)[6] = x >> 48;
-        ((u1*)p)[5] = x >> 40;
-        ((u1*)p)[4] = x >> 32;
-        ((u1*)p)[3] = x >> 24;
-        ((u1*)p)[2] = x >> 16;
-        ((u1*)p)[1] = x >>  8;
-        ((u1*)p)[0] = x;
-        break;
-    }
-  }
-
-#ifndef VM_LITTLE_ENDIAN
-#error RISC-V is little endian, the preprocessor macro VM_LITTLE_ENDIAN should be defined.
-#endif
-
-  // Efficient reading and writing of unaligned unsigned data in Java byte ordering (i.e. big-endian ordering)
-  static inline u2 get_Java_u2(address p) { return byteswap(get_native_u2(p)); }
-  static inline u4 get_Java_u4(address p) { return byteswap(get_native_u4(p)); }
-  static inline u8 get_Java_u8(address p) { return byteswap(get_native_u8(p)); }
-
-  static inline void put_Java_u2(address p, u2 x) { put_native_u2(p, byteswap(x)); }
-  static inline void put_Java_u4(address p, u4 x) { put_native_u4(p, byteswap(x)); }
-  static inline void put_Java_u8(address p, u8 x) { put_native_u8(p, byteswap(x)); }
-};
-
-#endif // CPU_RISCV_BYTES_RISCV_HPP

src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp

@@ -408,6 +408,7 @@ void C2_MacroAssembler::fast_unlock(Register obj, Register box,
 // StringLatin1.indexOfChar
 void C2_MacroAssembler::string_indexof_char_short(Register str1, Register cnt1,
                                                   Register ch, Register result,
+                                                  Register start_index,
                                                   bool isL)
 {
   Register ch1 = t0;
@@ -500,7 +501,7 @@ void C2_MacroAssembler::string_indexof_char_short(Register str1, Register cnt1,
   addi(index, index, 7);
 
   bind(MATCH);
-  mv(result, index);
+  add(result, start_index, index);
   bind(NOMATCH);
   BLOCK_COMMENT("} string_indexof_char_short");
 }
@@ -513,39 +514,39 @@ void C2_MacroAssembler::string_indexof_char(Register str1, Register cnt1,
                                             Register tmp3, Register tmp4,
                                             bool isL)
 {
-  Label CH1_LOOP, HIT, NOMATCH, DONE, DO_LONG;
+  Label CH1_LOOP, HIT, NOMATCH, DONE, SHORT;
   Register ch1 = t0;
   Register orig_cnt = t1;
   Register mask1 = tmp3;
   Register mask2 = tmp2;
   Register match_mask = tmp1;
-  Register trailing_char = tmp4;
-  Register unaligned_elems = tmp4;
+  Register loop_step = tmp4;
+  Register trailing_chars = tmp4;
+  Register unaligned_chars = tmp4;
+  Register start_index = tmp4;
 
   BLOCK_COMMENT("string_indexof_char {");
   beqz(cnt1, NOMATCH);
 
   subi(t0, cnt1, isL ? 32 : 16);
-  bgtz(t0, DO_LONG);
-  string_indexof_char_short(str1, cnt1, ch, result, isL);
-  j(DONE);
+  mv(start_index, zr);
+  blez(t0, SHORT);
 
-  bind(DO_LONG);
   mv(orig_cnt, cnt1);
   if (AvoidUnalignedAccesses) {
     Label ALIGNED;
-    andi(unaligned_elems, str1, 0x7);
-    beqz(unaligned_elems, ALIGNED);
-    sub(unaligned_elems, unaligned_elems, 8);
-    neg(unaligned_elems, unaligned_elems);
+    andi(unaligned_chars, str1, 0x7);
+    beqz(unaligned_chars, ALIGNED);
+    sub(unaligned_chars, unaligned_chars, 8);
+    neg(unaligned_chars, unaligned_chars);
     if (!isL) {
-      srli(unaligned_elems, unaligned_elems, 1);
+      srli(unaligned_chars, unaligned_chars, 1);
     }
     // do unaligned part per element
-    string_indexof_char_short(str1, unaligned_elems, ch, result, isL);
+    string_indexof_char_short(str1, unaligned_chars, ch, result, zr, isL);
     bgez(result, DONE);
     mv(orig_cnt, cnt1);
-    sub(cnt1, cnt1, unaligned_elems);
+    sub(cnt1, cnt1, unaligned_chars);
     bind(ALIGNED);
   }
 
@@ -570,29 +571,48 @@ void C2_MacroAssembler::string_indexof_char(Register str1, Register cnt1,
   uint64_t mask7fff = UCONST64(0x7fff7fff7fff7fff);
   mv(mask2, isL ? mask7f7f : mask7fff);
 
+  mv(loop_step, 8);
+
   bind(CH1_LOOP);
   ld(ch1, Address(str1));
   addi(str1, str1, 8);
   subi(cnt1, cnt1, 8);
   compute_match_mask(ch1, ch, match_mask, mask1, mask2);
   bnez(match_mask, HIT);
-  bgtz(cnt1, CH1_LOOP);
-  j(NOMATCH);
+  bge(cnt1, loop_step, CH1_LOOP);
+
+  beqz(cnt1, NOMATCH);
+  if (!isL) {
+    srli(cnt1, cnt1, 1);
+  }
+  // Tail (1..7 chars) after the SWAR loop has advanced str1. cnt1 holds the
+  // remaining char count; the number of chars already scanned by the loop is
+  // (orig_cnt - cnt1). string_indexof_char_short returns an index relative to
+  // the current str1, so we pass that prefix as start_index to recover the
+  // real index.
+  // Note: ch was broadcast across all 8 bytes for the SWAR loop above, but the
+  // short helper compares a single element, so restore ch to a single char.
+  isL ? zext(ch, ch, 8) : zext(ch, ch, 16);
+  sub(start_index, orig_cnt, cnt1);
+
+  bind(SHORT);
+  string_indexof_char_short(str1, cnt1, ch, result, start_index, isL);
+  j(DONE);
 
   bind(HIT);
   // count bits of trailing zero chars
-  ctzc_bits(trailing_char, match_mask, isL, ch1, result);
-  srli(trailing_char, trailing_char, 3);
+  ctzc_bits(trailing_chars, match_mask, isL, ch1, result);
+  srli(trailing_chars, trailing_chars, 3);
   addi(cnt1, cnt1, 8);
-  ble(cnt1, trailing_char, NOMATCH);
+
   // match case
   if (!isL) {
     srli(cnt1, cnt1, 1);
-    srli(trailing_char, trailing_char, 1);
+    srli(trailing_chars, trailing_chars, 1);
   }
 
   sub(result, orig_cnt, cnt1);
-  add(result, result, trailing_char);
+  add(result, result, trailing_chars);
   j(DONE);
 
   bind(NOMATCH);
@@ -2021,47 +2041,49 @@ void C2_MacroAssembler::enc_cmpEqNe_imm0_branch(int cmpFlag, Register op1, Label
 }
 
 void C2_MacroAssembler::enc_cmove(int cmpFlag, Register op1, Register op2, Register dst, Register src) {
-  bool is_unsigned = (cmpFlag & unsigned_branch_mask) == unsigned_branch_mask;
-  int op_select = cmpFlag & (~unsigned_branch_mask);
+  if (dst != src) {
+    bool is_unsigned = (cmpFlag & unsigned_branch_mask) == unsigned_branch_mask;
+    int op_select = cmpFlag & (~unsigned_branch_mask);
 
-  switch (op_select) {
-    case BoolTest::eq:
-      cmov_eq(op1, op2, dst, src);
-      break;
-    case BoolTest::ne:
-      cmov_ne(op1, op2, dst, src);
-      break;
-    case BoolTest::le:
-      if (is_unsigned) {
-        cmov_leu(op1, op2, dst, src);
-      } else {
-        cmov_le(op1, op2, dst, src);
-      }
-      break;
-    case BoolTest::ge:
-      if (is_unsigned) {
-        cmov_geu(op1, op2, dst, src);
-      } else {
-        cmov_ge(op1, op2, dst, src);
-      }
-      break;
-    case BoolTest::lt:
-      if (is_unsigned) {
-        cmov_ltu(op1, op2, dst, src);
-      } else {
-        cmov_lt(op1, op2, dst, src);
-      }
-      break;
-    case BoolTest::gt:
-      if (is_unsigned) {
-        cmov_gtu(op1, op2, dst, src);
-      } else {
-        cmov_gt(op1, op2, dst, src);
-      }
-      break;
-    default:
-      assert(false, "unsupported compare condition");
-      ShouldNotReachHere();
+    switch (op_select) {
+      case BoolTest::eq:
+        cmov_eq(op1, op2, dst, src);
+        break;
+      case BoolTest::ne:
+        cmov_ne(op1, op2, dst, src);
+        break;
+      case BoolTest::le:
+        if (is_unsigned) {
+          cmov_leu(op1, op2, dst, src);
+        } else {
+          cmov_le(op1, op2, dst, src);
+        }
+        break;
+      case BoolTest::ge:
+        if (is_unsigned) {
+          cmov_geu(op1, op2, dst, src);
+        } else {
+          cmov_ge(op1, op2, dst, src);
+        }
+        break;
+      case BoolTest::lt:
+        if (is_unsigned) {
+          cmov_ltu(op1, op2, dst, src);
+        } else {
+          cmov_lt(op1, op2, dst, src);
+        }
+        break;
+      case BoolTest::gt:
+        if (is_unsigned) {
+          cmov_gtu(op1, op2, dst, src);
+        } else {
+          cmov_gt(op1, op2, dst, src);
+        }
+        break;
+      default:
+        assert(false, "unsupported compare condition");
+        ShouldNotReachHere();
+    }
   }
 }
 

src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.hpp

@@ -64,6 +64,7 @@
 
   void string_indexof_char_short(Register str1, Register cnt1,
                                  Register ch, Register result,
+                                 Register start_index,
                                  bool isL);
 
   void string_indexof_char(Register str1, Register cnt1,

src/hotspot/cpu/riscv/downcallLinker_riscv.cpp

@@ -308,7 +308,7 @@ void DowncallLinker::StubGenerator::generate() {
     __ restore_cpu_control_state_after_jni(t0);
 
     __ block_comment("{ thread native2java");
-    __ mv(t0, _thread_in_native_trans);
+    __ mv(t0, _thread_in_vm);
     __ sw(t0, Address(xthread, JavaThread::thread_state_offset()));
 
     // Force this write out before the read below
@@ -383,5 +383,5 @@ void DowncallLinker::StubGenerator::generate() {
 
   //////////////////////////////////////////////////////////////////////////////
 
-  __ flush();
+  // Code will be copied. No ICache sync required.
 }

src/hotspot/cpu/riscv/gc/shared/barrierSetAssembler_riscv.cpp

@@ -24,6 +24,7 @@
  */
 
 #include "classfile/classLoaderData.hpp"
+#include "code/aotCodeCache.hpp"
 #include "gc/shared/barrierSet.hpp"
 #include "gc/shared/barrierSetAssembler.hpp"
 #include "gc/shared/barrierSetNMethod.hpp"
@@ -300,8 +301,10 @@ void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm, Label* slo
           // Because processors will not start the second load until the first comes back.
           // This means you can't overlap the two loads,
           // which is stronger than needed for ordering (stronger than TSO).
-          __ srli(ra, t0, 32);
-          __ orr(t1, t1, ra);
+          // XOR the guard into the epoch address twice. This preserves the
+          // address while making it dependent on the guard load.
+          __ xorr(t1, t1, t0);
+          __ xorr(t1, t1, t0);
         }
         // Read the global epoch value.
         __ lwu(t1, t1);
@@ -370,10 +373,20 @@ void BarrierSetAssembler::c2i_entry_barrier(MacroAssembler* masm) {
 }
 
 void BarrierSetAssembler::check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error) {
+  assert_different_registers(obj, tmp1, tmp2);
   // Check if the oop is in the right area of memory
-  __ mv(tmp2, (intptr_t) Universe::verify_oop_mask());
-  __ andr(tmp1, obj, tmp2);
-  __ mv(tmp2, (intptr_t) Universe::verify_oop_bits());
+#if INCLUDE_CDS
+  if (AOTCodeCache::is_on_for_dump()) {
+    __ ld(tmp2, ExternalAddress(AOTRuntimeConstants::verify_oop_mask_address()));
+    __ andr(tmp1, obj, tmp2);
+    __ ld(tmp2, ExternalAddress(AOTRuntimeConstants::verify_oop_bits_address()));
+  } else
+#endif
+  {
+    __ mv(tmp2, (intptr_t) Universe::verify_oop_mask());
+    __ andr(tmp1, obj, tmp2);
+    __ mv(tmp2, (intptr_t) Universe::verify_oop_bits());
+  }
 
   // Compare tmp1 and tmp2.
   __ bne(tmp1, tmp2, error);

src/hotspot/cpu/riscv/gc/shenandoah/shenandoahBarrierSetAssembler_riscv.cpp

@@ -24,6 +24,7 @@
  *
  */
 
+#include "code/aotCodeCache.hpp"
 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
 #include "gc/shenandoah/mode/shenandoahMode.hpp"
 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
@@ -219,11 +220,14 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm,
 
   // Test for in-cset
   if (is_strong) {
+#if INCLUDE_CDS
     if (AOTCodeCache::is_on_for_dump()) {
       __ ld(t1, ExternalAddress(AOTRuntimeConstants::cset_base_address()));
       __ lwu(t0, ExternalAddress(AOTRuntimeConstants::grain_shift_address()));
       __ srl(t0, x10, t0);
-    } else {
+    } else
+#endif
+    {
       __ mv(t1, ShenandoahHeap::in_cset_fast_test_addr());
       __ srli(t0, x10, ShenandoahHeapRegion::region_size_bytes_shift_jint());
     }
@@ -440,10 +444,20 @@ void ShenandoahBarrierSetAssembler::try_peek_weak_handle_in_nmethod(MacroAssembl
 }
 
 void ShenandoahBarrierSetAssembler::check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& L_error) {
+  assert_different_registers(obj, tmp1, tmp2);
   // Check if the oop is in the right area of memory
-  __ mv(tmp2, (intptr_t) Universe::verify_oop_mask());
-  __ andr(tmp1, obj, tmp2);
-  __ mv(tmp2, (intptr_t) Universe::verify_oop_bits());
+#if INCLUDE_CDS
+  if (AOTCodeCache::is_on_for_dump()) {
+    __ ld(tmp2, ExternalAddress(AOTRuntimeConstants::verify_oop_mask_address()));
+    __ andr(tmp1, obj, tmp2);
+    __ ld(tmp2, ExternalAddress(AOTRuntimeConstants::verify_oop_bits_address()));
+  } else
+#endif
+  {
+    __ mv(tmp2, (intptr_t) Universe::verify_oop_mask());
+    __ andr(tmp1, obj, tmp2);
+    __ mv(tmp2, (intptr_t) Universe::verify_oop_bits());
+  }
 
   // Compare tmp1 and tmp2.
   __ bne(tmp1, tmp2, L_error);

src/hotspot/cpu/riscv/gc/z/zBarrierSetAssembler_riscv.cpp

@@ -24,6 +24,7 @@
  */
 
 #include "asm/macroAssembler.inline.hpp"
+#include "code/aotCodeCache.hpp"
 #include "code/codeBlob.hpp"
 #include "code/vmreg.inline.hpp"
 #include "gc/z/zAddress.hpp"
@@ -1007,6 +1008,7 @@ void ZBarrierSetAssembler::generate_c1_store_barrier_stub(LIR_Assembler* ce,
 #define __ masm->
 
 void ZBarrierSetAssembler::check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error) {
+  assert_different_registers(obj, tmp1, tmp2);
   // C1 calls verify_oop in the middle of barriers, before they have been uncolored
   // and after being colored. Therefore, we must deal with colored oops as well.
   Label done;
@@ -1044,9 +1046,18 @@ void ZBarrierSetAssembler::check_oop(MacroAssembler* masm, Register obj, Registe
 
   __ bind(check_zaddress);
   // Check if the oop is the right area of memory
-  __ mv(tmp1, (intptr_t) Universe::verify_oop_mask());
-  __ andr(tmp1, tmp1, obj);
-  __ mv(obj, (intptr_t) Universe::verify_oop_bits());
+#if INCLUDE_CDS
+  if (AOTCodeCache::is_on_for_dump()) {
+    __ ld(tmp1, ExternalAddress(AOTRuntimeConstants::verify_oop_mask_address()));
+    __ andr(tmp1, tmp1, obj);
+    __ ld(obj, ExternalAddress(AOTRuntimeConstants::verify_oop_bits_address()));
+  } else
+#endif
+  {
+    __ mv(tmp1, (intptr_t) Universe::verify_oop_mask());
+    __ andr(tmp1, tmp1, obj);
+    __ mv(obj, (intptr_t) Universe::verify_oop_bits());
+  }
   __ bne(tmp1, obj, error);
 
   __ bind(done);

src/hotspot/cpu/riscv/gc/z/z_riscv.ad

@@ -33,7 +33,7 @@ source_hpp %{
 source %{
 #include "gc/z/zBarrierSetAssembler.hpp"
 
-static void z_color(MacroAssembler* masm, const MachNode* node, Register dst, Register src, Register tmp) {
+static void z_color(MacroAssembler* masm, Register dst, Register src, Register tmp) {
   assert_different_registers(dst, tmp);
 
   __ relocate(barrier_Relocation::spec(), [&] {
@@ -43,7 +43,7 @@ static void z_color(MacroAssembler* masm, const MachNode* node, Register dst, Re
   __ orr(dst, dst, tmp);
 }
 
-static void z_uncolor(MacroAssembler* masm, const MachNode* node, Register ref) {
+static void z_uncolor(MacroAssembler* masm, Register ref) {
   __ srli(ref, ref, ZPointerLoadShift);
 }
 
@@ -63,7 +63,7 @@ static void z_load_barrier(MacroAssembler* masm, const MachNode* node, Address r
       ((node->barrier_data() & ZBarrierPhantom) != 0);
 
   if (node->barrier_data() == ZBarrierElided) {
-    z_uncolor(masm, node, ref);
+    z_uncolor(masm, ref);
     return;
   }
 
@@ -74,14 +74,14 @@ static void z_load_barrier(MacroAssembler* masm, const MachNode* node, Address r
   __ j(*stub->entry());
 
   __ bind(good);
-  z_uncolor(masm, node, ref);
+  z_uncolor(masm, ref);
   __ bind(*stub->continuation());
 }
 
 static void z_store_barrier(MacroAssembler* masm, const MachNode* node, Address ref_addr, Register rnew_zaddress, Register rnew_zpointer, Register tmp, bool is_atomic) {
   Assembler::InlineSkippedInstructionsCounter skipped_counter(masm);
   if (node->barrier_data() == ZBarrierElided) {
-    z_color(masm, node, rnew_zpointer, rnew_zaddress, tmp);
+    z_color(masm, rnew_zpointer, rnew_zaddress, tmp);
   } else {
     bool is_native = (node->barrier_data() & ZBarrierNative) != 0;
     bool is_nokeepalive = (node->barrier_data() & ZBarrierNoKeepalive) != 0;
@@ -145,7 +145,7 @@ instruct zCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newva
   ins_encode %{
     guarantee($mem$$disp == 0, "impossible encoding");
     Address ref_addr($mem$$Register);
-    z_color(masm, this, $oldval_tmp$$Register, $oldval$$Register, $tmp1$$Register);
+    z_color(masm, $oldval_tmp$$Register, $oldval$$Register, $tmp1$$Register);
     z_store_barrier(masm, this, ref_addr, $newval$$Register, $newval_tmp$$Register, $tmp1$$Register, true /* is_atomic */);
     __ cmpxchg($mem$$Register, $oldval_tmp$$Register, $newval_tmp$$Register, Assembler::int64, Assembler::relaxed /* acquire */, Assembler::rl /* release */, $res$$Register, true /* result_as_bool */);
   %}
@@ -168,7 +168,7 @@ instruct zCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP ne
   ins_encode %{
     guarantee($mem$$disp == 0, "impossible encoding");
     Address ref_addr($mem$$Register);
-    z_color(masm, this, $oldval_tmp$$Register, $oldval$$Register, $tmp1$$Register);
+    z_color(masm, $oldval_tmp$$Register, $oldval$$Register, $tmp1$$Register);
     z_store_barrier(masm, this, ref_addr, $newval$$Register, $newval_tmp$$Register, $tmp1$$Register, true /* is_atomic */);
     __ cmpxchg($mem$$Register, $oldval_tmp$$Register, $newval_tmp$$Register, Assembler::int64, Assembler::aq /* acquire */, Assembler::rl /* release */, $res$$Register, true /* result_as_bool */);
   %}
@@ -189,10 +189,10 @@ instruct zCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP n
   ins_encode %{
     guarantee($mem$$disp == 0, "impossible encoding");
     Address ref_addr($mem$$Register);
-    z_color(masm, this, $oldval_tmp$$Register, $oldval$$Register, $tmp1$$Register);
+    z_color(masm, $oldval_tmp$$Register, $oldval$$Register, $tmp1$$Register);
     z_store_barrier(masm, this, ref_addr, $newval$$Register, $newval_tmp$$Register, $tmp1$$Register, true /* is_atomic */);
     __ cmpxchg($mem$$Register, $oldval_tmp$$Register, $newval_tmp$$Register, Assembler::int64, Assembler::relaxed /* acquire */, Assembler::rl /* release */, $res$$Register);
-    z_uncolor(masm, this, $res$$Register);
+    z_uncolor(masm, $res$$Register);
   %}
 
   ins_pipe(pipe_slow);
@@ -211,10 +211,10 @@ instruct zCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iReg
   ins_encode %{
     guarantee($mem$$disp == 0, "impossible encoding");
     Address ref_addr($mem$$Register);
-    z_color(masm, this, $oldval_tmp$$Register, $oldval$$Register, $tmp1$$Register);
+    z_color(masm, $oldval_tmp$$Register, $oldval$$Register, $tmp1$$Register);
     z_store_barrier(masm, this, ref_addr, $newval$$Register, $newval_tmp$$Register, $tmp1$$Register, true /* is_atomic */);
     __ cmpxchg($mem$$Register, $oldval_tmp$$Register, $newval_tmp$$Register, Assembler::int64, Assembler::aq /* acquire */, Assembler::rl /* release */, $res$$Register);
-    z_uncolor(masm, this, $res$$Register);
+    z_uncolor(masm, $res$$Register);
   %}
 
   ins_pipe(pipe_slow);
@@ -232,7 +232,7 @@ instruct zGetAndSetP(indirect mem, iRegP newv, iRegPNoSp prev, iRegPNoSp tmp, rF
   ins_encode %{
     z_store_barrier(masm, this, Address($mem$$Register), $newv$$Register, $prev$$Register, $tmp$$Register, true /* is_atomic */);
     __ atomic_xchg($prev$$Register, $prev$$Register, $mem$$Register);
-    z_uncolor(masm, this, $prev$$Register);
+    z_uncolor(masm, $prev$$Register);
   %}
 
   ins_pipe(pipe_serial);
@@ -250,7 +250,7 @@ instruct zGetAndSetPAcq(indirect mem, iRegP newv, iRegPNoSp prev, iRegPNoSp tmp,
   ins_encode %{
     z_store_barrier(masm, this, Address($mem$$Register), $newv$$Register, $prev$$Register, $tmp$$Register, true /* is_atomic */);
     __ atomic_xchgal($prev$$Register, $prev$$Register, $mem$$Register);
-    z_uncolor(masm, this, $prev$$Register);
+    z_uncolor(masm, $prev$$Register);
   %}
   ins_pipe(pipe_serial);
 %}

src/hotspot/cpu/riscv/interpreterRT_riscv.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
  * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -168,7 +168,7 @@ void InterpreterRuntime::SignatureHandlerGenerator::generate(uint64_t fingerprin
   __ movptr(x10, ExternalAddress(Interpreter::result_handler(method()->result_type())));
   __ ret();
 
-  __ flush();
+  __ invalidate_icache();
 }
 
 

src/hotspot/cpu/riscv/jniFastGetField_riscv.cpp

@@ -166,7 +166,7 @@ address JNI_FastGetField::generate_fast_get_int_field0(BasicType type) {
     __ leave();
     __ ret();
   }
-  __ flush();
+  __ invalidate_icache();
 
   return fast_entry;
 }

src/hotspot/cpu/riscv/macroAssembler_riscv.cpp

@@ -28,6 +28,7 @@
 #include "asm/assembler.inline.hpp"
 #include "cds/archiveBuilder.hpp"
 #include "ci/ciInlineKlass.hpp"
+#include "code/aotCodeCache.hpp"
 #include "code/compiledIC.hpp"
 #include "compiler/disassembler.hpp"
 #include "gc/shared/barrierSet.hpp"
@@ -161,8 +162,7 @@ uint32_t MacroAssembler::get_membar_kind(address addr) {
   assert_cond(addr != nullptr);
   assert(is_membar(addr), "no membar found");
 
-  uint32_t insn = Bytes::get_native_u4(addr);
-
+  uint32_t insn = Assembler::ld_instr(addr);
   uint32_t predecessor = Assembler::extract(insn, 27, 24);
   uint32_t successor = Assembler::extract(insn, 23, 20);
 
@@ -178,7 +178,7 @@ void MacroAssembler::set_membar_kind(address addr, uint32_t order_kind) {
 
   MacroAssembler::membar_mask_to_pred_succ(order_kind, predecessor, successor);
 
-  uint32_t insn = Bytes::get_native_u4(addr);
+  uint32_t insn = Assembler::ld_instr(addr);
   address pInsn = (address) &insn;
   Assembler::patch(pInsn, 27, 24, predecessor);
   Assembler::patch(pInsn, 23, 20, successor);
@@ -523,20 +523,25 @@ void MacroAssembler::_verify_oop(Register reg, const char* s, const char* file,
     ResourceMark rm;
     stringStream ss;
     ss.print("verify_oop: %s: %s (%s:%d)", reg->name(), s, file, line);
-    b = code_string(ss.as_string());
+#if INCLUDE_CDS
+    if (AOTCodeCache::is_on_for_dump() && !code_section()->scratch_emit()) {
+      // This will duplicate string to preserve it.
+      b = AOTCodeCache::add_C_string(ss.as_string());
+    } else
+#endif
+    {
+      b = code_string(ss.as_string());
+    }
   }
   BLOCK_COMMENT("verify_oop {");
 
   push_reg(RegSet::of(ra, t0, t1, c_rarg0), sp);
 
   mv(c_rarg0, reg); // c_rarg0 : x10
-  {
-    // The length of the instruction sequence emitted should not depend
-    // on the address of the char buffer so that the size of mach nodes for
-    // scratch emit and normal emit matches.
-    IncompressibleScope scope(this); // Fixed length
-    movptr(t0, (address) b);
-  }
+  // The length of the instruction sequence emitted should not depend
+  // on the address of the char buffer so that the size of mach nodes for
+  // scratch emit and normal emit matches.
+  la(t0, ExternalAddress((address)b));
 
   // Call indirectly to solve generation ordering problem
   ld(t1, RuntimeAddress(StubRoutines::verify_oop_subroutine_entry_address()));
@@ -709,7 +714,15 @@ void MacroAssembler::_verify_oop_addr(Address addr, const char* s, const char* f
     ResourceMark rm;
     stringStream ss;
     ss.print("verify_oop_addr: %s (%s:%d)", s, file, line);
-    b = code_string(ss.as_string());
+#if INCLUDE_CDS
+    if (AOTCodeCache::is_on_for_dump() && !code_section()->scratch_emit()) {
+      // This will duplicate string to preserve it.
+      b = AOTCodeCache::add_C_string(ss.as_string());
+    } else
+#endif
+    {
+      b = code_string(ss.as_string());
+    }
   }
   BLOCK_COMMENT("verify_oop_addr {");
 
@@ -722,13 +735,10 @@ void MacroAssembler::_verify_oop_addr(Address addr, const char* s, const char* f
     ld(x10, addr);
   }
 
-  {
-    // The length of the instruction sequence emitted should not depend
-    // on the address of the char buffer so that the size of mach nodes for
-    // scratch emit and normal emit matches.
-    IncompressibleScope scope(this); // Fixed length
-    movptr(t0, (address) b);
-  }
+  // The length of the instruction sequence emitted should not depend
+  // on the address of the char buffer so that the size of mach nodes for
+  // scratch emit and normal emit matches.
+  la(t0, ExternalAddress((address)b));
 
   // Call indirectly to solve generation ordering problem
   ld(t1, RuntimeAddress(StubRoutines::verify_oop_subroutine_entry_address()));
@@ -1255,13 +1265,33 @@ void MacroAssembler::wrap_label(Register r1, Register r2, Label &L,
 
 #undef INSN
 
-// cmov
+// cmov_zicond_eqz: dst = (cond == 0) ? src : dst
+void MacroAssembler::cmov_zicond_eqz(Register dst, Register src, Register cond, Register tmp) {
+  assert(UseZicond, "UseZicond must be enabled");
+  assert_different_registers(dst, src, cond);
+  czero_eqz(dst, dst, cond);
+  if (src != zr) {
+    czero_nez(tmp, src, cond);
+    add(dst, dst, tmp);
+  }
+}
+
+// cmov_zicond_nez: dst = (cond != 0) ? src : dst
+void MacroAssembler::cmov_zicond_nez(Register dst, Register src, Register cond, Register tmp) {
+  assert(UseZicond, "UseZicond must be enabled");
+  assert_different_registers(dst, src, cond);
+  czero_nez(dst, dst, cond);
+  if (src != zr) {
+    czero_eqz(tmp, src, cond);
+    add(dst, dst, tmp);
+  }
+}
+
 void MacroAssembler::cmov_eq(Register cmp1, Register cmp2, Register dst, Register src) {
+  assert_different_registers(dst, src);
   if (UseZicond) {
     xorr(t0, cmp1, cmp2);
-    czero_eqz(dst, dst, t0);
-    czero_nez(t0 , src, t0);
-    orr(dst, dst, t0);
+    cmov_zicond_eqz(dst, src, t0, t0);
     return;
   }
   Label no_set;
@@ -1271,11 +1301,10 @@ void MacroAssembler::cmov_eq(Register cmp1, Register cmp2, Register dst, Registe
 }
 
 void MacroAssembler::cmov_ne(Register cmp1, Register cmp2, Register dst, Register src) {
+  assert_different_registers(dst, src);
   if (UseZicond) {
     xorr(t0, cmp1, cmp2);
-    czero_nez(dst, dst, t0);
-    czero_eqz(t0 , src, t0);
-    orr(dst, dst, t0);
+    cmov_zicond_nez(dst, src, t0, t0);
     return;
   }
   Label no_set;
@@ -1285,11 +1314,10 @@ void MacroAssembler::cmov_ne(Register cmp1, Register cmp2, Register dst, Registe
 }
 
 void MacroAssembler::cmov_le(Register cmp1, Register cmp2, Register dst, Register src) {
+  assert_different_registers(dst, src);
   if (UseZicond) {
     slt(t0, cmp2, cmp1);
-    czero_eqz(dst, dst, t0);
-    czero_nez(t0,  src, t0);
-    orr(dst, dst, t0);
+    cmov_zicond_eqz(dst, src, t0, t0);
     return;
   }
   Label no_set;
@@ -1299,11 +1327,10 @@ void MacroAssembler::cmov_le(Register cmp1, Register cmp2, Register dst, Registe
 }
 
 void MacroAssembler::cmov_leu(Register cmp1, Register cmp2, Register dst, Register src) {
+  assert_different_registers(dst, src);
   if (UseZicond) {
     sltu(t0, cmp2, cmp1);
-    czero_eqz(dst, dst, t0);
-    czero_nez(t0,  src, t0);
-    orr(dst, dst, t0);
+    cmov_zicond_eqz(dst, src, t0, t0);
     return;
   }
   Label no_set;
@@ -1313,11 +1340,10 @@ void MacroAssembler::cmov_leu(Register cmp1, Register cmp2, Register dst, Regist
 }
 
 void MacroAssembler::cmov_ge(Register cmp1, Register cmp2, Register dst, Register src) {
+  assert_different_registers(dst, src);
   if (UseZicond) {
     slt(t0, cmp1, cmp2);
-    czero_eqz(dst, dst, t0);
-    czero_nez(t0,  src, t0);
-    orr(dst, dst, t0);
+    cmov_zicond_eqz(dst, src, t0, t0);
     return;
   }
   Label no_set;
@@ -1327,11 +1353,10 @@ void MacroAssembler::cmov_ge(Register cmp1, Register cmp2, Register dst, Registe
 }
 
 void MacroAssembler::cmov_geu(Register cmp1, Register cmp2, Register dst, Register src) {
+  assert_different_registers(dst, src);
   if (UseZicond) {
     sltu(t0, cmp1, cmp2);
-    czero_eqz(dst, dst, t0);
-    czero_nez(t0,  src, t0);
-    orr(dst, dst, t0);
+    cmov_zicond_eqz(dst, src, t0, t0);
     return;
   }
   Label no_set;
@@ -1341,11 +1366,10 @@ void MacroAssembler::cmov_geu(Register cmp1, Register cmp2, Register dst, Regist
 }
 
 void MacroAssembler::cmov_lt(Register cmp1, Register cmp2, Register dst, Register src) {
+  assert_different_registers(dst, src);
   if (UseZicond) {
     slt(t0, cmp1, cmp2);
-    czero_nez(dst, dst, t0);
-    czero_eqz(t0,  src, t0);
-    orr(dst, dst, t0);
+    cmov_zicond_nez(dst, src, t0, t0);
     return;
   }
   Label no_set;
@@ -1355,11 +1379,10 @@ void MacroAssembler::cmov_lt(Register cmp1, Register cmp2, Register dst, Registe
 }
 
 void MacroAssembler::cmov_ltu(Register cmp1, Register cmp2, Register dst, Register src) {
+  assert_different_registers(dst, src);
   if (UseZicond) {
     sltu(t0, cmp1, cmp2);
-    czero_nez(dst, dst, t0);
-    czero_eqz(t0,  src, t0);
-    orr(dst, dst, t0);
+    cmov_zicond_nez(dst, src, t0, t0);
     return;
   }
   Label no_set;
@@ -1369,11 +1392,10 @@ void MacroAssembler::cmov_ltu(Register cmp1, Register cmp2, Register dst, Regist
 }
 
 void MacroAssembler::cmov_gt(Register cmp1, Register cmp2, Register dst, Register src) {
+  assert_different_registers(dst, src);
   if (UseZicond) {
     slt(t0, cmp2, cmp1);
-    czero_nez(dst, dst, t0);
-    czero_eqz(t0,  src, t0);
-    orr(dst, dst, t0);
+    cmov_zicond_nez(dst, src, t0, t0);
     return;
   }
   Label no_set;
@@ -1383,11 +1405,10 @@ void MacroAssembler::cmov_gt(Register cmp1, Register cmp2, Register dst, Registe
 }
 
 void MacroAssembler::cmov_gtu(Register cmp1, Register cmp2, Register dst, Register src) {
+  assert_different_registers(dst, src);
   if (UseZicond) {
     sltu(t0, cmp2, cmp1);
-    czero_nez(dst, dst, t0);
-    czero_eqz(t0,  src, t0);
-    orr(dst, dst, t0);
+    cmov_zicond_nez(dst, src, t0, t0);
     return;
   }
   Label no_set;
@@ -3822,11 +3843,17 @@ void MacroAssembler::encode_heap_oop(Register d, Register s) {
       mv(d, s);
     }
   } else {
-    Label notNull;
-    sub(d, s, xheapbase);
-    bgez(d, notNull);
-    mv(d, zr);
-    bind(notNull);
+    if (UseZicond) {
+      assert_different_registers(s, t0);
+      sub(t0, s, xheapbase);
+      czero_eqz(d, t0, s);  // d = s == 0 ? 0 : t0
+    } else {
+      Label notNull;
+      sub(d, s, xheapbase);
+      bgez(d, notNull);
+      mv(d, zr);
+      bind(notNull);
+    }
     if (CompressedOops::shift() != 0) {
       assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
       srli(d, d, CompressedOops::shift());
@@ -4038,11 +4065,18 @@ void  MacroAssembler::decode_heap_oop(Register d, Register s) {
       slli(d, s, CompressedOops::shift());
     }
   } else {
-    Label done;
-    mv(d, s);
-    beqz(s, done);
-    shadd(d, s, xheapbase, d, LogMinObjAlignmentInBytes);
-    bind(done);
+    assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
+    if (UseZicond) {
+      assert_different_registers(s, t0);
+      shadd(t0, s, xheapbase, t0, LogMinObjAlignmentInBytes);
+      czero_eqz(d, t0, s);   // d = s == 0 ? 0 : t0
+    } else {
+      Label done;
+      mv(d, s);
+      beqz(s, done);
+      shadd(d, s, xheapbase, d, LogMinObjAlignmentInBytes);
+      bind(done);
+    }
   }
   verify_oop_msg(d, "broken oop in decode_heap_oop");
 }
@@ -5353,7 +5387,9 @@ void MacroAssembler::verify_secondary_supers_table(Register r_sub_klass,
     mv(x11, r_sub_klass);
     mv(x12, tmp3);
     mv(x13, result);
-    mv(x14, (address)("mismatch"));
+    const char* msg = "mismatch";
+    const char* str = (code_section()->scratch_emit()) ? msg : AOTCodeCache::add_C_string(msg);
+    la(x14, ExternalAddress((address) str));
     rt_call(CAST_FROM_FN_PTR(address, Klass::on_secondary_supers_verification_failure));
     should_not_reach_here();
   }

src/hotspot/cpu/riscv/macroAssembler_riscv.hpp

@@ -685,6 +685,9 @@ class MacroAssembler: public Assembler {
   void bltz(Register Rs, const address dest);
   void bgtz(Register Rs, const address dest);
 
+  void cmov_zicond_eqz(Register dst, Register src, Register cond, Register tmp = t0);
+  void cmov_zicond_nez(Register dst, Register src, Register cond, Register tmp = t0);
+
   void cmov_eq(Register cmp1, Register cmp2, Register dst, Register src);
   void cmov_ne(Register cmp1, Register cmp2, Register dst, Register src);
   void cmov_le(Register cmp1, Register cmp2, Register dst, Register src);
@@ -1839,7 +1842,7 @@ public:
   static bool is_pc_relative_at(address branch);
 
   static bool is_membar(address addr) {
-    return (Bytes::get_native_u4(addr) & 0x7f) == 0b1111 && extract_funct3(addr) == 0;
+    return (Assembler::ld_instr(addr) & 0x7f) == 0b1111 && extract_funct3(addr) == 0;
   }
   static uint32_t get_membar_kind(address addr);
   static void set_membar_kind(address addr, uint32_t order_kind);

src/hotspot/cpu/riscv/nativeInst_riscv.cpp

@@ -234,7 +234,7 @@ void NativeMovConstReg::verify() {
 intptr_t NativeMovConstReg::data() const {
   address addr = MacroAssembler::target_addr_for_insn(instruction_address());
   if (maybe_cpool_ref(instruction_address())) {
-    return Bytes::get_native_u8(addr);
+    return MacroAssembler::get_native_u8(addr);
   } else {
     return (intptr_t)addr;
   }
@@ -243,7 +243,7 @@ intptr_t NativeMovConstReg::data() const {
 void NativeMovConstReg::set_data(intptr_t x) {
   if (maybe_cpool_ref(instruction_address())) {
     address addr = MacroAssembler::target_addr_for_insn(instruction_address());
-    Bytes::put_native_u8(addr, x);
+    MacroAssembler::put_native_u8(addr, x);
   } else {
     // Store x into the instruction stream.
     MacroAssembler::pd_patch_instruction_size(instruction_address(), (address)x);
@@ -259,11 +259,11 @@ void NativeMovConstReg::set_data(intptr_t x) {
     while (iter.next()) {
       if (iter.type() == relocInfo::oop_type) {
         oop* oop_addr = iter.oop_reloc()->oop_addr();
-        Bytes::put_native_u8((address)oop_addr, x);
+        MacroAssembler::put_native_u8((address)oop_addr, x);
         break;
       } else if (iter.type() == relocInfo::metadata_type) {
         Metadata** metadata_addr = iter.metadata_reloc()->metadata_addr();
-        Bytes::put_native_u8((address)metadata_addr, x);
+        MacroAssembler::put_native_u8((address)metadata_addr, x);
         break;
       }
     }

src/hotspot/cpu/riscv/nativeInst_riscv.hpp

@@ -78,19 +78,19 @@ class NativeInstruction {
 
  protected:
   address addr_at(int offset) const { return address(this) + offset; }
-  jint     int_at(int offset) const { return (jint)      Bytes::get_native_u4(addr_at(offset));  }
-  juint   uint_at(int offset) const { return             Bytes::get_native_u4(addr_at(offset));  }
-  address  ptr_at(int offset) const { return (address)   Bytes::get_native_u8(addr_at(offset));  }
-  oop      oop_at(int offset) const { return cast_to_oop(Bytes::get_native_u8(addr_at(offset))); }
+  jint     int_at(int offset) const { return (jint)      MacroAssembler::get_native_u4(addr_at(offset));  }
+  juint   uint_at(int offset) const { return             MacroAssembler::get_native_u4(addr_at(offset));  }
+  address  ptr_at(int offset) const { return (address)   MacroAssembler::get_native_u8(addr_at(offset));  }
+  oop      oop_at(int offset) const { return cast_to_oop(MacroAssembler::get_native_u8(addr_at(offset))); }
 
 
-  void  set_int_at(int offset, jint i)      { Bytes::put_native_u4(addr_at(offset), i); }
-  void set_uint_at(int offset, jint i)      { Bytes::put_native_u4(addr_at(offset), i); }
-  void  set_ptr_at(int offset, address ptr) { Bytes::put_native_u8(addr_at(offset), (u8)ptr); }
-  void  set_oop_at(int offset, oop o)       { Bytes::put_native_u8(addr_at(offset), cast_from_oop<u8>(o)); }
+  void  set_int_at(int offset, jint i)      { MacroAssembler::put_native_u4(addr_at(offset), i); }
+  void set_uint_at(int offset, juint i)     { MacroAssembler::put_native_u4(addr_at(offset), i); }
+  void  set_ptr_at(int offset, address ptr) { MacroAssembler::put_native_u8(addr_at(offset), (u8)ptr); }
+  void  set_oop_at(int offset, oop o)       { MacroAssembler::put_native_u8(addr_at(offset), cast_from_oop<u8>(o)); }
 
-  static void     set_data64_at(address dest, uint64_t data) { Bytes::put_native_u8(dest, (u8)data); }
-  static uint64_t get_data64_at(address src)                 { return Bytes::get_native_u8(src); }
+  static void     set_data64_at(address dest, uint64_t data) { MacroAssembler::put_native_u8(dest, (u8)data); }
+  static uint64_t get_data64_at(address src)                 { return MacroAssembler::get_native_u8(src); }
 
  public:
   inline friend NativeInstruction* nativeInstruction_at(address addr);

src/hotspot/cpu/riscv/relocInfo_riscv.cpp

@@ -44,7 +44,7 @@ void Relocation::pd_set_data_value(address x, bool verify_only) {
       if (MacroAssembler::is_load_pc_relative_at(addr())) {
         address constptr = (address)code()->oop_addr_at(reloc->oop_index());
         bytes = MacroAssembler::pd_patch_instruction_size(addr(), constptr);
-        assert((address)Bytes::get_native_u8(constptr) == x, "error in oop relocation");
+        assert((address)MacroAssembler::get_native_u8(constptr) == x, "error in oop relocation");
       } else {
         bytes = MacroAssembler::patch_oop(addr(), x);
       }

src/hotspot/cpu/riscv/riscv.ad

@@ -1557,12 +1557,16 @@ uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *r
   // address also needs t0 to materialize an offset outside the 12-bit range.
   if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
     int last_dst_offset = dst_offset;
-    if (bottom_type()->isa_pvectmask()) {
-      int vmask_size_in_bytes = Matcher::scalable_predicate_reg_slots() * 32 / 8;
-      last_dst_offset += vmask_size_in_bytes - 4;
-    } else if (ideal_reg() == Op_VecA) {
-      int vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
-      last_dst_offset += vector_reg_size_in_bytes - 8;
+    if (bottom_type()->isa_vect() != nullptr) {
+      if (!bottom_type()->isa_pvectmask()) {
+        assert(ideal_reg() == Op_VecA, "Must be Op_VecA");
+        int vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
+        last_dst_offset += vector_reg_size_in_bytes - 8;
+      } else {
+        assert(ideal_reg() == Op_RegVectMask, "Must be Op_RegVectMask");
+        int vmask_size_in_bytes = Matcher::scalable_predicate_reg_slots() * 32 / 8;
+        last_dst_offset += vmask_size_in_bytes - 4;
+      }
     }
 
     if (masm != nullptr && !Assembler::is_simm12(last_dst_offset)) {
@@ -1574,9 +1578,9 @@ uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *r
     }
   }
 
-  if (bottom_type()->isa_vect() != nullptr) {
-    uint ireg = ideal_reg();
-    if (ireg == Op_VecA && masm) {
+  if (bottom_type()->isa_vect() != nullptr && masm != nullptr) {
+    if (!bottom_type()->isa_pvectmask()) {
+      assert(ideal_reg() == Op_VecA, "Must be Op_VecA");
       int vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
         // stack to stack
@@ -1595,7 +1599,8 @@ uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *r
       } else {
         ShouldNotReachHere();
       }
-    } else if (bottom_type()->isa_pvectmask() && masm) {
+    } else {
+      assert(ideal_reg() == Op_RegVectMask, "Must be Op_RegVectMask");
       int vmask_size_in_bytes = Matcher::scalable_predicate_reg_slots() * 32 / 8;
       if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
         // stack to stack
@@ -1698,14 +1703,11 @@ uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *r
       st->print("%s", Matcher::regName[dst_lo]);
     }
     if (bottom_type()->isa_vect() && !bottom_type()->isa_pvectmask()) {
-      int vsize = 0;
-      if (ideal_reg() == Op_VecA) {
-        vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
-      } else {
-        ShouldNotReachHere();
-      }
+      assert(ideal_reg() == Op_VecA, "Must be Op_VecA");
+      int vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
       st->print("\t# vector spill size = %d", vsize);
-    } else if (ideal_reg() == Op_RegVectMask) {
+    } else if (bottom_type()->isa_pvectmask()) {
+      assert(ideal_reg() == Op_RegVectMask, "Must be Op_RegVectMask");
       assert(Matcher::supports_scalable_vector(), "bad register type for spill");
       int vsize = Matcher::scalable_predicate_reg_slots() * 32;
       st->print("\t# vmask spill size = %d", vsize);
@@ -2848,6 +2850,16 @@ operand immIpowerOf2() %{
   interface(CONST_INTER);
 %}
 
+// Long Immediate: low 16-bit mask
+operand immL_16bits()
+%{
+  predicate(n->get_long() == 0xFFFFL);
+  match(ConL);
+  op_cost(0);
+  format %{ %}
+  interface(CONST_INTER);
+%}
+
 // Long Immediate: low 32-bit mask
 operand immL_32bits()
 %{
@@ -8575,6 +8587,157 @@ instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
   ins_pipe(ialu_reg);
 %}
 
+// Fused int-producer + ConvI2L rules.
+instruct convI2L_addI_reg_reg(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
+  match(Set dst (ConvI2L (AddI src1 src2)));
+
+  ins_cost(ALU_COST);
+  format %{ "addw  $dst, $src1, $src2\t#@convI2L_addI_reg_reg" %}
+
+  ins_encode %{
+    __ addw(as_Register($dst$$reg),
+            as_Register($src1$$reg),
+            as_Register($src2$$reg));
+  %}
+
+  ins_pipe(ialu_reg_reg);
+%}
+
+instruct convI2L_addI_reg_imm(iRegLNoSp dst, iRegIorL2I src1, immIAdd src2) %{
+  match(Set dst (ConvI2L (AddI src1 src2)));
+
+  ins_cost(ALU_COST);
+  format %{ "addiw  $dst, $src1, $src2\t#@convI2L_addI_reg_imm" %}
+
+  ins_encode %{
+    __ addiw(as_Register($dst$$reg),
+             as_Register($src1$$reg),
+             $src2$$constant);
+  %}
+
+  ins_pipe(ialu_reg_imm);
+%}
+
+instruct convI2L_subI_reg_reg(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
+  match(Set dst (ConvI2L (SubI src1 src2)));
+
+  ins_cost(ALU_COST);
+  format %{ "subw  $dst, $src1, $src2\t#@convI2L_subI_reg_reg" %}
+
+  ins_encode %{
+    __ subw(as_Register($dst$$reg),
+            as_Register($src1$$reg),
+            as_Register($src2$$reg));
+  %}
+
+  ins_pipe(ialu_reg_reg);
+%}
+
+instruct convI2L_mulI_reg_reg(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
+  match(Set dst (ConvI2L (MulI src1 src2)));
+
+  ins_cost(IMUL_COST);
+  format %{ "mulw  $dst, $src1, $src2\t#@convI2L_mulI_reg_reg" %}
+
+  ins_encode %{
+    __ mulw(as_Register($dst$$reg),
+            as_Register($src1$$reg),
+            as_Register($src2$$reg));
+  %}
+
+  ins_pipe(imul_reg_reg);
+%}
+
+instruct convI2L_lShiftI_reg_reg(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
+  match(Set dst (ConvI2L (LShiftI src1 src2)));
+
+  ins_cost(ALU_COST);
+  format %{ "sllw  $dst, $src1, $src2\t#@convI2L_lShiftI_reg_reg" %}
+
+  ins_encode %{
+    __ sllw(as_Register($dst$$reg),
+            as_Register($src1$$reg),
+            as_Register($src2$$reg));
+  %}
+
+  ins_pipe(ialu_reg_reg_vshift);
+%}
+
+instruct convI2L_lShiftI_reg_imm(iRegLNoSp dst, iRegIorL2I src1, immI src2) %{
+  match(Set dst (ConvI2L (LShiftI src1 src2)));
+
+  ins_cost(ALU_COST);
+  format %{ "slliw  $dst, $src1, ($src2 & 0x1f)\t#@convI2L_lShiftI_reg_imm" %}
+
+  ins_encode %{
+    __ slliw(as_Register($dst$$reg),
+             as_Register($src1$$reg),
+             (unsigned) $src2$$constant & 0x1f);
+  %}
+
+  ins_pipe(ialu_reg_shift);
+%}
+
+instruct convI2L_urShiftI_reg_reg(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
+  match(Set dst (ConvI2L (URShiftI src1 src2)));
+
+  ins_cost(ALU_COST);
+  format %{ "srlw  $dst, $src1, $src2\t#@convI2L_urShiftI_reg_reg" %}
+
+  ins_encode %{
+    __ srlw(as_Register($dst$$reg),
+            as_Register($src1$$reg),
+            as_Register($src2$$reg));
+  %}
+
+  ins_pipe(ialu_reg_reg_vshift);
+%}
+
+instruct convI2L_urShiftI_reg_imm(iRegLNoSp dst, iRegIorL2I src1, immI src2) %{
+  match(Set dst (ConvI2L (URShiftI src1 src2)));
+
+  ins_cost(ALU_COST);
+  format %{ "srliw  $dst, $src1, ($src2 & 0x1f)\t#@convI2L_urShiftI_reg_imm" %}
+
+  ins_encode %{
+    __ srliw(as_Register($dst$$reg),
+             as_Register($src1$$reg),
+             (unsigned) $src2$$constant & 0x1f);
+  %}
+
+  ins_pipe(ialu_reg_shift);
+%}
+
+instruct convI2L_rShiftI_reg_reg(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
+  match(Set dst (ConvI2L (RShiftI src1 src2)));
+
+  ins_cost(ALU_COST);
+  format %{ "sraw  $dst, $src1, $src2\t#@convI2L_rShiftI_reg_reg" %}
+
+  ins_encode %{
+    __ sraw(as_Register($dst$$reg),
+            as_Register($src1$$reg),
+            as_Register($src2$$reg));
+  %}
+
+  ins_pipe(ialu_reg_reg_vshift);
+%}
+
+instruct convI2L_rShiftI_reg_imm(iRegLNoSp dst, iRegIorL2I src1, immI src2) %{
+  match(Set dst (ConvI2L (RShiftI src1 src2)));
+
+  ins_cost(ALU_COST);
+  format %{ "sraiw  $dst, $src1, ($src2 & 0x1f)\t#@convI2L_rShiftI_reg_imm" %}
+
+  ins_encode %{
+    __ sraiw(as_Register($dst$$reg),
+             as_Register($src1$$reg),
+             (unsigned) $src2$$constant & 0x1f);
+  %}
+
+  ins_pipe(ialu_reg_shift);
+%}
+
 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
   match(Set dst (ConvL2I src));
 

src/hotspot/cpu/riscv/riscv_b.ad

@@ -183,7 +183,37 @@ instruct convI2UL_reg_reg_b(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask) %{
     __ zext_w(as_Register($dst$$reg), as_Register($src$$reg));
   %}
 
-  ins_pipe(ialu_reg_shift);
+  ins_pipe(ialu_reg);
+%}
+
+// And with a low 16-bit mask
+instruct andL_16bits_b(iRegLNoSp dst, iRegL src, immL_16bits mask) %{
+  predicate(UseZbb);
+  match(Set dst (AndL src mask));
+
+  format %{ "zext.h  $dst, $src\t#@andL_16bits_b" %}
+
+  ins_cost(ALU_COST);
+  ins_encode %{
+    __ zext_h(as_Register($dst$$reg), as_Register($src$$reg));
+  %}
+
+  ins_pipe(ialu_reg);
+%}
+
+// And with a low 32-bit mask
+instruct andL_32bits_b(iRegLNoSp dst, iRegL src, immL_32bits mask) %{
+  predicate(UseZba);
+  match(Set dst (AndL src mask));
+
+  format %{ "zext.w  $dst, $src\t#@andL_32bits_b" %}
+
+  ins_cost(ALU_COST);
+  ins_encode %{
+    __ zext_w(as_Register($dst$$reg), as_Register($src$$reg));
+  %}
+
+  ins_pipe(ialu_reg);
 %}
 
 // BSWAP instructions

src/hotspot/cpu/riscv/riscv_v.ad

@@ -1580,6 +1580,42 @@ instruct vnotL_masked(vReg dst_src, immI_M1 m1, vRegMask_V0 v0) %{
   ins_pipe(pipe_slow);
 %}
 
+// vector integer div
+
+instruct vdiv(vReg dst, vReg src1, vReg src2) %{
+  match(Set dst (DivVB src1 src2));
+  match(Set dst (DivVS src1 src2));
+  match(Set dst (DivVI src1 src2));
+  match(Set dst (DivVL src1 src2));
+  format %{ "vdiv $dst, $src1, $src2" %}
+  ins_encode %{
+    BasicType bt = Matcher::vector_element_basic_type(this);
+    __ vsetvli_helper(bt, Matcher::vector_length(this));
+    __ vdiv_vv(as_VectorRegister($dst$$reg),
+               as_VectorRegister($src1$$reg),
+               as_VectorRegister($src2$$reg));
+  %}
+  ins_pipe(pipe_slow);
+%}
+
+// vector integer div - predicated
+
+instruct vdiv_masked(vReg dst_src1, vReg src2, vRegMask_V0 v0) %{
+  match(Set dst_src1 (DivVB (Binary dst_src1 src2) v0));
+  match(Set dst_src1 (DivVS (Binary dst_src1 src2) v0));
+  match(Set dst_src1 (DivVI (Binary dst_src1 src2) v0));
+  match(Set dst_src1 (DivVL (Binary dst_src1 src2) v0));
+  format %{ "vdiv_masked $dst_src1, $dst_src1, $src2, $v0" %}
+  ins_encode %{
+    BasicType bt = Matcher::vector_element_basic_type(this);
+    __ vsetvli_helper(bt, Matcher::vector_length(this));
+    __ vdiv_vv(as_VectorRegister($dst_src1$$reg),
+               as_VectorRegister($dst_src1$$reg),
+               as_VectorRegister($src2$$reg), Assembler::v0_t);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
 // vector float div
 
 instruct vdiv_hfp(vReg dst, vReg src1, vReg src2) %{
@@ -4482,7 +4518,7 @@ instruct vmaskAllL(vRegMask dst, iRegL src) %{
 
 // ------------------------------ Vector mask basic OPs ------------------------
 
-// vector mask logical ops: and/and-not/or/xor
+// vector mask logical ops
 
 instruct vmask_and(vRegMask dst, vRegMask src1, vRegMask src2) %{
   match(Set dst (AndVMask src1 src2));
@@ -4523,6 +4559,136 @@ instruct vmask_and_notL(vRegMask dst, vRegMask src1, vRegMask src2, immL_M1 m1)
   ins_pipe(pipe_slow);
 %}
 
+instruct vmask_or_notI(vRegMask dst, vRegMask src1, vRegMask src2, immI_M1 m1) %{
+  match(Set dst (OrVMask src1 (XorVMask src2 (MaskAll m1))));
+  format %{ "vmask_or_notI $dst, $src1, $src2" %}
+  ins_encode %{
+    BasicType bt = Matcher::vector_element_basic_type(this);
+    __ vsetvli_helper(bt, Matcher::vector_length(this));
+    __ vmorn_mm(as_VectorRegister($dst$$reg),
+                as_VectorRegister($src1$$reg),
+                as_VectorRegister($src2$$reg));
+  %}
+  ins_pipe(pipe_slow);
+%}
+
+instruct vmask_or_notL(vRegMask dst, vRegMask src1, vRegMask src2, immL_M1 m1) %{
+  match(Set dst (OrVMask src1 (XorVMask src2 (MaskAll m1))));
+  format %{ "vmask_or_notL $dst, $src1, $src2" %}
+  ins_encode %{
+    BasicType bt = Matcher::vector_element_basic_type(this);
+    __ vsetvli_helper(bt, Matcher::vector_length(this));
+    __ vmorn_mm(as_VectorRegister($dst$$reg),
+                as_VectorRegister($src1$$reg),
+                as_VectorRegister($src2$$reg));
+  %}
+  ins_pipe(pipe_slow);
+%}
+
+instruct vmask_nandI(vRegMask dst, vRegMask src1, vRegMask src2, immI_M1 m1) %{
+  match(Set dst (XorVMask (AndVMask src1 src2) (MaskAll m1)));
+  format %{ "vmask_nandI $dst, $src1, $src2" %}
+  ins_encode %{
+    BasicType bt = Matcher::vector_element_basic_type(this);
+    __ vsetvli_helper(bt, Matcher::vector_length(this));
+    __ vmnand_mm(as_VectorRegister($dst$$reg),
+                 as_VectorRegister($src1$$reg),
+                 as_VectorRegister($src2$$reg));
+  %}
+  ins_pipe(pipe_slow);
+%}
+
+instruct vmask_nandL(vRegMask dst, vRegMask src1, vRegMask src2, immL_M1 m1) %{
+  match(Set dst (XorVMask (AndVMask src1 src2) (MaskAll m1)));
+  format %{ "vmask_nandL $dst, $src1, $src2" %}
+  ins_encode %{
+    BasicType bt = Matcher::vector_element_basic_type(this);
+    __ vsetvli_helper(bt, Matcher::vector_length(this));
+    __ vmnand_mm(as_VectorRegister($dst$$reg),
+                 as_VectorRegister($src1$$reg),
+                 as_VectorRegister($src2$$reg));
+  %}
+  ins_pipe(pipe_slow);
+%}
+
+instruct vmask_norI(vRegMask dst, vRegMask src1, vRegMask src2, immI_M1 m1) %{
+  match(Set dst (XorVMask (OrVMask src1 src2) (MaskAll m1)));
+  format %{ "vmask_norI $dst, $src1, $src2" %}
+  ins_encode %{
+    BasicType bt = Matcher::vector_element_basic_type(this);
+    __ vsetvli_helper(bt, Matcher::vector_length(this));
+    __ vmnor_mm(as_VectorRegister($dst$$reg),
+                as_VectorRegister($src1$$reg),
+                as_VectorRegister($src2$$reg));
+  %}
+  ins_pipe(pipe_slow);
+%}
+
+instruct vmask_norL(vRegMask dst, vRegMask src1, vRegMask src2, immL_M1 m1) %{
+  match(Set dst (XorVMask (OrVMask src1 src2) (MaskAll m1)));
+  format %{ "vmask_norL $dst, $src1, $src2" %}
+  ins_encode %{
+    BasicType bt = Matcher::vector_element_basic_type(this);
+    __ vsetvli_helper(bt, Matcher::vector_length(this));
+    __ vmnor_mm(as_VectorRegister($dst$$reg),
+                as_VectorRegister($src1$$reg),
+                as_VectorRegister($src2$$reg));
+  %}
+  ins_pipe(pipe_slow);
+%}
+
+instruct vmask_xnorI(vRegMask dst, vRegMask src1, vRegMask src2, immI_M1 m1) %{
+  match(Set dst (XorVMask (XorVMask src1 src2) (MaskAll m1)));
+  match(Set dst (XorVMask src1 (XorVMask src2 (MaskAll m1))));
+  format %{ "vmask_xnorI $dst, $src1, $src2" %}
+  ins_encode %{
+    BasicType bt = Matcher::vector_element_basic_type(this);
+    __ vsetvli_helper(bt, Matcher::vector_length(this));
+    __ vmxnor_mm(as_VectorRegister($dst$$reg),
+                 as_VectorRegister($src1$$reg),
+                 as_VectorRegister($src2$$reg));
+  %}
+  ins_pipe(pipe_slow);
+%}
+
+instruct vmask_xnorL(vRegMask dst, vRegMask src1, vRegMask src2, immL_M1 m1) %{
+  match(Set dst (XorVMask (XorVMask src1 src2) (MaskAll m1)));
+  match(Set dst (XorVMask src1 (XorVMask src2 (MaskAll m1))));
+  format %{ "vmask_xnorL $dst, $src1, $src2" %}
+  ins_encode %{
+    BasicType bt = Matcher::vector_element_basic_type(this);
+    __ vsetvli_helper(bt, Matcher::vector_length(this));
+    __ vmxnor_mm(as_VectorRegister($dst$$reg),
+                 as_VectorRegister($src1$$reg),
+                 as_VectorRegister($src2$$reg));
+  %}
+  ins_pipe(pipe_slow);
+%}
+
+instruct vmask_notI(vRegMask dst, vRegMask src, immI_M1 m1) %{
+  match(Set dst (XorVMask src (MaskAll m1)));
+  format %{ "vmask_notI $dst, $src" %}
+  ins_encode %{
+    BasicType bt = Matcher::vector_element_basic_type(this);
+    __ vsetvli_helper(bt, Matcher::vector_length(this));
+    __ vmnot_m(as_VectorRegister($dst$$reg),
+               as_VectorRegister($src$$reg));
+  %}
+  ins_pipe(pipe_slow);
+%}
+
+instruct vmask_notL(vRegMask dst, vRegMask src, immL_M1 m1) %{
+  match(Set dst (XorVMask src (MaskAll m1)));
+  format %{ "vmask_notL $dst, $src" %}
+  ins_encode %{
+    BasicType bt = Matcher::vector_element_basic_type(this);
+    __ vsetvli_helper(bt, Matcher::vector_length(this));
+    __ vmnot_m(as_VectorRegister($dst$$reg),
+               as_VectorRegister($src$$reg));
+  %}
+  ins_pipe(pipe_slow);
+%}
+
 instruct vmask_or(vRegMask dst, vRegMask src1, vRegMask src2) %{
   match(Set dst (OrVMask src1 src2));
   format %{ "vmask_or $dst, $src1, $src2" %}

src/hotspot/cpu/riscv/runtime_riscv.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2024, Red Hat Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -246,8 +246,7 @@ UncommonTrapBlob* OptoRuntime::generate_uncommon_trap_blob() {
   // Jump to interpreter
   __ ret();
 
-  // Make sure all code is generated
-  masm->flush();
+  // Code will be copied. No ICache sync required.
 
   UncommonTrapBlob* ut_blob = UncommonTrapBlob::create(&buffer, oop_maps,
                                                        SimpleRuntimeFrame::framesize >> 1);
@@ -389,8 +388,7 @@ ExceptionBlob* OptoRuntime::generate_exception_blob() {
 
   __ jr(t1);
 
-  // Make sure all code is generated
-  masm->flush();
+  // Code will be copied. No ICache sync required.
 
   // Set exception blob
   ExceptionBlob* ex_blob = ExceptionBlob::create(&buffer, oop_maps, SimpleRuntimeFrame::framesize >> 1);

src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp

@@ -1392,7 +1392,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
     assert(vep_offset != -1,        "Must be set");
 #endif
 
-    __ flush();
+    // Code will be copied. No ICache sync required.
     nmethod* nm = nmethod::new_native_nmethod(method,
                                               compile_id,
                                               masm->code(),
@@ -1430,7 +1430,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
                          in_sig_bt,
                          in_regs);
     int frame_complete = ((intptr_t)__ pc()) - start;  // not complete, period
-    __ flush();
+    // Code will be copied. No ICache sync required.
     int stack_slots = SharedRuntime::out_preserve_stack_slots();  // no out slots at all, actually
     return nmethod::new_native_nmethod(method,
                                        compile_id,
@@ -1817,14 +1817,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
 
   Label safepoint_in_progress, safepoint_in_progress_done;
 
-  // Switch thread to "native transition" state before reading the synchronization state.
-  // This additional state is necessary because reading and testing the synchronization
-  // state is not atomic w.r.t. GC, as this scenario demonstrates:
-  //     Java thread A, in _thread_in_native state, loads _not_synchronized and is preempted.
-  //     VM thread changes sync state to synchronizing and suspends threads for GC.
-  //     Thread A is resumed to finish this native method, but doesn't block here since it
-  //     didn't see any synchronization is progress, and escapes.
-  __ mv(t0, _thread_in_native_trans);
+  __ mv(t0, _thread_in_vm);
 
   __ sw(t0, Address(xthread, JavaThread::thread_state_offset()));
 
@@ -2089,7 +2082,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
     }
   }
 
-  __ flush();
+  // Code will be copied. No ICache sync required.
 
   nmethod *nm = nmethod::new_native_nmethod(method,
                                             compile_id,
@@ -2428,8 +2421,7 @@ void SharedRuntime::generate_deopt_blob() {
   // Jump to interpreter
   __ ret();
 
-  // Make sure all code is generated
-  masm->flush();
+  // Code will be copied. No ICache sync required.
 
   _deopt_blob = DeoptimizationBlob::create(&buffer, oop_maps, 0, exception_offset, reexecute_offset, frame_size_in_words);
   assert(_deopt_blob != nullptr, "create deoptimization blob fail!");
@@ -2574,8 +2566,7 @@ SafepointBlob* SharedRuntime::generate_handler_blob(StubId id, address call_ptr)
   __ stop("Attempting to adjust pc to skip safepoint poll but the return point is not what we expected");
 #endif
 
-  // Make sure all code is generated
-  masm->flush();
+  // Code will be copied. No ICache sync required.
 
   // Fill-out other meta info
   SafepointBlob* sp_blob = SafepointBlob::create(&buffer, oop_maps, frame_size_in_words);
@@ -2670,9 +2661,7 @@ RuntimeStub* SharedRuntime::generate_resolve_blob(StubId id, address destination
   __ ld(x10, Address(xthread, Thread::pending_exception_offset()));
   __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
 
-  // -------------
-  // make sure all code is generated
-  masm->flush();
+  // Code will be copied. No ICache sync required.
 
   // return the  blob
   RuntimeStub* rs_blob = RuntimeStub::new_runtime_stub(name, &buffer, frame_complete, frame_size_in_words, oop_maps, true);

src/hotspot/cpu/riscv/templateInterpreterGenerator_riscv.cpp

@@ -1213,7 +1213,7 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) {
   // Force all preceding writes to be observed prior to thread state change
   __ membar(MacroAssembler::LoadStore | MacroAssembler::StoreStore);
 
-  __ mv(t0, _thread_in_native_trans);
+  __ mv(t0, _thread_in_vm);
   __ sw(t0, Address(xthread, JavaThread::thread_state_offset()));
 
   // Force this write out before the read below

src/hotspot/cpu/riscv/upcallLinker_riscv.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -330,7 +330,7 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Symbol* signature,
 
   //////////////////////////////////////////////////////////////////////////////
 
-  __ flush();
+  // Code will be copied. No ICache sync required.
 
 #ifndef PRODUCT
   stringStream ss;

src/hotspot/cpu/riscv/vtableStubs_riscv.cpp

@@ -137,7 +137,7 @@ VtableStub* VtableStubs::create_vtable_stub(int vtable_index, bool caller_is_c1)
   __ ld(t1, Address(xmethod, entry_offset));
   __ jr(t1);
 
-  masm->flush();
+  masm->invalidate_icache();
   bookkeeping(masm, tty, s, npe_addr, ame_addr, true, vtable_index, slop_bytes, 0);
 
   return s;
@@ -246,7 +246,7 @@ VtableStub* VtableStubs::create_itable_stub(int itable_index, bool caller_is_c1)
   assert(SharedRuntime::get_handle_wrong_method_stub() != nullptr, "check initialization order");
   __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 
-  masm->flush();
+  masm->invalidate_icache();
   bookkeeping(masm, tty, s, npe_addr, ame_addr, false, itable_index, slop_bytes, 0);
 
   return s;

src/hotspot/cpu/s390/bytes_s390.hpp

@@ -1,63 +0,0 @@
-/*
- * Copyright (c) 2016, 2022, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2016, 2022 SAP SE. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#ifndef CPU_S390_BYTES_S390_HPP
-#define CPU_S390_BYTES_S390_HPP
-
-#include "memory/allStatic.hpp"
-
-class Bytes: AllStatic {
- public:
-  // Efficient reading and writing of unaligned unsigned data in
-  // platform-specific byte ordering.
-
-  // Use regular load and store for unaligned access.
-  //
-  // On z/Architecture, unaligned loads and stores are supported when using the
-  // "traditional" load (LH, L/LY, LG) and store (STH, ST/STY, STG) instructions.
-  // The penalty for unaligned access is just very few (two or three) ticks,
-  // plus another few (two or three) ticks if the access crosses a cache line boundary.
-  //
-  // In short, it makes no sense on z/Architecture to piecemeal get or put unaligned data.
-
-  static inline u2   get_native_u2(address p) { return *(u2*)p; }
-  static inline u4   get_native_u4(address p) { return *(u4*)p; }
-  static inline u8   get_native_u8(address p) { return *(u8*)p; }
-
-  static inline void put_native_u2(address p, u2 x) { *(u2*)p = x; }
-  static inline void put_native_u4(address p, u4 x) { *(u4*)p = x; }
-  static inline void put_native_u8(address p, u8 x) { *(u8*)p = x; }
-
-  // Efficient reading and writing of unaligned unsigned data in Java byte ordering (i.e. big-endian ordering)
-  static inline u2   get_Java_u2(address p) { return get_native_u2(p); }
-  static inline u4   get_Java_u4(address p) { return get_native_u4(p); }
-  static inline u8   get_Java_u8(address p) { return get_native_u8(p); }
-
-  static inline void put_Java_u2(address p, u2 x) { put_native_u2(p, x); }
-  static inline void put_Java_u4(address p, u4 x) { put_native_u4(p, x); }
-  static inline void put_Java_u8(address p, u8 x) { put_native_u8(p, x); }
-};
-
-#endif // CPU_S390_BYTES_S390_HPP

src/hotspot/cpu/s390/downcallLinker_s390.cpp

@@ -247,7 +247,7 @@ void DowncallLinker::StubGenerator::generate() {
 
   if (_needs_transition) {
     __ block_comment("thread_native2java {");
-    __ set_thread_state(_thread_in_native_trans);
+    __ set_thread_state(_thread_in_vm);
 
     if (!UseSystemMemoryBarrier) {
       __ z_fence(); // Order state change wrt. safepoint poll.
@@ -316,5 +316,5 @@ void DowncallLinker::StubGenerator::generate() {
 
   //////////////////////////////////////////////////////////////////////////////
 
-  __ flush();
+  // Code will be copied. No ICache sync required.
 }

src/hotspot/cpu/s390/interpreterRT_s390.cpp

@@ -1,6 +1,6 @@
 /*
- * Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2016, 2023 SAP SE. All rights reserved.
+ * Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2026 SAP SE. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -139,7 +139,7 @@ void InterpreterRuntime::SignatureHandlerGenerator::generate(uint64_t fingerprin
   iterate(fingerprint);
   __ load_const_optimized(Z_RET, AbstractInterpreter::result_handler(method()->result_type()));
   __ z_br(Z_R14);
-  __ flush();
+  __ invalidate_icache();
 }
 
 #undef  __

src/hotspot/cpu/s390/jniFastGetField_s390.cpp

@@ -1,6 +1,6 @@
 /*
- * Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2016 SAP SE. All rights reserved.
+ * Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2026 SAP SE. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -138,7 +138,7 @@ address JNI_FastGetField::generate_fast_get_int_field0(BasicType type) {
   __ load_const_optimized(Robj, slow_case_addr);
   __ z_br(Robj); // tail call
 
-  __ flush();
+  __ invalidate_icache();
 
   return fast_entry;
 }

src/hotspot/cpu/s390/runtime_s390.cpp

@@ -1,6 +1,6 @@
 /*
- * Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2016, 2023 SAP SE. All rights reserved.
+ * Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2026 SAP SE. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -142,8 +142,7 @@ ExceptionBlob* OptoRuntime::generate_exception_blob() {
 
   __ z_br(handle_exception);
 
-  // Make sure all code is generated.
-  masm->flush();
+  // Code will be copied. No ICache sync required.
 
   // Set exception blob.
   OopMapSet *oop_maps = nullptr;

src/hotspot/cpu/s390/sharedRuntime_s390.cpp

@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2016, 2024 SAP SE. All rights reserved.
+ * Copyright (c) 2016, 2026 SAP SE. All rights reserved.
  * Copyright (c) 2026 IBM Corporation. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -2220,7 +2220,7 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm,
     assert(vep_offset != -1,        "Must be set");
 #endif
 
-    __ flush();
+    // Code will be copied. No ICache sync required.
     nmethod* nm = nmethod::new_native_nmethod(method,
                                               compile_id,
                                               masm->code(),
@@ -2250,7 +2250,7 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm,
 
     int frame_complete = ((intptr_t)__ pc()) - start; // Not complete, period.
 
-    __ flush();
+    // Code will be copied. No ICache sync required.
 
     int stack_slots = SharedRuntime::out_preserve_stack_slots();  // No out slots at all, actually.
 
@@ -2745,16 +2745,8 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm,
       break;
   }
 
-  // Switch thread to "native transition" state before reading the synchronization state.
-  // This additional state is necessary because reading and testing the synchronization
-  // state is not atomic w.r.t. GC, as this scenario demonstrates:
-  //   - Java thread A, in _thread_in_native state, loads _not_synchronized and is preempted.
-  //   - VM thread changes sync state to synchronizing and suspends threads for GC.
-  //   - Thread A is resumed to finish this native method, but doesn't block here since it
-  //     didn't see any synchronization in progress, and escapes.
-
-  // Transition from _thread_in_native to _thread_in_native_trans.
-  __ set_thread_state(_thread_in_native_trans);
+  // Transition from _thread_in_native to _thread_in_vm.
+  __ set_thread_state(_thread_in_vm);
 
   // Safepoint synchronization
   //--------------------------------------------------------------------
@@ -2795,10 +2787,10 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm,
   }
 
   //--------------------------------------------------------------------
-  // Thread state is thread_in_native_trans. Any safepoint blocking has
+  // Thread state is _thread_in_vm. Any safepoint blocking has
   // already happened so we can now change state to _thread_in_Java.
   //--------------------------------------------------------------------
-  // Transition from _thread_in_native_trans to _thread_in_Java.
+  // Transition from _thread_in_vm to _thread_in_Java.
   __ set_thread_state(_thread_in_Java);
 
   // Check preemption for Object.wait()
@@ -2976,7 +2968,7 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm,
   __ restore_return_pc();
   __ z_br(Z_R1_scratch);
 
-  __ flush();
+  // Code will be copied. No ICache sync required.
   //////////////////////////////////////////////////////////////////////
   // end of code generation
   //////////////////////////////////////////////////////////////////////
@@ -3551,8 +3543,7 @@ void SharedRuntime::generate_deopt_blob() {
   // return to the interpreter entry point.
   __ z_br(Z_R14);
 
-  // Make sure all code is generated
-  masm->flush();
+  // Code will be copied. No ICache sync required.
 
   _deopt_blob = DeoptimizationBlob::create(&buffer, oop_maps, 0, exception_offset, reexecute_offset, RegisterSaver::live_reg_frame_size(RegisterSaver::all_registers, SuperwordUseVX)/wordSize);
   _deopt_blob->set_unpack_with_exception_in_tls_offset(exception_in_tls_offset);
@@ -3690,7 +3681,7 @@ UncommonTrapBlob* OptoRuntime::generate_uncommon_trap_blob() {
   // return to the interpreter entry point
   __ z_br(Z_R14);
 
-  masm->flush();
+  // Code will be copied. No ICache sync required.
   return UncommonTrapBlob::create(&buffer, nullptr, framesize_in_bytes/wordSize);
 }
 #endif // COMPILER2
@@ -3788,8 +3779,7 @@ SafepointBlob* SharedRuntime::generate_handler_blob(StubId id, address call_ptr)
 
   __ z_br(Z_R14);
 
-  // Make sure all code is generated
-  masm->flush();
+  // Code will be copied. No ICache sync required.
 
   // Fill-out other meta info
   return SafepointBlob::create(&buffer, oop_maps, RegisterSaver::live_reg_frame_size(RegisterSaver::all_registers, save_vectors)/wordSize);
@@ -3871,8 +3861,7 @@ RuntimeStub* SharedRuntime::generate_resolve_blob(StubId id, address destination
   __ z_br(Z_R1_scratch);
 
   // -------------
-  // make sure all code is generated
-  masm->flush();
+  // Code will be copied. No ICache sync required.
 
   // return the blob
   // frame_size_words or bytes??

src/hotspot/cpu/s390/templateInterpreterGenerator_s390.cpp

@@ -1575,26 +1575,14 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) {
   // In order for GC to work, don't clear the last_Java_sp until after
   // blocking.
 
-  //=============================================================================
-  // Switch thread to "native transition" state before reading the
-  // synchronization state. This additional state is necessary
-  // because reading and testing the synchronization state is not
-  // atomic w.r.t. GC, as this scenario demonstrates: Java thread A,
-  // in _thread_in_native state, loads _not_synchronized and is
-  // preempted. VM thread changes sync state to synchronizing and
-  // suspends threads for GC. Thread A is resumed to finish this
-  // native method, but doesn't block here since it didn't see any
-  // synchronization is progress, and escapes.
-
-  __ set_thread_state(_thread_in_native_trans);
+  __ set_thread_state(_thread_in_vm);
   if (!UseSystemMemoryBarrier) {
     __ z_fence();
   }
 
   // Now before we return to java we must look for a current safepoint
-  // (a new safepoint can not start since we entered native_trans).
-  // We must check here because a current safepoint could be modifying
-  // the callers registers right this moment.
+  // (a new safepoint can not start since we entered _thread_in_vm).
+  // We must check here because a current safepoint could be in progress.
 
   // Check for safepoint operation in progress and/or pending suspend requests.
   {
@@ -1612,7 +1600,7 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) {
   //=============================================================================
   // Back in Interpreter Frame.
 
-  // We are in thread_in_native_trans here and back in the normal
+  // We are in _thread_in_vm here and back in the normal
   // interpreter frame. We don't have to do anything special about
   // safepoints and we can switch to Java mode anytime we are ready.
 

src/hotspot/cpu/s390/upcallLinker_s390.cpp

@@ -271,7 +271,7 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Symbol* signature,
 
   //////////////////////////////////////////////////////////////////////////////
 
-  _masm->flush();
+  // Code will be copied. No ICache sync required.
 
 #ifndef PRODUCT
   stringStream ss;

src/hotspot/cpu/s390/vm_version_s390.cpp

@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2016, 2024 SAP SE. All rights reserved.
+ * Copyright (c) 2016, 2026 SAP SE. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -1142,7 +1142,7 @@ void VM_Version::determine_features() {
   a->z_br(Z_R14);
 
   address code_end = a->pc();
-  a->flush();
+  a->invalidate_icache();
 
   cbuf.insts()->set_end(code_end);
 

src/hotspot/cpu/s390/vtableStubs_s390.cpp

@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2016, 2026, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2016, 2023 SAP SE. All rights reserved.
+ * Copyright (c) 2016, 2026 SAP SE. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -141,7 +141,7 @@ VtableStub* VtableStubs::create_vtable_stub(int vtable_index, bool caller_is_c1)
   __ z_lg(Z_R1_scratch, in_bytes(Method::from_compiled_offset()), Z_method);
   __ z_br(Z_R1_scratch);
 
-  masm->flush();
+  masm->invalidate_icache();
   bookkeeping(masm, tty, s, npe_addr, ame_addr, true, vtable_index, slop_bytes, 0);
 
   return s;
@@ -235,7 +235,7 @@ VtableStub* VtableStubs::create_itable_stub(int itable_index, bool caller_is_c1)
   assert(slop_delta >= 0, "negative slop(%d) encountered, adjust code size estimate!", slop_delta);
   __ z_br(Z_R1_scratch);
 
-  masm->flush();
+  masm->invalidate_icache();
   bookkeeping(masm, tty, s, npe_addr, ame_addr, false, itable_index, slop_bytes, 0);
 
   return s;

src/hotspot/cpu/x86/bytes_x86.hpp

@@ -1,101 +0,0 @@
-/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#ifndef CPU_X86_BYTES_X86_HPP
-#define CPU_X86_BYTES_X86_HPP
-
-#include "memory/allStatic.hpp"
-#include "utilities/align.hpp"
-#include "utilities/byteswap.hpp"
-#include "utilities/macros.hpp"
-
-class Bytes: AllStatic {
- public:
-  // Efficient reading and writing of unaligned unsigned data in platform-specific byte ordering
-  template <typename T>
-  static inline T get_native(const void* p) {
-    assert(p != nullptr, "null pointer");
-
-    T x;
-
-    if (is_aligned(p, sizeof(T))) {
-      x = *(T*)p;
-    } else {
-      memcpy(&x, p, sizeof(T));
-    }
-
-    return x;
-  }
-
-  template <typename T>
-  static inline void put_native(void* p, T x) {
-    assert(p != nullptr, "null pointer");
-
-    if (is_aligned(p, sizeof(T))) {
-      *(T*)p = x;
-    } else {
-      memcpy(p, &x, sizeof(T));
-    }
-  }
-
-  static inline u2   get_native_u2(address p)         { return get_native<u2>((void*)p); }
-  static inline u4   get_native_u4(address p)         { return get_native<u4>((void*)p); }
-  static inline u8   get_native_u8(address p)         { return get_native<u8>((void*)p); }
-  static inline void put_native_u2(address p, u2 x)   { put_native<u2>((void*)p, x); }
-  static inline void put_native_u4(address p, u4 x)   { put_native<u4>((void*)p, x); }
-  static inline void put_native_u8(address p, u8 x)   { put_native<u8>((void*)p, x); }
-
-  // Efficient reading and writing of unaligned unsigned data in Java
-  // byte ordering (i.e. big-endian ordering). Byte-order reversal is
-  // needed since x86 CPUs use little-endian format.
-  template <typename T>
-  static inline T get_Java(const address p) {
-    T x = get_native<T>(p);
-
-    if (Endian::is_Java_byte_ordering_different()) {
-      x = byteswap(x);
-    }
-
-    return x;
-  }
-
-  template <typename T>
-  static inline void put_Java(address p, T x) {
-    if (Endian::is_Java_byte_ordering_different()) {
-      x = byteswap(x);
-    }
-
-    put_native<T>(p, x);
-  }
-
-  static inline u2   get_Java_u2(address p)           { return get_Java<u2>(p); }
-  static inline u4   get_Java_u4(address p)           { return get_Java<u4>(p); }
-  static inline u8   get_Java_u8(address p)           { return get_Java<u8>(p); }
-
-  static inline void put_Java_u2(address p, u2 x)     { put_Java<u2>(p, x); }
-  static inline void put_Java_u4(address p, u4 x)     { put_Java<u4>(p, x); }
-  static inline void put_Java_u8(address p, u8 x)     { put_Java<u8>(p, x); }
-};
-
-#endif // CPU_X86_BYTES_X86_HPP

src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp

@@ -262,7 +262,6 @@ void LIR_Assembler::osr_entry() {
   //
 
   // build frame
-  ciMethod* m = compilation()->method();
   __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
 
   // OSR buffer is
@@ -1339,7 +1338,6 @@ void LIR_Assembler::type_profile_helper(Register mdo,
 
 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
   // we always need a stub for the failure case.
-  CodeStub* stub = op->stub();
   Register obj = op->object()->as_register();
   Register k_RInfo = op->tmp1()->as_register();
   Register klass_RInfo = op->tmp2()->as_register();
@@ -2341,7 +2339,7 @@ void LIR_Assembler::emit_static_call_stub() {
     return;
   }
 
-  int start = __ offset();
+  DEBUG_ONLY(int start = __ offset();)
 
   // make sure that the displacement word of the call ends up word aligned
   __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
@@ -2937,7 +2935,6 @@ void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
   ciMethod* method = op->profiled_method();
   int bci          = op->profiled_bci();
-  ciMethod* callee = op->profiled_callee();
   Register tmp_load_klass = rscratch1;
 
   // Update counter for all call types

src/hotspot/cpu/x86/c1_LIRGenerator_x86.cpp

@@ -674,7 +674,7 @@ LIR_Opr LIRGenerator::atomic_cmpxchg(BasicType type, LIR_Opr addr, LIRItem& cmp_
 }
 
 LIR_Opr LIRGenerator::atomic_xchg(BasicType type, LIR_Opr addr, LIRItem& value) {
-  bool is_oop = is_reference_type(type);
+  DEBUG_ONLY(bool is_oop = is_reference_type(type);)
   LIR_Opr result = new_register(type);
   value.load_item();
   // Because we want a 2-arg form of xchg and xadd
@@ -920,7 +920,6 @@ void LIRGenerator::do_update_CRC32(Intrinsic* x) {
   assert(UseCRC32Intrinsics, "need AVX and CLMUL instructions support");
   // Make all state_for calls early since they can emit code
   LIR_Opr result = rlock_result(x);
-  int flags = 0;
   switch (x->id()) {
     case vmIntrinsics::_updateCRC32: {
       LIRItem crc(x->argument_at(0), this);

src/hotspot/cpu/x86/c1_Runtime1_x86.cpp

@@ -814,7 +814,6 @@ OopMapSet* Runtime1::generate_patching(StubAssembler* sasm, address target) {
 OopMapSet* Runtime1::generate_code_for(StubId id, StubAssembler* sasm) {
 
   // for better readability
-  const bool must_gc_arguments = true;
   const bool dont_gc_arguments = false;
 
   // default value; overwritten for some optimized stubs that are called from methods that do not use the fpu

src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp

@@ -2243,7 +2243,6 @@ void C2_MacroAssembler::reduce16S(int opcode, Register dst, Register src1, XMMRe
 
 void C2_MacroAssembler::reduce32S(int opcode, Register dst, Register src1, XMMRegister src2, XMMRegister vtmp1, XMMRegister vtmp2) {
   assert_different_registers(src2, vtmp1);
-  int vector_len = Assembler::AVX_256bit;
   vextracti64x4_high(vtmp1, src2);
   reduce_operation_256(T_SHORT, opcode, vtmp1, vtmp1, src2);
   reduce16S(opcode, dst, src1, vtmp1, vtmp1, vtmp2);
@@ -2507,7 +2506,6 @@ XMMRegister C2_MacroAssembler::get_lane(BasicType typ, XMMRegister dst, XMMRegis
   int esize =  type2aelembytes(typ);
   int elem_per_lane = 16/esize;
   int lane = elemindex / elem_per_lane;
-  int eindex = elemindex % elem_per_lane;
 
   if (lane >= 2) {
     assert(UseAVX > 2, "required");
@@ -5188,7 +5186,7 @@ void C2_MacroAssembler::vector_castF2X_avx(BasicType to_elem_bt, XMMRegister dst
 void C2_MacroAssembler::vector_castF2X_evex(BasicType to_elem_bt, XMMRegister dst, XMMRegister src, XMMRegister xtmp1,
                                             XMMRegister xtmp2, KRegister ktmp1, KRegister ktmp2, AddressLiteral float_sign_flip,
                                             Register rscratch, int vec_enc) {
-  int to_elem_sz = type2aelembytes(to_elem_bt);
+  DEBUG_ONLY(int to_elem_sz = type2aelembytes(to_elem_bt);)
   assert(to_elem_sz <= 4, "");
   vcvttps2dq(dst, src, vec_enc);
   vector_cast_fp_to_int_special_cases_evex(T_FLOAT, dst, src, xtmp1, xtmp2, ktmp1, ktmp2, rscratch, float_sign_flip, vec_enc);

src/hotspot/cpu/x86/c2_init_x86.cpp

@@ -36,7 +36,6 @@ void Compile::pd_compiler2_init() {
   if (UseAVX < 3) {
     int delta = XMMRegister::max_slots_per_register * XMMRegister::number_of_registers;
     int bottom = ConcreteRegisterImpl::max_fpr;
-    int top = bottom + delta;
     int middle = bottom + (delta / 2);
     int xmm_slots = XMMRegister::max_slots_per_register;
     int lower = xmm_slots / 2;

src/hotspot/cpu/x86/c2_stubGenerator_x86_64_string.cpp

@@ -220,7 +220,7 @@ static void generate_string_indexof_stubs(StubGenerator *stubgen, address *fnptr
 
   assert(StubInfo::entry_count(stub_id) == 1, "sanity check");
   GrowableArray<address> extras;
-  const int expected_extra_count = 2 * NUMBER_OF_CASES;
+  DEBUG_ONLY(const int expected_extra_count = 2 * NUMBER_OF_CASES;)
   address start = stubgen->load_archive_data(stub_id, nullptr, &extras);
   if (start != nullptr) {
     assert(extras.length() == expected_extra_count,
@@ -1009,7 +1009,6 @@ static void broadcast_first_and_last_needle(Register needle, Register needle_len
                                             MacroAssembler *_masm) {
   bool isUL = (ae == StrIntrinsicNode::UL);
   bool isUU = (ae == StrIntrinsicNode::UU);
-  bool isU = (isUU || isUL);
   Label L_short;
 
   // Always need needle broadcast to ymm registers
@@ -1776,8 +1775,6 @@ static void setup_jump_tables(StrIntrinsicNode::ArgEncoding ae, Label &L_error,
   bool isU = isUL || isUU;  // At least one is UTF-16
   const XMMRegister byte_1 = XMM_BYTE_1;
 
-  int jmp_ndx = 0;
-
   ////////////////////////////////////////////////
   //  On entry to each case, the register state is:
   //

src/hotspot/cpu/x86/downcallLinker_x86_64.cpp

@@ -310,7 +310,7 @@ void DowncallLinker::StubGenerator::generate() {
     __ block_comment("{ thread native2java");
     __ restore_cpu_control_state_after_jni(rscratch1);
 
-    __ movl(Address(r15_thread, JavaThread::thread_state_offset()), _thread_in_native_trans);
+    __ movl(Address(r15_thread, JavaThread::thread_state_offset()), _thread_in_vm);
 
     // Force this write out before the read below
     if (!UseSystemMemoryBarrier) {
@@ -381,5 +381,5 @@ void DowncallLinker::StubGenerator::generate() {
   }
   //////////////////////////////////////////////////////////////////////////////
 
-  __ flush();
+  // Code will be copied. No ICache sync required.
 }

src/hotspot/cpu/x86/gc/z/z_x86_64.ad

@@ -34,14 +34,14 @@ source %{
 #include "c2_intelJccErratum_x86.hpp"
 #include "gc/z/zBarrierSetAssembler.hpp"
 
-static void z_color(MacroAssembler* masm, const MachNode* node, Register ref) {
+static void z_color(MacroAssembler* masm, Register ref) {
   __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatLoadGoodBeforeShl);
   __ shlq(ref, barrier_Relocation::unpatched);
   __ orq_imm32(ref, barrier_Relocation::unpatched);
   __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatStoreGoodAfterOr);
 }
 
-static void z_uncolor(MacroAssembler* masm, const MachNode* node, Register ref) {
+static void z_uncolor(MacroAssembler* masm, Register ref) {
   __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatLoadGoodBeforeShl);
   __ shrq(ref, barrier_Relocation::unpatched);
 }
@@ -53,7 +53,7 @@ static void z_keep_alive_load_barrier(MacroAssembler* masm, const MachNode* node
   ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref);
   __ jcc(Assembler::notEqual, *stub->entry());
 
-  z_uncolor(masm, node, ref);
+  z_uncolor(masm, ref);
 
   __ bind(*stub->continuation());
 }
@@ -69,7 +69,7 @@ static void z_load_barrier(MacroAssembler* masm, const MachNode* node, Address r
     return;
   }
 
-  z_uncolor(masm, node, ref);
+  z_uncolor(masm, ref);
   if (node->barrier_data() == ZBarrierElided) {
     return;
   }
@@ -87,7 +87,7 @@ static void z_store_barrier(MacroAssembler* masm, const MachNode* node, Address
     if (rnew_zaddress != noreg) {
       // noreg means null; no need to color
       __ movptr(rnew_zpointer, rnew_zaddress);
-      z_color(masm, node, rnew_zpointer);
+      z_color(masm, rnew_zpointer);
     }
   } else {
     bool is_native = (node->barrier_data() & ZBarrierNative) != 0;
@@ -200,10 +200,10 @@ instruct zCompareAndExchangeP(indirect mem, no_rax_RegP newval, rRegP tmp, rax_R
     assert_different_registers($oldval$$Register, $newval$$Register);
     const Address mem_addr = Address($mem$$Register, 0);
     z_store_barrier(masm, this, mem_addr, $newval$$Register, $tmp$$Register, true /* is_atomic */);
-    z_color(masm, this, $oldval$$Register);
+    z_color(masm, $oldval$$Register);
     __ lock();
     __ cmpxchgptr($tmp$$Register, mem_addr);
-    z_uncolor(masm, this, $oldval$$Register);
+    z_uncolor(masm, $oldval$$Register);
   %}
 
   ins_pipe(pipe_cmpxchg);
@@ -223,7 +223,7 @@ instruct zCompareAndSwapP(rRegI res, indirect mem, rRegP newval, rRegP tmp, rax_
     assert_different_registers($oldval$$Register, $mem$$Register);
     const Address mem_addr = Address($mem$$Register, 0);
     z_store_barrier(masm, this, mem_addr, $newval$$Register, $tmp$$Register, true /* is_atomic */);
-    z_color(masm, this, $oldval$$Register);
+    z_color(masm, $oldval$$Register);
     __ lock();
     __ cmpxchgptr($tmp$$Register, mem_addr);
     __ setcc(Assembler::equal, $res$$Register);
@@ -245,7 +245,7 @@ instruct zXChgP(indirect mem, rRegP newval, rRegP tmp, rFlagsReg cr) %{
     z_store_barrier(masm, this, mem_addr, $newval$$Register, $tmp$$Register, true /* is_atomic */);
     __ movptr($newval$$Register, $tmp$$Register);
     __ xchgptr($newval$$Register, mem_addr);
-    z_uncolor(masm, this, $newval$$Register);
+    z_uncolor(masm, $newval$$Register);
   %}
 
   ins_pipe(pipe_cmpxchg);

src/hotspot/cpu/x86/globals_x86.hpp

@@ -108,7 +108,7 @@ define_pd_global(bool, InlineTypeReturnedAsFields, true);
           "Highest supported AVX instructions set on x86/x64")              \
           range(0, 3)                                                       \
                                                                             \
-  product(bool, UseAPX, false, EXPERIMENTAL,                                \
+  product(bool, UseAPX, false,                                              \
           "Use Intel Advanced Performance Extensions")                      \
                                                                             \
   product(bool, UseKNLSetting, false, DIAGNOSTIC,                           \

src/hotspot/cpu/x86/icache_x86.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -28,16 +28,6 @@
 // Interface for updating the instruction cache.  Whenever the VM modifies
 // code, part of the processor instruction cache potentially has to be flushed.
 
-// On the x86, this is a no-op -- the I-cache is guaranteed to be consistent
-// after the next jump, and the VM never modifies instructions directly ahead
-// of the instruction fetch path.
-
-// [phh] It's not clear that the above comment is correct, because on an MP
-// system where the dcaches are not snooped, only the thread doing the invalidate
-// will see the update.  Even in the snooped case, a memory fence would be
-// necessary if stores weren't ordered.  Fortunately, they are on all known
-// x86 implementations.
-
 class ICache : public AbstractICache {
  public:
   enum {

src/hotspot/cpu/x86/interpreterRT_x86_64.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -295,7 +295,7 @@ void InterpreterRuntime::SignatureHandlerGenerator::generate(uint64_t fingerprin
   __ lea(rax, ExternalAddress(Interpreter::result_handler(method()->result_type())));
   __ ret(0);
 
-  __ flush();
+  __ invalidate_icache();
 }
 
 

src/hotspot/cpu/x86/jniFastGetField_x86_64.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2004, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2004, 2026, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -120,7 +120,7 @@ address JNI_FastGetField::generate_fast_get_int_field0(BasicType type) {
   // tail call
   __ jump (RuntimeAddress(slow_case_addr), rscratch1);
 
-  __ flush ();
+  __ invalidate_icache();
 
   return fast_entry;
 }
@@ -208,7 +208,7 @@ address JNI_FastGetField::generate_fast_get_float_field0(BasicType type) {
   // tail call
   __ jump (RuntimeAddress(slow_case_addr), rscratch1);
 
-  __ flush ();
+  __ invalidate_icache();
 
   return fast_entry;
 }

src/hotspot/cpu/x86/macroAssembler_x86.cpp

@@ -4607,7 +4607,7 @@ void MacroAssembler::lookup_secondary_supers_table_var(Register r_sub_klass,
   assert(Array<Klass*>::length_offset_in_bytes() == 0, "Adjust this code");
 
   cmpq(r_super_klass, Address(r_array_base, r_array_index, Address::times_8));
-  jccb(Assembler::equal, L_success);
+  jcc(Assembler::equal, L_success);
 
   // Restore slot to its true value
   movb(slot, Address(r_super_klass, Klass::hash_slot_offset()));
@@ -4618,7 +4618,7 @@ void MacroAssembler::lookup_secondary_supers_table_var(Register r_sub_klass,
 
   // Is there another entry to check? Consult the bitmap.
   btq(r_bitmap, 1);
-  jccb(Assembler::carryClear, L_failure);
+  jcc(Assembler::carryClear, L_failure);
 
   // Calls into the stub generated by lookup_secondary_supers_table_slow_path.
   // Arguments: r_super_klass, r_array_base, r_array_index, r_bitmap.
@@ -4757,21 +4757,6 @@ void MacroAssembler::lookup_secondary_supers_table_slow_path(Register r_super_kl
   }
 }
 
-struct VerifyHelperArguments {
-  Klass* _super;
-  Klass* _sub;
-  intptr_t _linear_result;
-  intptr_t _table_result;
-};
-
-static void verify_secondary_supers_table_helper(const char* msg, VerifyHelperArguments* args) {
-  Klass::on_secondary_supers_verification_failure(args->_super,
-                                                  args->_sub,
-                                                  args->_linear_result,
-                                                  args->_table_result,
-                                                  msg);
-}
-
 // Make sure that the hashed lookup and a linear scan agree.
 void MacroAssembler::verify_secondary_supers_table(Register r_sub_klass,
                                                    Register r_super_klass,
@@ -4814,15 +4799,31 @@ void MacroAssembler::verify_secondary_supers_table(Register r_sub_klass,
   cmpl(linear_result, result);
   jcc(Assembler::equal, L_done);
 
-  { // To avoid calling convention issues, build a record on the stack
-    // and pass the pointer to that instead.
+  { // Push values on stack and load them into argument registers
+    // to avoid overlaping registers issue.
     push(result);
     push(linear_result);
     push(r_sub_klass);
     push(r_super_klass);
-    movptr(c_rarg1, rsp);
-    movptr(c_rarg0, (uintptr_t) "mismatch");
-    call(RuntimeAddress(CAST_FROM_FN_PTR(address, verify_secondary_supers_table_helper)));
+    movptr(c_rarg0, Address(rsp, 0 * wordSize)); // super
+    movptr(c_rarg1, Address(rsp, 1 * wordSize)); // sub
+    movptr(c_rarg2, Address(rsp, 2 * wordSize)); // linear_result
+    movptr(c_rarg3, Address(rsp, 3 * wordSize)); // table_result
+    const char* msg = "mismatch";
+    const char* str = (code_section()->scratch_emit()) ? msg : AOTCodeCache::add_C_string(msg);
+    lea(rscratch1, ExternalAddress((address)str));
+#ifdef _WIN64
+    // Win64 pass only 4 arguments in registers, push message on stack.
+    // Windows always allocates space for its register args and we
+    // need one more for 5th argument.
+    subq(rsp, (frame::arg_reg_save_area_bytes + wordSize));
+    andq(rsp, -StackAlignmentInBytes); // align stack as required by ABI
+    movptr(Address(rsp, frame::arg_reg_save_area_bytes), rscratch1);
+#else
+    movptr(c_rarg4, rscratch1);
+    andq(rsp, -StackAlignmentInBytes); // align stack as required by ABI
+#endif
+    call(RuntimeAddress(CAST_FROM_FN_PTR(address, Klass::on_secondary_supers_verification_failure)));
     should_not_reach_here();
   }
   bind(L_done);

src/hotspot/cpu/x86/runtime_x86_64.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -231,8 +231,7 @@ UncommonTrapBlob* OptoRuntime::generate_uncommon_trap_blob() {
   // Jump to interpreter
   __ ret(0);
 
-  // Make sure all code is generated
-  masm->flush();
+  // Code will be copied. No ICache sync required.
 
   UncommonTrapBlob *ut_blob = UncommonTrapBlob::create(&buffer, oop_maps,
                                                        SimpleRuntimeFrame::framesize >> 1);
@@ -370,8 +369,7 @@ ExceptionBlob* OptoRuntime::generate_exception_blob() {
 
   __ jmp(r8);
 
-  // Make sure all code is generated
-  masm->flush();
+  // Code will be copied. No ICache sync required.
 
   // Set exception blob
   ExceptionBlob* ex_blob = ExceptionBlob::create(&buffer, oop_maps, SimpleRuntimeFrame::framesize >> 1);

src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp

@@ -2021,7 +2021,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
     assert(vep_offset != -1,        "Must be set");
 #endif
 
-    __ flush();
+    // Code will be copied. No ICache sync required.
     nmethod* nm = nmethod::new_native_nmethod(method,
                                               compile_id,
                                               masm->code(),
@@ -2050,7 +2050,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
                          in_sig_bt,
                          in_regs);
     int frame_complete = ((intptr_t)__ pc()) - start;  // not complete, period
-    __ flush();
+    // Code will be copied. No ICache sync required.
     int stack_slots = SharedRuntime::out_preserve_stack_slots();  // no out slots at all, actually
     return nmethod::new_native_nmethod(method,
                                        compile_id,
@@ -2456,14 +2456,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
   default       : ShouldNotReachHere();
   }
 
-  // Switch thread to "native transition" state before reading the synchronization state.
-  // This additional state is necessary because reading and testing the synchronization
-  // state is not atomic w.r.t. GC, as this scenario demonstrates:
-  //     Java thread A, in _thread_in_native state, loads _not_synchronized and is preempted.
-  //     VM thread changes sync state to synchronizing and suspends threads for GC.
-  //     Thread A is resumed to finish this native method, but doesn't block here since it
-  //     didn't see any synchronization is progress, and escapes.
-  __ movl(Address(r15_thread, JavaThread::thread_state_offset()), _thread_in_native_trans);
+  __ movl(Address(r15_thread, JavaThread::thread_state_offset()), _thread_in_vm);
 
   // Force this write out before the read below
   if (!UseSystemMemoryBarrier) {
@@ -2719,7 +2712,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
 
 
 
-  __ flush();
+  // Code will be copied. No ICache sync required.
 
   nmethod *nm = nmethod::new_native_nmethod(method,
                                             compile_id,
@@ -3071,8 +3064,7 @@ void SharedRuntime::generate_deopt_blob() {
   // Jump to interpreter
   __ ret(0);
 
-  // Make sure all code is generated
-  masm->flush();
+  // Code will be copied. No ICache sync required.
 
   _deopt_blob = DeoptimizationBlob::create(&buffer, oop_maps, 0, exception_offset, reexecute_offset, frame_size_in_words);
   _deopt_blob->set_unpack_with_exception_in_tls_offset(exception_in_tls_offset);
@@ -3255,8 +3247,7 @@ SafepointBlob* SharedRuntime::generate_handler_blob(StubId id, address call_ptr)
   __ stop("Attempting to adjust pc to skip safepoint poll but the return point is not what we expected");
 #endif
 
-  // Make sure all code is generated
-  masm->flush();
+  // Code will be copied. No ICache sync required.
 
   // Fill-out other meta info
   SafepointBlob* sp_blob = SafepointBlob::create(&buffer, oop_maps, frame_size_in_words);
@@ -3347,9 +3338,7 @@ RuntimeStub* SharedRuntime::generate_resolve_blob(StubId id, address destination
   __ movptr(rax, Address(r15_thread, Thread::pending_exception_offset()));
   __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
 
-  // -------------
-  // make sure all code is generated
-  masm->flush();
+  // Code will be copied. No ICache sync required.
 
   // return the  blob
   // frame_size_words or bytes??
@@ -3871,7 +3860,7 @@ BufferedInlineTypeBlob* SharedRuntime::generate_buffered_inline_type_adapter(con
   __ bind(skip);
   __ ret(0);
 
-  __ flush();
+  // Code will be copied. No ICache sync required.
 
   return BufferedInlineTypeBlob::create(&buffer, pack_fields_off, pack_fields_jobject_off, unpack_fields_off);
 }
@@ -4028,9 +4017,7 @@ RuntimeStub* SharedRuntime::generate_return_value_stub(address destination) {
   __ movptr(rax, Address(r15_thread, Thread::pending_exception_offset()));
   __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
 
-  // -------------
-  // make sure all code is generated
-  masm->flush();
+  // Code will be copied. No ICache sync required.
 
   RuntimeStub* stub = RuntimeStub::new_runtime_stub(name, &buffer, frame_complete, frame_size_in_words, oop_maps, false);
   AOTCodeCache::store_code_blob(*stub, AOTCodeEntry::SharedBlob, StubInfo::blob(id));

src/hotspot/cpu/x86/templateInterpreterGenerator_x86.cpp

@@ -956,7 +956,7 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) {
 
   // change thread state
   __ movl(Address(thread, JavaThread::thread_state_offset()),
-          _thread_in_native_trans);
+          _thread_in_vm);
 
   // Force this write out before the read below
   if (!UseSystemMemoryBarrier) {

src/hotspot/cpu/x86/upcallLinker_x86_64.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -363,7 +363,7 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Symbol* signature,
 
   //////////////////////////////////////////////////////////////////////////////
 
-  _masm->flush();
+  // Code will be copied. No ICache sync required.
 
 #ifndef PRODUCT
   stringStream ss;

src/hotspot/cpu/x86/vm_version_x86.cpp

@@ -1061,8 +1061,7 @@ void VM_Version::get_processor_features() {
   // Currently APX support is only enabled for targets supporting AVX512VL feature.
   if (supports_apx_f() && os_supports_apx_egprs() && supports_avx512vl()) {
     if (FLAG_IS_DEFAULT(UseAPX)) {
-      FLAG_SET_DEFAULT(UseAPX, false); // by default UseAPX is false
-      clear_feature(CPU_APX_F);
+      FLAG_SET_DEFAULT(UseAPX, true); // by default UseAPX is false; enable if supported.
     } else if (!UseAPX) {
       clear_feature(CPU_APX_F);
     }
@@ -1077,6 +1076,14 @@ void VM_Version::get_processor_features() {
       FLAG_SET_DEFAULT(UseAPX, false);
     }
   }
+#if defined(COMPILER2)
+  if (UseAPX) {
+    // Increase InlineSmallCode by 10%
+    if (FLAG_IS_DEFAULT(InlineSmallCode)) {
+      FLAG_SET_DEFAULT(InlineSmallCode, InlineSmallCode * 1.10);
+    }
+  }
+#endif
 
   CHECK_CPU_FEATURE(UseCLMUL, CLMUL, supports_clmul(), "CLMUL" MULTI_INST_WARNING_MSG);
   CHECK_CPU_FEATURE(UseAES, AES, supports_aes(), "AES" MULTI_INST_WARNING_MSG);
@@ -1146,6 +1153,10 @@ void VM_Version::get_processor_features() {
            cpu_family(), _model, _stepping, os::cpu_microcode_revision());
   ss.print(", ");
   int features_offset = (int)ss.size();
+  if (compute_fast_bmi2()) {
+    _features.set_feature(CPU_FAST_BMI2);
+  }
+
   insert_features_names(_features, ss);
 
   _cpu_info_string = ss.as_string(true);
@@ -1999,6 +2010,51 @@ bool VM_Version::compute_has_intel_jcc_erratum() {
   }
 }
 
+// The BMI2 instruction set includes PEXT (parallel bits extract) and PDEP
+// (parallel bits deposit), which are used to intrinsify Integer/Long.compress
+// and Integer/Long.expand (added in JDK 19, see https://bugs.openjdk.org/browse/JDK-8283893).
+//
+// While all BMI2-capable CPUs can execute these instructions, PEXT and PDEP
+// are unique in that some vendors implement them via microcode rather than
+// native ALU hardware. The microcoded versions are significantly slower (high latency/low throughput)
+// than the manual bitwise fallback used in the Java implementation.
+// Conversely, all other BMI2 instructions (BZHI, MULX, RORX, SARX, SHRX, SHLX)
+// execute efficiently on every BMI2-capable CPU and are unaffected by this check.
+//
+// The logic in this method is based on official optimization guides from hardware vendors,
+// to guarantee that microcode implementations of PEXT/PDEP are not used.
+bool VM_Version::compute_fast_bmi2() {
+  if (!supports_bmi2()) {
+    return false;
+  }
+
+  if (is_intel()) {
+    // All Intel CPUs with BMI2 (Haswell+) implement PEXT/PDEP natively.
+    // 3-cycle latency, 1-per-cycle throughput on a dedicated ALU port.
+    // Source: Intel Intrinsics Guide, https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html
+    return true;
+  }
+
+  if (is_amd()) {
+    // AMD added BMI2 in Excavator (Family 0x15, model 0x60+) but used
+    // microcode for PEXT/PDEP through all of Zen 2 (Family 0x17).
+    // Native ALU hardware support arrived with Zen 3 (Family 0x19).
+    // Source: AMD Software Optimization Guide (doc #56665), Section 2.10.2, https://developer.amd.com/resources/developer-guides-manuals/
+    uint32_t family = extended_cpu_family();
+    return family >= CPU_FAMILY_AMD_19H;
+  }
+
+  // Zhaoxin added BMI2 support in Lujiazui (KX-6000+).
+  // Based on community benchmarks(https://uops.info/html-instr/PDEP_R64_R64_R64.html),
+  // PEXT/PDEP performance is known to be similarly poor to pre-Zen3 AMD, suggesting a microcode implementation.
+  // This cannot be confirmed as Zhaoxin publishes no public optimization guide.
+
+  // On VIA/Centaur CNS, BMI2 is implemented in hardware with PDEP/PEXT executing at two per cycle (better than Haswell).
+  // Intel acquired Centaur in 2021, and CNS never reached production, so we don't check for it.
+
+  return false;
+}
+
 // On Xen, the cpuid instruction returns
 //  eax / registers[0]: Version of Xen
 //  ebx / registers[1]: chars 'XenV'

src/hotspot/cpu/x86/vm_version_x86.hpp

@@ -439,7 +439,8 @@ protected:
     decl(AVX512_FP16,       avx512_fp16       ) /* AVX512 FP16 ISA support*/ \
     decl(AVX10_1,           avx10_1           ) /* AVX10 512 bit vector ISA Version 1 support*/ \
     decl(AVX10_2,           avx10_2           ) /* AVX10 512 bit vector ISA Version 2 support*/ \
-    decl(HYBRID,            hybrid            ) /* Hybrid architecture */
+    decl(HYBRID,            hybrid            ) /* Hybrid architecture */ \
+    decl(FAST_BMI2,         fast_bmi2         ) /* Native Hardware support for PEXT/PDEP BMI2 instructions */
 
 #define DECLARE_CPU_FEATURE_FLAG(id, name) CPU_##id,
     CPU_FEATURE_FLAGS(DECLARE_CPU_FEATURE_FLAG)
@@ -725,6 +726,7 @@ private:
   }
 
   static bool compute_has_intel_jcc_erratum();
+  static bool compute_fast_bmi2();
 
   static bool os_supports_avx_vectors();
   static bool os_supports_apx_egprs();
@@ -912,6 +914,7 @@ public:
   static bool supports_hv()           { return _features.supports_feature(CPU_HV); }
   static bool supports_serialize()    { return _features.supports_feature(CPU_SERIALIZE); }
   static bool supports_hybrid()       { return _features.supports_feature(CPU_HYBRID); }
+  static bool supports_fast_bmi2()    { return _features.supports_feature(CPU_FAST_BMI2); }
   static bool supports_f16c()         { return _features.supports_feature(CPU_F16C); }
   static bool supports_pku()          { return _features.supports_feature(CPU_PKU); }
   static bool supports_ospke()        { return _features.supports_feature(CPU_OSPKE); }

src/hotspot/cpu/x86/vtableStubs_x86_64.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -131,7 +131,7 @@ VtableStub* VtableStubs::create_vtable_stub(int vtable_index, bool caller_is_c1)
   address ame_addr = __ pc();
   __ jmp( Address(rbx, entry_offset));
 
-  masm->flush();
+  masm->invalidate_icache();
   slop_bytes += index_dependent_slop; // add'l slop for size variance due to large itable offsets
   bookkeeping(masm, tty, s, npe_addr, ame_addr, true, vtable_index, slop_bytes, index_dependent_slop);
 
@@ -248,7 +248,7 @@ VtableStub* VtableStubs::create_itable_stub(int itable_index, bool caller_is_c1)
   // dirty work.
   __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 
-  masm->flush();
+  masm->invalidate_icache();
   slop_bytes += index_dependent_slop; // add'l slop for size variance due to large itable offsets
   bookkeeping(masm, tty, s, npe_addr, ame_addr, false, itable_index, slop_bytes, index_dependent_slop);