8368247: RISC-V: enable vectorapi test for expand operation

8368086: G1: Use ThreadsClaimTokenScope in G1CMRemarkTask
Reviewed-by: fandreuzzi, stefank, iwalulya
2025-09-22 17:07:54 +08:00 · 2025-09-22 07:54:25 +00:00 · 2025-09-22 07:20:00 +00:00 · 2025-09-22 07:15:41 +00:00 · 2025-09-22 06:02:20 +00:00 · 2025-09-22 05:32:11 +00:00
1055 changed files with 26376 additions and 14699 deletions
@@ -1,6 +1,6 @@
 #!/bin/sh
 #
-# Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
@@ -52,12 +52,39 @@
 #   include the SCM state that was used to build it, which can be found in ${JDK_N_INSTALL}/release,
 #   in property "SOURCE".

+source_path="$(dirname ${0})"
+this_script_dir="$(cd -- "${source_path}" > /dev/null && pwd)"
+if test -z "${this_script_dir}"; then
+  echo "Error: Could not determine location of this script"
+  exit 1
+fi
+
+symbols_dir="$(dirname $this_script_dir)/src/jdk.compiler/share/data/symbols"
+if [ ! -d $symbols_dir ] ; then
+    echo "Cannot locate symbols directory: $symbols_dir" >&2
+    exit 1
+fi
+
+generator_dir="$(dirname $this_script_dir)/make/langtools/src/classes/build/tools/symbolgenerator"
+
 if [ "$1x" = "x" ] ; then
    echo "Must provide the target JDK as a parameter:" >&2
    echo "$0 <target-jdk>" >&2
    exit 1
 fi;

+if [ ! -d $1 ] ; then
+    echo "Target JDK argument is not a directory:" $1 >&2
+    exit 1
+fi;
+
+if [ ! -x $1/bin/java ] ; then
+    echo "Target JDK argument is not a valid JDK: $1" >&2
+    exit 1
+fi;
+
+cd $symbols_dir
+
 if [ ! -f symbols ] ; then
    echo "Must run inside the src/jdk.compiler/share/data/symbols directory" >&2
    exit 1
@@ -72,5 +99,5 @@ $1/bin/java --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.jvm=ALL-UNNAMED \
            --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED \
            --add-modules jdk.jdeps \
-            ../../../../../make/langtools/src/classes/build/tools/symbolgenerator/CreateSymbols.java \
+            $generator_dir/CreateSymbols.java \
            build-description-incremental symbols include.list
@@ -1,6 +1,6 @@
 #! /bin/sh -f
 #
-# Copyright (c) 2012, 2020, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2012, 2025, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
@@ -62,7 +62,7 @@ B=`basename "${script_directory}"`
 script_dir="`cd \"${D}\" 2>/dev/null && pwd || echo \"${D}\"`/${B}"

 # set up a variable for the template directory
-template_dir=${script_dir}/../data/license-templates
+template_dir=${script_dir}/../make/data/license-templates

 # Check existence of the template directory.
 if [ ! -d ${template_dir} ] ; then
@@ -1,191 +0,0 @@
-#
-# Copyright (c) 2014, 2025, Oracle and/or its affiliates. All rights reserved.
-# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
-#
-# This code is free software; you can redistribute it and/or modify it
-# under the terms of the GNU General Public License version 2 only, as
-# published by the Free Software Foundation.
-#
-# This code is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
-# version 2 for more details (a copy is included in the LICENSE file that
-# accompanied this code).
-#
-# You should have received a copy of the GNU General Public License version
-# 2 along with this work; if not, write to the Free Software Foundation,
-# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
-# or visit www.oracle.com if you need additional information or have any
-# questions.
-#
-
-src/bsd : jdk/src/bsd
-src/demo : jdk/src/demo
-src/java.activation : jaxws/src/java.activation
-src/java.base : jdk/src/java.base
-src/java.compiler : langtools/src/java.compiler
-src/java.corba : corba/src/java.corba
-src/java.datatransfer : jdk/src/java.datatransfer
-src/java.desktop : jdk/src/java.desktop
-src/java.instrument : jdk/src/java.instrument
-src/java.logging : jdk/src/java.logging
-src/java.management : jdk/src/java.management
-src/java.management.rmi : jdk/src/java.management.rmi
-src/java.naming : jdk/src/java.naming
-src/java.prefs : jdk/src/java.prefs
-src/java.rmi : jdk/src/java.rmi
-src/java.scripting : jdk/src/java.scripting
-src/java.se : jdk/src/java.se
-src/java.security.jgss : jdk/src/java.security.jgss
-src/java.security.sasl : jdk/src/java.security.sasl
-src/java.se.ee : jdk/src/java.se.ee
-src/java.smartcardio : jdk/src/java.smartcardio
-src/java.sql : jdk/src/java.sql
-src/java.sql.rowset : jdk/src/java.sql.rowset
-src/java.transaction : jdk/src/java.transaction
-src/java.xml : jaxp/src/java.xml
-src/java.xml.bind : jaxws/src/java.xml.bind
-src/java.xml.crypto : jdk/src/java.xml.crypto
-src/java.xml.ws : jaxws/src/java.xml.ws
-src/java.xml.ws.annotation : jaxws/src/java.xml.ws.annotation
-src/jdk.accessibility : jdk/src/jdk.accessibility
-src/jdk.aot : hotspot/src/jdk.aot
-src/jdk.attach : jdk/src/jdk.attach
-src/jdk.charsets : jdk/src/jdk.charsets
-src/jdk.compiler : jdk/src/jdk.compiler  langtools/src/jdk.compiler
-src/jdk.crypto.cryptoki : jdk/src/jdk.crypto.cryptoki
-src/jdk.crypto.ec : jdk/src/jdk.crypto.ec
-src/jdk.crypto.mscapi : jdk/src/jdk.crypto.mscapi
-src/jdk.dynalink : nashorn/src/jdk.dynalink
-src/jdk.editpad : jdk/src/jdk.editpad
-src/jdk.hotspot.agent : hotspot/src/jdk.hotspot.agent
-src/jdk.httpserver : jdk/src/jdk.httpserver
-src/jdk.incubator.httpclient : jdk/src/jdk.incubator.httpclient
-src/jdk.internal.ed : jdk/src/jdk.internal.ed
-src/jdk.internal.jvmstat : jdk/src/jdk.internal.jvmstat
-src/jdk.internal.le : jdk/src/jdk.internal.le
-src/jdk.internal.opt : jdk/src/jdk.internal.opt
-src/jdk.internal.vm.ci : hotspot/src/jdk.internal.vm.ci
-src/jdk.internal.vm.compiler : hotspot/src/jdk.internal.vm.compiler
-src/jdk.jartool : jdk/src/jdk.jartool
-src/jdk.javadoc : langtools/src/jdk.javadoc
-src/jdk.jcmd : jdk/src/jdk.jcmd
-src/jdk.jconsole : jdk/src/jdk.jconsole
-src/jdk.jdeps : langtools/src/jdk.jdeps
-src/jdk.jdi : jdk/src/jdk.jdi
-src/jdk.jdwp.agent : jdk/src/jdk.jdwp.agent
-src/jdk.jlink : jdk/src/jdk.jlink
-src/jdk.jshell : langtools/src/jdk.jshell
-src/jdk.jstatd : jdk/src/jdk.jstatd
-src/jdk.localedata : jdk/src/jdk.localedata
-src/jdk.management : jdk/src/jdk.management
-src/jdk.management.agent : jdk/src/jdk.management.agent
-src/jdk.naming.dns : jdk/src/jdk.naming.dns
-src/jdk.naming.rmi : jdk/src/jdk.naming.rmi
-src/jdk.net : jdk/src/jdk.net
-src/jdk.pack : jdk/src/jdk.pack
-src/jdk.scripting.nashorn : nashorn/src/jdk.scripting.nashorn
-src/jdk.scripting.nashorn.shell : nashorn/src/jdk.scripting.nashorn.shell
-src/jdk.sctp : jdk/src/jdk.sctp
-src/jdk.security.auth : jdk/src/jdk.security.auth
-src/jdk.security.jgss : jdk/src/jdk.security.jgss
-src/jdk.unsupported : jdk/src/jdk.unsupported
-src/jdk.xml.bind : jaxws/src/jdk.xml.bind
-src/jdk.xml.dom : jaxp/src/jdk.xml.dom
-src/jdk.xml.ws : jaxws/src/jdk.xml.ws
-src/jdk.zipfs : jdk/src/jdk.zipfs
-src/langtools/sample : langtools/src/sample
-src/linux : jdk/src/linux
-src/sample : jdk/src/sample
-src/hotspot/share : hotspot/src/share/vm
-src/hotspot/cpu/aarch64 : hotspot/src/cpu/aarch64/vm
-src/hotspot/cpu/arm : hotspot/src/cpu/arm/vm
-src/hotspot/cpu/ppc : hotspot/src/cpu/ppc/vm
-src/hotspot/cpu/s390 : hotspot/src/cpu/s390/vm
-src/hotspot/cpu/x86 : hotspot/src/cpu/x86/vm
-src/hotspot/cpu/zero : hotspot/src/cpu/zero/vm
-src/hotspot/os/aix : hotspot/src/os/aix/vm
-src/hotspot/os/bsd : hotspot/src/os/bsd/vm
-src/hotspot/os/linux : hotspot/src/os/linux/vm
-src/hotspot/os/posix/dtrace : hotspot/src/os/posix/dtrace
-src/hotspot/os/posix : hotspot/src/os/posix/vm
-src/hotspot/os/windows : hotspot/src/os/windows/vm
-src/hotspot/os_cpu/aix_ppc : hotspot/src/os_cpu/aix_ppc/vm
-src/hotspot/os_cpu/bsd_x86 : hotspot/src/os_cpu/bsd_x86/vm
-src/hotspot/os_cpu/bsd_zero : hotspot/src/os_cpu/bsd_zero/vm
-src/hotspot/os_cpu/linux_aarch64 : hotspot/src/os_cpu/linux_aarch64/vm
-src/hotspot/os_cpu/linux_arm : hotspot/src/os_cpu/linux_arm/vm
-src/hotspot/os_cpu/linux_ppc : hotspot/src/os_cpu/linux_ppc/vm
-src/hotspot/os_cpu/linux_s390 : hotspot/src/os_cpu/linux_s390/vm
-src/hotspot/os_cpu/linux_x86 : hotspot/src/os_cpu/linux_x86/vm
-src/hotspot/os_cpu/linux_zero : hotspot/src/os_cpu/linux_zero/vm
-src/hotspot/os_cpu/windows_x86 : hotspot/src/os_cpu/windows_x86/vm
-src/hotspot : hotspot/src
-src/utils/IdealGraphVisualizer : hotspot/src/share/tools/IdealGraphVisualizer
-src/utils/LogCompilation : hotspot/src/share/tools/LogCompilation
-src/utils/hsdis : hotspot/src/share/tools/hsdis
-src/utils/reorder : jdk/make/non-build-utils/reorder
-src/utils/src/build : jdk/make/non-build-utils/src/build
-make/BuildNashorn.gmk : nashorn/make/BuildNashorn.gmk
-make/CompileDemos.gmk : jdk/make/CompileDemos.gmk
-make/CompileInterimLangtools.gmk : langtools/make/CompileInterim.gmk
-make/CompileModuleTools.gmk : jdk/make/CompileModuleTools.gmk
-make/CompileToolsHotspot.gmk : hotspot/make/CompileTools.gmk
-make/CompileToolsJdk.gmk : jdk/make/CompileTools.gmk
-make/CopyInterimCLDRConverter.gmk : jdk/make/CopyInterimCLDRConverter.gmk
-make/GenerateModuleSummary.gmk : jdk/make/GenerateModuleSummary.gmk
-make/ModuleTools.gmk : jdk/make/ModuleTools.gmk
-make/ToolsJdk.gmk : jdk/make/Tools.gmk
-make/ToolsLangtools.gmk : langtools/make/Tools.gmk
-make/UnpackSecurity.gmk : jdk/make/UnpackSecurity.gmk
-make/autoconf : common/autoconf
-make/conf : common/conf
-make/copy : jdk/make/copy
-make/copy/Copy-java.corba.gmk : corba/make/copy/Copy-java.corba.gmk
-make/corba : corba/make
-make/data : jdk/make/data
-make/gendata : jdk/make/gendata
-make/gendata/Gendata-jdk.compiler.gmk : langtools/make/gendata/Gendata-jdk.compiler.gmk
-make/gensrc : jdk/make/gensrc
-make/gensrc/Gensrc-java.corba.gmk : corba/make/gensrc/Gensrc-java.corba.gmk
-make/gensrc/Gensrc-jdk.compiler.gmk : langtools/make/gensrc/Gensrc-jdk.compiler.gmk
-make/gensrc/Gensrc-jdk.hotspot.agent.gmk : hotspot/make/gensrc/Gensrc-jdk.hotspot.agent.gmk
-make/gensrc/Gensrc-jdk.internal.vm.compiler.gmk : hotspot/make/gensrc/Gensrc-jdk.internal.vm.compiler.gmk
-make/gensrc/Gensrc-jdk.javadoc.gmk : langtools/make/gensrc/Gensrc-jdk.javadoc.gmk
-make/gensrc/Gensrc-jdk.jdeps.gmk : langtools/make/gensrc/Gensrc-jdk.jdeps.gmk
-make/gensrc/Gensrc-jdk.jshell.gmk : langtools/make/gensrc/Gensrc-jdk.jshell.gmk
-make/gensrc/GensrcCommonLangtools.gmk : langtools/make/gensrc/GensrcCommon.gmk
-make/hotspot : hotspot/make
-make/jdk : jdk/make
-make/langtools : langtools/make
-make/launcher : jdk/make/launcher
-make/lib : jdk/make/lib
-make/lib/Lib-jdk.hotspot.agent.gmk : hotspot/make/lib/Lib-jdk.hotspot.agent.gmk
-make/mapfiles : jdk/make/mapfiles
-make/mapfiles/libjsig : hotspot/make/mapfiles/libjsig
-make/mapfiles/libjvm_db : hotspot/make/mapfiles/libjvm_db
-make/mapfiles/libjvm_dtrace : hotspot/make/mapfiles/libjvm_dtrace
-make/mapfiles/libsaproc : hotspot/make/mapfiles/libsaproc
-make/nashorn : nashorn/make
-make/nb_native : common/nb_native
-make/scripts/addNotices.sh : jdk/make/scripts/addNotices.sh
-make/scripts/compare.sh : common/bin/compare.sh
-make/scripts/compare_exceptions.sh.incl : common/bin/compare_exceptions.sh.incl
-make/scripts/genExceptions.sh : jdk/make/scripts/genExceptions.sh
-make/scripts/hide_important_warnings_from_javac.sh : common/bin/hide_important_warnings_from_javac.sh
-make/scripts/logger.sh : common/bin/logger.sh
-make/src/native/fixpath.c : common/src/fixpath.c
-make/test/JtregNativeHotspot.gmk : hotspot/make/test/JtregNative.gmk
-make/test/JtregNativeJdk.gmk : jdk/make/test/JtregNative.gmk
-test/jdk : jdk/test
-test/langtools : langtools/test
-test/nashorn : nashorn/test
-test/jaxp : jaxp/test
-test/hotspot/gtest : hotspot/test/native
-test/hotspot/jtreg : hotspot/test
-bin : common/bin
-bin/nashorn : nashorn/bin
-doc : common/doc
-doc/nashorn : nashorn/docs
@@ -1,237 +0,0 @@
-#!/bin/bash
-#
-# Copyright (c) 2014, 2017, Oracle and/or its affiliates. All rights reserved.
-# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
-#
-# This code is free software; you can redistribute it and/or modify it
-# under the terms of the GNU General Public License version 2 only, as
-# published by the Free Software Foundation.
-#
-# This code is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
-# version 2 for more details (a copy is included in the LICENSE file that
-# accompanied this code).
-#
-# You should have received a copy of the GNU General Public License version
-# 2 along with this work; if not, write to the Free Software Foundation,
-# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
-# or visit www.oracle.com if you need additional information or have any
-# questions.
-#
-
-# Script for updating a patch file as per the shuffled/unshuffled source location.
-
-usage() {
-  echo "Usage: $0 [-h|--help] [-v|--verbose] [-to9|-to10] [-r <repo>] <input_patch> <output_patch>"
-  echo "where:"
-  echo "  -to9            create patches appropriate for a JDK 9 source tree"
-  echo "                  When going to 9, the output patches will be suffixed with the"
-  echo "                  repo name"
-  echo "  -to10           create patches appropriate for a JDK 10 source tree"
-  echo "  -r <repo>       specify repo for source patch, set to 'top' for top repo"
-  echo "  <input_patch>   is the input patch file, that needs shuffling/unshuffling"
-  echo "  <output_patch>  is the updated patch file "
-  echo " "
-  exit 1
-}
-
-SCRIPT_DIR=`dirname $0`
-UNSHUFFLE_LIST=$SCRIPT_DIR"/unshuffle_list.txt"
-
-if [ ! -f "$UNSHUFFLE_LIST" ] ; then
-  echo "FATAL: cannot find $UNSHUFFLE_LIST" >&2
-  exit 1
-fi
-
-vflag="false"
-while [ $# -gt 0 ]
-do
-  case $1 in
-    -h | --help )
-      usage
-      ;;
-
-    -v | --verbose )
-      vflag="true"
-      ;;
-
-    -r)
-      repo="$2"
-      shift
-      ;;
-
-    -to9)
-      shuffle_to=9
-      ;;
-
-    -to10)
-      shuffle_to=10
-      ;;
-
-    -*)  # bad option
-      usage
-      ;;
-
-    * )  # non option
-      break
-      ;;
-  esac
-  shift
-done
-
-# Make sure we have the right number of arguments
-if [ ! $# -eq 2 ] ; then
-  echo "ERROR: Invalid number of arguments." >&2
-  usage
-fi
-
-# Check the given repo
-repos="top corba jaxp jaxws jdk langtools nashorn hotspot"
-found="false"
-if [ -n "$repo" ]; then
-  for r in $repos ; do
-    if [ $repo = "$r" ] ; then
-      found="true"
-      break;
-    fi
-  done
-  if [ $found = "false" ] ; then
-    echo "ERROR: Unknown repo: $repo. Should be one of [$repos]." >&2
-    usage
-  fi
-fi
-
-if [ "$shuffle_to" != "9" -a "$shuffle_to" != "10" ]; then
-  echo "ERROR: Must pick either -to9 or -to10"
-  exit 1
-fi
-
-# When going to 10, a repo must be specified for the source patch
-if [ "$shuffle_to" = "10" -a -z "$repo" ]; then
-  echo "ERROR: Must specify src repo for JDK 9 patch"
-  exit 1
-fi
-
-# Check given input/output files
-input="$1"
-if [ "x$input" = "x-" ] ; then
-  input="/dev/stdin"
-fi
-
-if [ ! -f $input -a "x$input" != "x/dev/stdin" ] ; then
-  echo "ERROR: Cannot find input patch file: $input" >&2
-  exit 1
-fi
-
-output="$2"
-if [ "x$output" = "x-" ] ; then
-  output="/dev/stdout"
-fi
-base_output="$output"
-
-if [ "$shuffle_to" = "10" ]; then
-  if [ -f $output -a "x$output" != "x/dev/stdout" ] ; then
-    echo "ERROR: Output patch already exists: $output" >&2
-    exit 1
-  fi
-else
-  for r in $repos; do
-    if [ -f "$output.$r" ]; then
-      echo "ERROR: Output patch already exists: $output.$r" >&2
-      exit 1
-    fi
-  done
-fi
-
-verbose() {
-  if [ ${vflag} = "true" ] ; then
-    echo "$@" >&2
-  fi
-}
-
-unshuffle() {
-  line=$@
-  verbose "Attempting to rewrite: \"$line\""
-
-  # Retrieve the file name
-  path=
-  if echo "$line" | egrep '^diff' > /dev/null ; then
-    if ! echo "$line" | egrep '\-\-git' > /dev/null ; then
-      echo "ERROR: Only git patches supported. Please use 'hg export --git ...'." >&2
-      exit 1
-    fi
-    path="`echo "$line" | sed -e s@'diff --git a/'@@ -e s@' b/.*$'@@`"
-  elif echo "$line" | egrep '^\-\-\-' > /dev/null ; then
-    path="`echo "$line" | sed -e s@'--- a/'@@`"
-  elif echo "$line" | egrep '^\+\+\+' > /dev/null ; then
-    path="`echo "$line" | sed s@'+++ b/'@@`"
-  fi
-  verbose "Extracted path: \"$path\""
-
-  # Find the most specific matches in the shuffle list
-  matches=
-  if [ -n "$repo" -a "$repo" != "top" ]; then
-    matchpath="$repo"/"$path"/x
-  else
-    matchpath="$path"/x
-  fi
-  while [ "$matchpath" != "" ] ; do
-    matchpath="`echo $matchpath | sed s@'\(.*\)/.*$'@'\1'@`"
-
-    if [ "$shuffle_to" =  "10" ] ; then
-      pattern=": $matchpath$"
-    else
-      pattern="^$matchpath :"
-    fi
-    verbose "Attempting to find \"$matchpath\""
-    matches=`egrep "$pattern" "$UNSHUFFLE_LIST"`
-    if ! [ "x${matches}" = "x" ] ; then
-      verbose "Got matches: [$matches]"
-      break;
-    fi
-
-    if ! echo "$matchpath" | egrep '.*/.*' > /dev/null ; then
-      break;
-    fi
-  done
-
-  # Rewrite the line, if we have a match
-  if ! [ "x${matches}" = "x" ] ; then
-    shuffled="${matches%% : *}"
-    unshuffled="${matches#* : }"
-    patch_suffix_9=""
-    for r in $repos; do
-      if [ "$unshuffled" != "${unshuffled#$r}" ]; then
-        unshuffled="${unshuffled#$r\/}"
-        patch_suffix_9=".$r"
-      fi
-    done
-    verbose "shuffled: $shuffled"
-    verbose "unshuffled: $unshuffled"
-    verbose "patch_suffix_9: $patch_suffix_9"
-    if [ "$shuffle_to" =  "10" ] ; then
-      newline="`echo "$line" | sed -e s@"$unshuffled"@"$shuffled"@g`"
-    else
-      newline="`echo "$line" | sed -e s@"$shuffled"@"$unshuffled"@g`"
-      output=$base_output$patch_suffix_9
-      verbose "Writing to $output"
-    fi
-    verbose "Rewriting to \"$newline\""
-    echo "$newline" >> $output
-  else
-    echo "WARNING: no match found for $path"
-    echo "$line" >> $output
-  fi
-}
-
-while IFS= read -r line
-do
-  if echo "$line" | egrep '^diff|^\-\-\-|^\+\+\+' > /dev/null ; then
-    unshuffle "$line"
-  else
-    printf "%s\n" "$line" >> $output
-  fi
-done < "$input"
@@ -23,9 +23,19 @@
 # The output of this script may require some degree of human curation:
 # - Redundant headers, e.g. both x.hpp, x.inline.hpp are included;
 # - Headers relative to a non-default feature should be protected by an
-#   appropriate 'if' clause to make sure all variants can build without 
+#   appropriate 'if' clause to make sure all variants can build without
 #   errors.

+source_path="$(dirname ${0})"
+this_script_dir="$(cd -- "${source_path}" > /dev/null && pwd)"
+if test -z "${this_script_dir}"; then
+  echo "Error: Could not determine location of this script"
+  exit 1
+fi
+
+# Work in top directory
+cd $this_script_dir/..
+
 # Time threshold for header compilation, if the time exceeds the
 # threshold the header will be precompiled.
 if [ -z "$MIN_MS" ]; then
@@ -75,6 +75,9 @@ Standard Library</a></li>
 Deduction</a></li>
 <li><a href="#expression-sfinae" id="toc-expression-sfinae">Expression
 SFINAE</a></li>
+<li><a href="#trailing-return-type-syntax-for-functions"
+id="toc-trailing-return-type-syntax-for-functions">Trailing return type
+syntax for functions</a></li>
 <li><a href="#non-type-template-parameter-values"
 id="toc-non-type-template-parameter-values">Non-type template parameter
 values</a></li>
@@ -83,8 +86,9 @@ values</a></li>
 <li><a href="#thread_local" id="toc-thread_local">thread_local</a></li>
 <li><a href="#nullptr" id="toc-nullptr">nullptr</a></li>
 <li><a href="#atomic" id="toc-atomic">&lt;atomic&gt;</a></li>
-<li><a href="#inline-variables" id="toc-inline-variables">Inline
-Variables</a></li>
+<li><a href="#variable-templates-and-inline-variables"
+id="toc-variable-templates-and-inline-variables">Variable Templates and
+Inline Variables</a></li>
 <li><a href="#initializing-variables-with-static-storage-duration"
 id="toc-initializing-variables-with-static-storage-duration">Initializing
 variables with static storage duration</a></li>
@@ -719,11 +723,14 @@ href="http://wg21.link/p0127r2">p0127r2</a>)<br> <code>auto</code> may
 be used as a placeholder for the type of a non-type template parameter.
 The type is deduced from the value provided in a template
 instantiation.</p></li>
-<li><p>Function return type deduction (<a
+</ul>
+<p><a name="function-return-type-deduction"></a> * Function return type
+deduction (<a
 href="https://isocpp.org/files/papers/N3638.html">n3638</a>)<br> Only
 use if the function body has a very small number of <code>return</code>
-statements, and generally relatively little other code.</p></li>
-<li><p>Class template argument deduction (<a
+statements, and generally relatively little other code.</p>
+<ul>
+<li>Class template argument deduction (<a
 href="http://wg21.link/n3602">n3602</a>, <a
 href="http://wg21.link/p0091r3">p0091r3</a>)<br> The template arguments
 of a class template may be deduced from the arguments to a constructor.
@@ -736,7 +743,7 @@ harder to understand, because explicit type information is lacking. But
 it can also remove the need to be explicit about types that are either
 obvious, or that are very hard to write. For example, these allow the
 addition of a scope-guard mechanism with nice syntax; something like
-this</p></li>
+this</li>
 </ul>
 <pre><code>  ScopeGuard guard{[&amp;]{ ... cleanup code ... }};</code></pre>
 <ul>
@@ -771,6 +778,44 @@ class="uri">https://gcc.gnu.org/bugzilla/show_bug.cgi?id=95468</a><br>
 <a
 href="https://developercommunity.visualstudio.com/content/problem/396562/sizeof-deduced-type-is-sometimes-not-a-constant-ex.html"
 class="uri">https://developercommunity.visualstudio.com/content/problem/396562/sizeof-deduced-type-is-sometimes-not-a-constant-ex.html</a></p>
+<h3 id="trailing-return-type-syntax-for-functions">Trailing return type
+syntax for functions</h3>
+<p>A function's return type may be specified after the parameters and
+qualifiers (<a
+href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2008/n2541.htm">n2541</a>).
+In such a declaration the normal return type is <code>auto</code> and
+the return type is indicated by <code>-&gt;</code> followed by the type.
+Although both use <code>auto</code> in the "normal" leading return type
+position, this differs from <a
+href="#function-return-type-deduction">function return type
+deduction</a>, in that the return type is explicit rather than deduced,
+but specified in a trailing position.</p>
+<p>Use of trailing return types is permitted. However, the normal,
+leading position for the return type is preferred. A trailing return
+type should only be used where it provides some benefit. Such benefits
+usually arise because a trailing return type is in a different scope
+than a leading return type.</p>
+<ul>
+<li><p>If the function identifier is a nested name specifier, then the
+trailing return type occurs in the nested scope. This may permit simpler
+naming in the return type because of the different name lookup
+context.</p></li>
+<li><p>The trailing return type is in the scope of the parameters,
+making their types accessible via <code>decltype</code>. For
+example</p></li>
+</ul>
+<pre><code>template&lt;typename T, typename U&gt; auto add(T t, U u) -&gt; decltype(t + u);</code></pre>
+<p>rather than</p>
+<pre><code>template&lt;typename T, typename U&gt; decltype((*(T*)0) + (*(U*)0)) add(T t, U u);</code></pre>
+<ul>
+<li>Complex calculated leading return types may obscure the normal
+syntactic boundaries, making it more difficult for a reader to find the
+function name and parameters. This is particularly common in cases where
+the return type is being used for <a
+href="https://en.cppreference.com/w/cpp/language/sfinae"
+title="Substitution Failure Is Not An Error">SFINAE</a>. A trailing
+return type may be preferable in such situations.</li>
+</ul>
 <h3 id="non-type-template-parameter-values">Non-type template parameter
 values</h3>
 <p>C++17 extended the arguments permitted for non-type template
@@ -893,12 +938,18 @@ differ from what the Java compilers implement.</p>
 "conservative" memory ordering, which may differ from (may be stronger
 than) sequentially consistent. There are algorithms in HotSpot that are
 believed to rely on that ordering.</p>
-<h3 id="inline-variables">Inline Variables</h3>
-<p>Variables with static storage duration may be declared
-<code>inline</code> (<a href="https://wg21.link/p0386r2">p0386r2</a>).
-This has similar effects as for declaring a function inline: it can be
-defined, identically, in multiple translation units, must be defined in
-every translation unit in which it is <a
+<h3 id="variable-templates-and-inline-variables">Variable Templates and
+Inline Variables</h3>
+<p>The use of variable templates (including static data member
+templates) (<a href="https://wg21.link/N3651">N3651</a>) is permitted.
+They provide parameterized variables and constants in a simple and
+direct form, instead of requiring the use of various workarounds.</p>
+<p>Variables with static storage duration and variable templates may be
+declared <code>inline</code> (<a
+href="https://wg21.link/p0386r2">p0386r2</a>), and this usage is
+permitted. This has similar effects as for declaring a function inline:
+it can be defined, identically, in multiple translation units, must be
+defined in every translation unit in which it is <a
 href="https://en.cppreference.com/w/cpp/language/definition"
 title="One Definition Rule">ODR used</a>, and the behavior of the
 program is as if there is exactly one variable.</p>
@@ -911,16 +962,17 @@ initializations can make initialization order problems worse. The few
 ordering constraints that exist for non-inline variables don't apply, as
 there isn't a single program-designated translation unit containing the
 definition.</p>
-<p>A <code>constexpr</code> static data member is implicitly
-<code>inline</code>. As a consequence, an <a
+<p>A <code>constexpr</code> static data member or static data member
+template is implicitly <code>inline</code>. As a consequence, an <a
 href="https://en.cppreference.com/w/cpp/language/definition"
-title="One Definition Rule">ODR use</a> of such a variable doesn't
-require a definition in some .cpp file. (This is a change from
-pre-C++17. Beginning with C++17, such a definition is considered a
-duplicate definition, and is deprecated.)</p>
-<p>Declaring a <code>thread_local</code> variable <code>inline</code> is
-forbidden for HotSpot code. <a href="#thread_local">The use of
-<code>thread_local</code></a> is already heavily restricted.</p>
+title="One Definition Rule">ODR use</a> of such a member doesn't require
+a definition in some .cpp file. (This is a change from pre-C++17.
+Beginning with C++17, such a definition is considered a duplicate
+definition, and is deprecated.)</p>
+<p>Declaring a <code>thread_local</code> variable template or
+<code>inline</code> variable is forbidden in HotSpot code. <a
+href="#thread_local">The use of <code>thread_local</code></a> is already
+heavily restricted.</p>
 <h3
 id="initializing-variables-with-static-storage-duration">Initializing
 variables with static storage duration</h3>
@@ -1807,11 +1859,6 @@ difference.</p>
 <h3 id="additional-undecided-features">Additional Undecided
 Features</h3>
 <ul>
-<li><p>Trailing return type syntax for functions (<a
-href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2008/n2541.htm">n2541</a>)</p></li>
-<li><p>Variable templates (<a
-href="https://isocpp.org/files/papers/N3651.pdf">n3651</a>, <a
-href="http://wg21.link/p0127r2">p0127r2</a>)</p></li>
 <li><p>Member initializers and aggregates (<a
 href="http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2013/n3653.html">n3653</a>)</p></li>
 <li><p>Rvalue references and move semantics</p></li>
@@ -642,6 +642,7 @@ use can make code much harder to understand.
 parameter. The type is deduced from the value provided in a template
 instantiation.

+<a name="function-return-type-deduction"></a>
 * Function return type deduction
 ([n3638](https://isocpp.org/files/papers/N3638.html))<br>
 Only use if the function body has a very small number of `return`
@@ -691,6 +692,42 @@ Here are a few closely related example bugs:<br>
 <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=95468><br>
 <https://developercommunity.visualstudio.com/content/problem/396562/sizeof-deduced-type-is-sometimes-not-a-constant-ex.html>

+### Trailing return type syntax for functions
+
+A function's return type may be specified after the parameters and qualifiers
+([n2541](http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2008/n2541.htm)).
+In such a declaration the normal return type is `auto` and the return type is
+indicated by `->` followed by the type.  Although both use `auto` in the
+"normal" leading return type position, this differs from
+[function return type deduction](#function-return-type-deduction),
+in that the return type is explicit rather than deduced, but specified in a
+trailing position.
+
+Use of trailing return types is permitted.  However, the normal, leading
+position for the return type is preferred. A trailing return type should only
+be used where it provides some benefit. Such benefits usually arise because a
+trailing return type is in a different scope than a leading return type.
+
+* If the function identifier is a nested name specifier, then the trailing
+return type occurs in the nested scope. This may permit simpler naming in the
+return type because of the different name lookup context.
+
+* The trailing return type is in the scope of the parameters, making their
+types accessible via `decltype`. For example
+```
+template<typename T, typename U> auto add(T t, U u) -> decltype(t + u);
+```
+rather than
+```
+template<typename T, typename U> decltype((*(T*)0) + (*(U*)0)) add(T t, U u);
+```
+
+* Complex calculated leading return types may obscure the normal syntactic
+boundaries, making it more difficult for a reader to find the function name and
+parameters. This is particularly common in cases where the return type is
+being used for [SFINAE]. A trailing return type may be preferable in such
+situations.
+
 ### Non-type template parameter values

 C++17 extended the arguments permitted for non-type template parameters
@@ -819,14 +856,19 @@ ordering, which may differ from (may be stronger than) sequentially
 consistent.  There are algorithms in HotSpot that are believed to rely
 on that ordering.

-### Inline Variables
+### Variable Templates and Inline Variables

-Variables with static storage duration may be declared `inline`
-([p0386r2](https://wg21.link/p0386r2)). This has similar effects as for
-declaring a function inline: it can be defined, identically, in multiple
-translation units, must be defined in every translation unit in which it is
-[ODR used][ODR], and the behavior of the program is as if there is exactly one
-variable.
+The use of variable templates (including static data member templates)
+([N3651](https://wg21.link/N3651)) is permitted. They provide parameterized
+variables and constants in a simple and direct form, instead of requiring the
+use of various workarounds.
+
+Variables with static storage duration and variable templates may be declared
+`inline` ([p0386r2](https://wg21.link/p0386r2)), and this usage is
+permitted. This has similar effects as for declaring a function inline: it can
+be defined, identically, in multiple translation units, must be defined in
+every translation unit in which it is [ODR used][ODR], and the behavior of the
+program is as if there is exactly one variable.

 Declaring a variable inline allows the complete definition to be in a header
 file, rather than having a declaration in a header and the definition in a
@@ -837,13 +879,15 @@ make initialization order problems worse. The few ordering constraints
 that exist for non-inline variables don't apply, as there isn't a single
 program-designated translation unit containing the definition.

-A `constexpr` static data member is implicitly `inline`. As a consequence, an
-[ODR use][ODR] of such a variable doesn't require a definition in some .cpp
+A `constexpr` static data member or static data member template
+is implicitly `inline`. As a consequence, an
+[ODR use][ODR] of such a member doesn't require a definition in some .cpp
 file. (This is a change from pre-C++17. Beginning with C++17, such a
 definition is considered a duplicate definition, and is deprecated.)

-Declaring a `thread_local` variable `inline` is forbidden for HotSpot code.
-[The use of `thread_local`](#thread_local) is already heavily restricted.
+Declaring a `thread_local` variable template or `inline` variable is forbidden
+in HotSpot code.  [The use of `thread_local`](#thread_local) is already
+heavily restricted.

 ### Initializing variables with static storage duration

@@ -1809,13 +1853,6 @@ See Object Lifetime: C++17 6.8/8, C++20 6.7.3/8

 ### Additional Undecided Features

-* Trailing return type syntax for functions
-([n2541](http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2008/n2541.htm))
-
-* Variable templates
-([n3651](https://isocpp.org/files/papers/N3651.pdf),
-[p0127r2](http://wg21.link/p0127r2))
-
 * Member initializers and aggregates
 ([n3653](http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2013/n3653.html))

@@ -301,7 +301,7 @@ ifneq ($(filter product-bundles% legacy-bundles, $(MAKECMDGOALS)), )
 	$(call LogWarn, Signing $(JDK_BUNDLE_NAME))
 	$(CODESIGN) -s "$(MACOSX_CODESIGN_IDENTITY)" \
 	    --timestamp --options runtime --deep --force \
-	    $(JDK_MACOSX_BUNDLE_DIR_SIGNED)/$(JDK_MACOSX_BUNDLE_TOP_DIR) $(LOG_DEBUG)
+	    $(JDK_MACOSX_BUNDLE_DIR_SIGNED)/$(JDK_MACOSX_BUNDLE_TOP_SUBDIR) $(LOG_DEBUG)
 	$(TOUCH) $@

    $(eval $(call SetupBundleFile, BUILD_JDK_BUNDLE, \
@@ -330,7 +330,7 @@ ifneq ($(filter product-bundles% legacy-bundles, $(MAKECMDGOALS)), )
 	$(call LogWarn, Signing $(JRE_BUNDLE_NAME))
 	$(CODESIGN) -s "$(MACOSX_CODESIGN_IDENTITY)" \
 	    --timestamp --options runtime --deep --force \
-	    $(JRE_MACOSX_BUNDLE_DIR_SIGNED)/$(JRE_MACOSX_BUNDLE_TOP_DIR) $(LOG_DEBUG)
+	    $(JRE_MACOSX_BUNDLE_DIR_SIGNED)/$(JRE_MACOSX_BUNDLE_TOP_SUBDIR) $(LOG_DEBUG)
 	$(TOUCH) $@

    $(eval $(call SetupBundleFile, BUILD_JRE_BUNDLE, \
@@ -210,17 +210,8 @@ AC_DEFUN([BASIC_SETUP_XCODE_SYSROOT],
    if test $? -ne 0; then
      AC_MSG_ERROR([The xcodebuild tool in the devkit reports an error: $XCODEBUILD_OUTPUT])
    fi
-  elif test "x$TOOLCHAIN_PATH" != x; then
-    UTIL_LOOKUP_PROGS(XCODEBUILD, xcodebuild, $TOOLCHAIN_PATH)
-    if test "x$XCODEBUILD" != x; then
-      XCODEBUILD_OUTPUT=`"$XCODEBUILD" -version 2>&1`
-      if test $? -ne 0; then
-        AC_MSG_WARN([Ignoring the located xcodebuild tool $XCODEBUILD due to an error: $XCODEBUILD_OUTPUT])
-        XCODEBUILD=
-      fi
-    fi
  else
-    UTIL_LOOKUP_PROGS(XCODEBUILD, xcodebuild)
+    UTIL_LOOKUP_TOOLCHAIN_PROGS(XCODEBUILD, xcodebuild)
    if test "x$XCODEBUILD" != x; then
      XCODEBUILD_OUTPUT=`"$XCODEBUILD" -version 2>&1`
      if test $? -ne 0; then
@@ -348,21 +339,11 @@ AC_DEFUN_ONCE([BASIC_SETUP_DEVKIT],

  # You can force the sysroot if the sysroot encoded into the compiler tools
  # is not correct.
-  AC_ARG_WITH(sys-root, [AS_HELP_STRING([--with-sys-root],
-      [alias for --with-sysroot for backwards compatibility])],
-      [SYSROOT=$with_sys_root]
-  )
-
  AC_ARG_WITH(sysroot, [AS_HELP_STRING([--with-sysroot],
      [use this directory as sysroot])],
      [SYSROOT=$with_sysroot]
  )

-  AC_ARG_WITH([tools-dir], [AS_HELP_STRING([--with-tools-dir],
-      [alias for --with-toolchain-path for backwards compatibility])],
-      [UTIL_PREPEND_TO_PATH([TOOLCHAIN_PATH],$with_tools_dir)]
-  )
-
  AC_ARG_WITH([toolchain-path], [AS_HELP_STRING([--with-toolchain-path],
      [prepend these directories when searching for toolchain binaries (compilers etc)])],
      [UTIL_PREPEND_TO_PATH([TOOLCHAIN_PATH],$with_toolchain_path)]
@@ -371,6 +352,9 @@ AC_DEFUN_ONCE([BASIC_SETUP_DEVKIT],
  AC_ARG_WITH([xcode-path], [AS_HELP_STRING([--with-xcode-path],
      [set up toolchain on Mac OS using a path to an Xcode installation])])

+  UTIL_DEPRECATED_ARG_WITH(sys-root)
+  UTIL_DEPRECATED_ARG_WITH(tools-dir)
+
  if test "x$with_xcode_path" != x; then
    if test "x$OPENJDK_BUILD_OS" = "xmacosx"; then
      UTIL_PREPEND_TO_PATH([TOOLCHAIN_PATH],
@@ -207,29 +207,14 @@ AC_DEFUN([BASIC_CHECK_GNU_MAKE],
  UTIL_SETUP_TOOL(MAKE,
  [
    # Try our hardest to locate a correct version of GNU make
-    UTIL_LOOKUP_PROGS(CHECK_GMAKE, gmake)
+    UTIL_LOOKUP_TOOLCHAIN_PROGS(CHECK_GMAKE, gmake)
    BASIC_CHECK_MAKE_VERSION("$CHECK_GMAKE", [gmake in PATH])

    if test "x$FOUND_MAKE" = x; then
-      UTIL_LOOKUP_PROGS(CHECK_MAKE, make)
+      UTIL_LOOKUP_TOOLCHAIN_PROGS(CHECK_MAKE, make)
      BASIC_CHECK_MAKE_VERSION("$CHECK_MAKE", [make in PATH])
    fi

-    if test "x$FOUND_MAKE" = x; then
-      if test "x$TOOLCHAIN_PATH" != x; then
-        # We have a toolchain path, check that as well before giving up.
-        OLD_PATH=$PATH
-        PATH=$TOOLCHAIN_PATH:$PATH
-        UTIL_LOOKUP_PROGS(CHECK_TOOLSDIR_GMAKE, gmake)
-        BASIC_CHECK_MAKE_VERSION("$CHECK_TOOLSDIR_GMAKE", [gmake in tools-dir])
-        if test "x$FOUND_MAKE" = x; then
-          UTIL_LOOKUP_PROGS(CHECK_TOOLSDIR_MAKE, make)
-          BASIC_CHECK_MAKE_VERSION("$CHECK_TOOLSDIR_MAKE", [make in tools-dir])
-        fi
-        PATH=$OLD_PATH
-      fi
-    fi
-
    if test "x$FOUND_MAKE" = x; then
      AC_MSG_ERROR([Cannot find GNU make $MAKE_REQUIRED_VERSION or newer! Please put it in the path, or add e.g. MAKE=/opt/gmake3.81/make as argument to configure.])
    fi
@@ -162,12 +162,7 @@ AC_DEFUN([BPERF_SETUP_CCACHE],
  # Check if ccache is available
  CCACHE_AVAILABLE=true

-  OLD_PATH="$PATH"
-  if test "x$TOOLCHAIN_PATH" != x; then
-    PATH=$TOOLCHAIN_PATH:$PATH
-  fi
-  UTIL_LOOKUP_PROGS(CCACHE, ccache)
-  PATH="$OLD_PATH"
+  UTIL_LOOKUP_TOOLCHAIN_PROGS(CCACHE, ccache)

  AC_MSG_CHECKING([if ccache is available])
  if test "x$TOOLCHAIN_TYPE" != "xgcc" && test "x$TOOLCHAIN_TYPE" != "xclang"; then
@@ -110,4 +110,4 @@ $MV $OUTPUTDIR/compare.log $OUTPUTDIR/compare.log.old 2> /dev/null

 export SCRIPT_DIR="$( cd "$( dirname "$0" )" > /dev/null && pwd )"

-$BASH $TOPDIR/make/scripts/logger.sh $OUTPUTDIR/compare.log $BASH "$REAL_COMPARE_SCRIPT" "$@"
+$BASH $TOPDIR/make/scripts/compare-logger.sh $OUTPUTDIR/compare.log $BASH "$REAL_COMPARE_SCRIPT" "$@"
@@ -934,48 +934,6 @@ AC_DEFUN([FLAGS_SETUP_CFLAGS_CPU_DEP],
        IF_FALSE: [$2FDLIBM_CFLAGS=""])
  fi
  AC_SUBST($2FDLIBM_CFLAGS)
-
-  # Check whether the compiler supports the Arm C Language Extensions (ACLE)
-  # for SVE. Set SVE_CFLAGS to -march=armv8-a+sve if it does.
-  # ACLE and this flag are required to build the aarch64 SVE related functions in
-  # libvectormath. Apple Silicon does not support SVE; use macOS as a proxy for
-  # that check.
-  if test "x$OPENJDK_TARGET_CPU" = "xaarch64" && test "x$OPENJDK_TARGET_OS" = "xlinux"; then
-    if test "x$TOOLCHAIN_TYPE" = xgcc || test "x$TOOLCHAIN_TYPE" = xclang; then
-      AC_LANG_PUSH(C)
-      OLD_CFLAGS="$CFLAGS"
-      CFLAGS="$CFLAGS -march=armv8-a+sve"
-      AC_MSG_CHECKING([if Arm SVE ACLE is supported])
-      AC_COMPILE_IFELSE([AC_LANG_PROGRAM([#include <arm_sve.h>],
-          [
-            svint32_t r = svdup_n_s32(1);
-            return 0;
-          ])],
-          [
-            AC_MSG_RESULT([yes])
-            $2SVE_CFLAGS="-march=armv8-a+sve"
-            # Switching the initialization mode with gcc from 'pattern' to 'zero'
-            # avoids the use of unsupported `__builtin_clear_padding` for variable
-            # length aggregates
-            if test "x$DEBUG_LEVEL" != xrelease && test "x$TOOLCHAIN_TYPE" = xgcc ; then
-              INIT_ZERO_FLAG="-ftrivial-auto-var-init=zero"
-              FLAGS_COMPILER_CHECK_ARGUMENTS(ARGUMENT: [$INIT_ZERO_FLAG],
-                IF_TRUE: [
-                  $2SVE_CFLAGS="${$2SVE_CFLAGS} $INIT_ZERO_FLAG"
-                ]
-              )
-            fi
-          ],
-          [
-            AC_MSG_RESULT([no])
-            $2SVE_CFLAGS=""
-          ]
-      )
-      CFLAGS="$OLD_CFLAGS"
-      AC_LANG_POP(C)
-    fi
-  fi
-  AC_SUBST($2SVE_CFLAGS)
 ])

 AC_DEFUN_ONCE([FLAGS_SETUP_BRANCH_PROTECTION],
@@ -74,7 +74,7 @@ AC_DEFUN([FLAGS_SETUP_LDFLAGS_HELPER],
      # Clang needs the lld linker to work correctly
      BASIC_LDFLAGS="-fuse-ld=lld -Wl,--exclude-libs,ALL"
      if test "x$CXX_IS_USER_SUPPLIED" = xfalse && test "x$CC_IS_USER_SUPPLIED" = xfalse; then
-        UTIL_REQUIRE_PROGS(LLD, lld, $TOOLCHAIN_PATH:$PATH)
+        UTIL_REQUIRE_TOOLCHAIN_PROGS(LLD, lld)
      fi
    fi
    if test "x$OPENJDK_TARGET_OS" = xaix; then
@@ -107,6 +107,62 @@ AC_DEFUN([FLAGS_SETUP_NMFLAGS],
  AC_SUBST(NMFLAGS)
 ])

+# Check whether the compiler supports the Arm C Language Extensions (ACLE)
+# for SVE. Set SVE_CFLAGS to -march=armv8-a+sve if it does.
+# ACLE and this flag are required to build the aarch64 SVE related functions
+# in libvectormath.
+AC_DEFUN([FLAGS_SETUP_SVE],
+[
+  AARCH64_SVE_AVAILABLE=false
+  # Apple Silicon does not support SVE; use macOS as a proxy for that check.
+  if test "x$OPENJDK_TARGET_CPU" = "xaarch64" && test "x$OPENJDK_TARGET_OS" = "xlinux"; then
+    if test "x$TOOLCHAIN_TYPE" = xgcc || test "x$TOOLCHAIN_TYPE" = xclang; then
+      # check the compiler and binutils support sve or not
+      AC_MSG_CHECKING([if Arm SVE ACLE is supported])
+      AC_LANG_PUSH([C])
+      saved_cflags="$CFLAGS"
+      CFLAGS="$CFLAGS -march=armv8-a+sve $CFLAGS_WARNINGS_ARE_ERRORS ARG_ARGUMENT"
+      AC_COMPILE_IFELSE([AC_LANG_PROGRAM(
+        [
+          #include <arm_sve.h>
+          svfloat64_t a() {}
+        ],
+        [
+          svint32_t r = svdup_n_s32(1)
+        ])],
+        [
+          AARCH64_SVE_AVAILABLE=true
+        ]
+      )
+      CFLAGS="$saved_cflags"
+      AC_LANG_POP([C])
+      AC_MSG_RESULT([$AARCH64_SVE_AVAILABLE])
+    fi
+  fi
+
+  UTIL_ARG_ENABLE(NAME: aarch64-sve, DEFAULT: auto,
+    RESULT: AARCH64_SVE_ENABLED,
+    DESC: [Use SVE when compiling libsleef],
+    AVAILABLE: $AARCH64_SVE_AVAILABLE)
+  SVE_CFLAGS=""
+  if test "x$AARCH64_SVE_ENABLED" = xtrue; then
+    SVE_CFLAGS="-march=armv8-a+sve"
+    # Switching the initialization mode with gcc from 'pattern' to 'zero'
+    # avoids the use of unsupported `__builtin_clear_padding` for variable
+    # length aggregates
+    if test "x$DEBUG_LEVEL" != xrelease && test "x$TOOLCHAIN_TYPE" = xgcc ; then
+      AC_MSG_CHECKING([Switching the initialization mode with gcc from pattern to zero])
+      INIT_ZERO_FLAG="-ftrivial-auto-var-init=zero"
+      FLAGS_COMPILER_CHECK_ARGUMENTS(ARGUMENT: [$INIT_ZERO_FLAG],
+        IF_TRUE: [
+          SVE_CFLAGS="${SVE_CFLAGS} $INIT_ZERO_FLAG"
+        ]
+      )
+    fi
+  fi
+  AC_SUBST(SVE_CFLAGS)
+])
+
 ################################################################################
 # platform independent
 AC_DEFUN([FLAGS_SETUP_ASFLAGS],
@@ -374,6 +374,7 @@ AC_DEFUN([FLAGS_SETUP_FLAGS],
  FLAGS_SETUP_RCFLAGS
  FLAGS_SETUP_NMFLAGS

+  FLAGS_SETUP_SVE
  FLAGS_SETUP_ASFLAGS
  FLAGS_SETUP_ASFLAGS_CPU_DEP([TARGET])
  FLAGS_SETUP_ASFLAGS_CPU_DEP([BUILD], [OPENJDK_BUILD_])
@@ -28,7 +28,7 @@
 ################################################################################

 # Minimum supported versions
-JTREG_MINIMUM_VERSION=7.5.2
+JTREG_MINIMUM_VERSION=8
 GTEST_MINIMUM_VERSION=1.14.0

 ################################################################################
@@ -898,12 +898,14 @@ JDK_MACOSX_BUNDLE_DIR = $(IMAGES_OUTPUTDIR)/$(JDK_MACOSX_BUNDLE_SUBDIR)
 JRE_MACOSX_BUNDLE_DIR = $(IMAGES_OUTPUTDIR)/$(JRE_MACOSX_BUNDLE_SUBDIR)
 JDK_MACOSX_BUNDLE_DIR_SIGNED = $(IMAGES_OUTPUTDIR)/$(JDK_MACOSX_BUNDLE_SUBDIR_SIGNED)
 JRE_MACOSX_BUNDLE_DIR_SIGNED = $(IMAGES_OUTPUTDIR)/$(JRE_MACOSX_BUNDLE_SUBDIR_SIGNED)
-JDK_MACOSX_BUNDLE_TOP_DIR = jdk-$(VERSION_NUMBER).jdk
-JRE_MACOSX_BUNDLE_TOP_DIR = jre-$(VERSION_NUMBER).jre
-JDK_MACOSX_CONTENTS_SUBDIR = $(JDK_MACOSX_BUNDLE_TOP_DIR)/Contents
-JRE_MACOSX_CONTENTS_SUBDIR = $(JRE_MACOSX_BUNDLE_TOP_DIR)/Contents
+JDK_MACOSX_BUNDLE_TOP_SUBDIR = jdk-$(VERSION_NUMBER).jdk
+JRE_MACOSX_BUNDLE_TOP_SUBDIR = jre-$(VERSION_NUMBER).jre
+JDK_MACOSX_CONTENTS_SUBDIR = $(JDK_MACOSX_BUNDLE_TOP_SUBDIR)/Contents
+JRE_MACOSX_CONTENTS_SUBDIR = $(JRE_MACOSX_BUNDLE_TOP_SUBDIR)/Contents
 JDK_MACOSX_CONTENTS_DIR = $(JDK_MACOSX_BUNDLE_DIR)/$(JDK_MACOSX_CONTENTS_SUBDIR)
 JRE_MACOSX_CONTENTS_DIR = $(JRE_MACOSX_BUNDLE_DIR)/$(JRE_MACOSX_CONTENTS_SUBDIR)
+JDK_MACOSX_BUNDLE_TOP_DIR = $(JDK_MACOSX_BUNDLE_DIR)/$(JDK_MACOSX_BUNDLE_TOP_SUBDIR)
+JRE_MACOSX_BUNDLE_TOP_DIR = $(JRE_MACOSX_BUNDLE_DIR)/$(JRE_MACOSX_BUNDLE_TOP_SUBDIR)

 # Bundle names
 ifneq ($(VERSION_BUILD), )
@@ -276,9 +276,6 @@ AC_DEFUN_ONCE([TOOLCHAIN_PRE_DETECTION],
  ORG_CFLAGS="$CFLAGS"
  ORG_CXXFLAGS="$CXXFLAGS"

-  # autoconf magic only relies on PATH, so update it if tools dir is specified
-  OLD_PATH="$PATH"
-
  if test "x$OPENJDK_BUILD_OS" = "xmacosx"; then
    if test "x$XCODEBUILD" != x; then
      XCODE_VERSION_OUTPUT=`"$XCODEBUILD" -version 2> /dev/null | $HEAD -n 1`
@@ -300,9 +297,10 @@ AC_DEFUN_ONCE([TOOLCHAIN_PRE_DETECTION],
  fi
  AC_SUBST(TOOLCHAIN_VERSION)

-  # Finally prepend TOOLCHAIN_PATH to the PATH, to allow --with-tools-dir to
-  # override all other locations.
-  if test "x$TOOLCHAIN_PATH" != x; then
+  # For the microsoft toolchain the toolchain path needs to be added to the
+  # normal path, or the compiler will not work in some situations in later
+  # configure checks.
+  if test "x$TOOLCHAIN_TYPE" = "xmicrosoft" && test "x$TOOLCHAIN_PATH" != x; then
    export PATH=$TOOLCHAIN_PATH:$PATH
  fi
 ])
@@ -310,13 +308,6 @@ AC_DEFUN_ONCE([TOOLCHAIN_PRE_DETECTION],
 # Restore path, etc
 AC_DEFUN_ONCE([TOOLCHAIN_POST_DETECTION],
 [
-  # Restore old path, except for the microsoft toolchain, which requires the
-  # toolchain path to remain in place. Otherwise the compiler will not work in
-  # some situations in later configure checks.
-  if test "x$TOOLCHAIN_TYPE" != "xmicrosoft"; then
-    PATH="$OLD_PATH"
-  fi
-
  # Restore the flags to the user specified values.
  # This is necessary since AC_PROG_CC defaults CFLAGS to "-g -O2"
  CFLAGS="$ORG_CFLAGS"
@@ -458,17 +458,18 @@ AC_DEFUN([UTIL_LOOKUP_PROGS],

 ################################################################################
 # Call UTIL_SETUP_TOOL with AC_CHECK_TOOLS to locate the tool. This will look
-# first for cross-compilation tools.
+# first for tools using the cross-compilation prefix, and then for tools without
+# this prefix. For each of these name variants, it will look first in the
+# toolchain path, and then in the normal path.
 # $1: variable to set
 # $2: executable name (or list of names) to look for
-# $3: [path]
 AC_DEFUN([UTIL_LOOKUP_TOOLCHAIN_PROGS],
 [
  if test "x$ac_tool_prefix" = x; then
-    UTIL_LOOKUP_PROGS($1, $2, $3)
+    UTIL_LOOKUP_PROGS($1, $2, [$TOOLCHAIN_PATH:$PATH])
  else
    prefixed_names=$(for name in $2; do echo ${ac_tool_prefix}${name} $name; done)
-    UTIL_LOOKUP_PROGS($1, $prefixed_names, $3)
+    UTIL_LOOKUP_PROGS($1, $prefixed_names, [$TOOLCHAIN_PATH:$PATH])
  fi
 ])

@@ -497,10 +498,9 @@ AC_DEFUN([UTIL_REQUIRE_PROGS],
 # Like UTIL_LOOKUP_PROGS but fails if no tool was found.
 # $1: variable to set
 # $2: executable name (or list of names) to look for
-# $3: [path]
 AC_DEFUN([UTIL_REQUIRE_TOOLCHAIN_PROGS],
 [
-  UTIL_LOOKUP_TOOLCHAIN_PROGS($1, $2, $3)
+  UTIL_LOOKUP_TOOLCHAIN_PROGS($1, $2)
  UTIL_CHECK_NONEMPTY($1)
 ])

@@ -82,6 +82,8 @@ ifeq ($(INCLUDE), true)
 #   INFO        : Message to display at LOG=info level when running command (optional)
 #   WARN        : Message to display at LOG=warn level when running command (optional)
 #   DEPS        : Dependencies for the execution to take place
+#   DRYRUN      : Set to true to perform everything but executing the command \
+#                 (defaults to false, primarily intended for debugging)
 #

 # Setup make rules for copying files, with an option to do more complex
@@ -146,9 +148,12 @@ define SetupExecuteBody
    $1_INFO := Running commands for $1
  endif

+  $1_VARDEPS := $$($1_COMMAND) $$($1_PRE_COMMAND) $$($1_POST_COMMAND)
+  $1_VARDEPS_FILE := $$(call DependOnVariable, $1_VARDEPS)
+
  ifneq ($$($1_PRE_COMMAND), )

-    $$($1_PRE_MARKER): $$($1_DEPS)
+    $$($1_PRE_MARKER): $$($1_DEPS) $$($1_VARDEPS_FILE)
        ifneq ($$($1_WARN), )
 	  $$(call LogWarn, $$($1_WARN))
        endif
@@ -161,15 +166,20 @@ define SetupExecuteBody
 	$$(TOUCH) $$@

    $$($1_EXEC_RESULT): $$($1_PRE_MARKER)
-	$$(call ExecuteWithLog, $$($1_BASE)_exec, \
-	    cd $$($1_WORKING_DIR) && $$($1_COMMAND))
+        ifneq ($$($1_DRYRUN), true)
+	  $$(call ExecuteWithLog, $$($1_BASE)_exec, \
+	      cd $$($1_WORKING_DIR) && $$($1_COMMAND))
+        else
+	  $$(call LogWarn, DRYRUN enabled for $1, not actually running command)
+	  $$(TOUCH) $$@
+        endif
        ifeq ($$($1_EXEC_RESULT), $$($1_EXEC_MARKER))
 	  $$(TOUCH) $$@
        endif

    $1 := $$($1_PRE_MARKER) $$($1_EXEC_RESULT)
  else
-    $$($1_EXEC_RESULT): $$($1_DEPS)
+    $$($1_EXEC_RESULT): $$($1_DEPS) $$($1_VARDEPS_FILE)
        ifneq ($$($1_WARN), )
 	  $$(call LogWarn, $$($1_WARN))
        endif
@@ -177,8 +187,13 @@ define SetupExecuteBody
 	  $$(call LogInfo, $$($1_INFO))
        endif
 	$$(call MakeDir, $$(call EncodeSpace, $$($1_WORKING_DIR)) $$(call EncodeSpace, $$($1_SUPPORT_DIR)) $$(call EncodeSpace, $$($1_OUTPUT_DIR)))
-	$$(call ExecuteWithLog, $$($1_BASE)_exec, \
-	    cd $$($1_WORKING_DIR) && $$($1_COMMAND))
+        ifneq ($$($1_DRYRUN), true)
+	  $$(call ExecuteWithLog, $$($1_BASE)_exec, \
+	      cd $$($1_WORKING_DIR) && $$($1_COMMAND))
+        else
+	  $$(call LogWarn, DRYRUN enabled for $1, not actually running command)
+	  $$(TOUCH) $$@
+        endif
        ifeq ($$($1_EXEC_RESULT), $$($1_EXEC_MARKER))
 	  $$(TOUCH) $$@
        endif
@@ -26,7 +26,7 @@
 # Versions and download locations for dependencies used by GitHub Actions (GHA)

 GTEST_VERSION=1.14.0
-JTREG_VERSION=7.5.2+1
+JTREG_VERSION=8+2

 LINUX_X64_BOOT_JDK_EXT=tar.gz
 LINUX_X64_BOOT_JDK_URL=https://download.java.net/java/GA/jdk24/1f9ff9062db4449d8ca828c504ffae90/36/GPL/openjdk-24_linux-x64_bin.tar.gz
@@ -1174,9 +1174,9 @@ var getJibProfilesDependencies = function (input, common) {
        jtreg: {
            server: "jpg",
            product: "jtreg",
-            version: "7.5.2",
-            build_number: "1",
-            file: "bundles/jtreg-7.5.2+1.zip",
+            version: "8",
+            build_number: "2",
+            file: "bundles/jtreg-8+2.zip",
            environment_name: "JT_HOME",
            environment_path: input.get("jtreg", "home_path") + "/bin",
            configure_args: "--with-jtreg=" + input.get("jtreg", "home_path"),
@@ -62,13 +62,13 @@ $(eval $(call SetupJdkLibrary, BUILD_GTEST_LIBGTEST, \
    DISABLED_WARNINGS_gcc := format-nonliteral maybe-uninitialized undef \
        unused-result zero-as-null-pointer-constant, \
    DISABLED_WARNINGS_clang := format-nonliteral undef unused-result, \
+    DISABLED_WARNINGS_microsoft := 4530, \
    DEFAULT_CFLAGS := false, \
    CFLAGS := $(JVM_CFLAGS) \
        -I$(GTEST_FRAMEWORK_SRC)/googletest \
        -I$(GTEST_FRAMEWORK_SRC)/googletest/include \
        -I$(GTEST_FRAMEWORK_SRC)/googlemock \
        -I$(GTEST_FRAMEWORK_SRC)/googlemock/include, \
-    CFLAGS_windows := -EHsc, \
    CFLAGS_macosx := -DGTEST_OS_MAC=1, \
    OPTIMIZATION := $(JVM_OPTIMIZATION), \
    COPY_DEBUG_SYMBOLS := $(GTEST_COPY_DEBUG_SYMBOLS), \
@@ -98,7 +98,6 @@ $(eval $(call SetupJdkLibrary, BUILD_GTEST_LIBJVM, \
        -I$(GTEST_FRAMEWORK_SRC)/googletest/include \
        -I$(GTEST_FRAMEWORK_SRC)/googlemock/include \
        $(addprefix -I, $(GTEST_TEST_SRC)), \
-    CFLAGS_windows := -EHsc, \
    CFLAGS_macosx := -DGTEST_OS_MAC=1, \
    DISABLED_WARNINGS_gcc := $(DISABLED_WARNINGS_gcc) \
        undef stringop-overflow, \
@@ -110,7 +109,7 @@ $(eval $(call SetupJdkLibrary, BUILD_GTEST_LIBJVM, \
        self-assign-overloaded, \
    DISABLED_WARNINGS_clang_test_g1ServiceThread.cpp := delete-abstract-non-virtual-dtor, \
    DISABLED_WARNINGS_clang_test_logDecorations.cpp := missing-field-initializers, \
-    DISABLED_WARNINGS_microsoft := $(DISABLED_WARNINGS_microsoft), \
+    DISABLED_WARNINGS_microsoft := $(DISABLED_WARNINGS_microsoft) 4530, \
    LD_SET_ORIGIN := false, \
    DEFAULT_LDFLAGS := false, \
    LDFLAGS := $(JVM_LDFLAGS), \
@@ -33,7 +33,6 @@ include gensrc/GensrcBuffer.gmk
 include gensrc/GensrcCharacterData.gmk
 include gensrc/GensrcCharsetCoder.gmk
 include gensrc/GensrcCharsetMapping.gmk
-include gensrc/GensrcExceptions.gmk
 include gensrc/GensrcMisc.gmk
 include gensrc/GensrcModuleLoaderMap.gmk
 include gensrc/GensrcRegex.gmk
@@ -1,57 +0,0 @@
-#
-# Copyright (c) 2011, 2025, Oracle and/or its affiliates. All rights reserved.
-# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
-#
-# This code is free software; you can redistribute it and/or modify it
-# under the terms of the GNU General Public License version 2 only, as
-# published by the Free Software Foundation.  Oracle designates this
-# particular file as subject to the "Classpath" exception as provided
-# by Oracle in the LICENSE file that accompanied this code.
-#
-# This code is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
-# version 2 for more details (a copy is included in the LICENSE file that
-# accompanied this code).
-#
-# You should have received a copy of the GNU General Public License version
-# 2 along with this work; if not, write to the Free Software Foundation,
-# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
-# or visit www.oracle.com if you need additional information or have any
-# questions.
-#
-
-include MakeIncludeStart.gmk
-ifeq ($(INCLUDE), true)
-
-################################################################################
-
-GENSRC_EXCEPTIONS :=
-
-GENSRC_EXCEPTIONS_DST := $(SUPPORT_OUTPUTDIR)/gensrc/java.base/java/nio
-
-GENSRC_EXCEPTIONS_SRC := $(MODULE_SRC)/share/classes/java/nio
-GENSRC_EXCEPTIONS_CMD := $(TOPDIR)/make/scripts/genExceptions.sh
-
-GENSRC_EXCEPTIONS_SRC_DIRS := . charset channels
-
-$(GENSRC_EXCEPTIONS_DST)/_the.%.marker: $(GENSRC_EXCEPTIONS_SRC)/%/exceptions \
-    $(GENSRC_EXCEPTIONS_CMD)
-	$(call LogInfo, Generating exceptions java.nio $*)
-	$(call MakeDir, $(@D)/$*)
-	SCRIPTS="$(TOPDIR)/make/scripts" AWK="$(AWK)" SH="$(SH)" $(SH) \
-	    $(GENSRC_EXCEPTIONS_CMD) $< $(@D)/$* $(LOG_DEBUG)
-	$(TOUCH) $@
-
-GENSRC_EXCEPTIONS += $(foreach D, $(GENSRC_EXCEPTIONS_SRC_DIRS), $(GENSRC_EXCEPTIONS_DST)/_the.$(D).marker)
-
-$(GENSRC_EXCEPTIONS): $(BUILD_TOOLS_JDK)
-
-TARGETS += $(GENSRC_EXCEPTIONS)
-
-################################################################################
-
-endif # include guard
-include MakeIncludeEnd.gmk
@@ -1,45 +0,0 @@
-#! /bin/sh
-#
-# Copyright (c) 2007, 2020, Oracle and/or its affiliates. All rights reserved.
-# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
-#
-# This code is free software; you can redistribute it and/or modify it
-# under the terms of the GNU General Public License version 2 only, as
-# published by the Free Software Foundation.  Oracle designates this
-# particular file as subject to the "Classpath" exception as provided
-# by Oracle in the LICENSE file that accompanied this code.
-#
-# This code is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
-# version 2 for more details (a copy is included in the LICENSE file that
-# accompanied this code).
-#
-# You should have received a copy of the GNU General Public License version
-# 2 along with this work; if not, write to the Free Software Foundation,
-# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
-# or visit www.oracle.com if you need additional information or have any
-# questions.
-#
-
-# Parse the first contiguous comment block in this script and generate
-# a java comment block. If this script is invoked with a copyright
-# year/year range, the java comment block will contain a Sun copyright.
-
-COPYRIGHT_YEARS="$1"
-
-cat <<__END__
-/*
-__END__
-
-if [ "x$COPYRIGHT_YEARS" != x ]; then
-  cat <<__END__
- * Copyright (c) $COPYRIGHT_YEARS Oracle and/or its affiliates. All rights reserved.
-__END__
-fi
-
-$AWK ' /^#.*Copyright.*Oracle/ { next }
-    /^#([^!]|$)/ { sub(/^#/, " *"); print }
-    /^$/ { print " */"; exit } ' $0
@@ -1,116 +0,0 @@
-#! /bin/sh
-#
-# Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved.
-# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
-#
-# This code is free software; you can redistribute it and/or modify it
-# under the terms of the GNU General Public License version 2 only, as
-# published by the Free Software Foundation.  Oracle designates this
-# particular file as subject to the "Classpath" exception as provided
-# by Oracle in the LICENSE file that accompanied this code.
-#
-# This code is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
-# version 2 for more details (a copy is included in the LICENSE file that
-# accompanied this code).
-#
-# You should have received a copy of the GNU General Public License version
-# 2 along with this work; if not, write to the Free Software Foundation,
-# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
-# or visit www.oracle.com if you need additional information or have any
-# questions.
-#
-
-# Generate exception classes
-
-SPEC=$1
-DST=$2
-
-gen() {
-  ID=$1
-  WHAT=$2
-  SVUID=$3
-  ARG_TYPE=$4
-  ARG_ID=$5
-  ARG_PROP=$6
-  ARG_PHRASE=$7
-  ARG_PARAM="$ARG_TYPE$ $ARG_ID"
-  echo '-->' $DST/$ID.java
-  out=$DST/${ID}.java
-
-  $SH ${SCRIPTS}/addNotices.sh "$COPYRIGHT_YEARS" > $out
-
-  cat >>$out <<__END__
-
-// -- This file was mechanically generated: Do not edit! -- //
-
-package $PACKAGE;
-
-
-/**$WHAT
- *
- * @since $SINCE
- */
-
-public `if [ ${ABSTRACT:-0} = 1 ];
-        then echo 'abstract '; fi`class $ID
-    extends ${SUPER}
-{
-
-    @java.io.Serial
-    private static final long serialVersionUID = $SVUID;
-__END__
-
-  if [ $ARG_ID ]; then
-
-    cat >>$out <<__END__
-
-    /**
-     * The $ARG_PHRASE.
-     *
-     * @serial
-     */
-    private $ARG_TYPE $ARG_ID;
-
-    /**
-     * Constructs an instance of this class.
-     *
-     * @param  $ARG_ID
-     *         The $ARG_PHRASE
-     */
-    public $ID($ARG_TYPE $ARG_ID) {
-        super(String.valueOf($ARG_ID));
-	this.$ARG_ID = $ARG_ID;
-    }
-
-    /**
-     * Retrieves the $ARG_PHRASE.
-     *
-     * @return  The $ARG_PHRASE
-     */
-    public $ARG_TYPE get$ARG_PROP() {
-        return $ARG_ID;
-    }
-
-}
-__END__
-
-  else
-
-    cat >>$out <<__END__
-
-    /**
-     * Constructs an instance of this class.
-     */
-    public $ID() { }
-
-}
-__END__
-
-  fi
-}
-
-. $SPEC
@@ -1,36 +0,0 @@
-#!/bin/bash
-#
-# Copyright (c) 2012, 2020, Oracle and/or its affiliates. All rights reserved.
-# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
-#
-# This code is free software; you can redistribute it and/or modify it
-# under the terms of the GNU General Public License version 2 only, as
-# published by the Free Software Foundation.
-#
-# This code is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
-# version 2 for more details (a copy is included in the LICENSE file that
-# accompanied this code).
-#
-# You should have received a copy of the GNU General Public License version
-# 2 along with this work; if not, write to the Free Software Foundation,
-# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
-# or visit www.oracle.com if you need additional information or have any
-# questions.
-#
-
-GREP=grep
-
-#
-EXP="Note: Some input files use or override a deprecated API."
-EXP="${EXP}|Note: Recompile with -Xlint:deprecation for details."
-EXP="${EXP}|Note: Some input files use unchecked or unsafe operations."
-EXP="${EXP}|Note: Recompile with -Xlint:unchecked for details."
-EXP="${EXP}| warning"
-EXP="${EXP}|uses or overrides a deprecated API."
-EXP="${EXP}|uses unchecked or unsafe operations."
-#
-${GREP} --line-buffered -v -E "${EXP}"
@@ -5965,9 +5965,6 @@ attributes %{
  instruction_unit_size = 4;         // An instruction is 4 bytes long
  instruction_fetch_unit_size = 64;  // The processor fetches one line
  instruction_fetch_units = 1;       // of 64 bytes
-
-  // List of nop instructions
-  nops( MachNop );
 %}

 // We don't use an actual pipeline model so don't care about resources
@@ -216,11 +216,6 @@ source %{
          return false;
        }
        break;
-      case Op_ExpandV:
-        if (UseSVE < 2 || is_subword_type(bt)) {
-          return false;
-        }
-        break;
      case Op_VectorMaskToLong:
        if (UseSVE > 0 && vlen > 64) {
          return false;
@@ -7113,10 +7108,39 @@ instruct vcompressS(vReg dst, vReg src, pReg pg,
  ins_pipe(pipe_slow);
 %}

-instruct vexpand(vReg dst, vReg src, pRegGov pg) %{
+instruct vexpand_neon(vReg dst, vReg src, vReg mask, vReg tmp1, vReg tmp2) %{
+  predicate(UseSVE == 0);
+  match(Set dst (ExpandV src mask));
+  effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2);
+  format %{ "vexpand_neon $dst, $src, $mask\t# KILL $tmp1, $tmp2" %}
+  ins_encode %{
+    BasicType bt = Matcher::vector_element_basic_type(this);
+    int length_in_bytes = (int) Matcher::vector_length_in_bytes(this);
+    __ vector_expand_neon($dst$$FloatRegister, $src$$FloatRegister, $mask$$FloatRegister,
+                          $tmp1$$FloatRegister, $tmp2$$FloatRegister, bt, length_in_bytes);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
+instruct vexpand_sve(vReg dst, vReg src, pRegGov pg, vReg tmp1, vReg tmp2) %{
+  predicate(UseSVE == 1 || (UseSVE == 2 && type2aelembytes(Matcher::vector_element_basic_type(n)) < 4));
+  match(Set dst (ExpandV src pg));
+  effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2);
+  format %{ "vexpand_sve $dst, $src, $pg\t# KILL $tmp1, $tmp2" %}
+  ins_encode %{
+    BasicType bt = Matcher::vector_element_basic_type(this);
+    int length_in_bytes = (int) Matcher::vector_length_in_bytes(this);
+    __ vector_expand_sve($dst$$FloatRegister, $src$$FloatRegister, $pg$$PRegister,
+                         $tmp1$$FloatRegister, $tmp2$$FloatRegister, bt, length_in_bytes);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
+instruct vexpand_sve2_SD(vReg dst, vReg src, pRegGov pg) %{
+  predicate(UseSVE == 2 && type2aelembytes(Matcher::vector_element_basic_type(n)) >= 4);
  match(Set dst (ExpandV src pg));
  effect(TEMP_DEF dst);
-  format %{ "vexpand $dst, $pg, $src" %}
+  format %{ "vexpand_sve2_SD $dst, $src, $pg" %}
  ins_encode %{
    // Example input:   src   = 1 2 3 4 5 6 7 8
    //                  pg    = 1 0 0 1 1 0 1 1
@@ -7127,7 +7151,6 @@ instruct vexpand(vReg dst, vReg src, pRegGov pg) %{
    // for TBL whose value is used to select the indexed element from src vector.

    BasicType bt = Matcher::vector_element_basic_type(this);
-    assert(UseSVE == 2 && !is_subword_type(bt), "unsupported");
    Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
    // dst = 0 0 0 0 0 0 0 0
    __ sve_dup($dst$$FloatRegister, size, 0);
@@ -206,11 +206,6 @@ source %{
          return false;
        }
        break;
-      case Op_ExpandV:
-        if (UseSVE < 2 || is_subword_type(bt)) {
-          return false;
-        }
-        break;
      case Op_VectorMaskToLong:
        if (UseSVE > 0 && vlen > 64) {
          return false;
@@ -5101,10 +5096,39 @@ instruct vcompressS(vReg dst, vReg src, pReg pg,
  ins_pipe(pipe_slow);
 %}

-instruct vexpand(vReg dst, vReg src, pRegGov pg) %{
+instruct vexpand_neon(vReg dst, vReg src, vReg mask, vReg tmp1, vReg tmp2) %{
+  predicate(UseSVE == 0);
+  match(Set dst (ExpandV src mask));
+  effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2);
+  format %{ "vexpand_neon $dst, $src, $mask\t# KILL $tmp1, $tmp2" %}
+  ins_encode %{
+    BasicType bt = Matcher::vector_element_basic_type(this);
+    int length_in_bytes = (int) Matcher::vector_length_in_bytes(this);
+    __ vector_expand_neon($dst$$FloatRegister, $src$$FloatRegister, $mask$$FloatRegister,
+                          $tmp1$$FloatRegister, $tmp2$$FloatRegister, bt, length_in_bytes);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
+instruct vexpand_sve(vReg dst, vReg src, pRegGov pg, vReg tmp1, vReg tmp2) %{
+  predicate(UseSVE == 1 || (UseSVE == 2 && type2aelembytes(Matcher::vector_element_basic_type(n)) < 4));
+  match(Set dst (ExpandV src pg));
+  effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2);
+  format %{ "vexpand_sve $dst, $src, $pg\t# KILL $tmp1, $tmp2" %}
+  ins_encode %{
+    BasicType bt = Matcher::vector_element_basic_type(this);
+    int length_in_bytes = (int) Matcher::vector_length_in_bytes(this);
+    __ vector_expand_sve($dst$$FloatRegister, $src$$FloatRegister, $pg$$PRegister,
+                         $tmp1$$FloatRegister, $tmp2$$FloatRegister, bt, length_in_bytes);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
+instruct vexpand_sve2_SD(vReg dst, vReg src, pRegGov pg) %{
+  predicate(UseSVE == 2 && type2aelembytes(Matcher::vector_element_basic_type(n)) >= 4);
  match(Set dst (ExpandV src pg));
  effect(TEMP_DEF dst);
-  format %{ "vexpand $dst, $pg, $src" %}
+  format %{ "vexpand_sve2_SD $dst, $src, $pg" %}
  ins_encode %{
    // Example input:   src   = 1 2 3 4 5 6 7 8
    //                  pg    = 1 0 0 1 1 0 1 1
@@ -5115,7 +5139,6 @@ instruct vexpand(vReg dst, vReg src, pRegGov pg) %{
    // for TBL whose value is used to select the indexed element from src vector.

    BasicType bt = Matcher::vector_element_basic_type(this);
-    assert(UseSVE == 2 && !is_subword_type(bt), "unsupported");
    Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
    // dst = 0 0 0 0 0 0 0 0
    __ sve_dup($dst$$FloatRegister, size, 0);
@@ -3814,7 +3814,11 @@ private:
    starti;
    assert(T != Q, "invalid size");
    int sh = 0;
-    if (imm8 <= 127 && imm8 >= -128) {
+    if (isFloat) {
+      assert(T != B, "invalid size");
+      assert((imm8 >> 8) == 0, "invalid immediate");
+      sh = 0;
+    } else if (imm8 <= 127 && imm8 >= -128) {
      sh = 0;
    } else if (T != B && imm8 <= 32512 && imm8 >= -32768 && (imm8 & 0xff) == 0) {
      sh = 1;
@@ -3824,7 +3828,7 @@ private:
    }
    int m = isMerge ? 1 : 0;
    f(0b00000101, 31, 24), f(T, 23, 22), f(0b01, 21, 20);
-    prf(Pg, 16), f(isFloat ? 1 : 0, 15), f(m, 14), f(sh, 13), sf(imm8, 12, 5), rf(Zd, 0);
+    prf(Pg, 16), f(isFloat ? 1 : 0, 15), f(m, 14), f(sh, 13), f(imm8 & 0xff, 12, 5), rf(Zd, 0);
  }

 public:
@@ -3834,7 +3838,7 @@ public:
  }
  // SVE copy floating-point immediate to vector elements (predicated)
  void sve_cpy(FloatRegister Zd, SIMD_RegVariant T, PRegister Pg, double d) {
-    sve_cpy(Zd, T, Pg, checked_cast<int8_t>(pack(d)), /*isMerge*/true, /*isFloat*/true);
+    sve_cpy(Zd, T, Pg, checked_cast<uint8_t>(pack(d)), /*isMerge*/true, /*isFloat*/true);
  }

  // SVE conditionally select elements from two vectors
@@ -4064,6 +4068,13 @@ public:
  INSN(sve_brkb, 0b10); // Break before first true condition
 #undef INSN

+  // SVE move prefix (unpredicated)
+  void sve_movprfx(FloatRegister Zd, FloatRegister Zn) {
+    starti;
+    f(0b00000100, 31, 24), f(0b00, 23, 22), f(0b1, 21), f(0b00000, 20, 16);
+    f(0b101111, 15, 10), rf(Zn, 5), rf(Zd, 0);
+  }
+
 // Element count and increment scalar (SVE)
 #define INSN(NAME, TYPE)                                                             \
  void NAME(Register Xdn, unsigned imm4 = 1, int pattern = 0b11111) {                \
@@ -26,7 +26,7 @@
 #ifndef CPU_AARCH64_ASSEMBLER_AARCH64_INLINE_HPP
 #define CPU_AARCH64_ASSEMBLER_AARCH64_INLINE_HPP

-#include "asm/assembler.inline.hpp"
+#include "asm/assembler.hpp"
 #include "asm/codeBuffer.hpp"
 #include "code/codeCache.hpp"

@@ -2585,11 +2585,6 @@ void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
 }


-void LIR_Assembler::emit_delay(LIR_OpDelay*) {
-  Unimplemented();
-}
-
-
 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
  __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
 }
@@ -2771,3 +2771,90 @@ void C2_MacroAssembler::select_from_two_vectors(FloatRegister dst, FloatRegister
    select_from_two_vectors_neon(dst, src1, src2, dst, tmp, vector_length_in_bytes);
  }
 }
+
+// Vector expand implementation. Elements from the src vector are expanded into
+// the dst vector under the control of the vector mask.
+// Since there are no native instructions directly corresponding to expand before
+// SVE2p2, the following implementations mainly leverages the TBL instruction to
+// implement expand. To compute the index input for TBL, the prefix sum algorithm
+// (https://en.wikipedia.org/wiki/Prefix_sum) is used. The same algorithm is used
+// for NEON and SVE, but with different instructions where appropriate.
+
+// Vector expand implementation for NEON.
+//
+// An example of 128-bit Byte vector:
+//   Data direction: high <== low
+//   Input:
+//         src   = g  f  e  d  c  b  a  9  8  7  6  5  4  3  2  1
+//         mask  = 0  0 -1 -1  0  0 -1 -1  0  0 -1 -1  0  0 -1 -1
+//   Expected result:
+//         dst   = 0  0  8  7  0  0  6  5  0  0  4  3  0  0  2  1
+void C2_MacroAssembler::vector_expand_neon(FloatRegister dst, FloatRegister src, FloatRegister mask,
+                                           FloatRegister tmp1, FloatRegister tmp2, BasicType bt,
+                                           int vector_length_in_bytes) {
+  assert(vector_length_in_bytes <= 16, "the vector length in bytes for NEON must be <= 16");
+  assert_different_registers(dst, src, mask, tmp1, tmp2);
+  // Since the TBL instruction only supports byte table, we need to
+  // compute indices in byte type for all types.
+  SIMD_Arrangement size = vector_length_in_bytes == 16 ? T16B : T8B;
+  // tmp1 =  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0
+  dup(tmp1, size, zr);
+  // dst  =  0  0  1  1  0  0  1  1  0  0  1  1  0  0  1  1
+  negr(dst, size, mask);
+  // Calculate vector index for TBL with prefix sum algorithm.
+  // dst  =  8  8  8  7  6  6  6  5  4  4  4  3  2  2  2  1
+  for (int i = 1; i < vector_length_in_bytes; i <<= 1) {
+    ext(tmp2, size, tmp1, dst, vector_length_in_bytes - i);
+    addv(dst, size, tmp2, dst);
+  }
+  // tmp2 =  0  0 -1 -1  0  0 -1 -1  0  0 -1 -1  0  0 -1 -1
+  orr(tmp2, size, mask, mask);
+  // tmp2 =  0  0  8  7  0  0  6  5  0  0  4  3  0  0  2  1
+  bsl(tmp2, size, dst, tmp1);
+  // tmp1 =  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1
+  movi(tmp1, size, 1);
+  // dst  = -1 -1  7  6 -1 -1  5  4 -1 -1  3  2 -1 -1  1  0
+  subv(dst, size, tmp2, tmp1);
+  // dst  =  0  0  8  7  0  0  6  5  0  0  4  3  0  0  2  1
+  tbl(dst, size, src, 1, dst);
+}
+
+// Vector expand implementation for SVE.
+//
+// An example of 128-bit Short vector:
+//   Data direction: high <== low
+//   Input:
+//         src   = gf ed cb a9 87 65 43 21
+//         pg    = 00 01 00 01 00 01 00 01
+//   Expected result:
+//         dst   = 00 87 00 65 00 43 00 21
+void C2_MacroAssembler::vector_expand_sve(FloatRegister dst, FloatRegister src, PRegister pg,
+                                          FloatRegister tmp1, FloatRegister tmp2, BasicType bt,
+                                          int vector_length_in_bytes) {
+  assert(UseSVE > 0, "expand implementation only for SVE");
+  assert_different_registers(dst, src, tmp1, tmp2);
+  SIMD_RegVariant size = elemType_to_regVariant(bt);
+
+  // tmp1 = 00 00 00 00 00 00 00 00
+  sve_dup(tmp1, size, 0);
+  sve_movprfx(tmp2, tmp1);
+  // tmp2 = 00 01 00 01 00 01 00 01
+  sve_cpy(tmp2, size, pg, 1, true);
+  // Calculate vector index for TBL with prefix sum algorithm.
+  // tmp2 = 04 04 03 03 02 02 01 01
+  for (int i = type2aelembytes(bt); i < vector_length_in_bytes; i <<= 1) {
+    sve_movprfx(dst, tmp1);
+    // The EXT instruction operates on the full-width sve register. The correct
+    // index calculation method is:
+    // vector_length_in_bytes - i + MaxVectorSize - vector_length_in_bytes =>
+    // MaxVectorSize - i.
+    sve_ext(dst, tmp2, MaxVectorSize - i);
+    sve_add(tmp2, size, dst, tmp2);
+  }
+  // dst  = 00 04 00 03 00 02 00 01
+  sve_sel(dst, size, pg, tmp2, tmp1);
+  // dst  = -1 03 -1 02 -1 01 -1 00
+  sve_sub(dst, size, 1);
+  // dst  = 00 87 00 65 00 43 00 21
+  sve_tbl(dst, size, src, dst);
+}
@@ -204,4 +204,10 @@
                               FloatRegister index, FloatRegister tmp, BasicType bt,
                               unsigned vector_length_in_bytes);

+  void vector_expand_neon(FloatRegister dst, FloatRegister src, FloatRegister mask,
+                          FloatRegister tmp1, FloatRegister tmp2, BasicType bt,
+                          int vector_length_in_bytes);
+  void vector_expand_sve(FloatRegister dst, FloatRegister src, PRegister pg,
+                         FloatRegister tmp1, FloatRegister tmp2, BasicType bt,
+                         int vector_length_in_bytes);
 #endif // CPU_AARCH64_C2_MACROASSEMBLER_AARCH64_HPP
@@ -275,7 +275,7 @@ address BarrierSetAssembler::patching_epoch_addr() {
 }

 void BarrierSetAssembler::increment_patching_epoch() {
-  Atomic::inc(&_patching_epoch);
+  AtomicAccess::inc(&_patching_epoch);
 }

 void BarrierSetAssembler::clear_patching_epoch() {
@@ -112,11 +112,25 @@ public:
  }

  int get_value() {
-    return Atomic::load_acquire(guard_addr());
+    return AtomicAccess::load_acquire(guard_addr());
  }

-  void set_value(int value) {
-    Atomic::release_store(guard_addr(), value);
+  void set_value(int value, int bit_mask) {
+    if (bit_mask == ~0) {
+      AtomicAccess::release_store(guard_addr(), value);
+      return;
+    }
+    assert((value & ~bit_mask) == 0, "trying to set bits outside the mask");
+    value &= bit_mask;
+    int old_value = AtomicAccess::load(guard_addr());
+    while (true) {
+      // Only bits in the mask are changed
+      int new_value = value | (old_value & ~bit_mask);
+      if (new_value == old_value) break;
+      int v = AtomicAccess::cmpxchg(guard_addr(), old_value, new_value, memory_order_release);
+      if (v == old_value) break;
+      old_value = v;
+    }
  }

  bool check_barrier(err_msg& msg) const;
@@ -179,7 +193,7 @@ void BarrierSetNMethod::deoptimize(nmethod* nm, address* return_address_ptr) {
  new_frame->pc = SharedRuntime::get_handle_wrong_method_stub();
 }

-void BarrierSetNMethod::set_guard_value(nmethod* nm, int value) {
+void BarrierSetNMethod::set_guard_value(nmethod* nm, int value, int bit_mask) {
  if (!supports_entry_barrier(nm)) {
    return;
  }
@@ -196,7 +210,7 @@ void BarrierSetNMethod::set_guard_value(nmethod* nm, int value) {
  }

  NativeNMethodBarrier barrier(nm);
-  barrier.set_value(value);
+  barrier.set_value(value, bit_mask);
 }

 int BarrierSetNMethod::guard_value(nmethod* nm) {
@@ -1623,7 +1623,7 @@ public:
                    FloatRegister p, FloatRegister z, FloatRegister t1);
  void ghash_reduce_wide(int index, FloatRegister result, FloatRegister lo, FloatRegister hi,
                    FloatRegister p, FloatRegister z, FloatRegister t1);
-  void ghash_processBlocks_wide(address p, Register state, Register subkeyH,
+  void ghash_processBlocks_wide(Label& p, Register state, Register subkeyH,
                                Register data, Register blocks, int unrolls);


@@ -507,7 +507,7 @@ void MacroAssembler::ghash_modmul(FloatRegister result,
 //
 // Clobbers all vector registers.
 //
-void MacroAssembler::ghash_processBlocks_wide(address field_polynomial, Register state,
+void MacroAssembler::ghash_processBlocks_wide(Label& field_polynomial, Register state,
                                              Register subkeyH,
                                              Register data, Register blocks, int unrolls) {
  int register_stride = 7;
@@ -531,7 +531,10 @@ void MacroAssembler::ghash_processBlocks_wide(address field_polynomial, Register
  FloatRegister p = v31;
  eor(vzr, T16B, vzr, vzr); // zero register

-  ldrq(p, field_polynomial);    // The field polynomial
+  // load polynomial via label which must identify local data in the
+  // same code stub
+  adr(rscratch1, field_polynomial);
+  ldrq(p, rscratch1);    // The field polynomial

  ldrq(v0, Address(state));
  ldrq(Hprime, Address(subkeyH));
@@ -42,7 +42,7 @@
 #include "prims/methodHandles.hpp"
 #include "prims/upcallLinker.hpp"
 #include "runtime/arguments.hpp"
-#include "runtime/atomic.hpp"
+#include "runtime/atomicAccess.hpp"
 #include "runtime/continuation.hpp"
 #include "runtime/continuationEntry.inline.hpp"
 #include "runtime/frame.inline.hpp"
@@ -802,7 +802,7 @@ class StubGenerator: public StubCodeGenerator {
  //
  // s and d are adjusted to point to the remaining words to copy
  //
-  void generate_copy_longs(StubId stub_id, DecoratorSet decorators, Label &start, Register s, Register d, Register count) {
+  address generate_copy_longs(StubId stub_id, DecoratorSet decorators, Register s, Register d, Register count) {
    BasicType type;
    copy_direction direction;

@@ -854,7 +854,7 @@ class StubGenerator: public StubCodeGenerator {

    StubCodeMark mark(this, stub_id);

-    __ bind(start);
+    address start = __ pc();

    Label unaligned_copy_long;
    if (AvoidUnalignedAccesses) {
@@ -894,9 +894,9 @@ class StubGenerator: public StubCodeGenerator {
    int prefetch = PrefetchCopyIntervalInBytes;
    bool use_stride = false;
    if (direction == copy_backwards) {
-       use_stride = prefetch > 256;
-       prefetch = -prefetch;
-       if (use_stride) __ mov(stride, prefetch);
+      use_stride = prefetch > 256;
+      prefetch = -prefetch;
+      if (use_stride) __ mov(stride, prefetch);
    }

    __ bind(again);
@@ -1026,9 +1026,9 @@ class StubGenerator: public StubCodeGenerator {
      int prefetch = PrefetchCopyIntervalInBytes;
      bool use_stride = false;
      if (direction == copy_backwards) {
-         use_stride = prefetch > 256;
-         prefetch = -prefetch;
-         if (use_stride) __ mov(stride, prefetch);
+        use_stride = prefetch > 256;
+        prefetch = -prefetch;
+        if (use_stride) __ mov(stride, prefetch);
      }

      __ bind(again);
@@ -1037,15 +1037,15 @@ class StubGenerator: public StubCodeGenerator {
        __ prfm(use_stride ? Address(s, stride) : Address(s, prefetch), PLDL1KEEP);

      if (direction == copy_forwards) {
-       // allowing for the offset of -8 the store instructions place
-       // registers into the target 64 bit block at the following
-       // offsets
-       //
-       // t0 at offset 0
-       // t1 at offset 8,  t2 at offset 16
-       // t3 at offset 24, t4 at offset 32
-       // t5 at offset 40, t6 at offset 48
-       // t7 at offset 56
+        // allowing for the offset of -8 the store instructions place
+        // registers into the target 64 bit block at the following
+        // offsets
+        //
+        // t0 at offset 0
+        // t1 at offset 8,  t2 at offset 16
+        // t3 at offset 24, t4 at offset 32
+        // t5 at offset 40, t6 at offset 48
+        // t7 at offset 56

        bs.copy_store_at_8(Address(d, 1 * unit), t0);
        bs.copy_store_at_16(Address(d, 2 * unit), t1, t2);
@@ -1057,18 +1057,18 @@ class StubGenerator: public StubCodeGenerator {
        bs.copy_store_at_8(Address(__ pre(d, 8 * unit)), t7);
        bs.copy_load_at_16(t6, t7, Address(__ pre(s, 8 * unit)));
      } else {
-       // d was not offset when we started so the registers are
-       // written into the 64 bit block preceding d with the following
-       // offsets
-       //
-       // t1 at offset -8
-       // t3 at offset -24, t0 at offset -16
-       // t5 at offset -48, t2 at offset -32
-       // t7 at offset -56, t4 at offset -48
-       //                   t6 at offset -64
-       //
-       // note that this matches the offsets previously noted for the
-       // loads
+        // d was not offset when we started so the registers are
+        // written into the 64 bit block preceding d with the following
+        // offsets
+        //
+        // t1 at offset -8
+        // t3 at offset -24, t0 at offset -16
+        // t5 at offset -48, t2 at offset -32
+        // t7 at offset -56, t4 at offset -48
+        //                   t6 at offset -64
+        //
+        // note that this matches the offsets previously noted for the
+        // loads

        bs.copy_store_at_8(Address(d, 1 * unit), t1);
        bs.copy_store_at_16(Address(d, 3 * unit), t3, t0);
@@ -1109,10 +1109,10 @@ class StubGenerator: public StubCodeGenerator {
      {
        Label L1, L2;
        __ tbz(count, exact_log2(4), L1);
-       // this is the same as above but copying only 4 longs hence
-       // with only one intervening stp between the str instructions
-       // but note that the offsets and registers still follow the
-       // same pattern
+        // this is the same as above but copying only 4 longs hence
+        // with only one intervening stp between the str instructions
+        // but note that the offsets and registers still follow the
+        // same pattern
        bs.copy_load_at_16(t0, t1, Address(s, 2 * unit));
        bs.copy_load_at_16(t2, t3, Address(__ pre(s, 4 * unit)));
        if (direction == copy_forwards) {
@@ -1127,10 +1127,10 @@ class StubGenerator: public StubCodeGenerator {
        __ bind(L1);

        __ tbz(count, 1, L2);
-       // this is the same as above but copying only 2 longs hence
-       // there is no intervening stp between the str instructions
-       // but note that the offset and register patterns are still
-       // the same
+        // this is the same as above but copying only 2 longs hence
+        // there is no intervening stp between the str instructions
+        // but note that the offset and register patterns are still
+        // the same
        bs.copy_load_at_16(t0, t1, Address(__ pre(s, 2 * unit)));
        if (direction == copy_forwards) {
          bs.copy_store_at_8(Address(d, 1 * unit), t0);
@@ -1141,18 +1141,20 @@ class StubGenerator: public StubCodeGenerator {
        }
        __ bind(L2);

-       // for forwards copy we need to re-adjust the offsets we
-       // applied so that s and d are follow the last words written
+        // for forwards copy we need to re-adjust the offsets we
+        // applied so that s and d are follow the last words written

-       if (direction == copy_forwards) {
-         __ add(s, s, 16);
-         __ add(d, d, 8);
-       }
+        if (direction == copy_forwards) {
+          __ add(s, s, 16);
+          __ add(d, d, 8);
+        }

      }

      __ ret(lr);
-      }
+    }
+
+    return start;
  }

  // Small copy: less than 16 bytes.
@@ -1206,10 +1208,6 @@ class StubGenerator: public StubCodeGenerator {
    }
  }

-  Label copy_f, copy_b;
-  Label copy_obj_f, copy_obj_b;
-  Label copy_obj_uninit_f, copy_obj_uninit_b;
-
  // All-singing all-dancing memory copy.
  //
  // Copy count units of memory from s to d.  The size of a unit is
@@ -1447,19 +1445,19 @@ class StubGenerator: public StubCodeGenerator {
    }
    if (direction == copy_forwards) {
      if (type != T_OBJECT) {
-        __ bl(copy_f);
+        __ bl(StubRoutines::aarch64::copy_byte_f());
      } else if ((decorators & IS_DEST_UNINITIALIZED) != 0) {
-        __ bl(copy_obj_uninit_f);
+        __ bl(StubRoutines::aarch64::copy_oop_uninit_f());
      } else {
-        __ bl(copy_obj_f);
+        __ bl(StubRoutines::aarch64::copy_oop_f());
      }
    } else {
      if (type != T_OBJECT) {
-        __ bl(copy_b);
+        __ bl(StubRoutines::aarch64::copy_byte_b());
      } else if ((decorators & IS_DEST_UNINITIALIZED) != 0) {
-        __ bl(copy_obj_uninit_b);
+        __ bl(StubRoutines::aarch64::copy_oop_uninit_b());
      } else {
-        __ bl(copy_obj_b);
+        __ bl(StubRoutines::aarch64::copy_oop_b());
      }
    }

@@ -1522,11 +1520,11 @@ class StubGenerator: public StubCodeGenerator {
  // the hardware handle it.  The two dwords within qwords that span
  // cache line boundaries will still be loaded and stored atomically.
  //
-  // Side Effects: entry is set to the (post push) entry point so it
-  //               can be used by the corresponding conjoint copy
-  //               method
+  // Side Effects: nopush_entry is set to the (post push) entry point
+  //               so it can be used by the corresponding conjoint
+  //               copy method
  //
-  address generate_disjoint_copy(StubId stub_id, address *entry) {
+  address generate_disjoint_copy(StubId stub_id, address *nopush_entry) {
    Register s = c_rarg0, d = c_rarg1, count = c_rarg2;
    RegSet saved_reg = RegSet::of(s, d, count);
    int size;
@@ -1615,8 +1613,8 @@ class StubGenerator: public StubCodeGenerator {
    address start = __ pc();
    __ enter();

-    if (entry != nullptr) {
-      *entry = __ pc();
+    if (nopush_entry != nullptr) {
+      *nopush_entry = __ pc();
      // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
      BLOCK_COMMENT("Entry:");
    }
@@ -1679,10 +1677,10 @@ class StubGenerator: public StubCodeGenerator {
  // cache line boundaries will still be loaded and stored atomically.
  //
  // Side Effects:
-  //   entry is set to the no-overlap entry point so it can be used by
-  //   some other conjoint copy method
+  //   nopush_entry is set to the no-overlap entry point so it can be
+  //   used by some other conjoint copy method
  //
-  address generate_conjoint_copy(StubId stub_id, address nooverlap_target, address *entry) {
+  address generate_conjoint_copy(StubId stub_id, address nooverlap_target, address *nopush_entry) {
    Register s = c_rarg0, d = c_rarg1, count = c_rarg2;
    RegSet saved_regs = RegSet::of(s, d, count);
    int size;
@@ -1769,16 +1767,19 @@ class StubGenerator: public StubCodeGenerator {
    address start = __ pc();
    __ enter();

-    if (entry != nullptr) {
-      *entry = __ pc();
+    if (nopush_entry != nullptr) {
+      *nopush_entry = __ pc();
      // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
      BLOCK_COMMENT("Entry:");
    }

    // use fwd copy when (d-s) above_equal (count*size)
+    Label L_overlapping;
    __ sub(rscratch1, d, s);
    __ cmp(rscratch1, count, Assembler::LSL, exact_log2(size));
-    __ br(Assembler::HS, nooverlap_target);
+    __ br(Assembler::LO, L_overlapping);
+    __ b(RuntimeAddress(nooverlap_target));
+    __ bind(L_overlapping);

    DecoratorSet decorators = IN_HEAP | IS_ARRAY;
    if (dest_uninitialized) {
@@ -1850,7 +1851,7 @@ class StubGenerator: public StubCodeGenerator {
  //    r0 ==  0  -  success
  //    r0 == -1^K - failure, where K is partial transfer count
  //
-  address generate_checkcast_copy(StubId stub_id, address *entry) {
+  address generate_checkcast_copy(StubId stub_id, address *nopush_entry) {
    bool dest_uninitialized;
    switch (stub_id) {
    case StubId::stubgen_checkcast_arraycopy_id:
@@ -1911,8 +1912,8 @@ class StubGenerator: public StubCodeGenerator {
 #endif //ASSERT

    // Caller of this entry point must set up the argument registers.
-    if (entry != nullptr) {
-      *entry = __ pc();
+    if (nopush_entry != nullptr) {
+      *nopush_entry = __ pc();
      BLOCK_COMMENT("Entry:");
    }

@@ -2724,13 +2725,21 @@ class StubGenerator: public StubCodeGenerator {
  }

  void generate_arraycopy_stubs() {
-    address entry;
-    address entry_jbyte_arraycopy;
-    address entry_jshort_arraycopy;
-    address entry_jint_arraycopy;
-    address entry_oop_arraycopy;
-    address entry_jlong_arraycopy;
-    address entry_checkcast_arraycopy;
+    // Some copy stubs publish a normal entry and then a 2nd 'fallback'
+    // entry immediately following their stack push. This can be used
+    // as a post-push branch target for compatible stubs when they
+    // identify a special case that can be handled by the fallback
+    // stub e.g a disjoint copy stub may be use as a special case
+    // fallback for its compatible conjoint copy stub.
+    //
+    // A no push entry is always returned in the following local and
+    // then published by assigning to the appropriate entry field in
+    // class StubRoutines. The entry value is then passed to the
+    // generator for the compatible stub. That means the entry must be
+    // listed when saving to/restoring from the AOT cache, ensuring
+    // that the inter-stub jumps are noted at AOT-cache save and
+    // relocated at AOT cache load.
+    address nopush_entry;

    // generate the common exit first so later stubs can rely on it if
    // they want an UnsafeMemoryAccess exit non-local to the stub
@@ -2738,83 +2747,123 @@ class StubGenerator: public StubCodeGenerator {
    // register the stub as the default exit with class UnsafeMemoryAccess
    UnsafeMemoryAccess::set_common_exit_stub_pc(StubRoutines::_unsafecopy_common_exit);

-    generate_copy_longs(StubId::stubgen_copy_byte_f_id, IN_HEAP | IS_ARRAY, copy_f, r0, r1, r15);
-    generate_copy_longs(StubId::stubgen_copy_byte_b_id, IN_HEAP | IS_ARRAY, copy_b, r0, r1, r15);
+    // generate and publish arch64-specific bulk copy routines first
+    // so we can call them from other copy stubs
+    StubRoutines::aarch64::_copy_byte_f = generate_copy_longs(StubId::stubgen_copy_byte_f_id, IN_HEAP | IS_ARRAY, r0, r1, r15);
+    StubRoutines::aarch64::_copy_byte_b = generate_copy_longs(StubId::stubgen_copy_byte_b_id, IN_HEAP | IS_ARRAY, r0, r1, r15);

-    generate_copy_longs(StubId::stubgen_copy_oop_f_id, IN_HEAP | IS_ARRAY, copy_obj_f, r0, r1, r15);
-    generate_copy_longs(StubId::stubgen_copy_oop_b_id, IN_HEAP | IS_ARRAY, copy_obj_b, r0, r1, r15);
+    StubRoutines::aarch64::_copy_oop_f = generate_copy_longs(StubId::stubgen_copy_oop_f_id, IN_HEAP | IS_ARRAY, r0, r1, r15);
+    StubRoutines::aarch64::_copy_oop_b = generate_copy_longs(StubId::stubgen_copy_oop_b_id, IN_HEAP | IS_ARRAY, r0, r1, r15);

-    generate_copy_longs(StubId::stubgen_copy_oop_uninit_f_id, IN_HEAP | IS_ARRAY | IS_DEST_UNINITIALIZED, copy_obj_uninit_f, r0, r1, r15);
-    generate_copy_longs(StubId::stubgen_copy_oop_uninit_b_id, IN_HEAP | IS_ARRAY | IS_DEST_UNINITIALIZED, copy_obj_uninit_b, r0, r1, r15);
+    StubRoutines::aarch64::_copy_oop_uninit_f = generate_copy_longs(StubId::stubgen_copy_oop_uninit_f_id, IN_HEAP | IS_ARRAY | IS_DEST_UNINITIALIZED, r0, r1, r15);
+    StubRoutines::aarch64::_copy_oop_uninit_b = generate_copy_longs(StubId::stubgen_copy_oop_uninit_b_id, IN_HEAP | IS_ARRAY | IS_DEST_UNINITIALIZED, r0, r1, r15);

    StubRoutines::aarch64::_zero_blocks = generate_zero_blocks();

    //*** jbyte
    // Always need aligned and unaligned versions
-    StubRoutines::_jbyte_disjoint_arraycopy         = generate_disjoint_copy(StubId::stubgen_jbyte_disjoint_arraycopy_id, &entry);
-    StubRoutines::_jbyte_arraycopy                  = generate_conjoint_copy(StubId::stubgen_jbyte_arraycopy_id, entry, &entry_jbyte_arraycopy);
-    StubRoutines::_arrayof_jbyte_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_arrayof_jbyte_disjoint_arraycopy_id, &entry);
-    StubRoutines::_arrayof_jbyte_arraycopy          = generate_conjoint_copy(StubId::stubgen_arrayof_jbyte_arraycopy_id, entry, nullptr);
+    StubRoutines::_jbyte_disjoint_arraycopy         = generate_disjoint_copy(StubId::stubgen_jbyte_disjoint_arraycopy_id, &nopush_entry);
+    // disjoint nopush entry is needed by conjoint copy
+    StubRoutines::_jbyte_disjoint_arraycopy_nopush  = nopush_entry;
+    StubRoutines::_jbyte_arraycopy                  = generate_conjoint_copy(StubId::stubgen_jbyte_arraycopy_id, StubRoutines::_jbyte_disjoint_arraycopy_nopush, &nopush_entry);
+    // conjoint nopush entry is needed by generic/unsafe copy
+    StubRoutines::_jbyte_arraycopy_nopush = nopush_entry;
+    StubRoutines::_arrayof_jbyte_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_arrayof_jbyte_disjoint_arraycopy_id, &nopush_entry);
+    // disjoint arrayof nopush entry is needed by conjoint copy
+    StubRoutines::_arrayof_jbyte_disjoint_arraycopy_nopush  = nopush_entry;
+    StubRoutines::_arrayof_jbyte_arraycopy          = generate_conjoint_copy(StubId::stubgen_arrayof_jbyte_arraycopy_id, StubRoutines::_arrayof_jbyte_disjoint_arraycopy_nopush, nullptr);

    //*** jshort
    // Always need aligned and unaligned versions
-    StubRoutines::_jshort_disjoint_arraycopy         = generate_disjoint_copy(StubId::stubgen_jshort_disjoint_arraycopy_id, &entry);
-    StubRoutines::_jshort_arraycopy                  = generate_conjoint_copy(StubId::stubgen_jshort_arraycopy_id, entry, &entry_jshort_arraycopy);
-    StubRoutines::_arrayof_jshort_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_arrayof_jshort_disjoint_arraycopy_id, &entry);
-    StubRoutines::_arrayof_jshort_arraycopy          = generate_conjoint_copy(StubId::stubgen_arrayof_jshort_arraycopy_id, entry, nullptr);
+    StubRoutines::_jshort_disjoint_arraycopy         = generate_disjoint_copy(StubId::stubgen_jshort_disjoint_arraycopy_id, &nopush_entry);
+    // disjoint nopush entry is needed by conjoint copy
+    StubRoutines::_jshort_disjoint_arraycopy_nopush  = nopush_entry;
+    StubRoutines::_jshort_arraycopy                  = generate_conjoint_copy(StubId::stubgen_jshort_arraycopy_id, StubRoutines::_jshort_disjoint_arraycopy_nopush, &nopush_entry);
+    // conjoint nopush entry is used by generic/unsafe copy
+    StubRoutines::_jshort_arraycopy_nopush = nopush_entry;
+    StubRoutines::_arrayof_jshort_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_arrayof_jshort_disjoint_arraycopy_id, &nopush_entry);
+    // disjoint arrayof nopush entry is needed by conjoint copy
+    StubRoutines::_arrayof_jshort_disjoint_arraycopy_nopush = nopush_entry;
+    StubRoutines::_arrayof_jshort_arraycopy          = generate_conjoint_copy(StubId::stubgen_arrayof_jshort_arraycopy_id, StubRoutines::_arrayof_jshort_disjoint_arraycopy_nopush, nullptr);

    //*** jint
    // Aligned versions
-    StubRoutines::_arrayof_jint_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_arrayof_jint_disjoint_arraycopy_id, &entry);
-    StubRoutines::_arrayof_jint_arraycopy          = generate_conjoint_copy(StubId::stubgen_arrayof_jint_arraycopy_id, entry, &entry_jint_arraycopy);
+    StubRoutines::_arrayof_jint_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_arrayof_jint_disjoint_arraycopy_id, &nopush_entry);
+    // disjoint arrayof nopush entry is needed by conjoint copy
+    StubRoutines::_arrayof_jint_disjoint_arraycopy_nopush = nopush_entry;
+    StubRoutines::_arrayof_jint_arraycopy          = generate_conjoint_copy(StubId::stubgen_arrayof_jint_arraycopy_id, StubRoutines::_arrayof_jint_disjoint_arraycopy_nopush, nullptr);
    // In 64 bit we need both aligned and unaligned versions of jint arraycopy.
-    // entry_jint_arraycopy always points to the unaligned version
-    StubRoutines::_jint_disjoint_arraycopy         = generate_disjoint_copy(StubId::stubgen_jint_disjoint_arraycopy_id, &entry);
-    StubRoutines::_jint_arraycopy                  = generate_conjoint_copy(StubId::stubgen_jint_arraycopy_id, entry, &entry_jint_arraycopy);
+    // jint_arraycopy_nopush always points to the unaligned version
+    StubRoutines::_jint_disjoint_arraycopy         = generate_disjoint_copy(StubId::stubgen_jint_disjoint_arraycopy_id, &nopush_entry);
+    // disjoint nopush entry is needed by conjoint copy
+    StubRoutines::_jint_disjoint_arraycopy_nopush  = nopush_entry;
+    StubRoutines::_jint_arraycopy                  = generate_conjoint_copy(StubId::stubgen_jint_arraycopy_id, StubRoutines::_jint_disjoint_arraycopy_nopush, &nopush_entry);
+    // conjoint nopush entry is needed by generic/unsafe copy
+    StubRoutines::_jint_arraycopy_nopush = nopush_entry;

    //*** jlong
    // It is always aligned
-    StubRoutines::_arrayof_jlong_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_arrayof_jlong_disjoint_arraycopy_id, &entry);
-    StubRoutines::_arrayof_jlong_arraycopy          = generate_conjoint_copy(StubId::stubgen_arrayof_jlong_arraycopy_id, entry, &entry_jlong_arraycopy);
+    StubRoutines::_arrayof_jlong_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_arrayof_jlong_disjoint_arraycopy_id, &nopush_entry);
+    // disjoint arrayof nopush entry is needed by conjoint copy
+    StubRoutines::_arrayof_jlong_disjoint_arraycopy_nopush = nopush_entry;
+    StubRoutines::_arrayof_jlong_arraycopy          = generate_conjoint_copy(StubId::stubgen_arrayof_jlong_arraycopy_id, StubRoutines::_arrayof_jlong_disjoint_arraycopy_nopush, &nopush_entry);
+    // conjoint nopush entry is needed by generic/unsafe copy
+    StubRoutines::_jlong_arraycopy_nopush = nopush_entry;
+    // disjoint normal/nopush and conjoint normal entries are not
+    // generated since the arrayof versions are the same
    StubRoutines::_jlong_disjoint_arraycopy         = StubRoutines::_arrayof_jlong_disjoint_arraycopy;
+    StubRoutines::_jlong_disjoint_arraycopy_nopush = StubRoutines::_arrayof_jlong_disjoint_arraycopy_nopush;
    StubRoutines::_jlong_arraycopy                  = StubRoutines::_arrayof_jlong_arraycopy;

    //*** oops
    {
-      // With compressed oops we need unaligned versions; notice that
-      // we overwrite entry_oop_arraycopy.
-      bool aligned = !UseCompressedOops;
-
      StubRoutines::_arrayof_oop_disjoint_arraycopy
-        = generate_disjoint_copy(StubId::stubgen_arrayof_oop_disjoint_arraycopy_id, &entry);
+        = generate_disjoint_copy(StubId::stubgen_arrayof_oop_disjoint_arraycopy_id, &nopush_entry);
+      // disjoint arrayof nopush entry is needed by conjoint copy
+      StubRoutines::_arrayof_oop_disjoint_arraycopy_nopush = nopush_entry;
      StubRoutines::_arrayof_oop_arraycopy
-        = generate_conjoint_copy(StubId::stubgen_arrayof_oop_arraycopy_id, entry, &entry_oop_arraycopy);
+        = generate_conjoint_copy(StubId::stubgen_arrayof_oop_arraycopy_id, StubRoutines::_arrayof_oop_disjoint_arraycopy_nopush, &nopush_entry);
+      // conjoint arrayof nopush entry is needed by generic/unsafe copy
+      StubRoutines::_oop_arraycopy_nopush = nopush_entry;
      // Aligned versions without pre-barriers
      StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit
-        = generate_disjoint_copy(StubId::stubgen_arrayof_oop_disjoint_arraycopy_uninit_id, &entry);
+        = generate_disjoint_copy(StubId::stubgen_arrayof_oop_disjoint_arraycopy_uninit_id, &nopush_entry);
+      // disjoint arrayof+uninit nopush entry is needed by conjoint copy
+      StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit_nopush = nopush_entry;
+      // note that we don't need a returned nopush entry because the
+      // generic/unsafe copy does not cater for uninit arrays.
      StubRoutines::_arrayof_oop_arraycopy_uninit
-        = generate_conjoint_copy(StubId::stubgen_arrayof_oop_arraycopy_uninit_id, entry, nullptr);
+        = generate_conjoint_copy(StubId::stubgen_arrayof_oop_arraycopy_uninit_id, StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit_nopush, nullptr);
    }

+    // for oop copies reuse arrayof entries for non-arrayof cases
    StubRoutines::_oop_disjoint_arraycopy            = StubRoutines::_arrayof_oop_disjoint_arraycopy;
+    StubRoutines::_oop_disjoint_arraycopy_nopush = StubRoutines::_arrayof_oop_disjoint_arraycopy_nopush;
    StubRoutines::_oop_arraycopy                     = StubRoutines::_arrayof_oop_arraycopy;
    StubRoutines::_oop_disjoint_arraycopy_uninit     = StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit;
+    StubRoutines::_oop_disjoint_arraycopy_uninit_nopush = StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit_nopush;
    StubRoutines::_oop_arraycopy_uninit              = StubRoutines::_arrayof_oop_arraycopy_uninit;

-    StubRoutines::_checkcast_arraycopy        = generate_checkcast_copy(StubId::stubgen_checkcast_arraycopy_id, &entry_checkcast_arraycopy);
+    StubRoutines::_checkcast_arraycopy        = generate_checkcast_copy(StubId::stubgen_checkcast_arraycopy_id, &nopush_entry);
+    // checkcast nopush entry is needed by generic copy
+    StubRoutines::_checkcast_arraycopy_nopush = nopush_entry;
+    // note that we don't need a returned nopush entry because the
+    // generic copy does not cater for uninit arrays.
    StubRoutines::_checkcast_arraycopy_uninit = generate_checkcast_copy(StubId::stubgen_checkcast_arraycopy_uninit_id, nullptr);

-    StubRoutines::_unsafe_arraycopy    = generate_unsafe_copy(entry_jbyte_arraycopy,
-                                                              entry_jshort_arraycopy,
-                                                              entry_jint_arraycopy,
-                                                              entry_jlong_arraycopy);
+    // unsafe arraycopy may fallback on conjoint stubs
+    StubRoutines::_unsafe_arraycopy    = generate_unsafe_copy(StubRoutines::_jbyte_arraycopy_nopush,
+                                                              StubRoutines::_jshort_arraycopy_nopush,
+                                                              StubRoutines::_jint_arraycopy_nopush,
+                                                              StubRoutines::_jlong_arraycopy_nopush);

-    StubRoutines::_generic_arraycopy   = generate_generic_copy(entry_jbyte_arraycopy,
-                                                               entry_jshort_arraycopy,
-                                                               entry_jint_arraycopy,
-                                                               entry_oop_arraycopy,
-                                                               entry_jlong_arraycopy,
-                                                               entry_checkcast_arraycopy);
+    // generic arraycopy may fallback on conjoint stubs
+    StubRoutines::_generic_arraycopy   = generate_generic_copy(StubRoutines::_jbyte_arraycopy_nopush,
+                                                               StubRoutines::_jshort_arraycopy_nopush,
+                                                               StubRoutines::_jint_arraycopy_nopush,
+                                                               StubRoutines::_oop_arraycopy_nopush,
+                                                               StubRoutines::_jlong_arraycopy_nopush,
+                                                               StubRoutines::_checkcast_arraycopy_nopush);

    StubRoutines::_jbyte_fill = generate_fill(StubId::stubgen_jbyte_fill_id);
    StubRoutines::_jshort_fill = generate_fill(StubId::stubgen_jshort_fill_id);
@@ -3402,14 +3451,9 @@ class StubGenerator: public StubCodeGenerator {
  // counter = c_rarg7 - 16 bytes of CTR
  // return - number of processed bytes
  address generate_galoisCounterMode_AESCrypt() {
-    address ghash_polynomial = __ pc();
-    __ emit_int64(0x87);  // The low-order bits of the field
-                          // polynomial (i.e. p = z^7+z^2+z+1)
-                          // repeated in the low and high parts of a
-                          // 128-bit vector
-    __ emit_int64(0x87);
+    Label ghash_polynomial; // local data generated after code

-    __ align(CodeEntryAlignment);
+   __ align(CodeEntryAlignment);
    StubId stub_id = StubId::stubgen_galoisCounterMode_AESCrypt_id;
    StubCodeMark mark(this, stub_id);
    address start = __ pc();
@@ -3514,7 +3558,17 @@ class StubGenerator: public StubCodeGenerator {

    __ leave(); // required for proper stackwalking of RuntimeStub frame
    __ ret(lr);
-     return start;
+
+    // bind label and generate polynomial data
+    __ align(wordSize * 2);
+    __ bind(ghash_polynomial);
+    __ emit_int64(0x87);  // The low-order bits of the field
+                          // polynomial (i.e. p = z^7+z^2+z+1)
+                          // repeated in the low and high parts of a
+                          // 128-bit vector
+    __ emit_int64(0x87);
+
+    return start;
  }

  class Cached64Bytes {
@@ -4559,16 +4613,6 @@ class StubGenerator: public StubCodeGenerator {
  // by the second lane from all vectors and so on.
  address generate_chacha20Block_blockpar() {
    Label L_twoRounds, L_cc20_const;
-    // The constant data is broken into two 128-bit segments to be loaded
-    // onto FloatRegisters.  The first 128 bits are a counter add overlay
-    // that adds +0/+1/+2/+3 to the vector holding replicated state[12].
-    // The second 128-bits is a table constant used for 8-bit left rotations.
-    __ BIND(L_cc20_const);
-    __ emit_int64(0x0000000100000000UL);
-    __ emit_int64(0x0000000300000002UL);
-    __ emit_int64(0x0605040702010003UL);
-    __ emit_int64(0x0E0D0C0F0A09080BUL);
-
    __ align(CodeEntryAlignment);
    StubId stub_id = StubId::stubgen_chacha20Block_id;
    StubCodeMark mark(this, stub_id);
@@ -4716,6 +4760,17 @@ class StubGenerator: public StubCodeGenerator {
    __ leave();
    __ ret(lr);

+    // bind label and generate local constant data used by this stub
+    // The constant data is broken into two 128-bit segments to be loaded
+    // onto FloatRegisters.  The first 128 bits are a counter add overlay
+    // that adds +0/+1/+2/+3 to the vector holding replicated state[12].
+    // The second 128-bits is a table constant used for 8-bit left rotations.
+    __ BIND(L_cc20_const);
+    __ emit_int64(0x0000000100000000UL);
+    __ emit_int64(0x0000000300000002UL);
+    __ emit_int64(0x0605040702010003UL);
+    __ emit_int64(0x0E0D0C0F0A09080BUL);
+
    return start;
  }

@@ -6036,10 +6091,6 @@ class StubGenerator: public StubCodeGenerator {
  address generate_kyber12To16() {
    Label L_F00, L_loop, L_end;

-    __ BIND(L_F00);
-    __ emit_int64(0x0f000f000f000f00);
-    __ emit_int64(0x0f000f000f000f00);
-
    __ align(CodeEntryAlignment);
    StubId stub_id = StubId::stubgen_kyber12To16_id;
    StubCodeMark mark(this, stub_id);
@@ -6233,6 +6284,11 @@ class StubGenerator: public StubCodeGenerator {
    __ mov(r0, zr); // return 0
    __ ret(lr);

+    // bind label and generate constant data used by this stub
+    __ BIND(L_F00);
+    __ emit_int64(0x0f000f000f000f00);
+    __ emit_int64(0x0f000f000f000f00);
+
    return start;
  }

@@ -9642,14 +9698,7 @@ class StubGenerator: public StubCodeGenerator {

    StubId stub_id = StubId::stubgen_ghash_processBlocks_id;
    StubCodeMark mark(this, stub_id);
-    __ align(wordSize * 2);
-    address p = __ pc();
-    __ emit_int64(0x87);  // The low-order bits of the field
-                          // polynomial (i.e. p = z^7+z^2+z+1)
-                          // repeated in the low and high parts of a
-                          // 128-bit vector
-    __ emit_int64(0x87);
-
+    Label polynomial; // local data generated at end of stub
    __ align(CodeEntryAlignment);
    address start = __ pc();

@@ -9661,7 +9710,8 @@ class StubGenerator: public StubCodeGenerator {
    FloatRegister vzr = v30;
    __ eor(vzr, __ T16B, vzr, vzr); // zero register

-    __ ldrq(v24, p);    // The field polynomial
+    __ adr(rscratch1, polynomial);
+    __ ldrq(v24, rscratch1);    // The field polynomial

    __ ldrq(v0, Address(state));
    __ ldrq(v1, Address(subkeyH));
@@ -9701,6 +9751,15 @@ class StubGenerator: public StubCodeGenerator {
    __ st1(v0, __ T16B, state);
    __ ret(lr);

+    // bind label and generate local polynomial data
+    __ align(wordSize * 2);
+    __ bind(polynomial);
+    __ emit_int64(0x87);  // The low-order bits of the field
+                          // polynomial (i.e. p = z^7+z^2+z+1)
+                          // repeated in the low and high parts of a
+                          // 128-bit vector
+    __ emit_int64(0x87);
+
    return start;
  }

@@ -9709,14 +9768,7 @@ class StubGenerator: public StubCodeGenerator {

    StubId stub_id = StubId::stubgen_ghash_processBlocks_wide_id;
    StubCodeMark mark(this, stub_id);
-    __ align(wordSize * 2);
-    address p = __ pc();
-    __ emit_int64(0x87);  // The low-order bits of the field
-                          // polynomial (i.e. p = z^7+z^2+z+1)
-                          // repeated in the low and high parts of a
-                          // 128-bit vector
-    __ emit_int64(0x87);
-
+    Label polynomial;           // local data generated after stub
    __ align(CodeEntryAlignment);
    address start = __ pc();

@@ -9738,7 +9790,7 @@ class StubGenerator: public StubCodeGenerator {
      __ st1(v8, v9, v10, v11, __ T16B, Address(sp));
    }

-    __ ghash_processBlocks_wide(p, state, subkeyH, data, blocks, unroll);
+    __ ghash_processBlocks_wide(polynomial, state, subkeyH, data, blocks, unroll);

    if (unroll > 1) {
      // And restore state
@@ -9751,7 +9803,17 @@ class StubGenerator: public StubCodeGenerator {

    __ ret(lr);

+    // bind label and generate polynomial data
+    __ align(wordSize * 2);
+    __ bind(polynomial);
+    __ emit_int64(0x87);  // The low-order bits of the field
+                          // polynomial (i.e. p = z^7+z^2+z+1)
+                          // repeated in the low and high parts of a
+                          // 128-bit vector
+    __ emit_int64(0x87);
+
    return start;
+
  }

  void generate_base64_encode_simdround(Register src, Register dst,
@@ -10265,7 +10327,7 @@ class StubGenerator: public StubCodeGenerator {

 #if defined (LINUX) && !defined (__ARM_FEATURE_ATOMICS)

-  // ARMv8.1 LSE versions of the atomic stubs used by Atomic::PlatformXX.
+  // ARMv8.1 LSE versions of the atomic stubs used by AtomicAccess::PlatformXX.
  //
  // If LSE is in use, generate LSE versions of all the stubs. The
  // non-LSE versions are in atomic_aarch64.S.
@@ -2638,9 +2638,6 @@ attributes %{
  instruction_unit_size = 4;         // An instruction is 4 bytes long
  instruction_fetch_unit_size = 16;  // The processor fetches one line
  instruction_fetch_units = 1;       // of 16 bytes
-
-  // List of nop instructions
-  nops( Nop_A0, Nop_A1, Nop_MS, Nop_FA, Nop_BR );
 %}

 //----------RESOURCES----------------------------------------------------------
@@ -3284,18 +3281,18 @@ pipe_class loadPollP(iRegP poll) %{
 %}

 pipe_class br(Universe br, label labl) %{
-    single_instruction_with_delay_slot;
+    single_instruction;
    BR  : R;
 %}

 pipe_class br_cc(Universe br, cmpOp cmp, flagsReg cr, label labl) %{
-    single_instruction_with_delay_slot;
+    single_instruction;
    cr    : E(read);
    BR    : R;
 %}

 pipe_class br_reg(Universe br, cmpOp cmp, iRegI op1, label labl) %{
-    single_instruction_with_delay_slot;
+    single_instruction;
    op1 : E(read);
    BR  : R;
    MS  : R;
@@ -3326,14 +3323,14 @@ pipe_class call(method meth) %{
 %}

 pipe_class tail_call(Universe ignore, label labl) %{
-    single_instruction; has_delay_slot;
+    single_instruction;
    fixed_latency(100);
    BR  : R(1);
    MS  : R(1);
 %}

 pipe_class ret(Universe ignore) %{
-    single_instruction; has_delay_slot;
+    single_instruction;
    BR  : R(1);
    MS  : R(1);
 %}
@@ -3376,14 +3373,6 @@ pipe_class cadd_cmpltmask( iRegI p, iRegI q, iRegI y ) %{
    IALU  : R(3)
 %}

-// Perform a compare, then move conditionally in a branch delay slot.
-pipe_class min_max( iRegI src2, iRegI srcdst ) %{
-    src2   : E(read);
-    srcdst : E(read);
-    IALU   : R;
-    BR     : R;
-%}
-
 // Define the class for the Nop node
 define %{
   MachNop = ialu_nop;
@@ -9056,7 +9045,7 @@ instruct clear_array(iRegX cnt, iRegP base, iRegI temp, iRegX zero, Universe dum
  format %{ "MOV    $zero,0\n"
      "        MOV    $temp,$cnt\n"
      "loop:   SUBS   $temp,$temp,4\t! Count down a dword of bytes\n"
-      "        STR.ge $zero,[$base+$temp]\t! delay slot"
+      "        STR.ge $zero,[$base+$temp]\n"
      "        B.gt   loop\t\t! Clearing loop\n" %}
  ins_encode %{
    __ mov($zero$$Register, 0);
@@ -2552,11 +2552,6 @@ void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
  fatal("Type profiling not implemented on this platform");
 }

-void LIR_Assembler::emit_delay(LIR_OpDelay*) {
-  Unimplemented();
-}
-
-
 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
  Address mon_addr = frame_map()->address_for_monitor_lock(monitor_no);
  __ add_slow(dst->as_pointer_register(), mon_addr.base(), mon_addr.disp());
@@ -48,11 +48,25 @@ class NativeNMethodBarrier: public NativeInstruction {

 public:
  int get_value() {
-    return Atomic::load_acquire(guard_addr());
+    return AtomicAccess::load_acquire(guard_addr());
  }

-  void set_value(int value) {
-    Atomic::release_store(guard_addr(), value);
+  void set_value(int value, int bit_mask) {
+    if (bit_mask == ~0) {
+      AtomicAccess::release_store(guard_addr(), value);
+      return;
+    }
+    assert((value & ~bit_mask) == 0, "trying to set bits outside the mask");
+    value &= bit_mask;
+    int old_value = AtomicAccess::load(guard_addr());
+    while (true) {
+      // Only bits in the mask are changed
+      int new_value = value | (old_value & ~bit_mask);
+      if (new_value == old_value) break;
+      int v = AtomicAccess::cmpxchg(guard_addr(), old_value, new_value, memory_order_release);
+      if (v == old_value) break;
+      old_value = v;
+    }
  }

  void verify() const;
@@ -115,7 +129,7 @@ void BarrierSetNMethod::deoptimize(nmethod* nm, address* return_address_ptr) {
  new_frame->pc = SharedRuntime::get_handle_wrong_method_stub();
 }

-void BarrierSetNMethod::set_guard_value(nmethod* nm, int value) {
+void BarrierSetNMethod::set_guard_value(nmethod* nm, int value, int bit_mask) {
  if (!supports_entry_barrier(nm)) {
    return;
  }
@@ -123,7 +137,7 @@ void BarrierSetNMethod::set_guard_value(nmethod* nm, int value) {
  // Disarms the nmethod guard emitted by BarrierSetAssembler::nmethod_entry_barrier.
  // Symmetric "LDR; DMB ISHLD" is in the nmethod barrier.
  NativeNMethodBarrier* barrier = native_nmethod_barrier(nm);
-  barrier->set_value(value);
+  barrier->set_value(value, bit_mask);
 }

 int BarrierSetNMethod::guard_value(nmethod* nm) {
@@ -421,7 +421,8 @@ class StubGenerator: public StubCodeGenerator {
  }


- // As per atomic.hpp the Atomic read-modify-write operations must be logically implemented as:
+ // As per atomicAccess.hpp the atomic read-modify-write operations must be
+ // logically implemented as:
 //  <fence>; <op>; <membar StoreLoad|StoreStore>
 // But for load-linked/store-conditional based systems a fence here simply means
 // no load/store can be reordered with respect to the initial load-linked, so we have:
@@ -440,7 +441,7 @@ class StubGenerator: public StubCodeGenerator {
  // be removed in the future.

  // Implementation of atomic_add(jint add_value, volatile jint* dest)
-  // used by Atomic::add(volatile jint* dest, jint add_value)
+  // used by AtomicAccess::add(volatile jint* dest, jint add_value)
  //
  // Arguments :
  //
@@ -492,7 +493,7 @@ class StubGenerator: public StubCodeGenerator {
  }

  // Implementation of jint atomic_xchg(jint exchange_value, volatile jint* dest)
-  // used by Atomic::add(volatile jint* dest, jint exchange_value)
+  // used by AtomicAccess::add(volatile jint* dest, jint exchange_value)
  //
  // Arguments :
  //
@@ -542,7 +543,7 @@ class StubGenerator: public StubCodeGenerator {
  }

  // Implementation of jint atomic_cmpxchg(jint exchange_value, volatile jint *dest, jint compare_value)
-  // used by Atomic::cmpxchg(volatile jint *dest, jint compare_value, jint exchange_value)
+  // used by AtomicAccess::cmpxchg(volatile jint *dest, jint compare_value, jint exchange_value)
  //
  // Arguments :
  //
@@ -582,7 +583,7 @@ class StubGenerator: public StubCodeGenerator {
    return start;
  }

-  // Support for jlong Atomic::cmpxchg(jlong exchange_value, volatile jlong *dest, jlong compare_value)
+  // Support for jlong AtomicAccess::cmpxchg(jlong exchange_value, volatile jlong *dest, jlong compare_value)
  // reordered before by a wrapper to (jlong compare_value, jlong exchange_value, volatile jlong *dest)
  //
  // Arguments :
@@ -3010,6 +3011,10 @@ class StubGenerator: public StubCodeGenerator {
    // Note:  the disjoint stubs must be generated first, some of
    //        the conjoint stubs use them.

+    // Note:   chaining of stubs does not rely on branching to an
+    //         auxiliary post-push entry because none of the stubs
+    //         push/pop a frame.
+
    // these need always status in case they are called from generic_arraycopy
    StubRoutines::_jbyte_disjoint_arraycopy  = generate_primitive_copy(StubId::stubgen_jbyte_disjoint_arraycopy_id);
    StubRoutines::_jshort_disjoint_arraycopy = generate_primitive_copy(StubId::stubgen_jshort_disjoint_arraycopy_id);
@@ -3023,6 +3028,7 @@ class StubGenerator: public StubCodeGenerator {
    StubRoutines::_arrayof_jlong_disjoint_arraycopy  = generate_primitive_copy(StubId::stubgen_arrayof_jlong_disjoint_arraycopy_id);
    StubRoutines::_arrayof_oop_disjoint_arraycopy    = generate_oop_copy      (StubId::stubgen_arrayof_oop_disjoint_arraycopy_id);

+    // disjoint copy entry is needed by conjoint copy
    // these need always status in case they are called from generic_arraycopy
    StubRoutines::_jbyte_arraycopy  = generate_primitive_copy(StubId::stubgen_jbyte_arraycopy_id, StubRoutines::_jbyte_disjoint_arraycopy);
    StubRoutines::_jshort_arraycopy = generate_primitive_copy(StubId::stubgen_jshort_arraycopy_id, StubRoutines::_jshort_disjoint_arraycopy);
@@ -26,7 +26,7 @@
 #ifndef CPU_PPC_ASSEMBLER_PPC_INLINE_HPP
 #define CPU_PPC_ASSEMBLER_PPC_INLINE_HPP

-#include "asm/assembler.inline.hpp"
+#include "asm/assembler.hpp"
 #include "asm/codeBuffer.hpp"
 #include "code/codeCache.hpp"
 #include "runtime/vm_version.hpp"
@@ -2747,11 +2747,6 @@ void LIR_Assembler::align_backward_branch_target() {
 }


-void LIR_Assembler::emit_delay(LIR_OpDelay* op) {
-  Unimplemented();
-}
-
-
 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
  // tmp must be unused
  assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
@@ -183,12 +183,9 @@ void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm, Register t
  BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
  assert_different_registers(tmp, R0);

-  __ block_comment("nmethod_entry_barrier (nmethod_entry_barrier) {");
+  __ align(8); // must align the following block which requires atomic updates

-  // Load stub address using toc (fixed instruction size, unlike load_const_optimized)
-  __ calculate_address_from_global_toc(tmp, StubRoutines::method_entry_barrier(),
-                                       true, true, false); // 2 instructions
-  __ mtctr(tmp);
+  __ block_comment("nmethod_entry_barrier (nmethod_entry_barrier) {");

  // This is a compound instruction. Patching support is provided by NativeMovRegMem.
  // Actual patching is done in (platform-specific part of) BarrierSetNMethod.
@@ -198,6 +195,11 @@ void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm, Register t
  __ ld(R0, in_bytes(bs_nm->thread_disarmed_guard_value_offset()), R16_thread);
  __ cmpw(CR0, R0, tmp);

+  // Load stub address using toc (fixed instruction size, unlike load_const_optimized)
+  __ calculate_address_from_global_toc(tmp, StubRoutines::method_entry_barrier(),
+                                       true, true, false); // 2 instructions
+  __ mtctr(tmp);
+
  __ bnectrl(CR0);

  // Oops may have been changed. Make those updates observable.
@@ -38,7 +38,7 @@ class NativeNMethodBarrier: public NativeInstruction {

  NativeMovRegMem* get_patchable_instruction_handle() const {
    // Endianness is handled by NativeMovRegMem
-    return reinterpret_cast<NativeMovRegMem*>(get_barrier_start_address() + 3 * 4);
+    return reinterpret_cast<NativeMovRegMem*>(get_barrier_start_address());
  }

 public:
@@ -47,7 +47,7 @@ public:
    return get_patchable_instruction_handle()->offset();
  }

-  void release_set_guard_value(int value) {
+  void release_set_guard_value(int value, int bit_mask) {
    // Patching is not atomic.
    // Stale observations of the "armed" state is okay as invoking the barrier stub in that case has no
    // unwanted side effects. Disarming is thus a non-critical operation.
@@ -55,8 +55,37 @@ public:

    OrderAccess::release(); // Release modified oops

-    // Set the guard value (naming of 'offset' function is misleading).
-    get_patchable_instruction_handle()->set_offset(value);
+    if (bit_mask == ~0) {
+      // Set the guard value (naming of 'offset' function is misleading).
+      get_patchable_instruction_handle()->set_offset(value);
+      return;
+    }
+
+    assert((value & ~bit_mask) == 0, "trying to set bits outside the mask");
+    value &= bit_mask;
+
+    NativeMovRegMem* mov = get_patchable_instruction_handle();
+    assert(align_up(mov->instruction_address(), sizeof(uint64_t)) ==
+           align_down(mov->instruction_address(), sizeof(uint64_t)), "instruction not aligned");
+    uint64_t *instr = (uint64_t*)mov->instruction_address();
+    assert(NativeMovRegMem::instruction_size == sizeof(*instr), "must be");
+    union {
+      u_char buf[NativeMovRegMem::instruction_size];
+      uint64_t u64;
+    } new_mov_instr, old_mov_instr;
+    new_mov_instr.u64 = old_mov_instr.u64 = AtomicAccess::load(instr);
+    while (true) {
+      // Only bits in the mask are changed
+      int old_value = nativeMovRegMem_at(old_mov_instr.buf)->offset();
+      int new_value = value | (old_value & ~bit_mask);
+      if (new_value == old_value) return; // skip icache flush if nothing changed
+      nativeMovRegMem_at(new_mov_instr.buf)->set_offset(new_value, false /* no icache flush */);
+      // Swap in the new value
+      uint64_t v = AtomicAccess::cmpxchg(instr, old_mov_instr.u64, new_mov_instr.u64, memory_order_relaxed);
+      if (v == old_mov_instr.u64) break;
+      old_mov_instr.u64 = v;
+    }
+    ICache::ppc64_flush_icache_bytes(addr_at(0), NativeMovRegMem::instruction_size);
  }

  void verify() const {
@@ -66,12 +95,6 @@ public:

    uint* current_instruction = reinterpret_cast<uint*>(get_barrier_start_address());

-    // calculate_address_from_global_toc (compound instruction)
-    verify_op_code_manually(current_instruction, MacroAssembler::is_addis(*current_instruction));
-    verify_op_code_manually(current_instruction, MacroAssembler::is_addi(*current_instruction));
-
-    verify_op_code_manually(current_instruction, MacroAssembler::is_mtctr(*current_instruction));
-
    get_patchable_instruction_handle()->verify();
    current_instruction += 2;

@@ -80,6 +103,12 @@ public:
    // cmpw (mnemonic)
    verify_op_code(current_instruction, Assembler::CMP_OPCODE);

+    // calculate_address_from_global_toc (compound instruction)
+    verify_op_code_manually(current_instruction, MacroAssembler::is_addis(*current_instruction));
+    verify_op_code_manually(current_instruction, MacroAssembler::is_addi(*current_instruction));
+
+    verify_op_code_manually(current_instruction, MacroAssembler::is_mtctr(*current_instruction));
+
    // bnectrl (mnemonic) (weak check; not checking the exact type)
    verify_op_code(current_instruction, Assembler::BCCTR_OPCODE);

@@ -117,13 +146,13 @@ void BarrierSetNMethod::deoptimize(nmethod* nm, address* return_address_ptr) {
  // Thus, there's nothing to do here.
 }

-void BarrierSetNMethod::set_guard_value(nmethod* nm, int value) {
+void BarrierSetNMethod::set_guard_value(nmethod* nm, int value, int bit_mask) {
  if (!supports_entry_barrier(nm)) {
    return;
  }

  NativeNMethodBarrier* barrier = get_nmethod_barrier(nm);
-  barrier->release_set_guard_value(value);
+  barrier->release_set_guard_value(value, bit_mask);
 }

 int BarrierSetNMethod::guard_value(nmethod* nm) {
@@ -347,7 +347,7 @@ void NativeGeneralJump::replace_mt_safe(address instr_addr, address code_buffer)
  // Finally patch out the jump.
  volatile juint *jump_addr = (volatile juint*)instr_addr;
  // Release not needed because caller uses invalidate_range after copying the remaining bytes.
-  //Atomic::release_store(jump_addr, *((juint*)code_buffer));
+  //AtomicAccess::release_store(jump_addr, *((juint*)code_buffer));
  *jump_addr = *((juint*)code_buffer); // atomically store code over branch instruction
  ICache::ppc64_flush_icache_bytes(instr_addr, NativeGeneralJump::instruction_size);
 }
@@ -462,7 +462,7 @@ class NativeMovRegMem: public NativeInstruction {
    return ((*hi_ptr) << 16) | ((*lo_ptr) & 0xFFFF);
  }

-  void set_offset(intptr_t x) {
+  void set_offset(intptr_t x, bool flush_icache = true) {
 #ifdef VM_LITTLE_ENDIAN
    short *hi_ptr = (short*)(addr_at(0));
    short *lo_ptr = (short*)(addr_at(4));
@@ -472,7 +472,9 @@ class NativeMovRegMem: public NativeInstruction {
 #endif
    *hi_ptr = x >> 16;
    *lo_ptr = x & 0xFFFF;
-    ICache::ppc64_flush_icache_bytes(addr_at(0), NativeMovRegMem::instruction_size);
+    if (flush_icache) {
+      ICache::ppc64_flush_icache_bytes(addr_at(0), NativeMovRegMem::instruction_size);
+    }
  }

  void add_offset_in_bytes(intptr_t radd_offset) {
@@ -4920,10 +4920,6 @@ attributes %{

  // ...in one line
  instruction_fetch_units = 1
-
-  // Unused, list one so that array generated by adlc is not empty.
-  // Aix compiler chokes if _nop_count = 0.
-  nops(fxNop);
 %}

 //----------RESOURCES----------------------------------------------------------
@@ -3277,8 +3277,12 @@ class StubGenerator: public StubCodeGenerator {
    // register the stub as the default exit with class UnsafeMemoryAccess
    UnsafeMemoryAccess::set_common_exit_stub_pc(StubRoutines::_unsafecopy_common_exit);

-    // Note: the disjoint stubs must be generated first, some of
-    // the conjoint stubs use them.
+    // Note: the disjoint stubs must be generated first, some of the
+    //       conjoint stubs use them.
+
+    // Note: chaining of stubs does not rely on branching to an
+    //       auxiliary post-push entry because none of the stubs
+    //       push/pop a frame.

    // non-aligned disjoint versions
    StubRoutines::_jbyte_disjoint_arraycopy       = generate_disjoint_byte_copy(StubId::stubgen_jbyte_disjoint_arraycopy_id);
@@ -912,6 +912,32 @@ protected:
    emit(insn);
  }

+ public:
+
+  static uint32_t encode_jal(Register Rd, const int32_t offset) {
+    guarantee(is_simm21(offset) && ((offset % 2) == 0), "offset is invalid.");
+    uint32_t insn = 0;
+    patch((address)&insn, 6, 0, 0b1101111);
+    patch_reg((address)&insn, 7, Rd);
+    patch((address)&insn, 19, 12, (uint32_t)((offset >> 12) & 0xff));
+    patch((address)&insn, 20, (uint32_t)((offset >> 11) & 0x1));
+    patch((address)&insn, 30, 21, (uint32_t)((offset >> 1) & 0x3ff));
+    patch((address)&insn, 31, (uint32_t)((offset >> 20) & 0x1));
+    return insn;
+  }
+
+  static uint32_t encode_jalr(Register Rd, Register Rs, const int32_t offset) {
+    guarantee(is_simm12(offset), "offset is invalid.");
+    uint32_t insn = 0;
+    patch((address)&insn, 6, 0, 0b1100111);
+    patch_reg((address)&insn, 7, Rd);
+    patch((address)&insn, 14, 12, 0b000);
+    patch_reg((address)&insn, 15, Rs);
+    int32_t val = offset & 0xfff;
+    patch((address)&insn, 31, 20, val);
+    return insn;
+  }
+
 protected:

  enum barrier {
@@ -27,7 +27,7 @@
 #ifndef CPU_RISCV_ASSEMBLER_RISCV_INLINE_HPP
 #define CPU_RISCV_ASSEMBLER_RISCV_INLINE_HPP

-#include "asm/assembler.inline.hpp"
+#include "asm/assembler.hpp"
 #include "asm/codeBuffer.hpp"
 #include "code/codeCache.hpp"

@@ -1590,8 +1590,6 @@ void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
  }
 }

-void LIR_Assembler::emit_delay(LIR_OpDelay*) { Unimplemented(); }
-
 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
  __ la(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
 }
@@ -217,7 +217,7 @@ address BarrierSetAssembler::patching_epoch_addr() {
 }

 void BarrierSetAssembler::increment_patching_epoch() {
-  Atomic::inc(&_patching_epoch);
+  AtomicAccess::inc(&_patching_epoch);
 }

 void BarrierSetAssembler::clear_patching_epoch() {
@@ -106,11 +106,25 @@ public:
  }

  int get_value() {
-    return Atomic::load_acquire(guard_addr());
+    return AtomicAccess::load_acquire(guard_addr());
  }

-  void set_value(int value) {
-    Atomic::release_store(guard_addr(), value);
+  void set_value(int value, int bit_mask) {
+    if (bit_mask == ~0) {
+      AtomicAccess::release_store(guard_addr(), value);
+      return;
+    }
+    assert((value & ~bit_mask) == 0, "trying to set bits outside the mask");
+    value &= bit_mask;
+    int old_value = AtomicAccess::load(guard_addr());
+    while (true) {
+      // Only bits in the mask are changed
+      int new_value = value | (old_value & ~bit_mask);
+      if (new_value == old_value) break;
+      int v = AtomicAccess::cmpxchg(guard_addr(), old_value, new_value, memory_order_release);
+      if (v == old_value) break;
+      old_value = v;
+    }
  }

  bool check_barrier(err_msg& msg) const;
@@ -192,7 +206,7 @@ void BarrierSetNMethod::deoptimize(nmethod* nm, address* return_address_ptr) {
  new_frame->pc = SharedRuntime::get_handle_wrong_method_stub();
 }

-void BarrierSetNMethod::set_guard_value(nmethod* nm, int value) {
+void BarrierSetNMethod::set_guard_value(nmethod* nm, int value, int bit_mask) {
  if (!supports_entry_barrier(nm)) {
    return;
  }
@@ -209,7 +223,7 @@ void BarrierSetNMethod::set_guard_value(nmethod* nm, int value) {
  }

  NativeNMethodBarrier barrier(nm);
-  barrier.set_value(value);
+  barrier.set_value(value, bit_mask);
 }

 int BarrierSetNMethod::guard_value(nmethod* nm) {
@@ -3402,6 +3402,8 @@ void MacroAssembler::decode_klass_not_null(Register r, Register tmp) {

 void MacroAssembler::decode_klass_not_null(Register dst, Register src, Register tmp) {
  assert(UseCompressedClassPointers, "should only be used for compressed headers");
+  assert_different_registers(dst, tmp);
+  assert_different_registers(src, tmp);

  if (CompressedKlassPointers::base() == nullptr) {
    if (CompressedKlassPointers::shift() != 0) {
@@ -3412,18 +3414,13 @@ void MacroAssembler::decode_klass_not_null(Register dst, Register src, Register
    return;
  }

-  Register xbase = dst;
-  if (dst == src) {
-    xbase = tmp;
-  }
+  Register xbase = tmp;

-  assert_different_registers(src, xbase);
  mv(xbase, (uintptr_t)CompressedKlassPointers::base());

  if (CompressedKlassPointers::shift() != 0) {
-    Register t = src == dst ? dst : t0;
-    assert_different_registers(t, xbase);
-    shadd(dst, src, xbase, t, CompressedKlassPointers::shift());
+    // dst = (src << shift) + xbase
+    shadd(dst, src, xbase, dst /* temporary, dst != xbase */, CompressedKlassPointers::shift());
  } else {
    add(dst, xbase, src);
  }
@@ -5874,13 +5871,14 @@ void MacroAssembler::fill_words(Register base, Register cnt, Register value) {
 // in cnt.
 //
 // NOTE: This is intended to be used in the zero_blocks() stub.  If
-// you want to use it elsewhere, note that cnt must be >= CacheLineSize.
+// you want to use it elsewhere, note that cnt must be >= zicboz_block_size.
 void MacroAssembler::zero_dcache_blocks(Register base, Register cnt, Register tmp1, Register tmp2) {
+  int zicboz_block_size = VM_Version::zicboz_block_size.value();
  Label initial_table_end, loop;

  // Align base with cache line size.
  neg(tmp1, base);
-  andi(tmp1, tmp1, CacheLineSize - 1);
+  andi(tmp1, tmp1, zicboz_block_size - 1);

  // tmp1: the number of bytes to be filled to align the base with cache line size.
  add(base, base, tmp1);
@@ -5890,16 +5888,16 @@ void MacroAssembler::zero_dcache_blocks(Register base, Register cnt, Register tm
  la(tmp1, initial_table_end);
  sub(tmp2, tmp1, tmp2);
  jr(tmp2);
-  for (int i = -CacheLineSize + wordSize; i < 0; i += wordSize) {
+  for (int i = -zicboz_block_size + wordSize; i < 0; i += wordSize) {
    sd(zr, Address(base, i));
  }
  bind(initial_table_end);

-  mv(tmp1, CacheLineSize / wordSize);
+  mv(tmp1, zicboz_block_size / wordSize);
  bind(loop);
  cbo_zero(base);
  sub(cnt, cnt, tmp1);
-  addi(base, base, CacheLineSize);
+  addi(base, base, zicboz_block_size);
  bge(cnt, tmp1, loop);
 }

@@ -28,11 +28,13 @@
 #include "code/compiledIC.hpp"
 #include "nativeInst_riscv.hpp"
 #include "oops/oop.inline.hpp"
+#include "runtime/atomicAccess.hpp"
 #include "runtime/handles.hpp"
 #include "runtime/orderAccess.hpp"
 #include "runtime/safepoint.hpp"
 #include "runtime/sharedRuntime.hpp"
 #include "runtime/stubRoutines.hpp"
+#include "utilities/align.hpp"
 #include "utilities/ostream.hpp"
 #ifdef COMPILER1
 #include "c1/c1_Runtime1.hpp"
@@ -52,15 +54,15 @@ address NativeCall::destination() const {
  address addr = instruction_address();
  assert(NativeCall::is_at(addr), "unexpected code at call site");

-  address destination = MacroAssembler::target_addr_for_insn(addr);
+  address stub_addr = MacroAssembler::target_addr_for_insn(addr);

  CodeBlob* cb = CodeCache::find_blob(addr);
  assert(cb != nullptr && cb->is_nmethod(), "nmethod expected");
  nmethod *nm = (nmethod *)cb;
-  assert(nm != nullptr, "Sanity");
-  assert(nm->stub_contains(destination), "Sanity");
-  assert(destination != nullptr, "Sanity");
-  return stub_address_destination_at(destination);
+  assert(nm->stub_contains(stub_addr), "Sanity");
+  assert(stub_addr != nullptr, "Sanity");
+
+  return stub_address_destination_at(stub_addr);
 }

 address NativeCall::reloc_destination() {
@@ -89,6 +91,30 @@ void NativeCall::print() {
  tty->print_cr(PTR_FORMAT ": auipc,ld,jalr x1, offset/reg, ", p2i(instruction_address()));
 }

+void NativeCall::optimize_call(address dest, bool mt_safe) {
+  // Skip over auipc + ld
+  address jmp_ins_pc = instruction_address() + 2 * NativeInstruction::instruction_size;
+  // Rutime calls may be unaligned, but they are never changed after relocation.
+  assert(!mt_safe || is_aligned(jmp_ins_pc, NativeInstruction::instruction_size), "Must be naturally aligned: %p", jmp_ins_pc);
+  // If reachable use JAL
+  if (Assembler::reachable_from_branch_at(jmp_ins_pc, dest)) {
+    int64_t distance = dest - jmp_ins_pc;
+    uint32_t new_jal = Assembler::encode_jal(ra, distance);
+    AtomicAccess::store((uint32_t *)jmp_ins_pc, new_jal);
+  } else if (!MacroAssembler::is_jalr_at(jmp_ins_pc)) { // The jalr is always identical: jalr ra, 0(t1)
+    uint32_t new_jalr = Assembler::encode_jalr(ra, t1, 0);
+    AtomicAccess::store((uint32_t *)jmp_ins_pc, new_jalr);
+  } else {
+    // No change to instruction stream
+    return;
+  }
+  // We changed instruction stream
+  if (mt_safe) {
+    // IC invalidate provides a leading full fence, it thus happens after we changed the instruction stream.
+    ICache::invalidate_range(jmp_ins_pc, NativeInstruction::instruction_size);
+  }
+}
+
 bool NativeCall::set_destination_mt_safe(address dest) {
  assert(NativeCall::is_at(instruction_address()), "unexpected code at call site");
  assert((CodeCache_lock->is_locked() || SafepointSynchronize::is_at_safepoint()) ||
@@ -96,15 +122,17 @@ bool NativeCall::set_destination_mt_safe(address dest) {
         "concurrent code patching");

  address stub_addr = stub_address();
-  if (stub_addr != nullptr) {
-    set_stub_address_destination_at(stub_addr, dest);
-    return true;
-  }
+  assert(stub_addr != nullptr, "No stub?");
+  set_stub_address_destination_at(stub_addr, dest); // release
+  // optimize_call happens after we stored new address in addr stub.
+  // patches jalr -> jal/jal -> jalr depending on dest
+  optimize_call(dest, true);

-  return false;
+  return true;
 }

-bool NativeCall::reloc_set_destination(address dest) {
+// The argument passed in is the address to the stub containing the destination
+bool NativeCall::reloc_set_destination(address stub_addr) {
  address call_addr = instruction_address();
  assert(NativeCall::is_at(call_addr), "unexpected code at call site");

@@ -113,10 +141,12 @@ bool NativeCall::reloc_set_destination(address dest) {

  if (code->is_nmethod()) {
    // TODO: Need to revisit this when porting the AOT features.
-    assert(dest != nullptr, "Sanity");
-    assert(dest == trampoline_stub_Relocation::get_trampoline_for(call_addr,
-                                                          code->as_nmethod()), "Sanity");
-    MacroAssembler::pd_patch_instruction_size(call_addr, dest);
+    assert(stub_addr != nullptr, "Sanity");
+    assert(stub_addr == trampoline_stub_Relocation::get_trampoline_for(call_addr, code->as_nmethod()), "Sanity");
+    MacroAssembler::pd_patch_instruction_size(call_addr, stub_addr); // patches auipc + ld to stub_addr
+
+    address dest = stub_address_destination_at(stub_addr);
+    optimize_call(dest, false); // patches jalr -> jal/jal -> jalr depending on dest
  }

  return true;
@@ -142,9 +172,9 @@ address NativeCall::stub_address() {
  CodeBlob *code = CodeCache::find_blob(call_addr);
  assert(code != nullptr, "Could not find the containing code blob");

-  address dest = MacroAssembler::target_addr_for_insn(call_addr);
-  assert(code->contains(dest), "Sanity");
-  return dest;
+  address stub_addr = MacroAssembler::target_addr_for_insn(call_addr);
+  assert(code->contains(stub_addr), "Sanity");
+  return stub_addr;
 }

 bool NativeCall::is_at(address addr) {
@@ -160,6 +190,15 @@ bool NativeCall::is_at(address addr) {
      (MacroAssembler::extract_rd(addr + 2 * instr_size)   == x1)) {
    return true;
  }
+  if (MacroAssembler::is_auipc_at(addr) &&
+      MacroAssembler::is_ld_at(addr + instr_size) &&
+      MacroAssembler::is_jal_at(addr + 2 * instr_size) &&
+      (MacroAssembler::extract_rd(addr)                    == x6) &&
+      (MacroAssembler::extract_rd(addr + instr_size)       == x6) &&
+      (MacroAssembler::extract_rs1(addr + instr_size)      == x6) &&
+      (MacroAssembler::extract_rd(addr + 2 * instr_size)   == x1)) {
+    return true;
+  }
  return false;
 }

@@ -156,6 +156,10 @@ class NativeCall: private NativeInstruction {
  static void set_stub_address_destination_at(address dest, address value);
  // return target address at stub
  static address stub_address_destination_at(address src);
+  // We either have a jalr or jal depending on distance to old destination.
+  // This method emits a new jal if new destination is within jal reach.
+  // Otherwise restores the jalr which can reach any destination.
+  void optimize_call(address dest, bool mt_safe = true);
 };

 // An interface for accessing/manipulating native mov reg, imm instructions.
@@ -3845,9 +3845,6 @@ attributes %{

  // ...in one line.
  instruction_fetch_units = 1;
-
-  // List of nop instructions
-  nops( MachNop );
 %}

 // We don't use an actual pipeline model so don't care about resources
@@ -8941,7 +8938,7 @@ instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src, iRegPNoSp tmp) %{
  match(Set dst (DecodeNKlass src));

-  effect(TEMP tmp);
+  effect(TEMP_DEF dst, TEMP tmp);

  ins_cost(ALU_COST);
  format %{ "decode_klass_not_null  $dst, $src\t#@decodeKlass_not_null" %}
@@ -683,10 +683,11 @@ class StubGenerator: public StubCodeGenerator {
    address start = __ pc();

    if (UseBlockZeroing) {
-      // Ensure count >= 2*CacheLineSize so that it still deserves a cbo.zero
-      // after alignment.
+      int zicboz_block_size = VM_Version::zicboz_block_size.value();
+      // Ensure count >= 2 * zicboz_block_size so that it still deserves
+      // a cbo.zero after alignment.
      Label small;
-      int low_limit = MAX2(2 * CacheLineSize, BlockZeroingLowLimit) / wordSize;
+      int low_limit = MAX2(2 * zicboz_block_size, (int)BlockZeroingLowLimit) / wordSize;
      __ mv(tmp1, low_limit);
      __ blt(cnt, tmp1, small);
      __ zero_dcache_blocks(base, cnt, tmp1, tmp2);
@@ -731,8 +732,7 @@ class StubGenerator: public StubCodeGenerator {
  //
  // s and d are adjusted to point to the remaining words to copy
  //
-  void generate_copy_longs(StubId stub_id, Label &start,
-                           Register s, Register d, Register count) {
+  address generate_copy_longs(StubId stub_id, Register s, Register d, Register count) {
    BasicType type;
    copy_direction direction;
    switch (stub_id) {
@@ -762,7 +762,7 @@ class StubGenerator: public StubCodeGenerator {
    Label again, drain;
    StubCodeMark mark(this, stub_id);
    __ align(CodeEntryAlignment);
-    __ bind(start);
+    address start = __ pc();

    if (direction == copy_forwards) {
      __ sub(s, s, bias);
@@ -878,9 +878,9 @@ class StubGenerator: public StubCodeGenerator {
    }

    __ ret();
-  }

-  Label copy_f, copy_b;
+    return start;
+  }

  typedef void (MacroAssembler::*copy_insn)(Register Rd, const Address &adr, Register temp);

@@ -1098,8 +1098,8 @@ class StubGenerator: public StubCodeGenerator {
  //   stub_id - is used to name the stub and identify all details of
  //             how to perform the copy.
  //
-  //   entry - is assigned to the stub's post push entry point unless
-  //           it is null
+  //   nopush_entry - is assigned to the stub's post push entry point
+  //                  unless it is null
  //
  // Inputs:
  //   c_rarg0   - source array address
@@ -1110,11 +1110,11 @@ class StubGenerator: public StubCodeGenerator {
  // the hardware handle it.  The two dwords within qwords that span
  // cache line boundaries will still be loaded and stored atomically.
  //
-  // Side Effects: entry is set to the (post push) entry point so it
-  //               can be used by the corresponding conjoint copy
-  //               method
+  // Side Effects: nopush_entry is set to the (post push) entry point
+  //               so it can be used by the corresponding conjoint
+  //               copy method
  //
-  address generate_disjoint_copy(StubId stub_id, address* entry) {
+  address generate_disjoint_copy(StubId stub_id, address* nopush_entry) {
    size_t size;
    bool aligned;
    bool is_oop;
@@ -1203,8 +1203,8 @@ class StubGenerator: public StubCodeGenerator {
    address start = __ pc();
    __ enter();

-    if (entry != nullptr) {
-      *entry = __ pc();
+    if (nopush_entry != nullptr) {
+     *nopush_entry = __ pc();
      // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
      BLOCK_COMMENT("Entry:");
    }
@@ -1255,8 +1255,8 @@ class StubGenerator: public StubCodeGenerator {
  //             corresponding disjoint copy routine which can be
  //             jumped to if the ranges do not actually overlap
  //
-  //   entry - is assigned to the stub's post push entry point unless
-  //           it is null
+  //   nopush_entry - is assigned to the stub's post push entry point
+  //                 unless it is null
  //
  // Inputs:
  //   c_rarg0   - source array address
@@ -1268,10 +1268,10 @@ class StubGenerator: public StubCodeGenerator {
  // cache line boundaries will still be loaded and stored atomically.
  //
  // Side Effects:
-  //   entry is set to the no-overlap entry point so it can be used by
-  //   some other conjoint copy method
+  //   nopush_entry is set to the no-overlap entry point so it can be
+  //   used by some other conjoint copy method
  //
-  address generate_conjoint_copy(StubId stub_id, address nooverlap_target, address *entry) {
+  address generate_conjoint_copy(StubId stub_id, address nooverlap_target, address *nopush_entry) {
    const Register s = c_rarg0, d = c_rarg1, count = c_rarg2;
    RegSet saved_regs = RegSet::of(s, d, count);
    int size;
@@ -1358,8 +1358,8 @@ class StubGenerator: public StubCodeGenerator {
    address start = __ pc();
    __ enter();

-    if (entry != nullptr) {
-      *entry = __ pc();
+    if (nopush_entry != nullptr) {
+      *nopush_entry = __ pc();
      // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
      BLOCK_COMMENT("Entry:");
    }
@@ -1369,7 +1369,7 @@ class StubGenerator: public StubCodeGenerator {
    __ slli(t1, count, exact_log2(size));
    Label L_continue;
    __ bltu(t0, t1, L_continue);
-    __ j(nooverlap_target);
+    __ j(RuntimeAddress(nooverlap_target));
    __ bind(L_continue);

    DecoratorSet decorators = IN_HEAP | IS_ARRAY;
@@ -1444,7 +1444,7 @@ class StubGenerator: public StubCodeGenerator {
  //    x10 ==  0  -  success
  //    x10 == -1^K - failure, where K is partial transfer count
  //
-  address generate_checkcast_copy(StubId stub_id, address* entry) {
+  address generate_checkcast_copy(StubId stub_id, address* nopush_entry) {
    bool dest_uninitialized;
    switch (stub_id) {
    case StubId::stubgen_checkcast_arraycopy_id:
@@ -1495,8 +1495,8 @@ class StubGenerator: public StubCodeGenerator {
    __ enter(); // required for proper stackwalking of RuntimeStub frame

    // Caller of this entry point must set up the argument registers.
-    if (entry != nullptr) {
-      *entry = __ pc();
+    if (nopush_entry != nullptr) {
+      *nopush_entry = __ pc();
      BLOCK_COMMENT("Entry:");
    }

@@ -2293,13 +2293,21 @@ class StubGenerator: public StubCodeGenerator {
  }

  void generate_arraycopy_stubs() {
-    address entry                     = nullptr;
-    address entry_jbyte_arraycopy     = nullptr;
-    address entry_jshort_arraycopy    = nullptr;
-    address entry_jint_arraycopy      = nullptr;
-    address entry_oop_arraycopy       = nullptr;
-    address entry_jlong_arraycopy     = nullptr;
-    address entry_checkcast_arraycopy = nullptr;
+    // Some copy stubs publish a normal entry and then a 2nd 'fallback'
+    // entry immediately following their stack push. This can be used
+    // as a post-push branch target for compatible stubs when they
+    // identify a special case that can be handled by the fallback
+    // stub e.g a disjoint copy stub may be use as a special case
+    // fallback for its compatible conjoint copy stub.
+    //
+    // A no push entry is always returned in the following local and
+    // then published by assigning to the appropriate entry field in
+    // class StubRoutines. The entry value is then passed to the
+    // generator for the compatible stub. That means the entry must be
+    // listed when saving to/restoring from the AOT cache, ensuring
+    // that the inter-stub jumps are noted at AOT-cache save and
+    // relocated at AOT cache load.
+    address nopush_entry = nullptr;

    // generate the common exit first so later stubs can rely on it if
    // they want an UnsafeMemoryAccess exit non-local to the stub
@@ -2307,72 +2315,117 @@ class StubGenerator: public StubCodeGenerator {
    // register the stub as the default exit with class UnsafeMemoryAccess
    UnsafeMemoryAccess::set_common_exit_stub_pc(StubRoutines::_unsafecopy_common_exit);

-    generate_copy_longs(StubId::stubgen_copy_byte_f_id, copy_f, c_rarg0, c_rarg1, t1);
-    generate_copy_longs(StubId::stubgen_copy_byte_b_id, copy_b, c_rarg0, c_rarg1, t1);
+    // generate and publish riscv-specific bulk copy routines first
+    // so we can call them from other copy stubs
+    StubRoutines::riscv::_copy_byte_f = generate_copy_longs(StubId::stubgen_copy_byte_f_id, c_rarg0, c_rarg1, t1);
+    StubRoutines::riscv::_copy_byte_b = generate_copy_longs(StubId::stubgen_copy_byte_b_id, c_rarg0, c_rarg1, t1);

    StubRoutines::riscv::_zero_blocks = generate_zero_blocks();

    //*** jbyte
    // Always need aligned and unaligned versions
-    StubRoutines::_jbyte_disjoint_arraycopy          = generate_disjoint_copy(StubId::stubgen_jbyte_disjoint_arraycopy_id, &entry);
-    StubRoutines::_jbyte_arraycopy                   = generate_conjoint_copy(StubId::stubgen_jbyte_arraycopy_id, entry, &entry_jbyte_arraycopy);
-    StubRoutines::_arrayof_jbyte_disjoint_arraycopy  = generate_disjoint_copy(StubId::stubgen_arrayof_jbyte_disjoint_arraycopy_id, &entry);
-    StubRoutines::_arrayof_jbyte_arraycopy           = generate_conjoint_copy(StubId::stubgen_arrayof_jbyte_arraycopy_id, entry, nullptr);
+    StubRoutines::_jbyte_disjoint_arraycopy          = generate_disjoint_copy(StubId::stubgen_jbyte_disjoint_arraycopy_id, &nopush_entry);
+    // disjoint nopush entry is needed by conjoint copy
+    StubRoutines::_jbyte_disjoint_arraycopy_nopush  = nopush_entry;
+    StubRoutines::_jbyte_arraycopy                   = generate_conjoint_copy(StubId::stubgen_jbyte_arraycopy_id, StubRoutines::_jbyte_disjoint_arraycopy_nopush, &nopush_entry);
+    // conjoint nopush entry is needed by generic/unsafe copy
+    StubRoutines::_jbyte_arraycopy_nopush = nopush_entry;
+    StubRoutines::_arrayof_jbyte_disjoint_arraycopy  = generate_disjoint_copy(StubId::stubgen_arrayof_jbyte_disjoint_arraycopy_id, &nopush_entry);
+    // disjoint arrayof nopush entry is needed by conjoint copy
+    StubRoutines::_arrayof_jbyte_disjoint_arraycopy_nopush  = nopush_entry;
+    StubRoutines::_arrayof_jbyte_arraycopy           = generate_conjoint_copy(StubId::stubgen_arrayof_jbyte_arraycopy_id, StubRoutines::_arrayof_jbyte_disjoint_arraycopy_nopush, nullptr);

    //*** jshort
    // Always need aligned and unaligned versions
-    StubRoutines::_jshort_disjoint_arraycopy         = generate_disjoint_copy(StubId::stubgen_jshort_disjoint_arraycopy_id, &entry);
-    StubRoutines::_jshort_arraycopy                  = generate_conjoint_copy(StubId::stubgen_jshort_arraycopy_id, entry, &entry_jshort_arraycopy);
-    StubRoutines::_arrayof_jshort_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_arrayof_jshort_disjoint_arraycopy_id, &entry);
-    StubRoutines::_arrayof_jshort_arraycopy          = generate_conjoint_copy(StubId::stubgen_arrayof_jshort_arraycopy_id, entry, nullptr);
+    StubRoutines::_jshort_disjoint_arraycopy         = generate_disjoint_copy(StubId::stubgen_jshort_disjoint_arraycopy_id, &nopush_entry);
+    // disjoint nopush entry is needed by conjoint copy
+    StubRoutines::_jshort_disjoint_arraycopy_nopush  = nopush_entry;
+    StubRoutines::_jshort_arraycopy                  = generate_conjoint_copy(StubId::stubgen_jshort_arraycopy_id, StubRoutines::_jshort_disjoint_arraycopy_nopush, &nopush_entry);
+    // conjoint nopush entry is used by generic/unsafe copy
+    StubRoutines::_jshort_arraycopy_nopush = nopush_entry;
+    StubRoutines::_arrayof_jshort_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_arrayof_jshort_disjoint_arraycopy_id, &nopush_entry);
+    // disjoint arrayof nopush entry is needed by conjoint copy
+    StubRoutines::_arrayof_jshort_disjoint_arraycopy_nopush = nopush_entry;
+    StubRoutines::_arrayof_jshort_arraycopy          = generate_conjoint_copy(StubId::stubgen_arrayof_jshort_arraycopy_id, StubRoutines::_arrayof_jshort_disjoint_arraycopy_nopush, nullptr);

    //*** jint
    // Aligned versions
-    StubRoutines::_arrayof_jint_disjoint_arraycopy   = generate_disjoint_copy(StubId::stubgen_arrayof_jint_disjoint_arraycopy_id, &entry);
-    StubRoutines::_arrayof_jint_arraycopy            = generate_conjoint_copy(StubId::stubgen_arrayof_jint_arraycopy_id, entry, &entry_jint_arraycopy);
+    StubRoutines::_arrayof_jint_disjoint_arraycopy   = generate_disjoint_copy(StubId::stubgen_arrayof_jint_disjoint_arraycopy_id, &nopush_entry);
+    // disjoint arrayof nopush entry is needed by conjoint copy
+    StubRoutines::_arrayof_jint_disjoint_arraycopy_nopush = nopush_entry;
+    StubRoutines::_arrayof_jint_arraycopy            = generate_conjoint_copy(StubId::stubgen_arrayof_jint_arraycopy_id, StubRoutines::_arrayof_jint_disjoint_arraycopy_nopush, nullptr);
    // In 64 bit we need both aligned and unaligned versions of jint arraycopy.
    // entry_jint_arraycopy always points to the unaligned version
-    StubRoutines::_jint_disjoint_arraycopy           = generate_disjoint_copy(StubId::stubgen_jint_disjoint_arraycopy_id, &entry);
-    StubRoutines::_jint_arraycopy                  = generate_conjoint_copy(StubId::stubgen_jint_arraycopy_id, entry, &entry_jint_arraycopy);
+    StubRoutines::_jint_disjoint_arraycopy           = generate_disjoint_copy(StubId::stubgen_jint_disjoint_arraycopy_id, &nopush_entry);
+    // disjoint nopush entry is needed by conjoint copy
+    StubRoutines::_jint_disjoint_arraycopy_nopush  = nopush_entry;
+    StubRoutines::_jint_arraycopy                  = generate_conjoint_copy(StubId::stubgen_jint_arraycopy_id, StubRoutines::_jint_disjoint_arraycopy_nopush, &nopush_entry);
+    // conjoint nopush entry is needed by generic/unsafe copy
+    StubRoutines::_jint_arraycopy_nopush = nopush_entry;

    //*** jlong
    // It is always aligned
-    StubRoutines::_arrayof_jlong_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_arrayof_jlong_disjoint_arraycopy_id, &entry);
-    StubRoutines::_arrayof_jlong_arraycopy          = generate_conjoint_copy(StubId::stubgen_arrayof_jlong_arraycopy_id, entry, &entry_jlong_arraycopy);
+    StubRoutines::_arrayof_jlong_disjoint_arraycopy = generate_disjoint_copy(StubId::stubgen_arrayof_jlong_disjoint_arraycopy_id, &nopush_entry);
+    // disjoint arrayof nopush entry is needed by conjoint copy
+    StubRoutines::_arrayof_jlong_disjoint_arraycopy_nopush = nopush_entry;
+    StubRoutines::_arrayof_jlong_arraycopy          = generate_conjoint_copy(StubId::stubgen_arrayof_jlong_arraycopy_id, StubRoutines::_arrayof_jlong_disjoint_arraycopy_nopush, &nopush_entry);
+    // conjoint nopush entry is needed by generic/unsafe copy
+    StubRoutines::_jlong_arraycopy_nopush = nopush_entry;
+    // disjoint normal/nopush and conjoint normal entries are not
+    // generated since the arrayof versions are the same
    StubRoutines::_jlong_disjoint_arraycopy         = StubRoutines::_arrayof_jlong_disjoint_arraycopy;
+    StubRoutines::_jlong_disjoint_arraycopy_nopush = StubRoutines::_arrayof_jlong_disjoint_arraycopy_nopush;
    StubRoutines::_jlong_arraycopy                  = StubRoutines::_arrayof_jlong_arraycopy;

    //*** oops
    StubRoutines::_arrayof_oop_disjoint_arraycopy
-      = generate_disjoint_copy(StubId::stubgen_arrayof_oop_disjoint_arraycopy_id, &entry);
+      = generate_disjoint_copy(StubId::stubgen_arrayof_oop_disjoint_arraycopy_id, &nopush_entry);
+      // disjoint arrayof nopush entry is needed by conjoint copy
+    StubRoutines::_arrayof_oop_disjoint_arraycopy_nopush = nopush_entry;
    StubRoutines::_arrayof_oop_arraycopy
-      = generate_conjoint_copy(StubId::stubgen_arrayof_oop_arraycopy_id, entry, &entry_oop_arraycopy);
+      = generate_conjoint_copy(StubId::stubgen_arrayof_oop_arraycopy_id, StubRoutines::_arrayof_oop_disjoint_arraycopy_nopush, &nopush_entry);
+    // conjoint arrayof nopush entry is needed by generic/unsafe copy
+    StubRoutines::_oop_arraycopy_nopush = nopush_entry;
    // Aligned versions without pre-barriers
    StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit
-      = generate_disjoint_copy(StubId::stubgen_arrayof_oop_disjoint_arraycopy_uninit_id, &entry);
-    StubRoutines::_arrayof_oop_arraycopy_uninit
-      = generate_conjoint_copy(StubId::stubgen_arrayof_oop_arraycopy_uninit_id, entry, nullptr);
+      = generate_disjoint_copy(StubId::stubgen_arrayof_oop_disjoint_arraycopy_uninit_id, &nopush_entry);
+    // disjoint arrayof+uninit nopush entry is needed by conjoint copy
+    StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit_nopush = nopush_entry;

+    // note that we don't need a returned nopush entry because the
+    // generic/unsafe copy does not cater for uninit arrays.
+    StubRoutines::_arrayof_oop_arraycopy_uninit
+      = generate_conjoint_copy(StubId::stubgen_arrayof_oop_arraycopy_uninit_id, StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit_nopush, nullptr);
+
+    // for oop copies reuse arrayof entries for non-arrayof cases
    StubRoutines::_oop_disjoint_arraycopy            = StubRoutines::_arrayof_oop_disjoint_arraycopy;
+    StubRoutines::_oop_disjoint_arraycopy_nopush = StubRoutines::_arrayof_oop_disjoint_arraycopy_nopush;
    StubRoutines::_oop_arraycopy                     = StubRoutines::_arrayof_oop_arraycopy;
    StubRoutines::_oop_disjoint_arraycopy_uninit     = StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit;
+    StubRoutines::_oop_disjoint_arraycopy_uninit_nopush = StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit_nopush;
    StubRoutines::_oop_arraycopy_uninit              = StubRoutines::_arrayof_oop_arraycopy_uninit;

-    StubRoutines::_checkcast_arraycopy        = generate_checkcast_copy(StubId::stubgen_checkcast_arraycopy_id, &entry_checkcast_arraycopy);
+    StubRoutines::_checkcast_arraycopy        = generate_checkcast_copy(StubId::stubgen_checkcast_arraycopy_id, &nopush_entry);
+    // checkcast nopush entry is needed by generic copy
+    StubRoutines::_checkcast_arraycopy_nopush = nopush_entry;
+    // note that we don't need a returned nopush entry because the
+    // generic copy does not cater for uninit arrays.
    StubRoutines::_checkcast_arraycopy_uninit = generate_checkcast_copy(StubId::stubgen_checkcast_arraycopy_uninit_id, nullptr);


-    StubRoutines::_unsafe_arraycopy    = generate_unsafe_copy(entry_jbyte_arraycopy,
-                                                              entry_jshort_arraycopy,
-                                                              entry_jint_arraycopy,
-                                                              entry_jlong_arraycopy);
+    // unsafe arraycopy may fallback on conjoint stubs
+    StubRoutines::_unsafe_arraycopy    = generate_unsafe_copy(StubRoutines::_jbyte_arraycopy_nopush,
+                                                              StubRoutines::_jshort_arraycopy_nopush,
+                                                              StubRoutines::_jint_arraycopy_nopush,
+                                                              StubRoutines::_jlong_arraycopy_nopush);

-    StubRoutines::_generic_arraycopy   = generate_generic_copy(entry_jbyte_arraycopy,
-                                                               entry_jshort_arraycopy,
-                                                               entry_jint_arraycopy,
-                                                               entry_oop_arraycopy,
-                                                               entry_jlong_arraycopy,
-                                                               entry_checkcast_arraycopy);
+    // generic arraycopy may fallback on conjoint stubs
+    StubRoutines::_generic_arraycopy   = generate_generic_copy(StubRoutines::_jbyte_arraycopy_nopush,
+                                                               StubRoutines::_jshort_arraycopy_nopush,
+                                                               StubRoutines::_jint_arraycopy_nopush,
+                                                               StubRoutines::_oop_arraycopy_nopush,
+                                                               StubRoutines::_jlong_arraycopy_nopush,
+                                                               StubRoutines::_checkcast_arraycopy_nopush);

    StubRoutines::_jbyte_fill = generate_fill(StubId::stubgen_jbyte_fill_id);
    StubRoutines::_jshort_fill = generate_fill(StubId::stubgen_jshort_fill_id);
@@ -181,12 +181,13 @@ void VM_Version::common_initialize() {
    FLAG_SET_DEFAULT(UsePopCountInstruction, false);
  }

-  if (UseZicboz) {
+  if (UseZicboz && zicboz_block_size.enabled() && zicboz_block_size.value() > 0) {
+    assert(is_power_of_2(zicboz_block_size.value()), "Sanity");
    if (FLAG_IS_DEFAULT(UseBlockZeroing)) {
      FLAG_SET_DEFAULT(UseBlockZeroing, true);
    }
    if (FLAG_IS_DEFAULT(BlockZeroingLowLimit)) {
-      FLAG_SET_DEFAULT(BlockZeroingLowLimit, 2 * CacheLineSize);
+      FLAG_SET_DEFAULT(BlockZeroingLowLimit, 4 * zicboz_block_size.value());
    }
  } else if (UseBlockZeroing) {
    warning("Block zeroing is not available");
@@ -86,25 +86,27 @@ class VM_Version : public Abstract_VM_Version {
    }                                    \
  }                                      \

-  #define UPDATE_DEFAULT_DEP(flag, dep)    \
-  void update_flag() {                     \
-      assert(enabled(), "Must be.");       \
-      /* dep must be declared before */    \
-      assert((uintptr_t)(this) >           \
-             (uintptr_t)(&dep), "Invalid");\
-      if (FLAG_IS_DEFAULT(flag)) {         \
-        if (dep.enabled()) {               \
-          FLAG_SET_DEFAULT(flag, true);    \
-        } else {                           \
-          FLAG_SET_DEFAULT(flag, false);   \
-        }                                  \
-      } else {                             \
-        /* Sync CPU features with flags */ \
-        if (!flag) {                       \
-          disable_feature();               \
-        }                                  \
-      }                                    \
-  }                                        \
+  #define UPDATE_DEFAULT_DEP(flag, dep)      \
+  void update_flag() {                       \
+      assert(enabled(), "Must be.");         \
+      /* dep must be declared before */      \
+      assert((uintptr_t)(this) >             \
+             (uintptr_t)(&dep), "Invalid");  \
+      if (FLAG_IS_DEFAULT(flag)) {           \
+        if (dep.enabled()) {                 \
+          FLAG_SET_DEFAULT(flag, true);      \
+        } else {                             \
+          FLAG_SET_DEFAULT(flag, false);     \
+          /* Sync CPU features with flags */ \
+          disable_feature();                 \
+        }                                    \
+      } else {                               \
+        /* Sync CPU features with flags */   \
+        if (!flag) {                         \
+          disable_feature();                 \
+        }                                    \
+      }                                      \
+  }                                          \

  #define NO_UPDATE_DEFAULT                \
  void update_flag() {}                    \
@@ -160,45 +162,46 @@ class VM_Version : public Abstract_VM_Version {

  // Note: the order matters, depender should be after their dependee. E.g. ext_V before ext_Zvbb.
  // declaration name  , extension name, bit pos       ,in str, mapped flag)
-  #define RV_FEATURE_FLAGS(decl)                                                                    \
-  decl(ext_I           , "i"           ,    ('I' - 'A'), true , NO_UPDATE_DEFAULT)                  \
-  decl(ext_M           , "m"           ,    ('M' - 'A'), true , NO_UPDATE_DEFAULT)                  \
-  decl(ext_A           , "a"           ,    ('A' - 'A'), true , NO_UPDATE_DEFAULT)                  \
-  decl(ext_F           , "f"           ,    ('F' - 'A'), true , NO_UPDATE_DEFAULT)                  \
-  decl(ext_D           , "d"           ,    ('D' - 'A'), true , NO_UPDATE_DEFAULT)                  \
-  decl(ext_C           , "c"           ,    ('C' - 'A'), true , UPDATE_DEFAULT(UseRVC))             \
-  decl(ext_Q           , "q"           ,    ('Q' - 'A'), true , NO_UPDATE_DEFAULT)                  \
-  decl(ext_H           , "h"           ,    ('H' - 'A'), true , NO_UPDATE_DEFAULT)                  \
-  decl(ext_V           , "v"           ,    ('V' - 'A'), true , UPDATE_DEFAULT(UseRVV))             \
-  decl(ext_Zicbom      , "Zicbom"      , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZicbom))          \
-  decl(ext_Zicboz      , "Zicboz"      , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZicboz))          \
-  decl(ext_Zicbop      , "Zicbop"      , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZicbop))          \
-  decl(ext_Zba         , "Zba"         , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZba))             \
-  decl(ext_Zbb         , "Zbb"         , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZbb))             \
-  decl(ext_Zbc         , "Zbc"         , RV_NO_FLAG_BIT, true , NO_UPDATE_DEFAULT)                  \
-  decl(ext_Zbs         , "Zbs"         , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZbs))             \
-  decl(ext_Zbkb        , "Zbkb"        , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZbkb))            \
-  decl(ext_Zcb         , "Zcb"         , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZcb))             \
-  decl(ext_Zfa         , "Zfa"         , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZfa))             \
-  decl(ext_Zfh         , "Zfh"         , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZfh))             \
-  decl(ext_Zfhmin      , "Zfhmin"      , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZfhmin))          \
-  decl(ext_Zicsr       , "Zicsr"       , RV_NO_FLAG_BIT, true , NO_UPDATE_DEFAULT)                  \
-  decl(ext_Zicntr      , "Zicntr"      , RV_NO_FLAG_BIT, true , NO_UPDATE_DEFAULT)                  \
-  decl(ext_Zifencei    , "Zifencei"    , RV_NO_FLAG_BIT, true , NO_UPDATE_DEFAULT)                  \
-  decl(ext_Zic64b      , "Zic64b"      , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZic64b))          \
-  decl(ext_Ztso        , "Ztso"        , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZtso))            \
-  decl(ext_Zihintpause , "Zihintpause" , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZihintpause))     \
-  decl(ext_Zacas       , "Zacas"       , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZacas))           \
-  decl(ext_Zvbb        , "Zvbb"        , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT_DEP(UseZvbb, ext_V)) \
-  decl(ext_Zvbc        , "Zvbc"        , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT_DEP(UseZvbc, ext_V)) \
-  decl(ext_Zvfh        , "Zvfh"        , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT_DEP(UseZvfh, ext_V)) \
-  decl(ext_Zvkn        , "Zvkn"        , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT_DEP(UseZvkn, ext_V)) \
-  decl(ext_Zicond      , "Zicond"      , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZicond))          \
-  decl(mvendorid       , "VendorId"    , RV_NO_FLAG_BIT, false, NO_UPDATE_DEFAULT)                  \
-  decl(marchid         , "ArchId"      , RV_NO_FLAG_BIT, false, NO_UPDATE_DEFAULT)                  \
-  decl(mimpid          , "ImpId"       , RV_NO_FLAG_BIT, false, NO_UPDATE_DEFAULT)                  \
-  decl(unaligned_access, "Unaligned"   , RV_NO_FLAG_BIT, false, NO_UPDATE_DEFAULT)                  \
-  decl(satp_mode       , "SATP"        , RV_NO_FLAG_BIT, false, NO_UPDATE_DEFAULT)                  \
+  #define RV_FEATURE_FLAGS(decl)                                                                        \
+  decl(ext_I            , "i"              ,    ('I' - 'A'), true , NO_UPDATE_DEFAULT)                  \
+  decl(ext_M            , "m"              ,    ('M' - 'A'), true , NO_UPDATE_DEFAULT)                  \
+  decl(ext_A            , "a"              ,    ('A' - 'A'), true , NO_UPDATE_DEFAULT)                  \
+  decl(ext_F            , "f"              ,    ('F' - 'A'), true , NO_UPDATE_DEFAULT)                  \
+  decl(ext_D            , "d"              ,    ('D' - 'A'), true , NO_UPDATE_DEFAULT)                  \
+  decl(ext_C            , "c"              ,    ('C' - 'A'), true , UPDATE_DEFAULT(UseRVC))             \
+  decl(ext_Q            , "q"              ,    ('Q' - 'A'), true , NO_UPDATE_DEFAULT)                  \
+  decl(ext_H            , "h"              ,    ('H' - 'A'), true , NO_UPDATE_DEFAULT)                  \
+  decl(ext_V            , "v"              ,    ('V' - 'A'), true , UPDATE_DEFAULT(UseRVV))             \
+  decl(ext_Zicbom       , "Zicbom"         , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZicbom))          \
+  decl(ext_Zicboz       , "Zicboz"         , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZicboz))          \
+  decl(ext_Zicbop       , "Zicbop"         , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZicbop))          \
+  decl(ext_Zba          , "Zba"            , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZba))             \
+  decl(ext_Zbb          , "Zbb"            , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZbb))             \
+  decl(ext_Zbc          , "Zbc"            , RV_NO_FLAG_BIT, true , NO_UPDATE_DEFAULT)                  \
+  decl(ext_Zbs          , "Zbs"            , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZbs))             \
+  decl(ext_Zbkb         , "Zbkb"           , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZbkb))            \
+  decl(ext_Zcb          , "Zcb"            , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZcb))             \
+  decl(ext_Zfa          , "Zfa"            , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZfa))             \
+  decl(ext_Zfh          , "Zfh"            , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZfh))             \
+  decl(ext_Zfhmin       , "Zfhmin"         , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZfhmin))          \
+  decl(ext_Zicsr        , "Zicsr"          , RV_NO_FLAG_BIT, true , NO_UPDATE_DEFAULT)                  \
+  decl(ext_Zicntr       , "Zicntr"         , RV_NO_FLAG_BIT, true , NO_UPDATE_DEFAULT)                  \
+  decl(ext_Zifencei     , "Zifencei"       , RV_NO_FLAG_BIT, true , NO_UPDATE_DEFAULT)                  \
+  decl(ext_Zic64b       , "Zic64b"         , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZic64b))          \
+  decl(ext_Ztso         , "Ztso"           , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZtso))            \
+  decl(ext_Zihintpause  , "Zihintpause"    , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZihintpause))     \
+  decl(ext_Zacas        , "Zacas"          , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZacas))           \
+  decl(ext_Zvbb         , "Zvbb"           , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT_DEP(UseZvbb, ext_V)) \
+  decl(ext_Zvbc         , "Zvbc"           , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT_DEP(UseZvbc, ext_V)) \
+  decl(ext_Zvfh         , "Zvfh"           , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT_DEP(UseZvfh, ext_V)) \
+  decl(ext_Zvkn         , "Zvkn"           , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT_DEP(UseZvkn, ext_V)) \
+  decl(ext_Zicond       , "Zicond"         , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZicond))          \
+  decl(mvendorid        , "VendorId"       , RV_NO_FLAG_BIT, false, NO_UPDATE_DEFAULT)                  \
+  decl(marchid          , "ArchId"         , RV_NO_FLAG_BIT, false, NO_UPDATE_DEFAULT)                  \
+  decl(mimpid           , "ImpId"          , RV_NO_FLAG_BIT, false, NO_UPDATE_DEFAULT)                  \
+  decl(unaligned_access , "Unaligned"      , RV_NO_FLAG_BIT, false, NO_UPDATE_DEFAULT)                  \
+  decl(satp_mode        , "SATP"           , RV_NO_FLAG_BIT, false, NO_UPDATE_DEFAULT)                  \
+  decl(zicboz_block_size, "ZicbozBlockSize", RV_NO_FLAG_BIT, false, NO_UPDATE_DEFAULT)                  \

  #define DECLARE_RV_FEATURE(NAME, PRETTY, BIT, FSTRING, FLAGF)        \
  struct NAME##RVFeatureValue : public RVFeatureValue {                \
@@ -26,7 +26,7 @@
 #ifndef CPU_S390_ASSEMBLER_S390_INLINE_HPP
 #define CPU_S390_ASSEMBLER_S390_INLINE_HPP

-#include "asm/assembler.inline.hpp"
+#include "asm/assembler.hpp"
 #include "asm/codeBuffer.hpp"
 #include "code/codeCache.hpp"

@@ -2816,10 +2816,6 @@ void LIR_Assembler::align_backward_branch_target() {
  __ align(OptoLoopAlignment);
 }

-void LIR_Assembler::emit_delay(LIR_OpDelay* op) {
-  ShouldNotCallThis(); // There are no delay slots on ZARCH_64.
-}
-
 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
  // tmp must be unused
  assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
@@ -171,6 +171,7 @@ void BarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Re

 void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm) {
  BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
+  __ align(4, __ offset() + OFFSET_TO_PATCHABLE_DATA); // must align the following block which requires atomic updates
  __ block_comment("nmethod_entry_barrier (nmethod_entry_barrier) {");

    // Load jump addr:
@@ -66,6 +66,14 @@ public:
  OptoReg::Name refine_register(const Node* node,
                                OptoReg::Name opto_reg) const;
 #endif // COMPILER2
+
+  static const int OFFSET_TO_PATCHABLE_DATA_INSTRUCTION = 6 + 6 + 6; // iihf(6) + iilf(6) + lg(6)
+  static const int BARRIER_TOTAL_LENGTH = OFFSET_TO_PATCHABLE_DATA_INSTRUCTION + 6 + 6 + 2; // cfi(6) + larl(6) + bcr(2)
+
+  // first 2 bytes are for cfi instruction opcode and next 4 bytes will be the value/data to be patched,
+  // so we are skipping first 2 bytes and returning the address of value/data field
+  static const int OFFSET_TO_PATCHABLE_DATA = 6 + 6 + 6 + 2; // iihf(6) + iilf(6) + lg(6) + CFI_OPCODE(2)
+
 };

 #ifdef COMPILER2
@@ -26,26 +26,32 @@
 #include "code/codeBlob.hpp"
 #include "code/nativeInst.hpp"
 #include "code/nmethod.hpp"
+#include "gc/shared/barrierSetAssembler.hpp"
 #include "gc/shared/barrierSetNMethod.hpp"
 #include "utilities/debug.hpp"

 class NativeMethodBarrier: public NativeInstruction {
  private:
-    static const int PATCHABLE_INSTRUCTION_OFFSET = 3*6; // bytes

    address get_barrier_start_address() const {
      return NativeInstruction::addr_at(0);
    }

    address get_patchable_data_address() const {
-      address inst_addr = get_barrier_start_address() + PATCHABLE_INSTRUCTION_OFFSET;
+      address start_address = get_barrier_start_address();
+#ifdef ASSERT
+      address inst_addr = start_address + BarrierSetAssembler::OFFSET_TO_PATCHABLE_DATA_INSTRUCTION;

-      DEBUG_ONLY(Assembler::is_z_cfi(*((long*)inst_addr)));
-      return inst_addr + 2;
+      unsigned long instr = 0;
+      Assembler::get_instruction(inst_addr, &instr);
+      assert(Assembler::is_z_cfi(instr), "sanity check");
+#endif // ASSERT
+
+      return start_address + BarrierSetAssembler::OFFSET_TO_PATCHABLE_DATA;
    }

  public:
-    static const int BARRIER_TOTAL_LENGTH = PATCHABLE_INSTRUCTION_OFFSET + 2*6 + 2; // bytes
+    static const int BARRIER_TOTAL_LENGTH = BarrierSetAssembler::BARRIER_TOTAL_LENGTH;

    int get_guard_value() const {
      address data_addr = get_patchable_data_address();
@@ -53,32 +59,54 @@ class NativeMethodBarrier: public NativeInstruction {
      return *((int32_t*)data_addr);
    }

-    void set_guard_value(int value) {
-      int32_t* data_addr = (int32_t*)get_patchable_data_address();
+    void set_guard_value(int value, int bit_mask) {
+      if (bit_mask == ~0) {
+        int32_t* data_addr = (int32_t*)get_patchable_data_address();

-      // Set guard instruction value
-      *data_addr = value;
+        // Set guard instruction value
+        *data_addr = value;
+        return;
+      }
+      assert((value & ~bit_mask) == 0, "trying to set bits outside the mask");
+      value &= bit_mask;
+      int32_t* data_addr = (int32_t*)get_patchable_data_address();
+      int old_value = AtomicAccess::load(data_addr);
+      while (true) {
+        // Only bits in the mask are changed
+        int new_value = value | (old_value & ~bit_mask);
+        if (new_value == old_value) break;
+        int v = AtomicAccess::cmpxchg(data_addr, old_value, new_value, memory_order_release);
+        if (v == old_value) break;
+        old_value = v;
+      }
    }

    #ifdef ASSERT
      void verify() const {
+        unsigned long instr = 0;
        int offset = 0; // bytes
        const address start = get_barrier_start_address();

-        MacroAssembler::is_load_const(/* address */ start + offset); // two instructions
+        assert(MacroAssembler::is_load_const(/* address */ start + offset), "sanity check"); // two instructions
        offset += Assembler::instr_len(&start[offset]);
        offset += Assembler::instr_len(&start[offset]);

-        Assembler::is_z_lg(*((long*)(start + offset)));
+        Assembler::get_instruction(start + offset, &instr);
+        assert(Assembler::is_z_lg(instr), "sanity check");
        offset += Assembler::instr_len(&start[offset]);

-        Assembler::is_z_cfi(*((long*)(start + offset)));
+        // it will be assignment operation, So it doesn't matter what value is already present in instr
+        // hence, no need to 0 it out.
+        Assembler::get_instruction(start + offset, &instr);
+        assert(Assembler::is_z_cfi(instr), "sanity check");
        offset += Assembler::instr_len(&start[offset]);

-        Assembler::is_z_larl(*((long*)(start + offset)));
+        Assembler::get_instruction(start + offset, &instr);
+        assert(Assembler::is_z_larl(instr), "sanity check");
        offset += Assembler::instr_len(&start[offset]);

-        Assembler::is_z_bcr(*((long*)(start + offset)));
+        Assembler::get_instruction(start + offset, &instr);
+        assert(Assembler::is_z_bcr(instr), "sanity check");
        offset += Assembler::instr_len(&start[offset]);

        assert(offset == BARRIER_TOTAL_LENGTH, "check offset == barrier length constant");
@@ -100,13 +128,13 @@ void BarrierSetNMethod::deoptimize(nmethod* nm, address* return_address_ptr) {
  return;
 }

-void BarrierSetNMethod::set_guard_value(nmethod* nm, int value) {
+void BarrierSetNMethod::set_guard_value(nmethod* nm, int value, int bit_mask) {
  if (!supports_entry_barrier(nm)) {
    return;
  }

  NativeMethodBarrier* barrier = get_nmethod_barrier(nm);
-  barrier->set_guard_value(value);
+  barrier->set_guard_value(value, bit_mask);
 }

 int BarrierSetNMethod::guard_value(nmethod* nm) {
@@ -1398,11 +1398,7 @@ void Assembler::addl(Address dst, Register src) {

 void Assembler::eaddl(Register dst, Address src1, Register src2, bool no_flags) {
  InstructionMark im(this);
-  InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
-  attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
-  eevex_prefix_ndd(src1, dst->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
-  emit_int8(0x01);
-  emit_operand(src2, src1, 0);
+  emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x01, no_flags, false /* is_map1 */, true /* is_commutative */);
 }

 void Assembler::addl(Register dst, int32_t imm32) {
@@ -1432,11 +1428,7 @@ void Assembler::addl(Register dst, Register src) {
 }

 void Assembler::eaddl(Register dst, Register src1, Register src2, bool no_flags) {
-  InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
-  // NDD shares its encoding bits with NDS bits for regular EVEX instruction.
-  // Therefore, DST is passed as the second argument to minimize changes in the leaf level routine.
-  (void)emit_eevex_prefix_or_demote_ndd(src1->encoding(), dst->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
-  emit_arith(0x03, 0xC0, src1, src2);
+  emit_eevex_prefix_or_demote_arith_ndd(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x03, 0xC0, no_flags, true /* is_commutative */);
 }

 void Assembler::addr_nop_4() {
@@ -1657,17 +1649,18 @@ void Assembler::eandl(Register dst, Register src1, Address src2, bool no_flags)
  emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x23, no_flags);
 }

+void Assembler::eandl(Register dst, Address src1, Register src2, bool no_flags) {
+  InstructionMark im(this);
+  emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x21, no_flags, false /* is_map1 */, true /* is_commutative */);
+}
+
 void Assembler::andl(Register dst, Register src) {
  (void) prefix_and_encode(dst->encoding(), src->encoding());
  emit_arith(0x23, 0xC0, dst, src);
 }

 void Assembler::eandl(Register dst, Register src1, Register src2, bool no_flags) {
-  InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
-  // NDD shares its encoding bits with NDS bits for regular EVEX instruction.
-  // Therefore, DST is passed as the second argument to minimize changes in the leaf level routine.
-  (void) emit_eevex_prefix_or_demote_ndd(src1->encoding(), dst->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
-  emit_arith(0x23, 0xC0, src1, src2);
+  emit_eevex_prefix_or_demote_arith_ndd(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x23, 0xC0, no_flags, true /* is_commutative */);
 }

 void Assembler::andnl(Register dst, Register src1, Register src2) {
@@ -2519,7 +2512,7 @@ void Assembler::imull(Register dst, Register src) {
 }

 void Assembler::eimull(Register dst, Register src1, Register src2, bool no_flags) {
-  emit_eevex_or_demote(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0xAF, no_flags, true /* is_map1 */,  true /* swap */);
+  emit_eevex_or_demote(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0xAF, no_flags, true /* is_map1 */,  true /* swap */, true /* is_commutative */);
 }

 void Assembler::imull(Register dst, Address src, int32_t value) {
@@ -4419,11 +4412,7 @@ void Assembler::enotl(Register dst, Register src) {
 }

 void Assembler::eorw(Register dst, Register src1, Register src2, bool no_flags) {
-  InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
-  // NDD shares its encoding bits with NDS bits for regular EVEX instruction.
-  // Therefore, DST is passed as the second argument to minimize changes in the leaf level routine.
-  (void) emit_eevex_prefix_or_demote_ndd(src1->encoding(), dst->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
-  emit_arith(0x0B, 0xC0, src1, src2);
+  emit_eevex_prefix_or_demote_arith_ndd(dst, src1, src2, VEX_SIMD_66, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_16bit, 0x0B, 0xC0, no_flags, true /* is_commutative */);
 }

 void Assembler::orl(Address dst, int32_t imm32) {
@@ -4467,11 +4456,7 @@ void Assembler::orl(Register dst, Register src) {
 }

 void Assembler::eorl(Register dst, Register src1, Register src2, bool no_flags) {
-  InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
-  // NDD shares its encoding bits with NDS bits for regular EVEX instruction.
-  // Therefore, DST is passed as the second argument to minimize changes in the leaf level routine.
-  (void) emit_eevex_prefix_or_demote_ndd(src1->encoding(), dst->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
-  emit_arith(0x0B, 0xC0, src1, src2);
+  emit_eevex_prefix_or_demote_arith_ndd(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x0B, 0xC0, no_flags, true /* is_commutative */);
 }

 void Assembler::orl(Address dst, Register src) {
@@ -4483,11 +4468,7 @@ void Assembler::orl(Address dst, Register src) {

 void Assembler::eorl(Register dst, Address src1, Register src2, bool no_flags) {
  InstructionMark im(this);
-  InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
-  attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
-  eevex_prefix_ndd(src1, dst->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
-  emit_int8(0x09);
-  emit_operand(src2, src1, 0);
+  emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x09, no_flags, false /* is_map1 */, true /* is_commutative */);
 }

 void Assembler::orb(Address dst, int imm8) {
@@ -4517,11 +4498,7 @@ void Assembler::orb(Address dst, Register src) {

 void Assembler::eorb(Register dst, Address src1, Register src2, bool no_flags) {
  InstructionMark im(this);
-  InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
-  attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_8bit);
-  eevex_prefix_ndd(src1, dst->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
-  emit_int8(0x08);
-  emit_operand(src2, src1, 0);
+  emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_8bit, 0x08, no_flags, false /* is_map1 */, true /* is_commutative */);
 }

 void Assembler::packsswb(XMMRegister dst, XMMRegister src) {
@@ -7323,11 +7300,7 @@ void Assembler::xorl(Register dst, Register src) {
 }

 void Assembler::exorl(Register dst, Register src1, Register src2, bool no_flags) {
-  InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
-  // NDD shares its encoding bits with NDS bits for regular EVEX instruction.
-  // Therefore, DST is passed as the second argument to minimize changes in the leaf level routine.
-  (void) emit_eevex_prefix_or_demote_ndd(src1->encoding(), dst->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
-  emit_arith(0x33, 0xC0, src1, src2);
+  emit_eevex_prefix_or_demote_arith_ndd(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x33, 0xC0, no_flags, true /* is_commutative */);
 }

 void Assembler::xorl(Address dst, Register src) {
@@ -7339,11 +7312,7 @@ void Assembler::xorl(Address dst, Register src) {

 void Assembler::exorl(Register dst, Address src1, Register src2, bool no_flags) {
  InstructionMark im(this);
-  InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
-  attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
-  eevex_prefix_ndd(src1, dst->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
-  emit_int8(0x31);
-  emit_operand(src2, src1, 0);
+  emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_32bit, 0x31, no_flags, false /* is_map1 */, true /* is_commutative */);
 }

 void Assembler::xorb(Register dst, Address src) {
@@ -7367,11 +7336,7 @@ void Assembler::xorb(Address dst, Register src) {

 void Assembler::exorb(Register dst, Address src1, Register src2, bool no_flags) {
  InstructionMark im(this);
-  InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
-  attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_8bit);
-  eevex_prefix_ndd(src1, dst->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
-  emit_int8(0x30);
-  emit_operand(src2, src1, 0);
+  emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_8bit, 0x30, no_flags, false /* is_map1 */, true /* is_commutative */);
 }

 void Assembler::xorw(Register dst, Address src) {
@@ -12955,6 +12920,31 @@ void Assembler::eevex_prefix_ndd(Address adr, int ndd_enc, int xreg_enc, VexSimd
  vex_prefix(adr, ndd_enc, xreg_enc, pre, opc, attributes, /* nds_is_ndd */ true, no_flags);
 }

+void Assembler::emit_eevex_or_demote(Register dst, Address src1, Register src2, VexSimdPrefix pre, VexOpcode opc,
+                                     int size, int opcode_byte, bool no_flags, bool is_map1, bool is_commutative) {
+  if (is_commutative && is_demotable(no_flags, dst->encoding(), src2->encoding())) {
+    // Opcode byte adjustment due to mismatch between NDD and equivalent demotable variant
+    opcode_byte += 2;
+    if (size == EVEX_64bit) {
+      emit_prefix_and_int8(get_prefixq(src1, dst, is_map1), opcode_byte);
+    } else {
+      // For 32-bit, 16-bit and 8-bit
+      if (size == EVEX_16bit) {
+        emit_int8(0x66);
+      }
+      prefix(src1, dst, false, is_map1);
+      emit_int8(opcode_byte);
+    }
+  } else {
+    bool vex_w = (size == EVEX_64bit) ? true : false;
+    InstructionAttr attributes(AVX_128bit, vex_w, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
+    attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, size);
+    eevex_prefix_ndd(src1, dst->encoding(), src2->encoding(), pre, opc, &attributes, no_flags);
+    emit_int8(opcode_byte);
+  }
+  emit_operand(src2, src1, 0);
+}
+
 void Assembler::emit_eevex_or_demote(Register dst, Register src1, Address src2, VexSimdPrefix pre, VexOpcode opc,
                                     int size, int opcode_byte, bool no_flags, bool is_map1) {
  if (is_demotable(no_flags, dst->encoding(), src1->encoding())) {
@@ -13055,18 +13045,20 @@ void Assembler::emit_eevex_or_demote(int dst_enc, int nds_enc, int src_enc, int8
 }

 void Assembler::emit_eevex_or_demote(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc,
-                                     int size, int opcode_byte, bool no_flags, bool is_map1, bool swap) {
+                                     int size, int opcode_byte, bool no_flags, bool is_map1, bool swap, bool is_commutative) {
  int encode;
  bool is_prefixq = (size == EVEX_64bit) ? true : false;
-  if (is_demotable(no_flags, dst_enc, nds_enc)) {
+  bool first_operand_demotable = is_demotable(no_flags, dst_enc, nds_enc);
+  bool second_operand_demotable = is_commutative && is_demotable(no_flags, dst_enc, src_enc);
+  if (first_operand_demotable || second_operand_demotable) {
    if (size == EVEX_16bit) {
      emit_int8(0x66);
    }
-
+    int src = first_operand_demotable ? src_enc : nds_enc;
    if (swap) {
-      encode = is_prefixq ? prefixq_and_encode(dst_enc, src_enc, is_map1) : prefix_and_encode(dst_enc, src_enc, is_map1);
+      encode = is_prefixq ? prefixq_and_encode(dst_enc, src, is_map1) : prefix_and_encode(dst_enc, src, is_map1);
    } else {
-      encode = is_prefixq ? prefixq_and_encode(src_enc, dst_enc, is_map1) : prefix_and_encode(src_enc, dst_enc, is_map1);
+      encode = is_prefixq ? prefixq_and_encode(src, dst_enc, is_map1) : prefix_and_encode(src, dst_enc, is_map1);
    }
    emit_opcode_prefix_and_encoding((unsigned char)opcode_byte, 0xC0, encode);
  } else {
@@ -13114,6 +13106,26 @@ int Assembler::eevex_prefix_and_encode_nf(int dst_enc, int nds_enc, int src_enc,
  return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, attributes, /* src_is_gpr */ true, /* nds_is_ndd */ false, no_flags);
 }

+void Assembler::emit_eevex_prefix_or_demote_arith_ndd(Register dst, Register src1, Register src2, VexSimdPrefix pre, VexOpcode opc,
+                                                      int size, int op1, int op2, bool no_flags, bool is_commutative) {
+  bool demotable = is_demotable(no_flags, dst->encoding(), src1->encoding());
+  if (!demotable && is_commutative) {
+    if (is_demotable(no_flags, dst->encoding(), src2->encoding())) {
+      // swap src1 and src2
+      Register tmp = src1;
+      src1 = src2;
+      src2 = tmp;
+    }
+  }
+  bool vex_w = (size == EVEX_64bit) ? true : false;
+  bool use_prefixq = vex_w;
+  InstructionAttr attributes(AVX_128bit, vex_w, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
+  // NDD shares its encoding bits with NDS bits for regular EVEX instruction.
+  // Therefore, DST is passed as the second argument to minimize changes in the leaf level routine.
+  (void)emit_eevex_prefix_or_demote_ndd(src1->encoding(), dst->encoding(), src2->encoding(), pre, opc, &attributes, no_flags, use_prefixq);
+  emit_arith(op1, op2, src1, src2);
+}
+
 void Assembler::emit_eevex_prefix_or_demote_arith_ndd(Register dst, Register nds, int32_t imm32, VexSimdPrefix pre, VexOpcode opc,
                                                      int size, int op1, int op2, bool no_flags) {
  int dst_enc = dst->encoding();
@@ -13124,7 +13136,6 @@ void Assembler::emit_eevex_prefix_or_demote_arith_ndd(Register dst, Register nds
  } else {
    bool vex_w = (size == EVEX_64bit) ? true : false;
    InstructionAttr attributes(AVX_128bit, vex_w, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
-    //attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, size);
    attributes.set_is_evex_instruction();
    vex_prefix_and_encode(0, dst_enc, nds_enc, pre, opc, &attributes, /* src_is_gpr */ true, /* nds_is_ndd */ true, no_flags);

@@ -13769,7 +13780,7 @@ void Assembler::pdepq(Register dst, Register src1, Address src2) {

 void Assembler::sarxl(Register dst, Register src1, Register src2) {
  assert(VM_Version::supports_bmi2(), "");
-  InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+  InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
  int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes, true);
  emit_int16((unsigned char)0xF7, (0xC0 | encode));
 }
@@ -13777,7 +13788,7 @@ void Assembler::sarxl(Register dst, Register src1, Register src2) {
 void Assembler::sarxl(Register dst, Address src1, Register src2) {
  assert(VM_Version::supports_bmi2(), "");
  InstructionMark im(this);
-  InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+  InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
  attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
  vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
  emit_int8((unsigned char)0xF7);
@@ -13786,7 +13797,7 @@ void Assembler::sarxl(Register dst, Address src1, Register src2) {

 void Assembler::sarxq(Register dst, Register src1, Register src2) {
  assert(VM_Version::supports_bmi2(), "");
-  InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+  InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
  int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes, true);
  emit_int16((unsigned char)0xF7, (0xC0 | encode));
 }
@@ -13794,7 +13805,7 @@ void Assembler::sarxq(Register dst, Register src1, Register src2) {
 void Assembler::sarxq(Register dst, Address src1, Register src2) {
  assert(VM_Version::supports_bmi2(), "");
  InstructionMark im(this);
-  InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+  InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
  attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit);
  vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
  emit_int8((unsigned char)0xF7);
@@ -13803,7 +13814,7 @@ void Assembler::sarxq(Register dst, Address src1, Register src2) {

 void Assembler::shlxl(Register dst, Register src1, Register src2) {
  assert(VM_Version::supports_bmi2(), "");
-  InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+  InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
  int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes, true);
  emit_int16((unsigned char)0xF7, (0xC0 | encode));
 }
@@ -13811,7 +13822,7 @@ void Assembler::shlxl(Register dst, Register src1, Register src2) {
 void Assembler::shlxl(Register dst, Address src1, Register src2) {
  assert(VM_Version::supports_bmi2(), "");
  InstructionMark im(this);
-  InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+  InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
  attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
  vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
  emit_int8((unsigned char)0xF7);
@@ -13820,7 +13831,7 @@ void Assembler::shlxl(Register dst, Address src1, Register src2) {

 void Assembler::shlxq(Register dst, Register src1, Register src2) {
  assert(VM_Version::supports_bmi2(), "");
-  InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+  InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
  int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes, true);
  emit_int16((unsigned char)0xF7, (0xC0 | encode));
 }
@@ -13828,7 +13839,7 @@ void Assembler::shlxq(Register dst, Register src1, Register src2) {
 void Assembler::shlxq(Register dst, Address src1, Register src2) {
  assert(VM_Version::supports_bmi2(), "");
  InstructionMark im(this);
-  InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+  InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
  attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit);
  vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
  emit_int8((unsigned char)0xF7);
@@ -13837,7 +13848,7 @@ void Assembler::shlxq(Register dst, Address src1, Register src2) {

 void Assembler::shrxl(Register dst, Register src1, Register src2) {
  assert(VM_Version::supports_bmi2(), "");
-  InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+  InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
  int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes, true);
  emit_int16((unsigned char)0xF7, (0xC0 | encode));
 }
@@ -13845,7 +13856,7 @@ void Assembler::shrxl(Register dst, Register src1, Register src2) {
 void Assembler::shrxl(Register dst, Address src1, Register src2) {
  assert(VM_Version::supports_bmi2(), "");
  InstructionMark im(this);
-  InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+  InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
  attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_32bit);
  vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
  emit_int8((unsigned char)0xF7);
@@ -13854,7 +13865,7 @@ void Assembler::shrxl(Register dst, Address src1, Register src2) {

 void Assembler::shrxq(Register dst, Register src1, Register src2) {
  assert(VM_Version::supports_bmi2(), "");
-  InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+  InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
  int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes, true);
  emit_int16((unsigned char)0xF7, (0xC0 | encode));
 }
@@ -13862,7 +13873,7 @@ void Assembler::shrxq(Register dst, Register src1, Register src2) {
 void Assembler::shrxq(Register dst, Address src1, Register src2) {
  assert(VM_Version::supports_bmi2(), "");
  InstructionMark im(this);
-  InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
+  InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
  attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit);
  vex_prefix(src1, src2->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
  emit_int8((unsigned char)0xF7);
@@ -14623,11 +14634,7 @@ void Assembler::addq(Address dst, Register src) {

 void Assembler::eaddq(Register dst, Address src1, Register src2, bool no_flags) {
  InstructionMark im(this);
-  InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
-  attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit);
-  eevex_prefix_ndd(src1, dst->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
-  emit_int8(0x01);
-  emit_operand(src2, src1, 0);
+  emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x01, no_flags, false /* is_map1 */, true /* is_commutative */);
 }

 void Assembler::addq(Register dst, int32_t imm32) {
@@ -14656,11 +14663,7 @@ void Assembler::addq(Register dst, Register src) {
 }

 void Assembler::eaddq(Register dst, Register src1, Register src2, bool no_flags) {
-  InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
-  // NDD shares its encoding bits with NDS bits for regular EVEX instruction.
-  // Therefore, DST is passed as the second argument to minimize changes in the leaf level routine.
-  (void) emit_eevex_prefix_or_demote_ndd(src1->encoding(), dst->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags, true /* use_prefixq */);
-  emit_arith(0x03, 0xC0, src1, src2);
+  emit_eevex_prefix_or_demote_arith_ndd(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x03, 0xC0, no_flags, true /* is_commutative */);
 }

 void Assembler::adcxq(Register dst, Register src) {
@@ -14753,11 +14756,7 @@ void Assembler::andq(Register dst, Register src) {
 }

 void Assembler::eandq(Register dst, Register src1, Register src2, bool no_flags) {
-  InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
-  // NDD shares its encoding bits with NDS bits for regular EVEX instruction.
-  // Therefore, DST is passed as the second argument to minimize changes in the leaf level routine.
-  (void) emit_eevex_prefix_or_demote_ndd(src1->encoding(), dst->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags, true /* use_prefixq */);
-  emit_arith(0x23, 0xC0, src1, src2);
+  emit_eevex_prefix_or_demote_arith_ndd(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x23, 0xC0, no_flags, true /* is_commutative */);
 }

 void Assembler::andq(Address dst, Register src) {
@@ -14768,11 +14767,7 @@ void Assembler::andq(Address dst, Register src) {

 void Assembler::eandq(Register dst, Address src1, Register src2, bool no_flags) {
  InstructionMark im(this);
-  InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
-  attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit);
-  eevex_prefix_ndd(src1, dst->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
-  emit_int8(0x21);
-  emit_operand(src2, src1, 0);
+  emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x21, no_flags, false /* is_map1 */, true /* is_commutative */);
 }

 void Assembler::andnq(Register dst, Register src1, Register src2) {
@@ -15118,7 +15113,7 @@ void Assembler::eimulq(Register dst, Register src, bool no_flags) {
 }

 void Assembler::eimulq(Register dst, Register src1, Register src2, bool no_flags) {
-  emit_eevex_or_demote(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0xAF, no_flags, true /* is_map1 */,  true /* swap */);
+  emit_eevex_or_demote(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0xAF, no_flags, true /* is_map1 */,  true /* swap */, true /* is_commutative */);
 }

 void Assembler::imulq(Register src) {
@@ -15580,11 +15575,7 @@ void Assembler::orq(Address dst, Register src) {

 void Assembler::eorq(Register dst, Address src1, Register src2, bool no_flags) {
  InstructionMark im(this);
-  InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
-  attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit);
-  eevex_prefix_ndd(src1, dst->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
-  emit_int8(0x09);
-  emit_operand(src2, src1, 0);
+  emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x09, no_flags, false /* is_map1 */, true /* is_commutative */);
 }

 void Assembler::orq(Register dst, int32_t imm32) {
@@ -15624,13 +15615,8 @@ void Assembler::orq(Register dst, Register src) {
 }

 void Assembler::eorq(Register dst, Register src1, Register src2, bool no_flags) {
-  InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
-  // NDD shares its encoding bits with NDS bits for regular EVEX instruction.
-  // Therefore, DST is passed as the second argument to minimize changes in the leaf level routine.
-  (void) emit_eevex_prefix_or_demote_ndd(src1->encoding(), dst->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags, true /* use_prefixq */);
-  emit_arith(0x0B, 0xC0, src1, src2);
+  emit_eevex_prefix_or_demote_arith_ndd(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x0B, 0xC0, no_flags, true /* is_commutative */);
 }
-
 void Assembler::popcntq(Register dst, Address src) {
  assert(VM_Version::supports_popcnt(), "must support");
  InstructionMark im(this);
@@ -16372,11 +16358,7 @@ void Assembler::xorq(Register dst, Register src) {
 }

 void Assembler::exorq(Register dst, Register src1, Register src2, bool no_flags) {
-  InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
-  // NDD shares its encoding bits with NDS bits for regular EVEX instruction.
-  // Therefore, DST is passed as the second argument to minimize changes in the leaf level routine.
-  (void) emit_eevex_prefix_or_demote_ndd(src1->encoding(), dst->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags, true /* use_prefixq */);
-  emit_arith(0x33, 0xC0, src1, src2);
+  emit_eevex_prefix_or_demote_arith_ndd(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x33, 0xC0, no_flags, true /* is_commutative */);
 }

 void Assembler::xorq(Register dst, Address src) {
@@ -16430,11 +16412,7 @@ void Assembler::esetzucc(Condition cc, Register dst) {

 void Assembler::exorq(Register dst, Address src1, Register src2, bool no_flags) {
  InstructionMark im(this);
-  InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
-  attributes.set_address_attributes(/* tuple_type */ EVEX_NOSCALE, /* input_size_in_bits */ EVEX_64bit);
-  eevex_prefix_ndd(src1, dst->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, &attributes, no_flags);
-  emit_int8(0x31);
-  emit_operand(src2, src1, 0);
+  emit_eevex_or_demote(dst, src1, src2, VEX_SIMD_NONE, VEX_OPCODE_0F_3C /* MAP4 */, EVEX_64bit, 0x31, no_flags, false /* is_map1 */, true /* is_commutative */);
 }

 void InstructionAttr::set_address_attributes(int tuple_type, int input_size_in_bits) {
@@ -807,14 +807,20 @@ private:
  int emit_eevex_prefix_or_demote_ndd(int dst_enc, int nds_enc, VexSimdPrefix pre, VexOpcode opc,
                                      InstructionAttr *attributes, bool no_flags = false, bool use_prefixq = false);

+  void emit_eevex_prefix_or_demote_arith_ndd(Register dst, Register src1, Register src2, VexSimdPrefix pre, VexOpcode opc,
+                                             int size, int op1, int op2, bool no_flags = false, bool is_commutative = false);
+
  void emit_eevex_prefix_or_demote_arith_ndd(Register dst, Register nds, int32_t imm32, VexSimdPrefix pre, VexOpcode opc,
                                             int size, int op1, int op2, bool no_flags);

  void emit_eevex_or_demote(Register dst, Register src1, Address src2, VexSimdPrefix pre, VexOpcode opc,
                            int size, int opcode_byte, bool no_flags = false, bool is_map1 = false);

+  void emit_eevex_or_demote(Register dst, Address src1, Register src2, VexSimdPrefix pre, VexOpcode opc,
+                            int size, int opcode_byte, bool no_flags = false, bool is_map1 = false, bool is_commutative = false);
+
  void emit_eevex_or_demote(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc,
-                            int size, int opcode_byte, bool no_flags, bool is_map1 = false, bool swap = false);
+                            int size, int opcode_byte, bool no_flags, bool is_map1 = false, bool swap = false, bool is_commutative = false);

  void emit_eevex_or_demote(int dst_enc, int nds_enc, int src_enc, int8_t imm8, VexSimdPrefix pre, VexOpcode opc,
                            int size, int opcode_byte, bool no_flags, bool is_map1 = false);
@@ -1149,6 +1155,7 @@ private:
  void eandl(Register dst, Register src, int32_t imm32, bool no_flags);
  void andl(Register dst, Address src);
  void eandl(Register dst, Register src1, Address src2, bool no_flags);
+  void eandl(Register dst, Address src1, Register src2, bool no_flags);
  void andl(Register dst, Register src);
  void eandl(Register dst, Register src1, Register src2, bool no_flags);
  void andl(Address dst, Register src);
@@ -3001,11 +3001,6 @@ void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
  __ bind(next);
 }

-void LIR_Assembler::emit_delay(LIR_OpDelay*) {
-  Unimplemented();
-}
-
-
 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
  __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
 }
@@ -50,8 +50,31 @@ public:
  address instruction_address() const { return addr_at(0); }
  address immediate_address() const { return addr_at(imm_offset); }

+  NativeNMethodCmpBarrier* nativeNMethodCmpBarrier_at(address a) { return (NativeNMethodCmpBarrier*)a; }
+
  jint get_immediate() const { return int_at(imm_offset); }
-  void set_immediate(jint imm) { set_int_at(imm_offset, imm); }
+  void set_immediate(jint imm, int bit_mask) {
+    if (bit_mask == ~0) {
+      set_int_at(imm_offset, imm);
+      return;
+    }
+
+    assert((imm & ~bit_mask) == 0, "trying to set bits outside the mask");
+    imm &= bit_mask;
+
+    assert(align_up(immediate_address(), sizeof(jint)) ==
+           align_down(immediate_address(), sizeof(jint)), "immediate not aligned");
+    jint* data_addr = (jint*)immediate_address();
+    jint old_value = AtomicAccess::load(data_addr);
+    while (true) {
+      // Only bits in the mask are changed
+      jint new_value = imm | (old_value & ~bit_mask);
+      if (new_value == old_value) break;
+      jint v = AtomicAccess::cmpxchg(data_addr, old_value, new_value, memory_order_release);
+      if (v == old_value) break;
+      old_value = v;
+    }
+  }
  bool check_barrier(err_msg& msg) const;
  void verify() const {
 #ifdef ASSERT
@@ -159,13 +182,13 @@ static NativeNMethodCmpBarrier* native_nmethod_barrier(nmethod* nm) {
  return barrier;
 }

-void BarrierSetNMethod::set_guard_value(nmethod* nm, int value) {
+void BarrierSetNMethod::set_guard_value(nmethod* nm, int value, int bit_mask) {
  if (!supports_entry_barrier(nm)) {
    return;
  }

  NativeNMethodCmpBarrier* cmp = native_nmethod_barrier(nm);
-  cmp->set_immediate(value);
+  cmp->set_immediate(value, bit_mask);
 }

 int BarrierSetNMethod::guard_value(nmethod* nm) {
@@ -76,50 +76,95 @@ static uint& get_profile_ctr(int shift) {
 #endif // !PRODUCT

 void StubGenerator::generate_arraycopy_stubs() {
-  address entry;
-  address entry_jbyte_arraycopy;
-  address entry_jshort_arraycopy;
-  address entry_jint_arraycopy;
-  address entry_oop_arraycopy;
-  address entry_jlong_arraycopy;
-  address entry_checkcast_arraycopy;
+  // Some copy stubs publish a normal entry and then a 2nd 'fallback'
+  // entry immediately following their stack push. This can be used
+  // as a post-push branch target for compatible stubs when they
+  // identify a special case that can be handled by the fallback
+  // stub e.g a disjoint copy stub may be use as a special case
+  // fallback for its compatible conjoint copy stub.
+  //
+  // A no push entry is always returned in the following local and
+  // then published by assigning to the appropriate entry field in
+  // class StubRoutines. The entry value is then passed to the
+  // generator for the compatible stub. That means the entry must be
+  // listed when saving to/restoring from the AOT cache, ensuring
+  // that the inter-stub jumps are noted at AOT-cache save and
+  // relocated at AOT cache load.
+  address nopush_entry;

-  StubRoutines::_jbyte_disjoint_arraycopy  = generate_disjoint_byte_copy(&entry);
-  StubRoutines::_jbyte_arraycopy           = generate_conjoint_byte_copy(entry, &entry_jbyte_arraycopy);
+  StubRoutines::_jbyte_disjoint_arraycopy  = generate_disjoint_byte_copy(&nopush_entry);
+  // disjoint nopush entry is needed by conjoint copy
+  StubRoutines::_jbyte_disjoint_arraycopy_nopush  = nopush_entry;
+  StubRoutines::_jbyte_arraycopy           = generate_conjoint_byte_copy(StubRoutines::_jbyte_disjoint_arraycopy_nopush, &nopush_entry);
+  // conjoint nopush entry is needed by generic/unsafe copy
+  StubRoutines::_jbyte_arraycopy_nopush    = nopush_entry;

-  StubRoutines::_jshort_disjoint_arraycopy = generate_disjoint_short_copy(&entry);
-  StubRoutines::_jshort_arraycopy          = generate_conjoint_short_copy(entry, &entry_jshort_arraycopy);
+  StubRoutines::_jshort_disjoint_arraycopy = generate_disjoint_short_copy(&nopush_entry);
+  // disjoint nopush entry is needed by conjoint copy
+  StubRoutines::_jshort_disjoint_arraycopy_nopush = nopush_entry;
+  StubRoutines::_jshort_arraycopy          = generate_conjoint_short_copy(StubRoutines::_jshort_disjoint_arraycopy_nopush, &nopush_entry);
+  // conjoint nopush entry is needed by generic/unsafe copy
+  StubRoutines::_jshort_arraycopy_nopush   = nopush_entry;

-  StubRoutines::_jint_disjoint_arraycopy   = generate_disjoint_int_oop_copy(StubId::stubgen_jint_disjoint_arraycopy_id, &entry);
-  StubRoutines::_jint_arraycopy            = generate_conjoint_int_oop_copy(StubId::stubgen_jint_arraycopy_id, entry, &entry_jint_arraycopy);
+  StubRoutines::_jint_disjoint_arraycopy   = generate_disjoint_int_oop_copy(StubId::stubgen_jint_disjoint_arraycopy_id, &nopush_entry);
+  // disjoint nopush entry is needed by conjoint copy
+  StubRoutines::_jint_disjoint_arraycopy_nopush = nopush_entry;
+  StubRoutines::_jint_arraycopy            = generate_conjoint_int_oop_copy(StubId::stubgen_jint_arraycopy_id, StubRoutines::_jint_disjoint_arraycopy_nopush, &nopush_entry);
+  // conjoint nopush entry is needed by generic/unsafe copy
+  StubRoutines::_jint_arraycopy_nopush     = nopush_entry;
+
+  StubRoutines::_jlong_disjoint_arraycopy  = generate_disjoint_long_oop_copy(StubId::stubgen_jlong_disjoint_arraycopy_id, &nopush_entry);
+  // disjoint nopush entry is needed by conjoint copy
+  StubRoutines::_jlong_disjoint_arraycopy_nopush  = nopush_entry;
+  StubRoutines::_jlong_arraycopy           = generate_conjoint_long_oop_copy(StubId::stubgen_jlong_arraycopy_id, StubRoutines::_jlong_disjoint_arraycopy_nopush, &nopush_entry);
+  // conjoint nopush entry is needed by generic/unsafe copy
+  StubRoutines::_jlong_arraycopy_nopush    = nopush_entry;

-  StubRoutines::_jlong_disjoint_arraycopy  = generate_disjoint_long_oop_copy(StubId::stubgen_jlong_disjoint_arraycopy_id, &entry);
-  StubRoutines::_jlong_arraycopy           = generate_conjoint_long_oop_copy(StubId::stubgen_jlong_arraycopy_id, entry, &entry_jlong_arraycopy);
  if (UseCompressedOops) {
-    StubRoutines::_oop_disjoint_arraycopy  = generate_disjoint_int_oop_copy(StubId::stubgen_oop_disjoint_arraycopy_id, &entry);
-    StubRoutines::_oop_arraycopy           = generate_conjoint_int_oop_copy(StubId::stubgen_oop_arraycopy_id, entry, &entry_oop_arraycopy);
-    StubRoutines::_oop_disjoint_arraycopy_uninit  = generate_disjoint_int_oop_copy(StubId::stubgen_oop_disjoint_arraycopy_uninit_id, &entry);
-    StubRoutines::_oop_arraycopy_uninit           = generate_conjoint_int_oop_copy(StubId::stubgen_oop_arraycopy_uninit_id, entry, nullptr);
+    StubRoutines::_oop_disjoint_arraycopy  = generate_disjoint_int_oop_copy(StubId::stubgen_oop_disjoint_arraycopy_id, &nopush_entry);
+    // disjoint nopush entry is needed by conjoint copy
+    StubRoutines::_oop_disjoint_arraycopy_nopush  = nopush_entry;
+    StubRoutines::_oop_arraycopy           = generate_conjoint_int_oop_copy(StubId::stubgen_oop_arraycopy_id, StubRoutines::_oop_disjoint_arraycopy_nopush, &nopush_entry);
+    // conjoint nopush entry is needed by generic/unsafe copy
+    StubRoutines::_oop_arraycopy_nopush    = nopush_entry;
+    StubRoutines::_oop_disjoint_arraycopy_uninit  = generate_disjoint_int_oop_copy(StubId::stubgen_oop_disjoint_arraycopy_uninit_id, &nopush_entry);
+    // disjoint nopush entry is needed by conjoint copy
+    StubRoutines::_oop_disjoint_arraycopy_uninit_nopush  = nopush_entry;
+    // note that we don't need a returned nopush entry because the
+    // generic/unsafe copy does not cater for uninit arrays.
+    StubRoutines::_oop_arraycopy_uninit           = generate_conjoint_int_oop_copy(StubId::stubgen_oop_arraycopy_uninit_id, StubRoutines::_oop_disjoint_arraycopy_uninit_nopush, nullptr);
  } else {
-    StubRoutines::_oop_disjoint_arraycopy  = generate_disjoint_long_oop_copy(StubId::stubgen_oop_disjoint_arraycopy_id, &entry);
-    StubRoutines::_oop_arraycopy           = generate_conjoint_long_oop_copy(StubId::stubgen_oop_arraycopy_id, entry, &entry_oop_arraycopy);
-    StubRoutines::_oop_disjoint_arraycopy_uninit  = generate_disjoint_long_oop_copy(StubId::stubgen_oop_disjoint_arraycopy_uninit_id, &entry);
-    StubRoutines::_oop_arraycopy_uninit           = generate_conjoint_long_oop_copy(StubId::stubgen_oop_arraycopy_uninit_id, entry, nullptr);
+    StubRoutines::_oop_disjoint_arraycopy  = generate_disjoint_long_oop_copy(StubId::stubgen_oop_disjoint_arraycopy_id, &nopush_entry);
+    // disjoint nopush entry is needed by conjoint copy
+    StubRoutines::_oop_disjoint_arraycopy_nopush  = nopush_entry;
+    StubRoutines::_oop_arraycopy           = generate_conjoint_long_oop_copy(StubId::stubgen_oop_arraycopy_id, StubRoutines::_oop_disjoint_arraycopy_nopush, &nopush_entry);
+    // conjoint nopush entry is needed by generic/unsafe copy
+    StubRoutines::_oop_arraycopy_nopush    = nopush_entry;
+    StubRoutines::_oop_disjoint_arraycopy_uninit  = generate_disjoint_long_oop_copy(StubId::stubgen_oop_disjoint_arraycopy_uninit_id, &nopush_entry);
+    // disjoint nopush entry is needed by conjoint copy
+    StubRoutines::_oop_disjoint_arraycopy_uninit_nopush  = nopush_entry;
+    // note that we don't need a returned nopush entry because the
+    // generic/unsafe copy does not cater for uninit arrays.
+    StubRoutines::_oop_arraycopy_uninit           = generate_conjoint_long_oop_copy(StubId::stubgen_oop_arraycopy_uninit_id, StubRoutines::_oop_disjoint_arraycopy_uninit_nopush, nullptr);
  }

-  StubRoutines::_checkcast_arraycopy        = generate_checkcast_copy(StubId::stubgen_checkcast_arraycopy_id, &entry_checkcast_arraycopy);
+  StubRoutines::_checkcast_arraycopy        = generate_checkcast_copy(StubId::stubgen_checkcast_arraycopy_id, &nopush_entry);
+  // checkcast nopush entry is needed by generic copy
+  StubRoutines::_checkcast_arraycopy_nopush = nopush_entry;
+  // note that we don't need a returned nopush entry because the
+  // generic copy does not cater for uninit arrays.
  StubRoutines::_checkcast_arraycopy_uninit = generate_checkcast_copy(StubId::stubgen_checkcast_arraycopy_uninit_id, nullptr);

-  StubRoutines::_unsafe_arraycopy    = generate_unsafe_copy(entry_jbyte_arraycopy,
-                                                            entry_jshort_arraycopy,
-                                                            entry_jint_arraycopy,
-                                                            entry_jlong_arraycopy);
-  StubRoutines::_generic_arraycopy   = generate_generic_copy(entry_jbyte_arraycopy,
-                                                             entry_jshort_arraycopy,
-                                                             entry_jint_arraycopy,
-                                                             entry_oop_arraycopy,
-                                                             entry_jlong_arraycopy,
-                                                             entry_checkcast_arraycopy);
+  StubRoutines::_unsafe_arraycopy    = generate_unsafe_copy(StubRoutines::_jbyte_arraycopy_nopush,
+                                                            StubRoutines::_jshort_arraycopy_nopush,
+                                                            StubRoutines::_jint_arraycopy_nopush,
+                                                            StubRoutines::_jlong_arraycopy_nopush);
+  StubRoutines::_generic_arraycopy   = generate_generic_copy(StubRoutines::_jbyte_arraycopy_nopush,
+                                                             StubRoutines::_jshort_arraycopy_nopush,
+                                                             StubRoutines::_jint_arraycopy_nopush,
+                                                             StubRoutines::_oop_arraycopy_nopush,
+                                                             StubRoutines::_jlong_arraycopy_nopush,
+                                                             StubRoutines::_checkcast_arraycopy_nopush);

  StubRoutines::_jbyte_fill = generate_fill(StubId::stubgen_jbyte_fill_id);
  StubRoutines::_jshort_fill = generate_fill(StubId::stubgen_jshort_fill_id);
@@ -139,7 +139,7 @@ class VM_Version_StubGenerator: public StubCodeGenerator {
    const uint32_t CPU_FAMILY_486 = (4 << CPU_FAMILY_SHIFT);
    bool use_evex = FLAG_IS_DEFAULT(UseAVX) || (UseAVX > 2);

-    Label detect_486, cpu486, detect_586, std_cpuid1, std_cpuid4, std_cpuid24;
+    Label detect_486, cpu486, detect_586, std_cpuid1, std_cpuid4, std_cpuid24, std_cpuid29;
    Label sef_cpuid, sefsl1_cpuid, ext_cpuid, ext_cpuid1, ext_cpuid5, ext_cpuid7;
    Label ext_cpuid8, done, wrapup, vector_save_restore, apx_save_restore_warning;
    Label legacy_setup, save_restore_except, legacy_save_restore, start_simd_check;
@@ -338,6 +338,16 @@ class VM_Version_StubGenerator: public StubCodeGenerator {
    __ movl(Address(rsi, 0), rax);
    __ movl(Address(rsi, 4), rdx);

+    //
+    // cpuid(0x29) APX NCI NDD NF (EAX = 29H, ECX = 0).
+    //
+    __ bind(std_cpuid29);
+    __ movl(rax, 0x29);
+    __ movl(rcx, 0);
+    __ cpuid();
+    __ lea(rsi, Address(rbp, in_bytes(VM_Version::std_cpuid29_offset())));
+    __ movl(Address(rsi, 0), rbx);
+
    //
    // cpuid(0x24) Converged Vector ISA Main Leaf (EAX = 24H, ECX = 0).
    //
@@ -1016,16 +1026,6 @@ void VM_Version::get_processor_features() {
    _features.clear_feature(CPU_AVX10_2);
  }

-  // Currently APX support is only enabled for targets supporting AVX512VL feature.
-  bool apx_supported = os_supports_apx_egprs() && supports_apx_f() && supports_avx512vl();
-  if (UseAPX && !apx_supported) {
-    warning("UseAPX is not supported on this CPU, setting it to false");
-    FLAG_SET_DEFAULT(UseAPX, false);
-  }
-
-  if (!UseAPX) {
-    _features.clear_feature(CPU_APX_F);
-  }

  if (UseAVX < 2) {
    _features.clear_feature(CPU_AVX2);
@@ -1049,6 +1049,7 @@ void VM_Version::get_processor_features() {
      _features.clear_feature(CPU_VZEROUPPER);
      _features.clear_feature(CPU_AVX512BW);
      _features.clear_feature(CPU_AVX512VL);
+      _features.clear_feature(CPU_APX_F);
      _features.clear_feature(CPU_AVX512DQ);
      _features.clear_feature(CPU_AVX512_VNNI);
      _features.clear_feature(CPU_AVX512_VAES);
@@ -1068,6 +1069,17 @@ void VM_Version::get_processor_features() {
    }
  }

+    // Currently APX support is only enabled for targets supporting AVX512VL feature.
+  bool apx_supported = os_supports_apx_egprs() && supports_apx_f() && supports_avx512vl();
+  if (UseAPX && !apx_supported) {
+    warning("UseAPX is not supported on this CPU, setting it to false");
+    FLAG_SET_DEFAULT(UseAPX, false);
+  }
+
+  if (!UseAPX) {
+    _features.clear_feature(CPU_APX_F);
+  }
+
  if (FLAG_IS_DEFAULT(IntelJccErratumMitigation)) {
    _has_intel_jcc_erratum = compute_has_intel_jcc_erratum();
    FLAG_SET_ERGO(IntelJccErratumMitigation, _has_intel_jcc_erratum);
@@ -2912,7 +2924,8 @@ VM_Version::VM_Features VM_Version::CpuidInfo::feature_flags() const {
  if (std_cpuid1_ecx.bits.popcnt != 0)
    vm_features.set_feature(CPU_POPCNT);
  if (sefsl1_cpuid7_edx.bits.apx_f != 0 &&
-      xem_xcr0_eax.bits.apx_f != 0) {
+      xem_xcr0_eax.bits.apx_f != 0 &&
+      std_cpuid29_ebx.bits.apx_nci_ndd_nf != 0) {
    vm_features.set_feature(CPU_APX_F);
  }
  if (std_cpuid1_ecx.bits.avx != 0 &&
@@ -306,6 +306,14 @@ class VM_Version : public Abstract_VM_Version {
    } bits;
  };

+  union StdCpuidEax29Ecx0 {
+    uint32_t value;
+    struct {
+      uint32_t  apx_nci_ndd_nf  : 1,
+                                : 31;
+    } bits;
+  };
+
  union StdCpuid24MainLeafEax {
    uint32_t value;
    struct {
@@ -591,6 +599,10 @@ protected:
    StdCpuid24MainLeafEax std_cpuid24_eax;
    StdCpuid24MainLeafEbx std_cpuid24_ebx;

+    // cpuid function 0x29 APX Advanced Performance Extensions Leaf
+    // eax = 0x29, ecx = 0
+    StdCpuidEax29Ecx0 std_cpuid29_ebx;
+
    // cpuid function 0xB (processor topology)
    // ecx = 0
    uint32_t     tpl_cpuidB0_eax;
@@ -711,6 +723,7 @@ public:
  static ByteSize std_cpuid0_offset() { return byte_offset_of(CpuidInfo, std_max_function); }
  static ByteSize std_cpuid1_offset() { return byte_offset_of(CpuidInfo, std_cpuid1_eax); }
  static ByteSize std_cpuid24_offset() { return byte_offset_of(CpuidInfo, std_cpuid24_eax); }
+  static ByteSize std_cpuid29_offset() { return byte_offset_of(CpuidInfo, std_cpuid29_ebx); }
  static ByteSize dcp_cpuid4_offset() { return byte_offset_of(CpuidInfo, dcp_cpuid4_eax); }
  static ByteSize sef_cpuid7_offset() { return byte_offset_of(CpuidInfo, sef_cpuid7_eax); }
  static ByteSize sefsl1_cpuid7_offset() { return byte_offset_of(CpuidInfo, sefsl1_cpuid7_eax); }
@@ -760,7 +773,9 @@ public:
    _features.set_feature(CPU_SSE2);
    _features.set_feature(CPU_VZEROUPPER);
  }
-  static void set_apx_cpuFeatures() { _features.set_feature(CPU_APX_F); }
+  static void set_apx_cpuFeatures() {
+    _features.set_feature(CPU_APX_F);
+  }
  static void set_bmi_cpuFeatures() {
    _features.set_feature(CPU_BMI1);
    _features.set_feature(CPU_BMI2);
@@ -3429,9 +3429,6 @@ attributes %{
  instruction_unit_size = 1;         // An instruction is 1 bytes long
  instruction_fetch_unit_size = 16;  // The processor fetches one line
  instruction_fetch_units = 1;       // of 16 bytes
-
-  // List of nop instructions
-  nops( MachNop );
 %}

 //----------RESOURCES----------------------------------------------------------
@@ -26,7 +26,7 @@
 #ifndef CPU_ZERO_ASSEMBLER_ZERO_INLINE_HPP
 #define CPU_ZERO_ASSEMBLER_ZERO_INLINE_HPP

-#include "asm/assembler.inline.hpp"
+#include "asm/assembler.hpp"
 #include "asm/codeBuffer.hpp"
 #include "code/codeCache.hpp"
 #include "runtime/handles.inline.hpp"
@@ -29,7 +29,7 @@ void BarrierSetNMethod::deoptimize(nmethod* nm, address* return_address_ptr) {
  ShouldNotReachHere();
 }

-void BarrierSetNMethod::set_guard_value(nmethod* nm, int value) {
+void BarrierSetNMethod::set_guard_value(nmethod* nm, int value, int bit_mask) {
  ShouldNotReachHere();
 }

@@ -43,7 +43,7 @@
 #include "prims/jniFastGetField.hpp"
 #include "prims/jvm_misc.hpp"
 #include "runtime/arguments.hpp"
-#include "runtime/atomic.hpp"
+#include "runtime/atomicAccess.hpp"
 #include "runtime/globals.hpp"
 #include "runtime/globals_extension.hpp"
 #include "runtime/interfaceSupport.inline.hpp"
@@ -1753,10 +1753,6 @@ void os::numa_make_global(char *addr, size_t bytes) {
 void os::numa_make_local(char *addr, size_t bytes, int lgrp_hint) {
 }

-bool os::numa_topology_changed() {
-  return false;
-}
-
 size_t os::numa_get_groups_num() {
  return 1;
 }
@@ -39,7 +39,7 @@
 #include "prims/jniFastGetField.hpp"
 #include "prims/jvm_misc.hpp"
 #include "runtime/arguments.hpp"
-#include "runtime/atomic.hpp"
+#include "runtime/atomicAccess.hpp"
 #include "runtime/globals.hpp"
 #include "runtime/globals_extension.hpp"
 #include "runtime/interfaceSupport.inline.hpp"
@@ -809,7 +809,7 @@ jlong os::javaTimeNanos() {
  if (now <= prev) {
    return prev;   // same or retrograde time;
  }
-  const uint64_t obsv = Atomic::cmpxchg(&Bsd::_max_abstime, prev, now);
+  const uint64_t obsv = AtomicAccess::cmpxchg(&Bsd::_max_abstime, prev, now);
  assert(obsv >= prev, "invariant");   // Monotonicity
  // If the CAS succeeded then we're done and return "now".
  // If the CAS failed and the observed value "obsv" is >= now then
@@ -1599,8 +1599,6 @@ void os::numa_make_global(char *addr, size_t bytes) {
 void os::numa_make_local(char *addr, size_t bytes, int lgrp_hint) {
 }

-bool os::numa_topology_changed()   { return false; }
-
 size_t os::numa_get_groups_num() {
  return 1;
 }
@@ -2135,14 +2133,14 @@ uint os::processor_id() {
  __asm__ ("cpuid\n\t" : "+a" (eax), "+b" (ebx), "+c" (ecx), "+d" (edx) : );

  uint apic_id = (ebx >> 24) & (processor_id_map_size - 1);
-  int processor_id = Atomic::load(&processor_id_map[apic_id]);
+  int processor_id = AtomicAccess::load(&processor_id_map[apic_id]);

  while (processor_id < 0) {
    // Assign processor id to APIC id
-    processor_id = Atomic::cmpxchg(&processor_id_map[apic_id], processor_id_unassigned, processor_id_assigning);
+    processor_id = AtomicAccess::cmpxchg(&processor_id_map[apic_id], processor_id_unassigned, processor_id_assigning);
    if (processor_id == processor_id_unassigned) {
-      processor_id = Atomic::fetch_then_add(&processor_id_next, 1) % os::processor_count();
-      Atomic::store(&processor_id_map[apic_id], processor_id);
+      processor_id = AtomicAccess::fetch_then_add(&processor_id_next, 1) % os::processor_count();
+      AtomicAccess::store(&processor_id_map[apic_id], processor_id);
    }
  }

@@ -46,6 +46,10 @@ class CompilerThreadTimeoutLinux : public CHeapObj<mtCompiler> {
  bool init_timeout();
  void arm();
  void disarm();
+  void reset() {
+    disarm();
+    arm();
+  };
 };

 #endif //LINUX_COMPILER_THREAD_TIMEOUT_LINUX_HPP
@@ -106,6 +106,7 @@ public:
      PRINTIF(info.swap > 0, "swap");
      PRINTIF(info.ht, "huge");
      PRINTIF(info.anonhugepages > 0, "thp");
+      PRINTIF(info.thpeligible, "thpel");
      PRINTIF(info.hg, "thpad");
      PRINTIF(info.nh, "nothp");
      if (num_printed == 0) {
@@ -135,6 +136,7 @@ public:
    st->print_cr("                       com: mapping committed (swap space reserved)");
    st->print_cr("                      swap: mapping partly or completely swapped out");
    st->print_cr("                       thp: mapping uses THP");
+    st->print_cr("                     thpel: mapping is THP-eligible");
    st->print_cr("                     thpad: mapping is THP-madvised");
    st->print_cr("                     nothp: mapping is forbidden to use THP");
    st->print_cr("                      huge: mapping uses hugetlb pages");
@@ -42,7 +42,7 @@
 #include "prims/jniFastGetField.hpp"
 #include "prims/jvm_misc.hpp"
 #include "runtime/arguments.hpp"
-#include "runtime/atomic.hpp"
+#include "runtime/atomicAccess.hpp"
 #include "runtime/globals.hpp"
 #include "runtime/globals_extension.hpp"
 #include "runtime/init.hpp"
@@ -370,11 +370,20 @@ size_t os::physical_memory() {
  return phys_mem;
 }

+// Returns the resident set size (RSS) of the process.
+// Falls back to using VmRSS from /proc/self/status if /proc/self/smaps_rollup is unavailable.
+// Note: On kernels with memory cgroups or shared memory, VmRSS may underreport RSS.
+// Users requiring accurate RSS values should be aware of this limitation.
 size_t os::rss() {
  size_t size = 0;
-  os::Linux::meminfo_t info;
-  if (os::Linux::query_process_memory_info(&info)) {
-    size = info.vmrss * K;
+  os::Linux::accurate_meminfo_t accurate_info;
+  if (os::Linux::query_accurate_process_memory_info(&accurate_info) && accurate_info.rss != -1) {
+    size = accurate_info.rss * K;
+  } else {
+    os::Linux::meminfo_t info;
+    if (os::Linux::query_process_memory_info(&info)) {
+      size = info.vmrss * K;
+    }
  }
  return size;
 }
@@ -2362,6 +2371,37 @@ bool os::Linux::query_process_memory_info(os::Linux::meminfo_t* info) {
  return false;
 }

+// Accurate memory information need Linux 4.14 or newer
+bool os::Linux::query_accurate_process_memory_info(os::Linux::accurate_meminfo_t* info) {
+  FILE* f = os::fopen("/proc/self/smaps_rollup", "r");
+  if (f == nullptr) {
+    return false;
+  }
+
+  const size_t num_values = sizeof(os::Linux::accurate_meminfo_t) / sizeof(size_t);
+  size_t num_found = 0;
+  char buf[256];
+  info->rss = info->pss = info->pssdirty = info->pssanon =
+      info->pssfile = info->pssshmem = info->swap = info->swappss = -1;
+
+  while (::fgets(buf, sizeof(buf), f) != nullptr && num_found < num_values) {
+    if ( (info->rss == -1        && sscanf(buf, "Rss: %zd kB", &info->rss) == 1) ||
+         (info->pss == -1        && sscanf(buf, "Pss: %zd kB", &info->pss) == 1) ||
+         (info->pssdirty == -1   && sscanf(buf, "Pss_Dirty: %zd kB", &info->pssdirty) == 1) ||
+         (info->pssanon == -1    && sscanf(buf, "Pss_Anon: %zd kB", &info->pssanon) == 1) ||
+         (info->pssfile == -1    && sscanf(buf, "Pss_File: %zd kB", &info->pssfile) == 1) ||
+         (info->pssshmem == -1   && sscanf(buf, "Pss_Shmem: %zd kB", &info->pssshmem) == 1) ||
+         (info->swap == -1       && sscanf(buf, "Swap: %zd kB", &info->swap) == 1) ||
+         (info->swappss == -1    && sscanf(buf, "SwapPss: %zd kB", &info->swappss) == 1)
+         )
+    {
+      num_found ++;
+    }
+  }
+  fclose(f);
+  return true;
+}
+
 #ifdef __GLIBC__
 // For Glibc, print a one-liner with the malloc tunables.
 // Most important and popular is MALLOC_ARENA_MAX, but we are
@@ -2988,8 +3028,6 @@ void os::numa_make_local(char *addr, size_t bytes, int lgrp_hint) {
  Linux::numa_tonode_memory(addr, bytes, lgrp_hint);
 }

-bool os::numa_topology_changed() { return false; }
-
 size_t os::numa_get_groups_num() {
  // Return just the number of nodes in which it's possible to allocate memory
  // (in numa terminology, configured nodes).
@@ -4783,8 +4821,8 @@ static bool should_warn_invalid_processor_id() {

  static volatile int warn_once = 1;

-  if (Atomic::load(&warn_once) == 0 ||
-      Atomic::xchg(&warn_once, 0) == 0) {
+  if (AtomicAccess::load(&warn_once) == 0 ||
+      AtomicAccess::xchg(&warn_once, 0) == 0) {
    // Don't warn more than once
    return false;
  }
@@ -181,6 +181,23 @@ class os::Linux {
  // fields will contain -1.
  static bool query_process_memory_info(meminfo_t* info);

+  // Output structure for query_accurate_process_memory_info() (all values in KB)
+  struct accurate_meminfo_t {
+    ssize_t rss;        // current resident set size
+    ssize_t pss;        // current proportional set size
+    ssize_t pssdirty;   // proportional set size (dirty)
+    ssize_t pssanon;    // proportional set size (anonymous mappings)
+    ssize_t pssfile;    // proportional set size (file mappings)
+    ssize_t pssshmem;   // proportional set size (shared mappings)
+    ssize_t swap;       // swapped out
+    ssize_t swappss;    // proportional set size (swapped out)
+  };
+
+  // Attempts to query accurate memory information from /proc/self/smaps_rollup and return it in the output structure.
+  // May fail (returns false) or succeed (returns true) but not all output fields are available; unavailable
+  // fields will contain -1.
+  static bool query_accurate_process_memory_info(accurate_meminfo_t* info);
+
  // Tells if the user asked for transparent huge pages.
  static bool _thp_requested;

@@ -76,8 +76,16 @@ void ProcSmapsParser::scan_additional_line(ProcSmapsInfo& out) {
  SCAN("Private_Hugetlb", out.private_hugetlb);
  SCAN("Shared_Hugetlb", out.shared_hugetlb);
  SCAN("Swap", out.swap);
-  int i = 0;
 #undef SCAN
+
+  // scan THPeligible into a bool
+  int thpel = 0;
+  if (::sscanf(_line, "THPeligible: %d", &thpel) == 1) {
+    assert(thpel == 1 || thpel == 0, "Unexpected value %d", thpel);
+    out.thpeligible = (thpel == 1);
+    return;
+  }
+
  // scan some flags too
  if (strncmp(_line, "VmFlags:", 8) == 0) {
 #define SCAN(flag) { out.flag = (::strstr(_line + 8, " " #flag) != nullptr); }
@@ -49,6 +49,7 @@ struct ProcSmapsInfo {
  size_t shared_hugetlb;
  size_t anonhugepages;
  size_t swap;
+  bool thpeligible;
  bool rd, wr, ex;
  bool sh; // shared
  bool nr; // no reserve
@@ -64,7 +65,7 @@ struct ProcSmapsInfo {
    from = to = nullptr;
    prot[0] = filename[0] = '\0';
    kernelpagesize = rss = private_hugetlb = shared_hugetlb = anonhugepages = swap = 0;
-    rd = wr = ex = sh = nr = hg = ht = nh = false;
+    thpeligible = rd = wr = ex = sh = nr = hg = ht = nh = false;
  }
 };

@@ -24,19 +24,19 @@

 #include "logging/log.hpp"
 #include "memory/allocation.inline.hpp"
+#include "os_posix.hpp"
 #include "runtime/interfaceSupport.inline.hpp"
 #include "runtime/os.inline.hpp"
-#include "os_posix.hpp"
 #include "services/attachListener.hpp"
 #include "utilities/checkedCast.hpp"
 #include "utilities/macros.hpp"

-#include <unistd.h>
 #include <signal.h>
-#include <sys/types.h>
 #include <sys/socket.h>
-#include <sys/un.h>
 #include <sys/stat.h>
+#include <sys/types.h>
+#include <sys/un.h>
+#include <unistd.h>

 #if INCLUDE_SERVICES
 #ifndef AIX
@@ -37,28 +37,28 @@
 #endif

 // If needed, add os::strndup and use that instead.
-FORBID_C_FUNCTION(char* strndup(const char*, size_t), "don't use");
+FORBID_C_FUNCTION(char* strndup(const char*, size_t), noexcept, "don't use");

 // These are unimplementable for Windows, and they aren't useful for a
 // POSIX implementation of NMT either.
 // https://stackoverflow.com/questions/62962839/stdaligned-alloc-missing-from-visual-studio-2019
-FORBID_C_FUNCTION(int posix_memalign(void**, size_t, size_t), "don't use");
-FORBID_C_FUNCTION(void* aligned_alloc(size_t, size_t), "don't use");
+FORBID_C_FUNCTION(int posix_memalign(void**, size_t, size_t), noexcept, "don't use");
+FORBID_C_FUNCTION(void* aligned_alloc(size_t, size_t), noexcept, "don't use");

 // realpath with a null second argument mallocs a string for the result.
 // With a non-null second argument, there is a risk of buffer overrun.
 PRAGMA_DIAG_PUSH
 FORBIDDEN_FUNCTION_IGNORE_CLANG_FORTIFY_WARNING
-FORBID_C_FUNCTION(char* realpath(const char*, char*), "use os::realpath");
+FORBID_C_FUNCTION(char* realpath(const char*, char*), noexcept, "use os::realpath");
 PRAGMA_DIAG_POP

 // Returns a malloc'ed string.
-FORBID_C_FUNCTION(char* get_current_dir_name(), "use os::get_current_directory");
+FORBID_C_FUNCTION(char* get_current_dir_name(), noexcept, "use os::get_current_directory");

 // Problematic API that should never be used.
-FORBID_C_FUNCTION(char* getwd(char*), "use os::get_current_directory");
+FORBID_C_FUNCTION(char* getwd(char*), noexcept, "use os::get_current_directory");

 // BSD utility that is subtly different from realloc.
-FORBID_C_FUNCTION(void* reallocf(void*, size_t), "use os::realloc");
+FORBID_C_FUNCTION(void* reallocf(void*, size_t), /* not noexcept */, "use os::realloc");

 #endif // OS_POSIX_FORBIDDENFUNCTIONS_POSIX_HPP
@@ -31,7 +31,7 @@
 #include "nmt/memTracker.hpp"
 #include "os_posix.inline.hpp"
 #include "runtime/arguments.hpp"
-#include "runtime/atomic.hpp"
+#include "runtime/atomicAccess.hpp"
 #include "runtime/frame.inline.hpp"
 #include "runtime/globals_extension.hpp"
 #include "runtime/interfaceSupport.inline.hpp"
@@ -70,14 +70,13 @@
 #include <grp.h>
 #include <locale.h>
 #include <netdb.h>
-#include <pwd.h>
 #include <pthread.h>
+#include <pwd.h>
 #include <signal.h>
+#include <spawn.h>
 #include <sys/mman.h>
 #include <sys/resource.h>
 #include <sys/socket.h>
-#include <spawn.h>
-#include <sys/resource.h>
 #include <sys/time.h>
 #include <sys/times.h>
 #include <sys/types.h>
@@ -1692,7 +1691,7 @@ void PlatformEvent::park() {       // AKA "down()"
  // atomically decrement _event
  for (;;) {
    v = _event;
-    if (Atomic::cmpxchg(&_event, v, v - 1) == v) break;
+    if (AtomicAccess::cmpxchg(&_event, v, v - 1) == v) break;
  }
  guarantee(v >= 0, "invariant");

@@ -1739,7 +1738,7 @@ int PlatformEvent::park_nanos(jlong nanos) {
  // atomically decrement _event
  for (;;) {
    v = _event;
-    if (Atomic::cmpxchg(&_event, v, v - 1) == v) break;
+    if (AtomicAccess::cmpxchg(&_event, v, v - 1) == v) break;
  }
  guarantee(v >= 0, "invariant");

@@ -1795,7 +1794,7 @@ void PlatformEvent::unpark() {
  // but only in the correctly written condition checking loops of ObjectMonitor,
  // Mutex/Monitor, and JavaThread::sleep

-  if (Atomic::xchg(&_event, 1) >= 0) return;
+  if (AtomicAccess::xchg(&_event, 1) >= 0) return;

  int status = pthread_mutex_lock(_mutex);
  assert_status(status == 0, status, "mutex_lock");
@@ -1848,9 +1847,9 @@ void Parker::park(bool isAbsolute, jlong time) {

  // Optional fast-path check:
  // Return immediately if a permit is available.
-  // We depend on Atomic::xchg() having full barrier semantics
+  // We depend on AtomicAccess::xchg() having full barrier semantics
  // since we are doing a lock-free update to _counter.
-  if (Atomic::xchg(&_counter, 0) > 0) return;
+  if (AtomicAccess::xchg(&_counter, 0) > 0) return;

  JavaThread *jt = JavaThread::current();

@@ -30,9 +30,9 @@
 #include "runtime/mutex.hpp"
 #include "runtime/os.hpp"

-#include <unistd.h>
-#include <sys/socket.h>
 #include <netdb.h>
+#include <sys/socket.h>
+#include <unistd.h>

 // Aix does not have NUMA support but need these for compilation.
 inline bool os::numa_has_group_homing()     { AIX_ONLY(ShouldNotReachHere();) return false;  }
@@ -40,15 +40,14 @@
 #include "os_linux.hpp"
 #endif

-// put OS-includes here
-# include <sys/types.h>
-# include <sys/mman.h>
 # include <errno.h>
-# include <stdio.h>
-# include <unistd.h>
-# include <sys/stat.h>
-# include <signal.h>
 # include <pwd.h>
+# include <signal.h>
+# include <stdio.h>
+# include <sys/mman.h>
+# include <sys/stat.h>
+# include <sys/types.h>
+# include <unistd.h>

 #if defined(LINUX)
 # include <sys/file.h>
@@ -32,8 +32,8 @@
 #ifdef SAFEFETCH_METHOD_SIGSETJMP

 // For SafeFetch we need POSIX TLS and sigsetjmp/longjmp.
-#include <setjmp.h>
 #include <pthread.h>
+#include <setjmp.h>
 static pthread_key_t g_jmpbuf_key;

 struct InitTLSKey { InitTLSKey() { pthread_key_create(&g_jmpbuf_key, nullptr); } };
@@ -25,9 +25,10 @@
 #ifndef __APPLE__
 #include "os_posix.hpp"
 #include "runtime/os.hpp"
-#include "utilities/debug.hpp"
 // POSIX unnamed semaphores are not supported on OS X.
 #include "semaphore_posix.hpp"
+#include "utilities/debug.hpp"
+
 #include <semaphore.h>

 PosixSemaphore::PosixSemaphore(uint value) {
@@ -28,7 +28,7 @@
 #include "jvm.h"
 #include "logging/log.hpp"
 #include "os_posix.hpp"
-#include "runtime/atomic.hpp"
+#include "runtime/atomicAccess.hpp"
 #include "runtime/globals.hpp"
 #include "runtime/interfaceSupport.inline.hpp"
 #include "runtime/java.hpp"
@@ -356,7 +356,7 @@ static void jdk_misc_signal_init() {

 void os::signal_notify(int sig) {
  if (sig_semaphore != nullptr) {
-    Atomic::inc(&pending_signals[sig]);
+    AtomicAccess::inc(&pending_signals[sig]);
    sig_semaphore->signal();
  } else {
    // Signal thread is not created with ReduceSignalUsage and jdk_misc_signal_init
@@ -369,7 +369,7 @@ static int check_pending_signals() {
  for (;;) {
    for (int i = 0; i < NSIG + 1; i++) {
      jint n = pending_signals[i];
-      if (n > 0 && n == Atomic::cmpxchg(&pending_signals[i], n, n - 1)) {
+      if (n > 0 && n == AtomicAccess::cmpxchg(&pending_signals[i], n, n - 1)) {
        return i;
      }
    }
--- a/Show More
+++ b/Show More