Update version for 9.1.3 release

Signed-off-by: Michael Tokarev <mjt@tls.msk.ru>
gdbstub/user-target: fix gdbserver int format (%d -> %x)
2025-02-08 14:25:57 +03:00 · 2025-02-01 12:21:08 +03:00 · 2025-02-01 12:01:02 +03:00 · 2025-02-01 12:00:13 +03:00 · 2025-02-01 11:59:14 +03:00 · 2025-02-01 11:56:57 +03:00
5747 changed files with 199573 additions and 321583 deletions
@@ -1,13 +0,0 @@
-#
-# Common b4 settings that can be used to send patches to QEMU upstream.
-# https://b4.docs.kernel.org/
-#
-
-[b4]
-    send-series-to = qemu-devel@nongnu.org
-    send-auto-to-cmd = echo
-    send-auto-cc-cmd = scripts/get_maintainer.pl --noroles --norolestats --nogit --nogit-fallback
-    am-perpatch-check-cmd = scripts/checkpatch.pl -q --terse --no-summary --mailback -
-    prep-perpatch-check-cmd = scripts/checkpatch.pl -q --terse --no-summary --mailback -
-    searchmask = https://lore.kernel.org/qemu-devel/?x=m&t=1&q=%s
-    linkmask = https://lore.kernel.org/qemu-devel/%s
@@ -47,16 +47,3 @@ emacs_mode = glsl
 [*.json]
 indent_style = space
 emacs_mode = python
-
-# by default follow QEMU's style
-[*.pl]
-indent_style = space
-indent_size = 4
-emacs_mode = perl
-
-# but user kernel "style" for imported scripts
-[scripts/{get_maintainer.pl,checkpatch.pl}]
-indent_style = tab
-indent_size = 8
-emacs_mode = perl
-
@@ -2,8 +2,3 @@
 *.h.inc         diff=c
 *.m             diff=objc
 *.py            diff=python
-*.rs            diff=rust
-*.rs.inc        diff=rust
-Cargo.lock      diff=toml merge=binary
-
-*.patch         -text -whitespace
@@ -69,6 +69,10 @@ variables:
    - if: '$QEMU_CI != "1" && $QEMU_CI != "2" && $CI_PROJECT_NAMESPACE != $QEMU_CI_UPSTREAM'
      when: never

+    # Avocado jobs don't run in forks unless $QEMU_CI_AVOCADO_TESTING is set
+    - if: '$QEMU_JOB_AVOCADO && $QEMU_CI_AVOCADO_TESTING != "1" && $CI_PROJECT_NAMESPACE != $QEMU_CI_UPSTREAM'
+      when: never
+

    #############################################################
    # Stage 2: fine tune execution of jobs in specific scenarios
@@ -97,8 +101,8 @@ variables:
      when: manual
      allow_failure: true

-    # Functional jobs can be manually started in forks
-    - if: '$QEMU_JOB_FUNCTIONAL && $QEMU_CI_FUNCTIONAL != "1" && $CI_PROJECT_NAMESPACE != $QEMU_CI_UPSTREAM'
+    # Avocado jobs can be manually start in forks if $QEMU_CI_AVOCADO_TESTING is unset
+    - if: '$QEMU_JOB_AVOCADO && $CI_PROJECT_NAMESPACE != $QEMU_CI_UPSTREAM'
      when: manual
      allow_failure: true

@@ -8,11 +8,8 @@
    key: "$CI_JOB_NAME"
    when: always
  before_script:
-    - source scripts/ci/gitlab-ci-section
-    - section_start setup "Pre-script setup"
    - JOBS=$(expr $(nproc) + 1)
    - cat /packages.txt
-    - section_end setup
  script:
    - export CCACHE_BASEDIR="$(pwd)"
    - export CCACHE_DIR="$CCACHE_BASEDIR/ccache"
@@ -22,9 +19,7 @@
    - mkdir build
    - cd build
    - ccache --zero-stats
-    - section_start configure "Running configure"
    - ../configure --enable-werror --disable-docs --enable-fdt=system
-          --disable-debug-info
          ${TARGETS:+--target-list="$TARGETS"}
          $CONFIGURE_ARGS ||
      { cat config.log meson-logs/meson-log.txt && exit 1; }
@@ -32,16 +27,11 @@
      then
        pyvenv/bin/meson configure . -Dbackend_max_links="$LD_JOBS" ;
      fi || exit 1;
-    - section_end configure
-    - section_start build "Building QEMU"
    - $MAKE -j"$JOBS"
-    - section_end build
-    - section_start test "Running tests"
    - if test -n "$MAKE_CHECK_ARGS";
      then
        $MAKE -j"$JOBS" $MAKE_CHECK_ARGS ;
      fi
-    - section_end test
    - ccache --show-stats

 # We jump some hoops in common_test_job_template to avoid
@@ -64,89 +54,53 @@
  stage: test
  image: $CI_REGISTRY_IMAGE/qemu/$IMAGE:$QEMU_CI_CONTAINER_TAG
  script:
-    - source scripts/ci/gitlab-ci-section
-    - section_start buildenv "Setting up to run tests"
    - scripts/git-submodule.sh update roms/SLOF
-    - build/pyvenv/bin/meson subprojects download $(cd build/subprojects && echo *)
+    - meson subprojects download $(cd build/subprojects && echo *)
    - cd build
    - find . -type f -exec touch {} +
    # Avoid recompiling by hiding ninja with NINJA=":"
-    # We also have to pre-cache the functional tests manually in this case
-    - if [ "x${QEMU_TEST_CACHE_DIR}" != "x" ]; then
-        $MAKE precache-functional ;
-      fi
-    - section_end buildenv
-    - section_start test "Running tests"
-    # doctests need all the compilation artifacts
-    - $MAKE NINJA=":" MTESTARGS="--no-suite doc" $MAKE_CHECK_ARGS
-    - section_end test
+    - $MAKE NINJA=":" $MAKE_CHECK_ARGS

 .native_test_job_template:
  extends: .common_test_job_template
-  before_script:
-    # Prevent logs from the build job that run earlier
-    # from being duplicated in the test job artifacts
-    - rm -f build/meson-logs/*
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG"
    when: always
    expire_in: 7 days
    paths:
-      - build/meson-logs
+      - build/meson-logs/testlog.txt
    reports:
-      junit: build/meson-logs/*.junit.xml
+      junit: build/meson-logs/testlog.junit.xml

-.functional_test_job_template:
+.avocado_test_job_template:
  extends: .common_test_job_template
  cache:
    key: "${CI_JOB_NAME}-cache"
    paths:
-      - ${CI_PROJECT_DIR}/functional-cache
+      - ${CI_PROJECT_DIR}/avocado-cache
    policy: pull-push
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG"
    when: always
    expire_in: 7 days
    paths:
-      - build/meson-logs
-      - build/tests/functional/*/*/*.log
+      - build/tests/results/latest/results.xml
+      - build/tests/results/latest/test-results
    reports:
-      junit: build/meson-logs/*.junit.xml
+      junit: build/tests/results/latest/results.xml
  before_script:
-    - export QEMU_TEST_ALLOW_UNTRUSTED_CODE=1
-    - export QEMU_TEST_CACHE_DIR=${CI_PROJECT_DIR}/functional-cache
-    # Prevent logs from the build job that run earlier
-    # from being duplicated in the test job artifacts
-    - rm -f build/meson-logs/*
+    - mkdir -p ~/.config/avocado
+    - echo "[datadir.paths]" > ~/.config/avocado/avocado.conf
+    - echo "cache_dirs = ['${CI_PROJECT_DIR}/avocado-cache']"
+           >> ~/.config/avocado/avocado.conf
+    - echo -e '[job.output.testlogs]\nstatuses = ["FAIL", "INTERRUPT"]'
+           >> ~/.config/avocado/avocado.conf
+    - if [ -d ${CI_PROJECT_DIR}/avocado-cache ]; then
+        du -chs ${CI_PROJECT_DIR}/avocado-cache ;
+      fi
+    - export AVOCADO_ALLOW_UNTRUSTED_CODE=1
  after_script:
    - cd build
-    - du -chs ${CI_PROJECT_DIR}/*-cache
+    - du -chs ${CI_PROJECT_DIR}/avocado-cache
  variables:
-    QEMU_JOB_FUNCTIONAL: 1
-
-.wasm_build_job_template:
-  extends: .base_job_template
-  stage: build
-  image: $CI_REGISTRY_IMAGE/qemu/$IMAGE:$QEMU_CI_CONTAINER_TAG
-  before_script:
-    - source scripts/ci/gitlab-ci-section
-    - section_start setup "Pre-script setup"
-    - JOBS=$(expr $(nproc) + 1)
-    - section_end setup
-  script:
-    - du -sh .git
-    - mkdir build
-    - cd build
-    - section_start configure "Running configure"
-    - emconfigure ../configure --disable-docs
-          ${TARGETS:+--target-list="$TARGETS"}
-          $CONFIGURE_ARGS ||
-      { cat config.log meson-logs/meson-log.txt && exit 1; }
-    - if test -n "$LD_JOBS";
-      then
-        pyvenv/bin/meson configure . -Dbackend_max_links="$LD_JOBS" ;
-      fi || exit 1;
-    - section_end configure
-    - section_start build "Building QEMU"
-    - emmake make -j"$JOBS"
-    - section_end build
+    QEMU_JOB_AVOCADO: 1
@@ -22,24 +22,25 @@ check-system-alpine:
    IMAGE: alpine
    MAKE_CHECK_ARGS: check-unit check-qtest

-functional-system-alpine:
-  extends: .functional_test_job_template
+avocado-system-alpine:
+  extends: .avocado_test_job_template
  needs:
    - job: build-system-alpine
      artifacts: true
  variables:
    IMAGE: alpine
-    MAKE_CHECK_ARGS: check-functional
+    MAKE_CHECK_ARGS: check-avocado
+    AVOCADO_TAGS: arch:avr arch:loongarch64 arch:mips64 arch:mipsel

 build-system-ubuntu:
  extends:
    - .native_build_job_template
    - .native_build_artifact_template
  needs:
-    - job: amd64-ubuntu2204-container
+    job: amd64-ubuntu2204-container
  variables:
    IMAGE: ubuntu2204
-    CONFIGURE_ARGS: --enable-docs --enable-rust
+    CONFIGURE_ARGS: --enable-docs
    TARGETS: alpha-softmmu microblazeel-softmmu mips64el-softmmu
    MAKE_CHECK_ARGS: check-build

@@ -52,24 +53,25 @@ check-system-ubuntu:
    IMAGE: ubuntu2204
    MAKE_CHECK_ARGS: check

-functional-system-ubuntu:
-  extends: .functional_test_job_template
+avocado-system-ubuntu:
+  extends: .avocado_test_job_template
  needs:
    - job: build-system-ubuntu
      artifacts: true
  variables:
    IMAGE: ubuntu2204
-    MAKE_CHECK_ARGS: check-functional
+    MAKE_CHECK_ARGS: check-avocado
+    AVOCADO_TAGS: arch:alpha arch:microblazeel arch:mips64el

 build-system-debian:
  extends:
    - .native_build_job_template
    - .native_build_artifact_template
  needs:
-    - job: amd64-debian-container
+    job: amd64-debian-container
  variables:
    IMAGE: debian
-    CONFIGURE_ARGS: --with-coroutine=sigaltstack --enable-rust
+    CONFIGURE_ARGS: --with-coroutine=sigaltstack
    TARGETS: arm-softmmu i386-softmmu riscv64-softmmu sh4eb-softmmu
      sparc-softmmu xtensa-softmmu
    MAKE_CHECK_ARGS: check-build
@@ -83,14 +85,15 @@ check-system-debian:
    IMAGE: debian
    MAKE_CHECK_ARGS: check

-functional-system-debian:
-  extends: .functional_test_job_template
+avocado-system-debian:
+  extends: .avocado_test_job_template
  needs:
    - job: build-system-debian
      artifacts: true
  variables:
    IMAGE: debian
-    MAKE_CHECK_ARGS: check-functional
+    MAKE_CHECK_ARGS: check-avocado
+    AVOCADO_TAGS: arch:arm arch:i386 arch:riscv64 arch:sh4 arch:sparc arch:xtensa

 crash-test-debian:
  extends: .native_test_job_template
@@ -109,27 +112,13 @@ build-system-fedora:
    - .native_build_job_template
    - .native_build_artifact_template
  needs:
-    - job: amd64-fedora-container
+    job: amd64-fedora-container
  variables:
    IMAGE: fedora
-    CONFIGURE_ARGS: --disable-gcrypt --enable-nettle --enable-docs --enable-crypto-afalg --enable-rust
+    CONFIGURE_ARGS: --disable-gcrypt --enable-nettle --enable-docs
    TARGETS: microblaze-softmmu mips-softmmu
      xtensa-softmmu m68k-softmmu riscv32-softmmu ppc-softmmu sparc64-softmmu
-    MAKE_CHECK_ARGS: check-build check-doc
-
-build-system-fedora-rust-nightly:
-  extends:
-    - .native_build_job_template
-    - .native_build_artifact_template
-  needs:
-    - job: amd64-fedora-rust-nightly-container
-  variables:
-    IMAGE: fedora-rust-nightly
-    CONFIGURE_ARGS: --disable-docs --enable-rust --enable-strict-rust-lints
-    TARGETS: aarch64-softmmu
-    MAKE_CHECK_ARGS: check-build check-doc
-
-  allow_failure: true
+    MAKE_CHECK_ARGS: check-build

 check-system-fedora:
  extends: .native_test_job_template
@@ -140,14 +129,16 @@ check-system-fedora:
    IMAGE: fedora
    MAKE_CHECK_ARGS: check

-functional-system-fedora:
-  extends: .functional_test_job_template
+avocado-system-fedora:
+  extends: .avocado_test_job_template
  needs:
    - job: build-system-fedora
      artifacts: true
  variables:
    IMAGE: fedora
-    MAKE_CHECK_ARGS: check-functional
+    MAKE_CHECK_ARGS: check-avocado
+    AVOCADO_TAGS: arch:microblaze arch:mips arch:xtensa arch:m68k
+      arch:riscv32 arch:ppc arch:sparc64

 crash-test-fedora:
  extends: .native_test_job_template
@@ -167,7 +158,7 @@ build-system-centos:
    - .native_build_job_template
    - .native_build_artifact_template
  needs:
-    - job: amd64-centos9-container
+    job: amd64-centos9-container
  variables:
    IMAGE: centos9
    CONFIGURE_ARGS: --disable-nettle --enable-gcrypt --enable-vfio-user-server
@@ -183,25 +174,20 @@ build-previous-qemu:
    when: on_success
    expire_in: 2 days
    paths:
-      - build-previous/qemu-bundle
-      - build-previous/qemu-system-aarch64
-      - build-previous/qemu-system-x86_64
-      - build-previous/tests/qtest/migration-test
-      - build-previous/scripts
+      - build-previous
+    exclude:
+      - build-previous/**/*.p
+      - build-previous/**/*.a.p
+      - build-previous/**/*.c.o
+      - build-previous/**/*.c.o.d
  needs:
-    - job: amd64-opensuse-leap-container
+    job: amd64-opensuse-leap-container
  variables:
    IMAGE: opensuse-leap
    TARGETS: x86_64-softmmu aarch64-softmmu
    # Override the default flags as we need more to grab the old version
    GIT_FETCH_EXTRA_FLAGS: --prune --quiet
  before_script:
-    - source scripts/ci/gitlab-ci-section
-    # Skip if this series contains the release bump commit. During the
-    # release process there might be a window of commits when the
-    # version tag is not yet present in the remote and git fetch would
-    # fail.
-    - if grep -q "\.0$" VERSION; then exit 0; fi
    - export QEMU_PREV_VERSION="$(sed 's/\([0-9.]*\)\.[0-9]*/v\1.0/' VERSION)"
    - git remote add upstream https://gitlab.com/qemu-project/qemu
    - git fetch upstream refs/tags/$QEMU_PREV_VERSION:refs/tags/$QEMU_PREV_VERSION
@@ -222,9 +208,6 @@ build-previous-qemu:
    IMAGE: opensuse-leap
    MAKE_CHECK_ARGS: check-build
  script:
-    # Skip for round release numbers, this job is only relevant for
-    # testing a development tree.
-    - if grep -q "\.0$" VERSION; then exit 0; fi
    # Use the migration-tests from the older QEMU tree. This avoids
    # testing an old QEMU against new features/tests that it is not
    # compatible with.
@@ -260,21 +243,23 @@ check-system-centos:
    IMAGE: centos9
    MAKE_CHECK_ARGS: check

-functional-system-centos:
-  extends: .functional_test_job_template
+avocado-system-centos:
+  extends: .avocado_test_job_template
  needs:
    - job: build-system-centos
      artifacts: true
  variables:
    IMAGE: centos9
-    MAKE_CHECK_ARGS: check-functional
+    MAKE_CHECK_ARGS: check-avocado
+    AVOCADO_TAGS: arch:ppc64 arch:or1k arch:s390x arch:x86_64 arch:rx
+      arch:sh4

 build-system-opensuse:
  extends:
    - .native_build_job_template
    - .native_build_artifact_template
  needs:
-    - job: amd64-opensuse-leap-container
+    job: amd64-opensuse-leap-container
  variables:
    IMAGE: opensuse-leap
    TARGETS: s390x-softmmu x86_64-softmmu aarch64-softmmu
@@ -289,14 +274,15 @@ check-system-opensuse:
    IMAGE: opensuse-leap
    MAKE_CHECK_ARGS: check

-functional-system-opensuse:
-  extends: .functional_test_job_template
+avocado-system-opensuse:
+  extends: .avocado_test_job_template
  needs:
    - job: build-system-opensuse
      artifacts: true
  variables:
    IMAGE: opensuse-leap
-    MAKE_CHECK_ARGS: check-functional
+    MAKE_CHECK_ARGS: check-avocado
+    AVOCADO_TAGS: arch:s390x arch:x86_64 arch:aarch64

 #
 # Flaky tests. We don't run these by default and they are allow fail
@@ -308,7 +294,7 @@ build-system-flaky:
    - .native_build_job_template
    - .native_build_artifact_template
  needs:
-    - job: amd64-debian-container
+    job: amd64-debian-container
  variables:
    IMAGE: debian
    QEMU_JOB_OPTIONAL: 1
@@ -316,17 +302,18 @@ build-system-flaky:
      ppc64-softmmu rx-softmmu s390x-softmmu sh4-softmmu x86_64-softmmu
    MAKE_CHECK_ARGS: check-build

-functional-system-flaky:
-  extends: .functional_test_job_template
+avocado-system-flaky:
+  extends: .avocado_test_job_template
  needs:
    - job: build-system-flaky
      artifacts: true
  allow_failure: true
  variables:
    IMAGE: debian
-    MAKE_CHECK_ARGS: check-functional
+    MAKE_CHECK_ARGS: check-avocado
    QEMU_JOB_OPTIONAL: 1
    QEMU_TEST_FLAKY_TESTS: 1
+    AVOCADO_TAGS: flaky

 # This jobs explicitly disable TCG (--disable-tcg), KVM is detected by
 # the configure script. The container doesn't contain Xen headers so
@@ -338,7 +325,7 @@ functional-system-flaky:
 build-tcg-disabled:
  extends: .native_build_job_template
  needs:
-    - job: amd64-centos9-container
+    job: amd64-centos9-container
  variables:
    IMAGE: centos9
  script:
@@ -364,7 +351,7 @@ build-tcg-disabled:
 build-user:
  extends: .native_build_job_template
  needs:
-    - job: amd64-debian-user-cross-container
+    job: amd64-debian-user-cross-container
  variables:
    IMAGE: debian-all-test-cross
    CONFIGURE_ARGS: --disable-tools --disable-system
@@ -374,7 +361,7 @@ build-user:
 build-user-static:
  extends: .native_build_job_template
  needs:
-    - job: amd64-debian-user-cross-container
+    job: amd64-debian-user-cross-container
  variables:
    IMAGE: debian-all-test-cross
    CONFIGURE_ARGS: --disable-tools --disable-system --static
@@ -385,7 +372,7 @@ build-user-static:
 build-legacy:
  extends: .native_build_job_template
  needs:
-    - job: amd64-debian-legacy-cross-container
+    job: amd64-debian-legacy-cross-container
  variables:
    IMAGE: debian-legacy-test-cross
    TARGETS: alpha-linux-user alpha-softmmu sh4-linux-user
@@ -395,7 +382,7 @@ build-legacy:
 build-user-hexagon:
  extends: .native_build_job_template
  needs:
-    - job: hexagon-cross-container
+    job: hexagon-cross-container
  variables:
    IMAGE: debian-hexagon-cross
    TARGETS: hexagon-linux-user
@@ -408,7 +395,7 @@ build-user-hexagon:
 build-some-softmmu:
  extends: .native_build_job_template
  needs:
-    - job: amd64-debian-user-cross-container
+    job: amd64-debian-user-cross-container
  variables:
    IMAGE: debian-all-test-cross
    CONFIGURE_ARGS: --disable-tools --enable-debug
@@ -419,7 +406,7 @@ build-some-softmmu:
 build-loongarch64:
  extends: .native_build_job_template
  needs:
-    - job: loongarch-debian-cross-container
+    job: loongarch-debian-cross-container
  variables:
    IMAGE: debian-loongarch-cross
    CONFIGURE_ARGS: --disable-tools --enable-debug
@@ -430,7 +417,7 @@ build-loongarch64:
 build-tricore-softmmu:
  extends: .native_build_job_template
  needs:
-    - job: tricore-debian-cross-container
+    job: tricore-debian-cross-container
  variables:
    IMAGE: debian-tricore-cross
    CONFIGURE_ARGS: --disable-tools --disable-fdt --enable-debug
@@ -440,24 +427,26 @@ build-tricore-softmmu:
 clang-system:
  extends: .native_build_job_template
  needs:
-    - job: amd64-fedora-container
+    job: amd64-fedora-container
  variables:
    IMAGE: fedora
-    CONFIGURE_ARGS: --cc=clang --cxx=clang++ --enable-ubsan
-      --extra-cflags=-fno-sanitize-recover=undefined
+    CONFIGURE_ARGS: --cc=clang --cxx=clang++
+      --extra-cflags=-fsanitize=undefined --extra-cflags=-fno-sanitize-recover=undefined
+      --extra-cflags=-fno-sanitize=function
    TARGETS: alpha-softmmu arm-softmmu m68k-softmmu mips64-softmmu s390x-softmmu
    MAKE_CHECK_ARGS: check-qtest check-tcg

 clang-user:
  extends: .native_build_job_template
  needs:
-    - job: amd64-debian-user-cross-container
+    job: amd64-debian-user-cross-container
  timeout: 70m
  variables:
    IMAGE: debian-all-test-cross
-    CONFIGURE_ARGS: --cc=clang --cxx=clang++ --disable-system --enable-ubsan
+    CONFIGURE_ARGS: --cc=clang --cxx=clang++ --disable-system
      --target-list-exclude=alpha-linux-user,microblazeel-linux-user,aarch64_be-linux-user,i386-linux-user,m68k-linux-user,mipsn32el-linux-user,xtensaeb-linux-user
-      --extra-cflags=-fno-sanitize-recover=undefined
+      --extra-cflags=-fsanitize=undefined --extra-cflags=-fno-sanitize-recover=undefined
+      --extra-cflags=-fno-sanitize=function
    MAKE_CHECK_ARGS: check-unit check-tcg

 # Set LD_JOBS=1 because this requires LTO and ld consumes a large amount of memory.
@@ -467,8 +456,8 @@ clang-user:
 # Since slirp callbacks are used in QEMU Timers, we cannot use libslirp with
 # CFI builds, and thus have to disable it here.
 #
-# Split in three sets of build/check/functional to limit the execution time
-# of each job
+# Split in three sets of build/check/avocado to limit the execution time of each
+# job
 build-cfi-aarch64:
  extends:
    - .native_build_job_template
@@ -479,7 +468,7 @@ build-cfi-aarch64:
    LD_JOBS: 1
    AR: llvm-ar
    IMAGE: fedora
-    CONFIGURE_ARGS: --cc=clang --cxx=clang++ --enable-cfi
+    CONFIGURE_ARGS: --cc=clang --cxx=clang++ --enable-cfi --enable-cfi-debug
      --enable-safe-stack --disable-slirp
    TARGETS: aarch64-softmmu
    MAKE_CHECK_ARGS: check-build
@@ -498,14 +487,14 @@ check-cfi-aarch64:
    IMAGE: fedora
    MAKE_CHECK_ARGS: check

-functional-cfi-aarch64:
-  extends: .functional_test_job_template
+avocado-cfi-aarch64:
+  extends: .avocado_test_job_template
  needs:
    - job: build-cfi-aarch64
      artifacts: true
  variables:
    IMAGE: fedora
-    MAKE_CHECK_ARGS: check-functional
+    MAKE_CHECK_ARGS: check-avocado

 build-cfi-ppc64-s390x:
  extends:
@@ -517,7 +506,7 @@ build-cfi-ppc64-s390x:
    LD_JOBS: 1
    AR: llvm-ar
    IMAGE: fedora
-    CONFIGURE_ARGS: --cc=clang --cxx=clang++ --enable-cfi
+    CONFIGURE_ARGS: --cc=clang --cxx=clang++ --enable-cfi --enable-cfi-debug
      --enable-safe-stack --disable-slirp
    TARGETS: ppc64-softmmu s390x-softmmu
    MAKE_CHECK_ARGS: check-build
@@ -536,14 +525,14 @@ check-cfi-ppc64-s390x:
    IMAGE: fedora
    MAKE_CHECK_ARGS: check

-functional-cfi-ppc64-s390x:
-  extends: .functional_test_job_template
+avocado-cfi-ppc64-s390x:
+  extends: .avocado_test_job_template
  needs:
    - job: build-cfi-ppc64-s390x
      artifacts: true
  variables:
    IMAGE: fedora
-    MAKE_CHECK_ARGS: check-functional
+    MAKE_CHECK_ARGS: check-avocado

 build-cfi-x86_64:
  extends:
@@ -555,7 +544,7 @@ build-cfi-x86_64:
    LD_JOBS: 1
    AR: llvm-ar
    IMAGE: fedora
-    CONFIGURE_ARGS: --cc=clang --cxx=clang++ --enable-cfi
+    CONFIGURE_ARGS: --cc=clang --cxx=clang++ --enable-cfi --enable-cfi-debug
      --enable-safe-stack --disable-slirp
    TARGETS: x86_64-softmmu
    MAKE_CHECK_ARGS: check-build
@@ -570,19 +559,19 @@ check-cfi-x86_64:
    IMAGE: fedora
    MAKE_CHECK_ARGS: check

-functional-cfi-x86_64:
-  extends: .functional_test_job_template
+avocado-cfi-x86_64:
+  extends: .avocado_test_job_template
  needs:
    - job: build-cfi-x86_64
      artifacts: true
  variables:
    IMAGE: fedora
-    MAKE_CHECK_ARGS: check-functional
+    MAKE_CHECK_ARGS: check-avocado

 tsan-build:
  extends: .native_build_job_template
  needs:
-    - job: amd64-ubuntu2204-container
+    job: amd64-ubuntu2204-container
  variables:
    IMAGE: ubuntu2204
    CONFIGURE_ARGS: --enable-tsan --cc=clang --cxx=clang++
@@ -596,7 +585,7 @@ tsan-build:
 gcov:
  extends: .native_build_job_template
  needs:
-    - job: amd64-ubuntu2204-container
+    job: amd64-ubuntu2204-container
  timeout: 80m
  variables:
    IMAGE: ubuntu2204
@@ -613,9 +602,9 @@ gcov:
    when: always
    expire_in: 2 days
    paths:
-      - build/meson-logs
+      - build/meson-logs/testlog.txt
    reports:
-      junit: build/meson-logs/*.junit.xml
+      junit: build/meson-logs/testlog.junit.xml
      coverage_report:
        coverage_format: cobertura
        path: build/coverage.xml
@@ -623,7 +612,7 @@ gcov:
 build-oss-fuzz:
  extends: .native_build_job_template
  needs:
-    - job: amd64-fedora-container
+    job: amd64-fedora-container
  variables:
    IMAGE: fedora
  script:
@@ -632,20 +621,17 @@ build-oss-fuzz:
    - CC="clang" CXX="clang++" CFLAGS="-fsanitize=address"
      ./scripts/oss-fuzz/build.sh
    - export ASAN_OPTIONS="fast_unwind_on_malloc=0"
-    - failures=0
    - for fuzzer in $(find ./build-oss-fuzz/DEST_DIR/ -executable -type f
                      | grep -v slirp); do
        grep "LLVMFuzzerTestOneInput" ${fuzzer} > /dev/null 2>&1 || continue ;
        echo Testing ${fuzzer} ... ;
-        "${fuzzer}" -runs=1 -seed=1 || { echo "FAILED:"" ${fuzzer} exit code is $?"; failures=$(($failures+1)); };
+        "${fuzzer}" -runs=1 -seed=1 || exit 1 ;
      done
-    - echo "Number of failures:"" $failures"
-    - test $failures = 0

 build-tci:
  extends: .native_build_job_template
  needs:
-    - job: amd64-debian-user-cross-container
+    job: amd64-debian-user-cross-container
  variables:
    IMAGE: debian-all-test-cross
  script:
@@ -667,10 +653,13 @@ build-tci:
    - make check-tcg

 # Check our reduced build configurations
+# requires libfdt: aarch64, arm, loongarch64, microblaze, microblazeel,
+#   or1k, ppc64, riscv32, riscv64, rx
+# fails qtest without boards: i386, x86_64
 build-without-defaults:
  extends: .native_build_job_template
  needs:
-    - job: amd64-centos9-container
+    job: amd64-centos9-container
  variables:
    IMAGE: centos9
    CONFIGURE_ARGS:
@@ -680,7 +669,11 @@ build-without-defaults:
      --disable-pie
      --disable-qom-cast-debug
      --disable-strip
-      --target-list-exclude=aarch64-softmmu,microblaze-softmmu,mips64-softmmu,mipsel-softmmu,ppc64-softmmu,sh4el-softmmu,xtensa-softmmu,x86_64-softmmu
+    TARGETS: alpha-softmmu avr-softmmu cris-softmmu hppa-softmmu m68k-softmmu
+      mips-softmmu mips64-softmmu mipsel-softmmu mips64el-softmmu
+      ppc-softmmu s390x-softmmu sh4-softmmu sh4eb-softmmu sparc-softmmu
+      sparc64-softmmu tricore-softmmu xtensa-softmmu xtensaeb-softmmu
+      hexagon-linux-user i386-linux-user s390x-linux-user
    MAKE_CHECK_ARGS: check

 build-libvhost-user:
@@ -688,7 +681,7 @@ build-libvhost-user:
  stage: build
  image: $CI_REGISTRY_IMAGE/qemu/fedora:$QEMU_CI_CONTAINER_TAG
  needs:
-    - job: amd64-fedora-container
+    job: amd64-fedora-container
  script:
    - mkdir subprojects/libvhost-user/build
    - cd subprojects/libvhost-user/build
@@ -702,9 +695,9 @@ build-tools-and-docs-debian:
    - .native_build_job_template
    - .native_build_artifact_template
  needs:
-    - job: amd64-debian-container
-      # when running on 'master' we use pre-existing container
-      optional: true
+    job: amd64-debian-container
+    # when running on 'master' we use pre-existing container
+    optional: true
  variables:
    IMAGE: debian
    MAKE_CHECK_ARGS: check-unit ctags TAGS cscope
@@ -736,7 +729,7 @@ pages:
    - make gtags
    # We unset variables to work around a bug in some htags versions
    # which causes it to fail when the environment is large
-    - CI_COMMIT_MESSAGE= CI_COMMIT_TAG_MESSAGE= CI_COMMIT_DESCRIPTION= htags
+    - CI_COMMIT_MESSAGE= CI_COMMIT_TAG_MESSAGE= htags
        -anT --tree-view=filetree -m qemu_init
        -t "Welcome to the QEMU sourcecode"
    - mv HTML public/src
@@ -759,7 +752,7 @@ coverity:
    - job: amd64-fedora-container
      optional: true
  before_script:
-    - dnf install -y curl wget file
+    - dnf install -y curl wget
  script:
    # would be nice to cancel the job if over quota (https://gitlab.com/gitlab-org/gitlab/-/issues/256089)
    # for example:
@@ -786,12 +779,3 @@ coverity:
      when: never
    # Always manual on forks even if $QEMU_CI == "2"
    - when: manual
-
-build-wasm:
-  extends: .wasm_build_job_template
-  timeout: 2h
-  needs:
-    - job: wasm-emsdk-cross-container
-  variables:
-    IMAGE: emsdk-wasm32-cross
-    CONFIGURE_ARGS: --static --disable-tools --enable-debug --enable-tcg-interpreter
@@ -21,7 +21,7 @@ repourl = "https://gitlab.com/%s/%s.git" % (namespace, reponame)

 print(f"adding upstream git repo @ {repourl}")
 subprocess.check_call(["git", "remote", "add", "check-dco", repourl])
-subprocess.check_call(["git", "fetch", "--refetch", "check-dco", "master"])
+subprocess.check_call(["git", "fetch", "check-dco", "master"])

 ancestor = subprocess.check_output(["git", "merge-base",
                                    "check-dco/master", "HEAD"],
@@ -78,10 +78,7 @@ of Origin 1.1 (DCO):

 To indicate acceptance of the DCO every commit must have a tag

-  Signed-off-by: YOUR NAME <EMAIL>
-
-where "YOUR NAME" is your commonly known identity in the context
-of the community.
+  Signed-off-by: REAL NAME <EMAIL>

 This can be achieved by passing the "-s" flag to the "git commit" command.

@@ -24,7 +24,7 @@ print(f"adding upstream git repo @ {repourl}")
 # base for the user's branch. We thus need to figure out a common
 # ancestor between the user's branch and current git master.
 subprocess.check_call(["git", "remote", "add", "check-patch", repourl])
-subprocess.check_call(["git", "fetch", "--refetch", "check-patch", "master"])
+subprocess.check_call(["git", "fetch", "check-patch", "master"])

 ancestor = subprocess.check_output(["git", "merge-base",
                                    "check-patch/master", "HEAD"],
@@ -1,71 +0,0 @@
-#!/usr/bin/env python3
-#
-# check-units.py: check the number of compilation units and identify
-#                 those that are rebuilt multiple times
-#
-# Copyright (C) 2025 Linaro Ltd.
-#
-# SPDX-License-Identifier: GPL-2.0-or-later
-
-from os import access, R_OK, path
-from sys import exit
-import json
-import argparse
-from pathlib import Path
-from collections import Counter
-
-
-def extract_build_units(cc_path):
-    """
-    Extract the build units and their counds from compile_commands.json file.
-
-    Returns:
-        Hash table of ["unit"] = count
-    """
-
-    j = json.load(open(cc_path, 'r'))
-    files = [f['file'] for f in j]
-    build_units = Counter(files)
-
-    return build_units
-
-
-def analyse_units(build_units, top_n):
-    """
-    Analyse the build units and report stats and the top 10 rebuilds
-    """
-
-    print(f"Total source files: {len(build_units.keys())}")
-    print(f"Total build units: {sum(units.values())}")
-
-    # Create a sorted list by number of rebuilds
-    sorted_build_units = sorted(build_units.items(),
-                                key=lambda item: item[1],
-                                reverse=True)
-
-    print("Most rebuilt units:")
-    for unit, count in sorted_build_units[:top_n]:
-        print(f"  {unit} built {count} times")
-
-    print("Least rebuilt units:")
-    for unit, count in sorted_build_units[-10:]:
-        print(f"  {unit} built {count} times")
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(
-        description="analyse number of build units in compile_commands.json")
-    parser.add_argument("cc_path", type=Path, default=None,
-                        help="Path to compile_commands.json")
-    parser.add_argument("-n", type=int, default=20,
-                        help="Dump the top <n> entries")
-
-    args = parser.parse_args()
-
-    if path.isfile(args.cc_path) and access(args.cc_path, R_OK):
-        units = extract_build_units(args.cc_path)
-        analyse_units(units, args.n)
-        exit(0)
-    else:
-        print(f"{args.cc_path} doesn't exist or isn't readable")
-        exit(1)
@@ -15,46 +15,79 @@
  stage: build
  image: registry.gitlab.com/libvirt/libvirt-ci/cirrus-run:latest
  needs: []
-  allow_failure:
-    exit_codes: 3
  # 20 mins larger than "timeout_in" in cirrus/build.yml
  # as there's often a 5-10 minute delay before Cirrus CI
  # actually starts the task
  timeout: 80m
  script:
-    - set -o allexport
    - source .gitlab-ci.d/cirrus/$NAME.vars
-    - set +o allexport
-    - cirrus-vars <.gitlab-ci.d/cirrus/build.yml >.gitlab-ci.d/cirrus/$NAME.yml
+    - sed -e "s|[@]CI_REPOSITORY_URL@|$CI_REPOSITORY_URL|g"
+          -e "s|[@]CI_COMMIT_REF_NAME@|$CI_COMMIT_REF_NAME|g"
+          -e "s|[@]CI_COMMIT_SHA@|$CI_COMMIT_SHA|g"
+          -e "s|[@]CIRRUS_VM_INSTANCE_TYPE@|$CIRRUS_VM_INSTANCE_TYPE|g"
+          -e "s|[@]CIRRUS_VM_IMAGE_SELECTOR@|$CIRRUS_VM_IMAGE_SELECTOR|g"
+          -e "s|[@]CIRRUS_VM_IMAGE_NAME@|$CIRRUS_VM_IMAGE_NAME|g"
+          -e "s|[@]CIRRUS_VM_CPUS@|$CIRRUS_VM_CPUS|g"
+          -e "s|[@]CIRRUS_VM_RAM@|$CIRRUS_VM_RAM|g"
+          -e "s|[@]UPDATE_COMMAND@|$UPDATE_COMMAND|g"
+          -e "s|[@]INSTALL_COMMAND@|$INSTALL_COMMAND|g"
+          -e "s|[@]PATH@|$PATH_EXTRA${PATH_EXTRA:+:}\$PATH|g"
+          -e "s|[@]PKG_CONFIG_PATH@|$PKG_CONFIG_PATH|g"
+          -e "s|[@]PKGS@|$PKGS|g"
+          -e "s|[@]MAKE@|$MAKE|g"
+          -e "s|[@]PYTHON@|$PYTHON|g"
+          -e "s|[@]PIP3@|$PIP3|g"
+          -e "s|[@]PYPI_PKGS@|$PYPI_PKGS|g"
+          -e "s|[@]CONFIGURE_ARGS@|$CONFIGURE_ARGS|g"
+          -e "s|[@]TEST_TARGETS@|$TEST_TARGETS|g"
+      <.gitlab-ci.d/cirrus/build.yml >.gitlab-ci.d/cirrus/$NAME.yml
    - cat .gitlab-ci.d/cirrus/$NAME.yml
    - cirrus-run -v --show-build-log always .gitlab-ci.d/cirrus/$NAME.yml
  variables:
    QEMU_JOB_CIRRUS: 1

-x64-freebsd-14-build:
+x64-freebsd-13-build:
  extends: .cirrus_build_job
  variables:
-    NAME: freebsd-14
+    NAME: freebsd-13
    CIRRUS_VM_INSTANCE_TYPE: freebsd_instance
    CIRRUS_VM_IMAGE_SELECTOR: image_family
-    CIRRUS_VM_IMAGE_NAME: freebsd-14-2
+    CIRRUS_VM_IMAGE_NAME: freebsd-13-3
    CIRRUS_VM_CPUS: 8
    CIRRUS_VM_RAM: 8G
    UPDATE_COMMAND: pkg update; pkg upgrade -y
    INSTALL_COMMAND: pkg install -y
-    CONFIGURE_ARGS: --target-list-exclude=arm-softmmu,i386-softmmu,microblaze-softmmu,mips64el-softmmu,mipsel-softmmu,mips-softmmu,ppc-softmmu,sh4eb-softmmu,xtensa-softmmu --enable-rust
+    CONFIGURE_ARGS: --target-list-exclude=arm-softmmu,i386-softmmu,microblaze-softmmu,mips64el-softmmu,mipsel-softmmu,mips-softmmu,ppc-softmmu,sh4eb-softmmu,xtensa-softmmu
    TEST_TARGETS: check

-aarch64-macos-build:
+aarch64-macos-13-base-build:
  extends: .cirrus_build_job
  variables:
-    NAME: macos-14
+    NAME: macos-13
    CIRRUS_VM_INSTANCE_TYPE: macos_instance
    CIRRUS_VM_IMAGE_SELECTOR: image
-    CIRRUS_VM_IMAGE_NAME: ghcr.io/cirruslabs/macos-runner:sonoma
+    CIRRUS_VM_IMAGE_NAME: ghcr.io/cirruslabs/macos-ventura-base:latest
+    CIRRUS_VM_CPUS: 12
+    CIRRUS_VM_RAM: 24G
    UPDATE_COMMAND: brew update
    INSTALL_COMMAND: brew install
    PATH_EXTRA: /opt/homebrew/ccache/libexec:/opt/homebrew/gettext/bin
    PKG_CONFIG_PATH: /opt/homebrew/curl/lib/pkgconfig:/opt/homebrew/ncurses/lib/pkgconfig:/opt/homebrew/readline/lib/pkgconfig
    CONFIGURE_ARGS: --target-list-exclude=arm-softmmu,i386-softmmu,microblazeel-softmmu,mips64-softmmu,mipsel-softmmu,mips-softmmu,ppc-softmmu,sh4-softmmu,xtensaeb-softmmu
    TEST_TARGETS: check-unit check-block check-qapi-schema check-softfloat check-qtest-x86_64
+
+aarch64-macos-14-base-build:
+  extends: .cirrus_build_job
+  variables:
+    NAME: macos-14
+    CIRRUS_VM_INSTANCE_TYPE: macos_instance
+    CIRRUS_VM_IMAGE_SELECTOR: image
+    CIRRUS_VM_IMAGE_NAME: ghcr.io/cirruslabs/macos-sonoma-base:latest
+    CIRRUS_VM_CPUS: 12
+    CIRRUS_VM_RAM: 24G
+    UPDATE_COMMAND: brew update
+    INSTALL_COMMAND: brew install
+    PATH_EXTRA: /opt/homebrew/ccache/libexec:/opt/homebrew/gettext/bin
+    PKG_CONFIG_PATH: /opt/homebrew/curl/lib/pkgconfig:/opt/homebrew/ncurses/lib/pkgconfig:/opt/homebrew/readline/lib/pkgconfig
+    TEST_TARGETS: check-unit check-block check-qapi-schema check-softfloat check-qtest-x86_64
+    QEMU_JOB_OPTIONAL: 1
@@ -8,7 +8,7 @@ env:
  CI_REPOSITORY_URL: "@CI_REPOSITORY_URL@"
  CI_COMMIT_REF_NAME: "@CI_COMMIT_REF_NAME@"
  CI_COMMIT_SHA: "@CI_COMMIT_SHA@"
-  PATH: "@PATH_EXTRA@:$PATH"
+  PATH: "@PATH@"
  PKG_CONFIG_PATH: "@PKG_CONFIG_PATH@"
  PYTHON: "@PYTHON@"
  MAKE: "@MAKE@"
@@ -0,0 +1,16 @@
+# THIS FILE WAS AUTO-GENERATED
+#
+#  $ lcitool variables freebsd-13 qemu
+#
+# https://gitlab.com/libvirt/libvirt-ci
+
+CCACHE='/usr/local/bin/ccache'
+CPAN_PKGS=''
+CROSS_PKGS=''
+MAKE='/usr/local/bin/gmake'
+NINJA='/usr/local/bin/ninja'
+PACKAGING_COMMAND='pkg'
+PIP3='/usr/local/bin/pip-3.8'
+PKGS='alsa-lib bash bison bzip2 ca_root_nss capstone4 ccache cmocka ctags curl cyrus-sasl dbus diffutils dtc flex fusefs-libs3 gettext git glib gmake gnutls gsed gtk-vnc gtk3 json-c libepoxy libffi libgcrypt libjpeg-turbo libnfs libslirp libspice-server libssh libtasn1 llvm lzo2 meson mtools ncurses nettle ninja opencv pixman pkgconf png py311-numpy py311-pillow py311-pip py311-sphinx py311-sphinx_rtd_theme py311-tomli py311-yaml python3 rpm2cpio sdl2 sdl2_image snappy sndio socat spice-protocol tesseract usbredir virglrenderer vte3 xorriso zstd'
+PYPI_PKGS=''
+PYTHON='/usr/local/bin/python3'
@@ -1,16 +0,0 @@
-# THIS FILE WAS AUTO-GENERATED
-#
-#  $ lcitool variables freebsd-14 qemu
-#
-# https://gitlab.com/libvirt/libvirt-ci
-
-CCACHE='/usr/local/bin/ccache'
-CPAN_PKGS=''
-CROSS_PKGS=''
-MAKE='/usr/local/bin/gmake'
-NINJA='/usr/local/bin/ninja'
-PACKAGING_COMMAND='pkg'
-PIP3='/usr/local/bin/pip'
-PKGS='alsa-lib bash bison bzip2 ca_root_nss capstone4 ccache4 cmocka ctags curl cyrus-sasl dbus diffutils dtc flex fusefs-libs3 gettext git glib gmake gnutls gsed gtk-vnc gtk3 json-c libepoxy libffi libgcrypt libjpeg-turbo libnfs libslirp libspice-server libssh libtasn1 llvm lzo2 meson mtools ncurses nettle ninja opencv pixman pkgconf png py311-numpy py311-pillow py311-pip py311-pyyaml py311-sphinx py311-sphinx_rtd_theme py311-tomli python3 rpm2cpio rust rust-bindgen-cli sdl2 sdl2_image snappy sndio socat spice-protocol tesseract usbredir virglrenderer vte3 vulkan-tools xorriso zstd'
-PYPI_PKGS=''
-PYTHON='/usr/local/bin/python3'
@@ -0,0 +1,16 @@
+# THIS FILE WAS AUTO-GENERATED
+#
+#  $ lcitool variables macos-13 qemu
+#
+# https://gitlab.com/libvirt/libvirt-ci
+
+CCACHE='/opt/homebrew/bin/ccache'
+CPAN_PKGS=''
+CROSS_PKGS=''
+MAKE='/opt/homebrew/bin/gmake'
+NINJA='/opt/homebrew/bin/ninja'
+PACKAGING_COMMAND='brew'
+PIP3='/opt/homebrew/bin/pip3'
+PKGS='bash bc bison bzip2 capstone ccache cmocka ctags curl dbus diffutils dtc flex gcovr gettext git glib gnu-sed gnutls gtk+3 gtk-vnc jemalloc jpeg-turbo json-c libepoxy libffi libgcrypt libiscsi libnfs libpng libslirp libssh libtasn1 libusb llvm lzo make meson mtools ncurses nettle ninja pixman pkg-config python3 rpm2cpio sdl2 sdl2_image snappy socat sparse spice-protocol swtpm tesseract usbredir vde vte3 xorriso zlib zstd'
+PYPI_PKGS='PyYAML numpy pillow sphinx sphinx-rtd-theme tomli'
+PYTHON='/opt/homebrew/bin/python3'
@@ -11,6 +11,6 @@ MAKE='/opt/homebrew/bin/gmake'
 NINJA='/opt/homebrew/bin/ninja'
 PACKAGING_COMMAND='brew'
 PIP3='/opt/homebrew/bin/pip3'
-PKGS='bash bc bindgen bison bzip2 capstone ccache cmocka ctags curl dbus diffutils dtc flex gcovr gettext git glib gnu-sed gnutls gtk+3 gtk-vnc jemalloc jpeg-turbo json-c libcbor libepoxy libffi libgcrypt libiscsi libnfs libpng libslirp libssh libtasn1 libusb llvm lzo make meson mtools ncurses nettle ninja pixman pkg-config python3 rpm2cpio rust sdl2 sdl2_image snappy socat sparse spice-protocol swtpm tesseract usbredir vde vte3 vulkan-tools xorriso zlib zstd'
+PKGS='bash bc bison bzip2 capstone ccache cmocka ctags curl dbus diffutils dtc flex gcovr gettext git glib gnu-sed gnutls gtk+3 gtk-vnc jemalloc jpeg-turbo json-c libepoxy libffi libgcrypt libiscsi libnfs libpng libslirp libssh libtasn1 libusb llvm lzo make meson mtools ncurses nettle ninja pixman pkg-config python3 rpm2cpio sdl2 sdl2_image snappy socat sparse spice-protocol swtpm tesseract usbredir vde vte3 xorriso zlib zstd'
 PYPI_PKGS='PyYAML numpy pillow sphinx sphinx-rtd-theme tomli'
 PYTHON='/opt/homebrew/bin/python3'
@@ -52,6 +52,12 @@ mips64el-debian-cross-container:
  variables:
    NAME: debian-mips64el-cross

+mipsel-debian-cross-container:
+  extends: .container_job_template
+  stage: containers
+  variables:
+    NAME: debian-mipsel-cross
+
 ppc64el-debian-cross-container:
  extends: .container_job_template
  stage: containers
@@ -61,8 +67,11 @@ ppc64el-debian-cross-container:
 riscv64-debian-cross-container:
  extends: .container_job_template
  stage: containers
+  # as we are currently based on 'sid/unstable' we may break so...
+  allow_failure: true
  variables:
    NAME: debian-riscv64-cross
+    QEMU_JOB_OPTIONAL: 1

 s390x-debian-cross-container:
  extends: .container_job_template
@@ -81,12 +90,12 @@ xtensa-debian-cross-container:
  variables:
    NAME: debian-xtensa-cross

+cris-fedora-cross-container:
+  extends: .container_job_template
+  variables:
+    NAME: fedora-cris-cross
+
 win64-fedora-cross-container:
  extends: .container_job_template
  variables:
    NAME: fedora-win64-cross
-
-wasm-emsdk-cross-container:
-  extends: .container_job_template
-  variables:
-    NAME: emsdk-wasm32-cross
@@ -27,9 +27,3 @@ python-container:
  extends: .container_job_template
  variables:
    NAME: python
-
-amd64-fedora-rust-nightly-container:
-  extends: .container_job_template
-  variables:
-    NAME: fedora-rust-nightly
-  allow_failure: true
@@ -9,11 +9,7 @@
    when: always
  timeout: 80m
  before_script:
-    - source scripts/ci/gitlab-ci-section
-    - section_start setup "Pre-script setup"
-    - JOBS=$(expr $(nproc) + 1)
    - cat /packages.txt
-    - section_end setup
  script:
    - export CCACHE_BASEDIR="$(pwd)"
    - export CCACHE_DIR="$CCACHE_BASEDIR/ccache"
@@ -22,30 +18,18 @@
    - mkdir build
    - cd build
    - ccache --zero-stats
-    - section_start configure "Running configure"
    - ../configure --enable-werror --disable-docs --enable-fdt=system
        --disable-user $QEMU_CONFIGURE_OPTS $EXTRA_CONFIGURE_OPTS
-        --target-list-exclude="arm-softmmu
+        --target-list-exclude="arm-softmmu cris-softmmu
          i386-softmmu microblaze-softmmu mips-softmmu mipsel-softmmu
          mips64-softmmu ppc-softmmu riscv32-softmmu sh4-softmmu
          sparc-softmmu xtensa-softmmu $CROSS_SKIP_TARGETS"
-    - section_end configure
-    - section_start build "Building QEMU"
-    - make -j"$JOBS" all check-build
-    - section_end build
-    - section_start test "Running tests"
-    - if test -n "$MAKE_CHECK_ARGS";
-      then
-        $MAKE -j"$JOBS" $MAKE_CHECK_ARGS ;
-      fi
-    - section_end test
-    - section_start installer "Building the installer"
+    - make -j$(expr $(nproc) + 1) all check-build $MAKE_CHECK_ARGS
    - if grep -q "EXESUF=.exe" config-host.mak;
      then make installer;
      version="$(git describe --match v[0-9]* 2>/dev/null || git rev-parse --short HEAD)";
      mv -v qemu-setup*.exe qemu-setup-${version}.exe;
      fi
-    - section_end installer
    - ccache --show-stats

 # Job to cross-build specific accelerators.
@@ -57,14 +41,11 @@
  extends: .base_job_template
  stage: build
  image: $CI_REGISTRY_IMAGE/qemu/$IMAGE:$QEMU_CI_CONTAINER_TAG
-  timeout: 60m
+  timeout: 30m
  cache:
    paths:
      - ccache/
    key: "$CI_JOB_NAME"
-  before_script:
-    - source scripts/ci/gitlab-ci-section
-    - JOBS=$(expr $(nproc) + 1)
  script:
    - export CCACHE_BASEDIR="$(pwd)"
    - export CCACHE_DIR="$CCACHE_BASEDIR/ccache"
@@ -72,19 +53,9 @@
    - export PATH="$CCACHE_WRAPPERSDIR:$PATH"
    - mkdir build
    - cd build
-    - section_start configure "Running configure"
    - ../configure --enable-werror --disable-docs $QEMU_CONFIGURE_OPTS
        --disable-tools --enable-${ACCEL:-kvm} $EXTRA_CONFIGURE_OPTS
-    - section_end configure
-    - section_start build "Building QEMU"
-    - make -j"$JOBS" all check-build
-    - section_end build
-    - section_start test "Running tests"
-    - if test -n "$MAKE_CHECK_ARGS";
-      then
-        $MAKE -j"$JOBS" $MAKE_CHECK_ARGS ;
-      fi
-    - section_end test
+    - make -j$(expr $(nproc) + 1) all check-build $MAKE_CHECK_ARGS

 .cross_user_build_job:
  extends: .base_job_template
@@ -94,31 +65,18 @@
    paths:
      - ccache/
    key: "$CI_JOB_NAME"
-  before_script:
-    - source scripts/ci/gitlab-ci-section
-    - JOBS=$(expr $(nproc) + 1)
  script:
    - export CCACHE_BASEDIR="$(pwd)"
    - export CCACHE_DIR="$CCACHE_BASEDIR/ccache"
    - export CCACHE_MAXSIZE="500M"
    - mkdir build
    - cd build
-    - section_start configure "Running configure"
    - ../configure --enable-werror --disable-docs $QEMU_CONFIGURE_OPTS
        --disable-system --target-list-exclude="aarch64_be-linux-user
-          alpha-linux-user m68k-linux-user microblazeel-linux-user
+          alpha-linux-user cris-linux-user m68k-linux-user microblazeel-linux-user
          or1k-linux-user ppc-linux-user sparc-linux-user
          xtensa-linux-user $CROSS_SKIP_TARGETS"
-    - section_end configure
-    - section_start build "Building QEMU"
-    - make -j"$JOBS" all check-build
-    - section_end build
-    - section_start test "Running tests"
-    - if test -n "$MAKE_CHECK_ARGS";
-      then
-        $MAKE -j"$JOBS" $MAKE_CHECK_ARGS ;
-      fi
-    - section_end test
+    - make -j$(expr $(nproc) + 1) all check-build $MAKE_CHECK_ARGS

 # We can still run some tests on some of our cross build jobs. They can add this
 # template to their extends to save the build logs and test results
@@ -128,6 +86,6 @@
    when: always
    expire_in: 7 days
    paths:
-      - build/meson-logs
+      - build/meson-logs/testlog.txt
    reports:
-      junit: build/meson-logs/*.junit.xml
+      junit: build/meson-logs/testlog.junit.xml
@@ -4,28 +4,28 @@ include:
 cross-armhf-user:
  extends: .cross_user_build_job
  needs:
-    - job: armhf-debian-cross-container
+    job: armhf-debian-cross-container
  variables:
    IMAGE: debian-armhf-cross

 cross-arm64-system:
  extends: .cross_system_build_job
  needs:
-    - job: arm64-debian-cross-container
+    job: arm64-debian-cross-container
  variables:
    IMAGE: debian-arm64-cross

 cross-arm64-user:
  extends: .cross_user_build_job
  needs:
-    - job: arm64-debian-cross-container
+    job: arm64-debian-cross-container
  variables:
    IMAGE: debian-arm64-cross

 cross-arm64-kvm-only:
  extends: .cross_accel_build_job
  needs:
-    - job: arm64-debian-cross-container
+    job: arm64-debian-cross-container
  variables:
    IMAGE: debian-arm64-cross
    EXTRA_CONFIGURE_OPTS: --disable-tcg --without-default-features
@@ -35,7 +35,7 @@ cross-i686-system:
    - .cross_system_build_job
    - .cross_test_artifacts
  needs:
-    - job: i686-debian-cross-container
+    job: i686-debian-cross-container
  variables:
    IMAGE: debian-i686-cross
    EXTRA_CONFIGURE_OPTS: --disable-kvm
@@ -46,7 +46,7 @@ cross-i686-user:
    - .cross_user_build_job
    - .cross_test_artifacts
  needs:
-    - job: i686-debian-cross-container
+    job: i686-debian-cross-container
  variables:
    IMAGE: debian-i686-cross
    MAKE_CHECK_ARGS: check
@@ -57,85 +57,100 @@ cross-i686-tci:
    - .cross_test_artifacts
  timeout: 60m
  needs:
-    - job: i686-debian-cross-container
+    job: i686-debian-cross-container
  variables:
    IMAGE: debian-i686-cross
    ACCEL: tcg-interpreter
-    EXTRA_CONFIGURE_OPTS: --target-list=i386-softmmu,i386-linux-user,arm-softmmu,arm-linux-user,ppc-softmmu,ppc-linux-user --disable-plugins --disable-kvm
-    # Force tests to run with reduced parallelism, to see whether this
-    # reduces the flakiness of this CI job. The CI
-    # environment by default shows us 8 CPUs and so we
-    # would otherwise be using a parallelism of 9.
-    MAKE_CHECK_ARGS: check check-tcg -j2
+    EXTRA_CONFIGURE_OPTS: --target-list=i386-softmmu,i386-linux-user,aarch64-softmmu,aarch64-linux-user,ppc-softmmu,ppc-linux-user --disable-plugins --disable-kvm
+    MAKE_CHECK_ARGS: check check-tcg
+
+cross-mipsel-system:
+  extends: .cross_system_build_job
+  needs:
+    job: mipsel-debian-cross-container
+  variables:
+    IMAGE: debian-mipsel-cross
+
+cross-mipsel-user:
+  extends: .cross_user_build_job
+  needs:
+    job: mipsel-debian-cross-container
+  variables:
+    IMAGE: debian-mipsel-cross

 cross-mips64el-system:
  extends: .cross_system_build_job
  needs:
-    - job: mips64el-debian-cross-container
+    job: mips64el-debian-cross-container
  variables:
    IMAGE: debian-mips64el-cross

 cross-mips64el-user:
  extends: .cross_user_build_job
  needs:
-    - job: mips64el-debian-cross-container
+    job: mips64el-debian-cross-container
  variables:
    IMAGE: debian-mips64el-cross

 cross-ppc64el-system:
  extends: .cross_system_build_job
  needs:
-    - job: ppc64el-debian-cross-container
+    job: ppc64el-debian-cross-container
  variables:
    IMAGE: debian-ppc64el-cross

 cross-ppc64el-user:
  extends: .cross_user_build_job
  needs:
-    - job: ppc64el-debian-cross-container
+    job: ppc64el-debian-cross-container
  variables:
    IMAGE: debian-ppc64el-cross

 cross-ppc64el-kvm-only:
  extends: .cross_accel_build_job
  needs:
-    - job: ppc64el-debian-cross-container
+    job: ppc64el-debian-cross-container
  variables:
    IMAGE: debian-ppc64el-cross
    EXTRA_CONFIGURE_OPTS: --disable-tcg --without-default-devices

+# The riscv64 cross-builds currently use a 'sid' container to get
+# compilers and libraries. Until something more stable is found we
+# allow_failure so as not to block CI.
 cross-riscv64-system:
  extends: .cross_system_build_job
+  allow_failure: true
  needs:
-    - job: riscv64-debian-cross-container
+    job: riscv64-debian-cross-container
  variables:
    IMAGE: debian-riscv64-cross

 cross-riscv64-user:
  extends: .cross_user_build_job
+  allow_failure: true
  needs:
-    - job: riscv64-debian-cross-container
+    job: riscv64-debian-cross-container
  variables:
    IMAGE: debian-riscv64-cross

 cross-s390x-system:
  extends: .cross_system_build_job
  needs:
-    - job: s390x-debian-cross-container
+    job: s390x-debian-cross-container
  variables:
    IMAGE: debian-s390x-cross

 cross-s390x-user:
  extends: .cross_user_build_job
  needs:
-    - job: s390x-debian-cross-container
+    job: s390x-debian-cross-container
  variables:
    IMAGE: debian-s390x-cross

 cross-s390x-kvm-only:
  extends: .cross_accel_build_job
  needs:
-    - job: s390x-debian-cross-container
+    job: s390x-debian-cross-container
  variables:
    IMAGE: debian-s390x-cross
    EXTRA_CONFIGURE_OPTS: --disable-tcg --enable-trace-backends=ftrace
@@ -143,7 +158,7 @@ cross-s390x-kvm-only:
 cross-mips64el-kvm-only:
  extends: .cross_accel_build_job
  needs:
-    - job: mips64el-debian-cross-container
+    job: mips64el-debian-cross-container
  variables:
    IMAGE: debian-mips64el-cross
    EXTRA_CONFIGURE_OPTS: --disable-tcg --target-list=mips64el-softmmu
@@ -151,7 +166,7 @@ cross-mips64el-kvm-only:
 cross-win64-system:
  extends: .cross_system_build_job
  needs:
-    - job: win64-fedora-cross-container
+    job: win64-fedora-cross-container
  variables:
    IMAGE: fedora-win64-cross
    EXTRA_CONFIGURE_OPTS: --enable-fdt=internal --disable-plugins
@@ -167,7 +182,7 @@ cross-win64-system:
 cross-amd64-xen-only:
  extends: .cross_accel_build_job
  needs:
-    - job: amd64-debian-cross-container
+    job: amd64-debian-cross-container
  variables:
    IMAGE: debian-amd64-cross
    ACCEL: xen
@@ -176,7 +191,7 @@ cross-amd64-xen-only:
 cross-arm64-xen-only:
  extends: .cross_accel_build_job
  needs:
-    - job: arm64-debian-cross-container
+    job: arm64-debian-cross-container
  variables:
    IMAGE: debian-arm64-cross
    ACCEL: xen
@@ -26,8 +26,9 @@
      - build/build.ninja
      - build/meson-logs
    reports:
-      junit: build/meson-logs/*.junit.xml
+      junit: build/meson-logs/testlog.junit.xml

 include:
-  - local: '/.gitlab-ci.d/custom-runners/ubuntu-24.04-s390x.yml'
-  - local: '/.gitlab-ci.d/custom-runners/ubuntu-24.04-aarch64.yml'
+  - local: '/.gitlab-ci.d/custom-runners/ubuntu-22.04-s390x.yml'
+  - local: '/.gitlab-ci.d/custom-runners/ubuntu-22.04-aarch64.yml'
+  - local: '/.gitlab-ci.d/custom-runners/ubuntu-22.04-aarch32.yml'
@@ -0,0 +1,25 @@
+# All ubuntu-22.04 jobs should run successfully in an environment
+# setup by the scripts/ci/setup/ubuntu/build-environment.yml task
+# "Install basic packages to build QEMU on Ubuntu 22.04"
+
+ubuntu-22.04-aarch32-all:
+ extends: .custom_runner_template
+ needs: []
+ stage: build
+ tags:
+ - ubuntu_22.04
+ - aarch32
+ rules:
+ - if: '$CI_PROJECT_NAMESPACE == "qemu-project" && $CI_COMMIT_BRANCH =~ /^staging/'
+   when: manual
+   allow_failure: true
+ - if: "$AARCH32_RUNNER_AVAILABLE"
+   when: manual
+   allow_failure: true
+ script:
+ - mkdir build
+ - cd build
+ - ../configure --cross-prefix=arm-linux-gnueabihf-
+   || { cat config.log meson-logs/meson-log.txt; exit 1; }
+ - make --output-sync -j`nproc --ignore=40`
+ - make --output-sync -j`nproc --ignore=40` check
@@ -0,0 +1,151 @@
+# All ubuntu-22.04 jobs should run successfully in an environment
+# setup by the scripts/ci/setup/ubuntu/build-environment.yml task
+# "Install basic packages to build QEMU on Ubuntu 22.04"
+
+ubuntu-22.04-aarch64-all-linux-static:
+ extends: .custom_runner_template
+ needs: []
+ stage: build
+ tags:
+ - ubuntu_22.04
+ - aarch64
+ rules:
+ - if: '$CI_PROJECT_NAMESPACE == "qemu-project" && $CI_COMMIT_BRANCH =~ /^staging/'
+ - if: "$AARCH64_RUNNER_AVAILABLE"
+ script:
+ - mkdir build
+ - cd build
+ # Disable -static-pie due to build error with system libc:
+ # https://bugs.launchpad.net/ubuntu/+source/glibc/+bug/1987438
+ - ../configure --enable-debug --static --disable-system --disable-pie
+   || { cat config.log meson-logs/meson-log.txt; exit 1; }
+ - make --output-sync -j`nproc --ignore=40`
+ - make check-tcg
+ - make --output-sync -j`nproc --ignore=40` check
+
+ubuntu-22.04-aarch64-all:
+ extends: .custom_runner_template
+ needs: []
+ stage: build
+ tags:
+ - ubuntu_22.04
+ - aarch64
+ rules:
+ - if: '$CI_PROJECT_NAMESPACE == "qemu-project" && $CI_COMMIT_BRANCH =~ /^staging/'
+   when: manual
+   allow_failure: true
+ - if: "$AARCH64_RUNNER_AVAILABLE"
+   when: manual
+   allow_failure: true
+ script:
+ - mkdir build
+ - cd build
+ - ../configure
+   || { cat config.log meson-logs/meson-log.txt; exit 1; }
+ - make --output-sync -j`nproc --ignore=40`
+ - make --output-sync -j`nproc --ignore=40` check
+
+ubuntu-22.04-aarch64-without-defaults:
+ extends: .custom_runner_template
+ needs: []
+ stage: build
+ tags:
+ - ubuntu_22.04
+ - aarch64
+ rules:
+ - if: '$CI_PROJECT_NAMESPACE == "qemu-project" && $CI_COMMIT_BRANCH =~ /^staging/'
+   when: manual
+   allow_failure: true
+ - if: "$AARCH64_RUNNER_AVAILABLE"
+   when: manual
+   allow_failure: true
+ script:
+ - mkdir build
+ - cd build
+ - ../configure --disable-user --without-default-devices --without-default-features
+   || { cat config.log meson-logs/meson-log.txt; exit 1; }
+ - make --output-sync -j`nproc --ignore=40`
+ - make --output-sync -j`nproc --ignore=40` check
+
+ubuntu-22.04-aarch64-alldbg:
+ extends: .custom_runner_template
+ needs: []
+ stage: build
+ tags:
+ - ubuntu_22.04
+ - aarch64
+ rules:
+ - if: '$CI_PROJECT_NAMESPACE == "qemu-project" && $CI_COMMIT_BRANCH =~ /^staging/'
+ - if: "$AARCH64_RUNNER_AVAILABLE"
+ script:
+ - mkdir build
+ - cd build
+ - ../configure --enable-debug
+   || { cat config.log meson-logs/meson-log.txt; exit 1; }
+ - make clean
+ - make --output-sync -j`nproc --ignore=40`
+ - make --output-sync -j`nproc --ignore=40` check
+
+ubuntu-22.04-aarch64-clang:
+ extends: .custom_runner_template
+ needs: []
+ stage: build
+ tags:
+ - ubuntu_22.04
+ - aarch64
+ rules:
+ - if: '$CI_PROJECT_NAMESPACE == "qemu-project" && $CI_COMMIT_BRANCH =~ /^staging/'
+   when: manual
+   allow_failure: true
+ - if: "$AARCH64_RUNNER_AVAILABLE"
+   when: manual
+   allow_failure: true
+ script:
+ - mkdir build
+ - cd build
+ - ../configure --disable-libssh --cc=clang --cxx=clang++ --enable-sanitizers
+   || { cat config.log meson-logs/meson-log.txt; exit 1; }
+ - make --output-sync -j`nproc --ignore=40`
+ - make --output-sync -j`nproc --ignore=40` check
+
+ubuntu-22.04-aarch64-tci:
+ needs: []
+ stage: build
+ tags:
+ - ubuntu_22.04
+ - aarch64
+ rules:
+ - if: '$CI_PROJECT_NAMESPACE == "qemu-project" && $CI_COMMIT_BRANCH =~ /^staging/'
+   when: manual
+   allow_failure: true
+ - if: "$AARCH64_RUNNER_AVAILABLE"
+   when: manual
+   allow_failure: true
+ script:
+ - mkdir build
+ - cd build
+ - ../configure --enable-tcg-interpreter
+   || { cat config.log meson-logs/meson-log.txt; exit 1; }
+ - make --output-sync -j`nproc --ignore=40`
+
+ubuntu-22.04-aarch64-notcg:
+ extends: .custom_runner_template
+ needs: []
+ stage: build
+ tags:
+ - ubuntu_22.04
+ - aarch64
+ rules:
+ - if: '$CI_PROJECT_NAMESPACE == "qemu-project" && $CI_COMMIT_BRANCH =~ /^staging/'
+   when: manual
+   allow_failure: true
+ - if: "$AARCH64_RUNNER_AVAILABLE"
+   when: manual
+   allow_failure: true
+ script:
+ - mkdir build
+ - cd build
+ - ../configure --disable-tcg --with-devices-aarch64=minimal
+   || { cat config.log meson-logs/meson-log.txt; exit 1; }
+ - make --output-sync -j`nproc --ignore=40`
+ - make --output-sync -j`nproc --ignore=40` check
@@ -1,13 +1,13 @@
-# All ubuntu-24.04 jobs should run successfully in an environment
+# All ubuntu-22.04 jobs should run successfully in an environment
 # setup by the scripts/ci/setup/ubuntu/build-environment.yml task
-# "Install basic packages to build QEMU on Ubuntu 24.04"
+# "Install basic packages to build QEMU on Ubuntu 22.04"

-ubuntu-24.04-s390x-all-linux:
+ubuntu-22.04-s390x-all-linux:
 extends: .custom_runner_template
 needs: []
 stage: build
 tags:
- - ubuntu_24.04
+ - ubuntu_22.04
 - s390x
 rules:
 - if: '$CI_PROJECT_NAMESPACE == "qemu-project" && $CI_COMMIT_BRANCH =~ /^staging/'
@@ -21,19 +21,17 @@ ubuntu-24.04-s390x-all-linux:
 - make --output-sync check-tcg
 - make --output-sync -j`nproc` check

-ubuntu-24.04-s390x-all-system:
+ubuntu-22.04-s390x-all-system:
 extends: .custom_runner_template
 needs: []
 stage: build
 tags:
- - ubuntu_24.04
+ - ubuntu_22.04
 - s390x
 timeout: 75m
 rules:
 - if: '$CI_PROJECT_NAMESPACE == "qemu-project" && $CI_COMMIT_BRANCH =~ /^staging/'
-   allow_failure: true
 - if: "$S390X_RUNNER_AVAILABLE"
-   allow_failure: true
 script:
 - mkdir build
 - cd build
@@ -42,12 +40,12 @@ ubuntu-24.04-s390x-all-system:
 - make --output-sync -j`nproc`
 - make --output-sync -j`nproc` check

-ubuntu-24.04-s390x-alldbg:
+ubuntu-22.04-s390x-alldbg:
 extends: .custom_runner_template
 needs: []
 stage: build
 tags:
- - ubuntu_24.04
+ - ubuntu_22.04
 - s390x
 rules:
 - if: '$CI_PROJECT_NAMESPACE == "qemu-project" && $CI_COMMIT_BRANCH =~ /^staging/'
@@ -65,12 +63,12 @@ ubuntu-24.04-s390x-alldbg:
 - make --output-sync -j`nproc`
 - make --output-sync -j`nproc` check

-ubuntu-24.04-s390x-clang:
+ubuntu-22.04-s390x-clang:
 extends: .custom_runner_template
 needs: []
 stage: build
 tags:
- - ubuntu_24.04
+ - ubuntu_22.04
 - s390x
 rules:
 - if: '$CI_PROJECT_NAMESPACE == "qemu-project" && $CI_COMMIT_BRANCH =~ /^staging/'
@@ -82,16 +80,16 @@ ubuntu-24.04-s390x-clang:
 script:
 - mkdir build
 - cd build
- - ../configure --cc=clang --cxx=clang++ --enable-ubsan
+ - ../configure --cc=clang --cxx=clang++ --enable-sanitizers
   || { cat config.log meson-logs/meson-log.txt; exit 1; }
 - make --output-sync -j`nproc`
 - make --output-sync -j`nproc` check

-ubuntu-24.04-s390x-tci:
+ubuntu-22.04-s390x-tci:
 needs: []
 stage: build
 tags:
- - ubuntu_24.04
+ - ubuntu_22.04
 - s390x
 rules:
 - if: '$CI_PROJECT_NAMESPACE == "qemu-project" && $CI_COMMIT_BRANCH =~ /^staging/'
@@ -107,12 +105,12 @@ ubuntu-24.04-s390x-tci:
   || { cat config.log meson-logs/meson-log.txt; exit 1; }
 - make --output-sync -j`nproc`

-ubuntu-24.04-s390x-notcg:
+ubuntu-22.04-s390x-notcg:
 extends: .custom_runner_template
 needs: []
 stage: build
 tags:
- - ubuntu_24.04
+ - ubuntu_22.04
 - s390x
 rules:
 - if: '$CI_PROJECT_NAMESPACE == "qemu-project" && $CI_COMMIT_BRANCH =~ /^staging/'
@@ -1,111 +0,0 @@
-# All ubuntu-24.04 jobs should run successfully in an environment
-# setup by the scripts/ci/setup/ubuntu/build-environment.yml task
-# "Install basic packages to build QEMU on Ubuntu 24.04"
-
-.ubuntu_aarch64_template:
-  extends: .custom_runner_template
-  needs: []
-  stage: build
-  tags:
-    - ubuntu_24.04
-    - aarch64
-  rules:
-    - if: '$CI_PROJECT_NAMESPACE == "qemu-project" && $CI_COMMIT_BRANCH =~ /^staging/'
-    - if: "$AARCH64_RUNNER_AVAILABLE"
-  before_script:
-    - source scripts/ci/gitlab-ci-section
-    - section_start setup "Pre-script setup"
-    - JOBS=$(expr $(nproc) - 4)
-    - section_end setup
-  script:
-    - mkdir build
-    - cd build
-    - section_start configure "Running configure"
-    - ../configure $CONFIGURE_ARGS ||
-          { cat config.log meson-logs/meson-log.txt && exit 1; }
-    - section_end configure
-    - section_start build "Building QEMU"
-    - make --output-sync -j"$JOBS"
-    - section_end build
-    - section_start test "Running tests"
-    - if test -n "$MAKE_CHECK_ARGS";
-      then
-        make -j"$JOBS" $MAKE_CHECK_ARGS ;
-      fi
-    - section_end test
-
-ubuntu-24.04-aarch64-all-linux-static:
-  extends: .ubuntu_aarch64_template
-  variables:
-    # Disable -static-pie due to build error with system libc:
-    # https://bugs.launchpad.net/ubuntu/+source/glibc/+bug/1987438
-    CONFIGURE_ARGS: --enable-debug --static --disable-system --disable-pie
-    MAKE_CHECK_ARGS: check-tcg
-
-ubuntu-24.04-aarch64-all:
-  extends: .ubuntu_aarch64_template
-  variables:
-    MAKE_CHECK_ARGS: check
-  rules:
-    - if: '$CI_PROJECT_NAMESPACE == "qemu-project" && $CI_COMMIT_BRANCH =~ /^staging/'
-      when: manual
-      allow_failure: true
-    - if: "$AARCH64_RUNNER_AVAILABLE"
-      when: manual
-      allow_failure: true
-
-ubuntu-24.04-aarch64-without-defaults:
-  extends: .ubuntu_aarch64_template
-  variables:
-    CONFIGURE_ARGS: --disable-user --without-default-devices --without-default-features
-    MAKE_CHECK_ARGS: check
-  rules:
-    - if: '$CI_PROJECT_NAMESPACE == "qemu-project" && $CI_COMMIT_BRANCH =~ /^staging/'
-      when: manual
-      allow_failure: true
-    - if: "$AARCH64_RUNNER_AVAILABLE"
-      when: manual
-      allow_failure: true
-
-ubuntu-24.04-aarch64-alldbg:
-  extends: .ubuntu_aarch64_template
-  variables:
-    CONFIGURE_ARGS: --enable-debug
-    MAKE_CHECK_ARGS: check-tcg
-
-ubuntu-24.04-aarch64-clang:
-  extends: .ubuntu_aarch64_template
-  variables:
-    CONFIGURE_ARGS: --cc=clang --cxx=clang++ --enable-ubsan
-    MAKE_CHECK_ARGS: check
-  rules:
-    - if: '$CI_PROJECT_NAMESPACE == "qemu-project" && $CI_COMMIT_BRANCH =~ /^staging/'
-      when: manual
-      allow_failure: true
-    - if: "$AARCH64_RUNNER_AVAILABLE"
-      when: manual
-      allow_failure: true
-
-ubuntu-24.04-aarch64-tci:
-  extends: .ubuntu_aarch64_template
-  variables:
-    CONFIGURE_ARGS: --enable-tcg-interpreter
-    MAKE_CHECK_ARGS: check
-  rules:
-    - if: '$CI_PROJECT_NAMESPACE == "qemu-project" && $CI_COMMIT_BRANCH =~ /^staging/'
-      when: manual
-      allow_failure: true
-    - if: "$AARCH64_RUNNER_AVAILABLE"
-      when: manual
-      allow_failure: true
-
-ubuntu-24.04-aarch64-notcg:
-  extends: .ubuntu_aarch64_template
-  variables:
-    CONFIGURE_ARGS: --disable-tcg --with-devices-aarch64=minimal
-    MAKE_CHECK_ARGS: check
-  rules:
-    - if: '$CI_PROJECT_NAMESPACE == "qemu-project" && $CI_COMMIT_BRANCH =~ /^staging/'
-      when: manual
-    - if: "$AARCH64_RUNNER_AVAILABLE"
-      when: manual
@@ -32,7 +32,7 @@ check-python-minreqs:
  variables:
    GIT_DEPTH: 1
  needs:
-    - job: python-container
+    job: python-container

 check-python-tox:
  extends: .base_job_template
@@ -45,50 +45,4 @@ check-python-tox:
    QEMU_TOX_EXTRA_ARGS: --skip-missing-interpreters=false
    QEMU_JOB_OPTIONAL: 1
  needs:
-    - job: python-container
-
-check-rust-tools-nightly:
-  extends: .base_job_template
-  stage: test
-  image: $CI_REGISTRY_IMAGE/qemu/fedora-rust-nightly:$QEMU_CI_CONTAINER_TAG
-  script:
-    - source scripts/ci/gitlab-ci-section
-    - section_start test "Running Rust code checks"
-    - cd build
-    - pyvenv/bin/meson devenv -w ../rust ${CARGO-cargo} fmt --check
-    - make clippy
-    - make rustdoc
-    - section_end test
-  variables:
-    GIT_DEPTH: 1
-  allow_failure: true
-  needs:
-    - job: build-system-fedora-rust-nightly
-      artifacts: true
-  artifacts:
-    when: on_success
-    expire_in: 2 days
-    paths:
-      - rust/target/doc
-
-check-build-units:
-  extends: .base_job_template
-  stage: build
-  image: $CI_REGISTRY_IMAGE/qemu/debian:$QEMU_CI_CONTAINER_TAG
-  needs:
-    - job: amd64-debian-container
-  before_script:
-    - source scripts/ci/gitlab-ci-section
-    - section_start setup "Install Tools"
-    - apt install --assume-yes --no-install-recommends jq
-    - section_end setup
-  script:
-    - mkdir build
-    - cd build
-    - section_start configure "Running configure"
-    - ../configure
-    - cd ..
-    - section_end configure
-    - section_start analyse "Analyse"
-    - .gitlab-ci.d/check-units.py build/compile_commands.json
-    - section_end analyse
+    job: python-container
@@ -24,10 +24,9 @@ msys2-64bit:
    name: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG"
    expire_in: 7 days
    paths:
-      - build/meson-logs
-      - build/cache-log.txt
+      - build/meson-logs/testlog.txt
    reports:
-      junit: build/meson-logs/*.junit.xml
+      junit: "build/meson-logs/testlog.junit.xml"
  before_script:
  - Write-Output "Acquiring msys2.exe installer at $(Get-Date -Format u)"
  - If ( !(Test-Path -Path msys64\var\cache ) ) {
@@ -78,7 +77,7 @@ msys2-64bit:
      git grep make sed
      mingw-w64-x86_64-binutils
      mingw-w64-x86_64-ccache
-      mingw-w64-x86_64-curl-winssl
+      mingw-w64-x86_64-curl
      mingw-w64-x86_64-gcc
      mingw-w64-x86_64-glib2
      mingw-w64-x86_64-libnfs
@@ -88,23 +87,19 @@ msys2-64bit:
      mingw-w64-x86_64-pkgconf
      mingw-w64-x86_64-python
      mingw-w64-x86_64-zstd"
-  - .\msys64\usr\bin\bash -lc "pacman -Sc --noconfirm"
  - Write-Output "Running build at $(Get-Date -Format u)"
-  - $env:JOBS = $(.\msys64\usr\bin\bash -lc nproc)
  - $env:CHERE_INVOKING = 'yes'  # Preserve the current working directory
  - $env:MSYS = 'winsymlinks:native' # Enable native Windows symlink
  - $env:CCACHE_BASEDIR = "$env:CI_PROJECT_DIR"
  - $env:CCACHE_DIR = "$env:CCACHE_BASEDIR/ccache"
-  - $env:CCACHE_MAXSIZE = "180M"
+  - $env:CCACHE_MAXSIZE = "500M"
  - $env:CCACHE_DEPEND = 1 # cache misses are too expensive with preprocessor mode
  - $env:CC = "ccache gcc"
  - mkdir build
  - cd build
  - ..\msys64\usr\bin\bash -lc "ccache --zero-stats"
  - ..\msys64\usr\bin\bash -lc "../configure $CONFIGURE_ARGS"
-  - ..\msys64\usr\bin\bash -lc "make -j$env:JOBS"
+  - ..\msys64\usr\bin\bash -lc "make"
  - ..\msys64\usr\bin\bash -lc "make check MTESTARGS='$TEST_ARGS' || { cat meson-logs/testlog.txt; exit 1; } ;"
-  - ..\msys64\usr\bin\bash -lc "ls -lR /var/cache > cache-log.txt"
-  - ..\msys64\usr\bin\bash -lc "du -sh ."
  - ..\msys64\usr\bin\bash -lc "ccache --show-stats"
  - Write-Output "Finished build at $(Get-Date -Format u)"
@@ -15,7 +15,6 @@
 	url = https://gitlab.com/qemu-project/qemu-palcode.git
 [submodule "roms/u-boot"]
 	path = roms/u-boot
-        # upstream is https://github.com/u-boot/u-boot
 	url = https://gitlab.com/qemu-project/u-boot.git
 [submodule "roms/skiboot"]
 	path = roms/skiboot
@@ -28,7 +27,6 @@
 	url = https://gitlab.com/qemu-project/seabios-hppa.git
 [submodule "roms/u-boot-sam460ex"]
 	path = roms/u-boot-sam460ex
-        # upstream is https://github.com/zbalaton/u-boot-sam460ex
 	url = https://gitlab.com/qemu-project/u-boot-sam460ex.git
 [submodule "roms/edk2"]
 	path = roms/edk2
@@ -4,48 +4,48 @@
 # See https://github.com/stefanha/git-publish for more information
 #
 [gitpublishprofile "default"]
-base = origin/master
+base = master
 to = qemu-devel@nongnu.org
 cccmd = scripts/get_maintainer.pl --noroles --norolestats --nogit --nogit-fallback 2>/dev/null

 [gitpublishprofile "rfc"]
-base = origin/master
+base = master
 prefix = RFC PATCH
 to = qemu-devel@nongnu.org
 cccmd = scripts/get_maintainer.pl --noroles --norolestats --nogit --nogit-fallback 2>/dev/null

 [gitpublishprofile "stable"]
-base = origin/master
+base = master
 to = qemu-devel@nongnu.org
 cc = qemu-stable@nongnu.org
 cccmd = scripts/get_maintainer.pl --noroles --norolestats --nogit --nogit-fallback 2>/dev/null

 [gitpublishprofile "trivial"]
-base = origin/master
+base = master
 to = qemu-devel@nongnu.org
 cc = qemu-trivial@nongnu.org
 cccmd = scripts/get_maintainer.pl --noroles --norolestats --nogit --nogit-fallback 2>/dev/null

 [gitpublishprofile "block"]
-base = origin/master
+base = master
 to = qemu-devel@nongnu.org
 cc = qemu-block@nongnu.org
 cccmd = scripts/get_maintainer.pl --noroles --norolestats --nogit --nogit-fallback 2>/dev/null

 [gitpublishprofile "arm"]
-base = origin/master
+base = master
 to = qemu-devel@nongnu.org
 cc = qemu-arm@nongnu.org
 cccmd = scripts/get_maintainer.pl --noroles --norolestats --nogit --nogit-fallback 2>/dev/null

 [gitpublishprofile "s390"]
-base = origin/master
+base = master
 to = qemu-devel@nongnu.org
 cc = qemu-s390@nongnu.org
 cccmd = scripts/get_maintainer.pl --noroles --norolestats --nogit --nogit-fallback 2>/dev/null

 [gitpublishprofile "ppc"]
-base = origin/master
+base = master
 to = qemu-devel@nongnu.org
 cc = qemu-ppc@nongnu.org
 cccmd = scripts/get_maintainer.pl --noroles --norolestats --nogit --nogit-fallback 2>/dev/null
@@ -67,19 +67,14 @@ Andrey Drobyshev <andrey.drobyshev@virtuozzo.com> Andrey Drobyshev via <qemu-blo
 BALATON Zoltan <balaton@eik.bme.hu> BALATON Zoltan via <qemu-ppc@nongnu.org>

 # Next, replace old addresses by a more recent one.
-Akihiko Odaki <odaki@rsg.ci.i.u-tokyo.ac.jp> <akihiko.odaki@daynix.com>
-Akihiko Odaki <odaki@rsg.ci.i.u-tokyo.ac.jp> <akihiko.odaki@gmail.com>
 Aleksandar Markovic <aleksandar.qemu.devel@gmail.com> <aleksandar.markovic@mips.com>
 Aleksandar Markovic <aleksandar.qemu.devel@gmail.com> <aleksandar.markovic@imgtec.com>
 Aleksandar Markovic <aleksandar.qemu.devel@gmail.com> <amarkovic@wavecomp.com>
 Aleksandar Rikalo <aleksandar.rikalo@syrmia.com> <arikalo@wavecomp.com>
 Aleksandar Rikalo <aleksandar.rikalo@syrmia.com> <aleksandar.rikalo@rt-rk.com>
-Alex Williamson <alex@shazbot.org> <alex.williamson@redhat.com>
 Alexander Graf <agraf@csgraf.de> <agraf@suse.de>
 Ani Sinha <anisinha@redhat.com> <ani@anisinha.ca>
 Anthony Liguori <anthony@codemonkey.ws> Anthony Liguori <aliguori@us.ibm.com>
-Brian Cain <brian.cain@oss.qualcomm.com> <bcain@quicinc.com>
-Brian Cain <brian.cain@oss.qualcomm.com> <quic_bcain@quicinc.com>
 Christian Borntraeger <borntraeger@linux.ibm.com> <borntraeger@de.ibm.com>
 Damien Hedde <damien.hedde@dahe.fr> <damien.hedde@greensocs.com>
 Filip Bozuta <filip.bozuta@syrmia.com> <filip.bozuta@rt-rk.com.com>
@@ -90,9 +85,8 @@ Huacai Chen <chenhuacai@kernel.org> <chenhc@lemote.com>
 Huacai Chen <chenhuacai@kernel.org> <chenhuacai@loongson.cn>
 James Hogan <jhogan@kernel.org> <james.hogan@imgtec.com>
 Juan Quintela <quintela@trasno.org> <quintela@redhat.com>
-Leif Lindholm <leif.lindholm@oss.qualcomm.com> <quic_llindhol@quicinc.com>
-Leif Lindholm <leif.lindholm@oss.qualcomm.com> <leif.lindholm@linaro.org>
-Leif Lindholm <leif.lindholm@oss.qualcomm.com> <leif@nuviainc.com>
+Leif Lindholm <quic_llindhol@quicinc.com> <leif.lindholm@linaro.org>
+Leif Lindholm <quic_llindhol@quicinc.com> <leif@nuviainc.com>
 Luc Michel <luc@lmichel.fr> <luc.michel@git.antfield.fr>
 Luc Michel <luc@lmichel.fr> <luc.michel@greensocs.com>
 Luc Michel <luc@lmichel.fr> <lmichel@kalray.eu>
@@ -137,7 +131,6 @@ Chen Gang <gang.chen.5i5j@gmail.com>
 Chen Gang <gang.chen@sunrus.com.cn>
 Chen Wei-Ren <chenwj@iis.sinica.edu.tw>
 Christophe Lyon <christophe.lyon@st.com>
-Clément Mathieu--Drif <clement.mathieu--drif@eviden.com>
 Collin L. Walling <walling@linux.ibm.com>
 Daniel P. Berrangé <berrange@redhat.com>
 Eduardo Otubo <otubo@redhat.com>
@@ -21,3 +21,5 @@ python:
  install:
    - requirements: docs/requirements.txt

+# We want all the document formats
+formats: all
@@ -79,6 +79,41 @@ after_script:
 jobs:
  include:

+    - name: "[aarch64] GCC check-tcg"
+      arch: arm64
+      addons:
+        apt_packages:
+          - libaio-dev
+          - libattr1-dev
+          - libbrlapi-dev
+          - libcacard-dev
+          - libcap-ng-dev
+          - libfdt-dev
+          - libgcrypt20-dev
+          - libgnutls28-dev
+          - libgtk-3-dev
+          - libiscsi-dev
+          - liblttng-ust-dev
+          - libncurses5-dev
+          - libnfs-dev
+          - libpixman-1-dev
+          - libpng-dev
+          - librados-dev
+          - libsdl2-dev
+          - libseccomp-dev
+          - liburcu-dev
+          - libusb-1.0-0-dev
+          - libvdeplug-dev
+          - libvte-2.91-dev
+          - ninja-build
+          - python3-tomli
+          # Tests dependencies
+          - genisoimage
+      env:
+        - TEST_CMD="make check check-tcg V=1"
+        - CONFIG="--disable-containers --enable-fdt=system
+                  --target-list=${MAIN_SYSTEM_TARGETS} --cxx=/bin/false"
+
    - name: "[ppc64] Clang check-tcg"
      arch: ppc64le
      compiler: clang
@@ -2,7 +2,7 @@
 		       Version 2, June 1991

 Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
- <https://fsf.org/>
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 Everyone is permitted to copy and distribute verbatim copies
 of this license document, but changing it is not allowed.

@@ -304,7 +304,8 @@ the "copyright" line and a pointer to where the full notice is found.
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License along
-    with this program; if not, see <https://www.gnu.org/licenses/>.
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

 Also add information on how to contact you by electronic and paper mail.

@@ -2,7 +2,7 @@
                       Version 2.1, February 1999

 Copyright (C) 1991, 1999 Free Software Foundation, Inc.
- <https://fsf.org/>
+ 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 Everyone is permitted to copy and distribute verbatim copies
 of this license document, but changing it is not allowed.

@@ -484,7 +484,8 @@ convey the exclusion of warranty; and each file should have at least the
    Lesser General Public License for more details.

    You should have received a copy of the GNU Lesser General Public
-    License along with this library; if not, see <https://www.gnu.org/licenses/>.
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA

 Also add information on how to contact you by electronic and paper mail.

@@ -4,4 +4,3 @@ source accel/Kconfig
 source target/Kconfig
 source hw/Kconfig
 source semihosting/Kconfig
-source rust/Kconfig
@@ -5,12 +5,6 @@
 config LINUX
    bool

-config LIBCBOR
-    bool
-
-config GNUTLS
-    bool
-
 config OPENGL
    bool

@@ -58,9 +52,3 @@ config VFIO_USER_SERVER_ALLOWED

 config HV_BALLOON_POSSIBLE
    bool
-
-config HAVE_RUST
-    bool
-
-config MAC_PVG
-    bool
@@ -187,6 +187,11 @@ SUBDIR_RULES=$(foreach t, all clean distclean, $(addsuffix /$(t), $(SUBDIRS)))
 $(SUBDIR_RULES):
 	$(call quiet-command,$(MAKE) $(SUBDIR_MAKEFLAGS) -C $(dir $@) V="$(V)" TARGET_DIR="$(dir $@)" $(notdir $@),)

+ifneq ($(filter contrib/plugins, $(SUBDIRS)),)
+.PHONY: plugins
+plugins: contrib/plugins/all
+endif
+
 .PHONY: recurse-all recurse-clean
 recurse-all: $(addsuffix /all, $(SUBDIRS))
 recurse-clean: $(addsuffix /clean, $(SUBDIRS))
@@ -207,10 +212,10 @@ clean: recurse-clean

 VERSION = $(shell cat $(SRC_PATH)/VERSION)

-dist: qemu-$(VERSION).tar.xz
+dist: qemu-$(VERSION).tar.bz2

-qemu-%.tar.xz:
-	$(SRC_PATH)/scripts/make-release "$(SRC_PATH)" "$(patsubst qemu-%.tar.xz,%,$@)"
+qemu-%.tar.bz2:
+	$(SRC_PATH)/scripts/make-release "$(SRC_PATH)" "$(patsubst qemu-%.tar.bz2,%,$@)"

 distclean: clean recurse-distclean
 	-$(quiet-@)test -f build.ninja && $(NINJA) $(NINJAFLAGS) -t clean -g || :
@@ -227,7 +232,6 @@ distclean: clean recurse-distclean
 	rm -Rf .sdk qemu-bundle

 find-src-path = find "$(SRC_PATH)" -path "$(SRC_PATH)/meson" -prune -o \
-	-path "$(SRC_PATH)/.pc" -prune -o \
 	-type l -prune -o \( -name "*.[chsS]" -o -name "*.[ch].inc" \)

 .PHONY: ctags
@@ -303,6 +307,11 @@ help:
 	$(call print-help,cscope,Generate cscope index)
 	$(call print-help,sparse,Run sparse on the QEMU source)
 	@echo  ''
+ifneq ($(filter contrib/plugins, $(SUBDIRS)),)
+	@echo  'Plugin targets:'
+	$(call print-help,plugins,Build the example TCG plugins)
+	@echo  ''
+endif
 	@echo  'Cleaning targets:'
 	$(call print-help,clean,Remove most generated files but keep the config)
 	$(call print-help,distclean,Remove all generated files)
@@ -1 +1 @@
-10.1.50
+9.1.3
@@ -13,11 +13,7 @@ config TCG
 config KVM
    bool

-config MSHV
-    bool
-
 config XEN
    bool
    select FSDEV_9P if VIRTFS
-    select PCI_EXPRESS_GENERIC_BRIDGE
    select XEN_BUS
@@ -25,11 +25,10 @@
 */

 #include "qemu/osdep.h"
-#include "qemu/lockcnt.h"
 #include "qemu/thread.h"
 #include "qemu/main-loop.h"
 #include "hw/core/cpu.h"
-#include "system/accel-blocker.h"
+#include "sysemu/accel-blocker.h"

 static QemuLockCnt accel_in_ioctl_lock;
 static QemuEvent accel_in_ioctl_event;
@@ -1,144 +0,0 @@
-/*
- * QEMU accel class, components common to system emulation and user mode
- *
- * Copyright (c) 2003-2008 Fabrice Bellard
- * Copyright (c) 2014 Red Hat Inc.
- *
- * SPDX-License-Identifier: MIT
- */
-
-#include "qemu/osdep.h"
-#include "qemu/accel.h"
-#include "qemu/target-info.h"
-#include "accel/accel-ops.h"
-#include "accel/accel-cpu.h"
-#include "accel/accel-cpu-ops.h"
-#include "accel-internal.h"
-
-/* Lookup AccelClass from opt_name. Returns NULL if not found */
-AccelClass *accel_find(const char *opt_name)
-{
-    char *class_name = g_strdup_printf(ACCEL_CLASS_NAME("%s"), opt_name);
-    AccelClass *ac = ACCEL_CLASS(module_object_class_by_name(class_name));
-    g_free(class_name);
-    return ac;
-}
-
-/* Return the name of the current accelerator */
-const char *current_accel_name(void)
-{
-    AccelClass *ac = ACCEL_GET_CLASS(current_accel());
-
-    return ac->name;
-}
-
-static void accel_init_cpu_int_aux(ObjectClass *klass, void *opaque)
-{
-    CPUClass *cc = CPU_CLASS(klass);
-    AccelCPUClass *accel_cpu = opaque;
-
-    /*
-     * The first callback allows accel-cpu to run initializations
-     * for the CPU, customizing CPU behavior according to the accelerator.
-     *
-     * The second one allows the CPU to customize the accel-cpu
-     * behavior according to the CPU.
-     *
-     * The second is currently only used by TCG, to specialize the
-     * TCGCPUOps depending on the CPU type.
-     */
-    cc->accel_cpu = accel_cpu;
-    if (accel_cpu->cpu_class_init) {
-        accel_cpu->cpu_class_init(cc);
-    }
-    if (cc->init_accel_cpu) {
-        cc->init_accel_cpu(accel_cpu, cc);
-    }
-}
-
-/* initialize the arch-specific accel CpuClass interfaces */
-static void accel_init_cpu_interfaces(AccelClass *ac)
-{
-    const char *ac_name; /* AccelClass name */
-    char *acc_name;      /* AccelCPUClass name */
-    ObjectClass *acc;    /* AccelCPUClass */
-    const char *cpu_resolving_type = target_cpu_type();
-
-    ac_name = object_class_get_name(OBJECT_CLASS(ac));
-    g_assert(ac_name != NULL);
-
-    acc_name = g_strdup_printf("%s-%s", ac_name, cpu_resolving_type);
-    acc = object_class_by_name(acc_name);
-    g_free(acc_name);
-
-    if (acc) {
-        object_class_foreach(accel_init_cpu_int_aux,
-                             cpu_resolving_type, false, acc);
-    }
-}
-
-void accel_init_interfaces(AccelClass *ac)
-{
-    accel_init_ops_interfaces(ac);
-    accel_init_cpu_interfaces(ac);
-}
-
-void accel_cpu_instance_init(CPUState *cpu)
-{
-    if (cpu->cc->accel_cpu && cpu->cc->accel_cpu->cpu_instance_init) {
-        cpu->cc->accel_cpu->cpu_instance_init(cpu);
-    }
-}
-
-bool accel_cpu_common_realize(CPUState *cpu, Error **errp)
-{
-    AccelState *accel = current_accel();
-    AccelClass *acc = ACCEL_GET_CLASS(accel);
-
-    /* target specific realization */
-    if (cpu->cc->accel_cpu
-        && cpu->cc->accel_cpu->cpu_target_realize
-        && !cpu->cc->accel_cpu->cpu_target_realize(cpu, errp)) {
-        return false;
-    }
-
-    /* generic realization */
-    if (acc->cpu_common_realize && !acc->cpu_common_realize(cpu, errp)) {
-        return false;
-    }
-
-    return true;
-}
-
-void accel_cpu_common_unrealize(CPUState *cpu)
-{
-    AccelState *accel = current_accel();
-    AccelClass *acc = ACCEL_GET_CLASS(accel);
-
-    /* generic unrealization */
-    if (acc->cpu_common_unrealize) {
-        acc->cpu_common_unrealize(cpu);
-    }
-}
-
-int accel_supported_gdbstub_sstep_flags(void)
-{
-    AccelState *accel = current_accel();
-    AccelClass *acc = ACCEL_GET_CLASS(accel);
-    if (acc->gdbstub_supported_sstep_flags) {
-        return acc->gdbstub_supported_sstep_flags(accel);
-    }
-    return 0;
-}
-
-static const TypeInfo accel_types[] = {
-    {
-        .name           = TYPE_ACCEL,
-        .parent         = TYPE_OBJECT,
-        .class_size     = sizeof(AccelClass),
-        .instance_size  = sizeof(AccelState),
-        .abstract       = true,
-    },
-};
-
-DEFINE_TYPES(accel_types)
@@ -1,106 +0,0 @@
-/*
- * Accelerated irqchip abstraction
- *
- * Copyright Microsoft, Corp. 2025
- *
- * Authors: Ziqiao Zhou <ziqiaozhou@microsoft.com>
- *          Magnus Kulke <magnuskulke@microsoft.com>
- *
- * SPDX-License-Identifier: GPL-2.0-or-later
- */
-
-#include "qemu/osdep.h"
-#include "hw/pci/msi.h"
-
-#include "system/kvm.h"
-#include "system/mshv.h"
-#include "system/accel-irq.h"
-
-int accel_irqchip_add_msi_route(KVMRouteChange *c, int vector, PCIDevice *dev)
-{
-#ifdef CONFIG_MSHV_IS_POSSIBLE
-    if (mshv_msi_via_irqfd_enabled()) {
-        return mshv_irqchip_add_msi_route(vector, dev);
-    }
-#endif
-    if (kvm_enabled()) {
-        return kvm_irqchip_add_msi_route(c, vector, dev);
-    }
-    return -ENOSYS;
-}
-
-int accel_irqchip_update_msi_route(int vector, MSIMessage msg, PCIDevice *dev)
-{
-#ifdef CONFIG_MSHV_IS_POSSIBLE
-    if (mshv_msi_via_irqfd_enabled()) {
-        return mshv_irqchip_update_msi_route(vector, msg, dev);
-    }
-#endif
-    if (kvm_enabled()) {
-        return kvm_irqchip_update_msi_route(kvm_state, vector, msg, dev);
-    }
-    return -ENOSYS;
-}
-
-void accel_irqchip_commit_route_changes(KVMRouteChange *c)
-{
-#ifdef CONFIG_MSHV_IS_POSSIBLE
-    if (mshv_msi_via_irqfd_enabled()) {
-        mshv_irqchip_commit_routes();
-    }
-#endif
-    if (kvm_enabled()) {
-        kvm_irqchip_commit_route_changes(c);
-    }
-}
-
-void accel_irqchip_commit_routes(void)
-{
-#ifdef CONFIG_MSHV_IS_POSSIBLE
-    if (mshv_msi_via_irqfd_enabled()) {
-        mshv_irqchip_commit_routes();
-    }
-#endif
-    if (kvm_enabled()) {
-        kvm_irqchip_commit_routes(kvm_state);
-    }
-}
-
-void accel_irqchip_release_virq(int virq)
-{
-#ifdef CONFIG_MSHV_IS_POSSIBLE
-    if (mshv_msi_via_irqfd_enabled()) {
-        mshv_irqchip_release_virq(virq);
-    }
-#endif
-    if (kvm_enabled()) {
-        kvm_irqchip_release_virq(kvm_state, virq);
-    }
-}
-
-int accel_irqchip_add_irqfd_notifier_gsi(EventNotifier *n, EventNotifier *rn,
-                                         int virq)
-{
-#ifdef CONFIG_MSHV_IS_POSSIBLE
-    if (mshv_msi_via_irqfd_enabled()) {
-        return mshv_irqchip_add_irqfd_notifier_gsi(n, rn, virq);
-    }
-#endif
-    if (kvm_enabled()) {
-        return kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, n, rn, virq);
-    }
-    return -ENOSYS;
-}
-
-int accel_irqchip_remove_irqfd_notifier_gsi(EventNotifier *n, int virq)
-{
-#ifdef CONFIG_MSHV_IS_POSSIBLE
-    if (mshv_msi_via_irqfd_enabled()) {
-        return mshv_irqchip_remove_irqfd_notifier_gsi(n, virq);
-    }
-#endif
-    if (kvm_enabled()) {
-        return kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state, n, virq);
-    }
-    return -ENOSYS;
-}
@@ -1,35 +0,0 @@
-/*
- * QMP commands related to accelerators
- *
- * Copyright (c) Linaro
- *
- * SPDX-License-Identifier: GPL-2.0-or-later
- */
-
-#include "qemu/osdep.h"
-#include "qemu/accel.h"
-#include "qapi/type-helpers.h"
-#include "qapi/qapi-commands-accelerator.h"
-#include "accel/accel-ops.h"
-#include "accel/accel-cpu-ops.h"
-#include "hw/core/cpu.h"
-
-HumanReadableText *qmp_x_accel_stats(Error **errp)
-{
-    AccelState *accel = current_accel();
-    AccelClass *acc = ACCEL_GET_CLASS(accel);
-    g_autoptr(GString) buf = g_string_new("");
-
-    if (acc->get_stats) {
-        acc->get_stats(accel, buf);
-    }
-    if (acc->ops->get_vcpu_stats) {
-        CPUState *cpu;
-
-        CPU_FOREACH(cpu) {
-            acc->ops->get_vcpu_stats(cpu, buf);
-        }
-    }
-
-    return human_readable_text_from_str(buf);
-}
@@ -25,15 +25,10 @@

 #include "qemu/osdep.h"
 #include "qemu/accel.h"
-#include "qapi/qapi-commands-accelerator.h"
-#include "monitor/monitor.h"
 #include "hw/boards.h"
-#include "hw/core/cpu.h"
-#include "accel/accel-ops.h"
-#include "accel/accel-cpu-ops.h"
-#include "system/cpus.h"
+#include "sysemu/cpus.h"
 #include "qemu/error-report.h"
-#include "accel-internal.h"
+#include "accel-system.h"

 int accel_init_machine(AccelState *accel, MachineState *ms)
 {
@@ -41,7 +36,7 @@ int accel_init_machine(AccelState *accel, MachineState *ms)
    int ret;
    ms->accelerator = accel;
    *(acc->allowed) = true;
-    ret = acc->init_machine(accel, ms);
+    ret = acc->init_machine(ms);
    if (ret < 0) {
        ms->accelerator = NULL;
        *(acc->allowed) = false;
@@ -62,21 +57,12 @@ void accel_setup_post(MachineState *ms)
    AccelState *accel = ms->accelerator;
    AccelClass *acc = ACCEL_GET_CLASS(accel);
    if (acc->setup_post) {
-        acc->setup_post(accel);
-    }
-}
-
-void accel_pre_resume(MachineState *ms, bool step_pending)
-{
-    AccelState *accel = ms->accelerator;
-    AccelClass *acc = ACCEL_GET_CLASS(accel);
-    if (acc->pre_resume_vm) {
-        acc->pre_resume_vm(accel, step_pending);
+        acc->setup_post(ms, accel);
    }
 }

 /* initialize the arch-independent accel operation interfaces */
-void accel_init_ops_interfaces(AccelClass *ac)
+void accel_system_init_ops_interfaces(AccelClass *ac)
 {
    const char *ac_name;
    char *ops_name;
@@ -87,35 +73,30 @@ void accel_init_ops_interfaces(AccelClass *ac)
    g_assert(ac_name != NULL);

    ops_name = g_strdup_printf("%s" ACCEL_OPS_SUFFIX, ac_name);
+    ops = ACCEL_OPS_CLASS(module_object_class_by_name(ops_name));
    oc = module_object_class_by_name(ops_name);
    if (!oc) {
        error_report("fatal: could not load module for type '%s'", ops_name);
        exit(1);
    }
    g_free(ops_name);
+    ops = ACCEL_OPS_CLASS(oc);
    /*
     * all accelerators need to define ops, providing at least a mandatory
     * non-NULL create_vcpu_thread operation.
     */
-    ops = ACCEL_OPS_CLASS(oc);
-    ac->ops = ops;
+    g_assert(ops != NULL);
    if (ops->ops_init) {
-        ops->ops_init(ac);
+        ops->ops_init(ops);
    }
    cpus_register_accel(ops);
 }

-static void accel_ops_class_init(ObjectClass *oc, const void *data)
-{
-    monitor_register_hmp_info_hrt("accel", qmp_x_accel_stats);
-}
-
 static const TypeInfo accel_ops_type_info = {
    .name = TYPE_ACCEL_OPS,
    .parent = TYPE_OBJECT,
    .abstract = true,
    .class_size = sizeof(AccelOpsClass),
-    .class_init = accel_ops_class_init,
 };

 static void accel_system_register_types(void)
@@ -1,5 +1,5 @@
 /*
- * QEMU accel internal functions
+ * QEMU System Emulation accel internal functions
 *
 * Copyright 2021 SUSE LLC
 *
@@ -7,11 +7,9 @@
 * See the COPYING file in the top-level directory.
 */

-#ifndef ACCEL_INTERNAL_H
-#define ACCEL_INTERNAL_H
+#ifndef ACCEL_SYSTEM_H
+#define ACCEL_SYSTEM_H

-#include "qemu/accel.h"
-
-void accel_init_ops_interfaces(AccelClass *ac);
+void accel_system_init_ops_interfaces(AccelClass *ac);

 #endif /* ACCEL_SYSTEM_H */
@@ -24,7 +24,141 @@
 */

 #include "qemu/osdep.h"
-#include "accel/accel-cpu-target.h"
+#include "qemu/accel.h"
+
+#include "cpu.h"
+#include "hw/core/accel-cpu.h"
+
+#ifndef CONFIG_USER_ONLY
+#include "accel-system.h"
+#endif /* !CONFIG_USER_ONLY */
+
+static const TypeInfo accel_type = {
+    .name = TYPE_ACCEL,
+    .parent = TYPE_OBJECT,
+    .class_size = sizeof(AccelClass),
+    .instance_size = sizeof(AccelState),
+};
+
+/* Lookup AccelClass from opt_name. Returns NULL if not found */
+AccelClass *accel_find(const char *opt_name)
+{
+    char *class_name = g_strdup_printf(ACCEL_CLASS_NAME("%s"), opt_name);
+    AccelClass *ac = ACCEL_CLASS(module_object_class_by_name(class_name));
+    g_free(class_name);
+    return ac;
+}
+
+/* Return the name of the current accelerator */
+const char *current_accel_name(void)
+{
+    AccelClass *ac = ACCEL_GET_CLASS(current_accel());
+
+    return ac->name;
+}
+
+static void accel_init_cpu_int_aux(ObjectClass *klass, void *opaque)
+{
+    CPUClass *cc = CPU_CLASS(klass);
+    AccelCPUClass *accel_cpu = opaque;
+
+    /*
+     * The first callback allows accel-cpu to run initializations
+     * for the CPU, customizing CPU behavior according to the accelerator.
+     *
+     * The second one allows the CPU to customize the accel-cpu
+     * behavior according to the CPU.
+     *
+     * The second is currently only used by TCG, to specialize the
+     * TCGCPUOps depending on the CPU type.
+     */
+    cc->accel_cpu = accel_cpu;
+    if (accel_cpu->cpu_class_init) {
+        accel_cpu->cpu_class_init(cc);
+    }
+    if (cc->init_accel_cpu) {
+        cc->init_accel_cpu(accel_cpu, cc);
+    }
+}
+
+/* initialize the arch-specific accel CpuClass interfaces */
+static void accel_init_cpu_interfaces(AccelClass *ac)
+{
+    const char *ac_name; /* AccelClass name */
+    char *acc_name;      /* AccelCPUClass name */
+    ObjectClass *acc;    /* AccelCPUClass */
+
+    ac_name = object_class_get_name(OBJECT_CLASS(ac));
+    g_assert(ac_name != NULL);
+
+    acc_name = g_strdup_printf("%s-%s", ac_name, CPU_RESOLVING_TYPE);
+    acc = object_class_by_name(acc_name);
+    g_free(acc_name);
+
+    if (acc) {
+        object_class_foreach(accel_init_cpu_int_aux,
+                             CPU_RESOLVING_TYPE, false, acc);
+    }
+}
+
+void accel_init_interfaces(AccelClass *ac)
+{
+#ifndef CONFIG_USER_ONLY
+    accel_system_init_ops_interfaces(ac);
+#endif /* !CONFIG_USER_ONLY */
+
+    accel_init_cpu_interfaces(ac);
+}
+
+void accel_cpu_instance_init(CPUState *cpu)
+{
+    CPUClass *cc = CPU_GET_CLASS(cpu);
+
+    if (cc->accel_cpu && cc->accel_cpu->cpu_instance_init) {
+        cc->accel_cpu->cpu_instance_init(cpu);
+    }
+}
+
+bool accel_cpu_common_realize(CPUState *cpu, Error **errp)
+{
+    CPUClass *cc = CPU_GET_CLASS(cpu);
+    AccelState *accel = current_accel();
+    AccelClass *acc = ACCEL_GET_CLASS(accel);
+
+    /* target specific realization */
+    if (cc->accel_cpu && cc->accel_cpu->cpu_target_realize
+        && !cc->accel_cpu->cpu_target_realize(cpu, errp)) {
+        return false;
+    }
+
+    /* generic realization */
+    if (acc->cpu_common_realize && !acc->cpu_common_realize(cpu, errp)) {
+        return false;
+    }
+
+    return true;
+}
+
+void accel_cpu_common_unrealize(CPUState *cpu)
+{
+    AccelState *accel = current_accel();
+    AccelClass *acc = ACCEL_GET_CLASS(accel);
+
+    /* generic unrealization */
+    if (acc->cpu_common_unrealize) {
+        acc->cpu_common_unrealize(cpu);
+    }
+}
+
+int accel_supported_gdbstub_sstep_flags(void)
+{
+    AccelState *accel = current_accel();
+    AccelClass *acc = ACCEL_GET_CLASS(accel);
+    if (acc->gdbstub_supported_sstep_flags) {
+        return acc->gdbstub_supported_sstep_flags();
+    }
+    return 0;
+}

 static const TypeInfo accel_cpu_type = {
    .name = TYPE_ACCEL_CPU,
@@ -35,6 +169,7 @@ static const TypeInfo accel_cpu_type = {

 static void register_accel_types(void)
 {
+    type_register_static(&accel_type);
    type_register_static(&accel_cpu_type);
 }

@@ -9,12 +9,6 @@

 #include "qemu/osdep.h"
 #include "qemu/accel.h"
-#include "accel-internal.h"
-
-void accel_init_ops_interfaces(AccelClass *ac)
-{
-    /* nothing */
-}

 AccelState *current_accel(void)
 {
@@ -13,11 +13,10 @@

 #include "qemu/osdep.h"
 #include "qemu/rcu.h"
-#include "system/cpus.h"
+#include "sysemu/cpus.h"
 #include "qemu/guest-random.h"
 #include "qemu/main-loop.h"
 #include "hw/core/cpu.h"
-#include "accel/dummy-cpus.h"

 static void *dummy_cpu_thread_fn(void *arg)
 {
@@ -43,7 +42,6 @@ static void *dummy_cpu_thread_fn(void *arg)
    qemu_guest_random_seed_thread_part2(cpu->random_seed);

    do {
-        qemu_process_cpu_events(cpu);
        bql_unlock();
 #ifndef _WIN32
        do {
@@ -58,6 +56,7 @@ static void *dummy_cpu_thread_fn(void *arg)
        qemu_sem_wait(&cpu->sem);
 #endif
        bql_lock();
+        qemu_wait_io_event(cpu);
    } while (!cpu->unplug);

    bql_unlock();
@@ -1,14 +0,0 @@
-/*
- * Dummy cpu thread code
- *
- * Copyright IBM, Corp. 2011
- *
- * SPDX-License-Identifier: GPL-2.0-or-later
- */
-
-#ifndef ACCEL_DUMMY_CPUS_H
-#define ACCEL_DUMMY_CPUS_H
-
-void dummy_start_vcpu_thread(CPUState *cpu);
-
-#endif
@@ -48,20 +48,23 @@
 */

 #include "qemu/osdep.h"
-#include "qemu/guest-random.h"
+#include "qemu/error-report.h"
 #include "qemu/main-loop.h"
-#include "qemu/queue.h"
+#include "exec/address-spaces.h"
+#include "exec/exec-all.h"
 #include "gdbstub/enums.h"
-#include "exec/cpu-common.h"
-#include "hw/core/cpu.h"
-#include "accel/accel-cpu-ops.h"
-#include "system/cpus.h"
-#include "system/hvf.h"
-#include "system/hvf_int.h"
-#include <mach/mach_time.h>
+#include "sysemu/cpus.h"
+#include "sysemu/hvf.h"
+#include "sysemu/hvf_int.h"
+#include "sysemu/runstate.h"
+#include "qemu/guest-random.h"

 HVFState *hvf_state;

+#ifdef __aarch64__
+#define HV_VM_DEFAULT NULL
+#endif
+
 /* Memory slots */

 hvf_slot *hvf_find_overlap_slot(uint64_t start, uint64_t size)
@@ -78,17 +81,138 @@ hvf_slot *hvf_find_overlap_slot(uint64_t start, uint64_t size)
    return NULL;
 }

+struct mac_slot {
+    int present;
+    uint64_t size;
+    uint64_t gpa_start;
+    uint64_t gva;
+};
+
+struct mac_slot mac_slots[32];
+
+static int do_hvf_set_memory(hvf_slot *slot, hv_memory_flags_t flags)
+{
+    struct mac_slot *macslot;
+    hv_return_t ret;
+
+    macslot = &mac_slots[slot->slot_id];
+
+    if (macslot->present) {
+        if (macslot->size != slot->size) {
+            macslot->present = 0;
+            ret = hv_vm_unmap(macslot->gpa_start, macslot->size);
+            assert_hvf_ok(ret);
+        }
+    }
+
+    if (!slot->size) {
+        return 0;
+    }
+
+    macslot->present = 1;
+    macslot->gpa_start = slot->start;
+    macslot->size = slot->size;
+    ret = hv_vm_map(slot->mem, slot->start, slot->size, flags);
+    assert_hvf_ok(ret);
+    return 0;
+}
+
+static void hvf_set_phys_mem(MemoryRegionSection *section, bool add)
+{
+    hvf_slot *mem;
+    MemoryRegion *area = section->mr;
+    bool writable = !area->readonly && !area->rom_device;
+    hv_memory_flags_t flags;
+    uint64_t page_size = qemu_real_host_page_size();
+
+    if (!memory_region_is_ram(area)) {
+        if (writable) {
+            return;
+        } else if (!memory_region_is_romd(area)) {
+            /*
+             * If the memory device is not in romd_mode, then we actually want
+             * to remove the hvf memory slot so all accesses will trap.
+             */
+             add = false;
+        }
+    }
+
+    if (!QEMU_IS_ALIGNED(int128_get64(section->size), page_size) ||
+        !QEMU_IS_ALIGNED(section->offset_within_address_space, page_size)) {
+        /* Not page aligned, so we can not map as RAM */
+        add = false;
+    }
+
+    mem = hvf_find_overlap_slot(
+            section->offset_within_address_space,
+            int128_get64(section->size));
+
+    if (mem && add) {
+        if (mem->size == int128_get64(section->size) &&
+            mem->start == section->offset_within_address_space &&
+            mem->mem == (memory_region_get_ram_ptr(area) +
+            section->offset_within_region)) {
+            return; /* Same region was attempted to register, go away. */
+        }
+    }
+
+    /* Region needs to be reset. set the size to 0 and remap it. */
+    if (mem) {
+        mem->size = 0;
+        if (do_hvf_set_memory(mem, 0)) {
+            error_report("Failed to reset overlapping slot");
+            abort();
+        }
+    }
+
+    if (!add) {
+        return;
+    }
+
+    if (area->readonly ||
+        (!memory_region_is_ram(area) && memory_region_is_romd(area))) {
+        flags = HV_MEMORY_READ | HV_MEMORY_EXEC;
+    } else {
+        flags = HV_MEMORY_READ | HV_MEMORY_WRITE | HV_MEMORY_EXEC;
+    }
+
+    /* Now make a new slot. */
+    int x;
+
+    for (x = 0; x < hvf_state->num_slots; ++x) {
+        mem = &hvf_state->slots[x];
+        if (!mem->size) {
+            break;
+        }
+    }
+
+    if (x == hvf_state->num_slots) {
+        error_report("No free slots");
+        abort();
+    }
+
+    mem->size = int128_get64(section->size);
+    mem->mem = memory_region_get_ram_ptr(area) + section->offset_within_region;
+    mem->start = section->offset_within_address_space;
+    mem->region = area;
+
+    if (do_hvf_set_memory(mem, flags)) {
+        error_report("Error registering new memory slot");
+        abort();
+    }
+}
+
 static void do_hvf_cpu_synchronize_state(CPUState *cpu, run_on_cpu_data arg)
 {
-    if (!cpu->vcpu_dirty) {
-        hvf_arch_get_registers(cpu);
-        cpu->vcpu_dirty = true;
+    if (!cpu->accel->dirty) {
+        hvf_get_registers(cpu);
+        cpu->accel->dirty = true;
    }
 }

 static void hvf_cpu_synchronize_state(CPUState *cpu)
 {
-    if (!cpu->vcpu_dirty) {
+    if (!cpu->accel->dirty) {
        run_on_cpu(cpu, do_hvf_cpu_synchronize_state, RUN_ON_CPU_NULL);
    }
 }
@@ -97,7 +221,7 @@ static void do_hvf_cpu_synchronize_set_dirty(CPUState *cpu,
                                             run_on_cpu_data arg)
 {
    /* QEMU state is the reference, push it to HVF now and on next entry */
-    cpu->vcpu_dirty = true;
+    cpu->accel->dirty = true;
 }

 static void hvf_cpu_synchronize_post_reset(CPUState *cpu)
@@ -115,16 +239,137 @@ static void hvf_cpu_synchronize_pre_loadvm(CPUState *cpu)
    run_on_cpu(cpu, do_hvf_cpu_synchronize_set_dirty, RUN_ON_CPU_NULL);
 }

+static void hvf_set_dirty_tracking(MemoryRegionSection *section, bool on)
+{
+    hvf_slot *slot;
+
+    slot = hvf_find_overlap_slot(
+            section->offset_within_address_space,
+            int128_get64(section->size));
+
+    /* protect region against writes; begin tracking it */
+    if (on) {
+        slot->flags |= HVF_SLOT_LOG;
+        hv_vm_protect((uintptr_t)slot->start, (size_t)slot->size,
+                      HV_MEMORY_READ | HV_MEMORY_EXEC);
+    /* stop tracking region*/
+    } else {
+        slot->flags &= ~HVF_SLOT_LOG;
+        hv_vm_protect((uintptr_t)slot->start, (size_t)slot->size,
+                      HV_MEMORY_READ | HV_MEMORY_WRITE | HV_MEMORY_EXEC);
+    }
+}
+
+static void hvf_log_start(MemoryListener *listener,
+                          MemoryRegionSection *section, int old, int new)
+{
+    if (old != 0) {
+        return;
+    }
+
+    hvf_set_dirty_tracking(section, 1);
+}
+
+static void hvf_log_stop(MemoryListener *listener,
+                         MemoryRegionSection *section, int old, int new)
+{
+    if (new != 0) {
+        return;
+    }
+
+    hvf_set_dirty_tracking(section, 0);
+}
+
+static void hvf_log_sync(MemoryListener *listener,
+                         MemoryRegionSection *section)
+{
+    /*
+     * sync of dirty pages is handled elsewhere; just make sure we keep
+     * tracking the region.
+     */
+    hvf_set_dirty_tracking(section, 1);
+}
+
+static void hvf_region_add(MemoryListener *listener,
+                           MemoryRegionSection *section)
+{
+    hvf_set_phys_mem(section, true);
+}
+
+static void hvf_region_del(MemoryListener *listener,
+                           MemoryRegionSection *section)
+{
+    hvf_set_phys_mem(section, false);
+}
+
+static MemoryListener hvf_memory_listener = {
+    .name = "hvf",
+    .priority = MEMORY_LISTENER_PRIORITY_ACCEL,
+    .region_add = hvf_region_add,
+    .region_del = hvf_region_del,
+    .log_start = hvf_log_start,
+    .log_stop = hvf_log_stop,
+    .log_sync = hvf_log_sync,
+};
+
 static void dummy_signal(int sig)
 {
 }

-static void do_hvf_get_vcpu_exec_time(CPUState *cpu, run_on_cpu_data arg)
+bool hvf_allowed;
+
+static int hvf_accel_init(MachineState *ms)
 {
-    int r = hv_vcpu_get_exec_time(cpu->accel->fd, arg.host_ptr);
-    assert_hvf_ok(r);
+    int x;
+    hv_return_t ret;
+    HVFState *s;
+
+    ret = hv_vm_create(HV_VM_DEFAULT);
+    assert_hvf_ok(ret);
+
+    s = g_new0(HVFState, 1);
+
+    s->num_slots = ARRAY_SIZE(s->slots);
+    for (x = 0; x < s->num_slots; ++x) {
+        s->slots[x].size = 0;
+        s->slots[x].slot_id = x;
+    }
+
+    QTAILQ_INIT(&s->hvf_sw_breakpoints);
+
+    hvf_state = s;
+    memory_listener_register(&hvf_memory_listener, &address_space_memory);
+
+    return hvf_arch_init();
 }

+static inline int hvf_gdbstub_sstep_flags(void)
+{
+    return SSTEP_ENABLE | SSTEP_NOIRQ;
+}
+
+static void hvf_accel_class_init(ObjectClass *oc, void *data)
+{
+    AccelClass *ac = ACCEL_CLASS(oc);
+    ac->name = "HVF";
+    ac->init_machine = hvf_accel_init;
+    ac->allowed = &hvf_allowed;
+    ac->gdbstub_supported_sstep_flags = hvf_gdbstub_sstep_flags;
+}
+
+static const TypeInfo hvf_accel_type = {
+    .name = TYPE_HVF_ACCEL,
+    .parent = TYPE_ACCEL,
+    .class_init = hvf_accel_class_init,
+};
+
+static void hvf_type_init(void)
+{
+    type_register_static(&hvf_accel_type);
+}
+
+type_init(hvf_type_init);
+
 static void hvf_vcpu_destroy(CPUState *cpu)
 {
    hv_return_t ret = hv_vcpu_destroy(cpu->accel->fd);
@@ -157,8 +402,8 @@ static int hvf_init_vcpu(CPUState *cpu)
 #else
    r = hv_vcpu_create(&cpu->accel->fd, HV_VCPU_DEFAULT);
 #endif
+    cpu->accel->dirty = true;
    assert_hvf_ok(r);
-    cpu->vcpu_dirty = true;

    cpu->accel->guest_debug_enabled = false;

@@ -192,13 +437,13 @@ static void *hvf_cpu_thread_fn(void *arg)
    qemu_guest_random_seed_thread_part2(cpu->random_seed);

    do {
-        qemu_process_cpu_events(cpu);
        if (cpu_can_run(cpu)) {
-            r = hvf_arch_vcpu_exec(cpu);
+            r = hvf_vcpu_exec(cpu);
            if (r == EXCP_DEBUG) {
                cpu_handle_guest_debug(cpu);
            }
        }
+        qemu_wait_io_event(cpu);
    } while (!cpu->unplug || cpu_can_run(cpu));

    hvf_vcpu_destroy(cpu);
@@ -224,34 +469,6 @@ static void hvf_start_vcpu_thread(CPUState *cpu)
                       cpu, QEMU_THREAD_JOINABLE);
 }

-struct hvf_sw_breakpoint *hvf_find_sw_breakpoint(CPUState *cpu, vaddr pc)
-{
-    struct hvf_sw_breakpoint *bp;
-
-    QTAILQ_FOREACH(bp, &hvf_state->hvf_sw_breakpoints, entry) {
-        if (bp->pc == pc) {
-            return bp;
-        }
-    }
-    return NULL;
-}
-
-int hvf_sw_breakpoints_active(CPUState *cpu)
-{
-    return !QTAILQ_EMPTY(&hvf_state->hvf_sw_breakpoints);
-}
-
-static void do_hvf_update_guest_debug(CPUState *cpu, run_on_cpu_data arg)
-{
-    hvf_arch_update_guest_debug(cpu);
-}
-
-int hvf_update_guest_debug(CPUState *cpu)
-{
-    run_on_cpu(cpu, do_hvf_update_guest_debug, RUN_ON_CPU_NULL);
-    return 0;
-}
-
 static int hvf_insert_breakpoint(CPUState *cpu, int type, vaddr addr, vaddr len)
 {
    struct hvf_sw_breakpoint *bp;
@@ -354,28 +571,12 @@ static void hvf_remove_all_breakpoints(CPUState *cpu)
    }
 }

-static void hvf_get_vcpu_stats(CPUState *cpu, GString *buf)
-{
-    uint64_t time_mach; /* units of mach_absolute_time() */
-
-    run_on_cpu(cpu, do_hvf_get_vcpu_exec_time, RUN_ON_CPU_HOST_PTR(&time_mach));
-
-    mach_timebase_info_data_t timebase;
-    mach_timebase_info(&timebase);
-    uint64_t time_ns = time_mach * timebase.numer / timebase.denom;
-
-    g_string_append_printf(buf, "HVF cumulative execution time: %llu.%.3llus\n",
-                                 time_ns / 1000000000,
-                                (time_ns % 1000000000) / 1000000);
-}
-
-static void hvf_accel_ops_class_init(ObjectClass *oc, const void *data)
+static void hvf_accel_ops_class_init(ObjectClass *oc, void *data)
 {
    AccelOpsClass *ops = ACCEL_OPS_CLASS(oc);

    ops->create_vcpu_thread = hvf_start_vcpu_thread;
    ops->kick_vcpu_thread = hvf_kick_vcpu_thread;
-    ops->handle_interrupt = generic_handle_interrupt;

    ops->synchronize_post_reset = hvf_cpu_synchronize_post_reset;
    ops->synchronize_post_init = hvf_cpu_synchronize_post_init;
@@ -387,10 +588,7 @@ static void hvf_accel_ops_class_init(ObjectClass *oc, const void *data)
    ops->remove_all_breakpoints = hvf_remove_all_breakpoints;
    ops->update_guest_debug = hvf_update_guest_debug;
    ops->supports_guest_debug = hvf_arch_supports_guest_debug;
-
-    ops->get_vcpu_stats = hvf_get_vcpu_stats;
 };
-
 static const TypeInfo hvf_accel_ops_type = {
    .name = ACCEL_OPS_NAME("hvf"),

@@ -398,10 +596,8 @@ static const TypeInfo hvf_accel_ops_type = {
    .class_init = hvf_accel_ops_class_init,
    .abstract = true,
 };
-
 static void hvf_accel_ops_register_types(void)
 {
    type_register_static(&hvf_accel_ops_type);
 }
-
 type_init(hvf_accel_ops_register_types);
@@ -10,25 +10,8 @@

 #include "qemu/osdep.h"
 #include "qemu/error-report.h"
-#include "accel/accel-ops.h"
-#include "system/address-spaces.h"
-#include "system/memory.h"
-#include "system/hvf.h"
-#include "system/hvf_int.h"
-#include "hw/core/cpu.h"
-#include "hw/boards.h"
-#include "trace.h"
-
-bool hvf_allowed;
-
-struct mac_slot {
-    int present;
-    uint64_t size;
-    uint64_t gpa_start;
-    uint64_t gva;
-};
-
-struct mac_slot mac_slots[32];
+#include "sysemu/hvf.h"
+#include "sysemu/hvf_int.h"

 const char *hvf_return_string(hv_return_t ret)
 {
@@ -58,257 +41,25 @@ void assert_hvf_ok_impl(hv_return_t ret, const char *file, unsigned int line,
    abort();
 }

-static int do_hvf_set_memory(hvf_slot *slot, hv_memory_flags_t flags)
+struct hvf_sw_breakpoint *hvf_find_sw_breakpoint(CPUState *cpu, vaddr pc)
 {
-    struct mac_slot *macslot;
-    hv_return_t ret;
+    struct hvf_sw_breakpoint *bp;

-    macslot = &mac_slots[slot->slot_id];
-
-    if (macslot->present) {
-        if (macslot->size != slot->size) {
-            macslot->present = 0;
-            trace_hvf_vm_unmap(macslot->gpa_start, macslot->size);
-            ret = hv_vm_unmap(macslot->gpa_start, macslot->size);
-            assert_hvf_ok(ret);
+    QTAILQ_FOREACH(bp, &hvf_state->hvf_sw_breakpoints, entry) {
+        if (bp->pc == pc) {
+            return bp;
        }
    }
+    return NULL;
+}

-    if (!slot->size) {
-        return 0;
-    }
+int hvf_sw_breakpoints_active(CPUState *cpu)
+{
+    return !QTAILQ_EMPTY(&hvf_state->hvf_sw_breakpoints);
+}

-    macslot->present = 1;
-    macslot->gpa_start = slot->start;
-    macslot->size = slot->size;
-    trace_hvf_vm_map(slot->start, slot->size, slot->mem, flags,
-                     flags & HV_MEMORY_READ ?  'R' : '-',
-                     flags & HV_MEMORY_WRITE ? 'W' : '-',
-                     flags & HV_MEMORY_EXEC ?  'X' : '-');
-    ret = hv_vm_map(slot->mem, slot->start, slot->size, flags);
-    assert_hvf_ok(ret);
+int hvf_update_guest_debug(CPUState *cpu)
+{
+    hvf_arch_update_guest_debug(cpu);
    return 0;
 }
-
-static void hvf_set_phys_mem(MemoryRegionSection *section, bool add)
-{
-    hvf_slot *mem;
-    MemoryRegion *area = section->mr;
-    bool writable = !area->readonly && !area->rom_device;
-    hv_memory_flags_t flags;
-    uint64_t page_size = qemu_real_host_page_size();
-
-    if (!memory_region_is_ram(area)) {
-        if (writable) {
-            return;
-        } else if (!memory_region_is_romd(area)) {
-            /*
-             * If the memory device is not in romd_mode, then we actually want
-             * to remove the hvf memory slot so all accesses will trap.
-             */
-             add = false;
-        }
-    }
-
-    if (!QEMU_IS_ALIGNED(int128_get64(section->size), page_size) ||
-        !QEMU_IS_ALIGNED(section->offset_within_address_space, page_size)) {
-        /* Not page aligned, so we can not map as RAM */
-        add = false;
-    }
-
-    mem = hvf_find_overlap_slot(
-            section->offset_within_address_space,
-            int128_get64(section->size));
-
-    if (mem && add) {
-        if (mem->size == int128_get64(section->size) &&
-            mem->start == section->offset_within_address_space &&
-            mem->mem == (memory_region_get_ram_ptr(area) +
-            section->offset_within_region)) {
-            return; /* Same region was attempted to register, go away. */
-        }
-    }
-
-    /* Region needs to be reset. set the size to 0 and remap it. */
-    if (mem) {
-        mem->size = 0;
-        if (do_hvf_set_memory(mem, 0)) {
-            error_report("Failed to reset overlapping slot");
-            abort();
-        }
-    }
-
-    if (!add) {
-        return;
-    }
-
-    if (area->readonly ||
-        (!memory_region_is_ram(area) && memory_region_is_romd(area))) {
-        flags = HV_MEMORY_READ | HV_MEMORY_EXEC;
-    } else {
-        flags = HV_MEMORY_READ | HV_MEMORY_WRITE | HV_MEMORY_EXEC;
-    }
-
-    /* Now make a new slot. */
-    int x;
-
-    for (x = 0; x < hvf_state->num_slots; ++x) {
-        mem = &hvf_state->slots[x];
-        if (!mem->size) {
-            break;
-        }
-    }
-
-    if (x == hvf_state->num_slots) {
-        error_report("No free slots");
-        abort();
-    }
-
-    mem->size = int128_get64(section->size);
-    mem->mem = memory_region_get_ram_ptr(area) + section->offset_within_region;
-    mem->start = section->offset_within_address_space;
-    mem->region = area;
-
-    if (do_hvf_set_memory(mem, flags)) {
-        error_report("Error registering new memory slot");
-        abort();
-    }
-}
-
-static void hvf_set_dirty_tracking(MemoryRegionSection *section, bool on)
-{
-    hvf_slot *slot;
-
-    slot = hvf_find_overlap_slot(
-            section->offset_within_address_space,
-            int128_get64(section->size));
-
-    /* protect region against writes; begin tracking it */
-    if (on) {
-        slot->flags |= HVF_SLOT_LOG;
-        hv_vm_protect((uintptr_t)slot->start, (size_t)slot->size,
-                      HV_MEMORY_READ | HV_MEMORY_EXEC);
-    /* stop tracking region*/
-    } else {
-        slot->flags &= ~HVF_SLOT_LOG;
-        hv_vm_protect((uintptr_t)slot->start, (size_t)slot->size,
-                      HV_MEMORY_READ | HV_MEMORY_WRITE | HV_MEMORY_EXEC);
-    }
-}
-
-static void hvf_log_start(MemoryListener *listener,
-                          MemoryRegionSection *section, int old, int new)
-{
-    if (old != 0) {
-        return;
-    }
-
-    hvf_set_dirty_tracking(section, 1);
-}
-
-static void hvf_log_stop(MemoryListener *listener,
-                         MemoryRegionSection *section, int old, int new)
-{
-    if (new != 0) {
-        return;
-    }
-
-    hvf_set_dirty_tracking(section, 0);
-}
-
-static void hvf_log_sync(MemoryListener *listener,
-                         MemoryRegionSection *section)
-{
-    /*
-     * sync of dirty pages is handled elsewhere; just make sure we keep
-     * tracking the region.
-     */
-    hvf_set_dirty_tracking(section, 1);
-}
-
-static void hvf_region_add(MemoryListener *listener,
-                           MemoryRegionSection *section)
-{
-    hvf_set_phys_mem(section, true);
-}
-
-static void hvf_region_del(MemoryListener *listener,
-                           MemoryRegionSection *section)
-{
-    hvf_set_phys_mem(section, false);
-}
-
-static MemoryListener hvf_memory_listener = {
-    .name = "hvf",
-    .priority = MEMORY_LISTENER_PRIORITY_ACCEL,
-    .region_add = hvf_region_add,
-    .region_del = hvf_region_del,
-    .log_start = hvf_log_start,
-    .log_stop = hvf_log_stop,
-    .log_sync = hvf_log_sync,
-};
-
-static int hvf_accel_init(AccelState *as, MachineState *ms)
-{
-    int x;
-    hv_return_t ret;
-    HVFState *s = HVF_STATE(as);
-    int pa_range = 36;
-    MachineClass *mc = MACHINE_GET_CLASS(ms);
-
-    if (mc->hvf_get_physical_address_range) {
-        pa_range = mc->hvf_get_physical_address_range(ms);
-        if (pa_range < 0) {
-            return -EINVAL;
-        }
-    }
-
-    ret = hvf_arch_vm_create(ms, (uint32_t)pa_range);
-    if (ret == HV_DENIED) {
-        error_report("Could not access HVF. Is the executable signed"
-                     " with com.apple.security.hypervisor entitlement?");
-        exit(1);
-    }
-    assert_hvf_ok(ret);
-
-    s->num_slots = ARRAY_SIZE(s->slots);
-    for (x = 0; x < s->num_slots; ++x) {
-        s->slots[x].size = 0;
-        s->slots[x].slot_id = x;
-    }
-
-    QTAILQ_INIT(&s->hvf_sw_breakpoints);
-
-    hvf_state = s;
-    memory_listener_register(&hvf_memory_listener, &address_space_memory);
-
-    return hvf_arch_init();
-}
-
-static int hvf_gdbstub_sstep_flags(AccelState *as)
-{
-    return SSTEP_ENABLE | SSTEP_NOIRQ;
-}
-
-static void hvf_accel_class_init(ObjectClass *oc, const void *data)
-{
-    AccelClass *ac = ACCEL_CLASS(oc);
-    ac->name = "HVF";
-    ac->init_machine = hvf_accel_init;
-    ac->allowed = &hvf_allowed;
-    ac->gdbstub_supported_sstep_flags = hvf_gdbstub_sstep_flags;
-}
-
-static const TypeInfo hvf_accel_type = {
-    .name = TYPE_HVF_ACCEL,
-    .parent = TYPE_ACCEL,
-    .instance_size = sizeof(HVFState),
-    .class_init = hvf_accel_class_init,
-};
-
-static void hvf_type_init(void)
-{
-    type_register_static(&hvf_accel_type);
-}
-
-type_init(hvf_type_init);
@@ -1,7 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-or-later
-#
-# See docs/devel/tracing.rst for syntax documentation.
-
-# hvf-accel-ops.c
-hvf_vm_map(uint64_t paddr, uint64_t size, void *vaddr, uint8_t flags, const char r, const char w, const char e) "paddr:0x%016"PRIx64" size:0x%08"PRIx64" vaddr:%p flags:0x%02x/%c%c%c"
-hvf_vm_unmap(uint64_t paddr, uint64_t size) "paddr:0x%016"PRIx64" size:0x%08"PRIx64
@@ -1,2 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-#include "trace/trace-accel_hvf.h"
@@ -16,11 +16,10 @@
 #include "qemu/osdep.h"
 #include "qemu/error-report.h"
 #include "qemu/main-loop.h"
-#include "accel/accel-cpu-ops.h"
-#include "system/kvm.h"
-#include "system/kvm_int.h"
-#include "system/runstate.h"
-#include "system/cpus.h"
+#include "sysemu/kvm.h"
+#include "sysemu/kvm_int.h"
+#include "sysemu/runstate.h"
+#include "sysemu/cpus.h"
 #include "qemu/guest-random.h"
 #include "qapi/error.h"

@@ -47,14 +46,13 @@ static void *kvm_vcpu_thread_fn(void *arg)
    qemu_guest_random_seed_thread_part2(cpu->random_seed);

    do {
-        qemu_process_cpu_events(cpu);
-
        if (cpu_can_run(cpu)) {
            r = kvm_cpu_exec(cpu);
            if (r == EXCP_DEBUG) {
                cpu_handle_guest_debug(cpu);
            }
        }
+        qemu_wait_io_event(cpu);
    } while (!cpu->unplug || cpu_can_run(cpu));

    kvm_destroy_vcpu(cpu);
@@ -91,7 +89,7 @@ static int kvm_update_guest_debug_ops(CPUState *cpu)
 }
 #endif

-static void kvm_accel_ops_class_init(ObjectClass *oc, const void *data)
+static void kvm_accel_ops_class_init(ObjectClass *oc, void *data)
 {
    AccelOpsClass *ops = ACCEL_OPS_CLASS(oc);

@@ -102,7 +100,6 @@ static void kvm_accel_ops_class_init(ObjectClass *oc, const void *data)
    ops->synchronize_post_init = kvm_cpu_synchronize_post_init;
    ops->synchronize_state = kvm_cpu_synchronize_state;
    ops->synchronize_pre_loadvm = kvm_cpu_synchronize_pre_loadvm;
-    ops->handle_interrupt = generic_handle_interrupt;

 #ifdef TARGET_KVM_HAVE_GUEST_DEBUG
    ops->update_guest_debug = kvm_update_guest_debug_ops;
@@ -28,17 +28,13 @@
 #include "hw/pci/msix.h"
 #include "hw/s390x/adapter.h"
 #include "gdbstub/enums.h"
-#include "system/kvm_int.h"
-#include "system/runstate.h"
-#include "system/cpus.h"
-#include "system/accel-blocker.h"
-#include "system/physmem.h"
-#include "system/ramblock.h"
-#include "accel/accel-ops.h"
+#include "sysemu/kvm_int.h"
+#include "sysemu/runstate.h"
+#include "sysemu/cpus.h"
+#include "sysemu/accel-blocker.h"
 #include "qemu/bswap.h"
-#include "exec/tswap.h"
-#include "exec/target_page.h"
-#include "system/memory.h"
+#include "exec/memory.h"
+#include "exec/ram_addr.h"
 #include "qemu/event_notifier.h"
 #include "qemu/main-loop.h"
 #include "trace.h"
@@ -46,26 +42,21 @@
 #include "qapi/visitor.h"
 #include "qapi/qapi-types-common.h"
 #include "qapi/qapi-visit-common.h"
-#include "system/reset.h"
+#include "sysemu/reset.h"
 #include "qemu/guest-random.h"
-#include "system/hw_accel.h"
+#include "sysemu/hw_accel.h"
 #include "kvm-cpus.h"
-#include "system/dirtylimit.h"
+#include "sysemu/dirtylimit.h"
 #include "qemu/range.h"

 #include "hw/boards.h"
-#include "system/stats.h"
+#include "sysemu/stats.h"

 /* This check must be after config-host.h is included */
 #ifdef CONFIG_EVENTFD
 #include <sys/eventfd.h>
 #endif

-#if defined(__i386__) || defined(__x86_64__) || defined(__aarch64__)
-# define KVM_HAVE_MCE_INJECTION 1
-#endif
-
-
 /* KVM uses PAGE_SIZE in its definition of KVM_COALESCED_MMIO_MAX. We
 * need to use the real host PAGE_SIZE, as that's what KVM will use.
 */
@@ -80,8 +71,6 @@

 /* Default num of memslots to be allocated when VM starts */
 #define  KVM_MEMSLOTS_NR_ALLOC_DEFAULT                      16
-/* Default max allowed memslots if kernel reported nothing */
-#define  KVM_MEMSLOTS_NR_MAX_DEFAULT                        32

 struct KVMParkedVcpu {
    unsigned long vcpu_id;
@@ -102,7 +91,6 @@ bool kvm_allowed;
 bool kvm_readonly_mem_allowed;
 bool kvm_vm_attributes_allowed;
 bool kvm_msi_use_devid;
-bool kvm_pre_fault_memory_supported;
 static bool kvm_has_guest_debug;
 static int kvm_sstep_flags;
 static bool kvm_immediate_exit;
@@ -193,8 +181,8 @@ static bool kvm_slots_grow(KVMMemoryListener *kml, unsigned int nr_slots_new)
    unsigned int i, cur = kml->nr_slots_allocated;
    KVMSlot *slots;

-    if (nr_slots_new > kvm_state->nr_slots_max) {
-        nr_slots_new = kvm_state->nr_slots_max;
+    if (nr_slots_new > kvm_state->nr_slots) {
+        nr_slots_new = kvm_state->nr_slots;
    }

    if (cur >= nr_slots_new) {
@@ -235,7 +223,7 @@ unsigned int kvm_get_max_memslots(void)
 {
    KVMState *s = KVM_STATE(current_accel());

-    return s->nr_slots_max;
+    return s->nr_slots;
 }

 unsigned int kvm_get_free_memslots(void)
@@ -249,11 +237,11 @@ unsigned int kvm_get_free_memslots(void)
        if (!s->as[i].ml) {
            continue;
        }
-        used_slots = MAX(used_slots, s->as[i].ml->nr_slots_used);
+        used_slots = MAX(used_slots, s->as[i].ml->nr_used_slots);
    }
    kvm_slots_unlock();

-    return s->nr_slots_max - used_slots;
+    return s->nr_slots - used_slots;
 }

 /* Called with KVMMemoryListener.slots_lock held */
@@ -360,7 +348,7 @@ int kvm_physical_memory_addr_from_host(KVMState *s, void *ram,
 static int kvm_set_user_memory_region(KVMMemoryListener *kml, KVMSlot *slot, bool new)
 {
    KVMState *s = kvm_state;
-    struct kvm_userspace_memory_region2 mem = {};
+    struct kvm_userspace_memory_region2 mem;
    int ret;

    mem.slot = slot->slot | (kml->as_id << 16);
@@ -416,7 +404,7 @@ err:
    return ret;
 }

-static void kvm_park_vcpu(CPUState *cpu)
+void kvm_park_vcpu(CPUState *cpu)
 {
    struct KVMParkedVcpu *vcpu;

@@ -428,7 +416,7 @@ static void kvm_park_vcpu(CPUState *cpu)
    QLIST_INSERT_HEAD(&kvm_state->kvm_parked_vcpus, vcpu, node);
 }

-static int kvm_unpark_vcpu(KVMState *s, unsigned long vcpu_id)
+int kvm_unpark_vcpu(KVMState *s, unsigned long vcpu_id)
 {
    struct KVMParkedVcpu *cpu;
    int kvm_fd = -ENOENT;
@@ -447,8 +435,9 @@ static int kvm_unpark_vcpu(KVMState *s, unsigned long vcpu_id)
    return kvm_fd;
 }

-static void kvm_reset_parked_vcpus(KVMState *s)
+static void kvm_reset_parked_vcpus(void *param)
 {
+    KVMState *s = param;
    struct KVMParkedVcpu *cpu;

    QLIST_FOREACH(cpu, &s->kvm_parked_vcpus, node) {
@@ -456,13 +445,7 @@ static void kvm_reset_parked_vcpus(KVMState *s)
    }
 }

-/**
- * kvm_create_vcpu - Gets a parked KVM vCPU or creates a KVM vCPU
- * @cpu: QOM CPUState object for which KVM vCPU has to be fetched/created.
- *
- * @returns: 0 when success, errno (<0) when failed.
- */
-static int kvm_create_vcpu(CPUState *cpu)
+int kvm_create_vcpu(CPUState *cpu)
 {
    unsigned long vcpu_id = kvm_arch_vcpu_id(cpu);
    KVMState *s = kvm_state;
@@ -481,9 +464,7 @@ static int kvm_create_vcpu(CPUState *cpu)

    cpu->kvm_fd = kvm_fd;
    cpu->kvm_state = s;
-    if (!s->guest_state_protected) {
-        cpu->vcpu_dirty = true;
-    }
+    cpu->vcpu_dirty = true;
    cpu->dirty_pages = 0;
    cpu->throttle_us_per_full = 0;

@@ -507,7 +488,7 @@ int kvm_create_and_park_vcpu(CPUState *cpu)
 static int do_kvm_destroy_vcpu(CPUState *cpu)
 {
    KVMState *s = kvm_state;
-    int mmap_size;
+    long mmap_size;
    int ret = 0;

    trace_kvm_destroy_vcpu(cpu->cpu_index, kvm_arch_vcpu_id(cpu));
@@ -524,24 +505,16 @@ static int do_kvm_destroy_vcpu(CPUState *cpu)
        goto err;
    }

-    /* If I am the CPU that created coalesced_mmio_ring, then discard it */
-    if (s->coalesced_mmio_ring ==
-           (void *)cpu->kvm_run + s->coalesced_mmio * PAGE_SIZE) {
-        s->coalesced_mmio_ring = NULL;
-    }
-
    ret = munmap(cpu->kvm_run, mmap_size);
    if (ret < 0) {
        goto err;
    }
-    cpu->kvm_run = NULL;

    if (cpu->kvm_dirty_gfns) {
        ret = munmap(cpu->kvm_dirty_gfns, s->kvm_dirty_ring_bytes);
        if (ret < 0) {
            goto err;
        }
-        cpu->kvm_dirty_gfns = NULL;
    }

    kvm_park_vcpu(cpu);
@@ -560,16 +533,11 @@ void kvm_destroy_vcpu(CPUState *cpu)
 int kvm_init_vcpu(CPUState *cpu, Error **errp)
 {
    KVMState *s = kvm_state;
-    int mmap_size;
+    long mmap_size;
    int ret;

    trace_kvm_init_vcpu(cpu->cpu_index, kvm_arch_vcpu_id(cpu));

-    ret = kvm_arch_pre_create_vcpu(cpu, errp);
-    if (ret < 0) {
-        goto err;
-    }
-
    ret = kvm_create_vcpu(cpu);
    if (ret < 0) {
        error_setg_errno(errp, -ret,
@@ -625,31 +593,6 @@ err:
    return ret;
 }

-void kvm_close(void)
-{
-    CPUState *cpu;
-
-    if (!kvm_state || kvm_state->fd == -1) {
-        return;
-    }
-
-    CPU_FOREACH(cpu) {
-        cpu_remove_sync(cpu);
-        close(cpu->kvm_fd);
-        cpu->kvm_fd = -1;
-        close(cpu->kvm_vcpu_stats_fd);
-        cpu->kvm_vcpu_stats_fd = -1;
-    }
-
-    if (kvm_state && kvm_state->fd != -1) {
-        close(kvm_state->vmfd);
-        kvm_state->vmfd = -1;
-        close(kvm_state->fd);
-        kvm_state->fd = -1;
-    }
-    kvm_state = NULL;
-}
-
 /*
 * dirty pages logging control
 */
@@ -759,7 +702,7 @@ static void kvm_slot_sync_dirty_pages(KVMSlot *slot)
    ram_addr_t start = slot->ram_start_offset;
    ram_addr_t pages = slot->memory_size / qemu_real_host_page_size();

-    physical_memory_set_dirty_lebitmap(slot->dirty_bmap, start, pages);
+    cpu_physical_memory_set_dirty_lebitmap(slot->dirty_bmap, start, pages);
 }

 static void kvm_slot_reset_dirty_pages(KVMSlot *slot)
@@ -1343,7 +1286,7 @@ static void kvm_unpoison_all(void *param)

    QLIST_FOREACH_SAFE(page, &hwpoison_page_list, list, next_page) {
        QLIST_REMOVE(page, list);
-        qemu_ram_remap(page->ram_addr);
+        qemu_ram_remap(page->ram_addr, TARGET_PAGE_SIZE);
        g_free(page);
    }
 }
@@ -1369,22 +1312,21 @@ bool kvm_hwpoisoned_mem(void)

 static uint32_t adjust_ioeventfd_endianness(uint32_t val, uint32_t size)
 {
-    if (target_needs_bswap()) {
-        /*
-         * The kernel expects ioeventfd values in HOST_BIG_ENDIAN
-         * endianness, but the memory core hands them in target endianness.
-         * For example, PPC is always treated as big-endian even if running
-         * on KVM and on PPC64LE.  Correct here, swapping back.
-         */
-        switch (size) {
-        case 2:
-            val = bswap16(val);
-            break;
-        case 4:
-            val = bswap32(val);
-            break;
-        }
+#if HOST_BIG_ENDIAN != TARGET_BIG_ENDIAN
+    /* The kernel expects ioeventfd values in HOST_BIG_ENDIAN
+     * endianness, but the memory core hands them in target endianness.
+     * For example, PPC is always treated as big-endian even if running
+     * on KVM and on PPC64LE.  Correct here.
+     */
+    switch (size) {
+    case 2:
+        val = bswap16(val);
+        break;
+    case 4:
+        val = bswap32(val);
+        break;
    }
+#endif
    return val;
 }

@@ -1582,7 +1524,7 @@ static void kvm_set_phys_mem(KVMMemoryListener *kml,
            }
            start_addr += slot_size;
            size -= slot_size;
-            kml->nr_slots_used--;
+            kml->nr_used_slots--;
        } while (size);
        return;
    }
@@ -1598,8 +1540,7 @@ static void kvm_set_phys_mem(KVMMemoryListener *kml,
        mem->ram = ram;
        mem->flags = kvm_mem_flags(mr);
        mem->guest_memfd = mr->ram_block->guest_memfd;
-        mem->guest_memfd_offset = mem->guest_memfd >= 0 ?
-                                  (uint8_t*)ram - mr->ram_block->host : 0;
+        mem->guest_memfd_offset = (uint8_t*)ram - mr->ram_block->host;

        kvm_slot_init_dirty_bitmap(mem);
        err = kvm_set_user_memory_region(kml, mem, true);
@@ -1622,7 +1563,7 @@ static void kvm_set_phys_mem(KVMMemoryListener *kml,
        ram_start_offset += slot_size;
        ram += slot_size;
        size -= slot_size;
-        kml->nr_slots_used++;
+        kml->nr_used_slots++;
    } while (size);
 }

@@ -1658,7 +1599,11 @@ static void *kvm_dirty_ring_reaper_thread(void *data)
        r->reaper_iteration++;
    }

-    g_assert_not_reached();
+    trace_kvm_dirty_ring_reaper("exit");
+
+    rcu_unregister_thread();
+
+    return NULL;
 }

 static void kvm_dirty_ring_reaper_init(KVMState *s)
@@ -2477,7 +2422,7 @@ static int kvm_recommended_vcpus(KVMState *s)

 static int kvm_max_vcpus(KVMState *s)
 {
-    int ret = kvm_vm_check_extension(s, KVM_CAP_MAX_VCPUS);
+    int ret = kvm_check_extension(s, KVM_CAP_MAX_VCPUS);
    return (ret) ? ret : kvm_recommended_vcpus(s);
 }

@@ -2507,61 +2452,171 @@ uint32_t kvm_dirty_ring_size(void)
    return kvm_state->kvm_dirty_ring_size;
 }

-static int do_kvm_create_vm(KVMState *s, int type)
-{
-    int ret;
-
-    do {
-        ret = kvm_ioctl(s, KVM_CREATE_VM, type);
-    } while (ret == -EINTR);
-
-    if (ret < 0) {
-        error_report("ioctl(KVM_CREATE_VM) failed: %s", strerror(-ret));
-
-#ifdef TARGET_S390X
-        if (ret == -EINVAL) {
-            error_printf("Host kernel setup problem detected."
-                         " Please verify:\n");
-            error_printf("- for kernels supporting the"
-                        " switch_amode or user_mode parameters, whether");
-            error_printf(" user space is running in primary address space\n");
-            error_printf("- for kernels supporting the vm.allocate_pgste"
-                         " sysctl, whether it is enabled\n");
-        }
-#elif defined(TARGET_PPC)
-        if (ret == -EINVAL) {
-            error_printf("PPC KVM module is not loaded. Try modprobe kvm_%s.\n",
-                         (type == 2) ? "pr" : "hv");
-        }
-#endif
-    }
-
-    return ret;
-}
-
-static int find_kvm_machine_type(MachineState *ms)
+static int kvm_init(MachineState *ms)
 {
    MachineClass *mc = MACHINE_GET_CLASS(ms);
+    static const char upgrade_note[] =
+        "Please upgrade to at least kernel 2.6.29 or recent kvm-kmod\n"
+        "(see http://sourceforge.net/projects/kvm).\n";
+    const struct {
+        const char *name;
+        int num;
+    } num_cpus[] = {
+        { "SMP",          ms->smp.cpus },
+        { "hotpluggable", ms->smp.max_cpus },
+        { /* end of list */ }
+    }, *nc = num_cpus;
+    int soft_vcpus_limit, hard_vcpus_limit;
+    KVMState *s;
+    const KVMCapabilityInfo *missing_cap;
+    int ret;
    int type;
+    uint64_t dirty_log_manual_caps;
+
+    qemu_mutex_init(&kml_slots_lock);
+
+    s = KVM_STATE(ms->accelerator);
+
+    /*
+     * On systems where the kernel can support different base page
+     * sizes, host page size may be different from TARGET_PAGE_SIZE,
+     * even with KVM.  TARGET_PAGE_SIZE is assumed to be the minimum
+     * page size for the system though.
+     */
+    assert(TARGET_PAGE_SIZE <= qemu_real_host_page_size());
+
+    s->sigmask_len = 8;
+    accel_blocker_init();
+
+#ifdef TARGET_KVM_HAVE_GUEST_DEBUG
+    QTAILQ_INIT(&s->kvm_sw_breakpoints);
+#endif
+    QLIST_INIT(&s->kvm_parked_vcpus);
+    s->fd = qemu_open_old(s->device ?: "/dev/kvm", O_RDWR);
+    if (s->fd == -1) {
+        fprintf(stderr, "Could not access KVM kernel module: %m\n");
+        ret = -errno;
+        goto err;
+    }
+
+    ret = kvm_ioctl(s, KVM_GET_API_VERSION, 0);
+    if (ret < KVM_API_VERSION) {
+        if (ret >= 0) {
+            ret = -EINVAL;
+        }
+        fprintf(stderr, "kvm version too old\n");
+        goto err;
+    }
+
+    if (ret > KVM_API_VERSION) {
+        ret = -EINVAL;
+        fprintf(stderr, "kvm version not supported\n");
+        goto err;
+    }
+
+    kvm_supported_memory_attributes = kvm_check_extension(s, KVM_CAP_MEMORY_ATTRIBUTES);
+    kvm_guest_memfd_supported =
+        kvm_check_extension(s, KVM_CAP_GUEST_MEMFD) &&
+        kvm_check_extension(s, KVM_CAP_USER_MEMORY2) &&
+        (kvm_supported_memory_attributes & KVM_MEMORY_ATTRIBUTE_PRIVATE);
+
+    kvm_immediate_exit = kvm_check_extension(s, KVM_CAP_IMMEDIATE_EXIT);
+    s->nr_slots = kvm_check_extension(s, KVM_CAP_NR_MEMSLOTS);
+
+    /* If unspecified, use the default value */
+    if (!s->nr_slots) {
+        s->nr_slots = 32;
+    }
+
+    s->nr_as = kvm_check_extension(s, KVM_CAP_MULTI_ADDRESS_SPACE);
+    if (s->nr_as <= 1) {
+        s->nr_as = 1;
+    }
+    s->as = g_new0(struct KVMAs, s->nr_as);

    if (object_property_find(OBJECT(current_machine), "kvm-type")) {
-        g_autofree char *kvm_type;
-        kvm_type = object_property_get_str(OBJECT(current_machine),
-                                           "kvm-type",
-                                           &error_abort);
+        g_autofree char *kvm_type = object_property_get_str(OBJECT(current_machine),
+                                                            "kvm-type",
+                                                            &error_abort);
        type = mc->kvm_type(ms, kvm_type);
    } else if (mc->kvm_type) {
        type = mc->kvm_type(ms, NULL);
    } else {
        type = kvm_arch_get_default_type(ms);
    }
-    return type;
-}

-static int kvm_setup_dirty_ring(KVMState *s)
-{
-    uint64_t dirty_log_manual_caps;
-    int ret;
+    if (type < 0) {
+        ret = -EINVAL;
+        goto err;
+    }
+
+    do {
+        ret = kvm_ioctl(s, KVM_CREATE_VM, type);
+    } while (ret == -EINTR);
+
+    if (ret < 0) {
+        fprintf(stderr, "ioctl(KVM_CREATE_VM) failed: %d %s\n", -ret,
+                strerror(-ret));
+
+#ifdef TARGET_S390X
+        if (ret == -EINVAL) {
+            fprintf(stderr,
+                    "Host kernel setup problem detected. Please verify:\n");
+            fprintf(stderr, "- for kernels supporting the switch_amode or"
+                    " user_mode parameters, whether\n");
+            fprintf(stderr,
+                    "  user space is running in primary address space\n");
+            fprintf(stderr,
+                    "- for kernels supporting the vm.allocate_pgste sysctl, "
+                    "whether it is enabled\n");
+        }
+#elif defined(TARGET_PPC)
+        if (ret == -EINVAL) {
+            fprintf(stderr,
+                    "PPC KVM module is not loaded. Try modprobe kvm_%s.\n",
+                    (type == 2) ? "pr" : "hv");
+        }
+#endif
+        goto err;
+    }
+
+    s->vmfd = ret;
+
+    /* check the vcpu limits */
+    soft_vcpus_limit = kvm_recommended_vcpus(s);
+    hard_vcpus_limit = kvm_max_vcpus(s);
+
+    while (nc->name) {
+        if (nc->num > soft_vcpus_limit) {
+            warn_report("Number of %s cpus requested (%d) exceeds "
+                        "the recommended cpus supported by KVM (%d)",
+                        nc->name, nc->num, soft_vcpus_limit);
+
+            if (nc->num > hard_vcpus_limit) {
+                fprintf(stderr, "Number of %s cpus requested (%d) exceeds "
+                        "the maximum cpus supported by KVM (%d)\n",
+                        nc->name, nc->num, hard_vcpus_limit);
+                exit(1);
+            }
+        }
+        nc++;
+    }
+
+    missing_cap = kvm_check_extension_list(s, kvm_required_capabilites);
+    if (!missing_cap) {
+        missing_cap =
+            kvm_check_extension_list(s, kvm_arch_required_capabilities);
+    }
+    if (missing_cap) {
+        ret = -EINVAL;
+        fprintf(stderr, "kvm does not support %s\n%s",
+                missing_cap->name, upgrade_note);
+        goto err;
+    }
+
+    s->coalesced_mmio = kvm_check_extension(s, KVM_CAP_COALESCED_MMIO);
+    s->coalesced_pio = s->coalesced_mmio &&
+                       kvm_check_extension(s, KVM_CAP_COALESCED_PIO);

    /*
     * Enable KVM dirty ring if supported, otherwise fall back to
@@ -2569,7 +2624,7 @@ static int kvm_setup_dirty_ring(KVMState *s)
     */
    ret = kvm_dirty_ring_init(s);
    if (ret < 0) {
-        return ret;
+        goto err;
    }

    /*
@@ -2604,135 +2659,6 @@ static int kvm_setup_dirty_ring(KVMState *s)
        }
    }

-    return 0;
-}
-
-static int kvm_init(AccelState *as, MachineState *ms)
-{
-    MachineClass *mc = MACHINE_GET_CLASS(ms);
-    static const char upgrade_note[] =
-        "Please upgrade to at least kernel 4.5.\n";
-    const struct {
-        const char *name;
-        int num;
-    } num_cpus[] = {
-        { "SMP",          ms->smp.cpus },
-        { "hotpluggable", ms->smp.max_cpus },
-        { /* end of list */ }
-    }, *nc = num_cpus;
-    int soft_vcpus_limit, hard_vcpus_limit;
-    KVMState *s = KVM_STATE(as);
-    const KVMCapabilityInfo *missing_cap;
-    int ret;
-    int type;
-
-    qemu_mutex_init(&kml_slots_lock);
-
-    /*
-     * On systems where the kernel can support different base page
-     * sizes, host page size may be different from TARGET_PAGE_SIZE,
-     * even with KVM.  TARGET_PAGE_SIZE is assumed to be the minimum
-     * page size for the system though.
-     */
-    assert(TARGET_PAGE_SIZE <= qemu_real_host_page_size());
-
-    s->sigmask_len = 8;
-    accel_blocker_init();
-
-#ifdef TARGET_KVM_HAVE_GUEST_DEBUG
-    QTAILQ_INIT(&s->kvm_sw_breakpoints);
-#endif
-    QLIST_INIT(&s->kvm_parked_vcpus);
-    s->fd = qemu_open_old(s->device ?: "/dev/kvm", O_RDWR);
-    if (s->fd == -1) {
-        error_report("Could not access KVM kernel module: %m");
-        ret = -errno;
-        goto err;
-    }
-
-    ret = kvm_ioctl(s, KVM_GET_API_VERSION, 0);
-    if (ret < KVM_API_VERSION) {
-        if (ret >= 0) {
-            ret = -EINVAL;
-        }
-        error_report("kvm version too old");
-        goto err;
-    }
-
-    if (ret > KVM_API_VERSION) {
-        ret = -EINVAL;
-        error_report("kvm version not supported");
-        goto err;
-    }
-
-    kvm_immediate_exit = kvm_check_extension(s, KVM_CAP_IMMEDIATE_EXIT);
-    s->nr_slots_max = kvm_check_extension(s, KVM_CAP_NR_MEMSLOTS);
-
-    /* If unspecified, use the default value */
-    if (!s->nr_slots_max) {
-        s->nr_slots_max = KVM_MEMSLOTS_NR_MAX_DEFAULT;
-    }
-
-    type = find_kvm_machine_type(ms);
-    if (type < 0) {
-        ret = -EINVAL;
-        goto err;
-    }
-
-    ret = do_kvm_create_vm(s, type);
-    if (ret < 0) {
-        goto err;
-    }
-
-    s->vmfd = ret;
-
-    s->nr_as = kvm_vm_check_extension(s, KVM_CAP_MULTI_ADDRESS_SPACE);
-    if (s->nr_as <= 1) {
-        s->nr_as = 1;
-    }
-    s->as = g_new0(struct KVMAs, s->nr_as);
-
-    /* check the vcpu limits */
-    soft_vcpus_limit = kvm_recommended_vcpus(s);
-    hard_vcpus_limit = kvm_max_vcpus(s);
-
-    while (nc->name) {
-        if (nc->num > soft_vcpus_limit) {
-            warn_report("Number of %s cpus requested (%d) exceeds "
-                        "the recommended cpus supported by KVM (%d)",
-                        nc->name, nc->num, soft_vcpus_limit);
-
-            if (nc->num > hard_vcpus_limit) {
-                error_report("Number of %s cpus requested (%d) exceeds "
-                             "the maximum cpus supported by KVM (%d)",
-                             nc->name, nc->num, hard_vcpus_limit);
-                exit(1);
-            }
-        }
-        nc++;
-    }
-
-    missing_cap = kvm_check_extension_list(s, kvm_required_capabilites);
-    if (!missing_cap) {
-        missing_cap =
-            kvm_check_extension_list(s, kvm_arch_required_capabilities);
-    }
-    if (missing_cap) {
-        ret = -EINVAL;
-        error_report("kvm does not support %s", missing_cap->name);
-        error_printf("%s", upgrade_note);
-        goto err;
-    }
-
-    s->coalesced_mmio = kvm_check_extension(s, KVM_CAP_COALESCED_MMIO);
-    s->coalesced_pio = s->coalesced_mmio &&
-                       kvm_check_extension(s, KVM_CAP_COALESCED_PIO);
-
-    ret = kvm_setup_dirty_ring(s);
-    if (ret < 0) {
-        goto err;
-    }
-
 #ifdef KVM_CAP_VCPU_EVENTS
    s->vcpu_events = kvm_check_extension(s, KVM_CAP_VCPU_EVENTS);
 #endif
@@ -2778,18 +2704,12 @@ static int kvm_init(AccelState *as, MachineState *ms)
        goto err;
    }

-    kvm_supported_memory_attributes = kvm_vm_check_extension(s, KVM_CAP_MEMORY_ATTRIBUTES);
-    kvm_guest_memfd_supported =
-        kvm_vm_check_extension(s, KVM_CAP_GUEST_MEMFD) &&
-        kvm_vm_check_extension(s, KVM_CAP_USER_MEMORY2) &&
-        (kvm_supported_memory_attributes & KVM_MEMORY_ATTRIBUTE_PRIVATE);
-    kvm_pre_fault_memory_supported = kvm_vm_check_extension(s, KVM_CAP_PRE_FAULT_MEMORY);
-
    if (s->kernel_irqchip_split == ON_OFF_AUTO_AUTO) {
        s->kernel_irqchip_split = mc->default_kernel_irqchip_split ? ON_OFF_AUTO_ON : ON_OFF_AUTO_OFF;
    }

    qemu_register_reset(kvm_unpoison_all, NULL);
+    qemu_register_reset(kvm_reset_parked_vcpus, s);

    if (s->kernel_irqchip_allowed) {
        kvm_irqchip_create(s);
@@ -2914,15 +2834,9 @@ void kvm_flush_coalesced_mmio_buffer(void)
 static void do_kvm_cpu_synchronize_state(CPUState *cpu, run_on_cpu_data arg)
 {
    if (!cpu->vcpu_dirty && !kvm_state->guest_state_protected) {
-        Error *err = NULL;
-        int ret = kvm_arch_get_registers(cpu, &err);
+        int ret = kvm_arch_get_registers(cpu);
        if (ret) {
-            if (err) {
-                error_reportf_err(err, "Failed to synchronize CPU state: ");
-            } else {
-                error_report("Failed to get registers: %s", strerror(-ret));
-            }
-
+            error_report("Failed to get registers: %s", strerror(-ret));
            cpu_dump_state(cpu, stderr, CPU_DUMP_CODE);
            vm_stop(RUN_STATE_INTERNAL_ERROR);
        }
@@ -2938,48 +2852,32 @@ void kvm_cpu_synchronize_state(CPUState *cpu)
    }
 }

-static bool kvm_cpu_synchronize_put(CPUState *cpu, KvmPutState state,
-                                    const char *desc)
-{
-    Error *err = NULL;
-    int ret = kvm_arch_put_registers(cpu, state, &err);
-    if (ret) {
-        if (err) {
-            error_reportf_err(err, "Restoring resisters %s: ", desc);
-        } else {
-            error_report("Failed to put registers %s: %s", desc,
-                         strerror(-ret));
-        }
-        return false;
-    }
-
-    cpu->vcpu_dirty = false;
-
-    return true;
-}
-
 static void do_kvm_cpu_synchronize_post_reset(CPUState *cpu, run_on_cpu_data arg)
 {
-    if (!kvm_cpu_synchronize_put(cpu, KVM_PUT_RESET_STATE, "after reset")) {
+    int ret = kvm_arch_put_registers(cpu, KVM_PUT_RESET_STATE);
+    if (ret) {
+        error_report("Failed to put registers after reset: %s", strerror(-ret));
        cpu_dump_state(cpu, stderr, CPU_DUMP_CODE);
        vm_stop(RUN_STATE_INTERNAL_ERROR);
    }
+
+    cpu->vcpu_dirty = false;
 }

 void kvm_cpu_synchronize_post_reset(CPUState *cpu)
 {
    run_on_cpu(cpu, do_kvm_cpu_synchronize_post_reset, RUN_ON_CPU_NULL);
-
-    if (cpu == first_cpu) {
-        kvm_reset_parked_vcpus(kvm_state);
-    }
 }

 static void do_kvm_cpu_synchronize_post_init(CPUState *cpu, run_on_cpu_data arg)
 {
-    if (!kvm_cpu_synchronize_put(cpu, KVM_PUT_FULL_STATE, "after init")) {
+    int ret = kvm_arch_put_registers(cpu, KVM_PUT_FULL_STATE);
+    if (ret) {
+        error_report("Failed to put registers after init: %s", strerror(-ret));
        exit(1);
    }
+
+    cpu->vcpu_dirty = false;
 }

 void kvm_cpu_synchronize_post_init(CPUState *cpu)
@@ -3033,6 +2931,10 @@ static void kvm_eat_signals(CPUState *cpu)

    if (kvm_immediate_exit) {
        qatomic_set(&cpu->kvm_run->immediate_exit, 0);
+        /* Write kvm_run->immediate_exit before the cpu->exit_request
+         * write in kvm_cpu_exec.
+         */
+        smp_wmb();
        return;
    }

@@ -3061,17 +2963,17 @@ int kvm_convert_memory(hwaddr start, hwaddr size, bool to_private)
    MemoryRegion *mr;
    RAMBlock *rb;
    void *addr;
-    int ret = -EINVAL;
+    int ret = -1;

    trace_kvm_convert_memory(start, size, to_private ? "shared_to_private" : "private_to_shared");

    if (!QEMU_PTR_IS_ALIGNED(start, qemu_real_host_page_size()) ||
        !QEMU_PTR_IS_ALIGNED(size, qemu_real_host_page_size())) {
-        return ret;
+        return -1;
    }

    if (!size) {
-        return ret;
+        return -1;
    }

    section = memory_region_find(get_system_memory(), start, size);
@@ -3089,7 +2991,7 @@ int kvm_convert_memory(hwaddr start, hwaddr size, bool to_private)
        if (!to_private) {
            return 0;
        }
-        return ret;
+        return -1;
    }

    if (!memory_region_has_guest_memfd(mr)) {
@@ -3124,15 +3026,6 @@ int kvm_convert_memory(hwaddr start, hwaddr size, bool to_private)
    addr = memory_region_get_ram_ptr(mr) + section.offset_within_region;
    rb = qemu_ram_block_from_host(addr, false, &offset);

-    ret = ram_block_attributes_state_change(RAM_BLOCK_ATTRIBUTES(mr->rdm),
-                                            offset, size, to_private);
-    if (ret) {
-        error_report("Failed to notify the listener the state change of "
-                     "(0x%"HWADDR_PRIx" + 0x%"HWADDR_PRIx") to %s",
-                     start, size, to_private ? "private" : "shared");
-        goto out_unref;
-    }
-
    if (to_private) {
        if (rb->page_size != qemu_real_host_page_size()) {
            /*
@@ -3159,6 +3052,7 @@ int kvm_cpu_exec(CPUState *cpu)
    trace_kvm_cpu_exec();

    if (kvm_arch_process_async_events(cpu)) {
+        qatomic_set(&cpu->exit_request, 0);
        return EXCP_HLT;
    }

@@ -3169,16 +3063,19 @@ int kvm_cpu_exec(CPUState *cpu)
        MemTxAttrs attrs;

        if (cpu->vcpu_dirty) {
-            if (!kvm_cpu_synchronize_put(cpu, KVM_PUT_RUNTIME_STATE,
-                                         "at runtime")) {
+            ret = kvm_arch_put_registers(cpu, KVM_PUT_RUNTIME_STATE);
+            if (ret) {
+                error_report("Failed to put registers after init: %s",
+                             strerror(-ret));
                ret = -1;
                break;
            }
+
+            cpu->vcpu_dirty = false;
        }

        kvm_arch_pre_run(cpu, run);
-        /* Corresponding store-release is in cpu_exit. */
-        if (qatomic_load_acquire(&cpu->exit_request)) {
+        if (qatomic_read(&cpu->exit_request)) {
            trace_kvm_interrupt_exit_request();
            /*
             * KVM requires us to reenter the kernel after IO exits to complete
@@ -3188,14 +3085,12 @@ int kvm_cpu_exec(CPUState *cpu)
            kvm_cpu_kick_self();
        }

-        run_ret = kvm_vcpu_ioctl(cpu, KVM_RUN, 0);
-
-        /*
-         * After writing cpu->exit_request, cpu_exit() sends a signal that writes
-         * kvm->run->immediate_exit.  The signal is already happening after the
-         * write to cpu->exit_request so, if KVM read kvm->run->immediate_exit
-         * as true, cpu->exit_request will always read as true.
+        /* Read cpu->exit_request before KVM_RUN reads run->immediate_exit.
+         * Matching barrier in kvm_eat_signals.
         */
+        smp_rmb();
+
+        run_ret = kvm_vcpu_ioctl(cpu, KVM_RUN, 0);

        attrs = kvm_arch_post_run(cpu, run);

@@ -3339,10 +3234,11 @@ int kvm_cpu_exec(CPUState *cpu)
        vm_stop(RUN_STATE_INTERNAL_ERROR);
    }

+    qatomic_set(&cpu->exit_request, 0);
    return ret;
 }

-int kvm_ioctl(KVMState *s, unsigned long type, ...)
+int kvm_ioctl(KVMState *s, int type, ...)
 {
    int ret;
    void *arg;
@@ -3360,7 +3256,7 @@ int kvm_ioctl(KVMState *s, unsigned long type, ...)
    return ret;
 }

-int kvm_vm_ioctl(KVMState *s, unsigned long type, ...)
+int kvm_vm_ioctl(KVMState *s, int type, ...)
 {
    int ret;
    void *arg;
@@ -3380,7 +3276,7 @@ int kvm_vm_ioctl(KVMState *s, unsigned long type, ...)
    return ret;
 }

-int kvm_vcpu_ioctl(CPUState *cpu, unsigned long type, ...)
+int kvm_vcpu_ioctl(CPUState *cpu, int type, ...)
 {
    int ret;
    void *arg;
@@ -3400,7 +3296,7 @@ int kvm_vcpu_ioctl(CPUState *cpu, unsigned long type, ...)
    return ret;
 }

-int kvm_device_ioctl(int fd, unsigned long type, ...)
+int kvm_device_ioctl(int fd, int type, ...)
 {
    int ret;
    void *arg;
@@ -3723,7 +3619,7 @@ int kvm_on_sigbus_vcpu(CPUState *cpu, int code, void *addr)
    have_sigbus_pending = true;
    pending_sigbus_addr = addr;
    pending_sigbus_code = code;
-    qatomic_set(&cpu->exit_request, true);
+    qatomic_set(&cpu->exit_request, 1);
    return 0;
 #else
    return 1;
@@ -3810,10 +3706,10 @@ int kvm_get_one_reg(CPUState *cs, uint64_t id, void *target)
    return r;
 }

-static bool kvm_accel_has_memory(AccelState *accel, AddressSpace *as,
+static bool kvm_accel_has_memory(MachineState *ms, AddressSpace *as,
                                 hwaddr start_addr, hwaddr size)
 {
-    KVMState *kvm = KVM_STATE(accel);
+    KVMState *kvm = KVM_STATE(ms->accelerator);
    int i;

    for (i = 0; i < kvm->nr_as; ++i) {
@@ -4004,12 +3900,12 @@ static void kvm_accel_instance_init(Object *obj)
 * Returns: SSTEP_* flags that KVM supports for guest debug. The
 * support is probed during kvm_init()
 */
-static int kvm_gdbstub_sstep_flags(AccelState *as)
+static int kvm_gdbstub_sstep_flags(void)
 {
    return kvm_sstep_flags;
 }

-static void kvm_accel_class_init(ObjectClass *oc, const void *data)
+static void kvm_accel_class_init(ObjectClass *oc, void *data)
 {
    AccelClass *ac = ACCEL_CLASS(oc);
    ac->name = "KVM";
@@ -10,6 +10,8 @@
 #ifndef KVM_CPUS_H
 #define KVM_CPUS_H

+#include "sysemu/cpus.h"
+
 int kvm_init_vcpu(CPUState *cpu, Error **errp);
 int kvm_cpu_exec(CPUState *cpu);
 void kvm_destroy_vcpu(CPUState *cpu);
@@ -1,11 +1,11 @@
 # See docs/devel/tracing.rst for syntax documentation.

 # kvm-all.c
-kvm_ioctl(unsigned long type, void *arg) "type 0x%lx, arg %p"
-kvm_vm_ioctl(unsigned long type, void *arg) "type 0x%lx, arg %p"
-kvm_vcpu_ioctl(int cpu_index, unsigned long type, void *arg) "cpu_index %d, type 0x%lx, arg %p"
+kvm_ioctl(int type, void *arg) "type 0x%x, arg %p"
+kvm_vm_ioctl(int type, void *arg) "type 0x%x, arg %p"
+kvm_vcpu_ioctl(int cpu_index, int type, void *arg) "cpu_index %d, type 0x%x, arg %p"
 kvm_run_exit(int cpu_index, uint32_t reason) "cpu_index %d, reason %d"
-kvm_device_ioctl(int fd, unsigned long type, void *arg) "dev fd %d, type 0x%lx, arg %p"
+kvm_device_ioctl(int fd, int type, void *arg) "dev fd %d, type 0x%x, arg %p"
 kvm_failed_reg_get(uint64_t id, const char *msg) "Warning: Unable to retrieve ONEREG %" PRIu64 " from KVM: %s"
 kvm_failed_reg_set(uint64_t id, const char *msg) "Warning: Unable to set ONEREG %" PRIu64 " to KVM: %s"
 kvm_init_vcpu(int cpu_index, unsigned long arch_cpu_id) "index: %d id: %lu"
@@ -1,6 +1,5 @@
-common_ss.add(files('accel-common.c'))
 specific_ss.add(files('accel-target.c'))
-system_ss.add(files('accel-system.c', 'accel-blocker.c', 'accel-qmp.c', 'accel-irq.c'))
+system_ss.add(files('accel-system.c', 'accel-blocker.c'))
 user_ss.add(files('accel-user.c'))

 subdir('tcg')
@@ -10,7 +9,6 @@ if have_system
  subdir('kvm')
  subdir('xen')
  subdir('stubs')
-  subdir('mshv')
 endif

 # qtest
@@ -1,399 +0,0 @@
-/*
- * QEMU MSHV support
- *
- * Copyright Microsoft, Corp. 2025
- *
- * Authors: Ziqiao Zhou <ziqiaozhou@microsoft.com>
- *          Magnus Kulke <magnuskulke@microsoft.com>
- *          Stanislav Kinsburskii <skinsburskii@microsoft.com>
- *
- * SPDX-License-Identifier: GPL-2.0-or-later
- */
-
-#include "linux/mshv.h"
-#include "qemu/osdep.h"
-#include "qemu/error-report.h"
-#include "hw/hyperv/hvhdk_mini.h"
-#include "hw/hyperv/hvgdk_mini.h"
-#include "hw/intc/ioapic.h"
-#include "hw/pci/msi.h"
-#include "system/mshv.h"
-#include "system/mshv_int.h"
-#include "trace.h"
-#include <stdint.h>
-#include <sys/ioctl.h>
-
-#define MSHV_IRQFD_RESAMPLE_FLAG (1 << MSHV_IRQFD_BIT_RESAMPLE)
-#define MSHV_IRQFD_BIT_DEASSIGN_FLAG (1 << MSHV_IRQFD_BIT_DEASSIGN)
-
-static MshvMsiControl *msi_control;
-static QemuMutex msi_control_mutex;
-
-void mshv_init_msicontrol(void)
-{
-    qemu_mutex_init(&msi_control_mutex);
-    msi_control = g_new0(MshvMsiControl, 1);
-    msi_control->gsi_routes = g_hash_table_new(g_direct_hash, g_direct_equal);
-    msi_control->updated = false;
-}
-
-static int set_msi_routing(uint32_t gsi, uint64_t addr, uint32_t data)
-{
-    struct mshv_user_irq_entry *entry;
-    uint32_t high_addr = addr >> 32;
-    uint32_t low_addr = addr & 0xFFFFFFFF;
-    GHashTable *gsi_routes;
-
-    trace_mshv_set_msi_routing(gsi, addr, data);
-
-    if (gsi >= MSHV_MAX_MSI_ROUTES) {
-        error_report("gsi >= MSHV_MAX_MSI_ROUTES");
-        return -1;
-    }
-
-    assert(msi_control);
-
-    WITH_QEMU_LOCK_GUARD(&msi_control_mutex) {
-        gsi_routes = msi_control->gsi_routes;
-        entry = g_hash_table_lookup(gsi_routes, GINT_TO_POINTER(gsi));
-
-        if (entry
-            && entry->address_hi == high_addr
-            && entry->address_lo == low_addr
-            && entry->data == data)
-        {
-            /* nothing to update */
-            return 0;
-        }
-
-        /* free old entry */
-        g_free(entry);
-
-        /* create new entry */
-        entry = g_new0(struct mshv_user_irq_entry, 1);
-        entry->gsi = gsi;
-        entry->address_hi = high_addr;
-        entry->address_lo = low_addr;
-        entry->data = data;
-
-        g_hash_table_insert(gsi_routes, GINT_TO_POINTER(gsi), entry);
-        msi_control->updated = true;
-    }
-
-    return 0;
-}
-
-static int add_msi_routing(uint64_t addr, uint32_t data)
-{
-    struct mshv_user_irq_entry *route_entry;
-    uint32_t high_addr = addr >> 32;
-    uint32_t low_addr = addr & 0xFFFFFFFF;
-    int gsi;
-    GHashTable *gsi_routes;
-
-    trace_mshv_add_msi_routing(addr, data);
-
-    assert(msi_control);
-
-    WITH_QEMU_LOCK_GUARD(&msi_control_mutex) {
-        /* find an empty slot */
-        gsi = 0;
-        gsi_routes = msi_control->gsi_routes;
-        while (gsi < MSHV_MAX_MSI_ROUTES) {
-            route_entry = g_hash_table_lookup(gsi_routes, GINT_TO_POINTER(gsi));
-            if (!route_entry) {
-                break;
-            }
-            gsi++;
-        }
-        if (gsi >= MSHV_MAX_MSI_ROUTES) {
-            error_report("No empty gsi slot available");
-            return -1;
-        }
-
-        /* create new entry */
-        route_entry = g_new0(struct mshv_user_irq_entry, 1);
-        route_entry->gsi = gsi;
-        route_entry->address_hi = high_addr;
-        route_entry->address_lo = low_addr;
-        route_entry->data = data;
-
-        g_hash_table_insert(gsi_routes, GINT_TO_POINTER(gsi), route_entry);
-        msi_control->updated = true;
-    }
-
-    return gsi;
-}
-
-static int commit_msi_routing_table(int vm_fd)
-{
-    guint len;
-    int i, ret;
-    size_t table_size;
-    struct mshv_user_irq_table *table;
-    GHashTableIter iter;
-    gpointer key, value;
-
-    assert(msi_control);
-
-    WITH_QEMU_LOCK_GUARD(&msi_control_mutex) {
-        if (!msi_control->updated) {
-            /* nothing to update */
-            return 0;
-        }
-
-        /* Calculate the size of the table */
-        len = g_hash_table_size(msi_control->gsi_routes);
-        table_size = sizeof(struct mshv_user_irq_table)
-                     + len * sizeof(struct mshv_user_irq_entry);
-        table = g_malloc0(table_size);
-
-        g_hash_table_iter_init(&iter, msi_control->gsi_routes);
-        i = 0;
-        while (g_hash_table_iter_next(&iter, &key, &value)) {
-            struct mshv_user_irq_entry *entry = value;
-            table->entries[i] = *entry;
-            i++;
-        }
-        table->nr = i;
-
-        trace_mshv_commit_msi_routing_table(vm_fd, len);
-
-        ret = ioctl(vm_fd, MSHV_SET_MSI_ROUTING, table);
-        g_free(table);
-        if (ret < 0) {
-            error_report("Failed to commit msi routing table");
-            return -1;
-        }
-        msi_control->updated = false;
-    }
-    return 0;
-}
-
-static int remove_msi_routing(uint32_t gsi)
-{
-    struct mshv_user_irq_entry *route_entry;
-    GHashTable *gsi_routes;
-
-    trace_mshv_remove_msi_routing(gsi);
-
-    if (gsi >= MSHV_MAX_MSI_ROUTES) {
-        error_report("Invalid GSI: %u", gsi);
-        return -1;
-    }
-
-    assert(msi_control);
-
-    WITH_QEMU_LOCK_GUARD(&msi_control_mutex) {
-        gsi_routes = msi_control->gsi_routes;
-        route_entry = g_hash_table_lookup(gsi_routes, GINT_TO_POINTER(gsi));
-        if (route_entry) {
-            g_hash_table_remove(gsi_routes, GINT_TO_POINTER(gsi));
-            g_free(route_entry);
-            msi_control->updated = true;
-        }
-    }
-
-    return 0;
-}
-
-/* Pass an eventfd which is to be used for injecting interrupts from userland */
-static int irqfd(int vm_fd, int fd, int resample_fd, uint32_t gsi,
-                 uint32_t flags)
-{
-    int ret;
-    struct mshv_user_irqfd arg = {
-        .fd = fd,
-        .resamplefd = resample_fd,
-        .gsi = gsi,
-        .flags = flags,
-    };
-
-    ret = ioctl(vm_fd, MSHV_IRQFD, &arg);
-    if (ret < 0) {
-        error_report("Failed to set irqfd: gsi=%u, fd=%d", gsi, fd);
-        return -1;
-    }
-    return ret;
-}
-
-static int register_irqfd(int vm_fd, int event_fd, uint32_t gsi)
-{
-    int ret;
-
-    trace_mshv_register_irqfd(vm_fd, event_fd, gsi);
-
-    ret = irqfd(vm_fd, event_fd, 0, gsi, 0);
-    if (ret < 0) {
-        error_report("Failed to register irqfd: gsi=%u", gsi);
-        return -1;
-    }
-    return 0;
-}
-
-static int register_irqfd_with_resample(int vm_fd, int event_fd,
-                                        int resample_fd, uint32_t gsi)
-{
-    int ret;
-    uint32_t flags = MSHV_IRQFD_RESAMPLE_FLAG;
-
-    ret = irqfd(vm_fd, event_fd, resample_fd, gsi, flags);
-    if (ret < 0) {
-        error_report("Failed to register irqfd with resample: gsi=%u", gsi);
-        return -errno;
-    }
-    return 0;
-}
-
-static int unregister_irqfd(int vm_fd, int event_fd, uint32_t gsi)
-{
-    int ret;
-    uint32_t flags = MSHV_IRQFD_BIT_DEASSIGN_FLAG;
-
-    ret = irqfd(vm_fd, event_fd, 0, gsi, flags);
-    if (ret < 0) {
-        error_report("Failed to unregister irqfd: gsi=%u", gsi);
-        return -errno;
-    }
-    return 0;
-}
-
-static int irqchip_update_irqfd_notifier_gsi(const EventNotifier *event,
-                                             const EventNotifier *resample,
-                                             int virq, bool add)
-{
-    int fd = event_notifier_get_fd(event);
-    int rfd = resample ? event_notifier_get_fd(resample) : -1;
-    int vm_fd = mshv_state->vm;
-
-    trace_mshv_irqchip_update_irqfd_notifier_gsi(fd, rfd, virq, add);
-
-    if (!add) {
-        return unregister_irqfd(vm_fd, fd, virq);
-    }
-
-    if (rfd > 0) {
-        return register_irqfd_with_resample(vm_fd, fd, rfd, virq);
-    }
-
-    return register_irqfd(vm_fd, fd, virq);
-}
-
-
-int mshv_irqchip_add_msi_route(int vector, PCIDevice *dev)
-{
-    MSIMessage msg = { 0, 0 };
-    int virq = 0;
-
-    if (pci_available && dev) {
-        msg = pci_get_msi_message(dev, vector);
-        virq = add_msi_routing(msg.address, le32_to_cpu(msg.data));
-    }
-
-    return virq;
-}
-
-void mshv_irqchip_release_virq(int virq)
-{
-    remove_msi_routing(virq);
-}
-
-int mshv_irqchip_update_msi_route(int virq, MSIMessage msg, PCIDevice *dev)
-{
-    int ret;
-
-    ret = set_msi_routing(virq, msg.address, le32_to_cpu(msg.data));
-    if (ret < 0) {
-        error_report("Failed to set msi routing");
-        return -1;
-    }
-
-    return 0;
-}
-
-int mshv_request_interrupt(MshvState *mshv_state, uint32_t interrupt_type, uint32_t vector,
-                           uint32_t vp_index, bool logical_dest_mode,
-                           bool level_triggered)
-{
-    int ret;
-    int vm_fd = mshv_state->vm;
-
-    if (vector == 0) {
-        warn_report("Ignoring request for interrupt vector 0");
-        return 0;
-    }
-
-    union hv_interrupt_control control = {
-        .interrupt_type = interrupt_type,
-        .level_triggered = level_triggered,
-        .logical_dest_mode = logical_dest_mode,
-        .rsvd = 0,
-    };
-
-    struct hv_input_assert_virtual_interrupt arg = {0};
-    arg.control = control;
-    arg.dest_addr = (uint64_t)vp_index;
-    arg.vector = vector;
-
-    struct mshv_root_hvcall args = {0};
-    args.code   = HVCALL_ASSERT_VIRTUAL_INTERRUPT;
-    args.in_sz  = sizeof(arg);
-    args.in_ptr = (uint64_t)&arg;
-
-    ret = mshv_hvcall(vm_fd, &args);
-    if (ret < 0) {
-        error_report("Failed to request interrupt");
-        return -errno;
-    }
-    return 0;
-}
-
-void mshv_irqchip_commit_routes(void)
-{
-    int ret;
-    int vm_fd = mshv_state->vm;
-
-    ret = commit_msi_routing_table(vm_fd);
-    if (ret < 0) {
-        error_report("Failed to commit msi routing table");
-        abort();
-    }
-}
-
-int mshv_irqchip_add_irqfd_notifier_gsi(const EventNotifier *event,
-                                        const EventNotifier *resample,
-                                        int virq)
-{
-    return irqchip_update_irqfd_notifier_gsi(event, resample, virq, true);
-}
-
-int mshv_irqchip_remove_irqfd_notifier_gsi(const EventNotifier *event,
-                                           int virq)
-{
-    return irqchip_update_irqfd_notifier_gsi(event, NULL, virq, false);
-}
-
-int mshv_reserve_ioapic_msi_routes(int vm_fd)
-{
-    int ret, gsi;
-
-    /*
-     * Reserve GSI 0-23 for IOAPIC pins, to avoid conflicts of legacy
-     * peripherals with MSI-X devices
-     */
-    for (gsi = 0; gsi < IOAPIC_NUM_PINS; gsi++) {
-        ret = add_msi_routing(0, 0);
-        if (ret < 0) {
-            error_report("Failed to reserve GSI %d", gsi);
-            return -1;
-        }
-    }
-
-    ret = commit_msi_routing_table(vm_fd);
-    if (ret < 0) {
-        error_report("Failed to commit reserved IOAPIC MSI routes");
-        return -1;
-    }
-
-    return 0;
-}
@@ -1,563 +0,0 @@
-/*
- * QEMU MSHV support
- *
- * Copyright Microsoft, Corp. 2025
- *
- * Authors:
- *  Magnus Kulke      <magnuskulke@microsoft.com>
- *
- * SPDX-License-Identifier: GPL-2.0-or-later
- *
- */
-
-#include "qemu/osdep.h"
-#include "qemu/lockable.h"
-#include "qemu/error-report.h"
-#include "qemu/rcu.h"
-#include "linux/mshv.h"
-#include "system/address-spaces.h"
-#include "system/mshv.h"
-#include "system/mshv_int.h"
-#include "exec/memattrs.h"
-#include <sys/ioctl.h>
-#include "trace.h"
-
-typedef struct SlotsRCUReclaim {
-    struct rcu_head rcu;
-    GList *old_head;
-    MshvMemorySlot *removed_slot;
-} SlotsRCUReclaim;
-
-static void rcu_reclaim_slotlist(struct rcu_head *rcu)
-{
-    SlotsRCUReclaim *r = container_of(rcu, SlotsRCUReclaim, rcu);
-    g_list_free(r->old_head);
-    g_free(r->removed_slot);
-    g_free(r);
-}
-
-static void publish_slots(GList *new_head, GList *old_head,
-                          MshvMemorySlot *removed_slot)
-{
-    MshvMemorySlotManager *manager = &mshv_state->msm;
-
-    assert(manager);
-    qatomic_store_release(&manager->slots, new_head);
-
-    SlotsRCUReclaim *r = g_new(SlotsRCUReclaim, 1);
-    r->old_head = old_head;
-    r->removed_slot = removed_slot;
-
-    call_rcu1(&r->rcu, rcu_reclaim_slotlist);
-}
-
-/* Needs to be called with mshv_state->msm.mutex held */
-static int remove_slot(MshvMemorySlot *slot)
-{
-    GList *old_head, *new_head;
-    MshvMemorySlotManager *manager = &mshv_state->msm;
-
-    assert(manager);
-    old_head = qatomic_load_acquire(&manager->slots);
-
-    if (!g_list_find(old_head, slot)) {
-        error_report("slot requested for removal not found");
-        return -1;
-    }
-
-    new_head = g_list_copy(old_head);
-    new_head = g_list_remove(new_head, slot);
-    manager->n_slots--;
-
-    publish_slots(new_head, old_head, slot);
-
-    return 0;
-}
-
-/* Needs to be called with mshv_state->msm.mutex held */
-static MshvMemorySlot *append_slot(uint64_t gpa, uint64_t userspace_addr,
-                                   uint64_t size, bool readonly)
-{
-    GList *old_head, *new_head;
-    MshvMemorySlot *slot;
-    MshvMemorySlotManager *manager = &mshv_state->msm;
-
-    assert(manager);
-
-    old_head = qatomic_load_acquire(&manager->slots);
-
-    if (manager->n_slots >= MSHV_MAX_MEM_SLOTS) {
-        error_report("no free memory slots available");
-        return NULL;
-    }
-
-    slot = g_new0(MshvMemorySlot, 1);
-    slot->guest_phys_addr = gpa;
-    slot->userspace_addr = userspace_addr;
-    slot->memory_size = size;
-    slot->readonly = readonly;
-
-    new_head = g_list_copy(old_head);
-    new_head = g_list_append(new_head, slot);
-    manager->n_slots++;
-
-    publish_slots(new_head, old_head, NULL);
-
-    return slot;
-}
-
-static int slot_overlaps(const MshvMemorySlot *slot1,
-                         const MshvMemorySlot *slot2)
-{
-    uint64_t start_1 = slot1->userspace_addr,
-             start_2 = slot2->userspace_addr;
-    size_t len_1 = slot1->memory_size,
-           len_2 = slot2->memory_size;
-
-    if (slot1 == slot2) {
-        return -1;
-    }
-
-    return ranges_overlap(start_1, len_1, start_2, len_2) ?  0 : -1;
-}
-
-static bool is_mapped(MshvMemorySlot *slot)
-{
-    /* Subsequent reads of mapped field see a fully-initialized slot */
-    return qatomic_load_acquire(&slot->mapped);
-}
-
-/*
- * Find slot that is:
- * - overlapping in userspace
- * - currently mapped in the guest
- *
- * Needs to be called with mshv_state->msm.mutex or RCU read lock held.
- */
-static MshvMemorySlot *find_overlap_mem_slot(GList *head, MshvMemorySlot *slot)
-{
-    GList *found;
-    MshvMemorySlot *overlap_slot;
-
-    found = g_list_find_custom(head, slot, (GCompareFunc) slot_overlaps);
-
-    if (!found) {
-        return NULL;
-    }
-
-    overlap_slot = found->data;
-    if (!overlap_slot || !is_mapped(overlap_slot)) {
-        return NULL;
-    }
-
-    return overlap_slot;
-}
-
-static int set_guest_memory(int vm_fd,
-                            const struct mshv_user_mem_region *region)
-{
-    int ret;
-
-    ret = ioctl(vm_fd, MSHV_SET_GUEST_MEMORY, region);
-    if (ret < 0) {
-        error_report("failed to set guest memory: %s", strerror(errno));
-        return -1;
-    }
-
-    return 0;
-}
-
-static int map_or_unmap(int vm_fd, const MshvMemorySlot *slot, bool map)
-{
-    struct mshv_user_mem_region region = {0};
-
-    region.guest_pfn = slot->guest_phys_addr >> MSHV_PAGE_SHIFT;
-    region.size = slot->memory_size;
-    region.userspace_addr = slot->userspace_addr;
-
-    if (!map) {
-        region.flags |= (1 << MSHV_SET_MEM_BIT_UNMAP);
-        trace_mshv_unmap_memory(slot->userspace_addr, slot->guest_phys_addr,
-                                slot->memory_size);
-        return set_guest_memory(vm_fd, &region);
-    }
-
-    region.flags = BIT(MSHV_SET_MEM_BIT_EXECUTABLE);
-    if (!slot->readonly) {
-        region.flags |= BIT(MSHV_SET_MEM_BIT_WRITABLE);
-    }
-
-    trace_mshv_map_memory(slot->userspace_addr, slot->guest_phys_addr,
-                          slot->memory_size);
-    return set_guest_memory(vm_fd, &region);
-}
-
-static int slot_matches_region(const MshvMemorySlot *slot1,
-                               const MshvMemorySlot *slot2)
-{
-    return (slot1->guest_phys_addr == slot2->guest_phys_addr &&
-            slot1->userspace_addr  == slot2->userspace_addr &&
-            slot1->memory_size     == slot2->memory_size) ? 0 : -1;
-}
-
-/* Needs to be called with mshv_state->msm.mutex held */
-static MshvMemorySlot *find_mem_slot_by_region(uint64_t gpa, uint64_t size,
-                                               uint64_t userspace_addr)
-{
-    MshvMemorySlot ref_slot = {
-        .guest_phys_addr = gpa,
-        .userspace_addr  = userspace_addr,
-        .memory_size     = size,
-    };
-    GList *found;
-    MshvMemorySlotManager *manager = &mshv_state->msm;
-
-    assert(manager);
-    found = g_list_find_custom(manager->slots, &ref_slot,
-                               (GCompareFunc) slot_matches_region);
-
-    return found ? found->data : NULL;
-}
-
-static int slot_covers_gpa(const MshvMemorySlot *slot, uint64_t *gpa_p)
-{
-    uint64_t gpa_offset, gpa = *gpa_p;
-
-    gpa_offset = gpa - slot->guest_phys_addr;
-    return (slot->guest_phys_addr <= gpa && gpa_offset < slot->memory_size)
-        ? 0 : -1;
-}
-
-/* Needs to be called with mshv_state->msm.mutex or RCU read lock held */
-static MshvMemorySlot *find_mem_slot_by_gpa(GList *head, uint64_t gpa)
-{
-    GList *found;
-    MshvMemorySlot *slot;
-
-    trace_mshv_find_slot_by_gpa(gpa);
-
-    found = g_list_find_custom(head, &gpa, (GCompareFunc) slot_covers_gpa);
-    if (found) {
-        slot = found->data;
-        trace_mshv_found_slot(slot->userspace_addr, slot->guest_phys_addr,
-                              slot->memory_size);
-        return slot;
-    }
-
-    return NULL;
-}
-
-/* Needs to be called with mshv_state->msm.mutex held */
-static void set_mapped(MshvMemorySlot *slot, bool mapped)
-{
-    /* prior writes to mapped field becomes visible before readers see slot */
-    qatomic_store_release(&slot->mapped, mapped);
-}
-
-MshvRemapResult mshv_remap_overlap_region(int vm_fd, uint64_t gpa)
-{
-    MshvMemorySlot *gpa_slot, *overlap_slot;
-    GList *head;
-    int ret;
-    MshvMemorySlotManager *manager = &mshv_state->msm;
-
-    /* fast path, called often by unmapped_gpa vm exit */
-    WITH_RCU_READ_LOCK_GUARD() {
-        assert(manager);
-        head = qatomic_load_acquire(&manager->slots);
-        /* return early if no slot is found */
-        gpa_slot = find_mem_slot_by_gpa(head, gpa);
-        if (gpa_slot == NULL) {
-            return MshvRemapNoMapping;
-        }
-
-        /* return early if no overlapping slot is found */
-        overlap_slot = find_overlap_mem_slot(head, gpa_slot);
-        if (overlap_slot == NULL) {
-            return MshvRemapNoOverlap;
-        }
-    }
-
-    /*
-     * We'll modify the mapping list, so we need to upgrade to mutex and
-     * recheck.
-     */
-    assert(manager);
-    QEMU_LOCK_GUARD(&manager->mutex);
-
-    /* return early if no slot is found */
-    gpa_slot = find_mem_slot_by_gpa(manager->slots, gpa);
-    if (gpa_slot == NULL) {
-        return MshvRemapNoMapping;
-    }
-
-    /* return early if no overlapping slot is found */
-    overlap_slot = find_overlap_mem_slot(manager->slots, gpa_slot);
-    if (overlap_slot == NULL) {
-        return MshvRemapNoOverlap;
-    }
-
-    /* unmap overlapping slot */
-    ret = map_or_unmap(vm_fd, overlap_slot, false);
-    if (ret < 0) {
-        error_report("failed to unmap overlap region");
-        abort();
-    }
-    set_mapped(overlap_slot, false);
-    warn_report("mapped out userspace_addr=0x%016lx gpa=0x%010lx size=0x%lx",
-                overlap_slot->userspace_addr,
-                overlap_slot->guest_phys_addr,
-                overlap_slot->memory_size);
-
-    /* map region for gpa */
-    ret = map_or_unmap(vm_fd, gpa_slot, true);
-    if (ret < 0) {
-        error_report("failed to map new region");
-        abort();
-    }
-    set_mapped(gpa_slot, true);
-    warn_report("mapped in  userspace_addr=0x%016lx gpa=0x%010lx size=0x%lx",
-                gpa_slot->userspace_addr, gpa_slot->guest_phys_addr,
-                gpa_slot->memory_size);
-
-    return MshvRemapOk;
-}
-
-static int handle_unmapped_mmio_region_read(uint64_t gpa, uint64_t size,
-                                            uint8_t *data)
-{
-    warn_report("read from unmapped mmio region gpa=0x%lx size=%lu", gpa, size);
-
-    if (size == 0 || size > 8) {
-        error_report("invalid size %lu for reading from unmapped mmio region",
-                     size);
-        return -1;
-    }
-
-    memset(data, 0xFF, size);
-
-    return 0;
-}
-
-int mshv_guest_mem_read(uint64_t gpa, uint8_t *data, uintptr_t size,
-                        bool is_secure_mode, bool instruction_fetch)
-{
-    int ret;
-    MemTxAttrs memattr = { .secure = is_secure_mode };
-
-    if (instruction_fetch) {
-        trace_mshv_insn_fetch(gpa, size);
-    } else {
-        trace_mshv_mem_read(gpa, size);
-    }
-
-    ret = address_space_rw(&address_space_memory, gpa, memattr, (void *)data,
-                           size, false);
-    if (ret == MEMTX_OK) {
-        return 0;
-    }
-
-    if (ret == MEMTX_DECODE_ERROR) {
-        return handle_unmapped_mmio_region_read(gpa, size, data);
-    }
-
-    error_report("failed to read guest memory at 0x%lx", gpa);
-    return -1;
-}
-
-int mshv_guest_mem_write(uint64_t gpa, const uint8_t *data, uintptr_t size,
-                         bool is_secure_mode)
-{
-    int ret;
-    MemTxAttrs memattr = { .secure = is_secure_mode };
-
-    trace_mshv_mem_write(gpa, size);
-    ret = address_space_rw(&address_space_memory, gpa, memattr, (void *)data,
-                           size, true);
-    if (ret == MEMTX_OK) {
-        return 0;
-    }
-
-    if (ret == MEMTX_DECODE_ERROR) {
-        warn_report("write to unmapped mmio region gpa=0x%lx size=%lu", gpa,
-                    size);
-        return 0;
-    }
-
-    error_report("Failed to write guest memory");
-    return -1;
-}
-
-static int tracked_unmap(int vm_fd, uint64_t gpa, uint64_t size,
-                        uint64_t userspace_addr)
-{
-    int ret;
-    MshvMemorySlot *slot;
-    MshvMemorySlotManager *manager = &mshv_state->msm;
-
-    assert(manager);
-
-    QEMU_LOCK_GUARD(&manager->mutex);
-
-    slot = find_mem_slot_by_region(gpa, size, userspace_addr);
-    if (!slot) {
-        trace_mshv_skip_unset_mem(userspace_addr, gpa, size);
-        /* no work to do */
-        return 0;
-    }
-
-    if (!is_mapped(slot)) {
-        /* remove slot, no need to unmap */
-        return remove_slot(slot);
-    }
-
-    ret = map_or_unmap(vm_fd, slot, false);
-    if (ret < 0) {
-        error_report("failed to unmap memory region");
-        return ret;
-    }
-    return remove_slot(slot);
-}
-
-static int tracked_map(int vm_fd, uint64_t gpa, uint64_t size, bool readonly,
-                       uint64_t userspace_addr)
-{
-    MshvMemorySlot *slot, *overlap_slot;
-    int ret;
-    MshvMemorySlotManager *manager = &mshv_state->msm;
-
-    assert(manager);
-
-    QEMU_LOCK_GUARD(&manager->mutex);
-
-    slot = find_mem_slot_by_region(gpa, size, userspace_addr);
-    if (slot) {
-        error_report("memory region already mapped at gpa=0x%lx, "
-                     "userspace_addr=0x%lx, size=0x%lx",
-                     slot->guest_phys_addr, slot->userspace_addr,
-                     slot->memory_size);
-        return -1;
-    }
-
-    slot = append_slot(gpa, userspace_addr, size, readonly);
-
-    overlap_slot = find_overlap_mem_slot(manager->slots, slot);
-    if (overlap_slot) {
-        trace_mshv_remap_attempt(slot->userspace_addr,
-                                 slot->guest_phys_addr,
-                                 slot->memory_size);
-        warn_report("attempt to map region [0x%lx-0x%lx], while "
-                    "[0x%lx-0x%lx] is already mapped in the guest",
-                    userspace_addr, userspace_addr + size - 1,
-                    overlap_slot->userspace_addr,
-                    overlap_slot->userspace_addr +
-                    overlap_slot->memory_size - 1);
-
-        /* do not register mem slot in hv, but record for later swap-in */
-        set_mapped(slot, false);
-
-        return 0;
-    }
-
-    ret = map_or_unmap(vm_fd, slot, true);
-    if (ret < 0) {
-        error_report("failed to map memory region");
-        return -1;
-    }
-    set_mapped(slot, true);
-
-    return 0;
-}
-
-static int set_memory(uint64_t gpa, uint64_t size, bool readonly,
-                      uint64_t userspace_addr, bool add)
-{
-    int vm_fd = mshv_state->vm;
-
-    if (add) {
-        return tracked_map(vm_fd, gpa, size, readonly, userspace_addr);
-    }
-
-    return tracked_unmap(vm_fd, gpa, size, userspace_addr);
-}
-
-/*
- * Calculate and align the start address and the size of the section.
- * Return the size. If the size is 0, the aligned section is empty.
- */
-static hwaddr align_section(MemoryRegionSection *section, hwaddr *start)
-{
-    hwaddr size = int128_get64(section->size);
-    hwaddr delta, aligned;
-
-    /*
-     * works in page size chunks, but the function may be called
-     * with sub-page size and unaligned start address. Pad the start
-     * address to next and truncate size to previous page boundary.
-     */
-    aligned = ROUND_UP(section->offset_within_address_space,
-                       qemu_real_host_page_size());
-    delta = aligned - section->offset_within_address_space;
-    *start = aligned;
-    if (delta > size) {
-        return 0;
-    }
-
-    return (size - delta) & qemu_real_host_page_mask();
-}
-
-void mshv_set_phys_mem(MshvMemoryListener *mml, MemoryRegionSection *section,
-                       bool add)
-{
-    int ret = 0;
-    MemoryRegion *area = section->mr;
-    bool writable = !area->readonly && !area->rom_device;
-    hwaddr start_addr, mr_offset, size;
-    void *ram;
-
-    size = align_section(section, &start_addr);
-    trace_mshv_set_phys_mem(add, section->mr->name, start_addr);
-
-    size = align_section(section, &start_addr);
-    trace_mshv_set_phys_mem(add, section->mr->name, start_addr);
-
-    /*
-     * If the memory device is a writable non-ram area, we do not
-     * want to map it into the guest memory. If it is not a ROM device,
-     * we want to remove mshv memory mapping, so accesses will trap.
-     */
-    if (!memory_region_is_ram(area)) {
-        if (writable) {
-            return;
-        } else if (!area->romd_mode) {
-            add = false;
-        }
-    }
-
-    if (!size) {
-        return;
-    }
-
-    mr_offset = section->offset_within_region + start_addr -
-                section->offset_within_address_space;
-
-    ram = memory_region_get_ram_ptr(area) + mr_offset;
-
-    ret = set_memory(start_addr, size, !writable, (uint64_t)ram, add);
-    if (ret < 0) {
-        error_report("failed to set memory region");
-        abort();
-    }
-}
-
-void mshv_init_memory_slot_manager(MshvState *mshv_state)
-{
-    MshvMemorySlotManager *manager;
-
-    assert(mshv_state);
-    manager = &mshv_state->msm;
-
-    manager->n_slots = 0;
-    manager->slots = NULL;
-    qemu_mutex_init(&manager->mutex);
-}
@@ -1,9 +0,0 @@
-mshv_ss = ss.source_set()
-mshv_ss.add(if_true: files(
-  'irq.c',
-  'mem.c',
-  'msr.c',
-  'mshv-all.c'
-))
-
-specific_ss.add_all(when: 'CONFIG_MSHV', if_true: mshv_ss)
@@ -1,730 +0,0 @@
-/*
- * QEMU MSHV support
- *
- * Copyright Microsoft, Corp. 2025
- *
- * Authors:
- *  Ziqiao Zhou       <ziqiaozhou@microsoft.com>
- *  Magnus Kulke      <magnuskulke@microsoft.com>
- *  Jinank Jain       <jinankjain@microsoft.com>
- *  Wei Liu           <liuwe@microsoft.com>
- *
- * SPDX-License-Identifier: GPL-2.0-or-later
- *
- */
-
-#include "qemu/osdep.h"
-#include "qapi/error.h"
-#include "qemu/error-report.h"
-#include "qemu/event_notifier.h"
-#include "qemu/module.h"
-#include "qemu/main-loop.h"
-#include "hw/boards.h"
-
-#include "hw/hyperv/hvhdk.h"
-#include "hw/hyperv/hvhdk_mini.h"
-#include "hw/hyperv/hvgdk.h"
-#include "hw/hyperv/hvgdk_mini.h"
-#include "linux/mshv.h"
-
-#include "qemu/accel.h"
-#include "qemu/guest-random.h"
-#include "accel/accel-ops.h"
-#include "accel/accel-cpu-ops.h"
-#include "system/cpus.h"
-#include "system/runstate.h"
-#include "system/accel-blocker.h"
-#include "system/address-spaces.h"
-#include "system/mshv.h"
-#include "system/mshv_int.h"
-#include "system/reset.h"
-#include "trace.h"
-#include <err.h>
-#include <stdint.h>
-#include <sys/ioctl.h>
-
-#define TYPE_MSHV_ACCEL ACCEL_CLASS_NAME("mshv")
-
-DECLARE_INSTANCE_CHECKER(MshvState, MSHV_STATE, TYPE_MSHV_ACCEL)
-
-bool mshv_allowed;
-
-MshvState *mshv_state;
-
-static int init_mshv(int *mshv_fd)
-{
-    int fd = open("/dev/mshv", O_RDWR | O_CLOEXEC);
-    if (fd < 0) {
-        error_report("Failed to open /dev/mshv: %s", strerror(errno));
-        return -1;
-    }
-    *mshv_fd = fd;
-    return 0;
-}
-
-/* freeze 1 to pause, 0 to resume */
-static int set_time_freeze(int vm_fd, int freeze)
-{
-    int ret;
-    struct hv_input_set_partition_property in = {0};
-    in.property_code = HV_PARTITION_PROPERTY_TIME_FREEZE;
-    in.property_value = freeze;
-
-    struct mshv_root_hvcall args = {0};
-    args.code = HVCALL_SET_PARTITION_PROPERTY;
-    args.in_sz = sizeof(in);
-    args.in_ptr = (uint64_t)&in;
-
-    ret = mshv_hvcall(vm_fd, &args);
-    if (ret < 0) {
-        error_report("Failed to set time freeze");
-        return -1;
-    }
-
-    return 0;
-}
-
-static int pause_vm(int vm_fd)
-{
-    int ret;
-
-    ret = set_time_freeze(vm_fd, 1);
-    if (ret < 0) {
-        error_report("Failed to pause partition: %s", strerror(errno));
-        return -1;
-    }
-
-    return 0;
-}
-
-static int resume_vm(int vm_fd)
-{
-    int ret;
-
-    ret = set_time_freeze(vm_fd, 0);
-    if (ret < 0) {
-        error_report("Failed to resume partition: %s", strerror(errno));
-        return -1;
-    }
-
-    return 0;
-}
-
-static int create_partition(int mshv_fd, int *vm_fd)
-{
-    int ret;
-    struct mshv_create_partition args = {0};
-
-    /* Initialize pt_flags with the desired features */
-    uint64_t pt_flags = (1ULL << MSHV_PT_BIT_LAPIC) |
-                        (1ULL << MSHV_PT_BIT_X2APIC) |
-                        (1ULL << MSHV_PT_BIT_GPA_SUPER_PAGES);
-
-    /* Set default isolation type */
-    uint64_t pt_isolation = MSHV_PT_ISOLATION_NONE;
-
-    args.pt_flags = pt_flags;
-    args.pt_isolation = pt_isolation;
-
-    ret = ioctl(mshv_fd, MSHV_CREATE_PARTITION, &args);
-    if (ret < 0) {
-        error_report("Failed to create partition: %s", strerror(errno));
-        return -1;
-    }
-
-    *vm_fd = ret;
-    return 0;
-}
-
-static int set_synthetic_proc_features(int vm_fd)
-{
-    int ret;
-    struct hv_input_set_partition_property in = {0};
-    union hv_partition_synthetic_processor_features features = {0};
-
-    /* Access the bitfield and set the desired features */
-    features.hypervisor_present = 1;
-    features.hv1 = 1;
-    features.access_partition_reference_counter = 1;
-    features.access_synic_regs = 1;
-    features.access_synthetic_timer_regs = 1;
-    features.access_partition_reference_tsc = 1;
-    features.access_frequency_regs = 1;
-    features.access_intr_ctrl_regs = 1;
-    features.access_vp_index = 1;
-    features.access_hypercall_regs = 1;
-    features.tb_flush_hypercalls = 1;
-    features.synthetic_cluster_ipi = 1;
-    features.direct_synthetic_timers = 1;
-
-    mshv_arch_amend_proc_features(&features);
-
-    in.property_code = HV_PARTITION_PROPERTY_SYNTHETIC_PROC_FEATURES;
-    in.property_value = features.as_uint64[0];
-
-    struct mshv_root_hvcall args = {0};
-    args.code = HVCALL_SET_PARTITION_PROPERTY;
-    args.in_sz = sizeof(in);
-    args.in_ptr = (uint64_t)&in;
-
-    trace_mshv_hvcall_args("synthetic_proc_features", args.code, args.in_sz);
-
-    ret = mshv_hvcall(vm_fd, &args);
-    if (ret < 0) {
-        error_report("Failed to set synthethic proc features");
-        return -errno;
-    }
-    return 0;
-}
-
-static int initialize_vm(int vm_fd)
-{
-    int ret = ioctl(vm_fd, MSHV_INITIALIZE_PARTITION);
-    if (ret < 0) {
-        error_report("Failed to initialize partition: %s", strerror(errno));
-        return -1;
-    }
-    return 0;
-}
-
-static int create_vm(int mshv_fd, int *vm_fd)
-{
-    int ret = create_partition(mshv_fd, vm_fd);
-    if (ret < 0) {
-        return -1;
-    }
-
-    ret = set_synthetic_proc_features(*vm_fd);
-    if (ret < 0) {
-        return -1;
-    }
-
-    ret = initialize_vm(*vm_fd);
-    if (ret < 0) {
-        return -1;
-    }
-
-    ret = mshv_reserve_ioapic_msi_routes(*vm_fd);
-    if (ret < 0) {
-        return -1;
-    }
-
-    ret = mshv_arch_post_init_vm(*vm_fd);
-    if (ret < 0) {
-        return -1;
-    }
-
-    /* Always create a frozen partition */
-    pause_vm(*vm_fd);
-
-    return 0;
-}
-
-static void mem_region_add(MemoryListener *listener,
-                           MemoryRegionSection *section)
-{
-    MshvMemoryListener *mml;
-    mml = container_of(listener, MshvMemoryListener, listener);
-    memory_region_ref(section->mr);
-    mshv_set_phys_mem(mml, section, true);
-}
-
-static void mem_region_del(MemoryListener *listener,
-                           MemoryRegionSection *section)
-{
-    MshvMemoryListener *mml;
-    mml = container_of(listener, MshvMemoryListener, listener);
-    mshv_set_phys_mem(mml, section, false);
-    memory_region_unref(section->mr);
-}
-
-typedef enum {
-    DATAMATCH_NONE,
-    DATAMATCH_U32,
-    DATAMATCH_U64,
-} DatamatchTag;
-
-typedef struct {
-    DatamatchTag tag;
-    union {
-        uint32_t u32;
-        uint64_t u64;
-    } value;
-} Datamatch;
-
-/* flags: determine whether to de/assign */
-static int ioeventfd(int vm_fd, int event_fd, uint64_t addr, Datamatch dm,
-                     uint32_t flags)
-{
-    struct mshv_user_ioeventfd args = {0};
-    args.fd = event_fd;
-    args.addr = addr;
-    args.flags = flags;
-
-    if (dm.tag == DATAMATCH_NONE) {
-        args.datamatch = 0;
-    } else {
-        flags |= BIT(MSHV_IOEVENTFD_BIT_DATAMATCH);
-        args.flags = flags;
-        if (dm.tag == DATAMATCH_U64) {
-            args.len = sizeof(uint64_t);
-            args.datamatch = dm.value.u64;
-        } else {
-            args.len = sizeof(uint32_t);
-            args.datamatch = dm.value.u32;
-        }
-    }
-
-    return ioctl(vm_fd, MSHV_IOEVENTFD, &args);
-}
-
-static int unregister_ioevent(int vm_fd, int event_fd, uint64_t mmio_addr)
-{
-    uint32_t flags = 0;
-    Datamatch dm = {0};
-
-    flags |= BIT(MSHV_IOEVENTFD_BIT_DEASSIGN);
-    dm.tag = DATAMATCH_NONE;
-
-    return ioeventfd(vm_fd, event_fd, mmio_addr, dm, flags);
-}
-
-static int register_ioevent(int vm_fd, int event_fd, uint64_t mmio_addr,
-                            uint64_t val, bool is_64bit, bool is_datamatch)
-{
-    uint32_t flags = 0;
-    Datamatch dm = {0};
-
-    if (!is_datamatch) {
-        dm.tag = DATAMATCH_NONE;
-    } else if (is_64bit) {
-        dm.tag = DATAMATCH_U64;
-        dm.value.u64 = val;
-    } else {
-        dm.tag = DATAMATCH_U32;
-        dm.value.u32 = val;
-    }
-
-    return ioeventfd(vm_fd, event_fd, mmio_addr, dm, flags);
-}
-
-static void mem_ioeventfd_add(MemoryListener *listener,
-                              MemoryRegionSection *section,
-                              bool match_data, uint64_t data,
-                              EventNotifier *e)
-{
-    int fd = event_notifier_get_fd(e);
-    int ret;
-    bool is_64 = int128_get64(section->size) == 8;
-    uint64_t addr = section->offset_within_address_space;
-
-    trace_mshv_mem_ioeventfd_add(addr, int128_get64(section->size), data);
-
-    ret = register_ioevent(mshv_state->vm, fd, addr, data, is_64, match_data);
-
-    if (ret < 0) {
-        error_report("Failed to register ioeventfd: %s (%d)", strerror(-ret),
-                     -ret);
-        abort();
-    }
-}
-
-static void mem_ioeventfd_del(MemoryListener *listener,
-                              MemoryRegionSection *section,
-                              bool match_data, uint64_t data,
-                              EventNotifier *e)
-{
-    int fd = event_notifier_get_fd(e);
-    int ret;
-    uint64_t addr = section->offset_within_address_space;
-
-    trace_mshv_mem_ioeventfd_del(section->offset_within_address_space,
-                                 int128_get64(section->size), data);
-
-    ret = unregister_ioevent(mshv_state->vm, fd, addr);
-    if (ret < 0) {
-        error_report("Failed to unregister ioeventfd: %s (%d)", strerror(-ret),
-                     -ret);
-        abort();
-    }
-}
-
-static MemoryListener mshv_memory_listener = {
-    .name = "mshv",
-    .priority = MEMORY_LISTENER_PRIORITY_ACCEL,
-    .region_add = mem_region_add,
-    .region_del = mem_region_del,
-    .eventfd_add = mem_ioeventfd_add,
-    .eventfd_del = mem_ioeventfd_del,
-};
-
-static MemoryListener mshv_io_listener = {
-    .name = "mshv", .priority = MEMORY_LISTENER_PRIORITY_DEV_BACKEND,
-    /* MSHV does not support PIO eventfd */
-};
-
-static void register_mshv_memory_listener(MshvState *s, MshvMemoryListener *mml,
-                                          AddressSpace *as, int as_id,
-                                          const char *name)
-{
-    int i;
-
-    mml->listener = mshv_memory_listener;
-    mml->listener.name = name;
-    memory_listener_register(&mml->listener, as);
-    for (i = 0; i < s->nr_as; ++i) {
-        if (!s->as[i].as) {
-            s->as[i].as = as;
-            s->as[i].ml = mml;
-            break;
-        }
-    }
-}
-
-int mshv_hvcall(int fd, const struct mshv_root_hvcall *args)
-{
-    int ret = 0;
-
-    ret = ioctl(fd, MSHV_ROOT_HVCALL, args);
-    if (ret < 0) {
-        error_report("Failed to perform hvcall: %s", strerror(errno));
-        return -1;
-    }
-    return ret;
-}
-
-static int mshv_init_vcpu(CPUState *cpu)
-{
-    int vm_fd = mshv_state->vm;
-    uint8_t vp_index = cpu->cpu_index;
-    int ret;
-
-    cpu->accel = g_new0(AccelCPUState, 1);
-    mshv_arch_init_vcpu(cpu);
-
-    ret = mshv_create_vcpu(vm_fd, vp_index, &cpu->accel->cpufd);
-    if (ret < 0) {
-        return -1;
-    }
-
-    cpu->accel->dirty = true;
-
-    return 0;
-}
-
-static int mshv_init(AccelState *as, MachineState *ms)
-{
-    MshvState *s;
-    int mshv_fd, vm_fd, ret;
-
-    if (mshv_state) {
-        warn_report("MSHV accelerator already initialized");
-        return 0;
-    }
-
-    s = MSHV_STATE(as);
-
-    accel_blocker_init();
-
-    s->vm = 0;
-
-    ret = init_mshv(&mshv_fd);
-    if (ret < 0) {
-        return -1;
-    }
-
-    mshv_init_mmio_emu();
-
-    mshv_init_msicontrol();
-
-    mshv_init_memory_slot_manager(s);
-
-    ret = create_vm(mshv_fd, &vm_fd);
-    if (ret < 0) {
-        close(mshv_fd);
-        return -1;
-    }
-
-    ret = resume_vm(vm_fd);
-    if (ret < 0) {
-        close(mshv_fd);
-        close(vm_fd);
-        return -1;
-    }
-
-    s->vm = vm_fd;
-    s->fd = mshv_fd;
-    s->nr_as = 1;
-    s->as = g_new0(MshvAddressSpace, s->nr_as);
-
-    mshv_state = s;
-
-    register_mshv_memory_listener(s, &s->memory_listener, &address_space_memory,
-                                  0, "mshv-memory");
-    memory_listener_register(&mshv_io_listener, &address_space_io);
-
-    return 0;
-}
-
-static int mshv_destroy_vcpu(CPUState *cpu)
-{
-    int cpu_fd = mshv_vcpufd(cpu);
-    int vm_fd = mshv_state->vm;
-
-    mshv_remove_vcpu(vm_fd, cpu_fd);
-    mshv_vcpufd(cpu) = 0;
-
-    mshv_arch_destroy_vcpu(cpu);
-    g_clear_pointer(&cpu->accel, g_free);
-    return 0;
-}
-
-static int mshv_cpu_exec(CPUState *cpu)
-{
-    hv_message mshv_msg;
-    enum MshvVmExit exit_reason;
-    int ret = 0;
-
-    bql_unlock();
-    cpu_exec_start(cpu);
-
-    do {
-        if (cpu->accel->dirty) {
-            ret = mshv_arch_put_registers(cpu);
-            if (ret) {
-                error_report("Failed to put registers after init: %s",
-                              strerror(-ret));
-                ret = -1;
-                break;
-            }
-            cpu->accel->dirty = false;
-        }
-
-        ret = mshv_run_vcpu(mshv_state->vm, cpu, &mshv_msg, &exit_reason);
-        if (ret < 0) {
-            error_report("Failed to run on vcpu %d", cpu->cpu_index);
-            abort();
-        }
-
-        switch (exit_reason) {
-        case MshvVmExitIgnore:
-            break;
-        default:
-            ret = EXCP_INTERRUPT;
-            break;
-        }
-    } while (ret == 0);
-
-    cpu_exec_end(cpu);
-    bql_lock();
-
-    if (ret < 0) {
-        cpu_dump_state(cpu, stderr, CPU_DUMP_CODE);
-        vm_stop(RUN_STATE_INTERNAL_ERROR);
-    }
-
-    return ret;
-}
-
-/*
- * The signal handler is triggered when QEMU's main thread receives a SIG_IPI
- * (SIGUSR1). This signal causes the current CPU thread to be kicked, forcing a
- * VM exit on the CPU. The VM exit generates an exit reason that breaks the loop
- * (see mshv_cpu_exec). If the exit is due to a Ctrl+A+x command, the system
- * will shut down. For other cases, the system will continue running.
- */
-static void sa_ipi_handler(int sig)
-{
-    /* TODO: call IOCTL to set_immediate_exit, once implemented. */
-
-    qemu_cpu_kick_self();
-}
-
-static void init_signal(CPUState *cpu)
-{
-    /* init cpu signals */
-    struct sigaction sigact;
-    sigset_t set;
-
-    memset(&sigact, 0, sizeof(sigact));
-    sigact.sa_handler = sa_ipi_handler;
-    sigaction(SIG_IPI, &sigact, NULL);
-
-    pthread_sigmask(SIG_BLOCK, NULL, &set);
-    sigdelset(&set, SIG_IPI);
-    pthread_sigmask(SIG_SETMASK, &set, NULL);
-}
-
-static void *mshv_vcpu_thread(void *arg)
-{
-    CPUState *cpu = arg;
-    int ret;
-
-    rcu_register_thread();
-
-    bql_lock();
-    qemu_thread_get_self(cpu->thread);
-    cpu->thread_id = qemu_get_thread_id();
-    current_cpu = cpu;
-    ret = mshv_init_vcpu(cpu);
-    if (ret < 0) {
-        error_report("Failed to init vcpu %d", cpu->cpu_index);
-        goto cleanup;
-    }
-    init_signal(cpu);
-
-    /* signal CPU creation */
-    cpu_thread_signal_created(cpu);
-    qemu_guest_random_seed_thread_part2(cpu->random_seed);
-
-    do {
-        qemu_process_cpu_events(cpu);
-        if (cpu_can_run(cpu)) {
-            mshv_cpu_exec(cpu);
-        }
-    } while (!cpu->unplug || cpu_can_run(cpu));
-
-    mshv_destroy_vcpu(cpu);
-cleanup:
-    cpu_thread_signal_destroyed(cpu);
-    bql_unlock();
-    rcu_unregister_thread();
-    return NULL;
-}
-
-static void mshv_start_vcpu_thread(CPUState *cpu)
-{
-    char thread_name[VCPU_THREAD_NAME_SIZE];
-
-    snprintf(thread_name, VCPU_THREAD_NAME_SIZE, "CPU %d/MSHV",
-             cpu->cpu_index);
-
-    cpu->thread = g_malloc0(sizeof(QemuThread));
-    cpu->halt_cond = g_malloc0(sizeof(QemuCond));
-
-    qemu_cond_init(cpu->halt_cond);
-
-    trace_mshv_start_vcpu_thread(thread_name, cpu->cpu_index);
-    qemu_thread_create(cpu->thread, thread_name, mshv_vcpu_thread, cpu,
-                       QEMU_THREAD_JOINABLE);
-}
-
-static void do_mshv_cpu_synchronize_post_init(CPUState *cpu,
-                                              run_on_cpu_data arg)
-{
-    int ret = mshv_arch_put_registers(cpu);
-    if (ret < 0) {
-        error_report("Failed to put registers after init: %s", strerror(-ret));
-        abort();
-    }
-
-    cpu->accel->dirty = false;
-}
-
-static void mshv_cpu_synchronize_post_init(CPUState *cpu)
-{
-    run_on_cpu(cpu, do_mshv_cpu_synchronize_post_init, RUN_ON_CPU_NULL);
-}
-
-static void mshv_cpu_synchronize_post_reset(CPUState *cpu)
-{
-    int ret = mshv_arch_put_registers(cpu);
-    if (ret) {
-        error_report("Failed to put registers after reset: %s",
-                     strerror(-ret));
-        cpu_dump_state(cpu, stderr, CPU_DUMP_CODE);
-        vm_stop(RUN_STATE_INTERNAL_ERROR);
-    }
-    cpu->accel->dirty = false;
-}
-
-static void do_mshv_cpu_synchronize_pre_loadvm(CPUState *cpu,
-                                               run_on_cpu_data arg)
-{
-    cpu->accel->dirty = true;
-}
-
-static void mshv_cpu_synchronize_pre_loadvm(CPUState *cpu)
-{
-    run_on_cpu(cpu, do_mshv_cpu_synchronize_pre_loadvm, RUN_ON_CPU_NULL);
-}
-
-static void do_mshv_cpu_synchronize(CPUState *cpu, run_on_cpu_data arg)
-{
-    if (!cpu->accel->dirty) {
-        int ret = mshv_load_regs(cpu);
-        if (ret < 0) {
-            error_report("Failed to load registers for vcpu %d",
-                         cpu->cpu_index);
-
-            cpu_dump_state(cpu, stderr, CPU_DUMP_CODE);
-            vm_stop(RUN_STATE_INTERNAL_ERROR);
-        }
-
-        cpu->accel->dirty = true;
-    }
-}
-
-static void mshv_cpu_synchronize(CPUState *cpu)
-{
-    if (!cpu->accel->dirty) {
-        run_on_cpu(cpu, do_mshv_cpu_synchronize, RUN_ON_CPU_NULL);
-    }
-}
-
-static bool mshv_cpus_are_resettable(void)
-{
-    return false;
-}
-
-static void mshv_accel_class_init(ObjectClass *oc, const void *data)
-{
-    AccelClass *ac = ACCEL_CLASS(oc);
-
-    ac->name = "MSHV";
-    ac->init_machine = mshv_init;
-    ac->allowed = &mshv_allowed;
-}
-
-static void mshv_accel_instance_init(Object *obj)
-{
-    MshvState *s = MSHV_STATE(obj);
-
-    s->vm = 0;
-}
-
-static const TypeInfo mshv_accel_type = {
-    .name = TYPE_MSHV_ACCEL,
-    .parent = TYPE_ACCEL,
-    .instance_init = mshv_accel_instance_init,
-    .class_init = mshv_accel_class_init,
-    .instance_size = sizeof(MshvState),
-};
-
-static void mshv_accel_ops_class_init(ObjectClass *oc, const void *data)
-{
-    AccelOpsClass *ops = ACCEL_OPS_CLASS(oc);
-
-    ops->create_vcpu_thread = mshv_start_vcpu_thread;
-    ops->synchronize_post_init = mshv_cpu_synchronize_post_init;
-    ops->synchronize_post_reset = mshv_cpu_synchronize_post_reset;
-    ops->synchronize_state = mshv_cpu_synchronize;
-    ops->synchronize_pre_loadvm = mshv_cpu_synchronize_pre_loadvm;
-    ops->cpus_are_resettable = mshv_cpus_are_resettable;
-    ops->handle_interrupt = generic_handle_interrupt;
-}
-
-static const TypeInfo mshv_accel_ops_type = {
-    .name = ACCEL_OPS_NAME("mshv"),
-    .parent = TYPE_ACCEL_OPS,
-    .class_init = mshv_accel_ops_class_init,
-    .abstract = true,
-};
-
-static void mshv_type_init(void)
-{
-    type_register_static(&mshv_accel_type);
-    type_register_static(&mshv_accel_ops_type);
-}
-
-type_init(mshv_type_init);
@@ -1,375 +0,0 @@
-/*
- * QEMU MSHV support
- *
- * Copyright Microsoft, Corp. 2025
- *
- * Authors: Magnus Kulke  <magnuskulke@microsoft.com>
- *
- * SPDX-License-Identifier: GPL-2.0-or-later
- */
-
-#include "qemu/osdep.h"
-#include "system/mshv.h"
-#include "system/mshv_int.h"
-#include "hw/hyperv/hvgdk_mini.h"
-#include "linux/mshv.h"
-#include "qemu/error-report.h"
-
-static uint32_t supported_msrs[64] = {
-    IA32_MSR_TSC,
-    IA32_MSR_EFER,
-    IA32_MSR_KERNEL_GS_BASE,
-    IA32_MSR_APIC_BASE,
-    IA32_MSR_PAT,
-    IA32_MSR_SYSENTER_CS,
-    IA32_MSR_SYSENTER_ESP,
-    IA32_MSR_SYSENTER_EIP,
-    IA32_MSR_STAR,
-    IA32_MSR_LSTAR,
-    IA32_MSR_CSTAR,
-    IA32_MSR_SFMASK,
-    IA32_MSR_MTRR_DEF_TYPE,
-    IA32_MSR_MTRR_PHYSBASE0,
-    IA32_MSR_MTRR_PHYSMASK0,
-    IA32_MSR_MTRR_PHYSBASE1,
-    IA32_MSR_MTRR_PHYSMASK1,
-    IA32_MSR_MTRR_PHYSBASE2,
-    IA32_MSR_MTRR_PHYSMASK2,
-    IA32_MSR_MTRR_PHYSBASE3,
-    IA32_MSR_MTRR_PHYSMASK3,
-    IA32_MSR_MTRR_PHYSBASE4,
-    IA32_MSR_MTRR_PHYSMASK4,
-    IA32_MSR_MTRR_PHYSBASE5,
-    IA32_MSR_MTRR_PHYSMASK5,
-    IA32_MSR_MTRR_PHYSBASE6,
-    IA32_MSR_MTRR_PHYSMASK6,
-    IA32_MSR_MTRR_PHYSBASE7,
-    IA32_MSR_MTRR_PHYSMASK7,
-    IA32_MSR_MTRR_FIX64K_00000,
-    IA32_MSR_MTRR_FIX16K_80000,
-    IA32_MSR_MTRR_FIX16K_A0000,
-    IA32_MSR_MTRR_FIX4K_C0000,
-    IA32_MSR_MTRR_FIX4K_C8000,
-    IA32_MSR_MTRR_FIX4K_D0000,
-    IA32_MSR_MTRR_FIX4K_D8000,
-    IA32_MSR_MTRR_FIX4K_E0000,
-    IA32_MSR_MTRR_FIX4K_E8000,
-    IA32_MSR_MTRR_FIX4K_F0000,
-    IA32_MSR_MTRR_FIX4K_F8000,
-    IA32_MSR_TSC_AUX,
-    IA32_MSR_DEBUG_CTL,
-    HV_X64_MSR_GUEST_OS_ID,
-    HV_X64_MSR_SINT0,
-    HV_X64_MSR_SINT1,
-    HV_X64_MSR_SINT2,
-    HV_X64_MSR_SINT3,
-    HV_X64_MSR_SINT4,
-    HV_X64_MSR_SINT5,
-    HV_X64_MSR_SINT6,
-    HV_X64_MSR_SINT7,
-    HV_X64_MSR_SINT8,
-    HV_X64_MSR_SINT9,
-    HV_X64_MSR_SINT10,
-    HV_X64_MSR_SINT11,
-    HV_X64_MSR_SINT12,
-    HV_X64_MSR_SINT13,
-    HV_X64_MSR_SINT14,
-    HV_X64_MSR_SINT15,
-    HV_X64_MSR_SCONTROL,
-    HV_X64_MSR_SIEFP,
-    HV_X64_MSR_SIMP,
-    HV_X64_MSR_REFERENCE_TSC,
-    HV_X64_MSR_EOM,
-};
-static const size_t msr_count = ARRAY_SIZE(supported_msrs);
-
-static int compare_msr_index(const void *a, const void *b)
-{
-    return *(uint32_t *)a - *(uint32_t *)b;
-}
-
-__attribute__((constructor))
-static void init_sorted_msr_map(void)
-{
-    qsort(supported_msrs, msr_count, sizeof(uint32_t), compare_msr_index);
-}
-
-static int mshv_is_supported_msr(uint32_t msr)
-{
-    return bsearch(&msr, supported_msrs, msr_count, sizeof(uint32_t),
-                   compare_msr_index) != NULL;
-}
-
-static int mshv_msr_to_hv_reg_name(uint32_t msr, uint32_t *hv_reg)
-{
-    switch (msr) {
-    case IA32_MSR_TSC:
-        *hv_reg = HV_X64_REGISTER_TSC;
-        return 0;
-    case IA32_MSR_EFER:
-        *hv_reg = HV_X64_REGISTER_EFER;
-        return 0;
-    case IA32_MSR_KERNEL_GS_BASE:
-        *hv_reg = HV_X64_REGISTER_KERNEL_GS_BASE;
-        return 0;
-    case IA32_MSR_APIC_BASE:
-        *hv_reg = HV_X64_REGISTER_APIC_BASE;
-        return 0;
-    case IA32_MSR_PAT:
-        *hv_reg = HV_X64_REGISTER_PAT;
-        return 0;
-    case IA32_MSR_SYSENTER_CS:
-        *hv_reg = HV_X64_REGISTER_SYSENTER_CS;
-        return 0;
-    case IA32_MSR_SYSENTER_ESP:
-        *hv_reg = HV_X64_REGISTER_SYSENTER_ESP;
-        return 0;
-    case IA32_MSR_SYSENTER_EIP:
-        *hv_reg = HV_X64_REGISTER_SYSENTER_EIP;
-        return 0;
-    case IA32_MSR_STAR:
-        *hv_reg = HV_X64_REGISTER_STAR;
-        return 0;
-    case IA32_MSR_LSTAR:
-        *hv_reg = HV_X64_REGISTER_LSTAR;
-        return 0;
-    case IA32_MSR_CSTAR:
-        *hv_reg = HV_X64_REGISTER_CSTAR;
-        return 0;
-    case IA32_MSR_SFMASK:
-        *hv_reg = HV_X64_REGISTER_SFMASK;
-        return 0;
-    case IA32_MSR_MTRR_CAP:
-        *hv_reg = HV_X64_REGISTER_MSR_MTRR_CAP;
-        return 0;
-    case IA32_MSR_MTRR_DEF_TYPE:
-        *hv_reg = HV_X64_REGISTER_MSR_MTRR_DEF_TYPE;
-        return 0;
-    case IA32_MSR_MTRR_PHYSBASE0:
-        *hv_reg = HV_X64_REGISTER_MSR_MTRR_PHYS_BASE0;
-        return 0;
-    case IA32_MSR_MTRR_PHYSMASK0:
-        *hv_reg = HV_X64_REGISTER_MSR_MTRR_PHYS_MASK0;
-        return 0;
-    case IA32_MSR_MTRR_PHYSBASE1:
-        *hv_reg = HV_X64_REGISTER_MSR_MTRR_PHYS_BASE1;
-        return 0;
-    case IA32_MSR_MTRR_PHYSMASK1:
-        *hv_reg = HV_X64_REGISTER_MSR_MTRR_PHYS_MASK1;
-        return 0;
-    case IA32_MSR_MTRR_PHYSBASE2:
-        *hv_reg = HV_X64_REGISTER_MSR_MTRR_PHYS_BASE2;
-        return 0;
-    case IA32_MSR_MTRR_PHYSMASK2:
-        *hv_reg = HV_X64_REGISTER_MSR_MTRR_PHYS_MASK2;
-        return 0;
-    case IA32_MSR_MTRR_PHYSBASE3:
-        *hv_reg = HV_X64_REGISTER_MSR_MTRR_PHYS_BASE3;
-        return 0;
-    case IA32_MSR_MTRR_PHYSMASK3:
-        *hv_reg = HV_X64_REGISTER_MSR_MTRR_PHYS_MASK3;
-        return 0;
-    case IA32_MSR_MTRR_PHYSBASE4:
-        *hv_reg = HV_X64_REGISTER_MSR_MTRR_PHYS_BASE4;
-        return 0;
-    case IA32_MSR_MTRR_PHYSMASK4:
-        *hv_reg = HV_X64_REGISTER_MSR_MTRR_PHYS_MASK4;
-        return 0;
-    case IA32_MSR_MTRR_PHYSBASE5:
-        *hv_reg = HV_X64_REGISTER_MSR_MTRR_PHYS_BASE5;
-        return 0;
-    case IA32_MSR_MTRR_PHYSMASK5:
-        *hv_reg = HV_X64_REGISTER_MSR_MTRR_PHYS_MASK5;
-        return 0;
-    case IA32_MSR_MTRR_PHYSBASE6:
-        *hv_reg = HV_X64_REGISTER_MSR_MTRR_PHYS_BASE6;
-        return 0;
-    case IA32_MSR_MTRR_PHYSMASK6:
-        *hv_reg = HV_X64_REGISTER_MSR_MTRR_PHYS_MASK6;
-        return 0;
-    case IA32_MSR_MTRR_PHYSBASE7:
-        *hv_reg = HV_X64_REGISTER_MSR_MTRR_PHYS_BASE7;
-        return 0;
-    case IA32_MSR_MTRR_PHYSMASK7:
-        *hv_reg = HV_X64_REGISTER_MSR_MTRR_PHYS_MASK7;
-        return 0;
-    case IA32_MSR_MTRR_FIX64K_00000:
-        *hv_reg = HV_X64_REGISTER_MSR_MTRR_FIX64K00000;
-        return 0;
-    case IA32_MSR_MTRR_FIX16K_80000:
-        *hv_reg = HV_X64_REGISTER_MSR_MTRR_FIX16K80000;
-        return 0;
-    case IA32_MSR_MTRR_FIX16K_A0000:
-        *hv_reg = HV_X64_REGISTER_MSR_MTRR_FIX16KA0000;
-        return 0;
-    case IA32_MSR_MTRR_FIX4K_C0000:
-        *hv_reg = HV_X64_REGISTER_MSR_MTRR_FIX4KC0000;
-        return 0;
-    case IA32_MSR_MTRR_FIX4K_C8000:
-        *hv_reg = HV_X64_REGISTER_MSR_MTRR_FIX4KC8000;
-        return 0;
-    case IA32_MSR_MTRR_FIX4K_D0000:
-        *hv_reg = HV_X64_REGISTER_MSR_MTRR_FIX4KD0000;
-        return 0;
-    case IA32_MSR_MTRR_FIX4K_D8000:
-        *hv_reg = HV_X64_REGISTER_MSR_MTRR_FIX4KD8000;
-        return 0;
-    case IA32_MSR_MTRR_FIX4K_E0000:
-        *hv_reg = HV_X64_REGISTER_MSR_MTRR_FIX4KE0000;
-        return 0;
-    case IA32_MSR_MTRR_FIX4K_E8000:
-        *hv_reg = HV_X64_REGISTER_MSR_MTRR_FIX4KE8000;
-        return 0;
-    case IA32_MSR_MTRR_FIX4K_F0000:
-        *hv_reg = HV_X64_REGISTER_MSR_MTRR_FIX4KF0000;
-        return 0;
-    case IA32_MSR_MTRR_FIX4K_F8000:
-        *hv_reg = HV_X64_REGISTER_MSR_MTRR_FIX4KF8000;
-        return 0;
-    case IA32_MSR_TSC_AUX:
-        *hv_reg = HV_X64_REGISTER_TSC_AUX;
-        return 0;
-    case IA32_MSR_BNDCFGS:
-        *hv_reg = HV_X64_REGISTER_BNDCFGS;
-        return 0;
-    case IA32_MSR_DEBUG_CTL:
-        *hv_reg = HV_X64_REGISTER_DEBUG_CTL;
-        return 0;
-    case IA32_MSR_TSC_ADJUST:
-        *hv_reg = HV_X64_REGISTER_TSC_ADJUST;
-        return 0;
-    case IA32_MSR_SPEC_CTRL:
-        *hv_reg = HV_X64_REGISTER_SPEC_CTRL;
-        return 0;
-    case HV_X64_MSR_GUEST_OS_ID:
-        *hv_reg = HV_REGISTER_GUEST_OS_ID;
-        return 0;
-    case HV_X64_MSR_SINT0:
-        *hv_reg = HV_REGISTER_SINT0;
-        return 0;
-    case HV_X64_MSR_SINT1:
-        *hv_reg = HV_REGISTER_SINT1;
-        return 0;
-    case HV_X64_MSR_SINT2:
-        *hv_reg = HV_REGISTER_SINT2;
-        return 0;
-    case HV_X64_MSR_SINT3:
-        *hv_reg = HV_REGISTER_SINT3;
-        return 0;
-    case HV_X64_MSR_SINT4:
-        *hv_reg = HV_REGISTER_SINT4;
-        return 0;
-    case HV_X64_MSR_SINT5:
-        *hv_reg = HV_REGISTER_SINT5;
-        return 0;
-    case HV_X64_MSR_SINT6:
-        *hv_reg = HV_REGISTER_SINT6;
-        return 0;
-    case HV_X64_MSR_SINT7:
-        *hv_reg = HV_REGISTER_SINT7;
-        return 0;
-    case HV_X64_MSR_SINT8:
-        *hv_reg = HV_REGISTER_SINT8;
-        return 0;
-    case HV_X64_MSR_SINT9:
-        *hv_reg = HV_REGISTER_SINT9;
-        return 0;
-    case HV_X64_MSR_SINT10:
-        *hv_reg = HV_REGISTER_SINT10;
-        return 0;
-    case HV_X64_MSR_SINT11:
-        *hv_reg = HV_REGISTER_SINT11;
-        return 0;
-    case HV_X64_MSR_SINT12:
-        *hv_reg = HV_REGISTER_SINT12;
-        return 0;
-    case HV_X64_MSR_SINT13:
-        *hv_reg = HV_REGISTER_SINT13;
-        return 0;
-    case HV_X64_MSR_SINT14:
-        *hv_reg = HV_REGISTER_SINT14;
-        return 0;
-    case HV_X64_MSR_SINT15:
-        *hv_reg = HV_REGISTER_SINT15;
-        return 0;
-    case IA32_MSR_MISC_ENABLE:
-        *hv_reg = HV_X64_REGISTER_MSR_IA32_MISC_ENABLE;
-        return 0;
-    case HV_X64_MSR_SCONTROL:
-        *hv_reg = HV_REGISTER_SCONTROL;
-        return 0;
-    case HV_X64_MSR_SIEFP:
-        *hv_reg = HV_REGISTER_SIEFP;
-        return 0;
-    case HV_X64_MSR_SIMP:
-        *hv_reg = HV_REGISTER_SIMP;
-        return 0;
-    case HV_X64_MSR_REFERENCE_TSC:
-        *hv_reg = HV_REGISTER_REFERENCE_TSC;
-        return 0;
-    case HV_X64_MSR_EOM:
-        *hv_reg = HV_REGISTER_EOM;
-        return 0;
-    default:
-        error_report("failed to map MSR %u to HV register name", msr);
-        return -1;
-    }
-}
-
-static int set_msrs(const CPUState *cpu, GList *msrs)
-{
-    size_t n_msrs;
-    GList *entries;
-    MshvMsrEntry *entry;
-    enum hv_register_name name;
-    struct hv_register_assoc *assoc;
-    int ret;
-    size_t i = 0;
-
-    n_msrs = g_list_length(msrs);
-    hv_register_assoc *assocs = g_new0(hv_register_assoc, n_msrs);
-
-    entries = msrs;
-    for (const GList *elem = entries; elem != NULL; elem = elem->next) {
-        entry = elem->data;
-        ret = mshv_msr_to_hv_reg_name(entry->index, &name);
-        if (ret < 0) {
-            g_free(assocs);
-            return ret;
-        }
-        assoc = &assocs[i];
-        assoc->name = name;
-        /* the union has been initialized to 0 */
-        assoc->value.reg64 = entry->data;
-        i++;
-    }
-    ret = mshv_set_generic_regs(cpu, assocs, n_msrs);
-    g_free(assocs);
-    if (ret < 0) {
-        error_report("failed to set msrs");
-        return -1;
-    }
-    return 0;
-}
-
-
-int mshv_configure_msr(const CPUState *cpu, const MshvMsrEntry *msrs,
-                       size_t n_msrs)
-{
-    GList *valid_msrs = NULL;
-    uint32_t msr_index;
-    int ret;
-
-    for (size_t i = 0; i < n_msrs; i++) {
-        msr_index = msrs[i].index;
-        /* check whether index of msrs is in SUPPORTED_MSRS */
-        if (mshv_is_supported_msr(msr_index)) {
-            valid_msrs = g_list_append(valid_msrs, (void *) &msrs[i]);
-        }
-    }
-
-    ret = set_msrs(cpu, valid_msrs);
-    g_list_free(valid_msrs);
-
-    return ret;
-}
@@ -1,33 +0,0 @@
-# Authors: Ziqiao Zhou <ziqiaozhou@microsoft.com>
-#          Magnus Kulke <magnuskulke@microsoft.com>
-#
-# SPDX-License-Identifier: GPL-2.0-or-later
-
-mshv_start_vcpu_thread(const char* thread, uint32_t cpu) "thread=%s cpu_index=%d"
-
-mshv_set_memory(bool add, uint64_t gpa, uint64_t size, uint64_t user_addr, bool readonly, int ret) "add=%d gpa=0x%" PRIx64 " size=0x%" PRIx64 " user=0x%" PRIx64 " readonly=%d result=%d"
-mshv_mem_ioeventfd_add(uint64_t addr, uint32_t size, uint32_t data) "addr=0x%" PRIx64 " size=%d data=0x%x"
-mshv_mem_ioeventfd_del(uint64_t addr, uint32_t size, uint32_t data) "addr=0x%" PRIx64 " size=%d data=0x%x"
-
-mshv_hvcall_args(const char* hvcall, uint16_t code, uint16_t in_sz) "built args for '%s' code: %d in_sz: %d"
-
-mshv_handle_interrupt(uint32_t cpu, int mask) "cpu_index=%d mask=0x%x"
-mshv_set_msi_routing(uint32_t gsi, uint64_t addr, uint32_t data) "gsi=%d addr=0x%" PRIx64 " data=0x%x"
-mshv_remove_msi_routing(uint32_t gsi) "gsi=%d"
-mshv_add_msi_routing(uint64_t addr, uint32_t data) "addr=0x%" PRIx64 " data=0x%x"
-mshv_commit_msi_routing_table(int vm_fd, int len) "vm_fd=%d table_size=%d"
-mshv_register_irqfd(int vm_fd, int event_fd, uint32_t gsi) "vm_fd=%d event_fd=%d gsi=%d"
-mshv_irqchip_update_irqfd_notifier_gsi(int event_fd, int resample_fd, int virq, bool add) "event_fd=%d resample_fd=%d virq=%d add=%d"
-
-mshv_insn_fetch(uint64_t addr, size_t size) "gpa=0x%" PRIx64 " size=%zu"
-mshv_mem_write(uint64_t addr, size_t size) "\tgpa=0x%" PRIx64 " size=%zu"
-mshv_mem_read(uint64_t addr, size_t size) "\tgpa=0x%" PRIx64 " size=%zu"
-mshv_map_memory(uint64_t userspace_addr, uint64_t gpa, uint64_t size) "\tu_a=0x%" PRIx64 " gpa=0x%010" PRIx64 " size=0x%08" PRIx64
-mshv_unmap_memory(uint64_t userspace_addr, uint64_t gpa, uint64_t size) "\tu_a=0x%" PRIx64 " gpa=0x%010" PRIx64 " size=0x%08" PRIx64
-mshv_set_phys_mem(bool add, const char *name, uint64_t gpa) "\tadd=%d name=%s gpa=0x%010" PRIx64
-mshv_handle_mmio(uint64_t gva, uint64_t gpa, uint64_t size, uint8_t access_type) "\tgva=0x%" PRIx64 " gpa=0x%010" PRIx64 " size=0x%" PRIx64 " access_type=%d"
-
-mshv_found_slot(uint64_t userspace_addr, uint64_t gpa, uint64_t size) "\tu_a=0x%" PRIx64 " gpa=0x%010" PRIx64 " size=0x%08" PRIx64
-mshv_skip_unset_mem(uint64_t userspace_addr, uint64_t gpa, uint64_t size) "\tu_a=0x%" PRIx64 " gpa=0x%010" PRIx64 " size=0x%08" PRIx64
-mshv_remap_attempt(uint64_t userspace_addr, uint64_t gpa, uint64_t size) "\tu_a=0x%" PRIx64 " gpa=0x%010" PRIx64 " size=0x%08" PRIx64
-mshv_find_slot_by_gpa(uint64_t gpa) "\tgpa=0x%010" PRIx64
@@ -1,14 +0,0 @@
-/*
- * QEMU MSHV support
- *
- * Copyright Microsoft, Corp. 2025
- *
- * Authors:
- *  Ziqiao Zhou       <ziqiaozhou@microsoft.com>
- *  Magnus Kulke      <magnuskulke@microsoft.com>
- *
- * SPDX-License-Identifier: GPL-2.0-or-later
- *
- */
-
-#include "trace/trace-accel_mshv.h"
@@ -18,14 +18,11 @@
 #include "qemu/option.h"
 #include "qemu/config-file.h"
 #include "qemu/accel.h"
-#include "accel/accel-ops.h"
-#include "accel/accel-cpu-ops.h"
-#include "system/qtest.h"
-#include "system/cpus.h"
+#include "sysemu/qtest.h"
+#include "sysemu/cpus.h"
 #include "qemu/guest-random.h"
 #include "qemu/main-loop.h"
 #include "hw/core/cpu.h"
-#include "accel/dummy-cpus.h"

 static int64_t qtest_clock_counter;

@@ -39,12 +36,12 @@ static void qtest_set_virtual_clock(int64_t count)
    qatomic_set_i64(&qtest_clock_counter, count);
 }

-static int qtest_init_accel(AccelState *as, MachineState *ms)
+static int qtest_init_accel(MachineState *ms)
 {
    return 0;
 }

-static void qtest_accel_class_init(ObjectClass *oc, const void *data)
+static void qtest_accel_class_init(ObjectClass *oc, void *data)
 {
    AccelClass *ac = ACCEL_CLASS(oc);
    ac->name = "QTest";
@@ -61,14 +58,13 @@ static const TypeInfo qtest_accel_type = {
 };
 module_obj(TYPE_QTEST_ACCEL);

-static void qtest_accel_ops_class_init(ObjectClass *oc, const void *data)
+static void qtest_accel_ops_class_init(ObjectClass *oc, void *data)
 {
    AccelOpsClass *ops = ACCEL_OPS_CLASS(oc);

    ops->create_vcpu_thread = dummy_start_vcpu_thread;
    ops->get_virtual_clock = qtest_get_virtual_clock;
    ops->set_virtual_clock = qtest_set_virtual_clock;
-    ops->handle_interrupt = generic_handle_interrupt;
 };

 static const TypeInfo qtest_accel_ops_type = {
@@ -1,12 +0,0 @@
-/*
- * HVF stubs for QEMU
- *
- *  Copyright (c) Linaro
- *
- * SPDX-License-Identifier: GPL-2.0-or-later
- */
-
-#include "qemu/osdep.h"
-#include "system/hvf.h"
-
-bool hvf_allowed;
@@ -11,7 +11,7 @@
 */

 #include "qemu/osdep.h"
-#include "system/kvm.h"
+#include "sysemu/kvm.h"
 #include "hw/pci/msi.h"

 KVMState *kvm_state;
@@ -29,6 +29,10 @@ void kvm_flush_coalesced_mmio_buffer(void)
 {
 }

+void kvm_cpu_synchronize_state(CPUState *cpu)
+{
+}
+
 bool kvm_has_sync_mmu(void)
 {
    return false;
@@ -101,6 +105,11 @@ unsigned int kvm_get_free_memslots(void)
    return 0;
 }

+void kvm_init_cpu_signals(CPUState *cpu)
+{
+    abort();
+}
+
 bool kvm_arm_supports_user_irq(void)
 {
    return false;
@@ -2,9 +2,5 @@ system_stubs_ss = ss.source_set()
 system_stubs_ss.add(when: 'CONFIG_XEN', if_false: files('xen-stub.c'))
 system_stubs_ss.add(when: 'CONFIG_KVM', if_false: files('kvm-stub.c'))
 system_stubs_ss.add(when: 'CONFIG_TCG', if_false: files('tcg-stub.c'))
-system_stubs_ss.add(when: 'CONFIG_HVF', if_false: files('hvf-stub.c'))
-system_stubs_ss.add(when: 'CONFIG_NVMM', if_false: files('nvmm-stub.c'))
-system_stubs_ss.add(when: 'CONFIG_WHPX', if_false: files('whpx-stub.c'))
-system_stubs_ss.add(when: 'CONFIG_MSHV', if_false: files('mshv-stub.c'))

 specific_ss.add_all(when: ['CONFIG_SYSTEM_ONLY'], if_true: system_stubs_ss)
@@ -1,44 +0,0 @@
-/*
- * QEMU MSHV stub
- *
- * Copyright Red Hat, Inc. 2025
- *
- * Author: Paolo Bonzini     <pbonzini@redhat.com>
- *
- * SPDX-License-Identifier: GPL-2.0-or-later
- */
-
-#include "qemu/osdep.h"
-#include "hw/pci/msi.h"
-#include "system/mshv.h"
-
-bool mshv_allowed;
-
-int mshv_irqchip_add_msi_route(int vector, PCIDevice *dev)
-{
-    return -ENOSYS;
-}
-
-void mshv_irqchip_release_virq(int virq)
-{
-}
-
-int mshv_irqchip_update_msi_route(int virq, MSIMessage msg, PCIDevice *dev)
-{
-    return -ENOSYS;
-}
-
-void mshv_irqchip_commit_routes(void)
-{
-}
-
-int mshv_irqchip_add_irqfd_notifier_gsi(const EventNotifier *n,
-                                        const EventNotifier *rn, int virq)
-{
-    return -ENOSYS;
-}
-
-int mshv_irqchip_remove_irqfd_notifier_gsi(const EventNotifier *n, int virq)
-{
-    return -ENOSYS;
-}
@@ -1,12 +0,0 @@
-/*
- * NVMM stubs for QEMU
- *
- *  Copyright (c) Linaro
- *
- * SPDX-License-Identifier: GPL-2.0-or-later
- */
-
-#include "qemu/osdep.h"
-#include "system/nvmm.h"
-
-bool nvmm_allowed;
@@ -11,9 +11,19 @@
 */

 #include "qemu/osdep.h"
-#include "exec/cpu-common.h"
+#include "exec/tb-flush.h"
+#include "exec/exec-all.h"
+
+void tb_flush(CPUState *cpu)
+{
+}

 G_NORETURN void cpu_loop_exit(CPUState *cpu)
 {
    g_assert_not_reached();
 }
+
+G_NORETURN void cpu_loop_exit_restore(CPUState *cpu, uintptr_t pc)
+{
+    g_assert_not_reached();
+}
@@ -1,12 +0,0 @@
-/*
- * WHPX stubs for QEMU
- *
- *  Copyright (c) Linaro
- *
- * SPDX-License-Identifier: GPL-2.0-or-later
- */
-
-#include "qemu/osdep.h"
-#include "system/whpx.h"
-
-bool whpx_allowed;
@@ -6,7 +6,7 @@
 */

 #include "qemu/osdep.h"
-#include "system/xen.h"
+#include "sysemu/xen.h"
 #include "qapi/qapi-commands-migration.h"

 bool xen_allowed;
@@ -14,20 +14,9 @@
 */

 static void atomic_trace_rmw_post(CPUArchState *env, uint64_t addr,
-                                  uint64_t read_value_low,
-                                  uint64_t read_value_high,
-                                  uint64_t write_value_low,
-                                  uint64_t write_value_high,
                                  MemOpIdx oi)
 {
-    if (cpu_plugin_mem_cbs_enabled(env_cpu(env))) {
-        qemu_plugin_vcpu_mem_cb(env_cpu(env), addr,
-                                read_value_low, read_value_high,
-                                oi, QEMU_PLUGIN_MEM_R);
-        qemu_plugin_vcpu_mem_cb(env_cpu(env), addr,
-                                write_value_low, write_value_high,
-                                oi, QEMU_PLUGIN_MEM_W);
-    }
+    qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_RW);
 }

 /*
@@ -122,14 +111,5 @@ GEN_ATOMIC_HELPERS(umax_fetch)

 GEN_ATOMIC_HELPERS(xchg)

-#if HAVE_CMPXCHG128
-ATOMIC_HELPER(xchgo_be, Int128)
-ATOMIC_HELPER(xchgo_le, Int128)
-ATOMIC_HELPER(fetch_ando_be, Int128)
-ATOMIC_HELPER(fetch_ando_le, Int128)
-ATOMIC_HELPER(fetch_oro_be, Int128)
-ATOMIC_HELPER(fetch_oro_le, Int128)
-#endif
-
 #undef ATOMIC_HELPER
 #undef GEN_ATOMIC_HELPERS
@@ -53,14 +53,6 @@
 # error unsupported data size
 #endif

-#if DATA_SIZE == 16
-# define VALUE_LOW(val) int128_getlo(val)
-# define VALUE_HIGH(val) int128_gethi(val)
-#else
-# define VALUE_LOW(val) val
-# define VALUE_HIGH(val) 0
-#endif
-
 #if DATA_SIZE >= 4
 # define ABI_TYPE  DATA_TYPE
 #else
@@ -77,7 +69,7 @@
 # define END  _le
 #endif

-ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, vaddr addr,
+ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, abi_ptr addr,
                              ABI_TYPE cmpv, ABI_TYPE newv,
                              MemOpIdx oi, uintptr_t retaddr)
 {
@@ -91,84 +83,33 @@ ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, vaddr addr,
    ret = qatomic_cmpxchg__nocheck(haddr, cmpv, newv);
 #endif
    ATOMIC_MMU_CLEANUP;
-    atomic_trace_rmw_post(env, addr,
-                          VALUE_LOW(ret),
-                          VALUE_HIGH(ret),
-                          VALUE_LOW(newv),
-                          VALUE_HIGH(newv),
-                          oi);
+    atomic_trace_rmw_post(env, addr, oi);
    return ret;
 }

-ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, vaddr addr, ABI_TYPE val,
+#if DATA_SIZE < 16
+ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, abi_ptr addr, ABI_TYPE val,
                           MemOpIdx oi, uintptr_t retaddr)
 {
    DATA_TYPE *haddr = atomic_mmu_lookup(env_cpu(env), addr, oi,
                                         DATA_SIZE, retaddr);
    DATA_TYPE ret;

-#if DATA_SIZE == 16
-    ret = atomic16_xchg(haddr, val);
-#else
    ret = qatomic_xchg__nocheck(haddr, val);
-#endif
    ATOMIC_MMU_CLEANUP;
-    atomic_trace_rmw_post(env, addr,
-                          VALUE_LOW(ret),
-                          VALUE_HIGH(ret),
-                          VALUE_LOW(val),
-                          VALUE_HIGH(val),
-                          oi);
+    atomic_trace_rmw_post(env, addr, oi);
    return ret;
 }

-#if DATA_SIZE == 16
-ABI_TYPE ATOMIC_NAME(fetch_and)(CPUArchState *env, vaddr addr, ABI_TYPE val,
-                                MemOpIdx oi, uintptr_t retaddr)
-{
-    DATA_TYPE *haddr = atomic_mmu_lookup(env_cpu(env), addr, oi,
-                                         DATA_SIZE, retaddr);
-    DATA_TYPE ret = atomic16_fetch_and(haddr, val);
-    ATOMIC_MMU_CLEANUP;
-    atomic_trace_rmw_post(env, addr,
-                          VALUE_LOW(ret),
-                          VALUE_HIGH(ret),
-                          VALUE_LOW(val),
-                          VALUE_HIGH(val),
-                          oi);
-    return ret;
-}
-
-ABI_TYPE ATOMIC_NAME(fetch_or)(CPUArchState *env, vaddr addr, ABI_TYPE val,
-                               MemOpIdx oi, uintptr_t retaddr)
-{
-    DATA_TYPE *haddr = atomic_mmu_lookup(env_cpu(env), addr, oi,
-                                         DATA_SIZE, retaddr);
-    DATA_TYPE ret = atomic16_fetch_or(haddr, val);
-    ATOMIC_MMU_CLEANUP;
-    atomic_trace_rmw_post(env, addr,
-                          VALUE_LOW(ret),
-                          VALUE_HIGH(ret),
-                          VALUE_LOW(val),
-                          VALUE_HIGH(val),
-                          oi);
-    return ret;
-}
-#else
 #define GEN_ATOMIC_HELPER(X)                                        \
-ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, vaddr addr,              \
+ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, abi_ptr addr,            \
                        ABI_TYPE val, MemOpIdx oi, uintptr_t retaddr) \
 {                                                                   \
    DATA_TYPE *haddr, ret;                                          \
    haddr = atomic_mmu_lookup(env_cpu(env), addr, oi, DATA_SIZE, retaddr);   \
    ret = qatomic_##X(haddr, val);                                  \
    ATOMIC_MMU_CLEANUP;                                             \
-    atomic_trace_rmw_post(env, addr,                                \
-                          VALUE_LOW(ret),                           \
-                          VALUE_HIGH(ret),                          \
-                          VALUE_LOW(val),                           \
-                          VALUE_HIGH(val),                          \
-                          oi);                                      \
+    atomic_trace_rmw_post(env, addr, oi);                           \
    return ret;                                                     \
 }

@@ -192,7 +133,7 @@ GEN_ATOMIC_HELPER(xor_fetch)
 * of CF_PARALLEL's value, we'll trace just a read and a write.
 */
 #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET)                \
-ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, vaddr addr,              \
+ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, abi_ptr addr,            \
                        ABI_TYPE xval, MemOpIdx oi, uintptr_t retaddr) \
 {                                                                   \
    XDATA_TYPE *haddr, cmp, old, new, val = xval;                   \
@@ -204,12 +145,7 @@ ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, vaddr addr,              \
        cmp = qatomic_cmpxchg__nocheck(haddr, old, new);            \
    } while (cmp != old);                                           \
    ATOMIC_MMU_CLEANUP;                                             \
-    atomic_trace_rmw_post(env, addr,                                \
-                          VALUE_LOW(old),                           \
-                          VALUE_HIGH(old),                          \
-                          VALUE_LOW(xval),                          \
-                          VALUE_HIGH(xval),                         \
-                          oi);                                      \
+    atomic_trace_rmw_post(env, addr, oi);                           \
    return RET;                                                     \
 }

@@ -224,7 +160,7 @@ GEN_ATOMIC_HELPER_FN(smax_fetch, MAX, SDATA_TYPE, new)
 GEN_ATOMIC_HELPER_FN(umax_fetch, MAX,  DATA_TYPE, new)

 #undef GEN_ATOMIC_HELPER_FN
-#endif /* DATA SIZE == 16 */
+#endif /* DATA SIZE < 16 */

 #undef END

@@ -238,7 +174,7 @@ GEN_ATOMIC_HELPER_FN(umax_fetch, MAX,  DATA_TYPE, new)
 # define END  _be
 #endif

-ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, vaddr addr,
+ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, abi_ptr addr,
                              ABI_TYPE cmpv, ABI_TYPE newv,
                              MemOpIdx oi, uintptr_t retaddr)
 {
@@ -252,84 +188,33 @@ ABI_TYPE ATOMIC_NAME(cmpxchg)(CPUArchState *env, vaddr addr,
    ret = qatomic_cmpxchg__nocheck(haddr, BSWAP(cmpv), BSWAP(newv));
 #endif
    ATOMIC_MMU_CLEANUP;
-    atomic_trace_rmw_post(env, addr,
-                          VALUE_LOW(ret),
-                          VALUE_HIGH(ret),
-                          VALUE_LOW(newv),
-                          VALUE_HIGH(newv),
-                          oi);
+    atomic_trace_rmw_post(env, addr, oi);
    return BSWAP(ret);
 }

-ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, vaddr addr, ABI_TYPE val,
+#if DATA_SIZE < 16
+ABI_TYPE ATOMIC_NAME(xchg)(CPUArchState *env, abi_ptr addr, ABI_TYPE val,
                           MemOpIdx oi, uintptr_t retaddr)
 {
    DATA_TYPE *haddr = atomic_mmu_lookup(env_cpu(env), addr, oi,
                                         DATA_SIZE, retaddr);
    ABI_TYPE ret;

-#if DATA_SIZE == 16
-    ret = atomic16_xchg(haddr, BSWAP(val));
-#else
    ret = qatomic_xchg__nocheck(haddr, BSWAP(val));
-#endif
    ATOMIC_MMU_CLEANUP;
-    atomic_trace_rmw_post(env, addr,
-                          VALUE_LOW(ret),
-                          VALUE_HIGH(ret),
-                          VALUE_LOW(val),
-                          VALUE_HIGH(val),
-                          oi);
+    atomic_trace_rmw_post(env, addr, oi);
    return BSWAP(ret);
 }

-#if DATA_SIZE == 16
-ABI_TYPE ATOMIC_NAME(fetch_and)(CPUArchState *env, vaddr addr, ABI_TYPE val,
-                                MemOpIdx oi, uintptr_t retaddr)
-{
-    DATA_TYPE *haddr = atomic_mmu_lookup(env_cpu(env), addr, oi,
-                                         DATA_SIZE, retaddr);
-    DATA_TYPE ret = atomic16_fetch_and(haddr, BSWAP(val));
-    ATOMIC_MMU_CLEANUP;
-    atomic_trace_rmw_post(env, addr,
-                          VALUE_LOW(ret),
-                          VALUE_HIGH(ret),
-                          VALUE_LOW(val),
-                          VALUE_HIGH(val),
-                          oi);
-    return BSWAP(ret);
-}
-
-ABI_TYPE ATOMIC_NAME(fetch_or)(CPUArchState *env, vaddr addr, ABI_TYPE val,
-                               MemOpIdx oi, uintptr_t retaddr)
-{
-    DATA_TYPE *haddr = atomic_mmu_lookup(env_cpu(env), addr, oi,
-                                         DATA_SIZE, retaddr);
-    DATA_TYPE ret = atomic16_fetch_or(haddr, BSWAP(val));
-    ATOMIC_MMU_CLEANUP;
-    atomic_trace_rmw_post(env, addr,
-                          VALUE_LOW(ret),
-                          VALUE_HIGH(ret),
-                          VALUE_LOW(val),
-                          VALUE_HIGH(val),
-                          oi);
-    return BSWAP(ret);
-}
-#else
 #define GEN_ATOMIC_HELPER(X)                                        \
-ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, vaddr addr,              \
+ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, abi_ptr addr,            \
                        ABI_TYPE val, MemOpIdx oi, uintptr_t retaddr) \
 {                                                                   \
    DATA_TYPE *haddr, ret;                                          \
    haddr = atomic_mmu_lookup(env_cpu(env), addr, oi, DATA_SIZE, retaddr);   \
    ret = qatomic_##X(haddr, BSWAP(val));                           \
    ATOMIC_MMU_CLEANUP;                                             \
-    atomic_trace_rmw_post(env, addr,                                \
-                          VALUE_LOW(ret),                           \
-                          VALUE_HIGH(ret),                          \
-                          VALUE_LOW(val),                           \
-                          VALUE_HIGH(val),                          \
-                          oi);                                      \
+    atomic_trace_rmw_post(env, addr, oi);                           \
    return BSWAP(ret);                                              \
 }

@@ -350,7 +235,7 @@ GEN_ATOMIC_HELPER(xor_fetch)
 * of CF_PARALLEL's value, we'll trace just a read and a write.
 */
 #define GEN_ATOMIC_HELPER_FN(X, FN, XDATA_TYPE, RET)                \
-ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, vaddr addr,              \
+ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, abi_ptr addr,            \
                        ABI_TYPE xval, MemOpIdx oi, uintptr_t retaddr) \
 {                                                                   \
    XDATA_TYPE *haddr, ldo, ldn, old, new, val = xval;              \
@@ -362,12 +247,7 @@ ABI_TYPE ATOMIC_NAME(X)(CPUArchState *env, vaddr addr,              \
        ldn = qatomic_cmpxchg__nocheck(haddr, ldo, BSWAP(new));     \
    } while (ldo != ldn);                                           \
    ATOMIC_MMU_CLEANUP;                                             \
-    atomic_trace_rmw_post(env, addr,                                \
-                          VALUE_LOW(old),                           \
-                          VALUE_HIGH(old),                          \
-                          VALUE_LOW(xval),                          \
-                          VALUE_HIGH(xval),                         \
-                          oi);                                      \
+    atomic_trace_rmw_post(env, addr, oi);                           \
    return RET;                                                     \
 }

@@ -389,7 +269,7 @@ GEN_ATOMIC_HELPER_FN(add_fetch, ADD, DATA_TYPE, new)
 #undef ADD

 #undef GEN_ATOMIC_HELPER_FN
-#endif /* DATA_SIZE == 16 */
+#endif /* DATA_SIZE < 16 */

 #undef END
 #endif /* DATA_SIZE > 1 */
@@ -401,5 +281,3 @@ GEN_ATOMIC_HELPER_FN(add_fetch, ADD, DATA_TYPE, new)
 #undef SUFFIX
 #undef DATA_SIZE
 #undef SHIFT
-#undef VALUE_LOW
-#undef VALUE_HIGH
@@ -1,41 +0,0 @@
-/*
- * Internal memory barrier helpers for QEMU (target agnostic)
- *
- *  Copyright (c) 2003 Fabrice Bellard
- *
- * SPDX-License-Identifier: LGPL-2.1-or-later
- */
-
-#ifndef ACCEL_TCG_BACKEND_LDST_H
-#define ACCEL_TCG_BACKEND_LDST_H
-
-#include "tcg-target-mo.h"
-
-/**
- * tcg_req_mo:
- * @guest_mo: Guest default memory order
- * @type: TCGBar
- *
- * Filter @type to the barrier that is required for the guest
- * memory ordering vs the host memory ordering.  A non-zero
- * result indicates that some barrier is required.
- */
-#define tcg_req_mo(guest_mo, type) \
-    ((type) & guest_mo & ~TCG_TARGET_DEFAULT_MO)
-
-/**
- * cpu_req_mo:
- * @cpu: CPUState
- * @type: TCGBar
- *
- * If tcg_req_mo indicates a barrier for @type is required
- * for the guest memory model, issue a host memory barrier.
- */
-#define cpu_req_mo(cpu, type)     \
-    do {                          \
-        if (tcg_req_mo(cpu->cc->tcg_ops->guest_default_memory_order, type)) { \
-            smp_mb();             \
-        }                         \
-    } while (0)
-
-#endif
@@ -18,45 +18,13 @@
 */

 #include "qemu/osdep.h"
-#include "exec/log.h"
-#include "system/tcg.h"
+#include "sysemu/cpus.h"
+#include "sysemu/tcg.h"
 #include "qemu/plugin.h"
 #include "internal-common.h"

 bool tcg_allowed;

-bool tcg_cflags_has(CPUState *cpu, uint32_t flags)
-{
-    return cpu->tcg_cflags & flags;
-}
-
-void tcg_cflags_set(CPUState *cpu, uint32_t flags)
-{
-    cpu->tcg_cflags |= flags;
-}
-
-uint32_t curr_cflags(CPUState *cpu)
-{
-    uint32_t cflags = cpu->tcg_cflags;
-
-    /*
-     * Record gdb single-step.  We should be exiting the TB by raising
-     * EXCP_DEBUG, but to simplify other tests, disable chaining too.
-     *
-     * For singlestep and -d nochain, suppress goto_tb so that
-     * we can log -d cpu,exec after every TB.
-     */
-    if (unlikely(cpu->singlestep_enabled)) {
-        cflags |= CF_NO_GOTO_TB | CF_NO_GOTO_PTR | CF_SINGLE_STEP | 1;
-    } else if (qatomic_read(&one_insn_per_tb)) {
-        cflags |= CF_NO_GOTO_TB | 1;
-    } else if (qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) {
-        cflags |= CF_NO_GOTO_TB;
-    }
-
-    return cflags;
-}
-
 /* exit the current TB, but without causing any exception to be raised */
 void cpu_loop_exit_noexc(CPUState *cpu)
 {
@@ -21,31 +21,26 @@
 #include "qemu/qemu-print.h"
 #include "qapi/error.h"
 #include "qapi/type-helpers.h"
-#include "hw/core/cpu.h"
-#include "accel/tcg/cpu-ops.h"
-#include "accel/tcg/helper-retaddr.h"
+#include "hw/core/tcg-cpu-ops.h"
 #include "trace.h"
 #include "disas/disas.h"
-#include "exec/cpu-common.h"
-#include "exec/cpu-interrupt.h"
-#include "exec/page-protection.h"
-#include "exec/mmap-lock.h"
-#include "exec/translation-block.h"
+#include "exec/exec-all.h"
 #include "tcg/tcg.h"
 #include "qemu/atomic.h"
 #include "qemu/rcu.h"
 #include "exec/log.h"
 #include "qemu/main-loop.h"
-#include "exec/icount.h"
+#include "sysemu/cpus.h"
+#include "exec/cpu-all.h"
+#include "sysemu/cpu-timers.h"
 #include "exec/replay-core.h"
-#include "system/tcg.h"
+#include "sysemu/tcg.h"
 #include "exec/helper-proto-common.h"
-#include "tcg-accel-ops.h"
 #include "tb-jmp-cache.h"
 #include "tb-hash.h"
 #include "tb-context.h"
-#include "tb-internal.h"
 #include "internal-common.h"
+#include "internal-target.h"

 /* -icount align implementation. */

@@ -149,10 +144,45 @@ static void init_delay_params(SyncClocks *sc, const CPUState *cpu)
 }
 #endif /* CONFIG USER ONLY */

+bool tcg_cflags_has(CPUState *cpu, uint32_t flags)
+{
+    return cpu->tcg_cflags & flags;
+}
+
+void tcg_cflags_set(CPUState *cpu, uint32_t flags)
+{
+    cpu->tcg_cflags |= flags;
+}
+
+uint32_t curr_cflags(CPUState *cpu)
+{
+    uint32_t cflags = cpu->tcg_cflags;
+
+    /*
+     * Record gdb single-step.  We should be exiting the TB by raising
+     * EXCP_DEBUG, but to simplify other tests, disable chaining too.
+     *
+     * For singlestep and -d nochain, suppress goto_tb so that
+     * we can log -d cpu,exec after every TB.
+     */
+    if (unlikely(cpu->singlestep_enabled)) {
+        cflags |= CF_NO_GOTO_TB | CF_NO_GOTO_PTR | CF_SINGLE_STEP | 1;
+    } else if (qatomic_read(&one_insn_per_tb)) {
+        cflags |= CF_NO_GOTO_TB | 1;
+    } else if (qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) {
+        cflags |= CF_NO_GOTO_TB;
+    }
+
+    return cflags;
+}
+
 struct tb_desc {
-    TCGTBCPUState s;
+    vaddr pc;
+    uint64_t cs_base;
    CPUArchState *env;
    tb_page_addr_t page_addr0;
+    uint32_t flags;
+    uint32_t cflags;
 };

 static bool tb_lookup_cmp(const void *p, const void *d)
@@ -160,11 +190,11 @@ static bool tb_lookup_cmp(const void *p, const void *d)
    const TranslationBlock *tb = p;
    const struct tb_desc *desc = d;

-    if ((tb_cflags(tb) & CF_PCREL || tb->pc == desc->s.pc) &&
+    if ((tb_cflags(tb) & CF_PCREL || tb->pc == desc->pc) &&
        tb_page_addr0(tb) == desc->page_addr0 &&
-        tb->cs_base == desc->s.cs_base &&
-        tb->flags == desc->s.flags &&
-        tb_cflags(tb) == desc->s.cflags) {
+        tb->cs_base == desc->cs_base &&
+        tb->flags == desc->flags &&
+        tb_cflags(tb) == desc->cflags) {
        /* check next page if needed */
        tb_page_addr_t tb_phys_page1 = tb_page_addr1(tb);
        if (tb_phys_page1 == -1) {
@@ -182,7 +212,7 @@ static bool tb_lookup_cmp(const void *p, const void *d)
             * is different for the new TB.  Therefore any exception raised
             * here by the faulting lookup is not premature.
             */
-            virt_page1 = TARGET_PAGE_ALIGN(desc->s.pc);
+            virt_page1 = TARGET_PAGE_ALIGN(desc->pc);
            phys_page1 = get_page_addr_code(desc->env, virt_page1);
            if (tb_phys_page1 == phys_page1) {
                return true;
@@ -192,65 +222,59 @@ static bool tb_lookup_cmp(const void *p, const void *d)
    return false;
 }

-static TranslationBlock *tb_htable_lookup(CPUState *cpu, TCGTBCPUState s)
+static TranslationBlock *tb_htable_lookup(CPUState *cpu, vaddr pc,
+                                          uint64_t cs_base, uint32_t flags,
+                                          uint32_t cflags)
 {
    tb_page_addr_t phys_pc;
    struct tb_desc desc;
    uint32_t h;

-    desc.s = s;
    desc.env = cpu_env(cpu);
-    phys_pc = get_page_addr_code(desc.env, s.pc);
+    desc.cs_base = cs_base;
+    desc.flags = flags;
+    desc.cflags = cflags;
+    desc.pc = pc;
+    phys_pc = get_page_addr_code(desc.env, pc);
    if (phys_pc == -1) {
        return NULL;
    }
    desc.page_addr0 = phys_pc;
-    h = tb_hash_func(phys_pc, (s.cflags & CF_PCREL ? 0 : s.pc),
-                     s.flags, s.cs_base, s.cflags);
+    h = tb_hash_func(phys_pc, (cflags & CF_PCREL ? 0 : pc),
+                     flags, cs_base, cflags);
    return qht_lookup_custom(&tb_ctx.htable, &desc, h, tb_lookup_cmp);
 }

-/**
- * tb_lookup:
- * @cpu: CPU that will execute the returned translation block
- * @pc: guest PC
- * @cs_base: arch-specific value associated with translation block
- * @flags: arch-specific translation block flags
- * @cflags: CF_* flags
- *
- * Look up a translation block inside the QHT using @pc, @cs_base, @flags and
- * @cflags. Uses @cpu's tb_jmp_cache. Might cause an exception, so have a
- * longjmp destination ready.
- *
- * Returns: an existing translation block or NULL.
- */
-static inline TranslationBlock *tb_lookup(CPUState *cpu, TCGTBCPUState s)
+/* Might cause an exception, so have a longjmp destination ready */
+static inline TranslationBlock *tb_lookup(CPUState *cpu, vaddr pc,
+                                          uint64_t cs_base, uint32_t flags,
+                                          uint32_t cflags)
 {
    TranslationBlock *tb;
    CPUJumpCache *jc;
    uint32_t hash;

    /* we should never be trying to look up an INVALID tb */
-    tcg_debug_assert(!(s.cflags & CF_INVALID));
+    tcg_debug_assert(!(cflags & CF_INVALID));

-    hash = tb_jmp_cache_hash_func(s.pc);
+    hash = tb_jmp_cache_hash_func(pc);
    jc = cpu->tb_jmp_cache;

    tb = qatomic_read(&jc->array[hash].tb);
    if (likely(tb &&
-               jc->array[hash].pc == s.pc &&
-               tb->cs_base == s.cs_base &&
-               tb->flags == s.flags &&
-               tb_cflags(tb) == s.cflags)) {
+               jc->array[hash].pc == pc &&
+               tb->cs_base == cs_base &&
+               tb->flags == flags &&
+               tb_cflags(tb) == cflags)) {
        goto hit;
    }

-    tb = tb_htable_lookup(cpu, s);
+    tb = tb_htable_lookup(cpu, pc, cs_base, flags, cflags);
    if (tb == NULL) {
        return NULL;
    }

-    jc->array[hash].pc = s.pc;
+    jc->array[hash].pc = pc;
    qatomic_set(&jc->array[hash].tb, tb);

 hit:
@@ -258,7 +282,7 @@ hit:
     * As long as tb is not NULL, the contents are consistent.  Therefore,
     * the virtual PC has to match for non-CF_PCREL translations.
     */
-    assert((tb_cflags(tb) & CF_PCREL) || tb->pc == s.pc);
+    assert((tb_cflags(tb) & CF_PCREL) || tb->pc == pc);
    return tb;
 }

@@ -275,11 +299,14 @@ static void log_cpu_exec(vaddr pc, CPUState *cpu,
        if (qemu_loglevel_mask(CPU_LOG_TB_CPU)) {
            FILE *logfile = qemu_log_trylock();
            if (logfile) {
-                int flags = CPU_DUMP_CCOP;
+                int flags = 0;

                if (qemu_loglevel_mask(CPU_LOG_TB_FPU)) {
                    flags |= CPU_DUMP_FPU;
                }
+#if defined(TARGET_I386)
+                flags |= CPU_DUMP_CCOP;
+#endif
                if (qemu_loglevel_mask(CPU_LOG_TB_VPU)) {
                    flags |= CPU_DUMP_VPU;
                }
@@ -375,6 +402,9 @@ const void *HELPER(lookup_tb_ptr)(CPUArchState *env)
 {
    CPUState *cpu = env_cpu(env);
    TranslationBlock *tb;
+    vaddr pc;
+    uint64_t cs_base;
+    uint32_t flags, cflags;

    /*
     * By definition we've just finished a TB, so I/O is OK.
@@ -384,36 +414,25 @@ const void *HELPER(lookup_tb_ptr)(CPUArchState *env)
     * The next TB, if we chain to it, will clear the flag again.
     */
    cpu->neg.can_do_io = true;
+    cpu_get_tb_cpu_state(env, &pc, &cs_base, &flags);

-    TCGTBCPUState s = cpu->cc->tcg_ops->get_tb_cpu_state(cpu);
-    s.cflags = curr_cflags(cpu);
-
-    if (check_for_breakpoints(cpu, s.pc, &s.cflags)) {
+    cflags = curr_cflags(cpu);
+    if (check_for_breakpoints(cpu, pc, &cflags)) {
        cpu_loop_exit(cpu);
    }

-    tb = tb_lookup(cpu, s);
+    tb = tb_lookup(cpu, pc, cs_base, flags, cflags);
    if (tb == NULL) {
        return tcg_code_gen_epilogue;
    }

    if (qemu_loglevel_mask(CPU_LOG_TB_CPU | CPU_LOG_EXEC)) {
-        log_cpu_exec(s.pc, cpu, tb);
+        log_cpu_exec(pc, cpu, tb);
    }

    return tb->tc.ptr;
 }

-/* Return the current PC from CPU, which may be cached in TB. */
-static vaddr log_pc(CPUState *cpu, const TranslationBlock *tb)
-{
-    if (tb_cflags(tb) & CF_PCREL) {
-        return cpu->cc->get_pc(cpu);
-    } else {
-        return tb->pc;
-    }
-}
-
 /* Execute a TB, and fix up the CPU state afterwards if necessary */
 /*
 * Disable CFI checks.
@@ -548,7 +567,11 @@ static void cpu_exec_longjmp_cleanup(CPUState *cpu)

 void cpu_exec_step_atomic(CPUState *cpu)
 {
+    CPUArchState *env = cpu_env(cpu);
    TranslationBlock *tb;
+    vaddr pc;
+    uint64_t cs_base;
+    uint32_t flags, cflags;
    int tb_exit;

    if (sigsetjmp(cpu->jmp_env, 0) == 0) {
@@ -557,13 +580,13 @@ void cpu_exec_step_atomic(CPUState *cpu)
        g_assert(!cpu->running);
        cpu->running = true;

-        TCGTBCPUState s = cpu->cc->tcg_ops->get_tb_cpu_state(cpu);
-        s.cflags = curr_cflags(cpu);
+        cpu_get_tb_cpu_state(env, &pc, &cs_base, &flags);

+        cflags = curr_cflags(cpu);
        /* Execute in a serial context. */
-        s.cflags &= ~CF_PARALLEL;
+        cflags &= ~CF_PARALLEL;
        /* After 1 insn, return and release the exclusive lock. */
-        s.cflags |= CF_NO_GOTO_TB | CF_NO_GOTO_PTR | 1;
+        cflags |= CF_NO_GOTO_TB | CF_NO_GOTO_PTR | 1;
        /*
         * No need to check_for_breakpoints here.
         * We only arrive in cpu_exec_step_atomic after beginning execution
@@ -571,16 +594,16 @@ void cpu_exec_step_atomic(CPUState *cpu)
         * Any breakpoint for this insn will have been recognized earlier.
         */

-        tb = tb_lookup(cpu, s);
+        tb = tb_lookup(cpu, pc, cs_base, flags, cflags);
        if (tb == NULL) {
            mmap_lock();
-            tb = tb_gen_code(cpu, s);
+            tb = tb_gen_code(cpu, pc, cs_base, flags, cflags);
            mmap_unlock();
        }

        cpu_exec_enter(cpu);
        /* execute the generated code */
-        trace_exec_tb(tb, s.pc);
+        trace_exec_tb(tb, pc);
        cpu_tb_exec(cpu, tb, &tb_exit);
        cpu_exec_exit(cpu);
    } else {
@@ -648,6 +671,7 @@ static inline void tb_add_jump(TranslationBlock *tb, int n,

 out_unlock_next:
    qemu_spin_unlock(&tb_next->jmp_lock);
+    return;
 }

 static inline bool cpu_handle_halt(CPUState *cpu)
@@ -713,10 +737,10 @@ static inline bool cpu_handle_exception(CPUState *cpu, int *ret)
     * If user mode only, we simulate a fake exception which will be
     * handled outside the cpu execution loop.
     */
+#if defined(TARGET_I386)
    const TCGCPUOps *tcg_ops = cpu->cc->tcg_ops;
-    if (tcg_ops->fake_user_interrupt) {
-        tcg_ops->fake_user_interrupt(cpu);
-    }
+    tcg_ops->fake_user_interrupt(cpu);
+#endif /* TARGET_I386 */
    *ret = cpu->exception_index;
    cpu->exception_index = -1;
    return true;
@@ -749,22 +773,6 @@ static inline bool cpu_handle_exception(CPUState *cpu, int *ret)
    return false;
 }

-void tcg_kick_vcpu_thread(CPUState *cpu)
-{
-#ifndef CONFIG_USER_ONLY
-    /*
-     * Ensure cpu_exec will see the reason why the exit request was set.
-     * FIXME: this is not always needed.  Other accelerators instead
-     * read interrupt_request and set exit_request on demand from the
-     * CPU thread; see kvm_arch_pre_run() for example.
-     */
-    qatomic_store_release(&cpu->exit_request, true);
-#endif
-
-    /* Ensure cpu_exec will see the exit request after TCG has exited.  */
-    qatomic_store_release(&cpu->neg.icount_decr.u16.high, -1);
-}
-
 static inline bool icount_exit_request(CPUState *cpu)
 {
    if (!icount_enabled()) {
@@ -791,53 +799,61 @@ static inline bool cpu_handle_interrupt(CPUState *cpu,
    /* Clear the interrupt flag now since we're processing
     * cpu->interrupt_request and cpu->exit_request.
     * Ensure zeroing happens before reading cpu->exit_request or
-     * cpu->interrupt_request (see also store-release in
-     * tcg_kick_vcpu_thread())
+     * cpu->interrupt_request (see also smp_wmb in cpu_exit())
     */
    qatomic_set_mb(&cpu->neg.icount_decr.u16.high, 0);

-#ifdef CONFIG_USER_ONLY
-    assert(!cpu_test_interrupt(cpu, ~0));
-#else
-    if (unlikely(cpu_test_interrupt(cpu, ~0))) {
+    if (unlikely(qatomic_read(&cpu->interrupt_request))) {
+        int interrupt_request;
        bql_lock();
-        if (cpu_test_interrupt(cpu, CPU_INTERRUPT_DEBUG)) {
-            cpu_reset_interrupt(cpu, CPU_INTERRUPT_DEBUG);
+        interrupt_request = cpu->interrupt_request;
+        if (unlikely(cpu->singlestep_enabled & SSTEP_NOIRQ)) {
+            /* Mask out external interrupts for this step. */
+            interrupt_request &= ~CPU_INTERRUPT_SSTEP_MASK;
+        }
+        if (interrupt_request & CPU_INTERRUPT_DEBUG) {
+            cpu->interrupt_request &= ~CPU_INTERRUPT_DEBUG;
            cpu->exception_index = EXCP_DEBUG;
            bql_unlock();
            return true;
        }
+#if !defined(CONFIG_USER_ONLY)
        if (replay_mode == REPLAY_MODE_PLAY && !replay_has_interrupt()) {
            /* Do nothing */
-        } else if (cpu_test_interrupt(cpu, CPU_INTERRUPT_HALT)) {
+        } else if (interrupt_request & CPU_INTERRUPT_HALT) {
            replay_interrupt();
-            cpu_reset_interrupt(cpu, CPU_INTERRUPT_HALT);
+            cpu->interrupt_request &= ~CPU_INTERRUPT_HALT;
            cpu->halted = 1;
            cpu->exception_index = EXCP_HLT;
            bql_unlock();
            return true;
-        } else {
+        }
+#if defined(TARGET_I386)
+        else if (interrupt_request & CPU_INTERRUPT_INIT) {
+            X86CPU *x86_cpu = X86_CPU(cpu);
+            CPUArchState *env = &x86_cpu->env;
+            replay_interrupt();
+            cpu_svm_check_intercept_param(env, SVM_EXIT_INIT, 0, 0);
+            do_cpu_init(x86_cpu);
+            cpu->exception_index = EXCP_HALTED;
+            bql_unlock();
+            return true;
+        }
+#else
+        else if (interrupt_request & CPU_INTERRUPT_RESET) {
+            replay_interrupt();
+            cpu_reset(cpu);
+            bql_unlock();
+            return true;
+        }
+#endif /* !TARGET_I386 */
+        /* The target hook has 3 exit conditions:
+           False when the interrupt isn't processed,
+           True when it is, and we should restart on a new TB,
+           and via longjmp via cpu_loop_exit.  */
+        else {
            const TCGCPUOps *tcg_ops = cpu->cc->tcg_ops;
-            int interrupt_request = cpu->interrupt_request;

-            if (cpu_test_interrupt(cpu, CPU_INTERRUPT_RESET)) {
-                replay_interrupt();
-                tcg_ops->cpu_exec_reset(cpu);
-                bql_unlock();
-                return true;
-            }
-
-            if (unlikely(cpu->singlestep_enabled & SSTEP_NOIRQ)) {
-                /* Mask out external interrupts for this step. */
-                interrupt_request &= ~CPU_INTERRUPT_SSTEP_MASK;
-            }
-
-            /*
-             * The target hook has 3 exit conditions:
-             * False when the interrupt isn't processed,
-             * True when it is, and we should restart on a new TB,
-             * and via longjmp via cpu_loop_exit.
-             */
            if (tcg_ops->cpu_exec_interrupt(cpu, interrupt_request)) {
                if (!tcg_ops->need_replay_interrupt ||
                    tcg_ops->need_replay_interrupt(interrupt_request)) {
@@ -856,9 +872,13 @@ static inline bool cpu_handle_interrupt(CPUState *cpu,
                cpu->exception_index = -1;
                *last_tb = NULL;
            }
+            /* The target hook may have updated the 'cpu->interrupt_request';
+             * reload the 'interrupt_request' value */
+            interrupt_request = cpu->interrupt_request;
        }
-        if (cpu_test_interrupt(cpu, CPU_INTERRUPT_EXITTB)) {
-            cpu_reset_interrupt(cpu, CPU_INTERRUPT_EXITTB);
+#endif /* !CONFIG_USER_ONLY */
+        if (interrupt_request & CPU_INTERRUPT_EXITTB) {
+            cpu->interrupt_request &= ~CPU_INTERRUPT_EXITTB;
            /* ensure that no TB jump will be modified as
               the program flow was changed */
            *last_tb = NULL;
@@ -867,13 +887,10 @@ static inline bool cpu_handle_interrupt(CPUState *cpu,
        /* If we exit via cpu_loop_exit/longjmp it is reset in cpu_exec */
        bql_unlock();
    }
-#endif /* !CONFIG_USER_ONLY */

-    /*
-     * Finally, check if we need to exit to the main loop.
-     * The corresponding store-release is in cpu_exit.
-     */
-    if (unlikely(qatomic_load_acquire(&cpu->exit_request)) || icount_exit_request(cpu)) {
+    /* Finally, check if we need to exit to the main loop.  */
+    if (unlikely(qatomic_read(&cpu->exit_request)) || icount_exit_request(cpu)) {
+        qatomic_set(&cpu->exit_request, 0);
        if (cpu->exception_index == -1) {
            cpu->exception_index = EXCP_INTERRUPT;
        }
@@ -943,8 +960,11 @@ cpu_exec_loop(CPUState *cpu, SyncClocks *sc)

        while (!cpu_handle_interrupt(cpu, &last_tb)) {
            TranslationBlock *tb;
-            TCGTBCPUState s = cpu->cc->tcg_ops->get_tb_cpu_state(cpu);
-            s.cflags = cpu->cflags_next_tb;
+            vaddr pc;
+            uint64_t cs_base;
+            uint32_t flags, cflags;
+
+            cpu_get_tb_cpu_state(cpu_env(cpu), &pc, &cs_base, &flags);

            /*
             * When requested, use an exact setting for cflags for the next
@@ -953,32 +973,33 @@ cpu_exec_loop(CPUState *cpu, SyncClocks *sc)
             * have CF_INVALID set, -1 is a convenient invalid value that
             * does not require tcg headers for cpu_common_reset.
             */
-            if (s.cflags == -1) {
-                s.cflags = curr_cflags(cpu);
+            cflags = cpu->cflags_next_tb;
+            if (cflags == -1) {
+                cflags = curr_cflags(cpu);
            } else {
                cpu->cflags_next_tb = -1;
            }

-            if (check_for_breakpoints(cpu, s.pc, &s.cflags)) {
+            if (check_for_breakpoints(cpu, pc, &cflags)) {
                break;
            }

-            tb = tb_lookup(cpu, s);
+            tb = tb_lookup(cpu, pc, cs_base, flags, cflags);
            if (tb == NULL) {
                CPUJumpCache *jc;
                uint32_t h;

                mmap_lock();
-                tb = tb_gen_code(cpu, s);
+                tb = tb_gen_code(cpu, pc, cs_base, flags, cflags);
                mmap_unlock();

                /*
                 * We add the TB in the virtual pc hash table
                 * for the fast lookup
                 */
-                h = tb_jmp_cache_hash_func(s.pc);
+                h = tb_jmp_cache_hash_func(pc);
                jc = cpu->tb_jmp_cache;
-                jc->array[h].pc = s.pc;
+                jc->array[h].pc = pc;
                qatomic_set(&jc->array[h].tb, tb);
            }

@@ -998,7 +1019,7 @@ cpu_exec_loop(CPUState *cpu, SyncClocks *sc)
                tb_add_jump(last_tb, tb_exit, tb);
            }

-            cpu_loop_exec_tb(cpu, tb, s.pc, &last_tb, &tb_exit);
+            cpu_loop_exec_tb(cpu, tb, pc, &last_tb, &tb_exit);

            /* Try to align the host and virtual clocks
               if the guest is in advance */
@@ -1053,17 +1074,11 @@ bool tcg_exec_realizefn(CPUState *cpu, Error **errp)

    if (!tcg_target_initialized) {
        /* Check mandatory TCGCPUOps handlers */
-        const TCGCPUOps *tcg_ops = cpu->cc->tcg_ops;
 #ifndef CONFIG_USER_ONLY
-        assert(tcg_ops->cpu_exec_halt);
-        assert(tcg_ops->cpu_exec_interrupt);
-        assert(tcg_ops->cpu_exec_reset);
-        assert(tcg_ops->pointer_wrap);
+        assert(cpu->cc->tcg_ops->cpu_exec_halt);
+        assert(cpu->cc->tcg_ops->cpu_exec_interrupt);
 #endif /* !CONFIG_USER_ONLY */
-        assert(tcg_ops->translate_code);
-        assert(tcg_ops->get_tb_cpu_state);
-        assert(tcg_ops->mmu_index);
-        tcg_ops->initialize();
+        cpu->cc->tcg_ops->initialize();
        tcg_target_initialized = true;
    }

@@ -19,18 +19,15 @@

 #include "qemu/osdep.h"
 #include "qemu/main-loop.h"
-#include "qemu/target-info.h"
-#include "accel/tcg/cpu-ops.h"
-#include "accel/tcg/iommu.h"
-#include "accel/tcg/probe.h"
+#include "hw/core/tcg-cpu-ops.h"
+#include "exec/exec-all.h"
 #include "exec/page-protection.h"
-#include "system/memory.h"
-#include "system/physmem.h"
-#include "accel/tcg/cpu-ldst-common.h"
-#include "accel/tcg/cpu-mmu-index.h"
+#include "exec/memory.h"
+#include "exec/cpu_ldst.h"
 #include "exec/cputlb.h"
 #include "exec/tb-flush.h"
-#include "system/ram_addr.h"
+#include "exec/memory-internal.h"
+#include "exec/ram_addr.h"
 #include "exec/mmu-access-type.h"
 #include "exec/tlb-common.h"
 #include "exec/vaddr.h"
@@ -38,21 +35,18 @@
 #include "qemu/error-report.h"
 #include "exec/log.h"
 #include "exec/helper-proto-common.h"
-#include "exec/tlb-flags.h"
 #include "qemu/atomic.h"
 #include "qemu/atomic128.h"
-#include "tb-internal.h"
+#include "exec/translate-all.h"
 #include "trace.h"
 #include "tb-hash.h"
-#include "tb-internal.h"
-#include "tlb-bounds.h"
 #include "internal-common.h"
+#include "internal-target.h"
 #ifdef CONFIG_PLUGIN
 #include "qemu/plugin-memory.h"
 #endif
 #include "tcg/tcg-ldst.h"
-#include "backend-ldst.h"
-
+#include "tcg/oversized-guest.h"

 /* DEBUG defines, enable DEBUG_TLB_LOG to log to the CPU_LOG_MMU target */
 /* #define DEBUG_TLB */
@@ -90,6 +84,9 @@
 */
 QEMU_BUILD_BUG_ON(sizeof(vaddr) > sizeof(run_on_cpu_data));

+/* We currently can't handle more than 16 bits in the MMUIDX bitmask.
+ */
+QEMU_BUILD_BUG_ON(NB_MMU_MODES > 16);
 #define ALL_MMUIDX_BITS ((1 << NB_MMU_MODES) - 1)

 static inline size_t tlb_n_entries(CPUTLBDescFast *fast)
@@ -107,15 +104,26 @@ static inline uint64_t tlb_read_idx(const CPUTLBEntry *entry,
 {
    /* Do not rearrange the CPUTLBEntry structure members. */
    QEMU_BUILD_BUG_ON(offsetof(CPUTLBEntry, addr_read) !=
-                      MMU_DATA_LOAD * sizeof(uintptr_t));
+                      MMU_DATA_LOAD * sizeof(uint64_t));
    QEMU_BUILD_BUG_ON(offsetof(CPUTLBEntry, addr_write) !=
-                      MMU_DATA_STORE * sizeof(uintptr_t));
+                      MMU_DATA_STORE * sizeof(uint64_t));
    QEMU_BUILD_BUG_ON(offsetof(CPUTLBEntry, addr_code) !=
-                      MMU_INST_FETCH * sizeof(uintptr_t));
+                      MMU_INST_FETCH * sizeof(uint64_t));

-    const uintptr_t *ptr = &entry->addr_idx[access_type];
+#if TARGET_LONG_BITS == 32
+    /* Use qatomic_read, in case of addr_write; only care about low bits. */
+    const uint32_t *ptr = (uint32_t *)&entry->addr_idx[access_type];
+    ptr += HOST_BIG_ENDIAN;
+    return qatomic_read(ptr);
+#else
+    const uint64_t *ptr = &entry->addr_idx[access_type];
+# if TCG_OVERSIZED_GUEST
+    return *ptr;
+# else
    /* ofs might correspond to .addr_write, so use qatomic_read */
    return qatomic_read(ptr);
+# endif
+#endif
 }

 static inline uint64_t tlb_addr_write(const CPUTLBEntry *entry)
@@ -127,7 +135,7 @@ static inline uint64_t tlb_addr_write(const CPUTLBEntry *entry)
 static inline uintptr_t tlb_index(CPUState *cpu, uintptr_t mmu_idx,
                                  vaddr addr)
 {
-    uintptr_t size_mask = cpu_tlb_fast(cpu, mmu_idx)->mask >> CPU_TLB_ENTRY_BITS;
+    uintptr_t size_mask = cpu->neg.tlb.f[mmu_idx].mask >> CPU_TLB_ENTRY_BITS;

    return (addr >> TARGET_PAGE_BITS) & size_mask;
 }
@@ -136,7 +144,7 @@ static inline uintptr_t tlb_index(CPUState *cpu, uintptr_t mmu_idx,
 static inline CPUTLBEntry *tlb_entry(CPUState *cpu, uintptr_t mmu_idx,
                                     vaddr addr)
 {
-    return &cpu_tlb_fast(cpu, mmu_idx)->table[tlb_index(cpu, mmu_idx, addr)];
+    return &cpu->neg.tlb.f[mmu_idx].table[tlb_index(cpu, mmu_idx, addr)];
 }

 static void tlb_window_reset(CPUTLBDesc *desc, int64_t ns,
@@ -290,7 +298,7 @@ static void tlb_flush_one_mmuidx_locked(CPUState *cpu, int mmu_idx,
                                        int64_t now)
 {
    CPUTLBDesc *desc = &cpu->neg.tlb.d[mmu_idx];
-    CPUTLBDescFast *fast = cpu_tlb_fast(cpu, mmu_idx);
+    CPUTLBDescFast *fast = &cpu->neg.tlb.f[mmu_idx];

    tlb_mmu_resize_locked(desc, fast, now);
    tlb_mmu_flush_locked(desc, fast);
@@ -329,7 +337,7 @@ void tlb_init(CPUState *cpu)
    cpu->neg.tlb.c.dirty = 0;

    for (i = 0; i < NB_MMU_MODES; i++) {
-        tlb_mmu_init(&cpu->neg.tlb.d[i], cpu_tlb_fast(cpu, i), now);
+        tlb_mmu_init(&cpu->neg.tlb.d[i], &cpu->neg.tlb.f[i], now);
    }
 }

@@ -340,7 +348,7 @@ void tlb_destroy(CPUState *cpu)
    qemu_spin_destroy(&cpu->neg.tlb.c.lock);
    for (i = 0; i < NB_MMU_MODES; i++) {
        CPUTLBDesc *desc = &cpu->neg.tlb.d[i];
-        CPUTLBDescFast *fast = cpu_tlb_fast(cpu, i);
+        CPUTLBDescFast *fast = &cpu->neg.tlb.f[i];

        g_free(fast->table);
        g_free(desc->fulltlb);
@@ -368,8 +376,8 @@ static void flush_all_helper(CPUState *src, run_on_cpu_func fn,

 static void tlb_flush_by_mmuidx_async_work(CPUState *cpu, run_on_cpu_data data)
 {
-    MMUIdxMap asked = data.host_int;
-    MMUIdxMap all_dirty, work, to_clean;
+    uint16_t asked = data.host_int;
+    uint16_t all_dirty, work, to_clean;
    int64_t now = get_clock_realtime();

    assert_cpu_is_self(cpu);
@@ -406,7 +414,7 @@ static void tlb_flush_by_mmuidx_async_work(CPUState *cpu, run_on_cpu_data data)
    }
 }

-void tlb_flush_by_mmuidx(CPUState *cpu, MMUIdxMap idxmap)
+void tlb_flush_by_mmuidx(CPUState *cpu, uint16_t idxmap)
 {
    tlb_debug("mmu_idx: 0x%" PRIx16 "\n", idxmap);

@@ -420,7 +428,7 @@ void tlb_flush(CPUState *cpu)
    tlb_flush_by_mmuidx(cpu, ALL_MMUIDX_BITS);
 }

-void tlb_flush_by_mmuidx_all_cpus_synced(CPUState *src_cpu, MMUIdxMap idxmap)
+void tlb_flush_by_mmuidx_all_cpus_synced(CPUState *src_cpu, uint16_t idxmap)
 {
    const run_on_cpu_func fn = tlb_flush_by_mmuidx_async_work;

@@ -529,7 +537,7 @@ static void tlb_flush_page_locked(CPUState *cpu, int midx, vaddr page)
 */
 static void tlb_flush_page_by_mmuidx_async_0(CPUState *cpu,
                                             vaddr addr,
-                                             MMUIdxMap idxmap)
+                                             uint16_t idxmap)
 {
    int mmu_idx;

@@ -568,14 +576,14 @@ static void tlb_flush_page_by_mmuidx_async_1(CPUState *cpu,
 {
    vaddr addr_and_idxmap = data.target_ptr;
    vaddr addr = addr_and_idxmap & TARGET_PAGE_MASK;
-    MMUIdxMap idxmap = addr_and_idxmap & ~TARGET_PAGE_MASK;
+    uint16_t idxmap = addr_and_idxmap & ~TARGET_PAGE_MASK;

    tlb_flush_page_by_mmuidx_async_0(cpu, addr, idxmap);
 }

 typedef struct {
    vaddr addr;
-    MMUIdxMap idxmap;
+    uint16_t idxmap;
 } TLBFlushPageByMMUIdxData;

 /**
@@ -597,7 +605,7 @@ static void tlb_flush_page_by_mmuidx_async_2(CPUState *cpu,
    g_free(d);
 }

-void tlb_flush_page_by_mmuidx(CPUState *cpu, vaddr addr, MMUIdxMap idxmap)
+void tlb_flush_page_by_mmuidx(CPUState *cpu, vaddr addr, uint16_t idxmap)
 {
    tlb_debug("addr: %016" VADDR_PRIx " mmu_idx:%" PRIx16 "\n", addr, idxmap);

@@ -616,7 +624,7 @@ void tlb_flush_page(CPUState *cpu, vaddr addr)

 void tlb_flush_page_by_mmuidx_all_cpus_synced(CPUState *src_cpu,
                                              vaddr addr,
-                                              MMUIdxMap idxmap)
+                                              uint16_t idxmap)
 {
    tlb_debug("addr: %016" VADDR_PRIx " mmu_idx:%"PRIx16"\n", addr, idxmap);

@@ -665,7 +673,7 @@ static void tlb_flush_range_locked(CPUState *cpu, int midx,
                                   unsigned bits)
 {
    CPUTLBDesc *d = &cpu->neg.tlb.d[midx];
-    CPUTLBDescFast *f = cpu_tlb_fast(cpu, midx);
+    CPUTLBDescFast *f = &cpu->neg.tlb.f[midx];
    vaddr mask = MAKE_64BIT_MASK(0, bits);

    /*
@@ -713,8 +721,8 @@ static void tlb_flush_range_locked(CPUState *cpu, int midx,
 typedef struct {
    vaddr addr;
    vaddr len;
-    MMUIdxMap idxmap;
-    unsigned bits;
+    uint16_t idxmap;
+    uint16_t bits;
 } TLBFlushRangeData;

 static void tlb_flush_range_by_mmuidx_async_0(CPUState *cpu,
@@ -764,26 +772,26 @@ static void tlb_flush_range_by_mmuidx_async_1(CPUState *cpu,
 }

 void tlb_flush_range_by_mmuidx(CPUState *cpu, vaddr addr,
-                               vaddr len, MMUIdxMap idxmap,
+                               vaddr len, uint16_t idxmap,
                               unsigned bits)
 {
    TLBFlushRangeData d;

    assert_cpu_is_self(cpu);

-    /* If no page bits are significant, this devolves to tlb_flush. */
-    if (bits < TARGET_PAGE_BITS) {
-        tlb_flush_by_mmuidx(cpu, idxmap);
-        return;
-    }
    /*
     * If all bits are significant, and len is small,
     * this devolves to tlb_flush_page.
     */
-    if (len <= TARGET_PAGE_SIZE && bits >= target_long_bits()) {
+    if (bits >= TARGET_LONG_BITS && len <= TARGET_PAGE_SIZE) {
        tlb_flush_page_by_mmuidx(cpu, addr, idxmap);
        return;
    }
+    /* If no page bits are significant, this devolves to tlb_flush. */
+    if (bits < TARGET_PAGE_BITS) {
+        tlb_flush_by_mmuidx(cpu, idxmap);
+        return;
+    }

    /* This should already be page aligned */
    d.addr = addr & TARGET_PAGE_MASK;
@@ -795,7 +803,7 @@ void tlb_flush_range_by_mmuidx(CPUState *cpu, vaddr addr,
 }

 void tlb_flush_page_bits_by_mmuidx(CPUState *cpu, vaddr addr,
-                                   MMUIdxMap idxmap, unsigned bits)
+                                   uint16_t idxmap, unsigned bits)
 {
    tlb_flush_range_by_mmuidx(cpu, addr, TARGET_PAGE_SIZE, idxmap, bits);
 }
@@ -803,25 +811,25 @@ void tlb_flush_page_bits_by_mmuidx(CPUState *cpu, vaddr addr,
 void tlb_flush_range_by_mmuidx_all_cpus_synced(CPUState *src_cpu,
                                               vaddr addr,
                                               vaddr len,
-                                               MMUIdxMap idxmap,
+                                               uint16_t idxmap,
                                               unsigned bits)
 {
    TLBFlushRangeData d, *p;
    CPUState *dst_cpu;

-    /* If no page bits are significant, this devolves to tlb_flush. */
-    if (bits < TARGET_PAGE_BITS) {
-        tlb_flush_by_mmuidx_all_cpus_synced(src_cpu, idxmap);
-        return;
-    }
    /*
     * If all bits are significant, and len is small,
     * this devolves to tlb_flush_page.
     */
-    if (len <= TARGET_PAGE_SIZE && bits >= target_long_bits()) {
+    if (bits >= TARGET_LONG_BITS && len <= TARGET_PAGE_SIZE) {
        tlb_flush_page_by_mmuidx_all_cpus_synced(src_cpu, addr, idxmap);
        return;
    }
+    /* If no page bits are significant, this devolves to tlb_flush. */
+    if (bits < TARGET_PAGE_BITS) {
+        tlb_flush_by_mmuidx_all_cpus_synced(src_cpu, idxmap);
+        return;
+    }

    /* This should already be page aligned */
    d.addr = addr & TARGET_PAGE_MASK;
@@ -845,7 +853,7 @@ void tlb_flush_range_by_mmuidx_all_cpus_synced(CPUState *src_cpu,

 void tlb_flush_page_bits_by_mmuidx_all_cpus_synced(CPUState *src_cpu,
                                                   vaddr addr,
-                                                   MMUIdxMap idxmap,
+                                                   uint16_t idxmap,
                                                   unsigned bits)
 {
    tlb_flush_range_by_mmuidx_all_cpus_synced(src_cpu, addr, TARGET_PAGE_SIZE,
@@ -856,7 +864,7 @@ void tlb_flush_page_bits_by_mmuidx_all_cpus_synced(CPUState *src_cpu,
   can be detected */
 void tlb_protect_code(ram_addr_t ram_addr)
 {
-    physical_memory_test_and_clear_dirty(ram_addr & TARGET_PAGE_MASK,
+    cpu_physical_memory_test_and_clear_dirty(ram_addr & TARGET_PAGE_MASK,
                                             TARGET_PAGE_SIZE,
                                             DIRTY_MEMORY_CODE);
 }
@@ -865,7 +873,7 @@ void tlb_protect_code(ram_addr_t ram_addr)
   tested for self modifying code */
 void tlb_unprotect_code(ram_addr_t ram_addr)
 {
-    physical_memory_set_dirty_flag(ram_addr, DIRTY_MEMORY_CODE);
+    cpu_physical_memory_set_dirty_flag(ram_addr, DIRTY_MEMORY_CODE);
 }


@@ -885,17 +893,26 @@ void tlb_unprotect_code(ram_addr_t ram_addr)
 *
 * Called with tlb_c.lock held.
 */
-static void tlb_reset_dirty_range_locked(CPUTLBEntryFull *full, CPUTLBEntry *ent,
+static void tlb_reset_dirty_range_locked(CPUTLBEntry *tlb_entry,
                                         uintptr_t start, uintptr_t length)
 {
-    const uintptr_t addr = ent->addr_write;
-    int flags = addr | full->slow_flags[MMU_DATA_STORE];
+    uintptr_t addr = tlb_entry->addr_write;

-    flags &= TLB_INVALID_MASK | TLB_MMIO | TLB_DISCARD_WRITE | TLB_NOTDIRTY;
-    if (flags == 0) {
-        uintptr_t host = (addr & TARGET_PAGE_MASK) + ent->addend;
-        if ((host - start) < length) {
-            qatomic_set(&ent->addr_write, addr | TLB_NOTDIRTY);
+    if ((addr & (TLB_INVALID_MASK | TLB_MMIO |
+                 TLB_DISCARD_WRITE | TLB_NOTDIRTY)) == 0) {
+        addr &= TARGET_PAGE_MASK;
+        addr += tlb_entry->addend;
+        if ((addr - start) < length) {
+#if TARGET_LONG_BITS == 32
+            uint32_t *ptr_write = (uint32_t *)&tlb_entry->addr_write;
+            ptr_write += HOST_BIG_ENDIAN;
+            qatomic_set(ptr_write, *ptr_write | TLB_NOTDIRTY);
+#elif TCG_OVERSIZED_GUEST
+            tlb_entry->addr_write |= TLB_NOTDIRTY;
+#else
+            qatomic_set(&tlb_entry->addr_write,
+                        tlb_entry->addr_write | TLB_NOTDIRTY);
+#endif
        }
    }
 }
@@ -914,25 +931,23 @@ static inline void copy_tlb_helper_locked(CPUTLBEntry *d, const CPUTLBEntry *s)
 * We must take tlb_c.lock to avoid racing with another vCPU update. The only
 * thing actually updated is the target TLB entry ->addr_write flags.
 */
-void tlb_reset_dirty(CPUState *cpu, uintptr_t start, uintptr_t length)
+void tlb_reset_dirty(CPUState *cpu, ram_addr_t start1, ram_addr_t length)
 {
    int mmu_idx;

    qemu_spin_lock(&cpu->neg.tlb.c.lock);
    for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
-        CPUTLBDesc *desc = &cpu->neg.tlb.d[mmu_idx];
-        CPUTLBDescFast *fast = cpu_tlb_fast(cpu, mmu_idx);
-        unsigned int n = tlb_n_entries(fast);
        unsigned int i;
+        unsigned int n = tlb_n_entries(&cpu->neg.tlb.f[mmu_idx]);

        for (i = 0; i < n; i++) {
-            tlb_reset_dirty_range_locked(&desc->fulltlb[i], &fast->table[i],
-                                         start, length);
+            tlb_reset_dirty_range_locked(&cpu->neg.tlb.f[mmu_idx].table[i],
+                                         start1, length);
        }

        for (i = 0; i < CPU_VTLB_SIZE; i++) {
-            tlb_reset_dirty_range_locked(&desc->vfulltlb[i], &desc->vtable[i],
-                                         start, length);
+            tlb_reset_dirty_range_locked(&cpu->neg.tlb.d[mmu_idx].vtable[i],
+                                         start1, length);
        }
    }
    qemu_spin_unlock(&cpu->neg.tlb.c.lock);
@@ -1083,7 +1098,7 @@ void tlb_set_page_full(CPUState *cpu, int mmu_idx,
        if (prot & PAGE_WRITE) {
            if (section->readonly) {
                write_flags |= TLB_DISCARD_WRITE;
-            } else if (physical_memory_is_clean(iotlb)) {
+            } else if (cpu_physical_memory_is_clean(iotlb)) {
                write_flags |= TLB_NOTDIRTY;
            }
        }
@@ -1184,7 +1199,7 @@ void tlb_set_page_full(CPUState *cpu, int mmu_idx,

 void tlb_set_page_with_attrs(CPUState *cpu, vaddr addr,
                             hwaddr paddr, MemTxAttrs attrs, int prot,
-                             int mmu_idx, vaddr size)
+                             int mmu_idx, uint64_t size)
 {
    CPUTLBEntryFull full = {
        .phys_addr = paddr,
@@ -1199,65 +1214,29 @@ void tlb_set_page_with_attrs(CPUState *cpu, vaddr addr,

 void tlb_set_page(CPUState *cpu, vaddr addr,
                  hwaddr paddr, int prot,
-                  int mmu_idx, vaddr size)
+                  int mmu_idx, uint64_t size)
 {
    tlb_set_page_with_attrs(cpu, addr, paddr, MEMTXATTRS_UNSPECIFIED,
                            prot, mmu_idx, size);
 }

-/**
- * tlb_hit_page: return true if page aligned @addr is a hit against the
- * TLB entry @tlb_addr
- *
- * @addr: virtual address to test (must be page aligned)
- * @tlb_addr: TLB entry address (a CPUTLBEntry addr_read/write/code value)
- */
-static inline bool tlb_hit_page(uint64_t tlb_addr, vaddr addr)
-{
-    return addr == (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK));
-}
-
-/**
- * tlb_hit: return true if @addr is a hit against the TLB entry @tlb_addr
- *
- * @addr: virtual address to test (need not be page aligned)
- * @tlb_addr: TLB entry address (a CPUTLBEntry addr_read/write/code value)
- */
-static inline bool tlb_hit(uint64_t tlb_addr, vaddr addr)
-{
-    return tlb_hit_page(tlb_addr, addr & TARGET_PAGE_MASK);
-}
-
 /*
- * Note: tlb_fill_align() can trigger a resize of the TLB.
- * This means that all of the caller's prior references to the TLB table
- * (e.g. CPUTLBEntry pointers) must be discarded and looked up again
- * (e.g. via tlb_entry()).
+ * Note: tlb_fill() can trigger a resize of the TLB. This means that all of the
+ * caller's prior references to the TLB table (e.g. CPUTLBEntry pointers) must
+ * be discarded and looked up again (e.g. via tlb_entry()).
 */
-static bool tlb_fill_align(CPUState *cpu, vaddr addr, MMUAccessType type,
-                           int mmu_idx, MemOp memop, int size,
-                           bool probe, uintptr_t ra)
+static void tlb_fill(CPUState *cpu, vaddr addr, int size,
+                     MMUAccessType access_type, int mmu_idx, uintptr_t retaddr)
 {
-    const TCGCPUOps *ops = cpu->cc->tcg_ops;
-    CPUTLBEntryFull full;
+    bool ok;

-    if (ops->tlb_fill_align) {
-        if (ops->tlb_fill_align(cpu, &full, addr, type, mmu_idx,
-                                memop, size, probe, ra)) {
-            tlb_set_page_full(cpu, mmu_idx, addr, &full);
-            return true;
-        }
-    } else {
-        /* Legacy behaviour is alignment before paging. */
-        if (addr & ((1u << memop_alignment_bits(memop)) - 1)) {
-            ops->do_unaligned_access(cpu, addr, type, mmu_idx, ra);
-        }
-        if (ops->tlb_fill(cpu, addr, size, type, mmu_idx, probe, ra)) {
-            return true;
-        }
-    }
-    assert(probe);
-    return false;
+    /*
+     * This is not a probe, so only valid return is success; failure
+     * should result in exception + longjmp to the cpu loop.
+     */
+    ok = cpu->cc->tcg_ops->tlb_fill(cpu, addr, size,
+                                    access_type, mmu_idx, false, retaddr);
+    assert(ok);
 }

 static inline void cpu_unaligned_access(CPUState *cpu, vaddr addr,
@@ -1314,7 +1293,7 @@ static bool victim_tlb_hit(CPUState *cpu, size_t mmu_idx, size_t index,

        if (cmp == page) {
            /* Found entry in victim tlb, swap tlb and iotlb.  */
-            CPUTLBEntry tmptlb, *tlb = &cpu_tlb_fast(cpu, mmu_idx)->table[index];
+            CPUTLBEntry tmptlb, *tlb = &cpu->neg.tlb.f[mmu_idx].table[index];

            qemu_spin_lock(&cpu->neg.tlb.c.lock);
            copy_tlb_helper_locked(&tmptlb, tlb);
@@ -1339,18 +1318,18 @@ static void notdirty_write(CPUState *cpu, vaddr mem_vaddr, unsigned size,

    trace_memory_notdirty_write_access(mem_vaddr, ram_addr, size);

-    if (!physical_memory_get_dirty_flag(ram_addr, DIRTY_MEMORY_CODE)) {
-        tb_invalidate_phys_range_fast(cpu, ram_addr, size, retaddr);
+    if (!cpu_physical_memory_get_dirty_flag(ram_addr, DIRTY_MEMORY_CODE)) {
+        tb_invalidate_phys_range_fast(ram_addr, size, retaddr);
    }

    /*
     * Set both VGA and migration bits for simplicity and to remove
     * the notdirty callback faster.
     */
-    physical_memory_set_dirty_range(ram_addr, size, DIRTY_CLIENTS_NOCODE);
+    cpu_physical_memory_set_dirty_range(ram_addr, size, DIRTY_CLIENTS_NOCODE);

    /* We remove the notdirty callback only if the code has been flushed. */
-    if (!physical_memory_is_clean(ram_addr)) {
+    if (!cpu_physical_memory_is_clean(ram_addr)) {
        trace_memory_notdirty_set_dirty(mem_vaddr);
        tlb_set_dirty(cpu, mem_vaddr);
    }
@@ -1372,22 +1351,22 @@ static int probe_access_internal(CPUState *cpu, vaddr addr,

    if (!tlb_hit_page(tlb_addr, page_addr)) {
        if (!victim_tlb_hit(cpu, mmu_idx, index, access_type, page_addr)) {
-            if (!tlb_fill_align(cpu, addr, access_type, mmu_idx,
-                                0, fault_size, nonfault, retaddr)) {
+            if (!cpu->cc->tcg_ops->tlb_fill(cpu, addr, fault_size, access_type,
+                                            mmu_idx, nonfault, retaddr)) {
                /* Non-faulting page table read failed.  */
                *phost = NULL;
                *pfull = NULL;
                return TLB_INVALID_MASK;
            }

-            /* TLB resize via tlb_fill_align may have moved the entry.  */
+            /* TLB resize via tlb_fill may have moved the entry.  */
            index = tlb_index(cpu, mmu_idx, addr);
            entry = tlb_entry(cpu, mmu_idx, addr);

            /*
             * With PAGE_WRITE_INV, we set TLB_INVALID_MASK immediately,
-             * to force the next access through tlb_fill_align.  We've just
-             * called tlb_fill_align, so we know that this entry *is* valid.
+             * to force the next access through tlb_fill.  We've just
+             * called tlb_fill, so we know that this entry *is* valid.
             */
            flags &= ~TLB_INVALID_MASK;
        }
@@ -1512,7 +1491,7 @@ void *probe_access(CPUArchState *env, vaddr addr, int size,
    return host;
 }

-void *tlb_vaddr_to_host(CPUArchState *env, vaddr addr,
+void *tlb_vaddr_to_host(CPUArchState *env, abi_ptr addr,
                        MMUAccessType access_type, int mmu_idx)
 {
    CPUTLBEntryFull *full;
@@ -1628,17 +1607,16 @@ typedef struct MMULookupLocals {
 * mmu_lookup1: translate one page
 * @cpu: generic cpu state
 * @data: lookup parameters
- * @memop: memory operation for the access, or 0
 * @mmu_idx: virtual address context
 * @access_type: load/store/code
 * @ra: return address into tcg generated code, or 0
 *
 * Resolve the translation for the one page at @data.addr, filling in
 * the rest of @data with the results.  If the translation fails,
- * tlb_fill_align will longjmp out.  Return true if the softmmu tlb for
+ * tlb_fill will longjmp out.  Return true if the softmmu tlb for
 * @mmu_idx may have resized.
 */
-static bool mmu_lookup1(CPUState *cpu, MMULookupPageData *data, MemOp memop,
+static bool mmu_lookup1(CPUState *cpu, MMULookupPageData *data,
                        int mmu_idx, MMUAccessType access_type, uintptr_t ra)
 {
    vaddr addr = data->addr;
@@ -1653,8 +1631,7 @@ static bool mmu_lookup1(CPUState *cpu, MMULookupPageData *data, MemOp memop,
    if (!tlb_hit(tlb_addr, addr)) {
        if (!victim_tlb_hit(cpu, mmu_idx, index, access_type,
                            addr & TARGET_PAGE_MASK)) {
-            tlb_fill_align(cpu, addr, access_type, mmu_idx,
-                           memop, data->size, false, ra);
+            tlb_fill(cpu, addr, data->size, access_type, mmu_idx, ra);
            maybe_resized = true;
            index = tlb_index(cpu, mmu_idx, addr);
            entry = tlb_entry(cpu, mmu_idx, addr);
@@ -1666,14 +1643,6 @@ static bool mmu_lookup1(CPUState *cpu, MMULookupPageData *data, MemOp memop,
    flags = tlb_addr & (TLB_FLAGS_MASK & ~TLB_FORCE_SLOW);
    flags |= full->slow_flags[access_type];

-    if (likely(!maybe_resized)) {
-        /* Alignment has not been checked by tlb_fill_align. */
-        int a_bits = memop_tlb_alignment_bits(memop, flags & TLB_CHECK_ALIGNED);
-        if (unlikely(addr & ((1 << a_bits) - 1))) {
-            cpu_unaligned_access(cpu, addr, access_type, mmu_idx, ra);
-        }
-    }
-
    data->full = full;
    data->flags = flags;
    /* Compute haddr speculatively; depending on flags it might be invalid. */
@@ -1730,8 +1699,8 @@ static void mmu_watch_or_dirty(CPUState *cpu, MMULookupPageData *data,
 static bool mmu_lookup(CPUState *cpu, vaddr addr, MemOpIdx oi,
                       uintptr_t ra, MMUAccessType type, MMULookupLocals *l)
 {
+    unsigned a_bits;
    bool crosspage;
-    vaddr last;
    int flags;

    l->memop = get_memop(oi);
@@ -1739,17 +1708,21 @@ static bool mmu_lookup(CPUState *cpu, vaddr addr, MemOpIdx oi,

    tcg_debug_assert(l->mmu_idx < NB_MMU_MODES);

+    /* Handle CPU specific unaligned behaviour */
+    a_bits = get_alignment_bits(l->memop);
+    if (addr & ((1 << a_bits) - 1)) {
+        cpu_unaligned_access(cpu, addr, type, l->mmu_idx, ra);
+    }
+
    l->page[0].addr = addr;
    l->page[0].size = memop_size(l->memop);
-    l->page[1].addr = 0;
+    l->page[1].addr = (addr + l->page[0].size - 1) & TARGET_PAGE_MASK;
    l->page[1].size = 0;
+    crosspage = (addr ^ l->page[1].addr) & TARGET_PAGE_MASK;

-    /* Lookup and recognize exceptions from the first page. */
-    mmu_lookup1(cpu, &l->page[0], l->memop, l->mmu_idx, type, ra);
-
-    last = addr + l->page[0].size - 1;
-    crosspage = (addr ^ last) & TARGET_PAGE_MASK;
    if (likely(!crosspage)) {
+        mmu_lookup1(cpu, &l->page[0], l->mmu_idx, type, ra);
+
        flags = l->page[0].flags;
        if (unlikely(flags & (TLB_WATCHPOINT | TLB_NOTDIRTY))) {
            mmu_watch_or_dirty(cpu, &l->page[0], type, ra);
@@ -1759,19 +1732,16 @@ static bool mmu_lookup(CPUState *cpu, vaddr addr, MemOpIdx oi,
        }
    } else {
        /* Finish compute of page crossing. */
-        vaddr addr1 = last & TARGET_PAGE_MASK;
-        int size0 = addr1 - addr;
+        int size0 = l->page[1].addr - addr;
        l->page[1].size = l->page[0].size - size0;
        l->page[0].size = size0;
-        l->page[1].addr = cpu->cc->tcg_ops->pointer_wrap(cpu, l->mmu_idx,
-                                                         addr1, addr);

        /*
-         * Lookup and recognize exceptions from the second page.
-         * If the lookup potentially resized the table, refresh the
-         * first CPUTLBEntryFull pointer.
+         * Lookup both pages, recognizing exceptions from either.  If the
+         * second lookup potentially resized, refresh first CPUTLBEntryFull.
         */
-        if (mmu_lookup1(cpu, &l->page[1], 0, l->mmu_idx, type, ra)) {
+        mmu_lookup1(cpu, &l->page[0], l->mmu_idx, type, ra);
+        if (mmu_lookup1(cpu, &l->page[1], l->mmu_idx, type, ra)) {
            uintptr_t index = tlb_index(cpu, l->mmu_idx, addr);
            l->page[0].full = &cpu->neg.tlb.d[l->mmu_idx].fulltlb[index];
        }
@@ -1790,6 +1760,31 @@ static bool mmu_lookup(CPUState *cpu, vaddr addr, MemOpIdx oi,
        tcg_debug_assert((flags & TLB_BSWAP) == 0);
    }

+    /*
+     * This alignment check differs from the one above, in that this is
+     * based on the atomicity of the operation. The intended use case is
+     * the ARM memory type field of each PTE, where access to pages with
+     * Device memory type require alignment.
+     */
+    if (unlikely(flags & TLB_CHECK_ALIGNED)) {
+        MemOp size = l->memop & MO_SIZE;
+
+        switch (l->memop & MO_ATOM_MASK) {
+        case MO_ATOM_NONE:
+            size = MO_8;
+            break;
+        case MO_ATOM_IFALIGN_PAIR:
+        case MO_ATOM_WITHIN16_PAIR:
+            size = size ? size - 1 : 0;
+            break;
+        default:
+            break;
+        }
+        if (addr & ((1 << size) - 1)) {
+            cpu_unaligned_access(cpu, addr, type, l->mmu_idx, ra);
+        }
+    }
+
    return crosspage;
 }

@@ -1802,18 +1797,34 @@ static void *atomic_mmu_lookup(CPUState *cpu, vaddr addr, MemOpIdx oi,
 {
    uintptr_t mmu_idx = get_mmuidx(oi);
    MemOp mop = get_memop(oi);
+    int a_bits = get_alignment_bits(mop);
    uintptr_t index;
    CPUTLBEntry *tlbe;
    vaddr tlb_addr;
    void *hostaddr;
    CPUTLBEntryFull *full;
-    bool did_tlb_fill = false;

    tcg_debug_assert(mmu_idx < NB_MMU_MODES);

    /* Adjust the given return address.  */
    retaddr -= GETPC_ADJ;

+    /* Enforce guest required alignment.  */
+    if (unlikely(a_bits > 0 && (addr & ((1 << a_bits) - 1)))) {
+        /* ??? Maybe indicate atomic op to cpu_unaligned_access */
+        cpu_unaligned_access(cpu, addr, MMU_DATA_STORE,
+                             mmu_idx, retaddr);
+    }
+
+    /* Enforce qemu required alignment.  */
+    if (unlikely(addr & (size - 1))) {
+        /* We get here if guest alignment was not requested,
+           or was not enforced by cpu_unaligned_access above.
+           We might widen the access and emulate, but for now
+           mark an exception and exit the cpu loop.  */
+        goto stop_the_world;
+    }
+
    index = tlb_index(cpu, mmu_idx, addr);
    tlbe = tlb_entry(cpu, mmu_idx, addr);

@@ -1822,9 +1833,8 @@ static void *atomic_mmu_lookup(CPUState *cpu, vaddr addr, MemOpIdx oi,
    if (!tlb_hit(tlb_addr, addr)) {
        if (!victim_tlb_hit(cpu, mmu_idx, index, MMU_DATA_STORE,
                            addr & TARGET_PAGE_MASK)) {
-            tlb_fill_align(cpu, addr, MMU_DATA_STORE, mmu_idx,
-                           mop, size, false, retaddr);
-            did_tlb_fill = true;
+            tlb_fill(cpu, addr, size,
+                     MMU_DATA_STORE, mmu_idx, retaddr);
            index = tlb_index(cpu, mmu_idx, addr);
            tlbe = tlb_entry(cpu, mmu_idx, addr);
        }
@@ -1838,38 +1848,17 @@ static void *atomic_mmu_lookup(CPUState *cpu, vaddr addr, MemOpIdx oi,
     * but addr_read will only be -1 if PAGE_READ was unset.
     */
    if (unlikely(tlbe->addr_read == -1)) {
-        tlb_fill_align(cpu, addr, MMU_DATA_LOAD, mmu_idx,
-                       0, size, false, retaddr);
+        tlb_fill(cpu, addr, size, MMU_DATA_LOAD, mmu_idx, retaddr);
        /*
         * Since we don't support reads and writes to different
         * addresses, and we do have the proper page loaded for
-         * write, this shouldn't ever return.
-         */
-        g_assert_not_reached();
-    }
-
-    /* Enforce guest required alignment, if not handled by tlb_fill_align. */
-    if (!did_tlb_fill && (addr & ((1 << memop_alignment_bits(mop)) - 1))) {
-        cpu_unaligned_access(cpu, addr, MMU_DATA_STORE, mmu_idx, retaddr);
-    }
-
-    /* Enforce qemu required alignment.  */
-    if (unlikely(addr & (size - 1))) {
-        /*
-         * We get here if guest alignment was not requested, or was not
-         * enforced by cpu_unaligned_access or tlb_fill_align above.
-         * We might widen the access and emulate, but for now
-         * mark an exception and exit the cpu loop.
+         * write, this shouldn't ever return.  But just in case,
+         * handle via stop-the-world.
         */
        goto stop_the_world;
    }
-
-    /* Finish collecting tlb flags for both read and write. */
-    full = &cpu->neg.tlb.d[mmu_idx].fulltlb[index];
+    /* Collect tlb flags for read. */
    tlb_addr |= tlbe->addr_read;
-    tlb_addr &= TLB_FLAGS_MASK & ~TLB_FORCE_SLOW;
-    tlb_addr |= full->slow_flags[MMU_DATA_STORE];
-    tlb_addr |= full->slow_flags[MMU_DATA_LOAD];

    /* Notice an IO access or a needs-MMU-lookup access */
    if (unlikely(tlb_addr & (TLB_MMIO | TLB_DISCARD_WRITE))) {
@@ -1879,12 +1868,13 @@ static void *atomic_mmu_lookup(CPUState *cpu, vaddr addr, MemOpIdx oi,
    }

    hostaddr = (void *)((uintptr_t)addr + tlbe->addend);
+    full = &cpu->neg.tlb.d[mmu_idx].fulltlb[index];

    if (unlikely(tlb_addr & TLB_NOTDIRTY)) {
        notdirty_write(cpu, addr, size, full, retaddr);
    }

-    if (unlikely(tlb_addr & TLB_WATCHPOINT)) {
+    if (unlikely(tlb_addr & TLB_FORCE_SLOW)) {
        int wp_flags = 0;

        if (full->slow_flags[MMU_DATA_STORE] & TLB_WATCHPOINT) {
@@ -1893,8 +1883,10 @@ static void *atomic_mmu_lookup(CPUState *cpu, vaddr addr, MemOpIdx oi,
        if (full->slow_flags[MMU_DATA_LOAD] & TLB_WATCHPOINT) {
            wp_flags |= BP_MEM_READ;
        }
-        cpu_check_watchpoint(cpu, addr, size,
-                             full->attrs, wp_flags, retaddr);
+        if (wp_flags) {
+            cpu_check_watchpoint(cpu, addr, size,
+                                 full->attrs, wp_flags, retaddr);
+        }
    }

    return hostaddr;
@@ -2321,7 +2313,7 @@ static uint8_t do_ld1_mmu(CPUState *cpu, vaddr addr, MemOpIdx oi,
    MMULookupLocals l;
    bool crosspage;

-    cpu_req_mo(cpu, TCG_MO_LD_LD | TCG_MO_ST_LD);
+    cpu_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
    crosspage = mmu_lookup(cpu, addr, oi, ra, access_type, &l);
    tcg_debug_assert(!crosspage);

@@ -2336,7 +2328,7 @@ static uint16_t do_ld2_mmu(CPUState *cpu, vaddr addr, MemOpIdx oi,
    uint16_t ret;
    uint8_t a, b;

-    cpu_req_mo(cpu, TCG_MO_LD_LD | TCG_MO_ST_LD);
+    cpu_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
    crosspage = mmu_lookup(cpu, addr, oi, ra, access_type, &l);
    if (likely(!crosspage)) {
        return do_ld_2(cpu, &l.page[0], l.mmu_idx, access_type, l.memop, ra);
@@ -2360,7 +2352,7 @@ static uint32_t do_ld4_mmu(CPUState *cpu, vaddr addr, MemOpIdx oi,
    bool crosspage;
    uint32_t ret;

-    cpu_req_mo(cpu, TCG_MO_LD_LD | TCG_MO_ST_LD);
+    cpu_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
    crosspage = mmu_lookup(cpu, addr, oi, ra, access_type, &l);
    if (likely(!crosspage)) {
        return do_ld_4(cpu, &l.page[0], l.mmu_idx, access_type, l.memop, ra);
@@ -2381,7 +2373,7 @@ static uint64_t do_ld8_mmu(CPUState *cpu, vaddr addr, MemOpIdx oi,
    bool crosspage;
    uint64_t ret;

-    cpu_req_mo(cpu, TCG_MO_LD_LD | TCG_MO_ST_LD);
+    cpu_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
    crosspage = mmu_lookup(cpu, addr, oi, ra, access_type, &l);
    if (likely(!crosspage)) {
        return do_ld_8(cpu, &l.page[0], l.mmu_idx, access_type, l.memop, ra);
@@ -2404,7 +2396,7 @@ static Int128 do_ld16_mmu(CPUState *cpu, vaddr addr,
    Int128 ret;
    int first;

-    cpu_req_mo(cpu, TCG_MO_LD_LD | TCG_MO_ST_LD);
+    cpu_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
    crosspage = mmu_lookup(cpu, addr, oi, ra, MMU_DATA_LOAD, &l);
    if (likely(!crosspage)) {
        if (unlikely(l.page[0].flags & TLB_MMIO)) {
@@ -2732,7 +2724,7 @@ static void do_st1_mmu(CPUState *cpu, vaddr addr, uint8_t val,
    MMULookupLocals l;
    bool crosspage;

-    cpu_req_mo(cpu, TCG_MO_LD_ST | TCG_MO_ST_ST);
+    cpu_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
    crosspage = mmu_lookup(cpu, addr, oi, ra, MMU_DATA_STORE, &l);
    tcg_debug_assert(!crosspage);

@@ -2746,7 +2738,7 @@ static void do_st2_mmu(CPUState *cpu, vaddr addr, uint16_t val,
    bool crosspage;
    uint8_t a, b;

-    cpu_req_mo(cpu, TCG_MO_LD_ST | TCG_MO_ST_ST);
+    cpu_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
    crosspage = mmu_lookup(cpu, addr, oi, ra, MMU_DATA_STORE, &l);
    if (likely(!crosspage)) {
        do_st_2(cpu, &l.page[0], val, l.mmu_idx, l.memop, ra);
@@ -2768,7 +2760,7 @@ static void do_st4_mmu(CPUState *cpu, vaddr addr, uint32_t val,
    MMULookupLocals l;
    bool crosspage;

-    cpu_req_mo(cpu, TCG_MO_LD_ST | TCG_MO_ST_ST);
+    cpu_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
    crosspage = mmu_lookup(cpu, addr, oi, ra, MMU_DATA_STORE, &l);
    if (likely(!crosspage)) {
        do_st_4(cpu, &l.page[0], val, l.mmu_idx, l.memop, ra);
@@ -2789,7 +2781,7 @@ static void do_st8_mmu(CPUState *cpu, vaddr addr, uint64_t val,
    MMULookupLocals l;
    bool crosspage;

-    cpu_req_mo(cpu, TCG_MO_LD_ST | TCG_MO_ST_ST);
+    cpu_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
    crosspage = mmu_lookup(cpu, addr, oi, ra, MMU_DATA_STORE, &l);
    if (likely(!crosspage)) {
        do_st_8(cpu, &l.page[0], val, l.mmu_idx, l.memop, ra);
@@ -2812,7 +2804,7 @@ static void do_st16_mmu(CPUState *cpu, vaddr addr, Int128 val,
    uint64_t a, b;
    int first;

-    cpu_req_mo(cpu, TCG_MO_LD_ST | TCG_MO_ST_ST);
+    cpu_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
    crosspage = mmu_lookup(cpu, addr, oi, ra, MMU_DATA_STORE, &l);
    if (likely(!crosspage)) {
        if (unlikely(l.page[0].flags & TLB_MMIO)) {
@@ -2897,45 +2889,54 @@ static void do_st16_mmu(CPUState *cpu, vaddr addr, Int128 val,

 /* Code access functions.  */

-uint8_t cpu_ldb_code_mmu(CPUArchState *env, vaddr addr,
+uint32_t cpu_ldub_code(CPUArchState *env, abi_ptr addr)
+{
+    CPUState *cs = env_cpu(env);
+    MemOpIdx oi = make_memop_idx(MO_UB, cpu_mmu_index(cs, true));
+    return do_ld1_mmu(cs, addr, oi, 0, MMU_INST_FETCH);
+}
+
+uint32_t cpu_lduw_code(CPUArchState *env, abi_ptr addr)
+{
+    CPUState *cs = env_cpu(env);
+    MemOpIdx oi = make_memop_idx(MO_TEUW, cpu_mmu_index(cs, true));
+    return do_ld2_mmu(cs, addr, oi, 0, MMU_INST_FETCH);
+}
+
+uint32_t cpu_ldl_code(CPUArchState *env, abi_ptr addr)
+{
+    CPUState *cs = env_cpu(env);
+    MemOpIdx oi = make_memop_idx(MO_TEUL, cpu_mmu_index(cs, true));
+    return do_ld4_mmu(cs, addr, oi, 0, MMU_INST_FETCH);
+}
+
+uint64_t cpu_ldq_code(CPUArchState *env, abi_ptr addr)
+{
+    CPUState *cs = env_cpu(env);
+    MemOpIdx oi = make_memop_idx(MO_TEUQ, cpu_mmu_index(cs, true));
+    return do_ld8_mmu(cs, addr, oi, 0, MMU_INST_FETCH);
+}
+
+uint8_t cpu_ldb_code_mmu(CPUArchState *env, abi_ptr addr,
                         MemOpIdx oi, uintptr_t retaddr)
 {
    return do_ld1_mmu(env_cpu(env), addr, oi, retaddr, MMU_INST_FETCH);
 }

-uint16_t cpu_ldw_code_mmu(CPUArchState *env, vaddr addr,
+uint16_t cpu_ldw_code_mmu(CPUArchState *env, abi_ptr addr,
                          MemOpIdx oi, uintptr_t retaddr)
 {
    return do_ld2_mmu(env_cpu(env), addr, oi, retaddr, MMU_INST_FETCH);
 }

-uint32_t cpu_ldl_code_mmu(CPUArchState *env, vaddr addr,
+uint32_t cpu_ldl_code_mmu(CPUArchState *env, abi_ptr addr,
                          MemOpIdx oi, uintptr_t retaddr)
 {
    return do_ld4_mmu(env_cpu(env), addr, oi, retaddr, MMU_INST_FETCH);
 }

-uint64_t cpu_ldq_code_mmu(CPUArchState *env, vaddr addr,
+uint64_t cpu_ldq_code_mmu(CPUArchState *env, abi_ptr addr,
                          MemOpIdx oi, uintptr_t retaddr)
 {
    return do_ld8_mmu(env_cpu(env), addr, oi, retaddr, MMU_INST_FETCH);
 }
-
-/*
- * Common pointer_wrap implementations.
- */
-
-/*
- * To be used for strict alignment targets.
- * Because no accesses are unaligned, no accesses wrap either.
- */
-vaddr cpu_pointer_wrap_notreached(CPUState *cs, int idx, vaddr res, vaddr base)
-{
-    g_assert_not_reached();
-}
-
-/* To be used for strict 32-bit targets. */
-vaddr cpu_pointer_wrap_uint32(CPUState *cs, int idx, vaddr res, vaddr base)
-{
-    return (uint32_t)res;
-}
@@ -27,16 +27,17 @@
 #include "migration/vmstate.h"
 #include "qapi/error.h"
 #include "qemu/error-report.h"
-#include "system/cpus.h"
-#include "system/qtest.h"
+#include "sysemu/cpus.h"
+#include "sysemu/qtest.h"
 #include "qemu/main-loop.h"
 #include "qemu/option.h"
 #include "qemu/seqlock.h"
-#include "system/replay.h"
-#include "system/runstate.h"
+#include "sysemu/replay.h"
+#include "sysemu/runstate.h"
 #include "hw/core/cpu.h"
-#include "exec/icount.h"
-#include "system/cpu-timers-internal.h"
+#include "sysemu/cpu-timers.h"
+#include "sysemu/cpu-throttle.h"
+#include "sysemu/cpu-timers-internal.h"

 /*
 * ICOUNT: Instruction Counter
@@ -48,8 +49,6 @@ static bool icount_sleep = true;
 /* Arbitrarily pick 1MIPS as the minimum allowable speed.  */
 #define MAX_ICOUNT_SHIFT 10

-bool icount_align_option;
-
 /* Do not count executed instructions */
 ICountMode use_icount = ICOUNT_DISABLED;

@@ -11,16 +11,12 @@

 #include "exec/cpu-common.h"
 #include "exec/translation-block.h"
-#include "exec/mmap-lock.h"
-#include "accel/tcg/tb-cpu-state.h"

 extern int64_t max_delay;
 extern int64_t max_advance;

 extern bool one_insn_per_tb;

-extern bool icount_align_option;
-
 /*
 * Return true if CS is not running in parallel with other cpus, either
 * because there are no other cpus or we are within an exclusive context.
@@ -47,7 +43,9 @@ static inline bool cpu_plugin_mem_cbs_enabled(const CPUState *cpu)
 #endif
 }

-TranslationBlock *tb_gen_code(CPUState *cpu, TCGTBCPUState s);
+TranslationBlock *tb_gen_code(CPUState *cpu, vaddr pc,
+                              uint64_t cs_base, uint32_t flags,
+                              int cflags);
 void page_init(void);
 void tb_htable_init(void);
 void tb_reset_jump(TranslationBlock *tb, int n);
@@ -55,90 +53,7 @@ TranslationBlock *tb_link_page(TranslationBlock *tb);
 void cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb,
                               uintptr_t host_pc);

-/**
- * tlb_init - initialize a CPU's TLB
- * @cpu: CPU whose TLB should be initialized
- */
-void tlb_init(CPUState *cpu);
-/**
- * tlb_destroy - destroy a CPU's TLB
- * @cpu: CPU whose TLB should be destroyed
- */
-void tlb_destroy(CPUState *cpu);
-
 bool tcg_exec_realizefn(CPUState *cpu, Error **errp);
 void tcg_exec_unrealizefn(CPUState *cpu);

-/* current cflags for hashing/comparison */
-uint32_t curr_cflags(CPUState *cpu);
-
-void tb_check_watchpoint(CPUState *cpu, uintptr_t retaddr);
-
-/**
- * get_page_addr_code_hostp()
- * @env: CPUArchState
- * @addr: guest virtual address of guest code
- *
- * See get_page_addr_code() (full-system version) for documentation on the
- * return value.
- *
- * Sets *@hostp (when @hostp is non-NULL) as follows.
- * If the return value is -1, sets *@hostp to NULL. Otherwise, sets *@hostp
- * to the host address where @addr's content is kept.
- *
- * Note: this function can trigger an exception.
- */
-tb_page_addr_t get_page_addr_code_hostp(CPUArchState *env, vaddr addr,
-                                        void **hostp);
-
-/**
- * get_page_addr_code()
- * @env: CPUArchState
- * @addr: guest virtual address of guest code
- *
- * If we cannot translate and execute from the entire RAM page, or if
- * the region is not backed by RAM, returns -1. Otherwise, returns the
- * ram_addr_t corresponding to the guest code at @addr.
- *
- * Note: this function can trigger an exception.
- */
-static inline tb_page_addr_t get_page_addr_code(CPUArchState *env,
-                                                vaddr addr)
-{
-    return get_page_addr_code_hostp(env, addr, NULL);
-}
-
-/*
- * Access to the various translations structures need to be serialised
- * via locks for consistency.  In user-mode emulation access to the
- * memory related structures are protected with mmap_lock.
- * In !user-mode we use per-page locks.
- */
-#ifdef CONFIG_USER_ONLY
-#define assert_memory_lock() tcg_debug_assert(have_mmap_lock())
-#else
-#define assert_memory_lock()
-#endif
-
-#if defined(CONFIG_SOFTMMU) && defined(CONFIG_DEBUG_TCG)
-void assert_no_pages_locked(void);
-#else
-static inline void assert_no_pages_locked(void) { }
-#endif
-
-#ifdef CONFIG_USER_ONLY
-static inline void page_table_config_init(void) { }
-#else
-void page_table_config_init(void);
-#endif
-
-#ifndef CONFIG_USER_ONLY
-G_NORETURN void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr);
-#endif /* CONFIG_USER_ONLY */
-
-void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr);
-void tb_set_jmp_target(TranslationBlock *tb, int n, uintptr_t addr);
-
-void tcg_get_stats(AccelState *accel, GString *buf);
-
 #endif
@@ -0,0 +1,118 @@
+/*
+ * Internal execution defines for qemu (target specific)
+ *
+ *  Copyright (c) 2003 Fabrice Bellard
+ *
+ * SPDX-License-Identifier: LGPL-2.1-or-later
+ */
+
+#ifndef ACCEL_TCG_INTERNAL_TARGET_H
+#define ACCEL_TCG_INTERNAL_TARGET_H
+
+#include "exec/exec-all.h"
+#include "exec/translate-all.h"
+
+/*
+ * Access to the various translations structures need to be serialised
+ * via locks for consistency.  In user-mode emulation access to the
+ * memory related structures are protected with mmap_lock.
+ * In !user-mode we use per-page locks.
+ */
+#ifdef CONFIG_USER_ONLY
+#define assert_memory_lock() tcg_debug_assert(have_mmap_lock())
+#else
+#define assert_memory_lock()
+#endif
+
+#if defined(CONFIG_SOFTMMU) && defined(CONFIG_DEBUG_TCG)
+void assert_no_pages_locked(void);
+#else
+static inline void assert_no_pages_locked(void) { }
+#endif
+
+#ifdef CONFIG_USER_ONLY
+static inline void page_table_config_init(void) { }
+#else
+void page_table_config_init(void);
+#endif
+
+#ifdef CONFIG_USER_ONLY
+/*
+ * For user-only, page_protect sets the page read-only.
+ * Since most execution is already on read-only pages, and we'd need to
+ * account for other TBs on the same page, defer undoing any page protection
+ * until we receive the write fault.
+ */
+static inline void tb_lock_page0(tb_page_addr_t p0)
+{
+    page_protect(p0);
+}
+
+static inline void tb_lock_page1(tb_page_addr_t p0, tb_page_addr_t p1)
+{
+    page_protect(p1);
+}
+
+static inline void tb_unlock_page1(tb_page_addr_t p0, tb_page_addr_t p1) { }
+static inline void tb_unlock_pages(TranslationBlock *tb) { }
+#else
+void tb_lock_page0(tb_page_addr_t);
+void tb_lock_page1(tb_page_addr_t, tb_page_addr_t);
+void tb_unlock_page1(tb_page_addr_t, tb_page_addr_t);
+void tb_unlock_pages(TranslationBlock *);
+#endif
+
+#ifdef CONFIG_SOFTMMU
+void tb_invalidate_phys_range_fast(ram_addr_t ram_addr,
+                                   unsigned size,
+                                   uintptr_t retaddr);
+G_NORETURN void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr);
+#endif /* CONFIG_SOFTMMU */
+
+bool tb_invalidate_phys_page_unwind(tb_page_addr_t addr, uintptr_t pc);
+
+/* Return the current PC from CPU, which may be cached in TB. */
+static inline vaddr log_pc(CPUState *cpu, const TranslationBlock *tb)
+{
+    if (tb_cflags(tb) & CF_PCREL) {
+        return cpu->cc->get_pc(cpu);
+    } else {
+        return tb->pc;
+    }
+}
+
+/**
+ * tcg_req_mo:
+ * @type: TCGBar
+ *
+ * Filter @type to the barrier that is required for the guest
+ * memory ordering vs the host memory ordering.  A non-zero
+ * result indicates that some barrier is required.
+ *
+ * If TCG_GUEST_DEFAULT_MO is not defined, assume that the
+ * guest requires strict ordering.
+ *
+ * This is a macro so that it's constant even without optimization.
+ */
+#ifdef TCG_GUEST_DEFAULT_MO
+# define tcg_req_mo(type) \
+    ((type) & TCG_GUEST_DEFAULT_MO & ~TCG_TARGET_DEFAULT_MO)
+#else
+# define tcg_req_mo(type) ((type) & ~TCG_TARGET_DEFAULT_MO)
+#endif
+
+/**
+ * cpu_req_mo:
+ * @type: TCGBar
+ *
+ * If tcg_req_mo indicates a barrier for @type is required
+ * for the guest memory model, issue a host memory barrier.
+ */
+#define cpu_req_mo(type)          \
+    do {                          \
+        if (tcg_req_mo(type)) {   \
+            smp_mb();             \
+        }                         \
+    } while (0)
+
+#endif /* ACCEL_TCG_INTERNAL_H */
@@ -168,7 +168,6 @@ static uint64_t load_atomic8_or_exit(CPUState *cpu, uintptr_t ra, void *pv)
 #endif

    /* Ultimate fallback: re-execute in serial context. */
-    trace_load_atom8_or_exit_fallback(ra);
    cpu_loop_exit_atomic(cpu, ra);
 }

@@ -213,7 +212,6 @@ static Int128 load_atomic16_or_exit(CPUState *cpu, uintptr_t ra, void *pv)
    }

    /* Ultimate fallback: re-execute in serial context. */
-    trace_load_atom16_or_exit_fallback(ra);
    cpu_loop_exit_atomic(cpu, ra);
 }

@@ -521,7 +519,6 @@ static uint64_t load_atom_8(CPUState *cpu, uintptr_t ra,
        if (HAVE_al8) {
            return load_atom_extract_al8x2(pv);
        }
-        trace_load_atom8_fallback(memop, ra);
        cpu_loop_exit_atomic(cpu, ra);
    default:
        g_assert_not_reached();
@@ -566,7 +563,6 @@ static Int128 load_atom_16(CPUState *cpu, uintptr_t ra,
        break;
    case MO_64:
        if (!HAVE_al8) {
-            trace_load_atom16_fallback(memop, ra);
            cpu_loop_exit_atomic(cpu, ra);
        }
        a = load_atomic8(pv);
@@ -574,7 +570,6 @@ static Int128 load_atom_16(CPUState *cpu, uintptr_t ra,
        break;
    case -MO_64:
        if (!HAVE_al8) {
-            trace_load_atom16_fallback(memop, ra);
            cpu_loop_exit_atomic(cpu, ra);
        }
        a = load_atom_extract_al8x2(pv);
@@ -902,7 +897,6 @@ static void store_atom_2(CPUState *cpu, uintptr_t ra,
        g_assert_not_reached();
    }

-    trace_store_atom2_fallback(memop, ra);
    cpu_loop_exit_atomic(cpu, ra);
 }

@@ -967,7 +961,6 @@ static void store_atom_4(CPUState *cpu, uintptr_t ra,
                return;
            }
        }
-        trace_store_atom4_fallback(memop, ra);
        cpu_loop_exit_atomic(cpu, ra);
    default:
        g_assert_not_reached();
@@ -1036,7 +1029,6 @@ static void store_atom_8(CPUState *cpu, uintptr_t ra,
    default:
        g_assert_not_reached();
    }
-    trace_store_atom8_fallback(memop, ra);
    cpu_loop_exit_atomic(cpu, ra);
 }

@@ -1115,6 +1107,5 @@ static void store_atom_16(CPUState *cpu, uintptr_t ra,
    default:
        g_assert_not_reached();
    }
-    trace_store_atom16_fallback(memop, ra);
    cpu_loop_exit_atomic(cpu, ra);
 }
@@ -123,69 +123,64 @@ void helper_st_i128(CPUArchState *env, uint64_t addr, Int128 val, MemOpIdx oi)
 * Load helpers for cpu_ldst.h
 */

-static void plugin_load_cb(CPUArchState *env, vaddr addr,
-                           uint64_t value_low,
-                           uint64_t value_high,
-                           MemOpIdx oi)
+static void plugin_load_cb(CPUArchState *env, abi_ptr addr, MemOpIdx oi)
 {
    if (cpu_plugin_mem_cbs_enabled(env_cpu(env))) {
-        qemu_plugin_vcpu_mem_cb(env_cpu(env), addr,
-                                value_low, value_high,
-                                oi, QEMU_PLUGIN_MEM_R);
+        qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_R);
    }
 }

-uint8_t cpu_ldb_mmu(CPUArchState *env, vaddr addr, MemOpIdx oi, uintptr_t ra)
+uint8_t cpu_ldb_mmu(CPUArchState *env, abi_ptr addr, MemOpIdx oi, uintptr_t ra)
 {
    uint8_t ret;

    tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_UB);
    ret = do_ld1_mmu(env_cpu(env), addr, oi, ra, MMU_DATA_LOAD);
-    plugin_load_cb(env, addr, ret, 0, oi);
+    plugin_load_cb(env, addr, oi);
    return ret;
 }

-uint16_t cpu_ldw_mmu(CPUArchState *env, vaddr addr,
+uint16_t cpu_ldw_mmu(CPUArchState *env, abi_ptr addr,
                     MemOpIdx oi, uintptr_t ra)
 {
    uint16_t ret;

    tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_16);
    ret = do_ld2_mmu(env_cpu(env), addr, oi, ra, MMU_DATA_LOAD);
-    plugin_load_cb(env, addr, ret, 0, oi);
+    plugin_load_cb(env, addr, oi);
    return ret;
 }

-uint32_t cpu_ldl_mmu(CPUArchState *env, vaddr addr,
+uint32_t cpu_ldl_mmu(CPUArchState *env, abi_ptr addr,
                     MemOpIdx oi, uintptr_t ra)
 {
    uint32_t ret;

    tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_32);
    ret = do_ld4_mmu(env_cpu(env), addr, oi, ra, MMU_DATA_LOAD);
-    plugin_load_cb(env, addr, ret, 0, oi);
+    plugin_load_cb(env, addr, oi);
    return ret;
 }

-uint64_t cpu_ldq_mmu(CPUArchState *env, vaddr addr,
+uint64_t cpu_ldq_mmu(CPUArchState *env, abi_ptr addr,
                     MemOpIdx oi, uintptr_t ra)
 {
    uint64_t ret;

    tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_64);
    ret = do_ld8_mmu(env_cpu(env), addr, oi, ra, MMU_DATA_LOAD);
-    plugin_load_cb(env, addr, ret, 0, oi);
+    plugin_load_cb(env, addr, oi);
    return ret;
 }

-Int128 cpu_ld16_mmu(CPUArchState *env, vaddr addr,
+Int128 cpu_ld16_mmu(CPUArchState *env, abi_ptr addr,
                    MemOpIdx oi, uintptr_t ra)
 {
    Int128 ret;

    tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_128);
    ret = do_ld16_mmu(env_cpu(env), addr, oi, ra);
-    plugin_load_cb(env, addr, int128_getlo(ret), int128_gethi(ret), oi);
+    plugin_load_cb(env, addr, oi);
    return ret;
 }

@@ -193,53 +188,363 @@ Int128 cpu_ld16_mmu(CPUArchState *env, vaddr addr,
 * Store helpers for cpu_ldst.h
 */

-static void plugin_store_cb(CPUArchState *env, vaddr addr,
-                            uint64_t value_low,
-                            uint64_t value_high,
-                            MemOpIdx oi)
+static void plugin_store_cb(CPUArchState *env, abi_ptr addr, MemOpIdx oi)
 {
    if (cpu_plugin_mem_cbs_enabled(env_cpu(env))) {
-        qemu_plugin_vcpu_mem_cb(env_cpu(env), addr,
-                                value_low, value_high,
-                                oi, QEMU_PLUGIN_MEM_W);
+        qemu_plugin_vcpu_mem_cb(env_cpu(env), addr, oi, QEMU_PLUGIN_MEM_W);
    }
 }

-void cpu_stb_mmu(CPUArchState *env, vaddr addr, uint8_t val,
+void cpu_stb_mmu(CPUArchState *env, abi_ptr addr, uint8_t val,
                 MemOpIdx oi, uintptr_t retaddr)
 {
    helper_stb_mmu(env, addr, val, oi, retaddr);
-    plugin_store_cb(env, addr, val, 0, oi);
+    plugin_store_cb(env, addr, oi);
 }

-void cpu_stw_mmu(CPUArchState *env, vaddr addr, uint16_t val,
+void cpu_stw_mmu(CPUArchState *env, abi_ptr addr, uint16_t val,
                 MemOpIdx oi, uintptr_t retaddr)
 {
    tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_16);
    do_st2_mmu(env_cpu(env), addr, val, oi, retaddr);
-    plugin_store_cb(env, addr, val, 0, oi);
+    plugin_store_cb(env, addr, oi);
 }

-void cpu_stl_mmu(CPUArchState *env, vaddr addr, uint32_t val,
+void cpu_stl_mmu(CPUArchState *env, abi_ptr addr, uint32_t val,
                    MemOpIdx oi, uintptr_t retaddr)
 {
    tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_32);
    do_st4_mmu(env_cpu(env), addr, val, oi, retaddr);
-    plugin_store_cb(env, addr, val, 0, oi);
+    plugin_store_cb(env, addr, oi);
 }

-void cpu_stq_mmu(CPUArchState *env, vaddr addr, uint64_t val,
+void cpu_stq_mmu(CPUArchState *env, abi_ptr addr, uint64_t val,
                 MemOpIdx oi, uintptr_t retaddr)
 {
    tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_64);
    do_st8_mmu(env_cpu(env), addr, val, oi, retaddr);
-    plugin_store_cb(env, addr, val, 0, oi);
+    plugin_store_cb(env, addr, oi);
 }

-void cpu_st16_mmu(CPUArchState *env, vaddr addr, Int128 val,
+void cpu_st16_mmu(CPUArchState *env, abi_ptr addr, Int128 val,
                  MemOpIdx oi, uintptr_t retaddr)
 {
    tcg_debug_assert((get_memop(oi) & MO_SIZE) == MO_128);
    do_st16_mmu(env_cpu(env), addr, val, oi, retaddr);
-    plugin_store_cb(env, addr, int128_getlo(val), int128_gethi(val), oi);
+    plugin_store_cb(env, addr, oi);
+}
+
+/*
+ * Wrappers of the above
+ */
+
+uint32_t cpu_ldub_mmuidx_ra(CPUArchState *env, abi_ptr addr,
+                            int mmu_idx, uintptr_t ra)
+{
+    MemOpIdx oi = make_memop_idx(MO_UB, mmu_idx);
+    return cpu_ldb_mmu(env, addr, oi, ra);
+}
+
+int cpu_ldsb_mmuidx_ra(CPUArchState *env, abi_ptr addr,
+                       int mmu_idx, uintptr_t ra)
+{
+    return (int8_t)cpu_ldub_mmuidx_ra(env, addr, mmu_idx, ra);
+}
+
+uint32_t cpu_lduw_be_mmuidx_ra(CPUArchState *env, abi_ptr addr,
+                               int mmu_idx, uintptr_t ra)
+{
+    MemOpIdx oi = make_memop_idx(MO_BEUW | MO_UNALN, mmu_idx);
+    return cpu_ldw_mmu(env, addr, oi, ra);
+}
+
+int cpu_ldsw_be_mmuidx_ra(CPUArchState *env, abi_ptr addr,
+                          int mmu_idx, uintptr_t ra)
+{
+    return (int16_t)cpu_lduw_be_mmuidx_ra(env, addr, mmu_idx, ra);
+}
+
+uint32_t cpu_ldl_be_mmuidx_ra(CPUArchState *env, abi_ptr addr,
+                              int mmu_idx, uintptr_t ra)
+{
+    MemOpIdx oi = make_memop_idx(MO_BEUL | MO_UNALN, mmu_idx);
+    return cpu_ldl_mmu(env, addr, oi, ra);
+}
+
+uint64_t cpu_ldq_be_mmuidx_ra(CPUArchState *env, abi_ptr addr,
+                              int mmu_idx, uintptr_t ra)
+{
+    MemOpIdx oi = make_memop_idx(MO_BEUQ | MO_UNALN, mmu_idx);
+    return cpu_ldq_mmu(env, addr, oi, ra);
+}
+
+uint32_t cpu_lduw_le_mmuidx_ra(CPUArchState *env, abi_ptr addr,
+                               int mmu_idx, uintptr_t ra)
+{
+    MemOpIdx oi = make_memop_idx(MO_LEUW | MO_UNALN, mmu_idx);
+    return cpu_ldw_mmu(env, addr, oi, ra);
+}
+
+int cpu_ldsw_le_mmuidx_ra(CPUArchState *env, abi_ptr addr,
+                          int mmu_idx, uintptr_t ra)
+{
+    return (int16_t)cpu_lduw_le_mmuidx_ra(env, addr, mmu_idx, ra);
+}
+
+uint32_t cpu_ldl_le_mmuidx_ra(CPUArchState *env, abi_ptr addr,
+                              int mmu_idx, uintptr_t ra)
+{
+    MemOpIdx oi = make_memop_idx(MO_LEUL | MO_UNALN, mmu_idx);
+    return cpu_ldl_mmu(env, addr, oi, ra);
+}
+
+uint64_t cpu_ldq_le_mmuidx_ra(CPUArchState *env, abi_ptr addr,
+                              int mmu_idx, uintptr_t ra)
+{
+    MemOpIdx oi = make_memop_idx(MO_LEUQ | MO_UNALN, mmu_idx);
+    return cpu_ldq_mmu(env, addr, oi, ra);
+}
+
+void cpu_stb_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint32_t val,
+                       int mmu_idx, uintptr_t ra)
+{
+    MemOpIdx oi = make_memop_idx(MO_UB, mmu_idx);
+    cpu_stb_mmu(env, addr, val, oi, ra);
+}
+
+void cpu_stw_be_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint32_t val,
+                          int mmu_idx, uintptr_t ra)
+{
+    MemOpIdx oi = make_memop_idx(MO_BEUW | MO_UNALN, mmu_idx);
+    cpu_stw_mmu(env, addr, val, oi, ra);
+}
+
+void cpu_stl_be_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint32_t val,
+                          int mmu_idx, uintptr_t ra)
+{
+    MemOpIdx oi = make_memop_idx(MO_BEUL | MO_UNALN, mmu_idx);
+    cpu_stl_mmu(env, addr, val, oi, ra);
+}
+
+void cpu_stq_be_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint64_t val,
+                          int mmu_idx, uintptr_t ra)
+{
+    MemOpIdx oi = make_memop_idx(MO_BEUQ | MO_UNALN, mmu_idx);
+    cpu_stq_mmu(env, addr, val, oi, ra);
+}
+
+void cpu_stw_le_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint32_t val,
+                          int mmu_idx, uintptr_t ra)
+{
+    MemOpIdx oi = make_memop_idx(MO_LEUW | MO_UNALN, mmu_idx);
+    cpu_stw_mmu(env, addr, val, oi, ra);
+}
+
+void cpu_stl_le_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint32_t val,
+                          int mmu_idx, uintptr_t ra)
+{
+    MemOpIdx oi = make_memop_idx(MO_LEUL | MO_UNALN, mmu_idx);
+    cpu_stl_mmu(env, addr, val, oi, ra);
+}
+
+void cpu_stq_le_mmuidx_ra(CPUArchState *env, abi_ptr addr, uint64_t val,
+                          int mmu_idx, uintptr_t ra)
+{
+    MemOpIdx oi = make_memop_idx(MO_LEUQ | MO_UNALN, mmu_idx);
+    cpu_stq_mmu(env, addr, val, oi, ra);
+}
+
+/*--------------------------*/
+
+uint32_t cpu_ldub_data_ra(CPUArchState *env, abi_ptr addr, uintptr_t ra)
+{
+    int mmu_index = cpu_mmu_index(env_cpu(env), false);
+    return cpu_ldub_mmuidx_ra(env, addr, mmu_index, ra);
+}
+
+int cpu_ldsb_data_ra(CPUArchState *env, abi_ptr addr, uintptr_t ra)
+{
+    return (int8_t)cpu_ldub_data_ra(env, addr, ra);
+}
+
+uint32_t cpu_lduw_be_data_ra(CPUArchState *env, abi_ptr addr, uintptr_t ra)
+{
+    int mmu_index = cpu_mmu_index(env_cpu(env), false);
+    return cpu_lduw_be_mmuidx_ra(env, addr, mmu_index, ra);
+}
+
+int cpu_ldsw_be_data_ra(CPUArchState *env, abi_ptr addr, uintptr_t ra)
+{
+    return (int16_t)cpu_lduw_be_data_ra(env, addr, ra);
+}
+
+uint32_t cpu_ldl_be_data_ra(CPUArchState *env, abi_ptr addr, uintptr_t ra)
+{
+    int mmu_index = cpu_mmu_index(env_cpu(env), false);
+    return cpu_ldl_be_mmuidx_ra(env, addr, mmu_index, ra);
+}
+
+uint64_t cpu_ldq_be_data_ra(CPUArchState *env, abi_ptr addr, uintptr_t ra)
+{
+    int mmu_index = cpu_mmu_index(env_cpu(env), false);
+    return cpu_ldq_be_mmuidx_ra(env, addr, mmu_index, ra);
+}
+
+uint32_t cpu_lduw_le_data_ra(CPUArchState *env, abi_ptr addr, uintptr_t ra)
+{
+    int mmu_index = cpu_mmu_index(env_cpu(env), false);
+    return cpu_lduw_le_mmuidx_ra(env, addr, mmu_index, ra);
+}
+
+int cpu_ldsw_le_data_ra(CPUArchState *env, abi_ptr addr, uintptr_t ra)
+{
+    return (int16_t)cpu_lduw_le_data_ra(env, addr, ra);
+}
+
+uint32_t cpu_ldl_le_data_ra(CPUArchState *env, abi_ptr addr, uintptr_t ra)
+{
+    int mmu_index = cpu_mmu_index(env_cpu(env), false);
+    return cpu_ldl_le_mmuidx_ra(env, addr, mmu_index, ra);
+}
+
+uint64_t cpu_ldq_le_data_ra(CPUArchState *env, abi_ptr addr, uintptr_t ra)
+{
+    int mmu_index = cpu_mmu_index(env_cpu(env), false);
+    return cpu_ldq_le_mmuidx_ra(env, addr, mmu_index, ra);
+}
+
+void cpu_stb_data_ra(CPUArchState *env, abi_ptr addr,
+                     uint32_t val, uintptr_t ra)
+{
+    int mmu_index = cpu_mmu_index(env_cpu(env), false);
+    cpu_stb_mmuidx_ra(env, addr, val, mmu_index, ra);
+}
+
+void cpu_stw_be_data_ra(CPUArchState *env, abi_ptr addr,
+                        uint32_t val, uintptr_t ra)
+{
+    int mmu_index = cpu_mmu_index(env_cpu(env), false);
+    cpu_stw_be_mmuidx_ra(env, addr, val, mmu_index, ra);
+}
+
+void cpu_stl_be_data_ra(CPUArchState *env, abi_ptr addr,
+                        uint32_t val, uintptr_t ra)
+{
+    int mmu_index = cpu_mmu_index(env_cpu(env), false);
+    cpu_stl_be_mmuidx_ra(env, addr, val, mmu_index, ra);
+}
+
+void cpu_stq_be_data_ra(CPUArchState *env, abi_ptr addr,
+                        uint64_t val, uintptr_t ra)
+{
+    int mmu_index = cpu_mmu_index(env_cpu(env), false);
+    cpu_stq_be_mmuidx_ra(env, addr, val, mmu_index, ra);
+}
+
+void cpu_stw_le_data_ra(CPUArchState *env, abi_ptr addr,
+                        uint32_t val, uintptr_t ra)
+{
+    int mmu_index = cpu_mmu_index(env_cpu(env), false);
+    cpu_stw_le_mmuidx_ra(env, addr, val, mmu_index, ra);
+}
+
+void cpu_stl_le_data_ra(CPUArchState *env, abi_ptr addr,
+                        uint32_t val, uintptr_t ra)
+{
+    int mmu_index = cpu_mmu_index(env_cpu(env), false);
+    cpu_stl_le_mmuidx_ra(env, addr, val, mmu_index, ra);
+}
+
+void cpu_stq_le_data_ra(CPUArchState *env, abi_ptr addr,
+                        uint64_t val, uintptr_t ra)
+{
+    int mmu_index = cpu_mmu_index(env_cpu(env), false);
+    cpu_stq_le_mmuidx_ra(env, addr, val, mmu_index, ra);
+}
+
+/*--------------------------*/
+
+uint32_t cpu_ldub_data(CPUArchState *env, abi_ptr addr)
+{
+    return cpu_ldub_data_ra(env, addr, 0);
+}
+
+int cpu_ldsb_data(CPUArchState *env, abi_ptr addr)
+{
+    return (int8_t)cpu_ldub_data(env, addr);
+}
+
+uint32_t cpu_lduw_be_data(CPUArchState *env, abi_ptr addr)
+{
+    return cpu_lduw_be_data_ra(env, addr, 0);
+}
+
+int cpu_ldsw_be_data(CPUArchState *env, abi_ptr addr)
+{
+    return (int16_t)cpu_lduw_be_data(env, addr);
+}
+
+uint32_t cpu_ldl_be_data(CPUArchState *env, abi_ptr addr)
+{
+    return cpu_ldl_be_data_ra(env, addr, 0);
+}
+
+uint64_t cpu_ldq_be_data(CPUArchState *env, abi_ptr addr)
+{
+    return cpu_ldq_be_data_ra(env, addr, 0);
+}
+
+uint32_t cpu_lduw_le_data(CPUArchState *env, abi_ptr addr)
+{
+    return cpu_lduw_le_data_ra(env, addr, 0);
+}
+
+int cpu_ldsw_le_data(CPUArchState *env, abi_ptr addr)
+{
+    return (int16_t)cpu_lduw_le_data(env, addr);
+}
+
+uint32_t cpu_ldl_le_data(CPUArchState *env, abi_ptr addr)
+{
+    return cpu_ldl_le_data_ra(env, addr, 0);
+}
+
+uint64_t cpu_ldq_le_data(CPUArchState *env, abi_ptr addr)
+{
+    return cpu_ldq_le_data_ra(env, addr, 0);
+}
+
+void cpu_stb_data(CPUArchState *env, abi_ptr addr, uint32_t val)
+{
+    cpu_stb_data_ra(env, addr, val, 0);
+}
+
+void cpu_stw_be_data(CPUArchState *env, abi_ptr addr, uint32_t val)
+{
+    cpu_stw_be_data_ra(env, addr, val, 0);
+}
+
+void cpu_stl_be_data(CPUArchState *env, abi_ptr addr, uint32_t val)
+{
+    cpu_stl_be_data_ra(env, addr, val, 0);
+}
+
+void cpu_stq_be_data(CPUArchState *env, abi_ptr addr, uint64_t val)
+{
+    cpu_stq_be_data_ra(env, addr, val, 0);
+}
+
+void cpu_stw_le_data(CPUArchState *env, abi_ptr addr, uint32_t val)
+{
+    cpu_stw_le_data_ra(env, addr, val, 0);
+}
+
+void cpu_stl_le_data(CPUArchState *env, abi_ptr addr, uint32_t val)
+{
+    cpu_stl_le_data_ra(env, addr, val, 0);
+}
+
+void cpu_stq_le_data(CPUArchState *env, abi_ptr addr, uint64_t val)
+{
+    cpu_stq_le_data_ra(env, addr, val, 0);
 }
@@ -1,39 +1,36 @@
-if not have_tcg
-   subdir_done()
-endif
-
-tcg_ss = ss.source_set()
-
-tcg_ss.add(files(
-  'cpu-exec.c',
+common_ss.add(when: 'CONFIG_TCG', if_true: files(
  'cpu-exec-common.c',
-  'tcg-runtime.c',
-  'tcg-runtime-gvec.c',
-  'tb-maint.c',
+))
+tcg_specific_ss = ss.source_set()
+tcg_specific_ss.add(files(
  'tcg-all.c',
-  'tcg-stats.c',
+  'cpu-exec.c',
+  'tb-maint.c',
+  'tcg-runtime-gvec.c',
+  'tcg-runtime.c',
  'translate-all.c',
  'translator.c',
 ))
+tcg_specific_ss.add(when: 'CONFIG_USER_ONLY', if_true: files('user-exec.c'))
+tcg_specific_ss.add(when: 'CONFIG_SYSTEM_ONLY', if_false: files('user-exec-stub.c'))
 if get_option('plugins')
-  tcg_ss.add(files('plugin-gen.c'))
+  tcg_specific_ss.add(files('plugin-gen.c'))
 endif
+specific_ss.add_all(when: 'CONFIG_TCG', if_true: tcg_specific_ss)

-user_ss.add_all(tcg_ss)
-system_ss.add_all(tcg_ss)
-
-user_ss.add(files(
-  'user-exec.c',
-  'user-exec-stub.c',
-))
-
-system_ss.add(files(
+specific_ss.add(when: ['CONFIG_SYSTEM_ONLY', 'CONFIG_TCG'], if_true: files(
  'cputlb.c',
-  'icount-common.c',
-  'monitor.c',
-  'tcg-accel-ops.c',
-  'tcg-accel-ops-icount.c',
-  'tcg-accel-ops-mttcg.c',
-  'tcg-accel-ops-rr.c',
  'watchpoint.c',
 ))
+
+system_ss.add(when: ['CONFIG_TCG'], if_true: files(
+  'icount-common.c',
+  'monitor.c',
+))
+
+tcg_module_ss.add(when: ['CONFIG_SYSTEM_ONLY', 'CONFIG_TCG'], if_true: files(
+  'tcg-accel-ops.c',
+  'tcg-accel-ops-mttcg.c',
+  'tcg-accel-ops-icount.c',
+  'tcg-accel-ops-rr.c',
+))
@@ -7,13 +7,197 @@
 */

 #include "qemu/osdep.h"
+#include "qemu/accel.h"
+#include "qemu/qht.h"
 #include "qapi/error.h"
 #include "qapi/type-helpers.h"
 #include "qapi/qapi-commands-machine.h"
 #include "monitor/monitor.h"
-#include "system/tcg.h"
+#include "sysemu/cpus.h"
+#include "sysemu/cpu-timers.h"
+#include "sysemu/tcg.h"
 #include "tcg/tcg.h"
 #include "internal-common.h"
+#include "tb-context.h"
+
+
+static void dump_drift_info(GString *buf)
+{
+    if (!icount_enabled()) {
+        return;
+    }
+
+    g_string_append_printf(buf, "Host - Guest clock  %"PRIi64" ms\n",
+                           (cpu_get_clock() - icount_get()) / SCALE_MS);
+    if (icount_align_option) {
+        g_string_append_printf(buf, "Max guest delay     %"PRIi64" ms\n",
+                               -max_delay / SCALE_MS);
+        g_string_append_printf(buf, "Max guest advance   %"PRIi64" ms\n",
+                               max_advance / SCALE_MS);
+    } else {
+        g_string_append_printf(buf, "Max guest delay     NA\n");
+        g_string_append_printf(buf, "Max guest advance   NA\n");
+    }
+}
+
+static void dump_accel_info(GString *buf)
+{
+    AccelState *accel = current_accel();
+    bool one_insn_per_tb = object_property_get_bool(OBJECT(accel),
+                                                    "one-insn-per-tb",
+                                                    &error_fatal);
+
+    g_string_append_printf(buf, "Accelerator settings:\n");
+    g_string_append_printf(buf, "one-insn-per-tb: %s\n\n",
+                           one_insn_per_tb ? "on" : "off");
+}
+
+static void print_qht_statistics(struct qht_stats hst, GString *buf)
+{
+    uint32_t hgram_opts;
+    size_t hgram_bins;
+    char *hgram;
+
+    if (!hst.head_buckets) {
+        return;
+    }
+    g_string_append_printf(buf, "TB hash buckets     %zu/%zu "
+                           "(%0.2f%% head buckets used)\n",
+                           hst.used_head_buckets, hst.head_buckets,
+                           (double)hst.used_head_buckets /
+                           hst.head_buckets * 100);
+
+    hgram_opts =  QDIST_PR_BORDER | QDIST_PR_LABELS;
+    hgram_opts |= QDIST_PR_100X   | QDIST_PR_PERCENT;
+    if (qdist_xmax(&hst.occupancy) - qdist_xmin(&hst.occupancy) == 1) {
+        hgram_opts |= QDIST_PR_NODECIMAL;
+    }
+    hgram = qdist_pr(&hst.occupancy, 10, hgram_opts);
+    g_string_append_printf(buf, "TB hash occupancy   %0.2f%% avg chain occ. "
+                           "Histogram: %s\n",
+                           qdist_avg(&hst.occupancy) * 100, hgram);
+    g_free(hgram);
+
+    hgram_opts = QDIST_PR_BORDER | QDIST_PR_LABELS;
+    hgram_bins = qdist_xmax(&hst.chain) - qdist_xmin(&hst.chain);
+    if (hgram_bins > 10) {
+        hgram_bins = 10;
+    } else {
+        hgram_bins = 0;
+        hgram_opts |= QDIST_PR_NODECIMAL | QDIST_PR_NOBINRANGE;
+    }
+    hgram = qdist_pr(&hst.chain, hgram_bins, hgram_opts);
+    g_string_append_printf(buf, "TB hash avg chain   %0.3f buckets. "
+                           "Histogram: %s\n",
+                           qdist_avg(&hst.chain), hgram);
+    g_free(hgram);
+}
+
+struct tb_tree_stats {
+    size_t nb_tbs;
+    size_t host_size;
+    size_t target_size;
+    size_t max_target_size;
+    size_t direct_jmp_count;
+    size_t direct_jmp2_count;
+    size_t cross_page;
+};
+
+static gboolean tb_tree_stats_iter(gpointer key, gpointer value, gpointer data)
+{
+    const TranslationBlock *tb = value;
+    struct tb_tree_stats *tst = data;
+
+    tst->nb_tbs++;
+    tst->host_size += tb->tc.size;
+    tst->target_size += tb->size;
+    if (tb->size > tst->max_target_size) {
+        tst->max_target_size = tb->size;
+    }
+    if (tb->page_addr[1] != -1) {
+        tst->cross_page++;
+    }
+    if (tb->jmp_reset_offset[0] != TB_JMP_OFFSET_INVALID) {
+        tst->direct_jmp_count++;
+        if (tb->jmp_reset_offset[1] != TB_JMP_OFFSET_INVALID) {
+            tst->direct_jmp2_count++;
+        }
+    }
+    return false;
+}
+
+static void tlb_flush_counts(size_t *pfull, size_t *ppart, size_t *pelide)
+{
+    CPUState *cpu;
+    size_t full = 0, part = 0, elide = 0;
+
+    CPU_FOREACH(cpu) {
+        full += qatomic_read(&cpu->neg.tlb.c.full_flush_count);
+        part += qatomic_read(&cpu->neg.tlb.c.part_flush_count);
+        elide += qatomic_read(&cpu->neg.tlb.c.elide_flush_count);
+    }
+    *pfull = full;
+    *ppart = part;
+    *pelide = elide;
+}
+
+static void tcg_dump_info(GString *buf)
+{
+    g_string_append_printf(buf, "[TCG profiler not compiled]\n");
+}
+
+static void dump_exec_info(GString *buf)
+{
+    struct tb_tree_stats tst = {};
+    struct qht_stats hst;
+    size_t nb_tbs, flush_full, flush_part, flush_elide;
+
+    tcg_tb_foreach(tb_tree_stats_iter, &tst);
+    nb_tbs = tst.nb_tbs;
+    /* XXX: avoid using doubles ? */
+    g_string_append_printf(buf, "Translation buffer state:\n");
+    /*
+     * Report total code size including the padding and TB structs;
+     * otherwise users might think "-accel tcg,tb-size" is not honoured.
+     * For avg host size we use the precise numbers from tb_tree_stats though.
+     */
+    g_string_append_printf(buf, "gen code size       %zu/%zu\n",
+                           tcg_code_size(), tcg_code_capacity());
+    g_string_append_printf(buf, "TB count            %zu\n", nb_tbs);
+    g_string_append_printf(buf, "TB avg target size  %zu max=%zu bytes\n",
+                           nb_tbs ? tst.target_size / nb_tbs : 0,
+                           tst.max_target_size);
+    g_string_append_printf(buf, "TB avg host size    %zu bytes "
+                           "(expansion ratio: %0.1f)\n",
+                           nb_tbs ? tst.host_size / nb_tbs : 0,
+                           tst.target_size ?
+                           (double)tst.host_size / tst.target_size : 0);
+    g_string_append_printf(buf, "cross page TB count %zu (%zu%%)\n",
+                           tst.cross_page,
+                           nb_tbs ? (tst.cross_page * 100) / nb_tbs : 0);
+    g_string_append_printf(buf, "direct jump count   %zu (%zu%%) "
+                           "(2 jumps=%zu %zu%%)\n",
+                           tst.direct_jmp_count,
+                           nb_tbs ? (tst.direct_jmp_count * 100) / nb_tbs : 0,
+                           tst.direct_jmp2_count,
+                           nb_tbs ? (tst.direct_jmp2_count * 100) / nb_tbs : 0);
+
+    qht_statistics_init(&tb_ctx.htable, &hst);
+    print_qht_statistics(hst, buf);
+    qht_statistics_destroy(&hst);
+
+    g_string_append_printf(buf, "\nStatistics:\n");
+    g_string_append_printf(buf, "TB flush count      %u\n",
+                           qatomic_read(&tb_ctx.tb_flush_count));
+    g_string_append_printf(buf, "TB invalidate count %u\n",
+                           qatomic_read(&tb_ctx.tb_phys_invalidate_count));
+
+    tlb_flush_counts(&flush_full, &flush_part, &flush_elide);
+    g_string_append_printf(buf, "TLB full flushes    %zu\n", flush_full);
+    g_string_append_printf(buf, "TLB partial flushes %zu\n", flush_part);
+    g_string_append_printf(buf, "TLB elided flushes  %zu\n", flush_elide);
+    tcg_dump_info(buf);
+}

 HumanReadableText *qmp_x_query_jit(Error **errp)
 {
@@ -24,7 +208,29 @@ HumanReadableText *qmp_x_query_jit(Error **errp)
        return NULL;
    }

-    tcg_dump_stats(buf);
+    dump_accel_info(buf);
+    dump_exec_info(buf);
+    dump_drift_info(buf);
+
+    return human_readable_text_from_str(buf);
+}
+
+static void tcg_dump_op_count(GString *buf)
+{
+    g_string_append_printf(buf, "[TCG profiler not compiled]\n");
+}
+
+HumanReadableText *qmp_x_query_opcount(Error **errp)
+{
+    g_autoptr(GString) buf = g_string_new("");
+
+    if (!tcg_enabled()) {
+        error_setg(errp,
+                   "Opcode count information is only available with accel=tcg");
+        return NULL;
+    }
+
+    tcg_dump_op_count(buf);

    return human_readable_text_from_str(buf);
 }
@@ -32,6 +238,7 @@ HumanReadableText *qmp_x_query_jit(Error **errp)
 static void hmp_tcg_register(void)
 {
    monitor_register_hmp_info_hrt("jit", qmp_x_query_jit);
+    monitor_register_hmp_info_hrt("opcount", qmp_x_query_opcount);
 }

 type_init(hmp_tcg_register);
@@ -22,12 +22,13 @@
 #include "qemu/osdep.h"
 #include "qemu/plugin.h"
 #include "qemu/log.h"
+#include "cpu.h"
 #include "tcg/tcg.h"
 #include "tcg/tcg-temp-internal.h"
-#include "tcg/tcg-op-common.h"
+#include "tcg/tcg-op.h"
+#include "exec/exec-all.h"
 #include "exec/plugin-gen.h"
 #include "exec/translator.h"
-#include "exec/translation-block.h"

 enum plugin_gen_from {
    PLUGIN_GEN_FROM_TB,
@@ -88,49 +89,32 @@ static void gen_enable_mem_helper(struct qemu_plugin_tb *ptb,
    qemu_plugin_add_dyn_cb_arr(arr);

    tcg_gen_st_ptr(tcg_constant_ptr((intptr_t)arr), tcg_env,
-                   offsetof(CPUState, neg.plugin_mem_cbs) - sizeof(CPUState));
+                   offsetof(CPUState, neg.plugin_mem_cbs) -
+                   offsetof(ArchCPU, env));
 }

 static void gen_disable_mem_helper(void)
 {
    tcg_gen_st_ptr(tcg_constant_ptr(0), tcg_env,
-                   offsetof(CPUState, neg.plugin_mem_cbs) - sizeof(CPUState));
+                   offsetof(CPUState, neg.plugin_mem_cbs) -
+                   offsetof(ArchCPU, env));
 }

 static TCGv_i32 gen_cpu_index(void)
 {
-    /*
-     * Optimize when we run with a single vcpu. All values using cpu_index,
-     * including scoreboard index, will be optimized out.
-     * User-mode flushes all TBs when setting this flag.
-     * In system-mode, all vcpus are created before generating code.
-     */
-    if (!tcg_cflags_has(current_cpu, CF_PARALLEL)) {
-        return tcg_constant_i32(current_cpu->cpu_index);
-    }
    TCGv_i32 cpu_index = tcg_temp_ebb_new_i32();
    tcg_gen_ld_i32(cpu_index, tcg_env,
-                   offsetof(CPUState, cpu_index) - sizeof(CPUState));
+                   -offsetof(ArchCPU, env) + offsetof(CPUState, cpu_index));
    return cpu_index;
 }

 static void gen_udata_cb(struct qemu_plugin_regular_cb *cb)
 {
    TCGv_i32 cpu_index = gen_cpu_index();
-    enum qemu_plugin_cb_flags cb_flags =
-        tcg_call_to_qemu_plugin_cb_flags(cb->info->flags);
-    TCGv_i32 flags = tcg_constant_i32(cb_flags);
-    TCGv_i32 clear_flags = tcg_constant_i32(QEMU_PLUGIN_CB_NO_REGS);
-    tcg_gen_st_i32(flags, tcg_env,
-           offsetof(CPUState, neg.plugin_cb_flags) - sizeof(CPUState));
    tcg_gen_call2(cb->f.vcpu_udata, cb->info, NULL,
                  tcgv_i32_temp(cpu_index),
                  tcgv_ptr_temp(tcg_constant_ptr(cb->userp)));
-    tcg_gen_st_i32(clear_flags, tcg_env,
-           offsetof(CPUState, neg.plugin_cb_flags) - sizeof(CPUState));
    tcg_temp_free_i32(cpu_index);
-    tcg_temp_free_i32(flags);
-    tcg_temp_free_i32(clear_flags);
 }

 static TCGv_ptr gen_plugin_u64_ptr(qemu_plugin_u64 entry)
@@ -183,20 +167,10 @@ static void gen_udata_cond_cb(struct qemu_plugin_conditional_cb *cb)
    tcg_gen_ld_i64(val, ptr, 0);
    tcg_gen_brcondi_i64(cond, val, cb->imm, after_cb);
    TCGv_i32 cpu_index = gen_cpu_index();
-    enum qemu_plugin_cb_flags cb_flags =
-        tcg_call_to_qemu_plugin_cb_flags(cb->info->flags);
-    TCGv_i32 flags = tcg_constant_i32(cb_flags);
-    TCGv_i32 clear_flags = tcg_constant_i32(QEMU_PLUGIN_CB_NO_REGS);
-    tcg_gen_st_i32(flags, tcg_env,
-           offsetof(CPUState, neg.plugin_cb_flags) - sizeof(CPUState));
    tcg_gen_call2(cb->f.vcpu_udata, cb->info, NULL,
                  tcgv_i32_temp(cpu_index),
                  tcgv_ptr_temp(tcg_constant_ptr(cb->userp)));
-    tcg_gen_st_i32(clear_flags, tcg_env,
-           offsetof(CPUState, neg.plugin_cb_flags) - sizeof(CPUState));
    tcg_temp_free_i32(cpu_index);
-    tcg_temp_free_i32(flags);
-    tcg_temp_free_i32(clear_flags);
    gen_set_label(after_cb);

    tcg_temp_free_i64(val);
@@ -230,22 +204,12 @@ static void gen_mem_cb(struct qemu_plugin_regular_cb *cb,
                       qemu_plugin_meminfo_t meminfo, TCGv_i64 addr)
 {
    TCGv_i32 cpu_index = gen_cpu_index();
-    enum qemu_plugin_cb_flags cb_flags =
-        tcg_call_to_qemu_plugin_cb_flags(cb->info->flags);
-    TCGv_i32 flags = tcg_constant_i32(cb_flags);
-    TCGv_i32 clear_flags = tcg_constant_i32(QEMU_PLUGIN_CB_NO_REGS);
-    tcg_gen_st_i32(flags, tcg_env,
-           offsetof(CPUState, neg.plugin_cb_flags) - sizeof(CPUState));
    tcg_gen_call4(cb->f.vcpu_mem, cb->info, NULL,
                  tcgv_i32_temp(cpu_index),
                  tcgv_i32_temp(tcg_constant_i32(meminfo)),
                  tcgv_i64_temp(addr),
                  tcgv_ptr_temp(tcg_constant_ptr(cb->userp)));
-    tcg_gen_st_i32(clear_flags, tcg_env,
-           offsetof(CPUState, neg.plugin_cb_flags) - sizeof(CPUState));
    tcg_temp_free_i32(cpu_index);
-    tcg_temp_free_i32(flags);
-    tcg_temp_free_i32(clear_flags);
 }

 static void inject_cb(struct qemu_plugin_dyn_cb *cb)
@@ -287,6 +251,7 @@ static void inject_mem_cb(struct qemu_plugin_dyn_cb *cb,
        break;
    default:
        g_assert_not_reached();
+        break;
    }
 }

@@ -20,9 +20,8 @@
 #ifndef EXEC_TB_HASH_H
 #define EXEC_TB_HASH_H

-#include "exec/vaddr.h"
-#include "exec/target_page.h"
-#include "exec/translation-block.h"
+#include "exec/cpu-defs.h"
+#include "exec/exec-all.h"
 #include "qemu/xxhash.h"
 #include "tb-jmp-cache.h"

@@ -1,55 +0,0 @@
-/*
- * TranslationBlock internal declarations (target specific)
- *
- *  Copyright (c) 2003 Fabrice Bellard
- *
- * SPDX-License-Identifier: LGPL-2.1-or-later
- */
-
-#ifndef ACCEL_TCG_TB_INTERNAL_TARGET_H
-#define ACCEL_TCG_TB_INTERNAL_TARGET_H
-
-#include "exec/translation-block.h"
-
-/*
- * The true return address will often point to a host insn that is part of
- * the next translated guest insn.  Adjust the address backward to point to
- * the middle of the call insn.  Subtracting one would do the job except for
- * several compressed mode architectures (arm, mips) which set the low bit
- * to indicate the compressed mode; subtracting two works around that.  It
- * is also the case that there are no host isas that contain a call insn
- * smaller than 4 bytes, so we don't worry about special-casing this.
- */
-#define GETPC_ADJ   2
-
-void tb_lock_page0(tb_page_addr_t);
-
-#ifdef CONFIG_USER_ONLY
-/*
- * For user-only, page_protect sets the page read-only.
- * Since most execution is already on read-only pages, and we'd need to
- * account for other TBs on the same page, defer undoing any page protection
- * until we receive the write fault.
- */
-static inline void tb_lock_page1(tb_page_addr_t p0, tb_page_addr_t p1)
-{
-    tb_lock_page0(p1);
-}
-
-static inline void tb_unlock_page1(tb_page_addr_t p0, tb_page_addr_t p1) { }
-static inline void tb_unlock_pages(TranslationBlock *tb) { }
-#else
-void tb_lock_page1(tb_page_addr_t, tb_page_addr_t);
-void tb_unlock_page1(tb_page_addr_t, tb_page_addr_t);
-void tb_unlock_pages(TranslationBlock *);
-#endif
-
-#ifdef CONFIG_SOFTMMU
-void tb_invalidate_phys_range_fast(CPUState *cpu, ram_addr_t ram_addr,
-                                   unsigned size, uintptr_t retaddr);
-#endif /* CONFIG_SOFTMMU */
-
-bool tb_invalidate_phys_page_unwind(CPUState *cpu, tb_page_addr_t addr,
-                                    uintptr_t pc);
-
-#endif
@@ -22,25 +22,17 @@
 #include "qemu/qtree.h"
 #include "exec/cputlb.h"
 #include "exec/log.h"
+#include "exec/exec-all.h"
 #include "exec/page-protection.h"
-#include "exec/mmap-lock.h"
 #include "exec/tb-flush.h"
-#include "exec/target_page.h"
-#include "accel/tcg/cpu-ops.h"
-#include "tb-internal.h"
-#include "system/tcg.h"
+#include "exec/translate-all.h"
+#include "sysemu/tcg.h"
 #include "tcg/tcg.h"
 #include "tb-hash.h"
 #include "tb-context.h"
-#include "tb-internal.h"
 #include "internal-common.h"
-#ifdef CONFIG_USER_ONLY
-#include "user/page-protection.h"
-#define runstate_is_running()  true
-#else
-#include "system/runstate.h"
-#endif
-#include "trace.h"
+#include "internal-target.h"
+

 /* List iterators for lists of tagged pointers in TranslationBlock. */
 #define TB_FOR_EACH_TAGGED(head, tb, n, field)                          \
@@ -91,10 +83,7 @@ static IntervalTreeRoot tb_root;

 static void tb_remove_all(void)
 {
-    /*
-     * Only called from tb_flush__exclusive_or_serial, where we have already
-     * asserted that we're in an exclusive state.
-     */
+    assert_memory_lock();
    memset(&tb_root, 0, sizeof(tb_root));
 }

@@ -163,7 +152,11 @@ static PageForEachNext foreach_tb_next(PageForEachNext tb,
 /*
 * In system mode we want L1_MAP to be based on ram offsets.
 */
-#define L1_MAP_ADDR_SPACE_BITS  HOST_LONG_BITS
+#if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
+# define L1_MAP_ADDR_SPACE_BITS  HOST_LONG_BITS
+#else
+# define L1_MAP_ADDR_SPACE_BITS  TARGET_PHYS_ADDR_SPACE_BITS
+#endif

 /* Size of the L2 (and L3, etc) page tables.  */
 #define V_L2_BITS 10
@@ -762,20 +755,17 @@ static void tb_remove(TranslationBlock *tb)
 }
 #endif /* CONFIG_USER_ONLY */

-/*
- * Flush all the translation blocks.
- * Must be called from a context in which no cpus are running,
- * e.g. start_exclusive() or vm_stop().
- */
-void tb_flush__exclusive_or_serial(void)
+/* flush all the translation blocks */
+static void do_tb_flush(CPUState *cpu, run_on_cpu_data tb_flush_count)
 {
-    CPUState *cpu;
+    bool did_flush = false;

-    trace_tb_flush();
-    assert(tcg_enabled());
-    /* Note that cpu_in_serial_context checks cpu_in_exclusive_context. */
-    assert(!runstate_is_running() ||
-           (current_cpu && cpu_in_serial_context(current_cpu)));
+    mmap_lock();
+    /* If it is already been done on request of another CPU, just retry. */
+    if (tb_ctx.tb_flush_count != tb_flush_count.host_int) {
+        goto done;
+    }
+    did_flush = true;

    CPU_FOREACH(cpu) {
        tcg_flush_jmp_cache(cpu);
@@ -787,23 +777,25 @@ void tb_flush__exclusive_or_serial(void)
    tcg_region_reset_all();
    /* XXX: flush processor icache at this point if cache flush is expensive */
    qatomic_inc(&tb_ctx.tb_flush_count);
-    qemu_plugin_flush_cb();
-}

-static void do_tb_flush(CPUState *cpu, run_on_cpu_data tb_flush_count)
-{
-    /* If it is already been done on request of another CPU, just retry. */
-    if (tb_ctx.tb_flush_count == tb_flush_count.host_int) {
-        tb_flush__exclusive_or_serial();
+done:
+    mmap_unlock();
+    if (did_flush) {
+        qemu_plugin_flush_cb();
    }
 }

-void queue_tb_flush(CPUState *cs)
+void tb_flush(CPUState *cpu)
 {
    if (tcg_enabled()) {
        unsigned tb_flush_count = qatomic_read(&tb_ctx.tb_flush_count);
-        async_safe_run_on_cpu(cs, do_tb_flush,
-                              RUN_ON_CPU_HOST_INT(tb_flush_count));
+
+        if (cpu_in_serial_context(cpu)) {
+            do_tb_flush(cpu, RUN_ON_CPU_HOST_INT(tb_flush_count));
+        } else {
+            async_safe_run_on_cpu(cpu, do_tb_flush,
+                                  RUN_ON_CPU_HOST_INT(tb_flush_count));
+        }
    }
 }

@@ -843,14 +835,6 @@ static inline void tb_remove_from_jmp_list(TranslationBlock *orig, int n_orig)
     * We first acquired the lock, and since the destination pointer matches,
     * we know for sure that @orig is in the jmp list.
     */
-    if (dest == orig) {
-        /*
-         * In the case of a TB that links to itself, removing the entry
-         * from the list means that it won't be present later during
-         * tb_jmp_unlink -- unlink now.
-         */
-        tb_reset_jump(orig, n_orig);
-    }
    pprev = &dest->jmp_list_head;
    TB_FOR_EACH_JMP(dest, tb, n) {
        if (tb == orig && n == n_orig) {
@@ -1021,8 +1005,7 @@ TranslationBlock *tb_link_page(TranslationBlock *tb)
 * Called with mmap_lock held for user-mode emulation.
 * NOTE: this function must not be called while a TB is running.
 */
-void tb_invalidate_phys_range(CPUState *cpu, tb_page_addr_t start,
-                              tb_page_addr_t last)
+void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t last)
 {
    TranslationBlock *tb;
    PageForEachNext n;
@@ -1045,16 +1028,17 @@ static void tb_invalidate_phys_page(tb_page_addr_t addr)

    start = addr & TARGET_PAGE_MASK;
    last = addr | ~TARGET_PAGE_MASK;
-    tb_invalidate_phys_range(NULL, start, last);
+    tb_invalidate_phys_range(start, last);
 }

 /*
 * Called with mmap_lock held. If pc is not 0 then it indicates the
 * host PC of the faulting store instruction that caused this invalidate.
- * Returns true if the caller needs to abort execution of the current TB.
+ * Returns true if the caller needs to abort execution of the current
+ * TB (because it was modified by this store and the guest CPU has
+ * precise-SMC semantics).
 */
-bool tb_invalidate_phys_page_unwind(CPUState *cpu, tb_page_addr_t addr,
-                                    uintptr_t pc)
+bool tb_invalidate_phys_page_unwind(tb_page_addr_t addr, uintptr_t pc)
 {
    TranslationBlock *current_tb;
    bool current_tb_modified;
@@ -1066,7 +1050,10 @@ bool tb_invalidate_phys_page_unwind(CPUState *cpu, tb_page_addr_t addr,
     * Without precise smc semantics, or when outside of a TB,
     * we can skip to invalidate.
     */
-    if (!pc || !cpu || !cpu->cc->tcg_ops->precise_smc) {
+#ifndef TARGET_HAS_PRECISE_SMC
+    pc = 0;
+#endif
+    if (!pc) {
        tb_invalidate_phys_page(addr);
        return false;
    }
@@ -1089,14 +1076,15 @@ bool tb_invalidate_phys_page_unwind(CPUState *cpu, tb_page_addr_t addr,
             * the CPU state.
             */
            current_tb_modified = true;
-            cpu_restore_state_from_tb(cpu, current_tb, pc);
+            cpu_restore_state_from_tb(current_cpu, current_tb, pc);
        }
        tb_phys_invalidate__locked(tb);
    }

    if (current_tb_modified) {
        /* Force execution of one insn next time.  */
-        cpu->cflags_next_tb = 1 | CF_NOIRQ | curr_cflags(cpu);
+        CPUState *cpu = current_cpu;
+        cpu->cflags_next_tb = 1 | CF_NOIRQ | curr_cflags(current_cpu);
        return true;
    }
    return false;
@@ -1105,28 +1093,23 @@ bool tb_invalidate_phys_page_unwind(CPUState *cpu, tb_page_addr_t addr,
 /*
 * @p must be non-NULL.
 * Call with all @pages locked.
- * (@cpu, @retaddr) may be (NULL, 0) outside of a cpu context,
- * in which case precise_smc need not be detected.
 */
 static void
-tb_invalidate_phys_page_range__locked(CPUState *cpu,
-                                      struct page_collection *pages,
+tb_invalidate_phys_page_range__locked(struct page_collection *pages,
                                      PageDesc *p, tb_page_addr_t start,
                                      tb_page_addr_t last,
                                      uintptr_t retaddr)
 {
    TranslationBlock *tb;
    PageForEachNext n;
+#ifdef TARGET_HAS_PRECISE_SMC
    bool current_tb_modified = false;
-    TranslationBlock *current_tb = NULL;
+    TranslationBlock *current_tb = retaddr ? tcg_tb_lookup(retaddr) : NULL;
+#endif /* TARGET_HAS_PRECISE_SMC */

    /* Range may not cross a page. */
    tcg_debug_assert(((start ^ last) & TARGET_PAGE_MASK) == 0);

-    if (retaddr && cpu && cpu->cc->tcg_ops->precise_smc) {
-        current_tb = tcg_tb_lookup(retaddr);
-    }
-
    /*
     * We remove all the TBs in the range [start, last].
     * XXX: see if in some cases it could be faster to invalidate all the code
@@ -1144,7 +1127,8 @@ tb_invalidate_phys_page_range__locked(CPUState *cpu,
            tb_last = tb_start + (tb_last & ~TARGET_PAGE_MASK);
        }
        if (!(tb_last < start || tb_start > last)) {
-            if (unlikely(current_tb == tb) &&
+#ifdef TARGET_HAS_PRECISE_SMC
+            if (current_tb == tb &&
                (tb_cflags(current_tb) & CF_COUNT_MASK) != 1) {
                /*
                 * If we are modifying the current TB, we must stop
@@ -1154,8 +1138,9 @@ tb_invalidate_phys_page_range__locked(CPUState *cpu,
                 * restore the CPU state.
                 */
                current_tb_modified = true;
-                cpu_restore_state_from_tb(cpu, current_tb, retaddr);
+                cpu_restore_state_from_tb(current_cpu, current_tb, retaddr);
            }
+#endif /* TARGET_HAS_PRECISE_SMC */
            tb_phys_invalidate__locked(tb);
        }
    }
@@ -1165,12 +1150,15 @@ tb_invalidate_phys_page_range__locked(CPUState *cpu,
        tlb_unprotect_code(start);
    }

-    if (unlikely(current_tb_modified)) {
+#ifdef TARGET_HAS_PRECISE_SMC
+    if (current_tb_modified) {
        page_collection_unlock(pages);
        /* Force execution of one insn next time.  */
-        cpu->cflags_next_tb = 1 | CF_NOIRQ | curr_cflags(cpu);
-        cpu_loop_exit_noexc(cpu);
+        current_cpu->cflags_next_tb = 1 | CF_NOIRQ | curr_cflags(current_cpu);
+        mmap_unlock();
+        cpu_loop_exit_noexc(current_cpu);
    }
+#endif
 }

 /*
@@ -1180,8 +1168,7 @@ tb_invalidate_phys_page_range__locked(CPUState *cpu,
 * access: the virtual CPU will exit the current TB if code is modified inside
 * this TB.
 */
-void tb_invalidate_phys_range(CPUState *cpu, tb_page_addr_t start,
-                              tb_page_addr_t last)
+void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t last)
 {
    struct page_collection *pages;
    tb_page_addr_t index, index_last;
@@ -1200,30 +1187,44 @@ void tb_invalidate_phys_range(CPUState *cpu, tb_page_addr_t start,
        page_start = index << TARGET_PAGE_BITS;
        page_last = page_start | ~TARGET_PAGE_MASK;
        page_last = MIN(page_last, last);
-        tb_invalidate_phys_page_range__locked(cpu, pages, pd,
+        tb_invalidate_phys_page_range__locked(pages, pd,
                                              page_start, page_last, 0);
    }
    page_collection_unlock(pages);
 }

+/*
+ * Call with all @pages in the range [@start, @start + len[ locked.
+ */
+static void tb_invalidate_phys_page_fast__locked(struct page_collection *pages,
+                                                 tb_page_addr_t start,
+                                                 unsigned len, uintptr_t ra)
+{
+    PageDesc *p;
+
+    p = page_find(start >> TARGET_PAGE_BITS);
+    if (!p) {
+        return;
+    }
+
+    assert_page_locked(p);
+    tb_invalidate_phys_page_range__locked(pages, p, start, start + len - 1, ra);
+}
+
 /*
 * len must be <= 8 and start must be a multiple of len.
 * Called via softmmu_template.h when code areas are written to with
 * iothread mutex not held.
 */
-void tb_invalidate_phys_range_fast(CPUState *cpu, ram_addr_t start,
-                                   unsigned len, uintptr_t ra)
+void tb_invalidate_phys_range_fast(ram_addr_t ram_addr,
+                                   unsigned size,
+                                   uintptr_t retaddr)
 {
-    PageDesc *p = page_find(start >> TARGET_PAGE_BITS);
+    struct page_collection *pages;

-    if (p) {
-        ram_addr_t last = start + len - 1;
-        struct page_collection *pages = page_collection_lock(start, last);
-
-        tb_invalidate_phys_page_range__locked(cpu, pages, p,
-                                              start, last, ra);
-        page_collection_unlock(pages);
-    }
+    pages = page_collection_lock(ram_addr, ram_addr + size - 1);
+    tb_invalidate_phys_page_fast__locked(pages, ram_addr, size, retaddr);
+    page_collection_unlock(pages);
 }

 #endif /* CONFIG_USER_ONLY */
@@ -24,11 +24,11 @@
 */

 #include "qemu/osdep.h"
-#include "system/replay.h"
-#include "exec/icount.h"
+#include "sysemu/replay.h"
+#include "sysemu/cpu-timers.h"
 #include "qemu/main-loop.h"
 #include "qemu/guest-random.h"
-#include "hw/core/cpu.h"
+#include "exec/exec-all.h"

 #include "tcg-accel-ops.h"
 #include "tcg-accel-ops-icount.h"
@@ -24,12 +24,13 @@
 */

 #include "qemu/osdep.h"
-#include "system/tcg.h"
-#include "system/replay.h"
-#include "exec/icount.h"
+#include "sysemu/tcg.h"
+#include "sysemu/replay.h"
+#include "sysemu/cpu-timers.h"
 #include "qemu/main-loop.h"
 #include "qemu/notify.h"
 #include "qemu/guest-random.h"
+#include "exec/exec-all.h"
 #include "hw/boards.h"
 #include "tcg/startup.h"
 #include "tcg-accel-ops.h"
@@ -84,9 +85,10 @@ static void *mttcg_cpu_thread_fn(void *arg)
    cpu_thread_signal_created(cpu);
    qemu_guest_random_seed_thread_part2(cpu->random_seed);

-    do {
-        qemu_process_cpu_events(cpu);
+    /* process any pending work */
+    cpu->exit_request = 1;

+    do {
        if (cpu_can_run(cpu)) {
            int r;
            bql_unlock();
@@ -111,6 +113,9 @@ static void *mttcg_cpu_thread_fn(void *arg)
                break;
            }
        }
+
+        qatomic_set_mb(&cpu->exit_request, 0);
+        qemu_wait_io_event(cpu);
    } while (!cpu->unplug || cpu_can_run(cpu));

    tcg_cpu_destroy(cpu);
@@ -120,6 +125,11 @@ static void *mttcg_cpu_thread_fn(void *arg)
    return NULL;
 }

+void mttcg_kick_vcpu_thread(CPUState *cpu)
+{
+    cpu_exit(cpu);
+}
+
 void mttcg_start_vcpu_thread(CPUState *cpu)
 {
    char thread_name[VCPU_THREAD_NAME_SIZE];
@@ -10,6 +10,9 @@
 #ifndef TCG_ACCEL_OPS_MTTCG_H
 #define TCG_ACCEL_OPS_MTTCG_H

+/* kick MTTCG vCPU thread */
+void mttcg_kick_vcpu_thread(CPUState *cpu);
+
 /* start an mttcg vCPU thread */
 void mttcg_start_vcpu_thread(CPUState *cpu);

@@ -25,13 +25,13 @@

 #include "qemu/osdep.h"
 #include "qemu/lockable.h"
-#include "system/tcg.h"
-#include "system/replay.h"
-#include "exec/icount.h"
+#include "sysemu/tcg.h"
+#include "sysemu/replay.h"
+#include "sysemu/cpu-timers.h"
 #include "qemu/main-loop.h"
 #include "qemu/notify.h"
 #include "qemu/guest-random.h"
-#include "exec/cpu-common.h"
+#include "exec/exec-all.h"
 #include "tcg/startup.h"
 #include "tcg-accel-ops.h"
 #include "tcg-accel-ops-rr.h"
@@ -43,7 +43,7 @@ void rr_kick_vcpu_thread(CPUState *unused)
    CPUState *cpu;

    CPU_FOREACH(cpu) {
-        tcg_kick_vcpu_thread(cpu);
+        cpu_exit(cpu);
    };
 }

@@ -117,7 +117,7 @@ static void rr_wait_io_event(void)
    rr_start_kick_timer();

    CPU_FOREACH(cpu) {
-        qemu_process_cpu_events_common(cpu);
+        qemu_wait_io_event_common(cpu);
    }
 }

@@ -197,13 +197,13 @@ static void *rr_cpu_thread_fn(void *arg)
    qemu_guest_random_seed_thread_part2(cpu->random_seed);

    /* wait for initial kick-off after machine start */
-    while (cpu_is_stopped(first_cpu)) {
+    while (first_cpu->stopped) {
        qemu_cond_wait_bql(first_cpu->halt_cond);

        /* process any pending work */
        CPU_FOREACH(cpu) {
            current_cpu = cpu;
-            qemu_process_cpu_events_common(cpu);
+            qemu_wait_io_event_common(cpu);
        }
    }

@@ -211,30 +211,13 @@ static void *rr_cpu_thread_fn(void *arg)

    cpu = first_cpu;

+    /* process any pending work */
+    cpu->exit_request = 1;
+
    while (1) {
        /* Only used for icount_enabled() */
        int64_t cpu_budget = 0;

-        if (cpu) {
-            /*
-             * This could even reset exit_request for all CPUs, but in practice
-             * races between CPU exits and changes to "cpu" are so rare that
-             * there's no advantage in doing so.
-             */
-            qatomic_set(&cpu->exit_request, false);
-        }
-
-        if (icount_enabled() && all_cpu_threads_idle()) {
-            /*
-             * When all cpus are sleeping (e.g in WFI), to avoid a deadlock
-             * in the main_loop, wake it up in order to start the warp timer.
-             */
-            qemu_notify_event();
-        }
-
-        rr_wait_io_event();
-        rr_deal_with_unplugged_cpus();
-
        bql_unlock();
        replay_mutex_lock();
        bql_lock();
@@ -259,17 +242,10 @@ static void *rr_cpu_thread_fn(void *arg)
            cpu = first_cpu;
        }

-        while (cpu && cpu_work_list_empty(cpu)) {
-            /*
-             * Store rr_current_cpu before evaluating cpu->exit_request.
-             * Pairs with rr_kick_next_cpu().
-             */
+        while (cpu && cpu_work_list_empty(cpu) && !cpu->exit_request) {
+            /* Store rr_current_cpu before evaluating cpu_can_run().  */
            qatomic_set_mb(&rr_current_cpu, cpu);

-            /* Pairs with store-release in cpu_exit.  */
-            if (qatomic_load_acquire(&cpu->exit_request)) {
-                break;
-            }
            current_cpu = cpu;

            qemu_clock_enable(QEMU_CLOCK_VIRTUAL,
@@ -309,9 +285,26 @@ static void *rr_cpu_thread_fn(void *arg)

        /* Does not need a memory barrier because a spurious wakeup is okay.  */
        qatomic_set(&rr_current_cpu, NULL);
+
+        if (cpu && cpu->exit_request) {
+            qatomic_set_mb(&cpu->exit_request, 0);
+        }
+
+        if (icount_enabled() && all_cpu_threads_idle()) {
+            /*
+             * When all cpus are sleeping (e.g in WFI), to avoid a deadlock
+             * in the main_loop, wake it up in order to start the warp timer.
+             */
+            qemu_notify_event();
+        }
+
+        rr_wait_io_event();
+        rr_deal_with_unplugged_cpus();
    }

-    g_assert_not_reached();
+    rcu_remove_force_rcu_notifier(&force_rcu);
+    rcu_unregister_thread();
+    return NULL;
 }

 void rr_start_vcpu_thread(CPUState *cpu)
@@ -26,19 +26,15 @@
 */

 #include "qemu/osdep.h"
-#include "accel/accel-ops.h"
-#include "accel/accel-cpu-ops.h"
-#include "system/tcg.h"
-#include "system/replay.h"
-#include "exec/icount.h"
+#include "sysemu/tcg.h"
+#include "sysemu/replay.h"
+#include "sysemu/cpu-timers.h"
 #include "qemu/main-loop.h"
 #include "qemu/guest-random.h"
 #include "qemu/timer.h"
-#include "exec/cputlb.h"
+#include "exec/exec-all.h"
 #include "exec/hwaddr.h"
 #include "exec/tb-flush.h"
-#include "exec/translation-block.h"
-#include "exec/watchpoint.h"
 #include "gdbstub/enums.h"

 #include "hw/core/cpu.h"
@@ -81,7 +77,6 @@ int tcg_cpu_exec(CPUState *cpu)
    cpu_exec_start(cpu);
    ret = cpu_exec(cpu);
    cpu_exec_end(cpu);
-
    return ret;
 }

@@ -95,7 +90,9 @@ static void tcg_cpu_reset_hold(CPUState *cpu)
 /* mask must never be zero, except for A20 change call */
 void tcg_handle_interrupt(CPUState *cpu, int mask)
 {
-    cpu_set_interrupt(cpu, mask);
+    g_assert(bql_locked());
+
+    cpu->interrupt_request |= mask;

    /*
     * If called from iothread context, wake the target cpu in
@@ -122,9 +119,10 @@ static inline int xlat_gdb_type(CPUState *cpu, int gdbtype)
        [GDB_WATCHPOINT_ACCESS] = BP_GDB | BP_MEM_ACCESS,
    };

+    CPUClass *cc = CPU_GET_CLASS(cpu);
    int cputype = xlat[gdbtype];

-    if (cpu->cc->gdb_stop_before_watchpoint) {
+    if (cc->gdb_stop_before_watchpoint) {
        cputype |= BP_STOP_BEFORE_ACCESS;
    }
    return cputype;
@@ -198,13 +196,11 @@ static inline void tcg_remove_all_breakpoints(CPUState *cpu)
    cpu_watchpoint_remove_all(cpu, BP_GDB);
 }

-static void tcg_accel_ops_init(AccelClass *ac)
+static void tcg_accel_ops_init(AccelOpsClass *ops)
 {
-    AccelOpsClass *ops = ac->ops;
-
    if (qemu_tcg_mttcg_enabled()) {
        ops->create_vcpu_thread = mttcg_start_vcpu_thread;
-        ops->kick_vcpu_thread = tcg_kick_vcpu_thread;
+        ops->kick_vcpu_thread = mttcg_kick_vcpu_thread;
        ops->handle_interrupt = tcg_handle_interrupt;
    } else {
        ops->create_vcpu_thread = rr_start_vcpu_thread;
@@ -226,7 +222,7 @@ static void tcg_accel_ops_init(AccelClass *ac)
    ops->remove_all_breakpoints = tcg_remove_all_breakpoints;
 }

-static void tcg_accel_ops_class_init(ObjectClass *oc, const void *data)
+static void tcg_accel_ops_class_init(ObjectClass *oc, void *data)
 {
    AccelOpsClass *ops = ACCEL_OPS_CLASS(oc);

@@ -12,12 +12,11 @@
 #ifndef TCG_ACCEL_OPS_H
 #define TCG_ACCEL_OPS_H

-#include "system/cpus.h"
+#include "sysemu/cpus.h"

 void tcg_cpu_destroy(CPUState *cpu);
 int tcg_cpu_exec(CPUState *cpu);
 void tcg_handle_interrupt(CPUState *cpu, int mask);
 void tcg_cpu_init_cflags(CPUState *cpu, bool parallel);
-void tcg_kick_vcpu_thread(CPUState *cpu);

 #endif /* TCG_ACCEL_OPS_H */
@@ -24,33 +24,26 @@
 */

 #include "qemu/osdep.h"
-#include "system/tcg.h"
+#include "sysemu/tcg.h"
 #include "exec/replay-core.h"
-#include "exec/icount.h"
+#include "sysemu/cpu-timers.h"
 #include "tcg/startup.h"
+#include "tcg/oversized-guest.h"
 #include "qapi/error.h"
 #include "qemu/error-report.h"
 #include "qemu/accel.h"
 #include "qemu/atomic.h"
-#include "qapi/qapi-types-common.h"
 #include "qapi/qapi-builtin-visit.h"
 #include "qemu/units.h"
-#include "qemu/target-info.h"
-#ifndef CONFIG_USER_ONLY
+#if !defined(CONFIG_USER_ONLY)
 #include "hw/boards.h"
-#include "exec/tb-flush.h"
-#include "system/runstate.h"
 #endif
-#include "accel/accel-ops.h"
-#include "accel/accel-cpu-ops.h"
-#include "accel/tcg/cpu-ops.h"
 #include "internal-common.h"

-
 struct TCGState {
    AccelState parent_obj;

-    OnOffAuto mttcg_enabled;
+    bool mttcg_enabled;
    bool one_insn_per_tb;
    int splitwx_enabled;
    unsigned long tb_size;
@@ -62,18 +55,40 @@ typedef struct TCGState TCGState;
 DECLARE_INSTANCE_CHECKER(TCGState, TCG_STATE,
                         TYPE_TCG_ACCEL)

-#ifndef CONFIG_USER_ONLY
-bool qemu_tcg_mttcg_enabled(void)
+/*
+ * We default to false if we know other options have been enabled
+ * which are currently incompatible with MTTCG. Otherwise when each
+ * guest (target) has been updated to support:
+ *   - atomic instructions
+ *   - memory ordering primitives (barriers)
+ * they can set the appropriate CONFIG flags in ${target}-softmmu.mak
+ *
+ * Once a guest architecture has been converted to the new primitives
+ * there is one remaining limitation to check:
+ *   - The guest can't be oversized (e.g. 64 bit guest on 32 bit host)
+ */
+
+static bool default_mttcg_enabled(void)
 {
-    TCGState *s = TCG_STATE(current_accel());
-    return s->mttcg_enabled == ON_OFF_AUTO_ON;
+    if (icount_enabled() || TCG_OVERSIZED_GUEST) {
+        return false;
+    }
+#ifdef TARGET_SUPPORTS_MTTCG
+# ifndef TCG_GUEST_DEFAULT_MO
+#  error "TARGET_SUPPORTS_MTTCG without TCG_GUEST_DEFAULT_MO"
+# endif
+    return true;
+#else
+    return false;
+#endif
 }
-#endif /* !CONFIG_USER_ONLY */

 static void tcg_accel_instance_init(Object *obj)
 {
    TCGState *s = TCG_STATE(obj);

+    s->mttcg_enabled = default_mttcg_enabled();
+
    /* If debugging enabled, default "auto on", otherwise off. */
 #if defined(CONFIG_DEBUG_TCG) && !defined(CONFIG_USER_ONLY)
    s->splitwx_enabled = -1;
@@ -82,76 +97,24 @@ static void tcg_accel_instance_init(Object *obj)
 #endif
 }

+bool mttcg_enabled;
 bool one_insn_per_tb;

-#ifndef CONFIG_USER_ONLY
-static void tcg_vm_change_state(void *opaque, bool running, RunState state)
+static int tcg_init_machine(MachineState *ms)
 {
-    if (state == RUN_STATE_RESTORE_VM) {
-        /*
-         * loadvm will update the content of RAM, bypassing the usual
-         * mechanisms that ensure we flush TBs for writes to memory
-         * we've translated code from, so we must flush all TBs.
-         *
-         * vm_stop() has just stopped all cpus, so we are exclusive.
-         */
-        assert(!running);
-        tb_flush__exclusive_or_serial();
-    }
-}
-#endif
-
-static int tcg_init_machine(AccelState *as, MachineState *ms)
-{
-    TCGState *s = TCG_STATE(as);
-    unsigned max_threads = 1;
-
-#ifndef CONFIG_USER_ONLY
-    CPUClass *cc = CPU_CLASS(object_class_by_name(target_cpu_type()));
-    bool mttcg_supported = cc->tcg_ops->mttcg_supported;
-
-    switch (s->mttcg_enabled) {
-    case ON_OFF_AUTO_AUTO:
-        /*
-         * We default to false if we know other options have been enabled
-         * which are currently incompatible with MTTCG. Otherwise when each
-         * guest (target) has been updated to support:
-         *   - atomic instructions
-         *   - memory ordering primitives (barriers)
-         * they can set the appropriate CONFIG flags in ${target}-softmmu.mak
-         *
-         * Once a guest architecture has been converted to the new primitives
-         * there is one remaining limitation to check:
-         *   - The guest can't be oversized (e.g. 64 bit guest on 32 bit host)
-         */
-        if (mttcg_supported && !icount_enabled()) {
-            s->mttcg_enabled = ON_OFF_AUTO_ON;
-            max_threads = ms->smp.max_cpus;
-        } else {
-            s->mttcg_enabled = ON_OFF_AUTO_OFF;
-        }
-        break;
-    case ON_OFF_AUTO_ON:
-        if (!mttcg_supported) {
-            warn_report("Guest not yet converted to MTTCG - "
-                        "you may get unexpected results");
-        }
-        max_threads = ms->smp.max_cpus;
-        break;
-    case ON_OFF_AUTO_OFF:
-        break;
-    default:
-        g_assert_not_reached();
-    }
-
-    qemu_add_vm_change_state_handler(tcg_vm_change_state, NULL);
+    TCGState *s = TCG_STATE(current_accel());
+#ifdef CONFIG_USER_ONLY
+    unsigned max_cpus = 1;
+#else
+    unsigned max_cpus = ms->smp.max_cpus;
 #endif

    tcg_allowed = true;
+    mttcg_enabled = s->mttcg_enabled;

    page_init();
    tb_htable_init();
-    tcg_init(s->tb_size * MiB, s->splitwx_enabled, max_threads);
+    tcg_init(s->tb_size * MiB, s->splitwx_enabled, max_cpus);

 #if defined(CONFIG_SOFTMMU)
    /*
@@ -161,10 +124,6 @@ static int tcg_init_machine(AccelState *as, MachineState *ms)
    tcg_prologue_init();
 #endif

-#ifdef CONFIG_USER_ONLY
-    qdev_create_fake_machine();
-#endif
-
    return 0;
 }

@@ -172,7 +131,7 @@ static char *tcg_get_thread(Object *obj, Error **errp)
 {
    TCGState *s = TCG_STATE(obj);

-    return g_strdup(s->mttcg_enabled == ON_OFF_AUTO_ON ? "multi" : "single");
+    return g_strdup(s->mttcg_enabled ? "multi" : "single");
 }

 static void tcg_set_thread(Object *obj, const char *value, Error **errp)
@@ -180,13 +139,19 @@ static void tcg_set_thread(Object *obj, const char *value, Error **errp)
    TCGState *s = TCG_STATE(obj);

    if (strcmp(value, "multi") == 0) {
-        if (icount_enabled()) {
+        if (TCG_OVERSIZED_GUEST) {
+            error_setg(errp, "No MTTCG when guest word size > hosts");
+        } else if (icount_enabled()) {
            error_setg(errp, "No MTTCG when icount is enabled");
        } else {
-            s->mttcg_enabled = ON_OFF_AUTO_ON;
+#ifndef TARGET_SUPPORTS_MTTCG
+            warn_report("Guest not yet converted to MTTCG - "
+                        "you may get unexpected results");
+#endif
+            s->mttcg_enabled = true;
        }
    } else if (strcmp(value, "single") == 0) {
-        s->mttcg_enabled = ON_OFF_AUTO_OFF;
+        s->mttcg_enabled = false;
    } else {
        error_setg(errp, "Invalid 'thread' setting %s", value);
    }
@@ -242,7 +207,7 @@ static void tcg_set_one_insn_per_tb(Object *obj, bool value, Error **errp)
    qatomic_set(&one_insn_per_tb, value);
 }

-static int tcg_gdbstub_supported_sstep_flags(AccelState *as)
+static int tcg_gdbstub_supported_sstep_flags(void)
 {
    /*
     * In replay mode all events will come from the log and can't be
@@ -257,14 +222,13 @@ static int tcg_gdbstub_supported_sstep_flags(AccelState *as)
    }
 }

-static void tcg_accel_class_init(ObjectClass *oc, const void *data)
+static void tcg_accel_class_init(ObjectClass *oc, void *data)
 {
    AccelClass *ac = ACCEL_CLASS(oc);
    ac->name = "tcg";
    ac->init_machine = tcg_init_machine;
    ac->cpu_common_realize = tcg_exec_realizefn;
    ac->cpu_common_unrealize = tcg_exec_unrealizefn;
-    ac->get_stats = tcg_get_stats;
    ac->allowed = &tcg_allowed;
    ac->gdbstub_supported_sstep_flags = tcg_gdbstub_supported_sstep_flags;

@@ -19,6 +19,7 @@

 #include "qemu/osdep.h"
 #include "qemu/host-utils.h"
+#include "cpu.h"
 #include "exec/helper-proto-common.h"
 #include "tcg/tcg-gvec-desc.h"

@@ -23,9 +23,13 @@
 */
 #include "qemu/osdep.h"
 #include "qemu/host-utils.h"
-#include "exec/cpu-common.h"
+#include "cpu.h"
 #include "exec/helper-proto-common.h"
-#include "accel/tcg/getpc.h"
+#include "exec/cpu_ldst.h"
+#include "exec/exec-all.h"
+#include "disas/disas.h"
+#include "exec/log.h"
+#include "tcg/tcg.h"

 #define HELPER_H  "accel/tcg/tcg-runtime.h"
 #include "exec/helper-info.c.inc"
--- a/Show More
+++ b/Show More
@@ -1 +1 @@
 .1.50
 .1.3