kvm tools: move kvm_cpus into struct kvm

There's no reason the array of guest specific vcpus is global. Move it into
struct kvm.

Also split up arch specific vcpu init from the generic code and call it from
the kvm_cpu initializer.

Signed-off-by: Sasha Levin <levinsasha928@gmail.com>
Signed-off-by: Pekka Enberg <penberg@kernel.org>

Makefile

@@ -243,7 +243,7 @@ DEFINES	+= -DKVMTOOLS_VERSION='"$(KVMTOOLS_VERSION)"'
 DEFINES	+= -DBUILD_ARCH='"$(ARCH)"'
 
 KVM_INCLUDE := include
-CFLAGS	+= $(CPPFLAGS) $(DEFINES) -I$(KVM_INCLUDE) -I$(ARCH_INCLUDE) -I$(KINCL_PATH)/include -I$(KINCL_PATH)/arch/$(ARCH)/include/ -O2 -fno-strict-aliasing -g -flto
+CFLAGS	+= $(CPPFLAGS) $(DEFINES) -I$(KVM_INCLUDE) -I$(ARCH_INCLUDE) -I$(KINCL_PATH)/include -I$(KINCL_PATH)/arch/$(ARCH)/include/ -O0 -fno-strict-aliasing -g -flto
 
 WARNINGS += -Wall
 WARNINGS += -Wcast-align

builtin-run.c

@@ -54,7 +54,6 @@
 #define GB_SHIFT		(30)
 
 struct kvm *kvm;
-struct kvm_cpu **kvm_cpus;
 __thread struct kvm_cpu *current_kvm_cpu;
 
 static int  kvm_run_wrapper;
@@ -520,15 +519,15 @@ static void handle_debug(int fd, u32 type, u32 len, u8 *msg)
 		if ((int)vcpu >= kvm->nrcpus)
 			return;
 
-		kvm_cpus[vcpu]->needs_nmi = 1;
-		pthread_kill(kvm_cpus[vcpu]->thread, SIGUSR1);
+		kvm->cpus[vcpu]->needs_nmi = 1;
+		pthread_kill(kvm->cpus[vcpu]->thread, SIGUSR1);
 	}
 
 	if (!(dbg_type & KVM_DEBUG_CMD_TYPE_DUMP))
 		return;
 
 	for (i = 0; i < kvm->nrcpus; i++) {
-		struct kvm_cpu *cpu = kvm_cpus[i];
+		struct kvm_cpu *cpu = kvm->cpus[i];
 
 		if (!cpu)
 			continue;
@@ -561,7 +560,7 @@ static void handle_stop(int fd, u32 type, u32 len, u8 *msg)
 	if (WARN_ON(type != KVM_IPC_STOP || len))
 		return;
 
-	kvm_cpu__reboot();
+	kvm_cpu__reboot(kvm);
 }
 
 static void *kvm_cpu_thread(void *arg)
@@ -873,7 +872,6 @@ static int kvm_cmd_run_init(int argc, const char **argv)
 	static char real_cmdline[2048], default_name[20];
 	struct framebuffer *fb = NULL;
 	unsigned int nr_online_cpus;
-	int max_cpus, recommended_cpus;
 	int i, r;
 
 	kvm = kvm__new();
@@ -1011,24 +1009,12 @@ static int kvm_cmd_run_init(int argc, const char **argv)
 		goto fail;
 	}
 
-	max_cpus = kvm__max_cpus(kvm);
-	recommended_cpus = kvm__recommended_cpus(kvm);
-
-	if (kvm->cfg.nrcpus > max_cpus) {
-		printf("  # Limit the number of CPUs to %d\n", max_cpus);
-		kvm->cfg.nrcpus = max_cpus;
-	} else if (kvm->cfg.nrcpus > recommended_cpus) {
-		printf("  # Warning: The maximum recommended amount of VCPUs"
-			" is %d\n", recommended_cpus);
+	r = kvm_cpu__init(kvm);
+	if (r < 0) {
+		pr_err("kvm_cpu__init() failed with error %d\n", r);
+		goto fail;
 	}
 
-	kvm->nrcpus = kvm->cfg.nrcpus;
-
-	/* Alloc one pointer too many, so array ends up 0-terminated */
-	kvm_cpus = calloc(kvm->nrcpus + 1, sizeof(void *));
-	if (!kvm_cpus)
-		die("Couldn't allocate array for %d CPUs", kvm->nrcpus);
-
 	r = irq__init(kvm);
 	if (r < 0) {
 		pr_err("irq__init() failed with error %d\n", r);
@@ -1217,7 +1203,8 @@ static int kvm_cmd_run_init(int argc, const char **argv)
 		goto fail;
 	}
 
-	/* Device init all done; firmware init must
+	/*
+	 * Device init all done; firmware init must
 	 * come after this (it may set up device trees etc.)
 	 */
 
@@ -1234,12 +1221,6 @@ static int kvm_cmd_run_init(int argc, const char **argv)
 		}
 	}
 
-	for (i = 0; i < kvm->nrcpus; i++) {
-		kvm_cpus[i] = kvm_cpu__init(kvm, i);
-		if (!kvm_cpus[i])
-			die("unable to initialize KVM VCPU");
-	}
-
 	thread_pool__init(nr_online_cpus);
 fail:
 	return r;
@@ -1247,33 +1228,16 @@ fail:
 
 static int kvm_cmd_run_work(void)
 {
-	int i, r = -1;
+	int i;
 	void *ret = NULL;
 
 	for (i = 0; i < kvm->nrcpus; i++) {
-		if (pthread_create(&kvm_cpus[i]->thread, NULL, kvm_cpu_thread, kvm_cpus[i]) != 0)
+		if (pthread_create(&kvm->cpus[i]->thread, NULL, kvm_cpu_thread, kvm->cpus[i]) != 0)
 			die("unable to create KVM VCPU thread");
 	}
 
 	/* Only VCPU #0 is going to exit by itself when shutting down */
-	if (pthread_join(kvm_cpus[0]->thread, &ret) != 0)
-		r = 0;
-
-	kvm_cpu__delete(kvm_cpus[0]);
-	kvm_cpus[0] = NULL;
-
-	for (i = 1; i < kvm->nrcpus; i++) {
-		if (kvm_cpus[i]->is_running) {
-			pthread_kill(kvm_cpus[i]->thread, SIGKVMEXIT);
-			if (pthread_join(kvm_cpus[i]->thread, &ret) != 0)
-				die("pthread_join");
-			kvm_cpu__delete(kvm_cpus[i]);
-		}
-		if (ret == NULL)
-			r = 0;
-	}
-
-	return r;
+	return pthread_join(kvm->cpus[0]->thread, &ret);
 }
 
 static void kvm_cmd_run_exit(int guest_ret)
@@ -1282,6 +1246,10 @@ static void kvm_cmd_run_exit(int guest_ret)
 
 	compat__print_all_messages();
 
+	r = kvm_cpu__exit(kvm);
+	if (r < 0)
+		pr_warning("kvm_cpu__exit() failed with error %d\n", r);
+
 	r = symbol_exit(kvm);
 	if (r < 0)
 		pr_warning("symbol_exit() failed with error %d\n", r);
@@ -1336,8 +1304,6 @@ static void kvm_cmd_run_exit(int guest_ret)
 	if (r < 0)
 		pr_warning("pci__exit() failed with error %d\n", r);
 
-	free(kvm_cpus);
-
 	if (guest_ret == 0)
 		printf("\n  # KVM session ended normally.\n");
 }

hw/i8042.c

@@ -162,7 +162,7 @@ static void kbd_write_command(struct kvm *kvm, u8 val)
 		state.mode &= ~MODE_DISABLE_AUX;
 		break;
 	case I8042_CMD_SYSTEM_RESET:
-		kvm_cpu__reboot();
+		kvm_cpu__reboot(kvm);
 		break;
 	default:
 		break;

hw/vesa.c

@@ -80,6 +80,7 @@ struct framebuffer *vesa__init(struct kvm *kvm)
 		.mem			= mem,
 		.mem_addr		= VESA_MEM_ADDR,
 		.mem_size		= VESA_MEM_SIZE,
+		.kvm			= kvm,
 	};
 	return fb__register(&vesafb);
 }

include/kvm/framebuffer.h

@@ -22,6 +22,7 @@ struct framebuffer {
 	char				*mem;
 	u64				mem_addr;
 	u64				mem_size;
+	struct kvm			*kvm;
 
 	unsigned long			nr_targets;
 	struct fb_target_operations	*targets[FB_MAX_TARGETS];

include/kvm/kvm-cpu.h

@@ -4,13 +4,15 @@
 #include "kvm/kvm-cpu-arch.h"
 #include <stdbool.h>
 
-struct kvm_cpu *kvm_cpu__init(struct kvm *kvm, unsigned long cpu_id);
+int kvm_cpu__init(struct kvm *kvm);
+int kvm_cpu__exit(struct kvm *kvm);
+struct kvm_cpu *kvm_cpu__arch_init(struct kvm *kvm, unsigned long cpu_id);
 void kvm_cpu__delete(struct kvm_cpu *vcpu);
 void kvm_cpu__reset_vcpu(struct kvm_cpu *vcpu);
 void kvm_cpu__setup_cpuid(struct kvm_cpu *vcpu);
 void kvm_cpu__enable_singlestep(struct kvm_cpu *vcpu);
 void kvm_cpu__run(struct kvm_cpu *vcpu);
-void kvm_cpu__reboot(void);
+void kvm_cpu__reboot(struct kvm *kvm);
 int kvm_cpu__start(struct kvm_cpu *cpu);
 bool kvm_cpu__handle_exit(struct kvm_cpu *vcpu);
 

include/kvm/kvm.h

@@ -40,6 +40,7 @@ struct kvm {
 	timer_t			timerid;	/* Posix timer for interrupts */
 
 	int			nrcpus;		/* Number of cpus to run */
+	struct kvm_cpu		**cpus;
 
 	u32			mem_slots;	/* for KVM_SET_USER_MEMORY_REGION */
 	u64			ram_size;

kvm-cpu.c

@@ -12,7 +12,6 @@
 #include <errno.h>
 #include <stdio.h>
 
-extern struct kvm_cpu **kvm_cpus;
 extern __thread struct kvm_cpu *current_kvm_cpu;
 
 void kvm_cpu__enable_singlestep(struct kvm_cpu *vcpu)
@@ -65,14 +64,14 @@ static void kvm_cpu__handle_coalesced_mmio(struct kvm_cpu *cpu)
 	}
 }
 
-void kvm_cpu__reboot(void)
+void kvm_cpu__reboot(struct kvm *kvm)
 {
 	int i;
 
-	/* The kvm_cpus array contains a null pointer in the last location */
+	/* The kvm->cpus array contains a null pointer in the last location */
 	for (i = 0; ; i++) {
-		if (kvm_cpus[i])
-			pthread_kill(kvm_cpus[i]->thread, SIGKVMEXIT);
+		if (kvm->cpus[i])
+			pthread_kill(kvm->cpus[i]->thread, SIGKVMEXIT);
 		else
 			break;
 	}
@@ -173,3 +172,69 @@ exit_kvm:
 panic_kvm:
 	return 1;
 }
+
+int kvm_cpu__init(struct kvm *kvm)
+{
+	int max_cpus, recommended_cpus, i;
+
+	max_cpus = kvm__max_cpus(kvm);
+	recommended_cpus = kvm__recommended_cpus(kvm);
+
+	if (kvm->cfg.nrcpus > max_cpus) {
+		printf("  # Limit the number of CPUs to %d\n", max_cpus);
+		kvm->cfg.nrcpus = max_cpus;
+	} else if (kvm->cfg.nrcpus > recommended_cpus) {
+		printf("  # Warning: The maximum recommended amount of VCPUs"
+			" is %d\n", recommended_cpus);
+	}
+
+	kvm->nrcpus = kvm->cfg.nrcpus;
+
+	/* Alloc one pointer too many, so array ends up 0-terminated */
+	kvm->cpus = calloc(kvm->nrcpus + 1, sizeof(void *));
+	if (!kvm->cpus) {
+		pr_warning("Couldn't allocate array for %d CPUs", kvm->nrcpus);
+		return -ENOMEM;
+	}
+
+	for (i = 0; i < kvm->nrcpus; i++) {
+		kvm->cpus[i] = kvm_cpu__arch_init(kvm, i);
+		if (!kvm->cpus[i]) {
+			pr_warning("unable to initialize KVM VCPU");
+			goto fail_alloc;
+		}
+	}
+
+	return 0;
+
+fail_alloc:
+	for (i = 0; i < kvm->nrcpus; i++)
+		free(kvm->cpus[i]);
+	return -ENOMEM;
+}
+
+int kvm_cpu__exit(struct kvm *kvm)
+{
+	int i, r;
+	void *ret = NULL;
+
+	kvm_cpu__delete(kvm->cpus[0]);
+	kvm->cpus[0] = NULL;
+
+	for (i = 1; i < kvm->nrcpus; i++) {
+		if (kvm->cpus[i]->is_running) {
+			pthread_kill(kvm->cpus[i]->thread, SIGKVMEXIT);
+			if (pthread_join(kvm->cpus[i]->thread, &ret) != 0)
+				die("pthread_join");
+			kvm_cpu__delete(kvm->cpus[i]);
+		}
+		if (ret == NULL)
+			r = 0;
+	}
+
+	free(kvm->cpus);
+
+	kvm->nrcpus = 0;
+
+	return r;
+}

kvm.c

@@ -536,7 +536,7 @@ void kvm__pause(void)
 	int i, paused_vcpus = 0;
 
 	/* Check if the guest is running */
-	if (!kvm_cpus[0] || kvm_cpus[0]->thread == 0)
+	if (!kvm->cpus[0] || kvm->cpus[0]->thread == 0)
 		return;
 
 	mutex_lock(&pause_lock);
@@ -545,7 +545,7 @@ void kvm__pause(void)
 	if (pause_event < 0)
 		die("Failed creating pause notification event");
 	for (i = 0; i < kvm->nrcpus; i++)
-		pthread_kill(kvm_cpus[i]->thread, SIGKVMPAUSE);
+		pthread_kill(kvm->cpus[i]->thread, SIGKVMPAUSE);
 
 	while (paused_vcpus < kvm->nrcpus) {
 		u64 cur_read;
@@ -560,7 +560,7 @@ void kvm__pause(void)
 void kvm__continue(void)
 {
 	/* Check if the guest is running */
-	if (!kvm_cpus[0] || kvm_cpus[0]->thread == 0)
+	if (!kvm->cpus[0] || kvm->cpus[0]->thread == 0)
 		return;
 
 	mutex_unlock(&pause_lock);

powerpc/kvm-cpu.c

@@ -57,7 +57,7 @@ void kvm_cpu__delete(struct kvm_cpu *vcpu)
 	free(vcpu);
 }
 
-struct kvm_cpu *kvm_cpu__init(struct kvm *kvm, unsigned long cpu_id)
+struct kvm_cpu *kvm_cpu__arch_init(struct kvm *kvm, unsigned long cpu_id)
 {
 	struct kvm_cpu *vcpu;
 	int mmap_size;

term.c

@@ -35,7 +35,7 @@ int term_getc(int term)
 	if (term_got_escape) {
 		term_got_escape = false;
 		if (c == 'x')
-			kvm_cpu__reboot();
+			kvm_cpu__reboot(kvm);
 		if (c == term_escape_char)
 			return c;
 	}

ui/sdl.c

@@ -261,7 +261,7 @@ static void *sdl__thread(void *p)
 		return NULL;
 	}
 exit:
-	kvm_cpu__reboot();
+	kvm_cpu__reboot(fb->kvm);
 
 	return NULL;
 }

x86/kvm-cpu.c

@@ -90,7 +90,7 @@ static int kvm_cpu__set_lint(struct kvm_cpu *vcpu)
 	return ioctl(vcpu->vcpu_fd, KVM_SET_LAPIC, &lapic);
 }
 
-struct kvm_cpu *kvm_cpu__init(struct kvm *kvm, unsigned long cpu_id)
+struct kvm_cpu *kvm_cpu__arch_init(struct kvm *kvm, unsigned long cpu_id)
 {
 	struct kvm_cpu *vcpu;
 	int mmap_size;