kvm tools: Use mmap by default for disk images

Use mmap() for disk images and fallback to read()/write() if it fails (for
example, with large images on 32 bit systems).

Performance (kB/s) test using bonnie++ shows 19% improvement in block writes,
7% improvement in block rewrites, and 6% improvement in block reads.

Guest was started with the following command:

  ./kvm run --mem=256 --image=./work/vms/gentoo.img --kernel=/boot/bzImage-git

and bonnie++ with:

  bonnie++ -u 0

Before:
------

  Version  1.96       ------Sequential Output----- --Sequential Input- -Random-
  Concurrency   1     -Per Chr- --Block-- -Rewrite- -Per Chr- --Block-- --Seeks--
  Machine        Size K/sec %CP K/sec %CP K/sec %CP K/sec %CP K/sec %CP  /sec %CP
  tux            480M   673 100 308017  61 288156  49  3286  99 892186  76 +++++ +++
  Latency             12998us   50992us   35993us    3000us    1999us     201ms
  Version  1.96       ------Sequential Create------ --------Random Create--------
  tux                 -Create-- --Read--- -Delete-- -Create-- --Read--- -Delete--
                files  /sec %CP  /sec %CP  /sec %CP  /sec %CP  /sec %CP  /sec %CP
                   16 +++++ +++ +++++ +++ +++++ +++ +++++ +++ +++++ +++ +++++ +++
  Latency              3000us    1000us    1000us    1000us    1998us    1000us

After:
------

  Version  1.96       ------Sequential Output------ --Sequential Input- --Random-
  Concurrency   1     -Per Chr- --Block-- -Rewrite- -Per Chr- --Block-- --Seeks--
  Machine        Size K/sec %CP K/sec %CP K/sec %CP K/sec %CP K/sec %CP  /sec %CP
  tux            480M   674  99 366588  42 309514  44  3539  99 954554  76 +++++ +++
  Latency             17997us   15997us   37993us    4999us    1998us     145ms
  Version  1.96       ------Sequential Create------ --------Random Create--------
  tux                 -Create-- --Read--- -Delete-- -Create-- --Read--- -Delete--
                files  /sec %CP  /sec %CP  /sec %CP  /sec %CP  /sec %CP  /sec %CP
                   16 +++++ +++ +++++ +++ +++++ +++ +++++ +++ +++++ +++ +++++ +++
  Latency              1000us    1000us    2000us    1999us    1000us    1000us

[ penberg@kernel.org: drop MAP_NORESERVE, it's no-op for MAP_SHARED ]
Signed-off-by: Sasha Levin <levinsasha928@gmail.com>
Signed-off-by: Pekka Enberg <penberg@kernel.org>

disk-image.c

@@ -13,21 +13,31 @@
 #include <unistd.h>
 #include <fcntl.h>
 
-struct disk_image *disk_image__new(int fd, uint64_t size, struct disk_image_operations *ops)
+static int raw_image__read_sector_mmap(struct disk_image *self, uint64_t sector, void *dst, uint32_t dst_len)
 {
-	struct disk_image *self;
+	uint64_t offset = sector << SECTOR_SHIFT;
 
-	self		= malloc(sizeof *self);
-	if (!self)
-		return NULL;
+	if (offset + dst_len > self->size)
+		return -1;
 
-	self->fd	= fd;
-	self->size	= size;
-	self->ops	= ops;
+	memcpy(dst, self->priv + offset, dst_len);
 
-	return self;
+	return 0;
 }
 
+static int raw_image__write_sector_mmap(struct disk_image *self, uint64_t sector, void *src, uint32_t src_len)
+{
+	uint64_t offset = sector << SECTOR_SHIFT;
+
+	if (offset + src_len > self->size)
+		return -1;
+
+	memcpy(self->priv + offset, src, src_len);
+
+	return 0;
+}
+
+
 static int raw_image__read_sector(struct disk_image *self, uint64_t sector, void *dst, uint32_t dst_len)
 {
 	uint64_t offset = sector << SECTOR_SHIFT;
@@ -59,6 +69,27 @@ static struct disk_image_operations raw_image_ops = {
 	.write_sector		= raw_image__write_sector,
 };
 
+static struct disk_image_operations raw_image_mmap_ops = {
+	.read_sector		= raw_image__read_sector_mmap,
+	.write_sector		= raw_image__write_sector_mmap,
+};
+
+struct disk_image *disk_image__new(int fd, uint64_t size)
+{
+	struct disk_image *self;
+
+	self		= malloc(sizeof *self);
+	if (!self)
+		return NULL;
+
+	self->fd	= fd;
+	self->size	= size;
+	self->priv	= mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+	self->ops	= (self->priv == MAP_FAILED) ? &raw_image_ops : &raw_image_mmap_ops;
+
+	return self;
+}
+
 static struct disk_image *raw_image__probe(int fd)
 {
 	struct stat st;
@@ -66,7 +97,7 @@ static struct disk_image *raw_image__probe(int fd)
 	if (fstat(fd, &st) < 0)
 		return NULL;
 
-	return disk_image__new(fd, st.st_size, &raw_image_ops);
+	return disk_image__new(fd, st.st_size);
 }
 
 struct disk_image *disk_image__open(const char *filename)
@@ -97,6 +128,9 @@ void disk_image__close(struct disk_image *self)
 	if (self->ops->close)
 		self->ops->close(self);
 
+	if (self->priv != MAP_FAILED)
+		munmap(self->priv, self->size);
+
 	if (close(self->fd) < 0)
 		warning("close() failed");
 

include/kvm/disk-image.h

@@ -22,7 +22,7 @@ struct disk_image {
 };
 
 struct disk_image *disk_image__open(const char *filename);
-struct disk_image *disk_image__new(int fd, uint64_t size, struct disk_image_operations *ops);
+struct disk_image *disk_image__new(int fd, uint64_t size);
 void disk_image__close(struct disk_image *self);
 
 static inline int disk_image__read_sector(struct disk_image *self, uint64_t sector, void *dst, uint32_t dst_len)