tests/qemu: Add boot-benchmark.

Add a new test program called 'boot-benchmark'.  This is similar to
'boot-analysis' but it simply boots and shuts down the appliance
several times in a row and measures how long it takes, calculating
mean and standard deviation.

.gitignore

@@ -511,6 +511,7 @@ Makefile.in
 /tests/parallel/test-parallel
 /tests/protocol/test-error-messages
 /tests/qemu/boot-analysis
+/tests/qemu/boot-benchmark
 /tests/qemu/qemu-boot
 /tests/qemu/qemu-speed-test
 /tests/regressions/rhbz501893

docs/guestfs-performance.pod

@@ -29,11 +29,20 @@ appliance:
 Run this command several times in a row and discard the first few
 runs, so that you are measuring a typical "hot cache" case.
 
+I<Side note for developers:> If you are compiling libguestfs from
+source, there is a program called F<tests/qemu/boot-benchmark> which
+does the same thing, but performs multiple runs and prints the mean
+and standard deviation.  To run it, do:
+
+ make
+ make -C tests/qemu boot-benchmark
+ ./run ./tests/qemu/boot-benchmark
+
 =head3 Explanation
 
-This command starts up the libguestfs appliance on a null disk, and
-then immediately shuts it down.  The first time you run the command,
-it will create an appliance and cache it (usually under
+The guestfish command above starts up the libguestfs appliance on a
+null disk, and then immediately shuts it down.  The first time you run
+the command, it will create an appliance and cache it (usually under
 F</var/tmp/.guestfs-*>).  Subsequent runs should reuse the cached
 appliance.
 

tests/qemu/Makefile.am

@@ -1,5 +1,5 @@
 # libguestfs
-# Copyright (C) 2011 Red Hat Inc.
+# Copyright (C) 2011-2016 Red Hat Inc.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -33,10 +33,11 @@ EXTRA_DIST = \
 	qemu-boot.c \
 	qemu-speed-test.c
 
-# qemu-boot, qemu-speed-test and boot-analysis are built but not run
-# by default as they are mainly qemu & kernel diagnostic tools.
+# qemu-boot, qemu-speed-test, boot-analysis and boot-benchmark are
+# built but not run by default as they are mainly qemu & kernel
+# diagnostic tools.
 
-check_PROGRAMS = qemu-boot qemu-speed-test boot-analysis
+check_PROGRAMS = qemu-boot qemu-speed-test boot-analysis boot-benchmark
 
 qemu_boot_SOURCES = \
 	../../df/estimate-max-threads.c \
@@ -76,7 +77,9 @@ qemu_speed_test_LDADD = \
 boot_analysis_SOURCES = \
 	boot-analysis.c \
 	boot-analysis.h \
-	boot-analysis-timeline.c
+	boot-analysis-timeline.c \
+	boot-analysis-utils.c \
+	boot-analysis-utils.h
 boot_analysis_CPPFLAGS = \
 	-I$(top_srcdir)/gnulib/lib -I$(top_builddir)/gnulib/lib \
 	-I$(top_srcdir)/src -I$(top_builddir)/src
@@ -93,6 +96,23 @@ boot_analysis_LDADD = \
 	$(top_builddir)/gnulib/lib/libgnu.la \
 	-lm
 
+boot_benchmark_SOURCES = \
+	boot-benchmark.c \
+	boot-analysis-utils.c \
+	boot-analysis-utils.h
+boot_benchmark_CPPFLAGS = \
+	-I$(top_srcdir)/gnulib/lib -I$(top_builddir)/gnulib/lib \
+	-I$(top_srcdir)/src -I$(top_builddir)/src
+boot_benchmark_CFLAGS = \
+	$(WARN_CFLAGS) $(WERROR_CFLAGS)
+boot_benchmark_LDADD = \
+	$(top_builddir)/src/libutils.la \
+	$(top_builddir)/src/libguestfs.la \
+	$(LIBXML2_LIBS) \
+	$(LTLIBINTL) \
+	$(top_builddir)/gnulib/lib/libgnu.la \
+	-lm
+
 # Don't run these tests in parallel, since they are designed to check
 # the integrity of qemu.
 .NOTPARALLEL:

tests/qemu/boot-analysis-utils.c

@@ -0,0 +1,47 @@
+/* libguestfs
+ * Copyright (C) 2016 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#include <config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <time.h>
+#include <error.h>
+#include <errno.h>
+
+#include "guestfs.h"
+#include "guestfs-internal-frontend.h"
+
+#include "boot-analysis-utils.h"
+
+void
+get_time (struct timespec *ts)
+{
+  if (clock_gettime (CLOCK_REALTIME, ts) == -1)
+    error (EXIT_FAILURE, errno, "clock_gettime: CLOCK_REALTIME");
+}
+
+int64_t
+timespec_diff (const struct timespec *x, const struct timespec *y)
+{
+  int64_t nsec;
+
+  nsec = (y->tv_sec - x->tv_sec) * UINT64_C(1000000000);
+  nsec += y->tv_nsec - x->tv_nsec;
+  return nsec;
+}

tests/qemu/boot-analysis-utils.h

@@ -0,0 +1,30 @@
+/* libguestfs
+ * Copyright (C) 2016 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#ifndef GUESTFS_BOOT_ANALYSIS_UTILS_H_
+#define GUESTFS_BOOT_ANALYSIS_UTILS_H_
+
+/* Get current time, returning it in *ts.  If there is a system call
+ * failure, this exits.
+ */
+extern void get_time (struct timespec *ts);
+
+/* Computes Y - X, returning nanoseconds. */
+extern int64_t timespec_diff (const struct timespec *x, const struct timespec *y);
+
+#endif /* GUESTFS_BOOT_ANALYSIS_UTILS_H_ */

tests/qemu/boot-analysis.c

@@ -79,6 +79,7 @@
 #include "guestfs-internal-frontend.h"
 
 #include "boot-analysis.h"
+#include "boot-analysis-utils.h"
 
 /* Activities taking longer than this % of the total time, except
  * those flagged as LONG_ACTIVITY, are highlighted in red.
@@ -96,8 +97,6 @@ static int smp = 1;
 static int verbose = 0;
 
 static void run_test (void);
-static void get_time (struct timespec *ts);
-static int64_t timespec_diff (const struct timespec *x, const struct timespec *y);
 static struct event *add_event (struct pass_data *, uint64_t source);
 static guestfs_h *create_handle (void);
 static void set_up_event_handlers (guestfs_h *g, size_t pass);
@@ -267,24 +266,6 @@ run_test (void)
   free_final_timeline ();
 }
 
-static void
-get_time (struct timespec *ts)
-{
-  if (clock_gettime (CLOCK_REALTIME, ts) == -1)
-    error (EXIT_FAILURE, errno, "clock_gettime: CLOCK_REALTIME");
-}
-
-/* Computes Y - X, returning nanoseconds. */
-static int64_t
-timespec_diff (const struct timespec *x, const struct timespec *y)
-{
-  int64_t nsec;
-
-  nsec = (y->tv_sec - x->tv_sec) * UINT64_C(1000000000);
-  nsec += y->tv_nsec - x->tv_nsec;
-  return nsec;
-}
-
 static struct event *
 add_event (struct pass_data *data, uint64_t source)
 {

tests/qemu/boot-benchmark.c

@@ -0,0 +1,230 @@
+/* libguestfs
+ * Copyright (C) 2016 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+/* Benchmark the time taken to boot the libguestfs appliance. */
+
+#include <config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <inttypes.h>
+#include <string.h>
+#include <getopt.h>
+#include <limits.h>
+#include <time.h>
+#include <errno.h>
+#include <error.h>
+#include <assert.h>
+#include <math.h>
+
+#include "guestfs.h"
+#include "guestfs-internal-frontend.h"
+
+#include "boot-analysis-utils.h"
+
+#define NR_WARMUP_PASSES 3
+#define NR_TEST_PASSES   10
+
+static const char *append = NULL;
+static int memsize = 0;
+static int smp = 1;
+
+static void run_test (void);
+static guestfs_h *create_handle (void);
+static void add_drive (guestfs_h *g);
+
+static void
+usage (int exitcode)
+{
+  guestfs_h *g;
+  int default_memsize = -1;
+
+  g = guestfs_create ();
+  if (g) {
+    default_memsize = guestfs_get_memsize (g);
+    guestfs_close (g);
+  }
+
+  fprintf (stderr,
+           "boot-benchmark: Benchmark the time taken to boot the libguestfs appliance.\n"
+           "Usage:\n"
+           "  boot-benchmark [--options]\n"
+           "Options:\n"
+           "  --help         Display this usage text and exit.\n"
+           "  --append OPTS  Append OPTS to kernel command line.\n"
+           "  -m MB\n"
+           "  --memsize MB   Set memory size in MB (default: %d).\n"
+           "  --smp N        Enable N virtual CPUs (default: 1).\n",
+           default_memsize);
+  exit (exitcode);
+}
+
+int
+main (int argc, char *argv[])
+{
+  enum { HELP_OPTION = CHAR_MAX + 1 };
+  static const char *options = "m:";
+  static const struct option long_options[] = {
+    { "help", 0, 0, HELP_OPTION },
+    { "append", 1, 0, 0 },
+    { "memsize", 1, 0, 'm' },
+    { "smp", 1, 0, 0 },
+    { 0, 0, 0, 0 }
+  };
+  int c, option_index;
+
+  for (;;) {
+    c = getopt_long (argc, argv, options, long_options, &option_index);
+    if (c == -1) break;
+
+    switch (c) {
+    case 0:                     /* Options which are long only. */
+      if (STREQ (long_options[option_index].name, "append")) {
+        append = optarg;
+        break;
+      }
+      else if (STREQ (long_options[option_index].name, "smp")) {
+        if (sscanf (optarg, "%d", &smp) != 1) {
+          fprintf (stderr, "%s: could not parse smp parameter: %s\n",
+                   guestfs_int_program_name, optarg);
+          exit (EXIT_FAILURE);
+        }
+        break;
+      }
+      fprintf (stderr, "%s: unknown long option: %s (%d)\n",
+               guestfs_int_program_name, long_options[option_index].name, option_index);
+      exit (EXIT_FAILURE);
+
+    case 'm':
+      if (sscanf (optarg, "%d", &memsize) != 1) {
+        fprintf (stderr, "%s: could not parse memsize parameter: %s\n",
+                 guestfs_int_program_name, optarg);
+        exit (EXIT_FAILURE);
+      }
+      break;
+
+    case HELP_OPTION:
+      usage (EXIT_SUCCESS);
+
+    default:
+      usage (EXIT_FAILURE);
+    }
+  }
+
+  run_test ();
+}
+
+static void
+run_test (void)
+{
+  guestfs_h *g;
+  size_t i;
+  int64_t ns[NR_TEST_PASSES];
+  double mean;
+  double variance;
+  double sd;
+
+  printf ("Warming up the libguestfs cache ...\n");
+  for (i = 0; i < NR_WARMUP_PASSES; ++i) {
+    g = create_handle ();
+    add_drive (g);
+    if (guestfs_launch (g) == -1)
+      exit (EXIT_FAILURE);
+    guestfs_close (g);
+  }
+
+  printf ("Running the tests ...\n");
+  for (i = 0; i < NR_TEST_PASSES; ++i) {
+    struct timespec start_t, end_t;
+
+    g = create_handle ();
+    add_drive (g);
+    get_time (&start_t);
+    if (guestfs_launch (g) == -1)
+      exit (EXIT_FAILURE);
+    guestfs_close (g);
+    get_time (&end_t);
+
+    ns[i] = timespec_diff (&start_t, &end_t);
+  }
+
+  /* Calculate the mean. */
+  mean = 0;
+  for (i = 0; i < NR_TEST_PASSES; ++i)
+    mean += ns[i];
+  mean /= NR_TEST_PASSES;
+
+  /* Calculate the variance and standard deviation. */
+  variance = 0;
+  for (i = 0; i < NR_TEST_PASSES; ++i)
+    variance = pow (ns[i] - mean, 2);
+  variance /= NR_TEST_PASSES;
+  sd = sqrt (variance);
+
+  /* Print the test parameters. */
+  printf ("\n");
+  printf (" passes %d\n", NR_TEST_PASSES);
+  g = create_handle ();
+  printf (" append %s\n", guestfs_get_append (g) ? : "");
+  printf ("backend %s\n", guestfs_get_backend (g));
+  printf ("     hv %s\n", guestfs_get_hv (g));
+  printf ("memsize %d\n", guestfs_get_memsize (g));
+  printf ("    smp %d\n", guestfs_get_smp (g));
+  guestfs_close (g);
+
+  /* Print the result. */
+  printf ("\n");
+  printf ("Result: %.1fms ±%.1fms\n", mean / 1000000, sd / 1000000);
+}
+
+/* Common function to create the handle and set various defaults. */
+static guestfs_h *
+create_handle (void)
+{
+  guestfs_h *g;
+  CLEANUP_FREE char *full_append = NULL;
+
+  g = guestfs_create ();
+  if (!g) error (EXIT_FAILURE, errno, "guestfs_create");
+
+  if (memsize != 0)
+    if (guestfs_set_memsize (g, memsize) == -1)
+      exit (EXIT_FAILURE);
+
+  if (smp >= 2)
+    if (guestfs_set_smp (g, smp) == -1)
+      exit (EXIT_FAILURE);
+
+  if (append != NULL)
+    if (guestfs_set_append (g, full_append) == -1)
+      exit (EXIT_FAILURE);
+
+  return g;
+}
+
+/* Common function to add the /dev/null drive. */
+static void
+add_drive (guestfs_h *g)
+{
+  if (guestfs_add_drive_opts (g, "/dev/null",
+                              GUESTFS_ADD_DRIVE_OPTS_FORMAT, "raw",
+                              GUESTFS_ADD_DRIVE_OPTS_READONLY, 1,
+                              -1) == -1)
+    exit (EXIT_FAILURE);
+}