Experimental implementation of the supermin appliance (passes most tests).

.gitignore

@@ -20,8 +20,14 @@ ChangeLog
 Makefile.in
 Makefile
 aclocal.m4
+appliance/guestfs-supermin-helper
 appliance/initramfs.*.img
+appliance/initramfs.*.supermin.hostfiles
+appliance/kmod.whitelist
 appliance/make.sh
+appliance/supermin-make.sh
+appliance/supermin-split.sh
+appliance/supermin.incfiles
 appliance/update.sh
 appliance/vmlinuz.*
 autom4te.cache

appliance/Makefile.am

@@ -16,20 +16,37 @@
 # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 
 EXTRA_DIST = \
-	make.sh update.sh
+	make.sh update.sh supermin-split.sh supermin-make.sh
 
 # Build the root filesystem (appliance).
 # Currently this is arch-dependent, so it seems like putting it in
 # $(libdir) is best.  When we build cross-architecture filesystems we
 # should probably move them to $(datadir).
-
 fsdir = $(libdir)/guestfs
+fs_DATA = $(APPLIANCE_FILES)
 
+# These are the resulting output files from the whole process:
+#   VMLINUZ        kernel for the full appliance
+#   INITRAMFSIMG   initramfs (ie. root fs) for the full appliance
+# For details of the supermin appliance, read the README file:
+#   SUPERMINIMG    initramfs (ie. partial root fs) for the supermin appliance
+#   SUPERMINFILES  list of missing files (the ones we will pull out of the
+#                    host filesystem at runtime) in the supermin appliance
+APPLIANCE_FILES = $(INITRAMFSIMG) $(VMLINUZ)
+if SUPERMIN
+APPLIANCE_FILES += $(SUPERMINIMG) $(SUPERMINFILES) kmod.whitelist
+bin_SCRIPTS = guestfs-supermin-helper
+endif
+
+# Don't change these names - they must be the same as in '*.sh' scripts.
 INITRAMFSIMG = initramfs.$(REPO).$(host_cpu).img
 VMLINUZ = vmlinuz.$(REPO).$(host_cpu)
+if SUPERMIN
+SUPERMINIMG = initramfs.$(REPO).$(host_cpu).supermin.img
+SUPERMINFILES = initramfs.$(REPO).$(host_cpu).supermin.hostfiles
+endif
 
-fs_DATA =  $(INITRAMFSIMG) $(VMLINUZ)
-
+# This is for building the normal appliance:
 $(INITRAMFSIMG) $(VMLINUZ): $(top_builddir)/initramfs/fakeroot.log
 
 $(top_builddir)/initramfs/fakeroot.log: make.sh kmod.whitelist
@@ -37,15 +54,49 @@ $(top_builddir)/initramfs/fakeroot.log: make.sh kmod.whitelist
 	-mv $(VMLINUZ) $(VMLINUZ).bak
 	if ! bash make.sh; then rm -f $@; exit 1; fi
 
-$(INITRAMFSIMG): $(top_builddir)/initramfs/fakeroot.log $(top_builddir)/daemon/guestfsd
+$(INITRAMFSIMG): $(top_builddir)/initramfs/fakeroot.log $(top_builddir)/daemon/guestfsd update.sh
 	rm -f $@
 	bash update.sh
 	touch $@
 
-make.sh: make.sh.in
-	cd .. && ./config.status appliance/$@
+kmod.whitelist: kmod.whitelist.in
+	grep -v '^[[:space:]]*$$' < $< | grep -v '^#' > $@
 
-# Test-boot the image.
+# This is for building the supermin appliance.  It has to be enabled
+# specifically with './configure --enable-supermin'.  You really need
+# to read the README file.
+
+if SUPERMIN
+
+# First we need to decide which files go in and out of the supermin
+# appliance.  This decision is made by 'supermin-split.sh'.
+$(SUPERMINFILES): supermin.incfiles
+supermin.incfiles: $(top_builddir)/initramfs/fakeroot.log supermin-split.sh
+	rm -f supermin.incfiles $(SUPERMINFILES)
+	bash supermin-split.sh
+
+# Second we need to create a supermin appliance with just the included
+# files (leaving out the host files, which we'll add back at runtime).
+$(SUPERMINIMG): supermin.incfiles supermin-make.sh
+	rm -f $@
+	bash supermin-make.sh
+
+endif
+
+# This should rebuild the scripts if the input files change, although
+# it doesn't always seem to work.
+%.sh: %.sh.in
+	cd .. && ./config.status appliance/$@
+	chmod +x $@
+
+guestfs-supermin-helper: guestfs-supermin-helper.in
+	cd .. && ./config.status appliance/$@
+	chmod +x $@
+
+#----------------------------------------------------------------------
+# Extra rules for testing the appliance.
+
+# Test-boot the appliance.
 
 test-boot: emptydisk
 	qemu-system-$(host_cpu) \
@@ -75,7 +126,7 @@ test-boot-realistic: emptydisk
 
 # Make clean.
 
-CLEANFILES = $(fs_DATA)
+CLEANFILES = $(APPLIANCE_FILES)
 
 clean-local:
 	rm -rf $(top_builddir)/initramfs

appliance/guestfs-supermin-helper.in

@@ -0,0 +1,80 @@
+#!/bin/bash -
+# @configure_input@
+# Copyright (C) 2009 Red Hat Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+# Helper script which constructs the supermin appliance at runtime.
+
+unset CDPATH
+
+set -e
+
+# Source directory containing the supermin input files.
+sourcedir=$(cd "$1" > /dev/null; pwd)
+
+# Output files.
+kernel="$2"
+initrd="$3"
+
+# Look for the kernel first.  This is very unsophisticated: We
+# just look for any kernel named vmlinuz-*.$host_cpu which has a
+# corresponding /lib/modules/*.$host_cpu directory.
+
+for f in /boot/vmlinuz-*.@host_cpu@; do
+    b=$(basename "$f")
+    b=$(echo "$b" | sed 's,vmlinuz-,,')
+    modpath="/lib/modules/$b"
+    if [ -d "$modpath" ]; then
+	ln -sf "$f" "$kernel"
+	break
+    fi
+    modpath=
+done
+
+if [ -z "$modpath" ]; then
+    echo "$0: failed to find a suitable kernel" >&2
+    exit 1
+fi
+
+# The initrd consists of these components:
+# (1) The base skeleton appliance that we constructed at build time.
+#     name = initramfs.@REPO@.@host_cpu@.supermin.img
+#     format = compressed cpio
+# (2) The modules from modpath which are on the module whitelist.
+#     format = plain cpio
+# (3) The host files which match wildcards in *.supermin.hostfiles.
+#     format = plain cpio
+
+cp "$sourcedir"/initramfs.@REPO@.@host_cpu@.supermin.img "$initrd"
+
+# Kernel modules (2).
+exec 5<"$sourcedir"/kmod.whitelist
+whitelist=
+while read kmod 0<&5; do
+    whitelist="$whitelist -a -not -name $kmod"
+done
+exec 5<&-
+
+find "$modpath" -not -name '*.ko' -o \( -name '*.ko' $whitelist \) -a -print0 |
+  cpio --quiet -o -0 -H newc >> "$initrd"
+
+# Host files (3).
+
+(cd / && \
+  ls -1df $(
+      cat "$sourcedir"/initramfs.@REPO@.@host_cpu@.supermin.hostfiles
+    ) 2>/dev/null |
+  cpio --quiet -o -H newc ) >> "$initrd"

appliance/make.sh.in

@@ -80,18 +80,22 @@ rm -f $koutput
 # Don't need any keyboard maps.
 @FEBOOTSTRAP_RUN@ initramfs -- rm -rf /lib/kbd
 
+# Remove anything in home directory.  Because this is potentially
+# liable to monstrous fuck-ups, we don't put a slash before 'home'.
+(cd initramfs && echo home/*) |
+  xargs @FEBOOTSTRAP_RUN@ initramfs -- rm -rf
+
+# Remove /var/lib/yum stuff.
+@FEBOOTSTRAP_RUN@ initramfs -- rm -rf /var/lib/yum
+
 # Kernel modules take up nearly half of the image.  Only include ones
 # which are on the whitelist.
-grep -v '^[[:space:]]*$' < appliance/kmod.whitelist |
-  grep -v '^#' > kmod.whitelist.tmp
-exec 5<kmod.whitelist.tmp
+exec 5<appliance/kmod.whitelist
 whitelist=
 while read kmod 0<&5; do
     whitelist="$whitelist -a -not -name $kmod"
 done
 exec 5<&-
-rm kmod.whitelist.tmp
-#echo whitelist=$whitelist
 
 (cd initramfs && \
   find lib/modules/*/kernel -name '*.ko' $whitelist -a -print0 ) |

appliance/supermin-make.sh.in

@@ -0,0 +1,33 @@
+#!/bin/bash -
+# @configure_input@
+# Copyright (C) 2009 Red Hat Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+# Build the supermin appliance.
+# Read the README file!
+
+unset CDPATH
+
+set -e
+
+cd @top_builddir@
+
+output=appliance/initramfs.@REPO@.@host_cpu@.supermin.img
+
+# Generate final image.
+@FEBOOTSTRAP_TO_INITRAMFS@ --files=$(pwd)/appliance/supermin.incfiles initramfs > $output-t
+mv $output-t $output
+ls -lh $output

appliance/supermin-split.sh.in

@@ -0,0 +1,93 @@
+#!/bin/bash -
+# @configure_input@
+# Copyright (C) 2009 Red Hat Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+# Decide which files will stay in the supermin appliance and which
+# files will be pulled out of the host at runtime.
+#
+# Read the README file!
+#
+# The basic idea is that we create two output files, one containing
+# the files that will stay, and the other listing the files that
+# will be pulled from the host (ie. not go into the appliance now).
+#
+# The list of files that stay ('supermin.incfiles') is just a straight
+# list of files and directories.
+#
+# The list of files that come from the host ('*.supermin.hostfiles')
+# can include wildcards, to allow libraries to be upgraded on the
+# host.
+
+unset CDPATH
+
+set -e
+
+cd @top_builddir@/initramfs
+
+incfiles=../appliance/supermin.incfiles
+hostfiles=../appliance/initramfs.@REPO@.@host_cpu@.supermin.hostfiles
+
+exec 5>$incfiles
+exec 6>$hostfiles
+
+# Note currently the initramfs contains ~2500 files, and none have
+# "funny characters" in the names.  So this is reasonable just to
+# simplify the script.
+for path in $(find -not -name fakeroot.log); do
+    dir=$(dirname "$path")
+    file=$(basename "$path")
+
+    # All we're going to keep are the special files /init, the daemon,
+    # configuration files (/etc), devices and modifiable stuff (/var).
+    if [ "$path" = "./init" -o "$file" = "guestfsd" ]; then
+	echo "$path" >&5
+
+    elif [[ "$path" =~ '^\./etc' || "$path" =~ '^./dev' || "$path" =~ '^\./var' ]]; then
+	echo "$path" >&5
+
+    # Kernel modules are always copied in from the host, including all
+    # the dependency files.
+    elif [[ "$path" =~ '^\./lib/modules/' ]]; then
+	:
+
+    elif [ -d "$path" ]; then
+	# Always write directory names to both output files.
+	echo "$path" >&5
+	echo "$path" >&6
+
+    # Some libraries need fixed version numbers replaced by wildcards.
+
+    elif [[ "$file" =~ '^ld-[.0-9]+\.so$' ]]; then
+	echo "$dir/ld-*.so" >&6
+
+    # libfoo-1.2.3.so
+    elif [[ "$file" =~ '^lib(.*)-[-.0-9]+\.so$' ]]; then
+	echo "$dir/lib${BASH_REMATCH[1]}-*.so" >&6
+
+    # libfoo-1.2.3.so.1.2.3 (but NOT '*.so.N')
+    elif [[ "$file" =~ '^lib(.*)-[-.0-9]+\.so\.([0-9]+)\.' ]]; then
+	echo "$dir/lib${BASH_REMATCH[1]}-*.so.${BASH_REMATCH[2]}.*" >&6
+
+    # libfoo.so.1.2.3 (but NOT '*.so.N')
+    elif [[ "$file" =~ '^lib(.*)\.so\.([0-9]+)\.' ]]; then
+	echo "$dir/lib${BASH_REMATCH[1]}.so.${BASH_REMATCH[2]}.*" >&6
+
+    else
+	# Anything else comes from the host directly.
+	echo "$path" >&6
+    fi
+done

configure.ac

@@ -480,6 +480,8 @@ AC_CONFIG_FILES([Makefile
 		 src/Makefile fish/Makefile po/Makefile.in examples/Makefile
 		 appliance/Makefile
 	         appliance/make.sh appliance/update.sh
+		 appliance/supermin-split.sh appliance/supermin-make.sh
+		 appliance/guestfs-supermin-helper
 		 images/Makefile
 		 capitests/Makefile
 		 regressions/Makefile
@@ -495,7 +497,7 @@ AC_CONFIG_FILES([Makefile
 AC_OUTPUT
 
 dnl WTF?
-chmod +x appliance/make.sh appliance/update.sh
+chmod +x appliance/*.sh appliance/guestfs-supermin-helper
 
 dnl Produce summary.
 echo

src/guestfs.c

@@ -31,6 +31,7 @@
 #include <time.h>
 #include <sys/stat.h>
 #include <sys/select.h>
+#include <dirent.h>
 
 #include <rpc/types.h>
 #include <rpc/xdr.h>
@@ -309,6 +310,12 @@ guestfs_close (guestfs_h *g)
     snprintf (filename, sizeof filename, "%s/sock", g->tmpdir);
     unlink (filename);
 
+    snprintf (filename, sizeof filename, "%s/initrd", g->tmpdir);
+    unlink (filename);
+
+    snprintf (filename, sizeof filename, "%s/kernel", g->tmpdir);
+    unlink (filename);
+
     rmdir (g->tmpdir);
 
     free (g->tmpdir);
@@ -706,6 +713,46 @@ guestfs_add_cdrom (guestfs_h *g, const char *filename)
   return guestfs_config (g, "-cdrom", filename);
 }
 
+/* Returns true iff file is contained in dir. */
+static int
+dir_contains_file (const char *dir, const char *file)
+{
+  int dirlen = strlen (dir);
+  int filelen = strlen (file);
+  int len = dirlen+filelen+2;
+  char path[len];
+
+  snprintf (path, len, "%s/%s", dir, file);
+  return access (path, F_OK) == 0;
+}
+
+/* Returns true iff every listed file is contained in 'dir'. */
+static int
+dir_contains_files (const char *dir, ...)
+{
+  va_list args;
+  const char *file;
+
+  va_start (args, dir);
+  while ((file = va_arg (args, const char *)) != NULL) {
+    if (!dir_contains_file (dir, file)) {
+      va_end (args);
+      return 0;
+    }
+  }
+  va_end (args);
+  return 1;
+}
+
+static int build_supermin_appliance (guestfs_h *g, const char *path, char **kernel, char **initrd);
+
+static const char *kernel_name = "vmlinuz." REPO "." host_cpu;
+static const char *initrd_name = "initramfs." REPO "." host_cpu ".img";
+static const char *supermin_name =
+  "initramfs." REPO "." host_cpu ".supermin.img";
+static const char *supermin_hostfiles_name =
+  "initramfs." REPO "." host_cpu ".supermin.hostfiles";
+
 int
 guestfs_launch (guestfs_h *g)
 {
@@ -714,8 +761,6 @@ guestfs_launch (guestfs_h *g)
   size_t len;
   int wfd[2], rfd[2];
   int tries;
-  const char *kernel_name = "vmlinuz." REPO "." host_cpu;
-  const char *initrd_name = "initramfs." REPO "." host_cpu ".img";
   char *path, *pelem, *pend;
   char *kernel = NULL, *initrd = NULL;
   char unixsock[256];
@@ -732,7 +777,20 @@ guestfs_launch (guestfs_h *g)
     return -1;
   }
 
-  /* Search g->path for the kernel and initrd. */
+  /* Make the temporary directory. */
+  if (!g->tmpdir) {
+    g->tmpdir = safe_strdup (g, dir_template);
+    if (mkdtemp (g->tmpdir) == NULL) {
+      perrorf (g, _("%s: cannot create temporary directory"), dir_template);
+      goto cleanup0;
+    }
+  }
+
+  /* First search g->path for the supermin appliance, and try to
+   * synthesize a kernel and initrd from that.  If it fails, we
+   * try the path search again looking for a backup ordinary
+   * appliance.
+   */
   pelem = path = safe_strdup (g, g->path);
   do {
     pend = strchrnul (pelem, ':');
@@ -740,32 +798,31 @@ guestfs_launch (guestfs_h *g)
     *pend = '\0';
     len = pend - pelem;
 
-    /* Empty element or "." means cwd. */
+    /* Empty element of "." means cwd. */
     if (len == 0 || (len == 1 && *pelem == '.')) {
       if (g->verbose)
 	fprintf (stderr,
-		 "looking for kernel and initrd in current directory\n");
-      if (access (kernel_name, F_OK) == 0 && access (initrd_name, F_OK) == 0) {
-	kernel = safe_strdup (g, kernel_name);
-	initrd = safe_strdup (g, initrd_name);
+		 "looking for supermin appliance in current directory\n");
+      if (dir_contains_files (".",
+			      supermin_name, supermin_hostfiles_name,
+			      "kmod.whitelist", NULL)) {
+	if (build_supermin_appliance (g, ".", &kernel, &initrd) == -1)
+	  return -1;
 	break;
       }
     }
-    /* Look at <path>/kernel etc. */
+    /* Look at <path>/supermin* etc. */
     else {
-      kernel = safe_malloc (g, len + strlen (kernel_name) + 2);
-      initrd = safe_malloc (g, len + strlen (initrd_name) + 2);
-      sprintf (kernel, "%s/%s", pelem, kernel_name);
-      sprintf (initrd, "%s/%s", pelem, initrd_name);
-
       if (g->verbose)
-	fprintf (stderr, "looking for %s and %s\n", kernel, initrd);
+	fprintf (stderr, "looking for supermin appliance in %s\n", pelem);
 
-      if (access (kernel, F_OK) == 0 && access (initrd, F_OK) == 0)
+      if (dir_contains_files (pelem,
+			      supermin_name, supermin_hostfiles_name,
+			      "kmod.whitelist", NULL)) {
+	if (build_supermin_appliance (g, pelem, &kernel, &initrd) == -1)
+	  return -1;
 	break;
-      free (kernel);
-      free (initrd);
-      kernel = initrd = NULL;
+      }
     }
 
     pelem = pend + 1;
@@ -773,6 +830,46 @@ guestfs_launch (guestfs_h *g)
 
   free (path);
 
+  if (kernel == NULL || initrd == NULL) {
+    /* Search g->path for the kernel and initrd. */
+    pelem = path = safe_strdup (g, g->path);
+    do {
+      pend = strchrnul (pelem, ':');
+      pmore = *pend == ':';
+      *pend = '\0';
+      len = pend - pelem;
+
+      /* Empty element or "." means cwd. */
+      if (len == 0 || (len == 1 && *pelem == '.')) {
+	if (g->verbose)
+	  fprintf (stderr,
+		   "looking for appliance in current directory\n");
+	if (dir_contains_files (".", kernel_name, initrd_name, NULL)) {
+	  kernel = safe_strdup (g, kernel_name);
+	  initrd = safe_strdup (g, initrd_name);
+	  break;
+	}
+      }
+      /* Look at <path>/kernel etc. */
+      else {
+	if (g->verbose)
+	  fprintf (stderr, "looking for appliance in %s\n", pelem);
+
+	if (dir_contains_files (pelem, kernel_name, initrd_name, NULL)) {
+	  kernel = safe_malloc (g, len + strlen (kernel_name) + 2);
+	  initrd = safe_malloc (g, len + strlen (initrd_name) + 2);
+	  sprintf (kernel, "%s/%s", pelem, kernel_name);
+	  sprintf (initrd, "%s/%s", pelem, initrd_name);
+	  break;
+	}
+      }
+
+      pelem = pend + 1;
+    } while (pmore);
+
+    free (path);
+  }
+
   if (kernel == NULL || initrd == NULL) {
     error (g, _("cannot find %s or %s on LIBGUESTFS_PATH (current path = %s)"),
 	   kernel_name, initrd_name, g->path);
@@ -788,15 +885,7 @@ guestfs_launch (guestfs_h *g)
    */
   memsize = 384;
 
-  /* Make the temporary directory containing the socket. */
-  if (!g->tmpdir) {
-    g->tmpdir = safe_strdup (g, dir_template);
-    if (mkdtemp (g->tmpdir) == NULL) {
-      perrorf (g, _("%s: cannot create temporary directory"), dir_template);
-      goto cleanup0;
-    }
-  }
-
+  /* Make the vmchannel socket. */
   snprintf (unixsock, sizeof unixsock, "%s/sock", g->tmpdir);
   unlink (unixsock);
 
@@ -1048,6 +1137,34 @@ guestfs_launch (guestfs_h *g)
   return -1;
 }
 
+/* This function does the hard work of building the supermin appliance
+ * on the fly.  'path' is the directory containing the control files.
+ * 'kernel' and 'initrd' are where we will return the names of the
+ * kernel and initrd (only initrd is built).  The work is done by
+ * an external script.  We just tell it where to put the result.
+ */
+static int
+build_supermin_appliance (guestfs_h *g, const char *path,
+			  char **kernel, char **initrd)
+{
+  char cmd[4096];
+  int r;
+
+  snprintf (cmd, sizeof cmd,
+	    "PATH='%s':$PATH "
+	    "guestfs-supermin-helper '%s' %s/kernel %s/initrd",
+	    path,
+	    path, g->tmpdir, g->tmpdir);
+
+  r = system (cmd);
+  if (r == -1 || WEXITSTATUS(r) != 0) {
+    error (g, _("external command failed: %s"), cmd);
+    return -1;
+  }
+
+  return 0;
+}
+
 static void
 finish_wait_ready (guestfs_h *g, void *vp)
 {