From 18b7f09f366d518050f467d0944c81c82fd5e39e Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Fri, 5 Oct 2012 14:56:35 +0100
Subject: [PATCH] Add support for hotplugging (adding disks) to the libvirt
 attach-method.

When libvirt is used, we can allow disks to be hotplugged.
guestfs_add_drive can be called after launch to hot-add a disk.

When a disk is hot-added, we first ask libvirt to add the disk to the
appliance, then we make an internal call into the appliance to get it
to wait for the disk to appear (ie. udev_settle ()).

Hot-added disks are tracked in the g->drives array.

This also adds a test.
---
 Makefile.am                   |  1 +
 configure.ac                  |  1 +
 daemon/Makefile.am            |  1 +
 daemon/hotplug.c              | 67 ++++++++++++++++++++++++
 fish/alloc.c                  |  5 --
 generator/actions.ml          | 24 +++++++--
 po/POTFILES                   |  1 +
 src/MAX_PROC_NR               |  2 +-
 src/guestfs-internal.h        |  3 ++
 src/guestfs.pod               | 38 ++++++++++++--
 src/launch-libvirt.c          | 99 +++++++++++++++++++++++++++++++----
 src/launch.c                  | 80 +++++++++++++++++++++++++---
 tests/hotplug/Makefile.am     | 26 +++++++++
 tests/hotplug/test-hot-add.pl | 66 +++++++++++++++++++++++
 14 files changed, 386 insertions(+), 28 deletions(-)
 create mode 100644 daemon/hotplug.c
 create mode 100644 tests/hotplug/Makefile.am
 create mode 100755 tests/hotplug/test-hot-add.pl

diff --git a/Makefile.am b/Makefile.am
index 4e476ea92..50bfb322f 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -52,6 +52,7 @@ SUBDIRS += tests/9p
 SUBDIRS += tests/rsync
 SUBDIRS += tests/bigdirs
 SUBDIRS += tests/disk-labels
+SUBDIRS += tests/hotplug
 SUBDIRS += tests/regressions
 endif
 
diff --git a/configure.ac b/configure.ac
index 0422bcb40..b4720bce3 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1379,6 +1379,7 @@ AC_CONFIG_FILES([Makefile
                  tests/disk-labels/Makefile
                  tests/extra/Makefile
                  tests/guests/Makefile
+                 tests/hotplug/Makefile
                  tests/luks/Makefile
                  tests/lvm/Makefile
                  tests/md/Makefile
diff --git a/daemon/Makefile.am b/daemon/Makefile.am
index 2a25c695f..9ffff152b 100644
--- a/daemon/Makefile.am
+++ b/daemon/Makefile.am
@@ -125,6 +125,7 @@ guestfsd_SOURCES = \
 	guestfsd.c \
 	headtail.c \
 	hexdump.c \
+	hotplug.c \
 	hivex.c \
 	htonl.c \
 	initrd.c \
diff --git a/daemon/hotplug.c b/daemon/hotplug.c
new file mode 100644
index 000000000..aae638e7a
--- /dev/null
+++ b/daemon/hotplug.c
@@ -0,0 +1,67 @@
+/* libguestfs - the guestfsd daemon
+ * Copyright (C) 2012 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#include <config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <errno.h>
+#include <time.h>
+
+#include "guestfs_protocol.h"
+#include "daemon.h"
+#include "actions.h"
+
+#define HOT_ADD_TIMEOUT 30 /* seconds */
+
+/* Wait for /dev/disk/guestfs/<label> to appear.  Timeout (and error)
+ * if it doesn't appear after a reasonable length of time.
+ */
+int
+do_internal_hot_add_drive (const char *label)
+{
+  time_t start_t, now_t;
+  size_t len = strlen (label);
+  char path[len+64];
+  int r;
+
+  snprintf (path, len+64, "/dev/disk/guestfs/%s", label);
+
+  time (&start_t);
+
+  while (time (&now_t) - start_t <= HOT_ADD_TIMEOUT) {
+    udev_settle ();
+
+    r = access (path, F_OK);
+    if (r == -1 && errno != ENOENT) {
+      reply_with_perror ("%s", path);
+      return -1;
+    }
+    if (r == 0)
+      return 0;
+
+    sleep (1);
+  }
+
+  reply_with_error ("hot-add drive: '%s' did not appear after %d seconds: "
+                    "this could mean that virtio-scsi (in qemu or kernel) "
+                    "or udev is not working",
+                    path, HOT_ADD_TIMEOUT);
+  return -1;
+}
diff --git a/fish/alloc.c b/fish/alloc.c
index f6e5b8ff7..7454fb737 100644
--- a/fish/alloc.c
+++ b/fish/alloc.c
@@ -72,11 +72,6 @@ alloc_disk (const char *filename, const char *size_str, int add, int sparse)
   if (parse_size (size_str, &size) == -1)
     return -1;
 
-  if (!guestfs_is_config (g)) {
-    fprintf (stderr, _("can't allocate or add disks after launching\n"));
-    return -1;
-  }
-
   fd = open (filename, O_WRONLY|O_CREAT|O_NOCTTY|O_TRUNC|O_CLOEXEC, 0666);
   if (fd == -1) {
     perror (filename);
diff --git a/generator/actions.ml b/generator/actions.ml
index 25d28098f..39c71ab8b 100644
--- a/generator/actions.ml
+++ b/generator/actions.ml
@@ -1186,14 +1186,22 @@ not all belong to a single logical operating system
     name = "add_drive";
     style = RErr, [String "filename"], [OBool "readonly"; OString "format"; OString "iface"; OString "name"; OString "label"];
     once_had_no_optargs = true;
-    fish_alias = ["add"]; config_only = true;
+    fish_alias = ["add"];
     shortdesc = "add an image to examine or modify";
     longdesc = "\
 This function adds a disk image called C<filename> to the handle.
 C<filename> may be a regular host file or a host device.
 
-The first time you call this function, the disk appears as
-C</dev/sda>, the second time as C</dev/sdb>, and so on.
+When this function is called before C<guestfs_launch> (the
+usual case) then the first time you call this function,
+the disk appears in the API as C</dev/sda>, the second time
+as C</dev/sdb>, and so on.
+
+In libguestfs E<ge> 1.20 you can also call this function
+after launch (with some restrictions).  This is called
+\"hotplugging\".  When hotplugging, you must specify a
+C<label> so that the new disk gets a predictable name.
+For more information see L<guestfs(3)/HOTPLUGGING>.
 
 You don't necessarily need to be root when using libguestfs.  However
 you obviously do need sufficient permissions to access the filename
@@ -9951,6 +9959,16 @@ This returns a hashtable, where keys are the disk labels
 are the full raw block device and partition names
 (eg. C</dev/sda> and C</dev/sda1>)." };
 
+  { defaults with
+    name = "internal_hot_add_drive";
+    style = RErr, [String "label"], [];
+    proc_nr = Some 370;
+    in_fish = false; in_docs = false;
+    tests = [];
+    shortdesc = "internal hotplugging operation";
+    longdesc = "\
+This function is used internally when hotplugging drives." };
+
 ]
 
 (* Non-API meta-commands available only in guestfish.
diff --git a/po/POTFILES b/po/POTFILES
index 548156c47..22cd1486e 100644
--- a/po/POTFILES
+++ b/po/POTFILES
@@ -42,6 +42,7 @@ daemon/guestfsd.c
 daemon/headtail.c
 daemon/hexdump.c
 daemon/hivex.c
+daemon/hotplug.c
 daemon/htonl.c
 daemon/initrd.c
 daemon/inotify.c
diff --git a/src/MAX_PROC_NR b/src/MAX_PROC_NR
index 446dfcc54..5b0cffbc0 100644
--- a/src/MAX_PROC_NR
+++ b/src/MAX_PROC_NR
@@ -1 +1 @@
-369
+370
diff --git a/src/guestfs-internal.h b/src/guestfs-internal.h
index ed9ada48c..db9818c3b 100644
--- a/src/guestfs-internal.h
+++ b/src/guestfs-internal.h
@@ -166,6 +166,9 @@ struct attach_ops {
 
   int (*get_pid) (guestfs_h *g);         /* get-pid API. */
   int (*max_disks) (guestfs_h *g);       /* max-disks API. */
+
+  /* Hotplugging drives. */
+  int (*hot_add_drive) (guestfs_h *g, struct drive *drv, size_t drv_index);
 };
 extern struct attach_ops attach_ops_appliance;
 extern struct attach_ops attach_ops_libvirt;
diff --git a/src/guestfs.pod b/src/guestfs.pod
index 48d810b0b..624e743e1 100644
--- a/src/guestfs.pod
+++ b/src/guestfs.pod
@@ -143,8 +143,11 @@ one you added), etc.
 
 Once L</guestfs_launch> has been called you cannot add any more images.
 You can call L</guestfs_list_devices> to get a list of the device
-names, in the order that you added them.  See also L</BLOCK DEVICE
-NAMING> below.
+names, in the order that you added them.
+See also L</BLOCK DEVICE NAMING> below.
+
+There are slightly different rules when hotplugging disks (in
+libguestfs E<ge> 1.20).  See L</HOTPLUGGING> below.
 
 =head2 MOUNTING
 
@@ -601,6 +604,35 @@ Libguestfs on top of FUSE performs quite poorly.  For best performance
 do not use it.  Use ordinary libguestfs filesystem calls, upload,
 download etc. instead.
 
+=head2 HOTPLUGGING
+
+In libguestfs E<ge> 1.20, you may add drives after calling
+L</guestfs_launch>.  There are some restrictions, see below.
+This is called I<hotplugging>.
+
+Only a subset of the attach-method backends support hotplugging
+(currently only the libvirt attach-method has support).  It also
+requires that you use libvirt E<ge> 0.10.3 and qemu E<ge> 1.2.
+
+To hot-add a disk, simply call L</guestfs_add_drive_opts> after
+L</guestfs_launch>.  It is mandatory to specify the C<label> parameter
+so that the newly added disk has a predictable name.  For example:
+
+ if (guestfs_launch (g) == -1)
+   error ("launch failed");
+ 
+ if (guestfs_add_drive_opts (g, filename,
+                             GUESTFS_ADD_DRIVE_OPTS_LABEL, "newdisk",
+                             -1) == -1)
+   error ("hot-add of disk failed");
+ 
+ if (guestfs_part_disk ("/dev/disk/guestfs/newdisk", "mbr") == -1)
+   error ("partitioning of hot-added disk failed");
+
+Backends that support hotplugging do not require that you add
+E<ge> 1 disk before calling launch.  When hotplugging is supported
+you don't need to add any disks.
+
 =head2 INSPECTION
 
 Libguestfs has APIs for inspecting an unknown disk image to find out
@@ -2639,7 +2671,7 @@ The guest may be killed by L</guestfs_kill_subprocess>, or may die
 asynchronously at any time (eg. due to some internal error), and that
 causes the state to transition back to CONFIG.
 
-Configuration commands for qemu such as L</guestfs_add_drive> can only
+Configuration commands for qemu such as L</guestfs_set_path> can only
 be issued when in the CONFIG state.
 
 The API offers one call that goes from CONFIG through LAUNCHING to
diff --git a/src/launch-libvirt.c b/src/launch-libvirt.c
index aa1c39f23..3eab56766 100644
--- a/src/launch-libvirt.c
+++ b/src/launch-libvirt.c
@@ -135,14 +135,6 @@ launch_libvirt (guestfs_h *g, const char *libvirt_uri)
   struct drive *drv;
   size_t i;
 
-  /* At present you must add drives before starting the appliance.  In
-   * future when we enable hotplugging you won't need to do this.
-   */
-  if (!g->nr_drives) {
-    error (g, _("you must call guestfs_add_drive before guestfs_launch"));
-    return -1;
-  }
-
   /* XXX: It should be possible to make this work. */
   if (g->direct) {
     error (g, _("direct mode flag is not supported yet for libvirt attach method"));
@@ -1376,6 +1368,89 @@ max_disks_libvirt (guestfs_h *g)
   return 255;
 }
 
+static xmlChar *construct_libvirt_xml_hot_add_disk (guestfs_h *g, struct drive *drv, size_t drv_index);
+
+/* Hot-add a drive.  Note the appliance is up when this is called. */
+static int
+hot_add_drive_libvirt (guestfs_h *g, struct drive *drv, size_t drv_index)
+{
+  virConnectPtr conn = g->virt.connv;
+  virDomainPtr dom = g->virt.domv;
+  xmlChar *xml = NULL;
+
+  if (!conn || !dom) {
+    /* This is essentially an internal error if it happens. */
+    error (g, "%s: conn == NULL or dom == NULL", __func__);
+    return -1;
+  }
+
+  /* Create overlay for read-only drive.  This works around lack of
+   * support for <transient/> disks in libvirt.
+   */
+  if (make_qcow2_overlay_for_drive (g, drv) == -1)
+    return -1;
+
+  /* Create the XML for the new disk. */
+  xml = construct_libvirt_xml_hot_add_disk (g, drv, drv_index);
+  if (xml == NULL)
+    return -1;
+
+  /* Attach it. */
+  if (virDomainAttachDeviceFlags (dom, (char *) xml,
+                                  VIR_DOMAIN_DEVICE_MODIFY_LIVE) == -1) {
+    libvirt_error (g, _("could not attach disk to libvirt domain"));
+    goto error;
+  }
+
+  free (xml);
+  return 0;
+
+ error:
+  free (xml);
+  return -1;
+}
+
+static xmlChar *
+construct_libvirt_xml_hot_add_disk (guestfs_h *g, struct drive *drv,
+                                    size_t drv_index)
+{
+  xmlChar *ret = NULL;
+  xmlBufferPtr xb = NULL;
+  xmlOutputBufferPtr ob;
+  xmlTextWriterPtr xo = NULL;
+
+  XMLERROR (NULL, xb = xmlBufferCreate ());
+  XMLERROR (NULL, ob = xmlOutputBufferCreateBuffer (xb, NULL));
+  XMLERROR (NULL, xo = xmlNewTextWriter (ob));
+
+  XMLERROR (-1, xmlTextWriterSetIndent (xo, 1));
+  XMLERROR (-1, xmlTextWriterSetIndentString (xo, BAD_CAST "  "));
+  XMLERROR (-1, xmlTextWriterStartDocument (xo, NULL, NULL, NULL));
+
+  if (construct_libvirt_xml_disk (g, xo, drv, drv_index) == -1)
+    goto err;
+
+  XMLERROR (-1, xmlTextWriterEndDocument (xo));
+  XMLERROR (NULL, ret = xmlBufferDetach (xb)); /* caller frees ret */
+
+  debug (g, "hot-add disk XML:\n%s", ret);
+
+ err:
+  if (xo)
+    xmlFreeTextWriter (xo); /* frees 'ob' too */
+  if (xb)
+    xmlBufferFree (xb);
+
+  return ret;
+}
+
+struct attach_ops attach_ops_libvirt = {
+  .launch = launch_libvirt,
+  .shutdown = shutdown_libvirt,
+  .max_disks = max_disks_libvirt,
+  .hot_add_drive = hot_add_drive_libvirt,
+};
+
 #else /* no libvirt or libxml2 at compile time */
 
 #define NOT_IMPL(r)                                                     \
@@ -1403,10 +1478,16 @@ max_disks_libvirt (guestfs_h *g)
   NOT_IMPL (-1);
 }
 
-#endif /* no libvirt or libxml2 at compile time */
+static int
+hot_add_drive_libvirt (guestfs_h *g, struct drive *drv, size_t drv_index)
+{
+  NOT_IMPL (-1);
+}
 
 struct attach_ops attach_ops_libvirt = {
   .launch = launch_libvirt,
   .shutdown = shutdown_libvirt,
   .max_disks = max_disks_libvirt,
 };
+
+#endif /* no libvirt or libxml2 at compile time */
diff --git a/src/launch.c b/src/launch.c
index 1b6cf4bdb..a4e10fefc 100644
--- a/src/launch.c
+++ b/src/launch.c
@@ -29,6 +29,7 @@
 #include <sys/types.h>
 #include <sys/wait.h>
 #include <errno.h>
+#include <assert.h>
 
 #include "c-ctype.h"
 
@@ -57,14 +58,29 @@ guestfs___rollback_drives (guestfs_h *g, size_t old_i)
   g->nr_drives = old_i;
 }
 
+/* Add struct drive to the g->drives vector at the given index. */
+static void
+add_drive_to_handle_at (guestfs_h *g, struct drive *d, size_t drv_index)
+{
+  if (drv_index >= g->nr_drives) {
+    g->drives = safe_realloc (g, g->drives,
+                              sizeof (struct drive *) * (drv_index + 1));
+    while (g->nr_drives <= drv_index) {
+      g->drives[g->nr_drives] = NULL;
+      g->nr_drives++;
+    }
+  }
+
+  assert (g->drives[drv_index] == NULL);
+
+  g->drives[drv_index] = d;
+}
+
 /* Add struct drive to the end of the g->drives vector in the handle. */
 static void
 add_drive_to_handle (guestfs_h *g, struct drive *d)
 {
-  g->nr_drives++;
-  g->drives = safe_realloc (g, g->drives,
-                            sizeof (struct drive *) * g->nr_drives);
-  g->drives[g->nr_drives-1] = d;
+  add_drive_to_handle_at (g, d, g->nr_drives);
 }
 
 static struct drive *
@@ -207,6 +223,48 @@ valid_disk_label (const char *str)
   return 1;
 }
 
+/* The low-level function that adds a drive. */
+static int
+add_drive (guestfs_h *g, struct drive *drv)
+{
+  size_t i, drv_index;
+
+  if (g->state == CONFIG) {
+    /* Not hotplugging, so just add it to the handle. */
+    add_drive_to_handle (g, drv);
+    return 0;
+  }
+
+  /* ... else, hotplugging case. */
+  if (!g->attach_ops || !g->attach_ops->hot_add_drive) {
+    error (g, _("the current attach-method does not support hotplugging drives"));
+    return -1;
+  }
+
+  if (!drv->disk_label) {
+    error (g, _("'label' is required when hotplugging drives"));
+    return -1;
+  }
+
+  /* Get the first free index, or add it at the end. */
+  drv_index = g->nr_drives;
+  for (i = 0; i < g->nr_drives; ++i)
+    if (g->drives[i] == NULL)
+      drv_index = i;
+
+  /* Hot-add the drive. */
+  if (g->attach_ops->hot_add_drive (g, drv, drv_index) == -1)
+    return -1;
+
+  add_drive_to_handle_at (g, drv, drv_index);
+
+  /* Call into the appliance to wait for the new drive to appear. */
+  if (guestfs_internal_hot_add_drive (g, drv->disk_label) == -1)
+    return -1;
+
+  return 0;
+}
+
 /* Traditionally you have been able to use /dev/null as a filename, as
  * many times as you like.  Ancient KVM (RHEL 5) cannot handle adding
  * /dev/null readonly.  qemu 1.2 + virtio-scsi segfaults when you use
@@ -220,7 +278,7 @@ add_null_drive (guestfs_h *g, int readonly, const char *format,
                 const char *iface, const char *name, const char *disk_label)
 {
   char *tmpfile = NULL;
-  int fd = -1;
+  int fd = -1, r;
   struct drive *drv;
 
   if (format && STRNEQ (format, "raw")) {
@@ -252,8 +310,12 @@ add_null_drive (guestfs_h *g, int readonly, const char *format,
   }
 
   drv = create_drive_struct (g, tmpfile, readonly, format, iface, name, disk_label, 0);
-  add_drive_to_handle (g, drv);
+  r = add_drive (g, drv);
   free (tmpfile);
+  if (r == -1) {
+    free_drive_struct (drv);
+    return -1;
+  }
 
   return 0;
 
@@ -328,7 +390,11 @@ guestfs__add_drive_opts (guestfs_h *g, const char *filename,
 
   drv = create_drive_struct (g, filename, readonly, format, iface, name, disk_label,
                              use_cache_none);
-  add_drive_to_handle (g, drv);
+  if (add_drive (g, drv) == -1) {
+    free_drive_struct (drv);
+    return -1;
+  }
+
   return 0;
 }
 
diff --git a/tests/hotplug/Makefile.am b/tests/hotplug/Makefile.am
new file mode 100644
index 000000000..6644930ab
--- /dev/null
+++ b/tests/hotplug/Makefile.am
@@ -0,0 +1,26 @@
+# libguestfs
+# Copyright (C) 2012 Red Hat Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+include $(top_srcdir)/subdir-rules.mk
+
+TESTS = \
+	test-hot-add.pl
+
+TESTS_ENVIRONMENT = $(top_builddir)/run --test
+
+EXTRA_DIST = \
+	$(TESTS)
diff --git a/tests/hotplug/test-hot-add.pl b/tests/hotplug/test-hot-add.pl
new file mode 100755
index 000000000..b1322e889
--- /dev/null
+++ b/tests/hotplug/test-hot-add.pl
@@ -0,0 +1,66 @@
+#!/usr/bin/perl
+# Copyright (C) 2012 Red Hat Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+# Test hot-adding disks.
+
+use strict;
+use warnings;
+
+use Sys::Guestfs;
+
+my $g = Sys::Guestfs->new ();
+
+# Skip the test if the default attach-method isn't libvirt, since only
+# the libvirt backend supports hotplugging.
+my $attach_method = $g->get_attach_method ();
+unless ($attach_method eq "libvirt" || $attach_method =~ /^libvirt:/) {
+    print "$0: test skipped because attach-method ($attach_method) is not libvirt\n";
+    exit 77
+}
+
+# We don't need to add disks before launch.
+$g->launch ();
+
+# Create some temporary disks.
+open FILE, ">test1.img" or die "test1.img: $!";
+truncate FILE, 512 * 1024 * 1024 or die "test1.img: truncate: $!";
+close FILE;
+
+open FILE, ">test2.img" or die "test2.img: $!";
+truncate FILE, 512 * 1024 * 1024 or die "test2.img: truncate: $!";
+close FILE;
+
+die unless system ("qemu-img create -f qcow2 test3.img 1G") == 0;
+
+# Hot-add them.  Labels are required.
+$g->add_drive ("test1.img", label => "a"); # autodetect format
+$g->add_drive ("test2.img", label => "b", format => "raw", readonly => 1);
+$g->add_drive ("test3.img", label => "c", format => "qcow2");
+
+# Check we can use the disks immediately.
+$g->part_disk ("/dev/disk/guestfs/a", "mbr");
+$g->mkfs ("ext2", "/dev/disk/guestfs/c");
+$g->mkfs ("ext2", "/dev/disk/guestfs/a1");
+
+$g->shutdown ();
+$g->close ();
+
+unlink "test1.img";
+unlink "test2.img";
+unlink "test3.img";
+
+exit 0