New tool: virt-diff.

This tool can be used to show the differences between two
disk images.

.gitignore

@@ -109,6 +109,9 @@ Makefile.in
 /df/stamp-virt-df.pod
 /df/virt-df
 /df/virt-df.1
+/diff/stamp-virt-diff.pod
+/diff/virt-diff
+/diff/virt-diff.1
 /edit/stamp-virt-*.pod
 /edit/virt-edit
 /edit/virt-edit.1
@@ -228,6 +231,7 @@ Makefile.in
 /html/virt-copy-in.1.html
 /html/virt-copy-out.1.html
 /html/virt-df.1.html
+/html/virt-diff.1.html
 /html/virt-edit.1.html
 /html/virt-filesystems.1.html
 /html/virt-format.1.html

Makefile.am

@@ -72,7 +72,7 @@ SUBDIRS += test-tool
 SUBDIRS += fish
 
 # virt-tools in C.
-SUBDIRS += align cat df edit format inspector rescue
+SUBDIRS += align cat diff df edit format inspector rescue
 
 # bash-completion
 SUBDIRS += bash
@@ -223,6 +223,7 @@ HTMLFILES = \
 	html/virt-copy-in.1.html \
 	html/virt-copy-out.1.html \
 	html/virt-df.1.html \
+	html/virt-diff.1.html \
 	html/virt-edit.1.html \
 	html/virt-filesystems.1.html \
 	html/virt-format.1.html \

cat/virt-ls.pod

@@ -68,24 +68,8 @@ Find regular files modified in the last 24 hours:
 
 =head2 DIFFERENCES IN SNAPSHOTS AND BACKING FILES
 
-Find the differences between files in a guest and an earlier snapshot
-of the same guest.
-
- virt-ls -lR -a snapshot.img / --uids > old
- virt-ls -lR -a current.img / --uids > new
- diff -u old new | less
-
-The commands above won't find files where the content has changed but
-the metadata (eg. file size and modification date) is the same.  To do
-that, you need to add the I<--checksum> parameter to both C<virt-ls>
-commands.  I<--checksum> can be quite slow since it has to read and
-compute a checksum of every regular file in the virtual machine.
-
-The commands above won't show changes in file times.  Add I<--time-t>
-to both C<virt-ls> commands if you want to show all time changes (that
-includes file access times).
-
-To diff the content of files, use L<virt-cat(1)> or L<guestfish(1)>.
+Although it is possible to use virt-ls to look for differences, since
+libguestfs E<ge> 1.26 a new tool is available called L<virt-diff(1)>.
 
 =head1 OUTPUT MODES
 
@@ -510,6 +494,7 @@ L<guestfs(3)>,
 L<guestfish(1)>,
 L<virt-cat(1)>,
 L<virt-copy-out(1)>,
+L<virt-diff(1)>,
 L<virt-tar-out(1)>,
 L<http://libguestfs.org/>.
 

configure.ac

@@ -1680,6 +1680,7 @@ AC_CONFIG_FILES([Makefile
                  csharp/Makefile
                  daemon/Makefile
                  df/Makefile
+                 diff/Makefile
                  edit/Makefile
                  erlang/Makefile
                  erlang/examples/Makefile

contrib/make-check-on-installed.pl

@@ -70,6 +70,7 @@ my %mapping = (
     '/bin/virt-copy-in$' => "fish",
     '/bin/virt-copy-out$' => "fish",
     '/bin/virt-df$' => "df",
+    '/bin/virt-diff$' => "diff",
     '/bin/virt-edit$' => "edit",
     '/bin/virt-filesystems$' => "cat",
     '/bin/virt-format$' => "format",

diff/Makefile.am

@@ -0,0 +1,88 @@
+# libguestfs virt-diff
+# Copyright (C) 2013 Red Hat Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+include $(top_srcdir)/subdir-rules.mk
+
+EXTRA_DIST = \
+	test-virt-diff.sh \
+	virt-diff.pod
+
+CLEANFILES = stamp-virt-diff.pod
+
+bin_PROGRAMS = virt-diff
+
+SHARED_SOURCE_FILES = \
+	../cat/visit.h \
+	../cat/visit.c \
+	../fish/domain.c \
+	../fish/inspect.c \
+	../fish/keys.c \
+	../fish/options.h \
+	../fish/options.c \
+	../fish/uri.h \
+	../fish/uri.c
+
+virt_diff_SOURCES = \
+	$(SHARED_SOURCE_FILES) \
+	diff.c
+
+virt_diff_CPPFLAGS = \
+	-DGUESTFS_WARN_DEPRECATED=1 \
+	-DLOCALEBASEDIR=\""$(datadir)/locale"\" \
+	-I$(top_srcdir)/src -I$(top_builddir)/src \
+	-I$(top_srcdir)/cat -I$(top_srcdir)/fish \
+	-I$(srcdir)/../gnulib/lib -I../gnulib/lib
+
+virt_diff_CFLAGS = \
+	$(WARN_CFLAGS) $(WERROR_CFLAGS) \
+	$(GPROF_CFLAGS) $(GCOV_CFLAGS) \
+	$(LIBXML2_CFLAGS)
+
+virt_diff_LDADD = \
+	$(top_builddir)/src/libutils.la \
+	$(top_builddir)/src/libguestfs.la \
+	$(LIBXML2_LIBS) \
+	$(LIBVIRT_LIBS) \
+	../gnulib/lib/libgnu.la
+
+# Manual pages and HTML files for the website.
+man_MANS = virt-diff.1
+
+noinst_DATA = \
+	$(top_builddir)/html/virt-diff.1.html
+
+virt-diff.1 $(top_builddir)/html/virt-diff.1.html: stamp-virt-diff.pod
+
+stamp-virt-diff.pod: virt-diff.pod
+	$(PODWRAPPER) \
+	  --man virt-diff.1 \
+	  --html $(top_builddir)/html/virt-diff.1.html \
+	  --license GPLv2+ \
+	  $<
+	touch $@
+
+# Tests.
+
+TESTS_ENVIRONMENT = $(top_builddir)/run --test
+
+if ENABLE_APPLIANCE
+TESTS = \
+	test-virt-diff.sh
+endif ENABLE_APPLIANCE
+
+check-valgrind:
+	$(MAKE) VG="$(top_builddir)/run @VG@" check

diff/test-virt-diff.sh

@@ -0,0 +1,62 @@
+#!/bin/bash -
+# libguestfs
+# Copyright (C) 2013 Red Hat Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+export LANG=C
+set -e
+
+if [ ! -f ../tests/guests/fedora.img ]; then
+    echo "$0: test skipped because there is no phony fedora test image"
+    exit 77
+fi
+
+if [ "$(../fish/guestfish get-backend)" = "uml" ]; then
+    echo "$0: test skipped because backend is UML"
+    exit 77
+fi
+
+rm -f fedora.qcow2
+
+# Modify a copy of the image.
+qemu-img create -f qcow2 -b ../tests/guests/fedora.img fedora.qcow2
+
+../fish/guestfish -a fedora.qcow2 -i <<EOF
+touch /diff
+write-append /etc/motd "Testing virt-diff\n"
+EOF
+
+output="$($VG ./virt-diff -a ../tests/guests/fedora.img -A fedora.qcow2)"
+
+expected="\
++ - 0644          0 /diff
+= - 0644         37 /etc/motd
+@@ -1 +1,2 @@
+ Welcome to Fedora release 14 (Phony)
++Testing virt-diff
+@@ End of diff @@"
+
+if [ "$output" != "$expected" ]; then
+    echo "$0: error: unexpected output from virt-diff"
+    echo "---- output: ------------------------------------------"
+    echo "$output"
+    echo "---- expected: ----------------------------------------"
+    echo "$expected"
+    echo "-------------------------------------------------------"
+    exit 1
+fi
+
+rm fedora.qcow2

diff/virt-diff.pod

@@ -0,0 +1,269 @@
+=encoding utf8
+
+=head1 NAME
+
+virt-diff - Differences between files in two virtual machines
+
+=head1 SYNOPSIS
+
+ virt-diff [--options] -d domain1 -D domain2
+
+ virt-diff [--options] -a disk1.img [-a ...] -A disk2.img [-A ...]
+
+=head1 DESCRIPTION
+
+C<virt-diff> lists the differences between files in two virtual
+machines or disk images.  The usual use case is to show the
+changes in a VM after it has been running for a while, by taking
+a snapshot, running the VM, and then using this tool to show
+what changed between the new VM state and the old snapshot.
+
+This tool will find differences in filenames, file sizes, checksums,
+extended attributes, file content and more from a virtual machine or
+disk image.  However it B<does not> look at the boot loader, unused
+space between partitions or within filesystems, "hidden" sectors and
+so on.  In other words, it is not a security or forensics tool.
+
+To specify two guests, you have to use the I<-a> or I<-d> option(s)
+for the first guest, and the I<-A> or I<-D> option(s) for the second
+guest.  The common case is:
+
+ virt-diff -a old.img -A new.img
+
+or using names known to libvirt:
+
+ virt-diff -d oldguest -D newguest
+
+=head1 OPTIONS
+
+=over 4
+
+=item B<--help>
+
+Display brief help.
+
+=item B<-a> file
+
+=item B<--add> file
+
+Add I<file> which should be a disk image from the first virtual
+machine.  If the virtual machine has multiple block devices, you must
+supply all of them with separate I<-a> options.
+
+The format of the disk image is auto-detected.  To override this and
+force a particular format use the I<--format=..> option.
+
+=item B<-a> URI
+
+=item B<--add> URI
+
+Add a remote disk.  See L<guestfish(1)/ADDING REMOTE STORAGE>.
+
+=item B<--all>
+
+Same as I<--extra-stats> I<--times> I<--uids> I<--xattrs>.
+
+=item B<--atime>
+
+The default is to ignore changes in file access times, since those are
+unlikely to be interesting.  Using this flag shows atime differences
+as well.
+
+=item B<-A> file
+
+=item B<-A> URI
+
+Add a disk image from the second virtual machine.
+
+=item B<--checksum>
+
+=item B<--checksum=crc|md5|sha1|sha224|sha256|sha384|sha512>
+
+Use a checksum over file contents to detect when regular files have
+changed content.
+
+With no argument, this defaults to using I<md5>.  Using an argument,
+you can select the checksum type to use.  If the flag is omitted then
+file times and size are used to determine if a file has changed.
+
+=item B<-c> URI
+
+=item B<--connect> URI
+
+If using libvirt, connect to the given I<URI>.  If omitted, then we
+connect to the default libvirt hypervisor.
+
+If you specify guest block devices directly (I<-a>), then libvirt is
+not used at all.
+
+=item B<--csv>
+
+Write out the results in CSV format (comma-separated values).  This
+format can be imported easily into databases and spreadsheets, but
+read L</NOTE ABOUT CSV FORMAT> below.
+
+=item B<--dir-links>
+
+The default is to ignore changed in the number of links in directory
+entries, since those are unlikely to be interesting.  Using this flag
+shows changes to the nlink field of directories.
+
+=item B<--dir-times>
+
+The default is to ignore changed times on directory entries, since
+those are unlikely to be interesting.  Using this flag shows changes
+to the time fields of directories.
+
+=item B<-d> guest
+
+=item B<--domain> guest
+
+Add all the disks from the named libvirt guest, as the first guest.
+Domain UUIDs can be used instead of names.
+
+=item B<-D> guest
+
+Add all the disks from the named libvirt guest, as the second guest.
+Domain UUIDs can be used instead of names.
+
+=item B<--echo-keys>
+
+When prompting for keys and passphrases, virt-diff normally turns
+echoing off so you cannot see what you are typing.  If you are not
+worried about Tempest attacks and there is no one else in the room you
+can specify this flag to see what you are typing.
+
+=item B<--extra-stats>
+
+Display extra stats.
+
+=item B<--format=raw|qcow2|..>
+
+=item B<--format>
+
+The default for the I<-a>/I<-A> option is to auto-detect the format of
+the disk image.  Using this forces the disk format for I<-a>/I<-A>
+options which follow on the command line.  Using I<--format> with no
+argument switches back to auto-detection for subsequent I<-a>/I<-A>
+options.
+
+For example:
+
+ virt-diff --format=raw -a disk.img [...]
+
+forces raw format (no auto-detection) for C<disk.img>.
+
+ virt-diff --format=raw -a disk.img --format -a another.img [...]
+
+forces raw format (no auto-detection) for C<disk.img> and reverts to
+auto-detection for C<another.img>.
+
+If you have untrusted raw-format guest disk images, you should use
+this option to specify the disk format.  This avoids a possible
+security problem with malicious guests (CVE-2010-3851).
+
+=item B<-h>
+
+=item B<--human-readable>
+
+Display file sizes in human-readable format.
+
+This option only has effect in I<-lR> output mode.  See
+L</RECURSIVE LONG LISTING> above.
+
+=item B<--keys-from-stdin>
+
+Read key or passphrase parameters from stdin.  The default is
+to try to read passphrases from the user by opening C</dev/tty>.
+
+=item B<--times>
+
+Display time fields.
+
+=item B<--time-days>
+
+Display time fields as days before now (negative if in the future).
+
+Note that C<0> in output means "up to 1 day before now", or that the
+age of the file is between 0 and 86399 seconds.
+
+=item B<--time-relative>
+
+Display time fields as seconds before now (negative if in the future).
+
+=item B<--time-t>
+
+Display time fields as seconds since the Unix epoch.
+
+=item B<--uids>
+
+Display UID and GID fields.
+
+=item B<-v>
+
+=item B<--verbose>
+
+Enable verbose messages for debugging.
+
+=item B<-V>
+
+=item B<--version>
+
+Display version number and exit.
+
+=item B<-x>
+
+Enable tracing of libguestfs API calls.
+
+=item B<--xattrs>
+
+Display extended attributes.
+
+=back
+
+=head1 NOTE ABOUT CSV FORMAT
+
+Comma-separated values (CSV) is a deceptive format.  It I<seems> like
+it should be easy to parse, but it is definitely not easy to parse.
+
+Myth: Just split fields at commas.  Reality: This does I<not> work
+reliably.  This example has two columns:
+
+ "foo,bar",baz
+
+Myth: Read the file one line at a time.  Reality: This does I<not>
+work reliably.  This example has one row:
+
+ "foo
+ bar",baz
+
+For shell scripts, use C<csvtool> (L<http://merjis.com/developers/csv>
+also packaged in major Linux distributions).
+
+For other languages, use a CSV processing library (eg. C<Text::CSV>
+for Perl or Python's built-in csv library).
+
+Most spreadsheets and databases can import CSV directly.
+
+=head1 EXIT STATUS
+
+This program returns 0 if successful, or non-zero if there was an
+error.
+
+=head1 SEE ALSO
+
+L<guestfs(3)>,
+L<guestfish(1)>,
+L<virt-cat(1)>,
+L<virt-copy-out(1)>,
+L<virt-ls(1)>,
+L<virt-tar-out(1)>,
+L<http://libguestfs.org/>.
+
+=head1 AUTHOR
+
+Richard W.M. Jones L<http://people.redhat.com/~rjones/>
+
+=head1 COPYRIGHT
+
+Copyright (C) 2009-2013 Red Hat Inc.

examples/guestfs-recipes.pod

@@ -180,13 +180,10 @@ of C<guestfish -h>
 
 =head1 Diff two guests; compare a snapshot to the current version
 
-L<virt-ls(1)> provides a simple way to find the differences between
-two guests (for example if they were originally cloned from the same
-source), or between two snapshots from the same guest.  See
-L<virt-ls(1)/DIFFERENCES IN SNAPSHOTS AND BACKING FILES>.
-
-There are also experimental patches on the mailing list for a
-"virt-diff" tool.
+Since libguestfs E<ge> 1.26, use L<virt-diff(1)> to look for
+differences between two guests (for example if they were originally
+cloned from the same source), or between two snapshots from the same
+guest.  In earlier versions of libguestfs, use L<virt-ls(1)>.
 
 =head1 Disable a systemd service
 

fish/guestfish.pod

@@ -1563,6 +1563,7 @@ L<virt-cat(1)>,
 L<virt-copy-in(1)>,
 L<virt-copy-out(1)>,
 L<virt-df(1)>,
+L<virt-diff(1)>,
 L<virt-edit(1)>,
 L<virt-filesystems(1)>,
 L<virt-inspector(1)>,

po/POTFILES

@@ -112,6 +112,7 @@ df/estimate-max-threads.c
 df/main.c
 df/output.c
 df/parallel.c
+diff/diff.c
 edit/edit.c
 erlang/erl-guestfs-proto.c
 erlang/erl-guestfs.c

run.in

@@ -74,7 +74,7 @@ fi
 
 # Set the PATH to contain all the libguestfs binaries.  There are a
 # lot of binaries, so a lot of path entries.
-PATH="$b/align:$b/builder:$b/cat:$b/df:$b/edit:$b/erlang:$b/fish:$b/format:$b/fuse:$b/rescue:$b/resize:$b/sparsify:$b/sysprep:$b/test-tool:$b/tools:$PATH"
+PATH="$b/align:$b/builder:$b/cat:$b/df:$b/diff:$b/edit:$b/erlang:$b/fish:$b/format:$b/fuse:$b/rescue:$b/resize:$b/sparsify:$b/sysprep:$b/test-tool:$b/tools:$PATH"
 export PATH
 
 # Set LD_LIBRARY_PATH to contain library.

src/guestfs.pod

@@ -4259,6 +4259,10 @@ actions.
 
 L<virt-df(1)> command and documentation.
 
+=item C<diff>
+
+L<virt-diff(1)> command and documentation.
+
 =item C<edit>
 
 L<virt-edit(1)> command and documentation.
@@ -4697,6 +4701,7 @@ L<virt-cat(1)>,
 L<virt-copy-in(1)>,
 L<virt-copy-out(1)>,
 L<virt-df(1)>,
+L<virt-diff(1)>,
 L<virt-edit(1)>,
 L<virt-filesystems(1)>,
 L<virt-format(1)>,