Version 1.27.24.

Currently it tests the upload and download speed of virtio-serial and
the read and write speed of the block device (eg. virtio-scsi).

.gitignore

@@ -9,9 +9,6 @@
 *.cmxa
 *.diff
 *.eml
-*.gcda
-*.gcno
-*.gcov
 *.hi
 *.jar
 *.la
@@ -26,7 +23,6 @@ bindtests.tmp
 cscope.out
 .deps
 .gdb_history
-gmon.out
 .libs
 Makefile
 Makefile.in
@@ -37,7 +33,6 @@ Makefile.in
 /align/stamp-virt-alignment-scan.pod
 /align/virt-alignment-scan
 /align/virt-alignment-scan.1
-/appliance/excludelist
 /appliance/libguestfs-make-fixed-appliance
 /appliance/libguestfs-make-fixed-appliance.1
 /appliance/make.sh
@@ -53,21 +48,33 @@ Makefile.in
 /bash/virt-filesystems
 /bash/virt-format
 /bash/virt-inspector
+/bash/virt-log
 /bash/virt-ls
 /bash/virt-sysprep
 /bash/virt-sparsify
 /build-aux
 /builder/.depend
+/builder/index-parse.c
+/builder/index-parse.h
+/builder/index-scan.c
+/builder/libguestfs.conf
+/builder/*.qcow2
 /builder/stamp-virt-builder.pod
-/builder/test-index
+/builder/stamp-virt-index-validate.pod
+/builder/test-config/virt-builder/repos.d/test-index.conf
+/builder/test-website/virt-builder/repos.d/libguestfs.conf
 /builder/virt-builder
 /builder/virt-builder.1
+/builder/virt-index-validate
+/builder/virt-index-validate.1
 /builder/*.xz
 /cat/stamp-virt-*.pod
 /cat/virt-cat
 /cat/virt-cat.1
 /cat/virt-filesystems
 /cat/virt-filesystems.1
+/cat/virt-log
+/cat/virt-log.1
 /cat/virt-ls
 /cat/virt-ls.1
 /ChangeLog
@@ -81,6 +88,14 @@ Makefile.in
 /config.sub
 /configure
 /csharp/Libguestfs.cs
+/customize/.depend
+/customize/customize_cmdline.ml
+/customize/customize_cmdline.mli
+/customize/customize-options.pod
+/customize/customize-synopsis.pod
+/customize/stamp-virt-customize.pod
+/customize/virt-customize
+/customize/virt-customize.1
 /daemon/actions.h
 /daemon/errnostring.c
 /daemon/errnostring-gperf.c
@@ -102,6 +117,9 @@ Makefile.in
 /df/stamp-virt-df.pod
 /df/virt-df
 /df/virt-df.1
+/diff/stamp-virt-diff.pod
+/diff/virt-diff
+/diff/virt-diff.1
 /edit/stamp-virt-*.pod
 /edit/virt-edit
 /edit/virt-edit.1
@@ -142,11 +160,13 @@ Makefile.in
 /fish/guestfish-actions.pod
 /fish/guestfish-commands.pod
 /fish/guestfish-prepopts.pod
+/fish/libguestfs-tools.conf.5
 /fish/prepopts.c
 /fish/prepopts.h
 /fish/rc_protocol.c
 /fish/rc_protocol.h
 /fish/stamp-guestfish.pod
+/fish/stamp-libguestfs-tools.conf.pod
 /fish/stamp-virt-copy-in.pod
 /fish/stamp-virt-copy-out.pod
 /fish/stamp-virt-tar-in.pod
@@ -164,6 +184,7 @@ Makefile.in
 /fuse/guestunmount.1
 /fuse/stamp-guestmount.pod
 /fuse/stamp-guestunmount.pod
+/fuse/test-guestmount-fd
 /fuse/test-guestunmount-fd
 /generator/.depend
 /generator/files-generated.txt
@@ -177,8 +198,6 @@ Makefile.in
 /gobject/bindtests.js
 /gobject/Guestfs-1.0.gir
 /gobject/Guestfs-1.0.typelib
-/gobject/guestfs-gobject*.c
-/gobject/guestfs-gobject*.h
 /golang/bindtests.go
 /golang/examples/guestfs-golang.3
 /golang/examples/stamp-guestfs-golang.pod
@@ -200,6 +219,7 @@ Makefile.in
 /html/guestfs-java.3.html
 /html/guestfs-lua.3.html
 /html/guestfs-ocaml.3.html
+/html/guestfs-p2v-iso.1.html
 /html/guestfs-performance.1.html
 /html/guestfs-perl.3.html
 /html/guestfs-python.3.html
@@ -212,20 +232,26 @@ Makefile.in
 /html/guestunmount.1.html
 /html/libguestfs-make-fixed-appliance.1.html
 /html/libguestfs-test-tool.1.html
+/html/libguestfs-tools.conf.5.html
 /html/virt-alignment-scan.1.html
 /html/virt-builder.1.html
 /html/virt-cat.1.html
 /html/virt-copy-in.1.html
 /html/virt-copy-out.1.html
+/html/virt-customize.1.html
 /html/virt-df.1.html
+/html/virt-diff.1.html
 /html/virt-edit.1.html
 /html/virt-filesystems.1.html
 /html/virt-format.1.html
+/html/virt-index-validate.1.html
 /html/virt-inspector.1.html
 /html/virt-list-filesystems.1.html
 /html/virt-list-partitions.1.html
+/html/virt-log.1.html
 /html/virt-ls.1.html
 /html/virt-make-fs.1.html
+/html/virt-p2v.1.html
 /html/virt-rescue.1.html
 /html/virt-resize.1.html
 /html/virt-sparsify.1.html
@@ -233,6 +259,7 @@ Makefile.in
 /html/virt-tar.1.html
 /html/virt-tar-in.1.html
 /html/virt-tar-out.1.html
+/html/virt-v2v.1.html
 /html/virt-win-reg.1.html
 /inspector/actual-*.xml
 /inspector/stamp-virt-inspector.pod
@@ -267,10 +294,14 @@ Makefile.in
 /m4/ltsugar.m4
 /m4/ltversion.m4
 /maint.mk
+/make-fs/stamp-virt-make-fs.pod
+/make-fs/virt-make-fs
+/make-fs/virt-make-fs.1
 /missing
 /mllib/.depend
 /mllib/common_gettext.ml
 /mllib/common_utils_tests
+/mllib/config.ml
 /mllib/dummy
 /mllib/libdir.ml
 /ocaml/bindtests.bc
@@ -290,6 +321,16 @@ Makefile.in
 /ocaml/stamp-mlguestfs
 /ocaml/t/*.bc
 /ocaml/t/*.opt
+/p2v/stamp-virt-p2v.pod
+/p2v/virt-p2v
+/p2v/virt-p2v.1
+/p2v-iso/guestfs-p2v-iso.1
+/p2v-iso/guestfs-p2v-iso.txt
+/p2v-iso/kickstart-target.sh
+/p2v-iso/launch-virt-p2v
+/p2v-iso/p2v.ks
+/p2v-iso/stamp-guestfs-p2v-iso.pod
+/p2v-iso/virt-builder-target.sh
 /perl/bindtests.pl
 /perl/blib
 /perl/examples/guestfs-perl.3
@@ -334,27 +375,38 @@ Makefile.in
 /php/extension/mkinstalldirs
 /php/extension/missing
 /php/extension/modules/
+/php/extension/php-for-tests.sh
 /php/extension/php_guestfs_php.h
 /php/extension/run-tests.php
 /php/extension/tmp-php.ini
 /pick-guests.pl
 /po-docs/*/*.1
 /po-docs/*/*.3
+/po-docs/*/*.5
 /po-docs/*/*.8
 /po-docs/po4a.conf
 /po-docs/*/*.pod
-/po-docs/*/stamp-update-po
 /podwrapper.1
 /podwrapper.pl
 /po/*.gmo
 /python/bindtests.py
+/python/build
+/python/config.h
+/python/dist
 /python/examples/guestfs-python.3
 /python/examples/stamp-guestfs-python.pod
 /python/guestfs.py
 /python/guestfs-py.c
 /python/guestfs.pyc
 /python/guestfs.pyo
+/python/guestfs-internal-all.h
+/python/guestfs-internal-frontend-cleanups.h
+/python/guestfs-internal-frontend.h
+/python/MANIFEST
 /python/__pycache__
+/python/setup.py
+/python/stamp-extra-files
+/python/utils.c
 /qemu-wrapper.sh
 /rescue/stamp-virt-rescue.pod
 /rescue/virt-rescue
@@ -376,19 +428,18 @@ Makefile.in
 /ruby/Rakefile
 /run
 /sparsify/.depend
+/sparsify/link.sh
 /sparsify/stamp-virt-sparsify.pod
 /sparsify/virt-sparsify
 /sparsify/virt-sparsify.1
 /src/actions-?.c
 /src/actions-variants.c
 /src/bindtests.c
-/src/cleanup-structs.c
 /src/errnostring.c
 /src/errnostring-gperf.c
 /src/errnostring-gperf.gperf
 /src/errnostring.h
 /src/event-string.c
-/src/free-structs.c
 /src/guestfs.3
 /src/guestfs-actions.pod
 /src/guestfs-availability.pod
@@ -402,12 +453,19 @@ Makefile.in
 /src/libguestfs.pc
 /src/libguestfs.syms
 /src/.libs/libguestfs.so
+/src/libvirt-is-version
 /src/stamp-guestfs.pod
+/src/structs-cleanup.c
+/src/structs-compare.c
+/src/structs-copy.c
+/src/structs-free.c
+/src/test-utils
 /stamp-guestfs-release-notes.pod
 /stamp-h1
 /sysprep/.depend
 /sysprep/stamp-script1.sh
 /sysprep/stamp-script2.sh
+/sysprep/stamp-script4.sh
 /sysprep/stamp-virt-sysprep.pod
 /sysprep/sysprep-extra-options.pod
 /sysprep/sysprep-operations.pod
@@ -417,6 +475,7 @@ Makefile.in
 /test.out
 /tests/c-api/test-add-drive-opts
 /tests/c-api/test-add-libvirt-dom
+/tests/c-api/test-backend-settings
 /tests/c-api/test-command
 /tests/c-api/test-config
 /tests/c-api/test-create-handle
@@ -449,6 +508,8 @@ Makefile.in
 /tests/data/initrd-x86_64.img.gz
 /tests/data/test-grep.txt.gz
 /tests/data/test.iso
+/tests/disks/test-qemu-drive-libvirt.xml
+/tests/events/test-libvirt-auth-callbacks
 /tests/guests/blank-*.img
 /tests/guests/debian.img
 /tests/guests/fedora.img
@@ -467,9 +528,14 @@ Makefile.in
 /tests/mount-local/test-parallel-mount-local
 /tests/mountable/test-internal-parse-mountable
 /tests/parallel/test-parallel
+/tests/protocol/test-error-messages
+/tests/qemu/qemu-boot
+/tests/qemu/qemu-speed-test
 /tests/regressions/rhbz501893
 /tests/regressions/rhbz790721
 /tests/regressions/rhbz914931
+/tests/regressions/rhbz1044014.out
+/tests/regressions/rhbz1055452
 /tests/rsync/rsyncd.pid
 /tests/syslinux/extlinux-guest.img
 /tests/syslinux/syslinux-guest.img
@@ -478,3 +544,8 @@ Makefile.in
 /test-tool/libguestfs-test-tool-helper
 /test-tool/stamp-libguestfs-test-tool.pod
 /tools/virt-*.1
+/v2v/.depend
+/v2v/link.sh
+/v2v/stamp-virt-v2v.pod
+/v2v/virt-v2v
+/v2v/virt-v2v.1

AUTHORS

@@ -1,44 +1,53 @@
-Angus Salkeld <asalkeld@redhat.com>
-Ani Peter <apeter@redhat.com>
-Charles Duffy <cduffy@messageone.com>
-Cole Robinson <crobinso@redhat.com>
-Daniel Berrange <berrange@redhat.com>
-Daniel Cabrera <logan@fedoraproject.org>
-Dave Vasilevsky <dave@vasilevsky.ca>
-Douglas Schilling Landgraf <dougsland@redhat.com>
-Eric Blake <eblake@redhat.com>
-Erik Nolte <erik_nolte@acm.org>
-Evaggelos Balaskas <ebalaskas@ebalaskas.gr>
-Geert Warrink <geert.warrink@onsnet.nu>
-Guido Günther <agx@sigxcpu.org>
-Hilko Bengen <bengen@debian.org>
-Hilko Bengen <bengen@hilluzination.de>
-infernix <infernix@infernix.net>
-Jaswinder Singh <jsingh@redhat.com>
-Jim Meyering <meyering@redhat.com>
-Jiri Popelka <jpopelka@redhat.com>
-John Eckersberg <jeckersb@redhat.com>
-Karel Klíč <kklic@redhat.com>
-Marcin Gibula <m.gibula@e24cloud.com>
-Martin Kletzander <mkletzan@redhat.com>
-Masami HIRATA <msmhrt@gmail.com>
-Matthew Booth <mbooth@redhat.com>
-Maxim Koltsov <kolmax94@gmail.com>
-Michael Scherer <misc@zarb.org>
-Mike Kelly <pioto@pioto.org>
-Nicholas Strugnell <nstrug@redhat.com>
-Nikita A Menkovich <menkovich@gmail.com>
-Nikita Menkovich <n.menkovich@sprinthost.ru>
-Nikos Skalkotos <skalkoto@gmail.com>
-Olaf Hering <olaf@aepfle.de>
-Or Goshen <oberonc@gmail.com>
-Paul Mackerras <paulus@samba.org>
-Piotr Drąg <piotrdrag@gmail.com>
-Qin Guan <qguan@redhat.com>
-Rajesh Ranjan <rranjan@redhat.com>
-Richard W.M. Jones <rjones@redhat.com>
-Sandeep Shedmake <sshedmak@redhat.com>
-Shankar Prasad <svenkate@redhat.com>
-Thomas S Hatch <thatch45@gmail.com>
-Wanlong Gao <gaowanlong@cn.fujitsu.com>
-Wulf C. Krueger <philantrop@exherbo.org>
+Adam Huffman
+Angus Salkeld
+Ani Peter
+Bastien ROUCARIÈS
+Charles Duffy
+Cole Robinson
+Colin Walters
+Dan Lipsitt
+Daniel Berrange
+Daniel Cabrera
+Daniel Exner
+Dave Vasilevsky
+Douglas Schilling Landgraf
+Eric Blake
+Erik Nolte
+Evaggelos Balaskas
+Geert Warrink
+Guido Günther
+Hilko Bengen
+infernix
+Jaswinder Singh
+Jim Meyering
+Jiri Popelka
+John Eckersberg
+Joseph Wang
+Karel Klíč
+Lee Yarwood
+Marcin Gibula
+Maros Zatko
+Martin Kletzander
+Masami HIRATA
+Matthew Booth
+Maxim Koltsov
+Michael Scherer
+Mike Kelly
+Nicholas Strugnell
+Nikita A Menkovich
+Nikita Menkovich
+Nikos Skalkotos
+Olaf Hering
+Or Goshen
+Paul Mackerras
+Pino Toscano
+Piotr Drąg
+Qin Guan
+Rajesh Ranjan
+Richard W.M. Jones
+Sandeep Shedmake
+Shankar Prasad
+Thomas S Hatch
+Török Edwin
+Wanlong Gao
+Wulf C. Krueger

BUGS

@@ -1,5 +1,5 @@
 NOTE: This file is automatically generated from "update-bugs.sh".
-Last updated: 2013-10-12
+Last updated: 2014-07-26
 
 This contains a local list of the bugs that are open against
 libguestfs.  Bugs are tracked in the Red Hat Bugzilla database
@@ -29,39 +29,21 @@ Bugs in NEW or ASSIGNED state are open and waiting for someone to fix.
 554829 NEW https://bugzilla.redhat.com/show_bug.cgi?id=554829
   SELinux handling could be done better.
 
-563450 NEW https://bugzilla.redhat.com/show_bug.cgi?id=563450
-  list-devices returns devices of different types out of order
-
 572337 NEW https://bugzilla.redhat.com/show_bug.cgi?id=572337
   libguestfs should support gptsync
 
 578103 NEW https://bugzilla.redhat.com/show_bug.cgi?id=578103
   [RFE] Tool to Compare Windows Registry Entries
 
-593511 NEW https://bugzilla.redhat.com/show_bug.cgi?id=593511
-  [RFE] function to get partition name
-
 604041 NEW https://bugzilla.redhat.com/show_bug.cgi?id=604041
   guestmount absolute symlinks don't work
 
 637251 NEW https://bugzilla.redhat.com/show_bug.cgi?id=637251
   virt-inspector fails to recognize data-only NTFS disk image
 
-660687 NEW https://bugzilla.redhat.com/show_bug.cgi?id=660687
-  guestmount: "touch" command fails: touch: setting times of `timestamp': Invalid argument
-
 693064 NEW https://bugzilla.redhat.com/show_bug.cgi?id=693064
   Symbolic links on ntfs-3g are not followed correctly by some commands
 
-696445 NEW https://bugzilla.redhat.com/show_bug.cgi?id=696445
-  Backport virt-inspector for virt-v2v
-
-696451 NEW https://bugzilla.redhat.com/show_bug.cgi?id=696451
-  libguestfs: unknown filesystem label SWAP-sda2
-
-700342 NEW https://bugzilla.redhat.com/show_bug.cgi?id=700342
-  virt-inspector resports unknown filesystem UUID
-
 709326 NEW https://bugzilla.redhat.com/show_bug.cgi?id=709326
   virt-inspector cannot detect ReactOS
 
@@ -92,9 +74,6 @@ Bugs in NEW or ASSIGNED state are open and waiting for someone to fix.
 803657 NEW https://bugzilla.redhat.com/show_bug.cgi?id=803657
   [RFE] inspect-is-netinst : support more distributions
 
-812970 NEW https://bugzilla.redhat.com/show_bug.cgi?id=812970
-  virt-rescue cannot set ><rescue> prompt, on Ubuntu 12.04
-
 813290 NEW https://bugzilla.redhat.com/show_bug.cgi?id=813290
   mdadm (md-detail) test occasionally fails with 'md device /dev/md125 does not appear to be active.'
 
@@ -164,27 +143,9 @@ Bugs in NEW or ASSIGNED state are open and waiting for someone to fix.
 920617 NEW https://bugzilla.redhat.com/show_bug.cgi?id=920617
   RFE: virt-inspector (or another tool) should produce OVF output for oVirt / RHEV
 
-923355 NEW https://bugzilla.redhat.com/show_bug.cgi?id=923355
-  guestfish prints literal "n" in error messages
-
 927447 NEW https://bugzilla.redhat.com/show_bug.cgi?id=927447
   [suse] virt tools hang on disk image, but libguestfs-test-tool runs OK
 
-950444 NEW https://bugzilla.redhat.com/show_bug.cgi?id=950444
-  inspection fails if libosinfo is not installed
-
-953907 NEW https://bugzilla.redhat.com/show_bug.cgi?id=953907
-  virt-sysprep does not correctly set the hostname on Debian/Ubuntu
-
-960522 NEW https://bugzilla.redhat.com/show_bug.cgi?id=960522
-  libguestfs is not built for F18 ppc64
-
-972292 NEW https://bugzilla.redhat.com/show_bug.cgi?id=972292
-  lgetxattrs can't show the file attribute list with ntfs FS in rhel7
-
-972294 NEW https://bugzilla.redhat.com/show_bug.cgi?id=972294
-  lsetxattr/lgetxattr doesn't work to softlink file with ntfs FS in rhel7
-
 973138 NEW https://bugzilla.redhat.com/show_bug.cgi?id=973138
   RFE: sh & debug sh should be able to capture all output and error return code
 
@@ -197,9 +158,6 @@ Bugs in NEW or ASSIGNED state are open and waiting for someone to fix.
 982516 NEW https://bugzilla.redhat.com/show_bug.cgi?id=982516
   sh "rpm -Va" fails because apparently /dev/shm is not created/bind-mounted correctly
 
-982979 NEW https://bugzilla.redhat.com/show_bug.cgi?id=982979
-  Disable fstrim API/command in RHEL 6
-
 983969 NEW https://bugzilla.redhat.com/show_bug.cgi?id=983969
   RFE: virt-sysprep should be SELinux-aware
 
@@ -221,14 +179,167 @@ Bugs in NEW or ASSIGNED state are open and waiting for someone to fix.
 994965 NEW https://bugzilla.redhat.com/show_bug.cgi?id=994965
   libguestfs doesn't work on armv7hl
 
-997160 NEW https://bugzilla.redhat.com/show_bug.cgi?id=997160
-  virt-sysprep prints: filesize: /etc/mdadm.conf: No such file or directory
+1019388 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1019388
+  firstboot scripts (virt-builder, virt-sysprep) don't work for Debian 6 & 7 guests
 
-998065 NEW https://bugzilla.redhat.com/show_bug.cgi?id=998065
-  libguestfs kernel hang in RHEL 6.5
+1023630 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1023630
+  enable-valgrind-daemon option breaks UML
 
-1012110 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1012110
-  libguestfs fails to parse /var/lib/rpm/Name during inspection
+1025942 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1025942
+  Parallel virt-df and virt-alignment-scan tests sometimes deadlock
+
+1027526 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1027526
+  RFE: Labels for logical partitions lost in Windows 2003 after virt-resize for primary partition
+
+1028676 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1028676
+  Add virt-builder to RHEL 7.1
+
+1029134 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1029134
+  Broken links in virt-builder web man page
+
+1033573 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1033573
+  libguestfs inspection does not recognize NAS4Free OS installed on QCOW2 image
+
+1040912 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1040912
+  [ARM] mount-local / FUSE support does not seem to work at all
+
+1041552 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1041552
+  virt-sysprep should remove Pegasus certificates
+
+1042780 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1042780
+  Support the virt-sysprep --delete option
+
+1051694 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1051694
+  libguestfs FTBFS on Ubuntu saucy
+
+1056047 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1056047
+  virt-df generates no output for Solaris 11 x86_64 guest.
+
+1057873 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1057873
+  libguestfs df/ subdirectory tests fail occasionally
+
+1059428 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1059428
+  Failed dependencies installing libguestfs with glibc ppc64p7
+
+1060423 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1060423
+  Support setting xattrs via FUSE
+
+1061040 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1061040
+  RFE: Enable ZFS support
+
+1064041 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1064041
+  virt-sparsify fails if a btrfs filesystem contains readonly snapshots
+
+1074005 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1074005
+  java bindings inspect_list_applications2 throws java.lang.ArrayIndexOutOfBoundsException:
+
+1075164 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1075164
+  RFE: allow passing in a pre-opened libvirt connection from python
+
+1075594 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1075594
+  RFE: python-libguestfs should be available on pypi
+
+1076478 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1076478
+  FTBFS: libguestfs-1.22.6-15.el7
+
+1079625 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1079625
+  virt-sparsify fails if a btrfs filesystem contains readonly snapshots
+
+1079734 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1079734
+  Tiny Core Linux is not recognized by libguestfs inspection
+
+1084221 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1084221
+  virt-builder --update causes vm not to allow logins
+
+1089100 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1089100
+  NetworkManager avc unlink denied for resolv.conf after using --selinux-relabel
+
+1091856 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1091856
+  scrub-file can't handle link file
+
+1091859 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1091859
+  scrub-file can't handle link file
+
+1092583 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1092583
+  guestfish removes query string from URI
+
+1097272 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1097272
+  Ruby bindings should use RB_GC_GUARD (instead of volatile) for enhanced safety
+
+1099284 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1099284
+  typo errors in man pages
+
+1099292 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1099292
+  sfdisk failed by "Device or resource busy" when using one command line
+
+1099475 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1099475
+  sfdisk: BLKRRPART: Device or resource busy
+
+1099976 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1099976
+  virt-builder gives GPG warning message
+
+1100138 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1100138
+  some operation cause "lvremove" can not find the object
+
+1100140 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1100140
+  some operation cause "lvremove" can not find the object
+
+1100528 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1100528
+  These APIs also need to add to `guestfish -h` command list
+
+1100530 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1100530
+  These APIs also need to add to `guestfish -h` command list
+
+1100533 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1100533
+  Remove 'user-cancel' from guestfish(1)
+
+1102241 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1102241
+  [RFE] libguestfs should detect OSTree (project-atomic) qcow2 disk image
+
+1102447 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1102447
+  libguestfs 1.27.13 does not build due to libxml library order
+
+1102619 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1102619
+  lvm-clear-filter failed when has two same name but different uuid's VG
+
+1102620 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1102620
+  lvm-clear-filter failed when has two same name but different uuid's VG
+
+1107317 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1107317
+  Ubuntu package fails to depend on linux-image-extra (causes missing virtio-serial)
+
+1108171 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1108171
+  RFE: virt-builder should be able to build PXE images for baremetal installation
+
+1109144 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1109144
+  virt-builder --get-kernel doesn't fetch the dtb from images
+
+1111662 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1111662
+  Guestfs.Error("vgchange_uuid_all:   Volume group has active logical volumes")
+
+1111794 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1111794
+  virt-sparsify leaves lots of data in TMPDIR
+
+1113020 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1113020
+  Support virt-sparsify option to allow specifying temporary file or block device
+
+1113153 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1113153
+  RFE: Inspection should support systemd mount units
+
+1114012 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1114012
+  "yum install guestfish" on fresh Fedora 20 results in broken libguestfs
+
+1117540 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1117540
+  Support virt-sparsify option to allow specifying temporary file or block device
+
+1118305 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1118305
+  RFE: Ability to set iSCSI-Initiator-IQN for iSCSI connections
+
+1123221 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1123221
+  make xfs faield, "mkfs.xfs: No such file or directory"
+
+1123281 NEW https://bugzilla.redhat.com/show_bug.cgi?id=1123281
+  lvm-set-filter stopped working in Rawhide
 
 503134 ASSIGNED https://bugzilla.redhat.com/show_bug.cgi?id=503134
   guestfish's list splitting does not recognize internal quoting
@@ -242,230 +353,86 @@ Bugs in NEW or ASSIGNED state are open and waiting for someone to fix.
 619334 ASSIGNED https://bugzilla.redhat.com/show_bug.cgi?id=619334
   RFE: Enable coredump capture in the appliance
 
-624335 ASSIGNED https://bugzilla.redhat.com/show_bug.cgi?id=624335
-  blockdev-setbsz succeeds, but does not affect blockdev-getbsz
-
 691389 ASSIGNED https://bugzilla.redhat.com/show_bug.cgi?id=691389
   SELinux labels don't work over guestmount (FUSE)
 
-965495 ASSIGNED https://bugzilla.redhat.com/show_bug.cgi?id=965495
-  New APIs required in libguestfs for virt-v2v in RHEL 6.5
+1069966 ASSIGNED https://bugzilla.redhat.com/show_bug.cgi?id=1069966
+  RFE: let virt-builder etc build ARM guests on x86 host
 
-998109 ASSIGNED https://bugzilla.redhat.com/show_bug.cgi?id=998109
-  Let's enable kvmclock in RHEL 7
-
-(78 bugs)
+(113 bugs)
 
 --------------------------------------------------
 Bugs in MODIFIED, POST or ON_QA state are fixed.
 You can help by testing the fixes.
 
-995712 MODIFIED https://bugzilla.redhat.com/show_bug.cgi?id=995712
-  list-filesystems command fails if there are no block devices
+1028650 POST https://bugzilla.redhat.com/show_bug.cgi?id=1028650
+  virt-* argument parser doesn't accept --arg=value (you have to use --arg value)
 
-998750 MODIFIED https://bugzilla.redhat.com/show_bug.cgi?id=998750
-  guestfish does not work due to conflict of remote and interactive mode
+1073917 POST https://bugzilla.redhat.com/show_bug.cgi?id=1073917
+  java bindings inspect_list_applications2 throws java.lang.ArrayIndexOutOfBoundsException:
 
-1000428 MODIFIED https://bugzilla.redhat.com/show_bug.cgi?id=1000428
-  virt-format uses wrong partition type for vfat filesystems
+1091803 POST https://bugzilla.redhat.com/show_bug.cgi?id=1091803
+  tar-in-opts execute failed that cause libguestfs appliance crashed
 
-1003291 MODIFIED https://bugzilla.redhat.com/show_bug.cgi?id=1003291
-  Backport cachemode patch to RHEL 7
+1122557 POST https://bugzilla.redhat.com/show_bug.cgi?id=1122557
+  virt-sparsify overwrites block devices if used as output files
 
-1011922 MODIFIED https://bugzilla.redhat.com/show_bug.cgi?id=1011922
-  Use host-passthrough instead of host-model
+1123007 POST https://bugzilla.redhat.com/show_bug.cgi?id=1123007
+  libguestfs 'direct' backend should close file descriptors before exec-ing qemu to avoid leaking !O_CLOEXEC fds
 
-672485 ON_QA https://bugzilla.redhat.com/show_bug.cgi?id=672485
-  [RFE] virt-edit/tar-in/tar-out/inspector do not support encrypted system
+1053847 MODIFIED https://bugzilla.redhat.com/show_bug.cgi?id=1053847
+  Recommended default clock/timer settings
 
-819086 ON_QA https://bugzilla.redhat.com/show_bug.cgi?id=819086
-  look for qemu-kvm on /usr/libexec
+1020950 ON_QA https://bugzilla.redhat.com/show_bug.cgi?id=1020950
+  CVE-2013-4419 libguestfs: insecure temporary directory handling for guestfish's network socket [epel-5]
 
-869179 ON_QA https://bugzilla.redhat.com/show_bug.cgi?id=869179
-  libguestfs fail to startup  on latest rhel7 tree
+1021149 ON_QA https://bugzilla.redhat.com/show_bug.cgi?id=1021149
+  Rebase libguestfs to 1.28 in RHEL 7.1
 
-880806 ON_QA https://bugzilla.redhat.com/show_bug.cgi?id=880806
-  virt-df with two -a options displays incorrect disk image name
+1079182 ON_QA https://bugzilla.redhat.com/show_bug.cgi?id=1079182
+  virt-df cannot report used disk space of windows guest when updated to 6.5
 
-892271 ON_QA https://bugzilla.redhat.com/show_bug.cgi?id=892271
-  virt-format fail to format  the same disk more than twice with lvm enable
+1097359 ON_QA https://bugzilla.redhat.com/show_bug.cgi?id=1097359
+  virt-sparsify hangs with 'No space left on device' while filling LV
 
-892275 ON_QA https://bugzilla.redhat.com/show_bug.cgi?id=892275
-  checksums-out fail to compute the checksums of all regular files in directory
+1106548 ON_QA https://bugzilla.redhat.com/show_bug.cgi?id=1106548
+  root gets an error accessing to a non-root dir on a snapshot guestmount VMDK img
 
-892291 ON_QA https://bugzilla.redhat.com/show_bug.cgi?id=892291
-  guestmount: link() incorrectly returns ENOENT, when it should be EXDEV
-
-905322 ON_QA https://bugzilla.redhat.com/show_bug.cgi?id=905322
-  "hivex-commit" should fail with a relative path
-
-909666 ON_QA https://bugzilla.redhat.com/show_bug.cgi?id=909666
-  Unexpected non-tail recursion in recv_from_daemon results in stack overflow in very long-running API calls that send progress messages
-
-909667 ON_QA https://bugzilla.redhat.com/show_bug.cgi?id=909667
-  Unexpected non-tail recursion in recv_from_daemon results in stack overflow in very long-running API calls that send progress messages
-
-921710 ON_QA https://bugzilla.redhat.com/show_bug.cgi?id=921710
-  9p support should be disabled in libguestfs in RHEL 7
-
-957796 ON_QA https://bugzilla.redhat.com/show_bug.cgi?id=957796
-  tar-out and base64-out quoting error
-
-958183 ON_QA https://bugzilla.redhat.com/show_bug.cgi?id=958183
-  Rebase libguestfs in RHEL 6.5
-
-962113 ON_QA https://bugzilla.redhat.com/show_bug.cgi?id=962113
-  Disable libguestfs block drivers which are not supported by RHEL 7 qemu + libvirt
-
-972776 ON_QA https://bugzilla.redhat.com/show_bug.cgi?id=972776
-  txz-out command produces a bzip2-compressed file (should be xz-compressed)
-
-974490 ON_QA https://bugzilla.redhat.com/show_bug.cgi?id=974490
-  Regression: Fedora inspection broken by change from guestfs_exists to guestfs_is_file
-
-975573 ON_QA https://bugzilla.redhat.com/show_bug.cgi?id=975573
-  virt-sysprep is in the wrong subpackage
-
-975771 ON_QA https://bugzilla.redhat.com/show_bug.cgi?id=975771
-  Specifying virtio interface ('iface' parameter) breaks the direct backend - libguestfs hangs
-
-976250 ON_QA https://bugzilla.redhat.com/show_bug.cgi?id=976250
-  Make xfs filesystem failed with specified blocksize, gives "unknown option -b" error
-
-976299 ON_QA https://bugzilla.redhat.com/show_bug.cgi?id=976299
-  mke2fs-J should give a meaningful error when specified type is anything except 'ext{2,3,4}'
-
-976662 ON_QA https://bugzilla.redhat.com/show_bug.cgi?id=976662
-  Create raid{4, 5, 6} by md-create leads to guestfish hang
-
-980338 ON_QA https://bugzilla.redhat.com/show_bug.cgi?id=980338
-  disk-format "qemu-img info: JSON parse error" when target file does not exist
-
-980502 ON_QA https://bugzilla.redhat.com/show_bug.cgi?id=980502
-  libguestfs is not able to be built with yum cache from multiple repos
-
-983691 ON_QA https://bugzilla.redhat.com/show_bug.cgi?id=983691
-  libguestfs double free when kernel link fails during launch
-
-985856 ON_QA https://bugzilla.redhat.com/show_bug.cgi?id=985856
-  Can't set acl value for a specified user with 'acl-set-file'
-
-988862 ON_QA https://bugzilla.redhat.com/show_bug.cgi?id=988862
-  virt-sysprep --firstboot option writes incorrect "99" (instead of "S99") sysv-init-style start up script
-
-988863 ON_QA https://bugzilla.redhat.com/show_bug.cgi?id=988863
-  virt-sysprep --firstboot option writes incorrect "99" (instead of "S99") sysv-init-style start up script
-
-989356 ON_QA https://bugzilla.redhat.com/show_bug.cgi?id=989356
-  cap-get-file will return error if the file has not be set capabilities
-
-995176 ON_QA https://bugzilla.redhat.com/show_bug.cgi?id=995176
-  RFE: Implement set-uuid command
-
-996039 ON_QA https://bugzilla.redhat.com/show_bug.cgi?id=996039
-  guestfish does not work due to conflict of remote and interactive mode
-
-997884 ON_QA https://bugzilla.redhat.com/show_bug.cgi?id=997884
-  9p support should be disabled in libguestfs in RHEL 6
-
-998108 ON_QA https://bugzilla.redhat.com/show_bug.cgi?id=998108
-  Let's enable kvmclock in RHEL 6
-
-998485 ON_QA https://bugzilla.redhat.com/show_bug.cgi?id=998485
-  guestfish remote prints "libguestfs: error: waitpid (qemu): No child processes"
-
-1000122 ON_QA https://bugzilla.redhat.com/show_bug.cgi?id=1000122
-  'sh' command before mount causes daemon to segfault
-
-1000123 ON_QA https://bugzilla.redhat.com/show_bug.cgi?id=1000123
-  'sh' command before mount causes daemon to segfault
-
-1001875 ON_QA https://bugzilla.redhat.com/show_bug.cgi?id=1001875
-  Argument 'excludes' of tar-out does not work
-
-1001876 ON_QA https://bugzilla.redhat.com/show_bug.cgi?id=1001876
-  Update "rsync-out" helpout for using wildcard
-
-1002032 ON_QA https://bugzilla.redhat.com/show_bug.cgi?id=1002032
-  mke2fs can't return the correct filesystem type when blockscount is less than 2048 for ext3
-
-(43 bugs)
+(11 bugs)
 
 --------------------------------------------------
 These bugs are in the VERIFIED state.
 
-889536 VERIFIED https://bugzilla.redhat.com/show_bug.cgi?id=889536
-  [RFE]It's better to emphasize "libguestfs-winsupport" in V2V manpage or error output
+624335 VERIFIED https://bugzilla.redhat.com/show_bug.cgi?id=624335
+  blockdev-setbsz succeeds, but does not affect blockdev-getbsz
 
-889537 VERIFIED https://bugzilla.redhat.com/show_bug.cgi?id=889537
-  Libguestfs live support should be disabled in RHEL 7 packages
+965495 VERIFIED https://bugzilla.redhat.com/show_bug.cgi?id=965495
+  New APIs required in libguestfs for virt-v2v in RHEL 6.5
 
-889538 VERIFIED https://bugzilla.redhat.com/show_bug.cgi?id=889538
-  libguestfs can not be installed with recent iptables
+982979 VERIFIED https://bugzilla.redhat.com/show_bug.cgi?id=982979
+  Disable fstrim API/command in RHEL 6
 
-892834 VERIFIED https://bugzilla.redhat.com/show_bug.cgi?id=892834
-  guestmount: rename() incorrectly follows symbolic links
+1025269 VERIFIED https://bugzilla.redhat.com/show_bug.cgi?id=1025269
+  libguestfs tests hang in libguestfs-1.20.11/tests/guests/guest-aux/make-fedora-img.pl
 
-894241 VERIFIED https://bugzilla.redhat.com/show_bug.cgi?id=894241
-  aug-init fail with the newest augeas package
+1056558 VERIFIED https://bugzilla.redhat.com/show_bug.cgi?id=1056558
+  virt-sparsify overwrites block devices if used as output files
 
-908255 VERIFIED https://bugzilla.redhat.com/show_bug.cgi?id=908255
-  error message didn't translate to user language
+1057510 VERIFIED https://bugzilla.redhat.com/show_bug.cgi?id=1057510
+  mount-local should give a clearer error if root is not mounted
 
-909573 VERIFIED https://bugzilla.redhat.com/show_bug.cgi?id=909573
-  patch libguestfs to use 'supermin' instead of 'febootstrap' in RHEL 7
+1072062 VERIFIED https://bugzilla.redhat.com/show_bug.cgi?id=1072062
+  Ruby bindings for libguestfs throws an error periodically
 
-947438 VERIFIED https://bugzilla.redhat.com/show_bug.cgi?id=947438
-  Some obsolete tools should be removed from libguestfs RHEL 7 package
+1091805 VERIFIED https://bugzilla.redhat.com/show_bug.cgi?id=1091805
+  tar-in-opts execute failed that cause libguestfs appliance crashed
 
-949486 VERIFIED https://bugzilla.redhat.com/show_bug.cgi?id=949486
-  libguestfs-1.20.5-3.el7.x86_64   dependency  problem
+1099105 VERIFIED https://bugzilla.redhat.com/show_bug.cgi?id=1099105
+  libguestfs Requires: /lib64/libldif-2.4.so.2 missing from openldap-2.4.39-6.el6.x86_64
 
-957616 VERIFIED https://bugzilla.redhat.com/show_bug.cgi?id=957616
-  libguestfs-1.21.31-1.1.el7.1.1.x86_64 won't install due to missing dependency
+1099332 VERIFIED https://bugzilla.redhat.com/show_bug.cgi?id=1099332
+  typo errors in man pages
 
-971207 VERIFIED https://bugzilla.redhat.com/show_bug.cgi?id=971207
-  guestfish aug-init command fails: libguestfs: error: aug_init: Augeas initialization failed
-
-971326 VERIFIED https://bugzilla.redhat.com/show_bug.cgi?id=971326
-  ntfsresize-opts execute failed when omitted the 'size' option
-
-971664 VERIFIED https://bugzilla.redhat.com/show_bug.cgi?id=971664
-  Need add some removed commands back into guestfish in RHEL 6.5
-
-972413 VERIFIED https://bugzilla.redhat.com/show_bug.cgi?id=972413
-  txz-out command produces a bzip2-compressed file (should be xz-compressed)
-
-973425 VERIFIED https://bugzilla.redhat.com/show_bug.cgi?id=973425
-  lsscsi is not available in 6client
-
-975377 VERIFIED https://bugzilla.redhat.com/show_bug.cgi?id=975377
-  inspect-get-hostname return unknown for linux guest in rhel6
-
-975572 VERIFIED https://bugzilla.redhat.com/show_bug.cgi?id=975572
-  virt-sysprep is in the wrong subpackage
-
-975753 VERIFIED https://bugzilla.redhat.com/show_bug.cgi?id=975753
-  "virt-resize --expand" and "virt-resize --resize" outputs error message for Win2008 32bit OS
-
-975760 VERIFIED https://bugzilla.redhat.com/show_bug.cgi?id=975760
-  Specifying virtio interface ('iface' parameter) breaks the appliance attach-method - libguestfs hangs
-
-980358 VERIFIED https://bugzilla.redhat.com/show_bug.cgi?id=980358
-  filesystem-available should return false for xfs in rhel6
-
-980372 VERIFIED https://bugzilla.redhat.com/show_bug.cgi?id=980372
-  "hivex-commit" should fail with a relative path
-
-983690 VERIFIED https://bugzilla.redhat.com/show_bug.cgi?id=983690
-  libguestfs double free when kernel link fails during launch
-
-985269 VERIFIED https://bugzilla.redhat.com/show_bug.cgi?id=985269
-  Can't set acl value for a specified user with 'acl-set-file'
-
-989352 VERIFIED https://bugzilla.redhat.com/show_bug.cgi?id=989352
-  cap-get-file will return error if the file has not be set capabilities
-
-(24 bugs)
+(10 bugs)
 
 End of BUGS file.

Makefile.am

@@ -1,5 +1,5 @@
 # libguestfs
-# Copyright (C) 2009-2013 Red Hat Inc.
+# Copyright (C) 2009-2014 Red Hat Inc.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -20,7 +20,10 @@ include $(top_srcdir)/common-rules.mk
 ACLOCAL_AMFLAGS = -I m4
 
 # Gnulib - must be built and tested before the library.
-SUBDIRS = gnulib/lib gnulib/tests
+SUBDIRS = gnulib/lib
+if ENABLE_GNULIB_TESTS
+SUBDIRS += gnulib/tests
+endif
 
 # Basic source for the library.
 SUBDIRS += tests/data generator src examples po
@@ -28,7 +31,9 @@ SUBDIRS += tests/data generator src examples po
 if ENABLE_DAEMON
 SUBDIRS += daemon
 endif
+if ENABLE_APPLIANCE
 SUBDIRS += appliance
+endif
 
 # Tests - order is important.
 if ENABLE_APPLIANCE
@@ -39,7 +44,9 @@ SUBDIRS += tests/tmpdirs
 SUBDIRS += tests/protocol
 SUBDIRS += tests/events
 SUBDIRS += tests/parallel
+SUBDIRS += tests/create
 SUBDIRS += tests/disks
+SUBDIRS += tests/discard
 SUBDIRS += tests/mountable
 SUBDIRS += tests/network
 SUBDIRS += tests/lvm
@@ -62,6 +69,7 @@ SUBDIRS += tests/http
 SUBDIRS += tests/syslinux
 SUBDIRS += tests/journal
 SUBDIRS += tests/fuzz
+SUBDIRS += tests/relative-paths
 SUBDIRS += tests/regressions
 endif
 
@@ -72,7 +80,10 @@ SUBDIRS += test-tool
 SUBDIRS += fish
 
 # virt-tools in C.
-SUBDIRS += align cat df edit format inspector rescue
+SUBDIRS += align cat diff df edit format inspector make-fs rescue
+if HAVE_P2V
+SUBDIRS += p2v p2v-iso
+endif
 
 # bash-completion
 SUBDIRS += bash
@@ -115,10 +126,17 @@ endif
 # Unconditional because nothing is built yet.
 SUBDIRS += csharp
 
-# OCaml tools.  Note 'mllib' contains random shared code used by
-# all of the OCaml tools.
+# OCaml tools.  Note 'mllib' and 'customize' contain shared code used
+# by other OCaml tools, so these must come first.
 if HAVE_OCAML
-SUBDIRS += mllib builder resize sparsify sysprep
+SUBDIRS += \
+	mllib \
+	customize \
+	builder builder/website \
+	resize \
+	sparsify \
+	sysprep \
+	v2v
 endif
 
 # Perl tools.
@@ -137,14 +155,16 @@ SUBDIRS += po-docs
 endif
 
 EXTRA_DIST = \
-	BUGS HACKING ROADMAP TODO \
+	BUGS HACKING TODO \
 	.gitignore \
 	.lvimrc \
 	.mailmap \
 	.tx/config \
+	bootstrap \
 	bugs-in-changelog.sh \
 	autogen.sh \
 	bindtests \
+	cfg.mk \
 	contrib/autobuild/autobuild.sh \
 	contrib/intro/libguestfs-intro.html \
 	contrib/intro/overview.png \
@@ -169,18 +189,31 @@ EXTRA_DIST = \
 	contrib/visualize-alignment/qemu-0.13-trace-block-device-access.patch \
 	contrib/visualize-alignment/README \
 	contrib/visualize-alignment/tracetops.ml \
+	contrib/windows-icons.pl \
 	guestfs-release-notes.pod \
 	guestfs-release-notes.txt \
 	html/draft.png \
 	html/draft.svg \
 	html/pod.css \
+	html/virt-builder.svg \
 	libtool-kill-dependency_libs.sh \
 	logo/fish.svg logo/fish.png \
+	logo/fish-5yrs.svg logo/fish-5yrs.png \
+	logo/virt-builder.svg \
 	m4/.gitignore \
+	tests/run-xml-to-junit.sh \
+	tests/run-xml-to-junit.xsl \
 	tmp/.gitignore \
 	tx-pull.sh \
 	update-bugs.sh \
-	valgrind-suppressions
+	valgrind-suppressions \
+	.x-sc_avoid_ctype_macros \
+	.x-sc_prohibit_have_config_h \
+	.x-sc_prohibit_magic_number_exit \
+	.x-sc_prohibit_strcmp \
+	.x-sc_prohibit_strcmp_and_strncmp \
+	.x-sc_TAB_in_indentation \
+	.x-sc_trailing_blank
 
 # The website.
 HTMLFILES = \
@@ -192,6 +225,7 @@ HTMLFILES = \
 	html/guestfs-java.3.html \
 	html/guestfs-lua.3.html \
 	html/guestfs-ocaml.3.html \
+	html/guestfs-p2v-iso.1.html \
 	html/guestfs-performance.1.html \
 	html/guestfs-perl.3.html \
 	html/guestfs-python.3.html \
@@ -210,13 +244,16 @@ HTMLFILES = \
 	html/virt-cat.1.html \
 	html/virt-copy-in.1.html \
 	html/virt-copy-out.1.html \
+	html/virt-customize.1.html \
 	html/virt-df.1.html \
+	html/virt-diff.1.html \
 	html/virt-edit.1.html \
 	html/virt-filesystems.1.html \
 	html/virt-format.1.html \
 	html/virt-inspector.1.html \
 	html/virt-list-filesystems.1.html \
 	html/virt-list-partitions.1.html \
+	html/virt-log.1.html \
 	html/virt-ls.1.html \
 	html/virt-make-fs.1.html \
 	html/virt-rescue.1.html \
@@ -226,22 +263,20 @@ HTMLFILES = \
 	html/virt-tar.1.html \
 	html/virt-tar-in.1.html \
 	html/virt-tar-out.1.html \
+	html/virt-v2v.1.html \
 	html/virt-win-reg.1.html
 
-HTMLSUPPORTFILES = html/draft.png html/pod.css
+HTMLSUPPORTFILES = \
+	html/draft.png \
+	html/pod.css \
+	html/virt-builder.svg
 
-TEXTFILES = BUGS README ROADMAP TODO
+TEXTFILES = BUGS README TODO
 
 BUILDERFILES = \
 	builder/website/README \
 	builder/website/index \
-	builder/website/index.asc \
-	builder/website/debian-7.xz \
-	builder/website/debian-7.xz.sig \
-	builder/website/fedora-18.xz \
-	builder/website/fedora-18.xz.sig \
-	builder/website/fedora-19.xz \
-	builder/website/fedora-19.xz.sig
+	builder/website/index.asc
 
 WEBSITEDIR = $(HOME)/d/websites/libguestfs
 
@@ -261,12 +296,12 @@ website: $(HTMLFILES) $(HTMLSUPPORTFILES) $(TEXTFILES) $(BUILDERFILES)
 
 # When doing 'make dist' update a few files automatically.
 dist-hook:
-	$(top_srcdir)/build-aux/gitlog-to-changelog > ChangeLog
+	git log --decorate=false > ChangeLog
 	cp ChangeLog $(distdir)/ChangeLog
 	$(top_srcdir)/update-bugs.sh > BUGS-t
 	mv BUGS-t BUGS
 	cp BUGS $(distdir)/BUGS
-	git shortlog -se | awk -F'\t' '{print $$2}' | sort -f > AUTHORS-t
+	git shortlog -s | $(AWK) -F'\t' '{print $$2}' | sort -f > AUTHORS-t
 	mv AUTHORS-t AUTHORS
 	cp AUTHORS $(distdir)/AUTHORS
 
@@ -286,9 +321,10 @@ all-local:
 	find $(DIST_SUBDIRS) -name '*.c' -o -name '*.pl' -o -name '*.pm' | \
 	grep -v -E '^(examples|gnulib|perl/(blib|examples)|po-docs|tests)/' | \
 	grep -v -E '/((guestfs|rc)_protocol\.c)$$' | \
+	grep -v -E '^python/utils.c$$' | \
 	LC_ALL=C sort > po/POTFILES
 	cd $(srcdir); \
-	find builder mllib resize sparsify sysprep -name '*.ml' | \
+	find builder customize mllib resize sparsify sysprep v2v -name '*.ml' | \
 	LC_ALL=C sort > po/POTFILES-ml
 
 # Manual pages in top level directory.
@@ -474,11 +510,28 @@ build-test-guests:
 print-subdirs:
 	@echo $(SUBDIRS)
 
+# Commit everything in current directory to HEAD, and set commit
+# message to current version (only for maintainer).
+
+maintainer-commit:
+	git commit -a -m "Version $(VERSION)."
+
 # Tag HEAD with current version (only for maintainer).
 
 maintainer-tag:
 	git tag -a $(VERSION) -m "Version $(VERSION) ($(BRANCH_TYPE))" -f
 
+# Maintainer only: check EXTRA_DIST rule is complete.
+# (Note you must have done 'make dist')
+
+maintainer-check-extra-dist:
+	zcat $(PACKAGE_NAME)-$(VERSION).tar.gz | tar tf - | sort | \
+	  sed 's,^$(PACKAGE_NAME)-$(VERSION)/,,' > tmp/tarfiles
+	git ls-files | sort > tmp/gitfiles
+	diff -ur tmp/tarfiles tmp/gitfiles | grep '^\+' | \
+	  grep -v src/api-support/[0-9]
+	rm tmp/tarfiles tmp/gitfiles
+
 # Provide help on common Makefile targets.
 
 help:

README

@@ -61,12 +61,11 @@ The full requirements are described below.
 |              |             |   |  - virtio-block                         |
 |              |             |   |  - virtio-net                           |
 +--------------+-------------+---+-----------------------------------------+
-| supermin     | 4.1.0       | R | This is required on all distros.        |
-| febootstrap  | 3.20        |   | 'supermin' is the new name for          |
+| supermin     | 5.1.0       | R | This is required on all distros.        |
+|              |             |   | 'supermin' is the new name for          |
 |              |             |   | 'febootstrap'.                          |
 |              |             |   | For alternatives, see:                  |
 |              |             |   | libguestfs.org/download/binaries/appliance/
-|              |             |   | febootstrap 2.x WILL NOT WORK           |
 +--------------+-------------+---+-----------------------------------------+
 | glibc        |             | R | We use various glibc-isms.              |
 |              |             |   | Also glibc provides XDR, rpcgen.        |
@@ -94,6 +93,10 @@ The full requirements are described below.
 +--------------+-------------+---+-----------------------------------------+
 | gperf        |             | R |                                         |
 +--------------+-------------+---+-----------------------------------------+
+| flex         |             | R | flex & bison are required for virt-     |
++--------------+-------------+---| builder.  We could make these           |
+| bison        |             | R | optional but automakes makes it hard.   |
++--------------+-------------+---+-----------------------------------------+
 | PCRE         |             | R | Perl-compatible Regular Expression lib. |
 +--------------+-------------+---+-----------------------------------------+
 | genisoimage  |             | R | mkisofs may work.                       |
@@ -102,6 +105,9 @@ The full requirements are described below.
 +--------------+-------------+---+-----------------------------------------+
 | augeas       | 1.0.0       | R |                                         |
 +--------------+-------------+---+-----------------------------------------+
+| xz           |             | R | Used to compress disk images.           |
+|              |             |   | Used by virt-builder for compression.   |
++--------------+-------------+---+-----------------------------------------+
 | po4a         |             |R/O| Required if compiling from git.         |
 |              |             |   | Optional if compiling from tarball.     |
 |              |             |   | For localizing man pages.               |
@@ -167,10 +173,13 @@ The full requirements are described below.
 | gpg          |             | O | Used by virt-builder for digital        |
 |              |             |   | signatures                              |
 +--------------+-------------+---+-----------------------------------------+
-| xz           |             | O | Used by virt-builder for compression    |
+| liblzma      |             | O | Can be used by virt-builder for fast    |
+|              |             |   | uncompression of templates.             |
 +--------------+-------------+---+-----------------------------------------+
-| nbdkit       |             | O | Used by virt-builder to speed up        |
-|              |             |   | template xz-decompression               |
+| gtk2         |             | O | Used by virt-p2v user interface.        |
++--------------+-------------+---+-----------------------------------------+
+| python-evtx  |             | O | Used by virt-log to parse Windows       |
+|              |             |   | Event Log files.                        |
 +--------------+-------------+---+-----------------------------------------+
 | findlib      |             | O | For the OCaml bindings.                 |
 +--------------+-------------+---+-----------------------------------------+
@@ -217,8 +226,6 @@ The full requirements are described below.
 +--------------+-------------+---+-----------------------------------------+
 | Test::More   |             | O | Perl module used by tests.              |
 +--------------+-------------+---+-----------------------------------------+
-| String::Shellquote         | O | Perl module used by some virt-* tools.  |
-+--------------+-------------+---+-----------------------------------------+
 | XML::XPath   |             | O | Perl module used by some virt-* tools.  |
 +--------------+-------------+---+-----------------------------------------+
 | XML::XPath::XMLParser      | O | Perl module used by some virt-* tools.  |
@@ -277,34 +284,31 @@ You can run guestfish, guestmount and the virt tools without needing
 to install, using the "./run" script in the top directory.  This
 script sets up some environment variables.  For example:
 
-  ./run ./fish/guestfish [usual guestfish args ...]
+  ./run guestfish [usual guestfish args ...]
 
-  ./run ./inspector/virt-inspector [usual virt-inspector args ...]
+  ./run virt-inspector [usual virt-inspector args ...]
 
 If you are already in the fish/ subdirectory, then the following
 command will also work:
 
-  ../run ./guestfish [...]
+  ../run guestfish [...]
 
-You can also make a link from your $PATH to the run script, eg:
-
-  cd ~/bin
-  ln -s ~/libguestfs/run libguestfs-run
-  cd ~/libguestfs
-  libguestfs-run ./inspector/virt-inspector [...]
+The ./run script adds every libguestfs binary to the $PATH, so the
+above example runs guestfish from the build directory (not the
+globally installed guestfish if there is one).
 
 You can also run the C programs under valgrind like this:
 
-  ./run valgrind [valgrind opts...] ./cat/virt-cat [virt-cat opts...]
+  ./run valgrind [valgrind opts...] virt-cat [virt-cat opts...]
 
 or under gdb:
 
-  ./run gdb --args ./cat/virt-cat [virt-cat opts...]
+  ./run gdb --args virt-cat [virt-cat opts...]
 
 This also works with sudo (eg. if you need root access for libvirt or
 to access a block device):
 
-  sudo ./run ./cat/virt-cat -d LinuxGuest /etc/passwd
+  sudo ./run virt-cat -d LinuxGuest /etc/passwd
 
 
 qemu
@@ -391,7 +395,7 @@ Don't enable GCC warnings (ie. *don't* use
 Copyright and license information
 ----------------------------------------------------------------------
 
-Copyright (C) 2009-2013 Red Hat Inc.
+Copyright (C) 2009-2014 Red Hat Inc.
 
 The library is distributed under the LGPLv2+.  The programs are
 distributed under the GPLv2+.  Please see the files COPYING and

ROADMAP

@@ -1,39 +0,0 @@
-Roadmap for future releases
----------------------------
-
-Before you read this:
-
-(1) To see what's in the current release, read 'guestfs-release-notes(1)'.
-
-(2) To see the list of bugs, read 'BUGS'.
-
-(3) To understand libguestfs versioning, read this:
-    http://libguestfs.org/guestfs.3.html#libguestfs_version_numbers
-
-(4) For general "might be good to have" items, see 'TODO'.
-
-For next major stable release (1.24)
-------------------------------------
-
-* Allow remote libvirt volumes to be accessed.  This requires some
-  enhancements to libvirt which have been agreed but not yet
-  implemented.
-
-Bugs assigned to 1.24 (put "1.24" in the Devel Whiteboard field in
-Bugzilla):
-
-https://bugzilla.redhat.com/buglist.cgi?query_format=advanced&field0-0-0=cf_devel_whiteboard&bug_status=NEW&bug_status=ASSIGNED&bug_status=MODIFIED&bug_status=ON_DEV&bug_status=ON_QA&bug_status=VERIFIED&bug_status=FAILS_QA&bug_status=RELEASE_PENDING&bug_status=POST&bug_status=PASSES_QA&type0-0-0=anywords&value0-0-0=1.24&component=libguestfs&product=Virtualization%20Tools
-
-Beyond 1.24
------------
-
-* Make 'guestfish --ro' be the default, and get users to use
-  'guestfish --rw' for write access (but allow the default to be
-  overridden in a configuration file).  This was originally planned
-  for 1.10 but there's not nearly enough adoption of the new
-  'guestfish --rw' option out there to do this yet.
-
-* Write a new partition handling library to replace parted, and use it
-  instead of parted.  (RHBZ#593511, RHBZ#642821).
-
-See TODO and BUGS files.

TODO

@@ -501,22 +501,6 @@ this approach works, it doesn't solve the MBR problem, so likely we'd
 have to write a library for that (or perhaps go back to sfdisk but
 using a very abstracted interface over sfdisk).
 
-qemu caching
-------------
-
-(Suggested by Paolo Bonzini and Kevin Wolf)
-
-Measure the effect of cache=none, cache=directsync,
-cache=writethrough, cache=writeback.
-
-It's doubtful that using cache=none is useful, since it disables the
-host cache making read-heavy workloads slower (they rely entirely on
-the smaller appliance kernel's cache).  And in libguestfs we don't
-necessarily care about ongoing data integrity while writing, as long
-as data is reliably written out when g.sync, g.shutdown or g.close
-return.  Also in libguestfs we effectively control the whole stack, so
-we can ensure write barriers happen when we want.
-
 virt-sparsify should use discard
 --------------------------------
 
@@ -558,9 +542,20 @@ virt-builder
 
  - set default timezone and language
 
- - how can we give users a shell?
+ - set permissions on uploaded files & created directories
 
- - Ubuntu templates
+ - how can we give users a shell for debugging purposes?
+
+ - allow non-xz-compressed templates (faster)
+
+ - let notes etc be localized, ie. notes[en]=...
+
+ - add a CLI option to print the in-built path/fingerprint(s)
+
+ - allow public keys to come from local file paths
+
+ - doing virt-builder then running (eg. via qemu, libvirt?) is common; is
+   it possible to make this more automatic?
 
  - more common code between virt-sysprep & virt-builder:
 
@@ -569,12 +564,19 @@ virt-builder
 
  - document:
 
-   * how to upload to glance
    * how to integrate with ansible, puppet, chef
+   * how to import to OpenStack / Glance
+   * how to import to EC2
 
  - /etc/resolv.conf handling works but is best described as a hack:
    https://github.com/libguestfs/libguestfs/commit/9521422ce60578f7196cc8b7977d998159238c19
 
+ - let's make UML work
+   + SLIRP is insecure, but we could allow just a bare web proxy which
+     gets proxied over virtio-serial to the outside world (except
+     virtio-serial can't be multiplexed)
+
+ - sometimes (not always) aug_init takes ages, why?
 
 Midnight Commander (mc) extension
 ---------------------------------
@@ -582,3 +584,17 @@ Midnight Commander (mc) extension
 Write an extension for mc that would let people browse into
 filesystems.  See
 http://repo.or.cz/w/midnight-commander.git/tree/HEAD:/misc/ext.d
+
+Python
+------
+
+It seems as if we should call PyErr_Clear() somewhere in every
+Python binding.
+
+Improvements in virt-log
+------------------------
+
+- Make it faster, especially if the user wants to grep the output.
+
+- Support Windows guests, see
+  http://rwmj.wordpress.com/2011/04/17/decoding-the-windows-event-log-using-guestfish/

align/Makefile.am

@@ -58,7 +58,6 @@ virt_alignment_scan_CPPFLAGS = \
 virt_alignment_scan_CFLAGS = \
 	-pthread \
 	$(WARN_CFLAGS) $(WERROR_CFLAGS) \
-	$(GPROF_CFLAGS) $(GCOV_CFLAGS) \
 	$(LIBCONFIG_CFLAGS) \
 	$(LIBXML2_CFLAGS) \
 	$(LIBVIRT_CFLAGS)

align/scan.c

@@ -121,7 +121,6 @@ main (int argc, char *argv[])
     { 0, 0, 0, 0 }
   };
   struct drv *drvs = NULL;
-  struct drv *drv;
   const char *format = NULL;
   int c;
   int option_index;

align/virt-alignment-scan.pod

@@ -1,5 +1,3 @@
-=encoding utf8
-
 =head1 NAME
 
 virt-alignment-scan - Check alignment of virtual machine partitions
@@ -349,13 +347,6 @@ contains an explicit C<%pre> section that creates aligned partitions
 using L<parted(8)>.  Do not use the Kickstart C<part> command.  The
 NetApp document above contains an example.
 
-=head1 SHELL QUOTING
-
-Libvirt guest names can contain arbitrary characters, some of which
-have meaning to the shell such as C<#> and space.  You may need to
-quote or escape these characters on the command line.  See the shell
-manual page L<sh(1)> for details.
-
 =head1 EXIT STATUS
 
 This program returns:

appliance/Makefile.am

@@ -1,5 +1,5 @@
 # libguestfs
-# Copyright (C) 2009 Red Hat Inc.
+# Copyright (C) 2009-2014 Red Hat Inc.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -19,8 +19,9 @@ include $(top_srcdir)/subdir-rules.mk
 
 EXTRA_DIST = \
 	99-guestfs-serial.rules \
-	excludelist.in \
+	excludefiles.in \
 	guestfsd.suppressions \
+	hostfiles.in \
 	init \
 	libguestfs-make-fixed-appliance.in \
 	libguestfs-make-fixed-appliance.pod \
@@ -32,23 +33,25 @@ superminfsdir = $(libdir)/guestfs/supermin.d
 
 fs_DATA =
 superminfs_DATA = \
-	supermin.d/init.img \
-	supermin.d/udev-rules.img
+	supermin.d/base.tar.gz \
+	supermin.d/daemon.tar.gz \
+	supermin.d/excludefiles \
+	supermin.d/hostfiles \
+	supermin.d/init.tar.gz \
+	supermin.d/packages \
+	supermin.d/udev-rules.tar.gz
 
-if SUPERMIN_HELPER_COMPRESSED_CPIO
-GZ = .gz
-endif
-
-if ENABLE_DAEMON
-superminfs_DATA += \
-	supermin.d/daemon.img$(GZ)
-endif
-
-if ENABLE_APPLIANCE
-superminfs_DATA += \
-	supermin.d/base.img$(GZ) \
-	supermin.d/hostfiles
-endif
+supermin.d/base.tar.gz supermin.d/packages: stamp-supermin
+stamp-supermin: make.sh packagelist
+	rm -f $@ supermin.d/base.tar.gz supermin.d/packages
+	./make.sh
+	$(MAKE) \
+	  supermin.d/daemon.tar.gz \
+	  supermin.d/excludefiles \
+	  supermin.d/hostfiles \
+	  supermin.d/init.tar.gz \
+	  supermin.d/udev-rules.tar.gz
+	touch $@
 
 # This used to be a configure-generated file.  However config.status
 # always touches the destination file, which means the appliance got
@@ -66,56 +69,49 @@ PACKAGELIST_CPP_FLAGS += -DVALGRIND_DAEMON=1
 endif
 
 packagelist: packagelist.in Makefile
-	cpp -undef $(PACKAGELIST_CPP_FLAGS) < $< | \
+	m4 $(PACKAGELIST_CPP_FLAGS) $< | \
 	grep -v '^[[:space:]]*$$' | grep -v '^#' > $@-t
 	cmp -s $@ $@-t || mv $@-t $@
 	rm -f $@-t
 
-excludelist: excludelist.in Makefile
-	cpp -undef $(PACKAGELIST_CPP_FLAGS) < $< | \
-	grep -v '^[[:space:]]*$$' | grep -v '^#' > $@-t
-	cmp -s $@ $@-t || mv $@-t $@
-	rm -f $@-t
-
-supermin.d/base.img$(GZ) supermin.d/hostfiles: stamp-supermin
-stamp-supermin: make.sh packagelist excludelist
-	rm -f $@ supermin.d/base.img$(GZ) supermin.d/hostfiles
-	./make.sh
-if SUPERMIN_HELPER_COMPRESSED_CPIO
-	gzip -9 supermin.d/base.img
-endif
-	touch $@
-
-supermin.d/daemon.img$(GZ): ../daemon/guestfsd guestfsd.suppressions
-	rm -f $@ $@-t $@-tt
+supermin.d/daemon.tar.gz: ../daemon/guestfsd guestfsd.suppressions
+	rm -f $@ $@-t
 	rm -rf tmp-d
 	mkdir -p tmp-d$(DAEMON_SUPERMIN_DIR) tmp-d/etc
 	ln ../daemon/guestfsd tmp-d$(DAEMON_SUPERMIN_DIR)/guestfsd
 	ln $(srcdir)/guestfsd.suppressions tmp-d/etc/guestfsd.suppressions
-	( cd tmp-d && find | cpio --quiet -o -H newc ) > $@-t
+	( cd tmp-d && tar zcf - * ) > $@-t
 	rm -r tmp-d
-if SUPERMIN_HELPER_COMPRESSED_CPIO
-	gzip -9 -c $@-t > $@-tt
-	mv $@-tt $@-t
-endif
 	mv $@-t $@
 
-supermin.d/init.img: init
-	rm -rf init.tmp $@ $@-t
-	mkdir init.tmp
-	cp $< init.tmp
-	(cd init.tmp; echo "init" | cpio --quiet -o -H newc) > $@-t
-	rm -r init.tmp
+# Note we must avoid touching supermin.d if possible, so don't create
+# the temporary files inside the supermin.d directory.
+
+$@supermin.d/excludefiles: excludefiles.in Makefile
+	m4 $(PACKAGELIST_CPP_FLAGS) $< | \
+	grep -v '^[[:space:]]*$$' | grep -v '^#' > excludefiles-t
+	cmp -s $@ excludefiles-t || mv excludefiles-t $@
+	rm -f excludefiles-t
+
+supermin.d/hostfiles: hostfiles.in Makefile
+	m4 $(PACKAGELIST_CPP_FLAGS) $< | \
+	grep -v '^[[:space:]]*$$' | grep -v '^#' > hostfiles-t
+	cmp -s $@ hostfiles-t || mv hostfiles-t $@
+	rm -f hostfiles-t
+
+supermin.d/init.tar.gz: init
+	rm -f $@ $@-t
+	( cd $(srcdir) && tar zcf - init ) > $@-t
 	mv $@-t $@
 
 # We should put this file in /lib/udev/rules.d, but put it in /etc so
 # we don't have to deal with all the UsrMove crap in Fedora.
-supermin.d/udev-rules.img: 99-guestfs-serial.rules
+supermin.d/udev-rules.tar.gz: 99-guestfs-serial.rules
 	rm -f $@ $@-t
 	rm -rf tmp-u
 	mkdir -p tmp-u/etc/udev/rules.d
 	for f in $^; do ln $$f tmp-u/etc/udev/rules.d/$$(basename $$f); done
-	( cd tmp-u && find | cpio --quiet -o -H newc ) > $@-t
+	( cd tmp-u && tar zcf - etc ) > $@-t
 	rm -r tmp-u
 	mv $@-t $@
 
@@ -144,7 +140,11 @@ stamp-libguestfs-make-fixed-appliance.pod: libguestfs-make-fixed-appliance.pod
 
 # Make clean.
 
-CLEANFILES = packagelist excludelist \
+CLEANFILES = \
+	*~ \
 	libguestfs-make-fixed-appliance.1 \
-	stamp-libguestfs-make-fixed-appliance.pod \
+	stamp-libguestfs-make-fixed-appliance.pod
+
+DISTCLEANFILES = \
+	stamp-supermin \
 	supermin.d/*

appliance/excludefiles.in

@@ -0,0 +1,44 @@
+dnl This is the list of files excluded from the appliance, even if
+dnl they appear in packagelist.in (or more likely, as dependencies of
+dnl packages in packagelist.in).
+dnl
+dnl List is a list of wildcards, one per line, prefixed by a '-' character.
+dnl
+dnl This file is processed by m4 with one of the
+dnl following symbols defined (depending on the distro):
+dnl
+dnl   REDHAT=1     For Fedora, RHEL, EPEL and workalikes.
+dnl   DEBIAN=1     For Debian.
+dnl   UBUNTU=1     For Ubuntu.
+dnl   ARCHLINUX=1  For Archlinux.
+dnl
+dnl Note that any matching file will be dropped from the appliance.
+dnl Of course, this may break the appliance, so be careful.
+
+dnl The right kernel modules are added back by supermin.
+-/boot/*
+-/lib/modules/*
+
+-/usr/lib/locale/*
+-/usr/share/locale/*
+-/usr/share/man/*
+-/usr/share/doc/*
+-/usr/share/info/*
+-/usr/share/gnome/help/*
+-/usr/share/cracklib/*
+-/usr/share/i18n/*
+-/usr/share/pkgconfig/*
+
+-/var/log/*.log*
+-/var/log/cron*
+-/var/log/lastlog*
+-/var/log/messages*
+-/var/log/secure*
+-/var/log/syslog*
+
+dnl For Debian:
+-/usr/share/lintian/*
+-/usr/share/initramfs-tools/*
+-/usr/share/doc-base/*
+-/usr/share/bug/*
+-/etc/initramfs-tools/*

appliance/excludelist.in

@@ -1,57 +0,0 @@
-/* This is the list of distro packages which are
- * excluded from the appliance, even if they appear in
- * packagelist.in (or more likely, as dependencies of
- * packages in packagelist.in).
- *
- * List is a list of basic regular expressions, one per line.
- *
- * This file is processed by cpp with one of the
- * following symbols defined (depending on the distro):
- *
- *   REDHAT=1     For Fedora, RHEL, EPEL and workalikes.
- *   DEBIAN=1     For Debian.
- *   UBUNTU=1     For Ubuntu.
- *   ARCHLINUX=1  For Archlinux.
- *
- * Note that any file provided by one of these packages will
- * be dropped from the appliance.  Of course, this may break
- * the appliance, so be careful.  Other files are also dropped
- * from the appliance such as docs and man pages: see 'make.sh.in'
- * for the full details.
- */
-
-/* Basically the same with a few minor tweaks. */
-#ifdef UBUNTU
-#define DEBIAN 1
-#endif
-
-/* Don't need any Perl or Python appearing in the appliance. */
-^perl
-^python
-
-/* Plymouth is a graphical boot thing - not needed. */
-^plymouth
-
-/* Linux firmware. */
-^linux-firmware
-
-/* Keyboard maps - appliance is not interactive. */
-^kbd-misc
-
-#ifdef REDHAT
-
-/* Linux kernel.  febootstrap <= 3.18 used to exclude the kernel
- * package (only) by default, but since 3.19 it doesn't do this any
- * longer.
- */
-^kernel
-
-^fedora-logos
-^redhat-logos
-^dracut
-
-#endif
-
-#ifdef DEBIAN
-^file-rc
-#endif

appliance/hostfiles.in

@@ -0,0 +1,14 @@
+dnl This is the list of extra files added to appliance.
+dnl
+dnl List is a list of wildcards, one per line.
+dnl
+dnl This file is processed by m4 with one of the
+dnl following symbols defined (depending on the distro):
+dnl
+dnl   REDHAT=1     For Fedora, RHEL, EPEL and workalikes.
+dnl   DEBIAN=1     For Debian.
+dnl   UBUNTU=1     For Ubuntu.
+dnl   ARCHLINUX=1  For Archlinux.
+
+/lib/lsb/*
+/usr/share/augeas/lenses/*.aug

appliance/init

@@ -10,6 +10,10 @@ RUNLEVEL=S
 PREVLEVEL=N
 export RUNLEVEL PREVLEVEL
 
+# Make sure to find all the libraries, also those in non-standard place
+# but with a proper ld.so configuration pointing at them
+ldconfig
+
 # Try to print a stack trace for segfaults inside the appliance.
 for d in /lib64 /lib; do
   f=$d/libSegFault.so
@@ -32,53 +36,25 @@ mount -t sysfs /sys /sys
 mkdir -p /run
 mount -t tmpfs -o "nosuid,size=20%,mode=0755" tmpfs /run
 mkdir -p /run/lock
+ln -s ../run/lock /var/lock
 
 # devtmpfs is required since udev 176
 mount -t devtmpfs /dev /dev
 
-if [ ! -L /etc/init.d/udev -a -x /etc/init.d/udev ]; then
-  if type service >/dev/null 2>&1; then
-     service udev start
-  else
-     /etc/init.d/udev start
-  fi
-elif [ -x /sbin/start_udev ] && /sbin/start_udev; then
-  :
-else
-  # Find udevd and run it directly.
-  for f in /sbin/udevd /lib/udev/udevd \
-      /lib/systemd/systemd-udevd /usr/lib/systemd/systemd-udevd \
-      /usr/lib/udev/udevd; do
-    if [ -x "$f" ]; then UDEVD="$f"; fi
-  done
-  if [ -n "$UDEVD" ]; then
-    echo -e '\000\000\000\000' > /proc/sys/kernel/hotplug
-    $UDEVD --daemon
-    udevadm trigger
-    udevadm settle --timeout=600
-  else
-    echo No udevd, creating /dev manually.
-    mount -t tmpfs none /dev
-    mkdir /dev/pts /dev/shm /dev/mapper
-    mount -t devpts -o gid=5,mode=620 /dev/pts /dev/pts
-    # Must do each MAKEDEV individually, because if one device fails,
-    # MAKEDEV will quit without creating the rest (RHBZ#507374).
-    for dev in mem null port zero core full ram tty console fd \
-      hda hdb hdc hdd sda sdb sdc sdd loop sd; do
-      MAKEDEV $dev ||:
-    done
-    mknod /dev/ptmx c 5 2;    chmod 0666 /dev/ptmx
-    mknod /dev/random c 1 8;  chmod 0666 /dev/random
-    mknod /dev/urandom c 1 9; chmod 0444 /dev/urandom
-    ln -sf /proc/self/fd/0 /dev/stdin
-    ln -sf /proc/self/fd/1 /dev/stdout
-    ln -sf /proc/self/fd/2 /dev/stderr
-
-    modprobe virtio_pci
-    modprobe virtio_net
-  fi
+# Find udevd and run it directly.
+for f in /sbin/udevd /lib/udev/udevd \
+    /lib/systemd/systemd-udevd /usr/lib/systemd/systemd-udevd \
+    /usr/lib/udev/udevd; do
+  if [ -x "$f" ]; then UDEVD="$f"; fi
+done
+if [ -z "$UDEVD" ]; then
+  echo "udev not found!  Things will probably not work ..."
 fi
 
+$UDEVD --daemon
+udevadm trigger
+udevadm settle --timeout=600
+
 if grep -sq selinux=1 /proc/cmdline; then
   mount -t selinuxfs none /sys/fs/selinux
 fi
@@ -108,6 +84,8 @@ ip link set dev eth0 up
 
 ip route add default via 169.254.2.2
 
+echo nameserver 169.254.2.3 > /etc/resolv.conf
+
 # Scan for MDs.
 mdadm -As --auto=yes --run
 
@@ -115,8 +93,7 @@ mdadm -As --auto=yes --run
 modprobe dm_mod ||:
 lvmetad ||:
 
-lvm vgscan --ignorelockingfailure
-lvm vgchange -ay --ignorelockingfailure
+lvm vgchange -aay --sysinit
 
 # Scan for Windows dynamic disks.
 ldmtool create all
@@ -134,6 +111,8 @@ if grep -sq guestfs_verbose=1 /proc/cmdline; then
     lsmod
     #hwclock -r
     date
+    echo -n "clocksource: "
+    cat /sys/devices/system/clocksource/clocksource0/current_clocksource
     #ping -n -v -c 5 10.0.2.2
     #ping -n -v -c 5 10.0.2.4
 
@@ -141,6 +120,8 @@ if grep -sq guestfs_verbose=1 /proc/cmdline; then
 fi
 
 if ! grep -sq guestfs_rescue=1 /proc/cmdline; then
+  # Run the daemon.
+
   # Run the daemon under valgrind if ./configure --enable-valgrind-daemon
   if grep -sq guestfs_valgrind_daemon=1 /proc/cmdline; then
     if [ -r /etc/guestfsd.suppressions ]; then
@@ -152,14 +133,24 @@ if ! grep -sq guestfs_rescue=1 /proc/cmdline; then
 
   # Run guestfsd, under valgrind if asked.
   $vg guestfsd
-  if [ $? -eq 119 ]; then echo "DAEMON VALGRIND FAILED"; fi
+  if [ $? -eq 119 ]; then
+      echo "DAEMON VALGRIND FAILED"
+      # Sleep so valgrind messages are seen by the host.  Note this
+      # only happens in non-production builds
+      # (--enable-valgrind-daemon) + on an error path.
+      sleep 10
+  fi
 else
+  # Run virt-rescue shell.
+
   # Remove LD_PRELOAD=libSegFault set above.
   unset LD_PRELOAD
-  # Use appliance in rescue mode, also used by the virt-rescue command.
-  eval $(grep -Eo 'TERM=[^[:space:]]+' /proc/cmdline)
-  PS1='><rescue> '
-  export TERM PS1
+
+  :> $HOME/.bashrc
+  grep -Eo 'TERM=[^[:space:]]+' /proc/cmdline >> $HOME/.bashrc
+  echo "PS1='><rescue> '" >> $HOME/.bashrc
+  echo "export TERM PS1" >> $HOME/.bashrc
+
   echo
   echo "------------------------------------------------------------"
   echo

appliance/libguestfs-make-fixed-appliance.in

@@ -105,7 +105,7 @@ guestfish -a /dev/null run
 # Find the location of the appliance.
 cachedir="$(guestfish get-cachedir)"
 euid="$(id -u)"
-appliancedir="$cachedir/.guestfs-$euid"
+appliancedir="$cachedir/.guestfs-$euid/appliance.d"
 
 cp "$appliancedir/kernel" "$outputdir/kernel"
 cp "$appliancedir/initrd" "$outputdir/initrd"
@@ -128,8 +128,8 @@ Then copy all four files:
 
 into a directory somewhere, eg. /usr/local/lib/guestfs/appliance/
 
-Then build libguestfs (>= 1.16.7 or >= 1.18) from source, disabling
-the normal appliance and daemon:
+Then build libguestfs from source, disabling the normal appliance
+and daemon:
 
   ./configure --disable-appliance --disable-daemon
   make

appliance/libguestfs-make-fixed-appliance.pod

@@ -1,5 +1,3 @@
-=encoding utf8
-
 =head1 NAME
 
 libguestfs-make-fixed-appliance - Make a "fixed appliance" for libguestfs
@@ -25,7 +23,7 @@ Some reasons why you I<might> want to use this include:
 
 You want to make a self-contained libguestfs appliance that can be
 copied to another machine or platform that doesn't support
-L<supermin(8)>.
+L<supermin(1)>.
 
 =item *
 
@@ -129,8 +127,8 @@ looking for an appliance.  The path is built into libguestfs, or can
 be set using the C<LIBGUESTFS_PATH> environment variable.
 
 Normally a supermin appliance is located on this path (see
-L<supermin(8)/SUPERMIN APPLIANCE>).  libguestfs reconstructs this
-into a full appliance by running L<supermin-helper(8)>.
+L<supermin(1)/SUPERMIN APPLIANCE>).  libguestfs reconstructs this
+into a full appliance by running C<supermin --build>.
 
 However, a simpler "fixed appliance" can also be used.  libguestfs
 detects this by looking for a directory on the path containing four
@@ -166,8 +164,7 @@ libguestfs, please see the L<guestfs(3)> manual page.
 =head1 SEE ALSO
 
 L<guestfs(3)>,
-L<supermin(8)>,
-L<supermin-helper(8)>,
+L<supermin(1)>,
 L<xz(1)>,
 L<http://libguestfs.org/>,
 L<http://qemu.org/>.
@@ -178,4 +175,4 @@ Richard W.M. Jones (C<rjones at redhat dot com>)
 
 =head1 COPYRIGHT
 
-Copyright (C) 2009-2013 Red Hat Inc.
+Copyright (C) 2009-2014 Red Hat Inc.

appliance/make.sh.in

@@ -1,6 +1,6 @@
 #!/bin/bash -
 # @configure_input@
-# Copyright (C) 2009-2013 Red Hat Inc.
+# Copyright (C) 2009-2014 Red Hat Inc.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -20,48 +20,14 @@ unset CDPATH
 
 set -e
 
-# Turn excludelist file into command line arguments.
-exec 5<excludelist
-while read regexp <&5; do
-    excludes="$excludes --exclude $regexp"
-done
-exec 5<&-
+# Run supermin.
 
-# Run supermin on the package list.
-# NB: Keep using --yum-config (deprecated alias) here since both old
-# and new supermin still support it.
 if [ "x@SUPERMIN_PACKAGER_CONFIG@" != "xno" ]; then
-    extra="--yum-config @SUPERMIN_PACKAGER_CONFIG@"
+    extra="--packager-config @SUPERMIN_PACKAGER_CONFIG@"
 fi
 if [ "x@SUPERMIN_EXTRA_OPTIONS@" != "xno" ]; then
     extra="$extra @SUPERMIN_EXTRA_OPTIONS@"
 fi
 
-echo @SUPERMIN@ -v -o supermin.d --names $(< packagelist ) $excludes $extra
-@SUPERMIN@ -v -o supermin.d --names $(< packagelist ) $excludes $extra
-
-# Remove some things that we don't want in the appliance.  This is
-# copied from the old febootstrap-minimize.  However minimization is
-# not so important now that we are caching the appliance.
-< supermin.d/hostfiles \
-grep -v '^/usr/lib/locale' |
-grep -v '^/usr/share/locale' |
-grep -v '^/usr/share/man/' |
-grep -v '^/usr/share/doc/' |
-grep -v '^/usr/share/info/' |
-grep -v '^/usr/share/gnome/help/' |
-grep -v '^/usr/share/cracklib/' |
-grep -v '^/usr/share/i18n/' > supermin.d/hostfiles-t
-
-# XXX resolv.conf?  The old script had:
-# echo nameserver 169.254.2.3 > resolv.conf
-if ! grep -q /etc/resolv.conf supermin.d/hostfiles-t; then
-    echo /etc/resolv.conf >> supermin.d/hostfiles-t
-fi
-
-# Include any Augeas lenses from the host.
-if grep -q /usr/share/augeas/lenses supermin.d/hostfiles-t; then
-    echo "/usr/share/augeas/lenses/*.aug" >> supermin.d/hostfiles-t
-fi
-
-mv supermin.d/hostfiles-t supermin.d/hostfiles
+echo @SUPERMIN@ --prepare -v -o supermin.d $(< packagelist ) $extra
+@SUPERMIN@ --prepare -v -o supermin.d $(< packagelist ) $extra

appliance/packagelist.in

@@ -1,34 +1,32 @@
-/* This is the list of distro packages which are
- * installed on the appliance.
- *
- * This file is processed by cpp with one of the
- * following symbols defined (depending on the distro):
- *
- *   REDHAT=1     For Fedora, RHEL, EPEL and workalikes.
- *   DEBIAN=1     For Debian.
- *   UBUNTU=1     For Ubuntu.
- *   ARCHLINUX=1  For Archlinux.
- *
- * There is also a list of packages which are excluded if they appear
- * as dependencies of the packages below.  See: excludelist.in
- *
- * To add arbitrary extra packages, use:
- *
- *   ./configure --with-extra-packages="gdb valgrind [etc]"
- */
+dnl This is the list of distro packages which are
+dnl installed on the appliance.
+dnl
+dnl This file is processed by m4 with one of the
+dnl following symbols defined (depending on the distro):
+dnl
+dnl   REDHAT=1     For Fedora, RHEL, EPEL and workalikes.
+dnl   DEBIAN=1     For Debian.
+dnl   UBUNTU=1     For Ubuntu.
+dnl   MAGEIA=1     For Mageia
+dnl   ARCHLINUX=1  For Archlinux.
+dnl
+dnl There is also a list of packages which are excluded if they appear
+dnl as dependencies of the packages below.  See: excludelist.in
+dnl
+dnl To add arbitrary extra packages, use:
+dnl
+dnl   ./configure --with-extra-packages="gdb valgrind [etc]"
 
-/* Basically the same with a few minor tweaks. */
-#ifdef UBUNTU
-#define DEBIAN 1
-#endif
+dnl Basically the same with a few minor tweaks.
+ifelse(UBUNTU,1,`define(`DEBIAN',1)')
 
-#ifdef REDHAT
+ifelse(REDHAT,1,
   augeas-libs
   btrfs-progs
   cryptsetup
-  cryptsetup-luks /* old name used before Fedora 17 */
+  cryptsetup-luks      dnl old name used before Fedora 17
   e2fsprogs
-  /* e4fsprogs only exists on RHEL 5, will be ignored everywhere else. */
+  dnl e4fsprogs only exists on RHEL 5, will be ignored everywhere else.
   e4fsprogs
   genisoimage
   gfs-utils
@@ -40,8 +38,7 @@
   iputils
   kernel
   libcap
-  libldm /* only Fedora has this for now, but we should add it to others later*/
-  MAKEDEV
+  libldm               dnl only Fedora for now, others later
   nilfs-utils
   ntfsprogs
   ntfs-3g
@@ -50,46 +47,54 @@
   reiserfs-utils
   libselinux
   syslinux-extlinux
-  systemd /* for /sbin/reboot and udevd */
+  systemd              dnl for /sbin/reboot and udevd
   vim-minimal
   xz
   yajl
   zfs-fuse
-#endif /* REDHAT */
+)
 
-#ifdef DEBIAN
+ifelse(DEBIAN,1,
   bsdmainutils
   btrfs-tools
   cryptsetup
   e2fsprogs
   extlinux
   genisoimage
+dnl gfs-tools, gfs2-tools have been renamed to gfs2-utils
   gfs-tools
   gfs2-tools
-  grub-pc
+  gfs2-utils
+  grub2-common
   hfsplus
+dnl iproute has been renamed to iproute2
   iproute
+  iproute2
+  iputils-ping
+  iputils-arping
+  iputils-tracepath
   libaugeas0
   libcap2
   libhivex0
   libpcre3
+  libsystemd-id128-0
+  libsystemd-journal0
   libyajl2
   linux-image
-  /* syslinux 'suggests' mtools, but in reality it's a hard dependency: */
+  dnl syslinux 'suggests' mtools, but in reality it's a hard dependency:
   mtools
   nilfs-tools
   ntfs-3g
-  ntfsprogs
   openssh-client
   reiserfsprogs
-  sysvinit /* for /sbin/reboot */
+  sysvinit                  dnl for /sbin/reboot
   ufsutils
   vim-tiny
   xz-utils
   zfs-fuse
-#endif /* DEBIAN */
+)
 
-#ifdef ARCHLINUX
+ifelse(ARCHLINUX,1,
   augeas
   btrfs-progs
   cdrkit
@@ -112,7 +117,97 @@
   xz
   yajl
   zfs-fuse
-#endif /* ARCHLINUX */
+)
+
+ifelse(FRUGALWARE,1,
+  augeas
+  btrfs-progs
+  cryptsetup-luks
+  e2fsprogs
+  cdrkit
+  grub2
+  hfsplus
+  iproute2
+  iputils
+  kernel
+  libcap
+  ntfsprogs
+  ntfs-3g
+  openssh
+  pcre
+  reiserfsprogs
+  syslinux
+  systemd
+  vim
+  xz
+  yajl
+  xfsprogs-acl
+  xfsprogs-attr
+  bash
+  binutils
+  bzip2
+  coreutils
+  cpio
+  diffutils
+  dosfstools
+  file
+  findutils
+  gawk
+  gptfdisk
+  grep
+  gzip
+  jfsutils
+  kmod
+  less
+  libxml2
+  lsof
+  lsscsi
+  lvm2
+  mdadm
+  module-init-tools
+  parted
+  procps
+  psmisc
+  rsync
+  sed
+  strace
+  syslinux
+  tar
+  util-linux
+  xfsprogs
+)
+
+ifelse(MAGEIA,1,
+  augeas
+  btrfs-progs
+  cryptsetup
+  chkconfig /* for /etc/init.d */
+  e2fsprogs
+  cdrkit-genisoimage
+  extlinux
+  gfs2-utils
+  grub
+  hfsplus-tools
+  hivex
+  iproute2
+  iputils
+  libcap
+  nilfs-utils
+  ntfsprogs
+  ntfs-3g
+  openssh-clients
+  pcre
+  reiserfs-utils
+  libselinux
+  systemd /* for /sbin/reboot and udevd */
+  vim-minimal
+  xz
+  yajl
+  module-init-tools
+  procps-ng
+  udev
+  util-linux-ng
+)
 
 acl
 attr
@@ -139,11 +234,9 @@ lvm2
 lzop
 mdadm
 module-init-tools
-/*
-Enabling this pulls out 140 extra packages
-into the appliance:
-ocfs2-tools
-*/
+dnl Enabling this pulls out 140 extra packages
+dnl into the appliance:
+dnl ocfs2-tools
 parted
 procps
 procps-ng
@@ -157,17 +250,10 @@ tar
 udev
 util-linux
 util-linux-ng
-#ifndef UBUNTU
-/* on Ubuntu contains a file in /lib64 which conflicts with libc6 that has
- * /lib64 as a symbolic link
- */
 xfsprogs
-#endif
 zerofree
 
-#ifdef VALGRIND_DAEMON
-valgrind
-#endif
+ifelse(VALGRIND_DAEMON,1,valgrind)
 
-/* Define this by doing: ./configure --with-extra-packages="..." */
+dnl Define this by doing: ./configure --with-extra-packages="..."
 EXTRA_PACKAGES

bash/Makefile.am

@@ -28,6 +28,7 @@ scripts = \
 	virt-filesystems \
 	virt-format \
 	virt-inspector \
+	virt-log \
 	virt-ls \
 	virt-rescue \
 	virt-resize \
@@ -51,6 +52,8 @@ virt-format:
 	ln -sf virt-alignment-scan $@
 virt-inspector:
 	ln -sf virt-alignment-scan $@
+virt-log:
+	ln -sf virt-alignment-scan $@
 virt-ls:
 	ln -sf virt-alignment-scan $@
 virt-sysprep:

bash/guestfish

@@ -1,5 +1,5 @@
 # guestfish bash completion script -*- shell-script -*-
-# Copyright (C) 2010-2013 Red Hat Inc.
+# Copyright (C) 2010-2014 Red Hat Inc.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

bash/guestmount

@@ -1,5 +1,5 @@
 # guestmount bash completion script -*- shell-script -*-
-# Copyright (C) 2010-2013 Red Hat Inc.
+# Copyright (C) 2010-2014 Red Hat Inc.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

bash/virt-alignment-scan

@@ -1,5 +1,5 @@
 # virt-tools bash completion script -*- shell-script -*-
-# Copyright (C) 2010-2013 Red Hat Inc.
+# Copyright (C) 2010-2014 Red Hat Inc.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

bash/virt-rescue

@@ -1,5 +1,5 @@
 # virt-rescue bash completion script -*- shell-script -*-
-# Copyright (C) 2010-2013 Red Hat Inc.
+# Copyright (C) 2010-2014 Red Hat Inc.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

bash/virt-resize

@@ -1,6 +1,6 @@
 # virt-resize, virt-builder, virt-sparsify bash completion script
 # -*- shell-script -*-
-# Copyright (C) 2010-2013 Red Hat Inc.
+# Copyright (C) 2010-2014 Red Hat Inc.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

bugs-in-changelog.sh

@@ -1,6 +1,6 @@
 #!/bin/bash -
 # bugs-in-changelog.sh
-# Copyright (C) 2009-2013 Red Hat Inc.
+# Copyright (C) 2009-2014 Red Hat Inc.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by

builder/Makefile.am

@@ -17,64 +17,127 @@
 
 include $(top_srcdir)/subdir-rules.mk
 
+AM_YFLAGS = -d
+AM_CFLAGS = \
+	-I$(shell $(OCAMLC) -where) \
+	-I$(top_srcdir)/gnulib/lib \
+	-I$(top_srcdir)/src \
+	-I$(top_srcdir)/fish \
+	-pthread \
+	$(LIBLZMA_CFLAGS)
+
 EXTRA_DIST = \
 	$(SOURCES) \
-	virt-builder.pod \
+	libguestfs.gpg \
+	test-index \
 	test-virt-builder.sh \
 	test-virt-builder-list.sh \
-	website/.gitignore \
-	website/README \
-	website/index \
-	website/index.asc \
-	website/debian-7.preseed \
-	website/debian-7.sh \
-	website/debian-7.xz.sig \
-	website/fedora-18.ks \
-	website/fedora-18.sh \
-	website/fedora-18.xz.sig \
-	website/fedora-19.ks \
-	website/fedora-19.sh \
-	website/fedora-19.xz.sig
+	test-virt-builder-planner.sh \
+	test-virt-index-validate.sh \
+	test-virt-index-validate-bad-1 \
+	test-virt-index-validate-good-1 \
+	test-virt-index-validate-good-2 \
+	virt-builder.pod \
+	virt-index-validate.pod
 
 CLEANFILES = *~ *.cmi *.cmo *.cmx *.cmxa *.o virt-builder
 
 # Alphabetical order.
 SOURCES = \
+	architecture.ml \
 	builder.ml \
+	cache.mli \
+	cache.ml \
+	cmdline.ml \
 	downloader.mli \
 	downloader.ml \
 	get_kernel.mli \
 	get_kernel.ml \
 	index_parser.mli \
 	index_parser.ml \
+	index-parser-c.c \
+	ini_reader.mli \
+	ini_reader.ml \
+	languages.mli \
+	languages.ml \
 	list_entries.mli \
 	list_entries.ml \
-	perl_edit.ml \
-	perl_edit.mli \
+	paths.ml \
+	pxzcat.ml \
+	pxzcat.mli \
+	pxzcat-c.c \
+	setlocale.ml \
+	setlocale.mli \
+	setlocale-c.c \
 	sigchecker.mli \
-	sigchecker.ml
+	sigchecker.ml \
+	sources.mli \
+	sources.ml \
+	uname.ml \
+	uname.mli \
+	uname-c.c
+
+man_MANS =
+noinst_DATA =
 
 if HAVE_OCAML
 
 # Note this list must be in dependency order.
-OBJECTS = \
+deps = \
 	$(top_builddir)/mllib/libdir.cmx \
+	$(top_builddir)/mllib/config.cmx \
 	$(top_builddir)/mllib/common_gettext.cmx \
 	$(top_builddir)/mllib/common_utils.cmx \
-	$(top_builddir)/mllib/random_seed.cmx \
-	$(top_builddir)/mllib/hostname.cmx \
-	$(top_builddir)/mllib/firstboot.cmx \
-	$(top_builddir)/mllib/crypt-c.o \
-	$(top_builddir)/mllib/crypt.cmx \
-	$(top_builddir)/mllib/password.cmx \
+	$(top_builddir)/mllib/fsync-c.o \
+	$(top_builddir)/mllib/fsync.cmx \
+	$(top_builddir)/mllib/planner.cmx \
+	$(top_builddir)/mllib/regedit.cmx \
+	$(top_builddir)/mllib/uri-c.o \
+	$(top_builddir)/mllib/uRI.cmx \
+	$(top_builddir)/mllib/mkdtemp-c.o \
+	$(top_builddir)/mllib/mkdtemp.cmx \
+	$(top_builddir)/customize/urandom.cmx \
+	$(top_builddir)/customize/random_seed.cmx \
+	$(top_builddir)/customize/hostname.cmx \
+	$(top_builddir)/customize/timezone.cmx \
+	$(top_builddir)/customize/firstboot.cmx \
+	$(top_builddir)/customize/perl_edit.cmx \
+	$(top_builddir)/customize/crypt-c.o \
+	$(top_builddir)/customize/crypt.cmx \
+	$(top_builddir)/customize/password.cmx \
+	$(top_builddir)/customize/customize_cmdline.cmx \
+	$(top_builddir)/customize/customize_run.cmx \
+	$(top_builddir)/fish/guestfish-uri.o \
+	index-scan.o \
+	index-struct.o \
+	index-parse.o \
+	index-parser-c.o \
+	pxzcat-c.o \
+	pxzcat.cmx \
+	setlocale-c.o \
+	setlocale.cmx \
+	uname-c.o \
+	uname.cmx \
+	architecture.cmx \
+	ini_reader.cmx \
+	paths.cmx \
+	languages.cmx \
 	get_kernel.cmx \
+	cache.cmx \
 	downloader.cmx \
 	sigchecker.cmx \
 	index_parser.cmx \
 	list_entries.cmx \
-	perl_edit.cmx \
+	sources.cmx \
+	cmdline.cmx \
 	builder.cmx
 
+if HAVE_OCAMLOPT
+OBJECTS = $(deps)
+else
+OBJECTS = $(patsubst %.cmx,%.cmo,$(deps))
+endif
+
 bin_SCRIPTS = virt-builder
 
 # -I $(top_builddir)/src/.libs is a hack which forces corresponding -L
@@ -84,20 +147,37 @@ OCAMLPACKAGES = \
 	-package str,unix \
 	-I $(top_builddir)/src/.libs \
 	-I $(top_builddir)/ocaml \
-	-I $(top_builddir)/mllib
+	-I $(top_builddir)/mllib \
+	-I $(top_builddir)/customize
 if HAVE_OCAML_PKG_GETTEXT
 OCAMLPACKAGES += -package gettext-stub
 endif
 
-OCAMLCFLAGS = -g -warn-error CDEFLMPSUVYZX $(OCAMLPACKAGES)
+OCAMLCFLAGS = -g -warn-error CDEFLMPSUVYZX-3 $(OCAMLPACKAGES)
 OCAMLOPTFLAGS = $(OCAMLCFLAGS)
 
+OCAMLCLIBS  = \
+	$(LIBLZMA_LIBS) \
+	$(LIBXML2_LIBS) \
+	-L../src/.libs -lutils \
+	-L../gnulib/lib/.libs -lgnu \
+	-pthread -lpthread \
+	-lncurses -lcrypt
+
+if HAVE_OCAMLOPT
 virt-builder: $(OBJECTS)
 	$(OCAMLFIND) ocamlopt $(OCAMLOPTFLAGS) \
 	  mlguestfs.cmxa -linkpkg $^ \
-	  -cclib '-lncurses -lcrypt' \
-	  $(OCAML_GCOV_LDFLAGS) \
+	  -cclib '$(OCAMLCLIBS)' \
 	  -o $@
+else
+virt-builder: $(OBJECTS)
+	$(OCAMLFIND) ocamlc $(OCAMLCFLAGS) \
+	  mlguestfs.cma -linkpkg $^ \
+	  -cclib '$(OCAMLCLIBS)' \
+	  -custom \
+	  -o $@
+endif
 
 .mli.cmi:
 	$(OCAMLFIND) ocamlc $(OCAMLCFLAGS) -c $< -o $@
@@ -106,31 +186,19 @@ virt-builder: $(OBJECTS)
 .ml.cmx:
 	$(OCAMLFIND) ocamlopt $(OCAMLOPTFLAGS) -c $< -o $@
 
-# automake will decide we don't need C support in this file.  Really
-# we do, so we have to provide it ourselves.
-
-DEFAULT_INCLUDES = \
-	-I. \
-	-I$(top_builddir) \
-	-I$(shell $(OCAMLC) -where) \
-	-I$(top_srcdir)/src \
-	-I$(top_srcdir)/fish
-
-.c.o:
-	$(CC) $(CFLAGS) $(PROF_CFLAGS) $(DEFAULT_INCLUDES) -c $< -o $@
-
 # Manual pages and HTML files for the website.
 
-man_MANS = virt-builder.1
-
-noinst_DATA = $(top_builddir)/html/virt-builder.1.html
+man_MANS += virt-builder.1
+noinst_DATA += $(top_builddir)/html/virt-builder.1.html
 
 virt-builder.1 $(top_builddir)/html/virt-builder.1.html: stamp-virt-builder.pod
 
-stamp-virt-builder.pod: virt-builder.pod
+stamp-virt-builder.pod: virt-builder.pod $(top_srcdir)/customize/customize-synopsis.pod $(top_srcdir)/customize/customize-options.pod
 	$(PODWRAPPER) \
 	  --man virt-builder.1 \
 	  --html $(top_builddir)/html/virt-builder.1.html \
+	  --insert $(top_srcdir)/customize/customize-synopsis.pod:__CUSTOMIZE_SYNOPSIS__ \
+	  --insert $(top_srcdir)/customize/customize-options.pod:__CUSTOMIZE_OPTIONS__ \
 	  --license GPLv2+ \
 	  $<
 	touch $@
@@ -141,17 +209,32 @@ CLEANFILES += stamp-virt-builder.pod
 
 TESTS_ENVIRONMENT = $(top_builddir)/run --test
 
-all_disk_images := debian.xz fedora.xz ubuntu.xz windows.xz
-disk_images := $(shell for f in $(all_disk_images); do b=`basename $$f .xz`; if [ -f "../tests/guests/$$b.img" ]; then echo $$f; fi; done)
+disk_images := \
+	$(shell for f in debian fedora ubuntu windows; do if [ -s "../tests/guests/$$f.img" ]; then echo $$f.xz; fi; done) \
+	$(shell if [ -s "../tests/guests/fedora.img" ]; then echo fedora.qcow2 fedora.qcow2.xz; fi)
 
-CLEANFILES += $(all_disk_images)
+CLEANFILES += *.qcow2 *.xz
 
 check_DATA = $(disk_images)
 
-%.xz: ../tests/guests/%.img
-	xz --best --block-size=16777216 -c $< > $@
+fedora.qcow2: ../tests/guests/fedora.img
+	rm -f $@ $@-t
+	qemu-img convert -f raw -O qcow2 $< $@-t
+	mv $@-t $@
 
-TESTS = test-virt-builder-list.sh
+fedora.qcow2.xz: fedora.qcow2
+	rm -f $@ $@-t
+	xz --best --block-size=16777216 -c $< > $@-t
+	mv $@-t $@
+
+%.xz: ../tests/guests/%.img
+	rm -f $@ $@-t
+	xz --best --block-size=16777216 -c $< > $@-t
+	mv $@-t $@
+
+TESTS = \
+	test-virt-builder-list.sh \
+	test-virt-index-validate.sh
 
 if ENABLE_APPLIANCE
 TESTS += test-virt-builder.sh
@@ -160,12 +243,15 @@ endif ENABLE_APPLIANCE
 check-valgrind:
 	$(MAKE) VG="$(top_builddir)/run @VG@" check
 
+check-slow:
+	$(MAKE) TESTS="test-virt-builder-planner.sh" check
+
 # Dependencies.
 depend: .depend
 
 .depend: $(wildcard $(abs_srcdir)/*.mli) $(wildcard $(abs_srcdir)/*.ml)
 	rm -f $@ $@-t
-	$(OCAMLFIND) ocamldep -I ../ocaml -I $(abs_srcdir) -I $(top_srcdir)/mllib $^ | \
+	$(OCAMLFIND) ocamldep -I ../ocaml -I $(abs_srcdir) -I $(abs_top_builddir)/mllib -I $(abs_top_builddir)/customize $^ | \
 	  $(SED) 's/ *$$//' | \
 	  $(SED) -e :a -e '/ *\\$$/N; s/ *\\\n */ /; ta' | \
 	  $(SED) -e 's,$(abs_srcdir)/,$(builddir)/,g' | \
@@ -179,3 +265,55 @@ endif
 DISTCLEANFILES = .depend
 
 .PHONY: depend docs
+
+# virt-builder's default repository
+
+repoconfdir = $(sysconfdir)/xdg/virt-builder/repos.d
+repoconf_DATA = libguestfs.conf libguestfs.gpg
+
+install-exec-hook:
+	$(LN_S) xdg/virt-builder $(DESTDIR)$(sysconfdir)/virt-builder
+
+# Build a small C index validator program.
+bin_PROGRAMS = virt-index-validate
+
+virt_index_validate_SOURCES = \
+	index-parse.y \
+	index-scan.l \
+	index-struct.h \
+	index-struct.c \
+	index-validate.c
+
+virt_index_validate_CPPFLAGS = \
+	-DLOCALEBASEDIR=\""$(datadir)/locale"\"
+
+virt_index_validate_LDADD = \
+	../gnulib/lib/libgnu.la
+
+man_MANS += virt-index-validate.1
+noinst_DATA += $(top_builddir)/html/virt-index-validate.1.html
+
+virt-index-validate.1 $(top_builddir)/html/virt-index-validate.1.html: stamp-virt-index-validate.pod
+
+stamp-virt-index-validate.pod: virt-index-validate.pod
+	$(PODWRAPPER) \
+	  --man virt-index-validate.1 \
+	  --html $(top_builddir)/html/virt-index-validate.1.html \
+	  --license GPLv2+ \
+	  $<
+	touch $@
+
+CLEANFILES += \
+	index-parse.c \
+	index-parse.h \
+	index-scan.c \
+	stamp-virt-index-validate.pod
+
+if HAVE_OCAML
+# Automake-generated makefile has a rule ".l.c:" but lacks a rule ".l.h:".
+# Also it doesn't generate dependencies for the C files that include
+# index-parse.h.
+index-parser-c.c index-scan.c index-validate.c: index-parse.h
+index-parse.h: index-parse.y
+	$(MAKE) index-parse.h
+endif

builder/architecture.ml

@@ -0,0 +1,41 @@
+(* virt-builder
+ * Copyright (C) 2014 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+open Common_gettext.Gettext
+open Common_utils
+
+open Unix
+
+let filter_arch = function
+  | "amd64" | "x86_64" | "x64" -> "x86_64"
+  | "powerpc" | "ppc" -> "ppc"
+  | arch -> arch
+
+let arch_is_compatible nativearch otherarch =
+  let nativearch = filter_arch nativearch in
+  let otherarch = filter_arch otherarch in
+  match nativearch, otherarch with
+  | a, b when a = b -> true
+  | "x86_64", "i386" -> true
+  | "ppc64", "ppc" -> true
+  | "sparc64", "sparc" -> true
+  | a, b -> false
+
+let current_arch =
+  try filter_arch ((Uname.uname ()).Uname.machine)
+  with Unix_error _ -> "unknown"

builder/cache.ml

@@ -0,0 +1,60 @@
+(* virt-builder
+ * Copyright (C) 2013-2014 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+open Common_gettext.Gettext
+open Common_utils
+
+open Unix
+open Printf
+
+let quote = Filename.quote
+
+let clean_cachedir dir =
+  let cmd = sprintf "rm -rf %s" (quote dir) in
+  ignore (Sys.command cmd);
+
+type t = {
+  verbose : bool;
+  directory : string;
+}
+
+let create ~verbose ~directory =
+  if not (is_directory directory) then
+    mkdir directory 0o755;
+  {
+    verbose = verbose;
+    directory = directory;
+  }
+
+let cache_of_name t name arch revision =
+  t.directory // sprintf "%s.%s.%d" name arch revision
+
+let is_cached t name arch revision =
+  let filename = cache_of_name t name arch revision in
+  Sys.file_exists filename
+
+let print_item_status t ~header l =
+  if header then (
+    printf (f_"cache directory: %s\n") t.directory
+  );
+  List.iter (
+    fun (name, arch, revision) ->
+      let cached = is_cached t name arch revision in
+      printf "%-24s %-10s %s\n" name arch
+        (if cached then s_"cached" else (*s_*)"no")
+  ) l

builder/cache.mli

@@ -0,0 +1,45 @@
+(* virt-builder
+ * Copyright (C) 2013-2014 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+(** This module represents a local cache. *)
+
+val clean_cachedir : string -> unit
+(** [clean_cachedir dir] clean the specified cache directory. *)
+
+type t
+(** The abstract data type. *)
+
+val create : verbose:bool -> directory:string -> t
+(** Create the abstract type. *)
+
+val cache_of_name : t -> string -> string -> int -> string
+(** [cache_of_name t name arch revision] return the filename
+    of the cached file.  (Note: It doesn't check if the filename
+    exists, this is just a simple string transformation). *)
+
+val is_cached : t -> string -> string -> int -> bool
+(** [is_cached t name arch revision] return whether the file with
+    specified name, architecture and revision is cached. *)
+
+val print_item_status : t -> header:bool -> (string * string * int) list -> unit
+(** [print_item_status t header items] print the status in the cache
+    of the specified items (which are tuples of name, architecture,
+    and revision).
+
+    If [~header:true] then display a header with the path of the
+    cache. *)

builder/cmdline.ml

@@ -0,0 +1,348 @@
+(* virt-builder
+ * Copyright (C) 2013 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+(* Command line argument parsing. *)
+
+open Common_gettext.Gettext
+open Common_utils
+
+open Customize_cmdline
+
+module G = Guestfs
+
+open Unix
+open Printf
+
+let prog = Filename.basename Sys.executable_name
+
+let parse_cmdline () =
+  let display_version () =
+    printf "virt-builder %s\n" Config.package_version;
+    exit 0
+  in
+
+  let mode = ref `Install in
+  let list_mode () = mode := `List in
+  let notes_mode () = mode := `Notes in
+  let get_kernel_mode () = mode := `Get_kernel in
+  let cache_all_mode () = mode := `Cache_all in
+  let print_cache_mode () = mode := `Print_cache in
+  let delete_cache_mode () = mode := `Delete_cache in
+
+  let arch = ref "" in
+
+  let attach = ref [] in
+  let attach_format = ref None in
+  let set_attach_format = function
+    | "auto" -> attach_format := None
+    | s -> attach_format := Some s
+  in
+  let attach_disk s = attach := (!attach_format, s) :: !attach in
+
+  let cache = ref Paths.xdg_cache_home in
+  let set_cache arg = cache := Some arg in
+  let no_cache () = cache := None in
+
+  let check_signature = ref true in
+  let curl = ref "curl" in
+
+  let delete_on_failure = ref true in
+
+  let fingerprints = ref [] in
+  let add_fingerprint arg = fingerprints := arg :: !fingerprints in
+
+  let format = ref "" in
+  let gpg = ref "gpg" in
+
+  let list_format = ref `Short in
+  let list_set_long () = list_format := `Long in
+  let list_set_format arg =
+    list_format := match arg with
+    | "short" -> `Short
+    | "long" -> `Long
+    | "json" -> `Json
+    | fmt ->
+      eprintf (f_"%s: invalid --list-format type '%s', see the man page.\n") prog fmt;
+      exit 1 in
+
+  let machine_readable = ref false in
+
+  let memsize = ref None in
+  let set_memsize arg = memsize := Some arg in
+
+  let network = ref true in
+  let output = ref "" in
+
+  let quiet = ref false in
+
+  let size = ref None in
+  let set_size arg = size := Some (parse_size ~prog arg) in
+
+  let smp = ref None in
+  let set_smp arg = smp := Some arg in
+
+  let sources = ref [] in
+  let add_source arg = sources := arg :: !sources in
+
+  let sync = ref true in
+  let trace = ref false in
+  let verbose = ref false in
+
+  let argspec = [
+    "--arch",    Arg.Set_string arch,       "arch" ^ " " ^ s_"Set the output architecture";
+    "--attach",  Arg.String attach_disk,    "iso" ^ " " ^ s_"Attach data disk/ISO during install";
+    "--attach-format",  Arg.String set_attach_format,
+                                            "format" ^ " " ^ s_"Set attach disk format";
+    "--cache",   Arg.String set_cache,      "dir" ^ " " ^ s_"Set template cache dir";
+    "--no-cache", Arg.Unit no_cache,        " " ^ s_"Disable template cache";
+    "--cache-all-templates", Arg.Unit cache_all_mode,
+                                            " " ^ s_"Download all templates to the cache";
+    "--check-signature", Arg.Set check_signature,
+                                            " " ^ s_"Check digital signatures";
+    "--check-signatures", Arg.Set check_signature,
+                                            " " ^ s_"Check digital signatures";
+    "--no-check-signature", Arg.Clear check_signature,
+                                            " " ^ s_"Disable digital signatures";
+    "--no-check-signatures", Arg.Clear check_signature,
+                                            " " ^ s_"Disable digital signatures";
+    "--curl",    Arg.Set_string curl,       "curl" ^ " " ^ s_"Set curl binary/command";
+    "--delete-cache", Arg.Unit delete_cache_mode,
+                                            " " ^ s_"Delete the template cache";
+    "--no-delete-on-failure", Arg.Clear delete_on_failure,
+                                            " " ^ s_"Don't delete output file on failure";
+    "--fingerprint", Arg.String add_fingerprint,
+                                            "AAAA.." ^ " " ^ s_"Fingerprint of valid signing key";
+    "--format",  Arg.Set_string format,     "raw|qcow2" ^ " " ^ s_"Output format (default: raw)";
+    "--get-kernel", Arg.Unit get_kernel_mode,
+                                            "image" ^ " " ^ s_"Get kernel from image";
+    "--gpg",    Arg.Set_string gpg,         "gpg" ^ " " ^ s_"Set GPG binary/command";
+    "-l",        Arg.Unit list_mode,        " " ^ s_"List available templates";
+    "--list",    Arg.Unit list_mode,        " " ^ s_"List available templates";
+    "--long",    Arg.Unit list_set_long,    " " ^ s_"Shortcut for --list-format short";
+    "--list-format", Arg.String list_set_format,
+                                            "short|long|json" ^ " " ^ s_"Set the format for --list (default: short)";
+    "--long-options", Arg.Unit display_long_options, " " ^ s_"List long options";
+    "--machine-readable", Arg.Set machine_readable, " " ^ s_"Make output machine readable";
+    "-m",        Arg.Int set_memsize,       "mb" ^ " " ^ s_"Set memory size";
+    "--memsize", Arg.Int set_memsize,       "mb" ^ " " ^ s_"Set memory size";
+    "--network", Arg.Set network,           " " ^ s_"Enable appliance network (default)";
+    "--no-network", Arg.Clear network,      " " ^ s_"Disable appliance network";
+    "--notes",   Arg.Unit notes_mode,       " " ^ s_"Display installation notes";
+    "-o",        Arg.Set_string output,     "file" ^ " " ^ s_"Set output filename";
+    "--output",  Arg.Set_string output,     "file" ^ " " ^ s_"Set output filename";
+    "--print-cache", Arg.Unit print_cache_mode,
+                                            " " ^ s_"Print info about template cache";
+    "--quiet",   Arg.Set quiet,             " " ^ s_"No progress messages";
+    "--size",    Arg.String set_size,       "size" ^ " " ^ s_"Set output disk size";
+    "--smp",     Arg.Int set_smp,           "vcpus" ^ " " ^ s_"Set number of vCPUs";
+    "--source",  Arg.String add_source,     "URL" ^ " " ^ s_"Set source URL";
+    "--no-sync", Arg.Clear sync,            " " ^ s_"Do not fsync output file on exit";
+    "-v",        Arg.Set verbose,           " " ^ s_"Enable debugging messages";
+    "--verbose", Arg.Set verbose,           " " ^ s_"Enable debugging messages";
+    "-V",        Arg.Unit display_version,  " " ^ s_"Display version and exit";
+    "--version", Arg.Unit display_version,  " " ^ s_"Display version and exit";
+    "-x",        Arg.Set trace,             " " ^ s_"Enable tracing of libguestfs calls";
+  ] in
+  let customize_argspec, get_customize_ops =
+    Customize_cmdline.argspec ~prog () in
+  let customize_argspec =
+    List.map (fun (spec, _, _) -> spec) customize_argspec in
+  let argspec = argspec @ customize_argspec in
+  let argspec =
+    let cmp (arg1, _, _) (arg2, _, _) =
+      let arg1 = skip_dashes arg1 and arg2 = skip_dashes arg2 in
+      compare (String.lowercase arg1) (String.lowercase arg2)
+    in
+    List.sort cmp argspec in
+  let argspec = Arg.align argspec in
+  long_options := argspec;
+
+  let args = ref [] in
+  let anon_fun s = args := s :: !args in
+  let usage_msg =
+    sprintf (f_"\
+%s: build virtual machine images quickly
+
+ virt-builder OS-VERSION
+ virt-builder -l
+ virt-builder --notes OS-VERSION
+ virt-builder --print-cache
+ virt-builder --cache-all-templates
+ virt-builder --delete-cache
+ virt-builder --get-kernel IMAGE
+
+A short summary of the options is given below.  For detailed help please
+read the man page virt-builder(1).
+")
+      prog in
+  Arg.parse argspec anon_fun usage_msg;
+
+  (* Dereference options. *)
+  let args = List.rev !args in
+  let mode = !mode in
+  let arch = !arch in
+  let attach = List.rev !attach in
+  let cache = !cache in
+  let check_signature = !check_signature in
+  let curl = !curl in
+  let delete_on_failure = !delete_on_failure in
+  let fingerprints = List.rev !fingerprints in
+  let format = match !format with "" -> None | s -> Some s in
+  let gpg = !gpg in
+  let list_format = !list_format in
+  let machine_readable = !machine_readable in
+  let memsize = !memsize in
+  let network = !network in
+  let ops = get_customize_ops () in
+  let output = match !output with "" -> None | s -> Some s in
+  let quiet = !quiet in
+  let size = !size in
+  let smp = !smp in
+  let sources = List.rev !sources in
+  let sync = !sync in
+  let trace = !trace in
+  let verbose = !verbose in
+
+  (* No arguments and machine-readable mode?  Print some facts. *)
+  if args = [] && machine_readable then (
+    printf "virt-builder\n";
+    printf "arch\n";
+    printf "config-file\n";
+    printf "customize\n";
+    printf "json-list\n";
+    if Pxzcat.using_parallel_xzcat () then printf "pxzcat\n";
+    exit 0
+  );
+
+  (* Check options. *)
+  let arg =
+    match mode with
+    | `Install ->
+      (match args with
+      | [arg] -> arg
+      | [] ->
+        eprintf (f_"%s: virt-builder os-version\nMissing 'os-version'. Use '--list' to list available template names.\n") prog;
+        exit 1
+      | _ ->
+        eprintf (f_"%s: virt-builder: too many parameters, expecting 'os-version'\n") prog;
+        exit 1
+      )
+    | `List ->
+      (match args with
+      | [] -> ""
+      | _ ->
+        eprintf (f_"%s: virt-builder --list does not need any extra arguments.\n") prog;
+        exit 1
+      )
+    | `Notes ->
+      (match args with
+      | [arg] -> arg
+      | [] ->
+        eprintf (f_"%s: virt-builder --notes os-version\nMissing 'os-version'. Use '--list' to list available template names.\n") prog;
+        exit 1
+      | _ ->
+        eprintf (f_"%s: virt-builder: too many parameters, expecting 'os-version'\n") prog;
+        exit 1
+      )
+    | `Cache_all
+    | `Print_cache
+    | `Delete_cache ->
+      (match args with
+      | [] -> ""
+      | _ ->
+        eprintf (f_"%s: virt-builder --cache-all-templates/--print-cache/--delete-cache does not need any extra arguments.\n") prog;
+        exit 1
+      )
+    | `Get_kernel ->
+      (match args with
+      | [arg] -> arg
+      | [] ->
+        eprintf (f_"%s: virt-builder --get-kernel image\nMissing 'image' (disk image file) argument.\n") prog;
+        exit 1
+      | _ ->
+        eprintf (f_"%s: virt-builder --get-kernel: too many parameters\n") prog;
+        exit 1
+      ) in
+
+  (* Check source(s) and fingerprint(s). *)
+  let sources =
+    let rec repeat x = function
+      | 0 -> [] | 1 -> [x]
+      | n -> x :: repeat x (n-1)
+    in
+
+    let nr_sources = List.length sources in
+    let fingerprints =
+      match fingerprints with
+      | [fingerprint] ->
+        (* You're allowed to have multiple sources and one fingerprint: it
+         * means that the same fingerprint is used for all sources.
+         *)
+        repeat fingerprint nr_sources
+      | xs -> xs in
+
+    if List.length fingerprints <> nr_sources then (
+      eprintf (f_"%s: source and fingerprint lists are not the same length\n")
+        prog;
+      exit 1
+    );
+
+    (* Combine the sources and fingerprints into a single list of pairs. *)
+    List.combine sources fingerprints in
+
+  (* Check the architecture. *)
+  let arch =
+    match arch with
+    | "" -> Architecture.current_arch
+    | arch ->
+      let target_arch = Architecture.filter_arch arch in
+      if Architecture.arch_is_compatible Architecture.current_arch target_arch <> true then (
+        let requires_execute_on_guest = List.exists (
+          function
+          | `Command _ | `InstallPackages _ | `Script _ | `Update -> true
+          | `Delete _ | `Edit _ | `FirstbootCommand _ | `FirstbootPackages _
+          | `FirstbootScript _ | `Hostname _ | `Link _ | `Mkdir _
+          | `Password _ | `RootPassword _ | `Scrub _ | `Timezone _ | `Upload _
+          | `Write _ | `Chmod _ -> false
+        ) ops.ops in
+        if requires_execute_on_guest then (
+          eprintf (f_"%s: sorry, cannot run commands on a guest with a different architecture\n")
+            prog;
+          exit 1
+        );
+      );
+      target_arch in
+
+  (* If user didn't elect any root password, that means we set a random
+   * root password.
+   *)
+  let ops =
+    let has_set_root_password = List.exists (
+      function `RootPassword _ -> true | _ -> false
+    ) ops.ops in
+    if has_set_root_password then ops
+    else (
+      let pw = Password.parse_selector ~prog "random" in
+      { ops with ops = ops.ops @ [ `RootPassword pw ] }
+    ) in
+
+  mode, arg,
+  arch, attach, cache, check_signature, curl,
+  delete_on_failure, format, gpg, list_format, memsize,
+  network, ops, output, quiet, size, smp, sources, sync,
+  trace, verbose

builder/downloader.ml

@@ -23,105 +23,141 @@ open Unix
 open Printf
 
 let quote = Filename.quote
-let (//) = Filename.concat
-
-let cache_of_name cachedir name revision =
-  cachedir // sprintf "%s.%d" name revision
 
 type uri = string
 type filename = string
 
 type t = {
-  debug : bool;
+  verbose : bool;
   curl : string;
-  cache : string option;                (* cache directory for templates *)
+  cache : Cache.t option;               (* cache for templates *)
 }
 
-let create ~debug ~curl ~cache = {
-  debug = debug;
+type proxy_mode =
+  | UnsetProxy
+  | SystemProxy
+  | ForcedProxy of string
+
+let create ~verbose ~curl ~cache = {
+  verbose = verbose;
   curl = curl;
   cache = cache;
 }
 
-let rec download t ?template ?progress_bar uri =
+let rec download ~prog t ?template ?progress_bar ?(proxy = SystemProxy) uri =
   match template with
   | None ->                       (* no cache, simple download *)
     (* Create a temporary name. *)
     let tmpfile = Filename.temp_file "vbcache" ".txt" in
-    download_to t ?progress_bar uri tmpfile;
+    download_to ~prog t ?progress_bar ~proxy uri tmpfile;
     (tmpfile, true)
 
-  | Some (name, revision) ->
+  | Some (name, arch, revision) ->
     match t.cache with
     | None ->
       (* Not using the cache at all? *)
-      download t ?progress_bar uri
+      download t ~prog ?progress_bar ~proxy uri
 
-    | Some cachedir ->
-      let filename = cache_of_name cachedir name revision in
+    | Some cache ->
+      let filename = Cache.cache_of_name cache name arch revision in
 
       (* Is the requested template name + revision in the cache already?
        * If not, download it.
        *)
       if not (Sys.file_exists filename) then
-        download_to t ?progress_bar uri filename;
+        download_to ~prog t ?progress_bar ~proxy uri filename;
 
       (filename, false)
 
-and download_to t ?(progress_bar = false) uri filename =
-  (* Get the status code first to ensure the file exists. *)
-  let cmd = sprintf "%s%s -g -o /dev/null -I -w '%%{http_code}' %s"
-    t.curl
-    (if t.debug then "" else " -s -S")
-    (quote uri) in
-  let chan = open_process_in cmd in
-  let status_code = input_line chan in
-  let stat = close_process_in chan in
-  (match stat with
-  | WEXITED 0 -> ()
-  | WEXITED i ->
-    eprintf (f_"virt-builder: curl (download) command failed downloading '%s'\n") uri;
-    exit 1
-  | WSIGNALED i ->
-    eprintf (f_"virt-builder: external command '%s' killed by signal %d\n")
-      cmd i;
-    exit 1
-  | WSTOPPED i ->
-    eprintf (f_"virt-builder: external command '%s' stopped by signal %d\n")
-      cmd i;
-    exit 1
-  );
-  let bad_status_code = function
-    | "" -> true
-    | s when s.[0] = '4' -> true (* 4xx *)
-    | s when s.[0] = '5' -> true (* 5xx *)
-    | _ -> false
-  in
-  if bad_status_code status_code then (
-    eprintf (f_"virt-builder: failed to download %s: HTTP status code %s\n")
-      uri status_code;
-    exit 1
-  );
+and download_to ~prog t ?(progress_bar = false) ~proxy uri filename =
+  let parseduri =
+    try URI.parse_uri uri
+    with Invalid_argument "URI.parse_uri" ->
+      eprintf (f_"Error parsing URI '%s'. Look for error messages printed above.\n") uri;
+      exit 1 in
 
-  (* Now download the file.
-   * 
-   * Note because there may be parallel virt-builder instances running
-   * and also to avoid partial downloads in the cachedir if the network
+  (* Note because there may be parallel virt-builder instances running
+   * and also to avoid partial downloads in the cache if the network
    * fails, we download to a random name in the cache and then
    * atomically rename it to the final filename.
    *)
   let filename_new = filename ^ "." ^ string_random8 () in
   unlink_on_exit filename_new;
-  let cmd = sprintf "%s%s -g -o %s %s"
-    t.curl
-    (if t.debug then "" else if progress_bar then " -#" else " -s -S")
-    (quote filename_new) (quote uri) in
-  if t.debug then eprintf "%s\n%!" cmd;
-  let r = Sys.command cmd in
-  if r <> 0 then (
-    eprintf (f_"virt-builder: curl (download) command failed downloading '%s'\n") uri;
-    exit 1
+
+  (match parseduri.URI.protocol with
+  | "file" ->
+    let path = parseduri.URI.path in
+    let cmd = sprintf "cp%s %s %s"
+      (if t.verbose then " -v" else "")
+      (quote path) (quote filename_new) in
+    let r = Sys.command cmd in
+    if r <> 0 then (
+      eprintf (f_"%s: cp (download) command failed copying '%s'\n")
+        prog path;
+      exit 1
+    )
+  | _ as protocol -> (* Any other protocol. *)
+    let outenv = proxy_envvar protocol proxy in
+    (* Get the status code first to ensure the file exists. *)
+    let cmd = sprintf "%s%s%s -g -o /dev/null -I -w '%%{http_code}' %s"
+      outenv
+      t.curl
+      (if t.verbose then "" else " -s -S")
+      (quote uri) in
+    if t.verbose then eprintf "%s\n%!" cmd;
+    let lines = external_command ~prog cmd in
+    if List.length lines < 1 then (
+      eprintf (f_"%s: unexpected output from curl command, enable debug and look at previous messages\n")
+        prog;
+      exit 1
+    );
+    let status_code = List.hd lines in
+    let bad_status_code = function
+      | "" -> true
+      | s when s.[0] = '4' -> true (* 4xx *)
+      | s when s.[0] = '5' -> true (* 5xx *)
+      | _ -> false
+    in
+    if bad_status_code status_code then (
+      eprintf (f_"%s: failed to download %s: HTTP status code %s\n")
+        prog uri status_code;
+      exit 1
+    );
+
+    (* Now download the file. *)
+    let cmd = sprintf "%s%s%s -g -o %s %s"
+      outenv
+      t.curl
+      (if t.verbose then "" else if progress_bar then " -#" else " -s -S")
+      (quote filename_new) (quote uri) in
+    if t.verbose then eprintf "%s\n%!" cmd;
+    let r = Sys.command cmd in
+    if r <> 0 then (
+      eprintf (f_"%s: curl (download) command failed downloading '%s'\n")
+        prog uri;
+      exit 1
+    )
   );
 
-  (* Rename the file if curl was successful. *)
+  (* Rename the file if the download was successful. *)
   rename filename_new filename
+
+and proxy_envvar protocol = function
+  | UnsetProxy ->
+    (match protocol with
+    | "http" -> "env http_proxy= no_proxy=* "
+    | "https" -> "env https_proxy= no_proxy=* "
+    | "ftp" -> "env ftp_proxy= no_proxy=* "
+    | _ -> "env no_proxy=* "
+    )
+  | SystemProxy ->
+    (* No changes required. *)
+    ""
+  | ForcedProxy proxy ->
+    let proxy = Filename.quote proxy in
+    (match protocol with
+    | "http" -> sprintf "env http_proxy=%s no_proxy= " proxy
+    | "https" -> sprintf "env https_proxy=%s no_proxy= " proxy
+    | "ftp" -> sprintf "env ftp_proxy=%s no_proxy= " proxy
+    | _ -> ""
+    )

builder/downloader.mli

@@ -18,30 +18,36 @@
 
 (** This module is a wrapper around curl, plus local caching. *)
 
-val cache_of_name : string -> string -> int -> string
-(** [cache_of_name cachedir name revision] returns the filename
-    of the cached file.  (Note: It doesn't check if the filename
-    exists, this is just a simple string transformation). *)
-
 type uri = string
 type filename = string
 
 type t
 (** The abstract data type. *)
 
-val create : debug:bool -> curl:string -> cache:string option -> t
+(** Type of proxy. *)
+type proxy_mode =
+  | UnsetProxy                 (* The proxy is forced off. *)
+  | SystemProxy                (* The proxy is not changed (follows the
+                                * system configuration).
+                                *)
+  | ForcedProxy of string      (* The proxy is forced to the specified URL. *)
+
+val create : verbose:bool -> curl:string -> cache:Cache.t option -> t
 (** Create the abstract type. *)
 
-val download : t -> ?template:(string*int) -> ?progress_bar:bool -> uri -> (filename * bool)
+val download : prog:string -> t -> ?template:(string*string*int) -> ?progress_bar:bool -> ?proxy:proxy_mode -> uri -> (filename * bool)
 (** Download the URI, returning the downloaded filename and a
     temporary file flag.  The temporary file flag is [true] iff
     the downloaded file is temporary and should be deleted by the
     caller (otherwise it's in the cache and you shouldn't delete it).
 
-    For templates, you must supply [~template:(name, revision)].  This
-    causes the cache to be used (if possible).  Name and revision are
-    used for cache control (see the man page for details).
+    For templates, you must supply [~template:(name, arch, revision)].
+    This causes the cache to be used (if possible).  Name, arch(itecture)
+    and revision are used for cache control (see the man page for details).
 
     If [~progress_bar:true] then display a progress bar if the file
-    doesn't come from the cache.  In debug mode, progress messages
-    are always displayed. *)
+    doesn't come from the cache.  In verbose mode, progress messages
+    are always displayed.
+
+    [proxy] specifies the type of proxy to be used in the transfer,
+    if possible. *)

builder/get_kernel.ml

@@ -23,15 +23,13 @@ module G = Guestfs
 
 open Printf
 
-let rex_numbers = Str.regexp "^\\([0-9]+\\)\\(.*\\)$"
-let rex_letters = Str.regexp_case_fold "^\\([a-z]+\\)\\(.*\\)$"
-
 (* Originally:
  * http://rwmj.wordpress.com/2013/09/13/get-kernel-and-initramfs-from-a-disk-image/
  *)
-let rec get_kernel ~debug ?format ?output disk =
+let rec get_kernel ~trace ~verbose ?format ?output disk =
   let g = new G.guestfs () in
-  if debug then g#set_trace true;
+  if trace then g#set_trace true;
+  if verbose then g#set_verbose true;
   g#add_drive_opts ?format ~readonly:true disk;
   g#launch ();
 
@@ -41,7 +39,7 @@ let rec get_kernel ~debug ?format ?output disk =
     exit 1
   );
   if Array.length roots > 1 then (
-    eprintf (f_"virt-builder: get-kernel: daual/mult-boot images are not supported by this tool\n");
+    eprintf (f_"virt-builder: get-kernel: dual/multi-boot images are not supported by this tool\n");
     exit 1
   );
   let root = roots.(0) in
@@ -96,26 +94,3 @@ let rec get_kernel ~debug ?format ?output disk =
   (* Shutdown. *)
   g#shutdown ();
   g#close ()
-
-and compare_version v1 v2 =
-  compare (split_version v1) (split_version v2)
-
-and split_version = function
-  | "" -> []
-  | str ->
-    let first, rest =
-      if Str.string_match rex_numbers str 0 then (
-        let n = Str.matched_group 1 str in
-        let rest = Str.matched_group 2 str in
-        let n =
-          try `Number (int_of_string n)
-          with Failure "int_of_string" -> `String n in
-        n, rest
-      )
-      else if Str.string_match rex_letters str 0 then
-        `String (Str.matched_group 1 str), Str.matched_group 2 str
-      else (
-        let len = String.length str in
-        `Char str.[0], String.sub str 1 (len-1)
-      ) in
-    first :: split_version rest

builder/get_kernel.mli

@@ -16,4 +16,4 @@
  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  *)
 
-val get_kernel : debug:bool -> ?format:string -> ?output:string -> string -> unit
+val get_kernel : trace:bool -> verbose:bool -> ?format:string -> ?output:string -> string -> unit

builder/index-parse.y

@@ -0,0 +1,176 @@
+/* libguestfs virt-builder tool -*- fundamental -*-
+ * Copyright (C) 2013 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+%{
+#include <config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "index-struct.h"
+#include "index-parse.h"
+
+#define YY_EXTRA_TYPE struct parse_context *
+
+extern void yyerror (YYLTYPE * yylloc, yyscan_t scanner, struct parse_context *context, const char *msg);
+extern int yylex (YYSTYPE * yylval, YYLTYPE * yylloc, yyscan_t scanner);
+extern void scanner_init (yyscan_t *scanner, struct parse_context *context, FILE *in);
+extern void scanner_destroy (yyscan_t scanner);
+
+/* Join two strings with \n */
+static char *
+concat_newline (const char *str1, const char *str2)
+{
+  size_t len1, len2, len;
+  char *ret;
+
+  if (str2 == NULL)
+    return strdup (str1);
+
+  len1 = strlen (str1);
+  len2 = strlen (str2);
+  len = len1 + 1 /* \n */ + len2 + 1 /* \0 */;
+  ret = malloc (len);
+  memcpy (ret, str1, len1);
+  ret[len1] = '\n';
+  memcpy (ret + len1 + 1, str2, len2);
+  ret[len-1] = '\0';
+
+  return ret;
+}
+
+%}
+
+%code requires {
+#ifndef YY_TYPEDEF_YY_SCANNER_T
+#define YY_TYPEDEF_YY_SCANNER_T
+typedef void *yyscan_t;
+#endif
+}
+
+%locations
+
+%union {
+  struct section *section;
+  struct field *field;
+  char *str;
+}
+
+%token <str>   SECTION_HEADER
+%token <field> FIELD
+%token <str>   VALUE_CONT
+%token         EMPTY_LINE
+%token         PGP_PROLOGUE
+%token         PGP_EPILOGUE
+%token         UNKNOWN_LINE
+
+%type <section> sections section
+%type <field>   fields field
+%type <str>     continuations
+
+%pure-parser
+
+%lex-param   { yyscan_t scanner }
+%parse-param { yyscan_t scanner }
+%parse-param { struct parse_context *context }
+
+%destructor { section_free ($$); } <sections>
+%destructor { section_free ($$); } <section>
+%destructor { field_free ($$); } <fields>
+%destructor { field_free ($$); } <field>
+
+%%
+
+index:
+      sections
+        { context->parsed_index = $1; }
+    | PGP_PROLOGUE sections PGP_EPILOGUE
+        { context->parsed_index = $2; }
+
+sections:
+      section emptylines
+        { $$ = $1; }
+    | section EMPTY_LINE emptylines sections
+        { $$ = $1; $$->next = $4; }
+    | emptylines
+        { $$ = NULL; }
+
+section:
+      SECTION_HEADER fields
+        { $$ = malloc (sizeof (struct section));
+          $$->next = NULL;
+          $$->name = $1;
+          $$->fields = $2; }
+
+fields:
+      /* empty */
+        { $$ = NULL; }
+    | field fields
+        { $$ = $1; $$->next = $2; }
+
+field: FIELD continuations
+        { $$ = $1;
+          char *old_value = $$->value;
+          $$->value = concat_newline (old_value, $2);
+          free (old_value);
+          free ($2); }
+
+continuations:
+      /* empty */
+        { $$ = NULL; }
+    | VALUE_CONT continuations
+        { $$ = concat_newline ($1, $2);
+          free ($1);
+          free ($2); }
+
+emptylines:
+      /* empty */
+        {}
+    | EMPTY_LINE emptylines
+        {}
+
+%%
+
+void
+yyerror (YYLTYPE * yylloc, yyscan_t scanner, struct parse_context *context, const char *msg)
+{
+  int has_suffix = context->error_suffix != NULL && context->error_suffix[0] != 0;
+
+  fprintf (stderr, "%s%s%s%ssyntax error at line %d: %s%s%s\n",
+           context->program_name ? context->program_name : "",
+           context->program_name ? ": " : "",
+           context->input_file ? context->input_file : "",
+           context->input_file ? ": " : "",
+           yylloc->first_line, msg,
+           has_suffix ? " " : "",
+           has_suffix ? context->error_suffix : "");
+}
+
+int
+do_parse (struct parse_context *context, FILE *in)
+{
+  yyscan_t scanner;
+  int res;
+
+  scanner_init (&scanner, context, in);
+  res = yyparse (scanner, context);
+  scanner_destroy (scanner);
+
+  return res;
+}

builder/index-parser-c.c

@@ -0,0 +1,119 @@
+/* virt-builder
+ * Copyright (C) 2013 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+/* This file handles the interface between the C/lex/yacc index file
+ * parser, and the OCaml world.  See index_parser.ml for the OCaml
+ * type definition.
+ */
+
+#include <config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <unistd.h>
+
+#include <caml/alloc.h>
+#include <caml/fail.h>
+#include <caml/memory.h>
+#include <caml/mlvalues.h>
+
+#ifdef HAVE_CAML_UNIXSUPPORT_H
+#include <caml/unixsupport.h>
+#else
+#define Nothing ((value) 0)
+extern void unix_error (int errcode, char * cmdname, value arg) Noreturn;
+#endif
+
+#include "index-struct.h"
+#include "index-parse.h"
+
+extern int do_parse (struct parse_context *context, FILE *in);
+
+value
+virt_builder_parse_index (value progv, value error_suffixv, value filenamev)
+{
+  CAMLparam2 (progv, filenamev);
+  CAMLlocal5 (rv, v, sv, sv2, fv);
+  struct section *sections;
+  size_t i, nr_sections;
+  struct parse_context context;
+  FILE *in;
+
+  parse_context_init (&context);
+  context.program_name = String_val (progv);
+  context.input_file = String_val (filenamev);
+  context.error_suffix = String_val (error_suffixv);
+
+  in = fopen (String_val (filenamev), "r");
+  if (in == NULL)
+    unix_error (errno, (char *) "fopen", filenamev);
+
+  if (do_parse (&context, in) != 0) {
+    fclose (in);
+    caml_invalid_argument ("parse error");
+  }
+
+  if (fclose (in) == EOF)
+    unix_error (errno, (char *) "fclose", filenamev);
+
+  /* Convert the parsed data to OCaml structures. */
+  nr_sections = 0;
+  for (sections = context.parsed_index; sections != NULL; sections = sections->next)
+    nr_sections++;
+  rv = caml_alloc (nr_sections, 0);
+
+  for (i = 0, sections = context.parsed_index; sections != NULL;
+       i++, sections = sections->next) {
+    struct field *fields;
+    size_t j, nr_fields;
+
+    nr_fields = 0;
+    for (fields = sections->fields; fields != NULL; fields = fields->next)
+      nr_fields++;
+    fv = caml_alloc (nr_fields, 0);
+
+    for (j = 0, fields = sections->fields; fields != NULL;
+         j++, fields = fields->next) {
+      v = caml_alloc_tuple (3);
+      sv = caml_copy_string (fields->key);
+      Store_field (v, 0, sv);   /* (key, Some subkey, value) */
+      if (fields->subkey) {
+        sv2 = caml_copy_string (fields->subkey);
+        sv = caml_alloc (1, 0);
+        Store_field (sv, 0, sv2);
+      } else
+        sv = Val_int (0);
+      Store_field (v, 1, sv);
+      sv = caml_copy_string (fields->value);
+      Store_field (v, 2, sv);
+      Store_field (fv, j, v);   /* assign to return array of fields */
+    }
+
+    v = caml_alloc_tuple (2);
+    sv = caml_copy_string (sections->name);
+    Store_field (v, 0, sv);     /* (name, fields) */
+    Store_field (v, 1, fv);
+    Store_field (rv, i, v);     /* assign to return array of sections */
+  }
+
+  /* Free parsed data. */
+  parse_context_free (&context);
+
+  CAMLreturn (rv);
+}

builder/index-scan.l

@@ -0,0 +1,128 @@
+/* libguestfs virt-builder tool -*- fundamental -*-
+ * Copyright (C) 2013 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+%{
+#include <config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "index-struct.h"
+#include "index-parse.h"
+
+#define YY_EXTRA_TYPE struct parse_context *
+#define YY_USER_ACTION yylloc->first_line = yylloc->last_line = yylineno;
+
+%}
+
+%option nounput
+%option noyywrap
+%option yylineno
+%option reentrant
+%option bison-bridge
+%option bison-locations
+
+%%
+
+ /* Apart from the PGP prologue/epilogue which is a hack, the
+  * scanning strategy is to deal with the file strictly line by
+  * line, and pass those lines up to the parser which deals with
+  * whether they appear in the right order to be meaningful.
+  * Note that flex does longest-match.
+  */
+
+  /* Ignore comments - '#' MUST appear at the start of a line. */
+^"#".*\n                { yyextra->seen_comments++; }
+
+  /* An empty line is significant. */
+^\n                                     { return EMPTY_LINE; }
+
+  /* [...] marks beginning of a section. */
+^"["[-A-Za-z0-9._]+"]"\n {
+                      yylval->str = strndup (yytext+1, yyleng-3);
+                      return SECTION_HEADER;
+                    }
+
+  /* field=value or field[subfield]=value */
+^[A-Za-z0-9_.]+("["[A-Za-z0-9_,.]+"]")?"=".*\n {
+                      size_t i = strcspn (yytext, "=[");
+                      yylval->field = malloc (sizeof (struct field));
+                      yylval->field->next = NULL;
+                      yylval->field->key = strndup (yytext, i);
+                      if (yytext[i] == '[') {
+                        size_t j = strcspn (yytext+i+1, "]");
+                        yylval->field->subkey = strndup (yytext+i+1, j);
+                        i += 1+j+1;
+                      } else {
+                        yylval->field->subkey = NULL;
+                      }
+                      /* Note we chop the final \n off here. */
+                      yylval->field->value = strndup (yytext+i+1, yyleng-(i+2));
+                      return FIELD;
+                    }
+
+  /* Continuation line for multi-line values. */
+^[[:blank:]].*\n        {
+                      yylval->str = strndup (yytext+1, yyleng-2);
+                      return VALUE_CONT;
+                    }
+
+ /* Hack to eat the PGP prologue. */
+^"-----BEGIN PGP SIGNED MESSAGE-----\n"  {
+  int c, prevnl = 0;
+
+  /* Eat everything to the first blank line. */
+  while ((c = input (yyscanner)) != EOF) {
+    if (c == '\n' && prevnl)
+      break;
+    prevnl = c == '\n';
+  }
+
+  return PGP_PROLOGUE;
+}
+
+ /* Hack to eat the PGP epilogue. */
+^"-----BEGIN PGP SIGNATURE-----\n"  {
+  /* Eat everything to the end of the file. */
+  while (input (yyscanner) != EOF)
+    ;
+
+  return PGP_EPILOGUE;
+}
+
+ /* anything else is an error */
+. {
+  return UNKNOWN_LINE;
+}
+
+%%
+
+void
+scanner_init (yyscan_t *scanner, struct parse_context *context, FILE *in)
+{
+  yylex_init (scanner);
+  yyset_extra (context, *scanner);
+  yyset_in (in, *scanner);
+}
+
+void
+scanner_destroy (yyscan_t scanner)
+{
+  yylex_destroy (scanner);
+}

builder/index-struct.c

@@ -0,0 +1,60 @@
+/* libguestfs virt-builder tool
+ * Copyright (C) 2013 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#include <config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "index-struct.h"
+
+void
+parse_context_init (struct parse_context *context)
+{
+  memset (context, 0, sizeof *context);
+}
+
+void
+parse_context_free (struct parse_context *context)
+{
+  section_free (context->parsed_index);
+}
+
+void
+section_free (struct section *section)
+{
+  if (section) {
+    section_free (section->next);
+    free (section->name);
+    field_free (section->fields);
+    free (section);
+  }
+}
+
+void
+field_free (struct field *field)
+{
+  if (field) {
+    field_free (field->next);
+    free (field->key);
+    free (field->subkey);
+    free (field->value);
+    free (field);
+  }
+}

builder/index-struct.h

@@ -0,0 +1,67 @@
+/* libguestfs virt-builder tool
+ * Copyright (C) 2013 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+/* The data structures produced when parsing the index file. */
+
+#ifndef INDEX_STRUCT_H
+#define INDEX_STRUCT_H
+
+/* A section or list of sections. */
+struct section {
+  struct section *next;
+  char *name;
+  struct field *fields;
+};
+
+/* A field or list of fields. */
+struct field {
+  struct field *next;
+  char *key;
+  char *subkey;
+  char *value;
+};
+
+/* A struct holding the data needed during the parsing. */
+struct parse_context {
+  struct section *parsed_index;        /* The result of the parsing. */
+  /* yyparse sets this if any comments were seen.  Required for checking
+   * compatibility with virt-builder 1.24.
+   */
+  int seen_comments;
+  const char *input_file;
+  const char *program_name;
+  const char *error_suffix;
+};
+
+/* Initialize the content of a parse_context. */
+extern void parse_context_init (struct parse_context *state);
+
+/* Free the content of a parse_context.  The actual pointer is not freed. */
+extern void parse_context_free (struct parse_context *state);
+
+/* Free the content of a section, recursively freeing also its fields.
+ * The actual pointer is not freed.
+ */
+extern void section_free (struct section *section);
+
+/* Free the content of a field, recursively freeing also its next field.
+ * The actual pointer is not freed.
+ */
+extern void field_free (struct field *field);
+
+#endif /* INDEX_STRUCT_H */

builder/index-validate.c

@@ -0,0 +1,185 @@
+/* libguestfs virt-builder tool
+ * Copyright (C) 2013 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#include <config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <limits.h>
+#include <getopt.h>
+#include <errno.h>
+#include <libintl.h>
+
+#include <guestfs.h>
+
+#include "guestfs-internal-frontend.h"
+
+#include "index-struct.h"
+#include "index-parse.h"
+
+extern int do_parse (struct parse_context *context, FILE *in);
+
+static void
+usage (int exit_status)
+{
+  printf ("%s index\n", program_name);
+  exit (exit_status);
+}
+
+int
+main (int argc, char *argv[])
+{
+  enum { HELP_OPTION = CHAR_MAX + 1 };
+  static const char *options = "V";
+  static const struct option long_options[] = {
+    { "help", 0, 0, HELP_OPTION },
+    { "compat-1.24.0", 0, 0, 0 },
+    { "compat-1.24.1", 0, 0, 0 },
+    { "version", 0, 0, 'V' },
+    { 0, 0, 0, 0 }
+  };
+  int c;
+  int option_index;
+  int compat_1_24_0 = 0;
+  int compat_1_24_1 = 0;
+  const char *input;
+  struct section *sections;
+  struct parse_context context;
+  FILE *in;
+  int ret;
+
+  setlocale (LC_ALL, "");
+  bindtextdomain (PACKAGE, LOCALEBASEDIR);
+  textdomain (PACKAGE);
+
+  parse_context_init (&context);
+
+  for (;;) {
+    c = getopt_long (argc, argv, options, long_options, &option_index);
+    if (c == -1) break;
+
+    switch (c) {
+    case 0:                     /* options which are long only */
+      if (STREQ (long_options[option_index].name, "compat-1.24.0"))
+        compat_1_24_0 = compat_1_24_1 = 1;
+      else if (STREQ (long_options[option_index].name, "compat-1.24.1"))
+        compat_1_24_1 = 1;
+      else {
+        fprintf (stderr, _("%s: unknown long option: %s (%d)\n"),
+                 program_name, long_options[option_index].name, option_index);
+        exit (EXIT_FAILURE);
+      }
+      break;
+
+    case 'V':
+      printf ("%s %s\n", PACKAGE_NAME, PACKAGE_VERSION);
+      exit (EXIT_SUCCESS);
+
+    case HELP_OPTION:
+      usage (EXIT_SUCCESS);
+
+    default:
+      usage (EXIT_FAILURE);
+    }
+  }
+
+  if (optind != argc-1)
+    usage (EXIT_FAILURE);
+
+  input = argv[optind++];
+
+  in = fopen (input, "r");
+  if (in == NULL) {
+    perror (input);
+    exit (EXIT_FAILURE);
+  }
+
+  ret = do_parse (&context, in);
+
+  if (fclose (in) == EOF) {
+    fprintf (stderr, _("%s: %s: error closing input file: %m (ignored)\n"),
+             program_name, input);
+  }
+
+  if (ret != 0) {
+    parse_context_free (&context);
+    fprintf (stderr, _("%s: '%s' could not be validated, see errors above\n"),
+             program_name, input);
+    exit (EXIT_FAILURE);
+  }
+
+  if (compat_1_24_1 && context.seen_comments) {
+    parse_context_free (&context);
+    fprintf (stderr, _("%s: %s contains comments which will not work with virt-builder 1.24.1\n"),
+             program_name, input);
+    exit (EXIT_FAILURE);
+  }
+
+  /* Iterate over the parsed sections, semantically validating it. */
+  for (sections = context.parsed_index; sections != NULL; sections = sections->next) {
+    int seen_sig = 0;
+    struct field *fields;
+
+    if (compat_1_24_0) {
+      if (strchr (sections->name, '_')) {
+        parse_context_free (&context);
+        fprintf (stderr, _("%s: %s: section [%s] has invalid characters which will not work with virt-builder 1.24.0\n"),
+                 program_name, input, sections->name);
+        exit (EXIT_FAILURE);
+      }
+    }
+
+    for (fields = sections->fields; fields != NULL; fields = fields->next) {
+      if (compat_1_24_0) {
+        if (strchr (fields->key, '[') ||
+            strchr (fields->key, ']')) {
+          parse_context_free (&context);
+          fprintf (stderr, _("%s: %s: section [%s], field '%s' has invalid characters which will not work with virt-builder 1.24.0\n"),
+                   program_name, input, sections->name, fields->key);
+          exit (EXIT_FAILURE);
+        }
+      }
+      if (compat_1_24_1) {
+        if (strchr (fields->key, '.') ||
+            strchr (fields->key, ',')) {
+          parse_context_free (&context);
+          fprintf (stderr, _("%s: %s: section [%s], field '%s' has invalid characters which will not work with virt-builder 1.24.1\n"),
+                   program_name, input, sections->name, fields->key);
+          exit (EXIT_FAILURE);
+        }
+      }
+      if (STREQ (fields->key, "sig"))
+        seen_sig = 1;
+    }
+
+    if (compat_1_24_0 && !seen_sig) {
+      parse_context_free (&context);
+      fprintf (stderr, _("%s: %s: section [%s] is missing a 'sig' field which will not work with virt-builder 1.24.0\n"),
+               program_name, input, sections->name);
+      exit (EXIT_FAILURE);
+    }
+  }
+
+  /* Free the parsed data. */
+  parse_context_free (&context);
+
+  printf ("%s validated OK\n", input);
+
+  exit (EXIT_SUCCESS);
+}

builder/index_parser.ml

@@ -27,21 +27,29 @@ and entry = {
   printable_name : string option;       (* the name= field *)
   osinfo : string option;
   file_uri : string;
-  signature_uri : string option;
+  arch : string;
+  signature_uri : string option;        (* deprecated, will be removed in 1.26 *)
+  checksum_sha512 : string option;
   revision : int;
   format : string option;
   size : int64;
   compressed_size : int64 option;
   expand : string option;
   lvexpand : string option;
-  notes : string option;
+  notes : (string * string) list;
   hidden : bool;
+  aliases : string list option;
+
+  sigchecker : Sigchecker.t;
+  proxy : Downloader.proxy_mode;
 }
 
 let print_entry chan (name, { printable_name = printable_name;
                               file_uri = file_uri;
+                              arch = arch;
                               osinfo = osinfo;
                               signature_uri = signature_uri;
+                              checksum_sha512 = checksum_sha512;
                               revision = revision;
                               format = format;
                               size = size;
@@ -49,6 +57,7 @@ let print_entry chan (name, { printable_name = printable_name;
                               expand = expand;
                               lvexpand = lvexpand;
                               notes = notes;
+                              aliases = aliases;
                               hidden = hidden }) =
   let fp fs = fprintf chan fs in
   fp "[%s]\n" name;
@@ -61,10 +70,15 @@ let print_entry chan (name, { printable_name = printable_name;
   | Some id -> fp "osinfo=%s\n" id
   );
   fp "file=%s\n" file_uri;
+  fp "arch=%s\n" arch;
   (match signature_uri with
   | None -> ()
   | Some uri -> fp "sig=%s\n" uri
   );
+  (match checksum_sha512 with
+  | None -> ()
+  | Some uri -> fp "checksum[sha512]=%s\n" uri
+  );
   fp "revision=%d\n" revision;
   (match format with
   | None -> ()
@@ -83,20 +97,20 @@ let print_entry chan (name, { printable_name = printable_name;
   | None -> ()
   | Some lvexpand -> fp "lvexpand=%s\n" lvexpand
   );
-  (match notes with
+  List.iter (
+    fun (lang, notes) ->
+      match lang with
+      | "" -> fp "notes=%s\n" notes
+      | lang -> fp "notes[%s]=%s\n" lang notes
+  ) notes;
+  (match aliases with
   | None -> ()
-  | Some notes -> fp "notes=%s\n" notes
+  | Some l -> fp "aliases=%s\n" (String.concat " " l)
   );
   if hidden then fp "hidden=true\n"
 
-let fieldname_rex = Str.regexp "^\\([a-z_]+\\)=\\(.*\\)$"
-
-let get_index ~debug ~downloader ~sigchecker source =
-  let rec corrupt_line line =
-    eprintf (f_"virt-builder: error parsing index near this line:\n\n%s\n")
-      line;
-    corrupt_file ()
-  and corrupt_file () =
+let get_index ~prog ~verbose ~downloader ~sigchecker ~proxy source =
+  let corrupt_file () =
     eprintf (f_"\nThe index file downloaded from '%s' is corrupt.\nYou need to ask the supplier of this file to fix it and upload a fixed version.\n")
       source;
     exit 1
@@ -104,162 +118,55 @@ let get_index ~debug ~downloader ~sigchecker source =
 
   let rec get_index () =
     (* Get the index page. *)
-    let tmpfile, delete_tmpfile = Downloader.download downloader source in
+    let tmpfile, delete_tmpfile = Downloader.download ~prog downloader ~proxy source in
 
     (* Check index file signature (also verifies it was fully
      * downloaded and not corrupted in transit).
      *)
     Sigchecker.verify sigchecker tmpfile;
 
-    (* Check the index page is not too huge. *)
-    let st = stat tmpfile in
-    if st.st_size > 1_000_000 then (
-      eprintf (f_"virt-builder: index page '%s' is too large (size %d bytes)\n")
-        source st.st_size;
-      exit 1
-    );
-
-    (* Load the file into memory. *)
-    let index = read_whole_file tmpfile in
+    (* Try parsing the file. *)
+    let sections = Ini_reader.read_ini ~prog tmpfile in
     if delete_tmpfile then
       (try Unix.unlink tmpfile with _ -> ());
 
-    (* Split file into lines. *)
-    let index = string_nsplit "\n" index in
-
-    (* If there is a signature (checked above) then remove it. *)
-    let index =
-      match index with
-      | "-----BEGIN PGP SIGNED MESSAGE-----" :: lines ->
-        (* Ignore all lines until we get to first blank. *)
-        let lines = dropwhile ((<>) "") lines in
-        (* Ignore the blank line too. *)
-        let lines = List.tl lines in
-        (* Take lines until we get to the end signature. *)
-        let lines = takewhile ((<>) "-----BEGIN PGP SIGNATURE-----") lines in
-        lines
-      | _ -> index in
-
-    (* Split into sections around each /^[/ *)
-    let rec loop = function
-      | [] -> []
-      | x :: xs when String.length x >= 1 && x.[0] = '[' ->
-        let lines = takewhile ((<>) "") xs in
-        let rest = dropwhile ((<>) "") xs in
-        if rest = [] then
-          [x, lines]
-        else (
-          let rest = List.tl rest in
-          let rest = loop rest in
-          (x, lines) :: rest
-        )
-      | x :: _ -> corrupt_line x
-    in
-    let sections = loop index in
-
-    (* Parse the fields in each section. *)
-    let isspace = function ' ' | '\t' -> true | _ -> false in
-    let starts_space str = String.length str >= 1 && isspace str.[0] in
-    let rec loop = function
-      | [] -> []
-      | x :: xs when not (starts_space x) && String.contains x '=' ->
-        let xs' = takewhile starts_space xs in
-        let ys = dropwhile starts_space xs in
-        (x :: xs') :: loop ys
-      | x :: _ -> corrupt_line x
-    in
-    let sections = List.map (fun (n, lines) -> n, loop lines) sections in
-
-    if debug then (
-      eprintf "index file (%s) after splitting:\n" source;
-      List.iter (
-        fun (n, fields) ->
-          eprintf "  os-version: %s\n" n;
-          let i = ref 0 in
-          List.iter (
-            fun field ->
-              eprintf "    %d: " !i;
-              List.iter prerr_endline field;
-              incr i
-          ) fields
-      ) sections
-    );
-
-    (* Now we've parsed the file into the correct sections, we
-     * interpret the meaning of the fields.
-   *)
-    let sections = List.map (
+    (* Check for repeated os-version+arch combination. *)
+    let name_arch_map = List.map (
       fun (n, fields) ->
-        let len = String.length n in
-        if len < 3 || n.[0] <> '[' || n.[len-1] <> ']' then
-          corrupt_line n;
-        let n = String.sub n 1 (len-2) in
-
-        let fields = List.map (
-          function
-          | [] -> assert false (* can never happen, I think? *)
-          | x :: xs when Str.string_match fieldname_rex x 0 ->
-            let field = Str.matched_group 1 x in
-            let rest_of_line = Str.matched_group 2 x in
-            let allow_multiline =
-              match field with
-              | "name" -> false
-              | "osinfo" -> false
-              | "file" -> false
-              | "sig" -> false
-              | "revision" -> false
-              | "format" -> false
-              | "size" -> false
-              | "compressed_size" -> false
-              | "expand" -> false
-              | "lvexpand" -> false
-              | "notes" -> true
-              | "hidden" -> false
-              | _ ->
-                eprintf "warning: unknown field '%s' in index (ignored)\n%!"
-                  field;
-                true in
-            let value =
-              if not allow_multiline then (
-                if xs <> [] then (
-                  eprintf (f_"virt-builder: field '%s' cannot span multiple lines\n")
-                    field;
-                  corrupt_line (List.hd xs)
-                );
-                rest_of_line
-              ) else (
-                String.concat "\n" (rest_of_line :: xs)
-              ) in
-            field, value
-          | x :: _ ->
-            corrupt_line x
-        ) fields in
-
-        (n, fields)
+        let rec find_arch = function
+          | ("arch", None, value) :: y -> value
+          | _ :: y -> find_arch y
+          | [] -> ""
+        in
+        n, (find_arch fields)
     ) sections in
-
-    (* Check for repeated os-version names. *)
     let nseen = Hashtbl.create 13 in
     List.iter (
-      fun (n, _) ->
-        if Hashtbl.mem nseen n then (
-          eprintf (f_"virt-builder: index is corrupt: os-version '%s' appears two or more times\n") n;
+      fun (n, arch) ->
+        let id = n, arch in
+        if Hashtbl.mem nseen id then (
+          eprintf (f_"virt-builder: index is corrupt: os-version '%s' with architecture '%s' appears two or more times\n") n arch;
           corrupt_file ()
         );
-        Hashtbl.add nseen n true
-    ) sections;
+        Hashtbl.add nseen id true
+    ) name_arch_map;
 
     (* Check for repeated fields. *)
     List.iter (
       fun (n, fields) ->
         let fseen = Hashtbl.create 13 in
         List.iter (
-          fun (field, _) ->
-            if Hashtbl.mem fseen field then (
-              eprintf (f_"virt-builder: index is corrupt: %s: field '%s' appears two or more times\n") n field;
+          fun (field, subkey, _) ->
+            let hashkey = (field, subkey) in
+            if Hashtbl.mem fseen hashkey then (
+              (match subkey with
+              | Some value ->
+                eprintf (f_"virt-builder: index is corrupt: %s: field '%s[%s]' appears two or more times\n") n field value
+              | None ->
+                eprintf (f_"virt-builder: index is corrupt: %s: field '%s' appears two or more times\n") n field);
               corrupt_file ()
             );
-            Hashtbl.add fseen field true
+            Hashtbl.add fseen hashkey true
         ) fields
     ) sections;
 
@@ -267,20 +174,31 @@ let get_index ~debug ~downloader ~sigchecker source =
     let entries =
       List.map (
         fun (n, fields) ->
+          let fields = List.map (fun (k, sk, v) -> (k, sk), v) fields in
           let printable_name =
-            try Some (List.assoc "name" fields) with Not_found -> None in
+            try Some (List.assoc ("name", None) fields) with Not_found -> None in
           let osinfo =
-            try Some (List.assoc "osinfo" fields) with Not_found -> None in
+            try Some (List.assoc ("osinfo", None) fields) with Not_found -> None in
           let file_uri =
-            try make_absolute_uri (List.assoc "file" fields)
+            try make_absolute_uri (List.assoc ("file", None) fields)
             with Not_found ->
               eprintf (f_"virt-builder: no 'file' (URI) entry for '%s'\n") n;
             corrupt_file () in
+          let arch =
+            try List.assoc ("arch", None) fields
+            with Not_found ->
+              eprintf (f_"virt-builder: no 'arch' entry for '%s'\n") n;
+            corrupt_file () in
           let signature_uri =
-            try Some (make_absolute_uri (List.assoc "sig" fields))
+            try Some (make_absolute_uri (List.assoc ("sig", None) fields))
             with Not_found -> None in
+          let checksum_sha512 =
+            try Some (List.assoc ("checksum", Some "sha512") fields)
+            with Not_found ->
+              try Some (List.assoc ("checksum", None) fields)
+              with Not_found -> None in
           let revision =
-            try int_of_string (List.assoc "revision" fields)
+            try int_of_string (List.assoc ("revision", None) fields)
             with
             | Not_found -> 1
             | Failure "int_of_string" ->
@@ -288,9 +206,9 @@ let get_index ~debug ~downloader ~sigchecker source =
                 n;
               corrupt_file () in
           let format =
-            try Some (List.assoc "format" fields) with Not_found -> None in
+            try Some (List.assoc ("format", None) fields) with Not_found -> None in
           let size =
-            try Int64.of_string (List.assoc "size" fields)
+            try Int64.of_string (List.assoc ("size", None) fields)
             with
             | Not_found ->
               eprintf (f_"virt-builder: no 'size' field for '%s'\n") n;
@@ -300,7 +218,7 @@ let get_index ~debug ~downloader ~sigchecker source =
                 n;
               corrupt_file () in
           let compressed_size =
-            try Some (Int64.of_string (List.assoc "compressed_size" fields))
+            try Some (Int64.of_string (List.assoc ("compressed_size", None) fields))
             with
             | Not_found ->
               None
@@ -309,24 +227,44 @@ let get_index ~debug ~downloader ~sigchecker source =
                 n;
               corrupt_file () in
           let expand =
-            try Some (List.assoc "expand" fields) with Not_found -> None in
+            try Some (List.assoc ("expand", None) fields) with Not_found -> None in
           let lvexpand =
-            try Some (List.assoc "lvexpand" fields) with Not_found -> None in
+            try Some (List.assoc ("lvexpand", None) fields) with Not_found -> None in
           let notes =
-            try Some (List.assoc "notes" fields) with Not_found -> None in
+            let rec loop = function
+              | [] -> []
+              | (("notes", subkey), value) :: xs ->
+                let subkey = match subkey with
+                | None -> ""
+                | Some v -> v in
+                (subkey, value) :: loop xs
+              | _ :: xs -> loop xs in
+            List.sort (
+              fun (k1, _) (k2, _) ->
+                String.compare k1 k2
+            ) (loop fields) in
           let hidden =
-            try bool_of_string (List.assoc "hidden" fields)
+            try bool_of_string (List.assoc ("hidden", None) fields)
             with
             | Not_found -> false
             | Failure "bool_of_string" ->
               eprintf (f_"virt-builder: cannot parse 'hidden' field for '%s'\n")
                 n;
               corrupt_file () in
+          let aliases =
+            let l =
+              try string_nsplit " " (List.assoc ("aliases", None) fields)
+              with Not_found -> [] in
+            match l with
+            | [] -> None
+            | l -> Some l in
 
           let entry = { printable_name = printable_name;
                         osinfo = osinfo;
                         file_uri = file_uri;
+                        arch = arch;
                         signature_uri = signature_uri;
+                        checksum_sha512 = checksum_sha512;
                         revision = revision;
                         format = format;
                         size = size;
@@ -334,12 +272,15 @@ let get_index ~debug ~downloader ~sigchecker source =
                         expand = expand;
                         lvexpand = lvexpand;
                         notes = notes;
-                        hidden = hidden } in
+                        hidden = hidden;
+                        aliases = aliases;
+                        proxy = proxy;
+                        sigchecker = sigchecker } in
           n, entry
       ) sections in
 
-    if debug then (
-      eprintf "index file (%s) after parsing:\n" source;
+    if verbose then (
+      eprintf "index file (%s) after parsing (C parser):\n" source;
       List.iter (print_entry Pervasives.stderr) entries
     );
 

builder/index_parser.mli

@@ -21,15 +21,21 @@ and entry = {
   printable_name : string option;       (* the name= field *)
   osinfo : string option;
   file_uri : string;
-  signature_uri : string option;
+  arch : string;
+  signature_uri : string option;        (* deprecated, will be removed in 1.26 *)
+  checksum_sha512 : string option;
   revision : int;
   format : string option;
   size : int64;
   compressed_size : int64 option;
   expand : string option;
   lvexpand : string option;
-  notes : string option;
+  notes : (string * string) list;
   hidden : bool;
+  aliases : string list option;
+
+  sigchecker : Sigchecker.t;
+  proxy : Downloader.proxy_mode;
 }
 
-val get_index : debug:bool -> downloader:Downloader.t -> sigchecker:Sigchecker.t -> string -> index
+val get_index : prog:string -> verbose:bool -> downloader:Downloader.t -> sigchecker:Sigchecker.t -> proxy:Downloader.proxy_mode -> string -> index

builder/ini_reader.ml

@@ -0,0 +1,38 @@
+(* virt-builder
+ * Copyright (C) 2013-2014 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+type sections = section list
+and section = string * fields                (* [name] + fields *)
+and fields = field list
+and field = string * string option * string  (* key + subkey + value *)
+
+(* Types returned by the C index parser. *)
+type c_sections = c_section array
+and c_section = string * c_fields             (* [name] + fields *)
+and c_fields = field array
+
+(* Calls yyparse in the C code. *)
+external parse_index : prog:string -> error_suffix:string -> string -> c_sections = "virt_builder_parse_index"
+
+let read_ini ~prog ?(error_suffix = "") file =
+  let sections = parse_index ~prog ~error_suffix file in
+  let sections = Array.to_list sections in
+  List.map (
+    fun (n, fields) ->
+      n, Array.to_list fields
+  ) sections

builder/ini_reader.mli

@@ -0,0 +1,24 @@
+(* virt-builder
+ * Copyright (C) 2013-2014 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+type sections = section list
+and section = string * fields                (* [name] + fields *)
+and fields = field list
+and field = string * string option * string  (* key + subkey + value *)
+
+val read_ini : prog:string -> ?error_suffix:string -> string -> sections

builder/languages.ml

@@ -0,0 +1,57 @@
+(* virt-builder
+ * Copyright (C) 2013-2014 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+open Common_utils
+
+let split_locale loc =
+  let regex = Str.regexp "^\\([A-Za-z]+\\)\\(_\\([A-Za-z]+\\)\\)?\\(\\.\\([A-Za-z0-9-]+\\)\\)?\\(@\\([A-Za-z]+\\)\\)?$" in
+  let l = ref [] in
+  if Str.string_match regex loc 0 then (
+    let match_or_empty n =
+      try Str.matched_group n loc with
+      | Not_found -> ""
+    in
+    let lang = Str.matched_group 1 loc in
+    let territory = match_or_empty 3 in
+    (match territory with
+    | "" -> ()
+    | territory -> l := (lang ^ "_" ^ territory) :: !l);
+    l := lang :: !l;
+  );
+  l := "" :: !l;
+  List.rev !l
+
+let languages () =
+  match Setlocale.setlocale Setlocale.LC_MESSAGES None with
+  | None -> [""]
+  | Some locale -> split_locale locale
+
+let find_notes languages notes =
+  let notes = List.fold_left (
+    fun acc lang ->
+      let res = List.filter (
+        fun (langkey, _) ->
+          match langkey with
+          | "C" -> lang = ""
+          | langkey -> langkey = lang
+      ) notes in
+      match res with
+      | (_, noteskey) :: _ -> noteskey :: acc
+      | [] -> acc
+  ) [] languages in
+  List.rev notes

builder/languages.mli

@@ -0,0 +1,21 @@
+(* virt-builder
+ * Copyright (C) 2014 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+val languages : unit -> string list
+
+val find_notes : string list -> (string * string) list -> string list

builder/libguestfs.conf.in

@@ -0,0 +1,3 @@
+[libguestfs.org]
+uri=http://libguestfs.org/download/builder/index.asc
+gpgkey=file://@SYSCONFDIR@/xdg/virt-builder/repos.d/libguestfs.gpg

builder/libguestfs.gpg

@@ -0,0 +1,64 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v1.4.14 (GNU/Linux)
+
+mQINBE6UMMEBEADM811hfTulaF4JpkVpAI10FImyb4ArvOiu8NdcUwTFo+cyWno3
+U85B86H1Bsk/LgLTYtthSrTgsCtdxy+i5OaMjxZDIwKQ2+IYI3FCn9T3Mn28Idyh
+kLHzrO9ph0Dv0BNfrlDZhQEC53aAFe/QxN7+A49BNBV7D1VAOOCsHjxMEDzcZkCa
+oCrtXw1aNm2vkkj5ukbfukHAyLcQL7kow0qKPSVa1G4lfQP0WiG259Ydy+sUmbVb
+TGdb6MEC84PQRDuw6/ZeoV04tn7ZNtQEMOS0uiciHOGfr2hBxQf9VIPNrHg42yaL
+dOv51D99GuaxZ9E0HSoH/RwB1oXgd6rFdqVNYaBIQnnkwJANUEeGBArtIOZNCADT
+Bt8vkSDm+lLEAFS+V8CACyW/LMIrGCvLdHeqtoAv0GDVyR2GPxldYfdtEmCUMWcb
+Jlf71V9iAse2gUdoiHp5FfpGMkA5j7idKuxIws11XxRZJXXbBqiBqmVEAQ/v0m6p
+kdo0MYTHydmecLuUK2bAGhpysfX97EfTSrxfrYphYWjTfKRD9GrADeZNfuz1DbKs
+7LSqVaQJSjQrfgAwcnZLRaU0V4P5zxiz50gz1Aj3AZRL+Y3meZenzZTXcLFdnusg
+wUfhhCuL3tluMtEh6tznumyxb43WO1yLwj6J6LtveiuJN1Z+KSQ6OieZcwARAQAB
+tCVSaWNoYXJkIFcuTS4gSm9uZXMgPHJpY2hAYW5uZXhpYS5vcmc+iQI4BBMBAgAi
+BQJOlDDBAhsDBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRCRc49z4bdooHQY
+D/wJLklSZNyXIW+rG5sUbg7j9cTIF5p/lB9kI2yx6KodJp/2knKyvnmzz0gBw/OE
+HL4E4UW26oWKo+36I8wkBnuGa6UtANeITcJqFE19VpHEXHsxre64jNQnO8/w748W
+1ROW+Ry43xmrlRWKuCm4oPYUzlp0fq9ATAne8eblfG+NOs8DYuA8xZNQzFaI2kDC
+QLD4YoXLoNsP27Koga36b0KwxPFD9tyVZiu9XDH/3hMN7Nb15B66PFr+HcMmQ67G
+nUIN5ulcIwj38i40cyaTs1VRheOzTHXE/a6Q2AhMKiKqOoEjQ73/mV7cAVoPtM3o
+83Q/8aVKBH0bVRwAeV1tju6b14fqKoG0zNBEcXdlSkht6ScxJYIc/LPUxAMDwgSE
+OWshjmeRzKXypBbHn/DP8QVyM2gk5wY+mMSH7MpR0p/hgj+rFO8H9L7pC4dCog3E
+qzrYhRN+TaP6MPH3WkOwPH4d4IfQRFnHp+VPYPijKEiLrUl/o8k3DyAanAPBpJ/x
+na4wXAjlFBctOq6g+SrCUiHpwk7b2YNwGgr5Vl3GmZELzK/G8gg3uJYKQ9Bpv16t
+WWOz+IFiOFa0UULeo0QPmFAIMZiDojNsY1SwBKB3ZL1YWZezgMdQAbpze/IXoSt7
+zxWJoKH2jK7q9mvFiaY12l2YnKuCcegWVAViLxRpBnrbz7QmUmljaGFyZCBXLk0u
+IEpvbmVzIDxyam9uZXNAcmVkaGF0LmNvbT6JAjgEEwECACIFAk6UOQsCGwMGCwkI
+BwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJEJFzj3Pht2igIUYQAKomI0edLakahsUQ
+MxOZuhBbXJ4/VWF8bXYChDNPKvJp5nB7fBXujJ+39cIUM5fe2ViO6qSDpFC29imx
+F5pPbAqspZBPBkLLiZLji8R42hGarntdtTW0UWSBpq+nC5+G1psrnATI3uXGNxKQ
+R99c5HoMY7dBC2Y8TCGE64NINZ/XVh472s6IGLPn8MTn26YdRKC9BrVkCFMP2OBr
+6D4IprnyTAWAzb68ew20QmyWO+NBi9MplaDNQVl8PIOgfpyWlkgX1z9m67pcSDkw
+46hksp0yuOD1VwR4iVZ2/CmIsGRUlx41vWD6BIp9KxKyDIU1CYTRhq72dahHsl/8
+BjCndV5PO0GphqfCzmCv4DXjUwmrMTbH/GFnt5rfwcMcXUgcK0vV9vQ2SOU56Zd1
+fb27ZCFJKZc0Fu8krwFldCp/NYILf6ogUL/C1hfuCGSSuyDVY16Gg3dla1x+6zpF
+asnWQlaw8xT5LlMWvTZs5WsoSVHu7dVZWlgxINP++hlZrTz/S8l38yyQ15YFFl3W
+9M7dzkegOeDTPfx6B89WgfvfJjA/D0/FYxxWPXEtrn9DlJ4daEJqNsrvfLErz9R8
+4IQmfmhR93j+rdotner+6keC/wVByEfbW1wmXtmFKXQ6srdpj8VKRFrvkyXVgepM
+DypLgRH2v7lL2kdWhUu2y4EAgrwzuQINBE6UMMEBEADxQxMgUuDrw5GT4tqARTPI
+SSdNcUsRxRhVA8srYOyECliE+B3TwcRDFBs+MyPFJVEuX8fi4eGj/AK5t1GHerfk
+orUGlz72q4c7LLhkfZrsuJbk2dgkjvldKJnIazQJa6epGLqdsE5RlmSgwedIbtMd
+naGJBQH8aKP/Wi1+wUxsm5N3p7+R2WRx48VfpEhYB+Zf/FkFm1Ycjwh57KQ0+OHw
+ykf8VfMisxuH30tDxOCV+VptWKfOF2rDNdaNPWhij2YIjhJXRpkuRR+1PpI4jLaD
+JxcVZmG/0zucacupUN2g5OUH59ySU/totD6YMnmp3FONoyF1uIEJo6Vs30npHGkO
+XgBo3Pxt7oLJeykLPtdSLgm3cwXIYMWarVsAkKNXitQIVGpVRLeaK373VwmXFqoi
+M2SMHeawTUdOORFjpQzkknlJWM1TmUVtHHKt8Pl9+/5+wXKyt2IDdcUkMrB6K5qF
+fb7EwVhoI8ehJQK+eeDCjFwCAiwB3iV8JlyW+tEU7JuyXOQlwY1VWm/WqMD8gaRi
+rT+RFDFliZ3tQbW2pqUoZBROV5HN4tieDfwxGKCvk6Tsdb30zA9DPQp93+238bYf
+312sg9R+CD0AqxoxFG5FJu4HShcPRrPnYtRZqKRe40GDWvBEArXZprwL1qrP+Kl/
+mRrEQpxAGIoFG8HbVvD3EQARAQABiQIfBBgBAgAJBQJOlDDBAhsMAAoJEJFzj3Ph
+t2igSLQP/2uIrAY2CDr0kWBJiD3TztiHy8IdxwUpyTBTebwmAbi44/EvtJfIisrG
+YjKIEv/w0E61gO7O1JBG4+IG93W+v9fTT/e39JMyxsYqoZZHUhP11Okx5grDS5b0
+O8VXOmXVRMdVNfstRBr10HD9uNDq7ruKD18TxYTwN0GPD4gj1dbHQDR77Tr5cyBs
+6Ou5PBOH4r3qcqf/cJUSMeUUu75xLwixux6E7tD2S+t6F07wlWxntUcPtzyAHj20
+J89orUC+dT6r6MypBoI0jdJCp9JPGtR7i+fE5Gm4E5+AUSubLPtZGRY9Um2eMoS2
+DnQpGOKx1VvsixR/Kw44j2tRAvmYMS4iDKcuZU+nZ+xokAgObILj/b9n/Qe2/fXy
+CFdcgSvbm+dV1fZxsdMF/P9OU8aqdT9A9Fv5y+cDMEg4DVnhwMJTxGh/TCkw/H+A
+frHEtRc98lSQN5odpITNG17mG6JOdHM+wA57qHH0uy4+5RsbyAJahcdBcmObK/RF
+i4WZlThpbHftX5O/LH98aYQ2fJayIxv1EAjzOBOQ0MfBHI0KCJR1pysEisX28sJA
+Ic73gnJJ3BLZbqfBRgxjNMNroxC+5Tw6uPGFHa3YnuIAxxw0HcDVZ9vnTWBWFPGw
+ZvXkQ3FVJwZoLmHw47vvlVpLD/4gi1SuHWieRvZ+UdDq00E348pm
+=neBW
+-----END PGP PUBLIC KEY BLOCK-----

builder/list_entries.ml

@@ -21,46 +21,168 @@ open Common_utils
 
 open Printf
 
-let list_entries ?(list_long = false) ~source index =
-  if list_long then (
-    printf (f_"Source URI: %s\n") source;
-    printf "\n"
-  );
+let rec list_entries ~list_format ~sources index =
+  match list_format with
+  | `Short -> list_entries_short index
+  | `Long -> list_entries_long ~sources index
+  | `Json -> list_entries_json ~sources index
+
+and list_entries_short index =
+  List.iter (
+    fun (name, { Index_parser.printable_name = printable_name;
+                 arch = arch;
+                 hidden = hidden }) ->
+      if not hidden then (
+        printf "%-24s" name;
+        printf " %-10s" arch;
+        (match printable_name with
+        | None -> ()
+        | Some s -> printf " %s" s
+        );
+        printf "\n"
+      )
+  ) index
+
+and list_entries_long ~sources index =
+  let langs = Languages.languages () in
+
+  List.iter (
+    fun (source, key, proxy) ->
+      printf (f_"Source URI: %s\n") source;
+      (match key with
+      | Sigchecker.No_Key -> ()
+      | Sigchecker.Fingerprint fp ->
+        printf (f_"Fingerprint: %s\n") fp;
+      | Sigchecker.KeyFile kf ->
+        printf (f_"Key: %s\n") kf;
+      );
+      printf "\n"
+  ) sources;
 
   List.iter (
     fun (name, { Index_parser.printable_name = printable_name;
+                 arch = arch;
                  size = size;
                  compressed_size = compressed_size;
                  notes = notes;
+                 aliases = aliases;
                  hidden = hidden }) ->
       if not hidden then (
-        if not list_long then (         (* Short *)
-          printf "%-24s" name;
-          (match printable_name with
-          | None -> ()
-          | Some s -> printf " %s" s
-          );
-          printf "\n"
-        )
-        else (                          (* Long *)
-          printf "%-24s %s\n" "os-version:" name;
-          (match printable_name with
-          | None -> ()
-          | Some name -> printf "%-24s %s\n" (s_"Full name:") name;
-          );
-          printf "%-24s %s\n" (s_"Minimum/default size:") (human_size size);
-          (match compressed_size with
-          | None -> ()
-          | Some size ->
-            printf "%-24s %s\n" (s_"Download size:") (human_size size);
-          );
-          (match notes with
-          | None -> ()
-          | Some notes ->
-            printf "\n";
-            printf "Notes:\n %s\n" notes
-          );
-          printf "\n"
-        )
+        printf "%-24s %s\n" "os-version:" name;
+        (match printable_name with
+        | None -> ()
+        | Some name -> printf "%-24s %s\n" (s_"Full name:") name;
+        );
+        printf "%-24s %s\n" (s_"Architecture:") arch;
+        printf "%-24s %s\n" (s_"Minimum/default size:") (human_size size);
+        (match compressed_size with
+        | None -> ()
+        | Some size ->
+          printf "%-24s %s\n" (s_"Download size:") (human_size size);
+        );
+        (match aliases with
+        | None -> ()
+        | Some l -> printf "%-24s %s\n" (s_"Aliases:")
+                      (String.concat " " l);
+        );
+        let notes = Languages.find_notes langs notes in
+        (match notes with
+        | notes :: _ ->
+          printf "\n";
+          printf (f_"Notes:\n\n%s\n") notes
+        | [] -> ()
+        );
+        printf "\n"
       )
   ) index
+
+and list_entries_json ~sources index =
+  let trailing_comma index size =
+    if index = size - 1 then "" else "," in
+  let json_string_of_bool b =
+    if b then "true" else "false" in
+  let json_string_escape str =
+    let res = ref "" in
+    for i = 0 to String.length str - 1 do
+      res := !res ^ (match str.[i] with
+        | '"' -> "\\\""
+        | '\\' -> "\\\\"
+        | '\b' -> "\\b"
+        | '\n' -> "\\n"
+        | '\r' -> "\\r"
+        | '\t' -> "\\t"
+        | c -> String.make 1 c)
+    done;
+    !res in
+  let json_optional_printf_string key = function
+    | None -> ()
+    | Some str ->
+      printf "    \"%s\": \"%s\",\n" key (json_string_escape str) in
+  let json_optional_printf_int64 key = function
+    | None -> ()
+    | Some n ->
+      printf "    \"%s\": \"%Ld\",\n" key n in
+  let json_optional_printf_stringlist key = function
+    | None -> ()
+    | Some l ->
+      printf "    \"%s\": [" key;
+      iteri (
+        fun i alias ->
+          printf " \"%s\"%s" alias (trailing_comma i (List.length l))
+      ) l;
+      printf " ],\n" in
+  let print_notes = function
+    | [] -> ()
+    | notes ->
+      printf "    \"notes\": {\n";
+      iteri (
+        fun i (lang, langnotes) ->
+          let lang =
+            match lang with
+            | "" -> "C"
+            | x -> x in
+          printf "      \"%s\": \"%s\"%s\n"
+            (json_string_escape lang) (json_string_escape langnotes)
+            (trailing_comma i (List.length notes))
+      ) notes;
+      printf "    },\n" in
+
+  printf "{\n";
+  printf "  \"version\": %d,\n" 1;
+  printf "  \"sources\": [\n";
+  iteri (
+    fun i (source, key, proxy) ->
+      printf "  {\n";
+      (match key with
+      | Sigchecker.No_Key -> ()
+      | Sigchecker.Fingerprint fp ->
+        printf "    \"fingerprint\": \"%s\",\n" fp;
+      | Sigchecker.KeyFile kf ->
+        printf "    \"key\": \"%s\",\n" kf;
+      );
+      printf "    \"uri\": \"%s\"\n" source;
+      printf "  }%s\n" (trailing_comma i (List.length sources))
+  ) sources;
+  printf "  ],\n";
+  printf "  \"templates\": [\n";
+  iteri (
+    fun i (name, { Index_parser.printable_name = printable_name;
+                   arch = arch;
+                   size = size;
+                   compressed_size = compressed_size;
+                   notes = notes;
+                   aliases = aliases;
+                   hidden = hidden }) ->
+      printf "  {\n";
+      printf "    \"os-version\": \"%s\",\n" name;
+      json_optional_printf_string "full-name" printable_name;
+      printf "    \"arch\": \"%s\",\n" arch;
+      printf "    \"size\": %Ld,\n" size;
+      json_optional_printf_int64 "compressed-size" compressed_size;
+      print_notes notes;
+      json_optional_printf_stringlist "aliases" aliases;
+      printf "    \"hidden\": %s\n" (json_string_of_bool hidden);
+      printf "  }%s\n" (trailing_comma i (List.length index))
+  ) index;
+  printf "  ]\n";
+ printf "}\n"

builder/list_entries.mli

@@ -16,4 +16,4 @@
  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  *)
 
-val list_entries : ?list_long:bool -> source:string -> Index_parser.index -> unit
+val list_entries : list_format:([ `Short | `Long | `Json ]) -> sources:(string * Sigchecker.gpgkey_type * Downloader.proxy_mode) list -> Index_parser.index -> unit

builder/paths.ml

@@ -0,0 +1,41 @@
+(* virt-builder
+ * Copyright (C) 2014 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+open Common_utils
+
+let xdg_cache_home =
+  try Some (Sys.getenv "XDG_CACHE_HOME" // "virt-builder")
+  with Not_found ->
+    try Some (Sys.getenv "HOME" // ".cache" // "virt-builder")
+    with Not_found ->
+      None (* no cache directory *)
+
+let xdg_config_home ~prog =
+  try Some (Sys.getenv "XDG_CONFIG_HOME" // prog)
+  with Not_found ->
+    try Some (Sys.getenv "HOME" // ".config" // prog)
+    with Not_found ->
+      None (* no config directory *)
+
+let xdg_config_dirs ~prog =
+  let dirs =
+    try Sys.getenv "XDG_CONFIG_DIRS"
+    with Not_found -> "/etc/xdg" in
+  let dirs = string_nsplit ":" dirs in
+  let dirs = List.filter (fun x -> x <> "") dirs in
+  List.map (fun x -> x // prog) dirs

builder/pxzcat-c.c

@@ -0,0 +1,668 @@
+/* virt-builder
+ * Copyright (C) 2013 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#include <config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+
+#include <pthread.h>
+
+#include <caml/alloc.h>
+#include <caml/fail.h>
+#include <caml/memory.h>
+#include <caml/mlvalues.h>
+
+#include "ignore-value.h"
+
+#if HAVE_LIBLZMA
+#include <lzma.h>
+#endif
+
+#ifdef HAVE_CAML_UNIXSUPPORT_H
+#include <caml/unixsupport.h>
+#else
+#define Nothing ((value) 0)
+extern void unix_error (int errcode, char * cmdname, value arg) Noreturn;
+#endif
+
+#if defined (HAVE_LIBLZMA) && \
+  defined (HAVE_LZMA_INDEX_STREAM_FLAGS) && \
+  defined (HAVE_LZMA_INDEX_STREAM_PADDING)
+#define PARALLEL_XZCAT 1
+#else
+#define PARALLEL_XZCAT 0
+#endif
+
+value
+virt_builder_using_parallel_xzcat (value unitv)
+{
+  return PARALLEL_XZCAT ? Val_true : Val_false;
+}
+
+#if PARALLEL_XZCAT
+static void pxzcat (value filenamev, value outputfilev, unsigned nr_threads);
+#endif /* PARALLEL_XZCAT */
+
+value
+virt_builder_pxzcat (value inputfilev, value outputfilev)
+{
+  CAMLparam2 (inputfilev, outputfilev);
+
+#if PARALLEL_XZCAT
+
+  /* Parallel implementation of xzcat (pxzcat). */
+  /* XXX Make number of threads configurable? */
+  long i;
+  unsigned nr_threads;
+
+  i = sysconf (_SC_NPROCESSORS_ONLN);
+  if (i <= 0) {
+    perror ("could not get number of cores");
+    i = 1;
+  }
+  nr_threads = (unsigned) i;
+
+  /* NB: This might throw an exception if something fails.  If it
+   * does, this function won't return as a regular C function.
+   */
+  pxzcat (inputfilev, outputfilev, nr_threads);
+
+#else /* !PARALLEL_XZCAT */
+
+  /* Fallback: use regular xzcat. */
+  int fd;
+  pid_t pid;
+  int status;
+
+  fd = open (String_val (outputfilev), O_WRONLY|O_CREAT|O_TRUNC|O_NOCTTY, 0666);
+  if (fd == -1)
+    unix_error (errno, "open", outputfilev);
+
+  pid = fork ();
+  if (pid == -1) {
+    int err = errno;
+    close (fd);
+    unix_error (err, "fork", Nothing);
+  }
+
+  if (pid == 0) {               /* child - run xzcat */
+    dup2 (fd, 1);
+    execlp (XZCAT, XZCAT, String_val (inputfilev), NULL);
+    perror (XZCAT);
+    _exit (EXIT_FAILURE);
+  }
+
+  close (fd);
+
+  if (waitpid (pid, &status, 0) == -1)
+    unix_error (errno, "waitpid", Nothing);
+  if (!WIFEXITED (status) || WEXITSTATUS (status) != 0)
+    caml_failwith (XZCAT " program failed, see earlier error messages");
+
+#endif /* !PARALLEL_XZCAT */
+
+  CAMLreturn (Val_unit);
+}
+
+#if PARALLEL_XZCAT
+
+#define DEBUG 0
+
+#if DEBUG
+#define debug(fs,...) fprintf (stderr, "pxzcat: debug: " fs "\n", ## __VA_ARGS__)
+#else
+#define debug(fs,...) /* nothing */
+#endif
+
+/* Size of buffers used in decompression loop. */
+#define BUFFER_SIZE (64*1024)
+
+#define XZ_HEADER_MAGIC     "\xfd" "7zXZ\0"
+#define XZ_HEADER_MAGIC_LEN 6
+#define XZ_FOOTER_MAGIC     "YZ"
+#define XZ_FOOTER_MAGIC_LEN 2
+
+static int check_header_magic (int fd);
+static lzma_index *parse_indexes (value filenamev, int fd);
+static void iter_blocks (lzma_index *idx, unsigned nr_threads, value filenamev, int fd, value outputfilev, int ofd);
+
+static void
+pxzcat (value filenamev, value outputfilev, unsigned nr_threads)
+{
+  int fd, ofd;
+  uint64_t size;
+  lzma_index *idx;
+
+  /* Open the file. */
+  fd = open (String_val (filenamev), O_RDONLY);
+  if (fd == -1)
+    unix_error (errno, "open", filenamev);
+
+  /* Check file magic. */
+  if (!check_header_magic (fd)) {
+    close (fd);
+    caml_invalid_argument ("input file is not an xz file");
+  }
+
+  /* Read and parse the indexes. */
+  idx = parse_indexes (filenamev, fd);
+
+  /* Get the file uncompressed size, create the output file. */
+  size = lzma_index_uncompressed_size (idx);
+  debug ("uncompressed size = %" PRIu64 " bytes", size);
+
+  /* Avoid annoying ext4 auto_da_alloc which causes a flush on close
+   * unless we are very careful about not truncating a regular file
+   * from non-zero size to zero size.  (Thanks Eric Sandeen)
+   */
+  ofd = open (String_val (outputfilev), O_WRONLY|O_CREAT|O_NOCTTY, 0644);
+  if (ofd == -1) {
+    int err = errno;
+    close (fd);
+    unix_error (err, "open", outputfilev);
+  }
+
+  if (ftruncate (ofd, 1) == -1) {
+    int err = errno;
+    close (fd);
+    unix_error (err, "ftruncate", outputfilev);
+  }
+
+  if (lseek (ofd, 0, SEEK_SET) == -1) {
+    int err = errno;
+    close (fd);
+    unix_error (err, "lseek", outputfilev);
+  }
+
+  if (write (ofd, "\0", 1) == -1) {
+    int err = errno;
+    close (fd);
+    unix_error (err, "write", outputfilev);
+  }
+
+  if (ftruncate (ofd, size) == -1) {
+    int err = errno;
+    close (fd);
+    unix_error (err, "ftruncate", outputfilev);
+  }
+
+  /* Tell the kernel we won't read the output file. */
+  ignore_value (posix_fadvise (fd, 0, 0, POSIX_FADV_RANDOM|POSIX_FADV_DONTNEED));
+
+  /* Iterate over blocks. */
+  iter_blocks (idx, nr_threads, filenamev, fd, outputfilev, ofd);
+
+  lzma_index_end (idx, NULL);
+
+  if (close (fd) == -1)
+    unix_error (errno, "close", filenamev);
+}
+
+static int
+check_header_magic (int fd)
+{
+  char buf[XZ_HEADER_MAGIC_LEN];
+
+  if (lseek (fd, 0, SEEK_SET) == -1)
+    return 0;
+  if (read (fd, buf, XZ_HEADER_MAGIC_LEN) != XZ_HEADER_MAGIC_LEN)
+    return 0;
+  if (memcmp (buf, XZ_HEADER_MAGIC, XZ_HEADER_MAGIC_LEN) != 0)
+    return 0;
+  return 1;
+}
+
+/* For explanation of this function, see src/xz/list.c:parse_indexes
+ * in the xz sources.
+ */
+static lzma_index *
+parse_indexes (value filenamev, int fd)
+{
+  lzma_ret r;
+  off_t pos, index_size;
+  uint8_t footer[LZMA_STREAM_HEADER_SIZE];
+  uint8_t header[LZMA_STREAM_HEADER_SIZE];
+  lzma_stream_flags footer_flags;
+  lzma_stream_flags header_flags;
+  lzma_stream strm = LZMA_STREAM_INIT;
+  ssize_t n;
+  lzma_index *combined_index = NULL;
+  lzma_index *this_index = NULL;
+  lzma_vli stream_padding = 0;
+  size_t nr_streams = 0;
+
+  /* Check file size is a multiple of 4 bytes. */
+  pos = lseek (fd, 0, SEEK_END);
+  if (pos == (off_t) -1)
+    unix_error (errno, "lseek", filenamev);
+
+  if ((pos & 3) != 0)
+    caml_invalid_argument ("input not an xz file: size is not a multiple of 4 bytes");
+
+  /* Jump backwards through the file identifying each stream. */
+  while (pos > 0) {
+    debug ("looping through streams: pos = %" PRIu64, (uint64_t) pos);
+
+    if (pos < LZMA_STREAM_HEADER_SIZE)
+      caml_invalid_argument ("corrupted xz file");
+
+    if (lseek (fd, -LZMA_STREAM_HEADER_SIZE, SEEK_CUR) == -1)
+      unix_error (errno, "lseek", filenamev);
+
+    if (read (fd, footer, LZMA_STREAM_HEADER_SIZE) != LZMA_STREAM_HEADER_SIZE)
+      unix_error (errno, "read", filenamev);
+
+    /* Skip stream padding. */
+    if (footer[8] == 0 && footer[9] == 0 &&
+        footer[10] == 0 && footer[11] == 0) {
+      stream_padding += 4;
+      pos -= 4;
+      continue;
+    }
+
+    pos -= LZMA_STREAM_HEADER_SIZE;
+    nr_streams++;
+
+    debug ("decode stream footer at pos = %" PRIu64, (uint64_t) pos);
+
+    /* Does the stream footer look reasonable? */
+    r = lzma_stream_footer_decode (&footer_flags, footer);
+    if (r != LZMA_OK) {
+      fprintf (stderr, "invalid stream footer - error %d\n", r);
+      caml_invalid_argument ("invalid stream footer");
+    }
+
+    debug ("backward_size = %" PRIu64, (uint64_t) footer_flags.backward_size);
+    index_size = footer_flags.backward_size;
+    if (pos < index_size + LZMA_STREAM_HEADER_SIZE)
+      caml_invalid_argument ("invalid stream footer");
+
+    pos -= index_size;
+    debug ("decode index at pos = %" PRIu64, (uint64_t) pos);
+
+    /* Seek backwards to the index of this stream. */
+    if (lseek (fd, pos, SEEK_SET) == -1)
+      unix_error (errno, "lseek", filenamev);
+
+    /* Decode the index. */
+    r = lzma_index_decoder (&strm, &this_index, UINT64_MAX);
+    if (r != LZMA_OK) {
+      fprintf (stderr, "invalid stream index - error %d\n", r);
+      caml_invalid_argument ("invalid stream index");
+    }
+
+    do {
+      uint8_t buf[BUFSIZ];
+
+      strm.avail_in = index_size;
+      if (strm.avail_in > BUFSIZ)
+        strm.avail_in = BUFSIZ;
+
+      n = read (fd, &buf, strm.avail_in);
+      if (n == -1)
+        unix_error (errno, "read", filenamev);
+
+      index_size -= strm.avail_in;
+
+      strm.next_in = buf;
+      r = lzma_code (&strm, LZMA_RUN);
+    } while (r == LZMA_OK);
+
+    if (r != LZMA_STREAM_END) {
+      fprintf (stderr, "could not parse index - error %d\n", r);
+      caml_invalid_argument ("could not parse index");
+    }
+
+    pos -= lzma_index_total_size (this_index) + LZMA_STREAM_HEADER_SIZE;
+
+    debug ("decode stream header at pos = %" PRIu64, (uint64_t) pos);
+
+    /* Read and decode the stream header. */
+    if (lseek (fd, pos, SEEK_SET) == -1)
+      unix_error (errno, "lseek", filenamev);
+
+    if (read (fd, header, LZMA_STREAM_HEADER_SIZE) != LZMA_STREAM_HEADER_SIZE)
+      unix_error (errno, "read stream header", filenamev);
+
+    r = lzma_stream_header_decode (&header_flags, header);
+    if (r != LZMA_OK) {
+      fprintf (stderr, "invalid stream header - error %d\n", r);
+      caml_invalid_argument ("invalid stream header");
+    }
+
+    /* Header and footer of the stream should be equal. */
+    r = lzma_stream_flags_compare (&header_flags, &footer_flags);
+    if (r != LZMA_OK) {
+      fprintf (stderr, "header and footer of stream are not equal - error %d\n",
+               r);
+      caml_invalid_argument ("header and footer of stream are not equal");
+    }
+
+    /* Store the decoded stream flags in this_index. */
+    r = lzma_index_stream_flags (this_index, &footer_flags);
+    if (r != LZMA_OK) {
+      fprintf (stderr, "cannot read stream_flags from index - error %d\n", r);
+      caml_invalid_argument ("cannot read stream_flags from index");
+    }
+
+    /* Store the amount of stream padding so far.  Needed to calculate
+     * compressed offsets correctly in multi-stream files.
+     */
+    r = lzma_index_stream_padding (this_index, stream_padding);
+    if (r != LZMA_OK) {
+      fprintf (stderr, "cannot set stream_padding in index - error %d\n", r);
+      caml_invalid_argument ("cannot set stream_padding in index");
+    }
+
+    if (combined_index != NULL) {
+      r = lzma_index_cat (this_index, combined_index, NULL);
+      if (r != LZMA_OK) {
+        fprintf (stderr, "cannot combine indexes - error %d\n", r);
+        caml_invalid_argument ("cannot combine indexes");
+      }
+    }
+
+    combined_index = this_index;
+    this_index = NULL;
+  }
+
+  lzma_end (&strm);
+
+  return combined_index;
+}
+
+/* Return true iff the buffer is all zero bytes.
+ *
+ * Note that gcc is smart enough to optimize this properly:
+ * http://stackoverflow.com/questions/1493936/faster-means-of-checking-for-an-empty-buffer-in-c/1493989#1493989
+ */
+static inline int
+is_zero (const unsigned char *buffer, size_t size)
+{
+  size_t i;
+
+  for (i = 0; i < size; ++i) {
+    if (buffer[i] != 0)
+      return 0;
+  }
+
+  return 1;
+}
+
+struct global_state {
+  /* Current iterator.  Threads update this, but it is protected by a
+   * mutex, and each thread takes a copy of it when working on it.
+   */
+  lzma_index_iter iter;
+  lzma_bool iter_finished;
+  pthread_mutex_t iter_mutex;
+
+  /* Note that all threads are accessing these fds, so you have
+   * to use pread/pwrite instead of lseek!
+   */
+
+  /* Input file. */
+  const char *filename;
+  int fd;
+
+  /* Output file. */
+  const char *outputfile;
+  int ofd;
+};
+
+struct per_thread_state {
+  unsigned thread_num;
+  struct global_state *global;
+  int status;
+};
+
+/* Create threads to iterate over the blocks and uncompress. */
+static void *worker_thread (void *vp);
+
+static void
+iter_blocks (lzma_index *idx, unsigned nr_threads,
+             value filenamev, int fd, value outputfilev, int ofd)
+{
+  struct global_state global;
+  struct per_thread_state per_thread[nr_threads];
+  pthread_t thread[nr_threads];
+  unsigned u, nr_errors;
+  int err;
+  void *status;
+
+  lzma_index_iter_init (&global.iter, idx);
+  global.iter_finished = 0;
+  err = pthread_mutex_init (&global.iter_mutex, NULL);
+  if (err != 0)
+    unix_error (err, "pthread_mutex_init", Nothing);
+
+  global.filename = String_val (filenamev);
+  global.fd = fd;
+  global.outputfile = String_val (outputfilev);
+  global.ofd = ofd;
+
+  for (u = 0; u < nr_threads; ++u) {
+    per_thread[u].thread_num = u;
+    per_thread[u].global = &global;
+  }
+
+  /* Start the threads. */
+  for (u = 0; u < nr_threads; ++u) {
+    err = pthread_create (&thread[u], NULL, worker_thread, &per_thread[u]);
+    if (err != 0)
+      unix_error (err, "pthread_create", Nothing);
+  }
+
+  /* Wait for the threads to exit. */
+  nr_errors = 0;
+  for (u = 0; u < nr_threads; ++u) {
+    err = pthread_join (thread[u], &status);
+    if (err != 0) {
+      fprintf (stderr, "pthread_join (%u): %s\n", u, strerror (err));
+      nr_errors++;
+    }
+    if (*(int *)status == -1)
+      nr_errors++;
+  }
+
+  if (nr_errors > 0)
+    caml_invalid_argument ("some threads failed, see earlier errors");
+}
+
+/* Iterate over the blocks and uncompress. */
+static void *
+worker_thread (void *vp)
+{
+  struct per_thread_state *state = vp;
+  struct global_state *global = state->global;
+  lzma_index_iter iter;
+  int err;
+  off_t position, oposition;
+  uint8_t header[LZMA_BLOCK_HEADER_SIZE_MAX];
+  ssize_t n;
+  lzma_block block;
+  lzma_filter filters[LZMA_FILTERS_MAX + 1];
+  lzma_ret r;
+  lzma_stream strm = LZMA_STREAM_INIT;
+  uint8_t buf[BUFFER_SIZE];
+  unsigned char outbuf[BUFFER_SIZE];
+  size_t i;
+  lzma_bool iter_finished;
+
+  state->status = -1;
+
+  for (;;) {
+    /* Get the next block. */
+    err = pthread_mutex_lock (&global->iter_mutex);
+    if (err != 0) abort ();
+    iter_finished = global->iter_finished;
+    if (!iter_finished) {
+      iter_finished = global->iter_finished =
+        lzma_index_iter_next (&global->iter, LZMA_INDEX_ITER_NONEMPTY_BLOCK);
+      if (!iter_finished)
+        /* Take a local copy of this iterator since another thread will
+         * update the global version.
+         */
+        iter = global->iter;
+    }
+    err = pthread_mutex_unlock (&global->iter_mutex);
+    if (err != 0) abort ();
+    if (iter_finished)
+      break;
+
+    /* Read the block header.  Start by reading a single byte which
+     * tell us how big the block header is.
+     */
+    position = iter.block.compressed_file_offset;
+    n = pread (global->fd, header, 1, position);
+    if (n == 0) {
+      fprintf (stderr,
+               "%s: read: unexpected end of file reading block header byte\n",
+               global->filename);
+      return &state->status;
+    }
+    if (n == -1) {
+      perror (String_val (global->filename));
+      return &state->status;
+    }
+    position++;
+
+    if (header[0] == '\0') {
+      fprintf (stderr,
+               "%s: read: unexpected invalid block in file, header[0] = 0\n",
+               global->filename);
+      return &state->status;
+    }
+
+    block.version = 0;
+    block.check = iter.stream.flags->check;
+    block.filters = filters;
+    block.header_size = lzma_block_header_size_decode (header[0]);
+
+    /* Now read and decode the block header. */
+    n = pread (global->fd, &header[1], block.header_size-1, position);
+    if (n >= 0 && n != block.header_size-1) {
+      fprintf (stderr,
+               "%s: read: unexpected end of file reading block header\n",
+               global->filename);
+      return &state->status;
+    }
+    if (n == -1) {
+      perror (global->filename);
+      return &state->status;
+    }
+    position += n;
+
+    r = lzma_block_header_decode (&block, NULL, header);
+    if (r != LZMA_OK) {
+      fprintf (stderr, "%s: invalid block header (error %d)\n",
+               global->filename, r);
+      return &state->status;
+    }
+
+    /* What this actually does is it checks that the block header
+     * matches the index.
+     */
+    r = lzma_block_compressed_size (&block, iter.block.unpadded_size);
+    if (r != LZMA_OK) {
+      fprintf (stderr,
+               "%s: cannot calculate compressed size (error %d)\n",
+               global->filename, r);
+      return &state->status;
+    }
+
+    /* Where we will start writing to. */
+    oposition = iter.block.uncompressed_file_offset;
+
+    /* Read the block data and uncompress it. */
+    r = lzma_block_decoder (&strm, &block);
+    if (r != LZMA_OK) {
+      fprintf (stderr, "%s: invalid block (error %d)\n", global->filename, r);
+      return &state->status;
+    }
+
+    strm.next_in = NULL;
+    strm.avail_in = 0;
+    strm.next_out = outbuf;
+    strm.avail_out = sizeof outbuf;
+
+    for (;;) {
+      lzma_action action = LZMA_RUN;
+
+      if (strm.avail_in == 0) {
+        strm.next_in = buf;
+        n = pread (global->fd, buf, sizeof buf, position);
+        if (n == -1) {
+          perror (global->filename);
+          return &state->status;
+        }
+        position += n;
+        strm.avail_in = n;
+        if (n == 0)
+          action = LZMA_FINISH;
+      }
+
+      r = lzma_code (&strm, action);
+
+      if (strm.avail_out == 0 || r == LZMA_STREAM_END) {
+        size_t wsz = sizeof outbuf - strm.avail_out;
+
+        /* Don't write if the block is all zero, to preserve output file
+         * sparseness.  However we have to update oposition.
+         */
+        if (!is_zero (outbuf, wsz)) {
+          if (pwrite (global->ofd, outbuf, wsz, oposition) != wsz) {
+            /* XXX Handle short writes. */
+            perror (global->filename);
+            return &state->status;
+          }
+        }
+        oposition += wsz;
+
+        strm.next_out = outbuf;
+        strm.avail_out = sizeof outbuf;
+      }
+
+      if (r == LZMA_STREAM_END)
+        break;
+      if (r != LZMA_OK) {
+        fprintf (stderr,
+                 "%s: could not parse block data (error %d)\n",
+                 global->filename, r);
+        return &state->status;
+      }
+    }
+
+    lzma_end (&strm);
+
+    for (i = 0; filters[i].id != LZMA_VLI_UNKNOWN; ++i)
+      free (filters[i].options);
+  }
+
+  state->status = 0;
+  return &state->status;
+}
+
+#endif /* PARALLEL_XZCAT */

builder/pxzcat.ml

@@ -0,0 +1,20 @@
+(* virt-builder
+ * Copyright (C) 2013 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+external pxzcat : string -> string -> unit = "virt_builder_pxzcat"
+external using_parallel_xzcat : unit -> bool = "virt_builder_using_parallel_xzcat" "noalloc"

builder/pxzcat.mli

@@ -0,0 +1,34 @@
+(* virt-builder
+ * Copyright (C) 2013 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+(** {1 Parallel xzcat (or fall back to regular xzcat).}
+
+    Eventually regular xzcat will be able to work in parallel and this
+    code can go away.
+*)
+
+val pxzcat : string -> string -> unit
+    (** [pxzcat input output] uncompresses the file [input] to the file
+        [output].  The input and output must both be seekable.
+
+        If liblzma was found at compile time, this uses an internal
+        implementation of parallel xzcat.  Otherwise regular xzcat is
+        used. *)
+
+val using_parallel_xzcat : unit -> bool
+(** Returns [true] iff the implementation uses parallel xzcat. *)

builder/setlocale-c.c

@@ -0,0 +1,59 @@
+/* virt-builder
+ * Copyright (C) 2014 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#include <config.h>
+
+#include <locale.h>
+
+#include <caml/alloc.h>
+#include <caml/fail.h>
+#include <caml/memory.h>
+#include <caml/mlvalues.h>
+
+static const int lc_string_table[7] = {
+  LC_ALL,
+  LC_CTYPE,
+  LC_NUMERIC,
+  LC_TIME,
+  LC_COLLATE,
+  LC_MONETARY,
+  LC_MESSAGES
+};
+
+#define Val_none (Val_int (0))
+
+value
+virt_builder_setlocale (value val_category, value val_name)
+{
+  CAMLparam2 (val_category, val_name);
+  CAMLlocal2 (rv, rv2);
+  char *ret, *locstring;
+  int category;
+
+  category = lc_string_table[Int_val (val_category)];
+  locstring = val_name == Val_none ? NULL : String_val (Field (val_name, 0));
+  ret = setlocale (category, locstring);
+  if (ret) {
+    rv2 = caml_copy_string (ret);
+    rv = caml_alloc (1, 0);
+    Store_field (rv, 0, rv2);
+  } else
+    rv = Val_none;
+
+  CAMLreturn (rv);
+}

builder/setlocale.ml

@@ -0,0 +1,29 @@
+(* virt-builder
+ * Copyright (C) 2014 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+type localecategory =
+  | LC_ALL
+  | LC_CTYPE
+  | LC_NUMERIC
+  | LC_TIME
+  | LC_COLLATE
+  | LC_MONETARY
+  | LC_MESSAGES
+;;
+
+external setlocale : localecategory -> string option -> string option = "virt_builder_setlocale"

builder/setlocale.mli

@@ -0,0 +1,30 @@
+(* virt-builder
+ * Copyright (C) 2014 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+type localecategory =
+  | LC_ALL
+  | LC_CTYPE
+  | LC_NUMERIC
+  | LC_TIME
+  | LC_COLLATE
+  | LC_MONETARY
+  | LC_MESSAGES
+;;
+
+val setlocale : localecategory -> string option -> string option
+(** [setlocale category newlocale] Tiny wrapper to the C [setlocale]. *)

builder/sigchecker.ml

@@ -24,96 +24,98 @@ open Unix
 
 let quote = Filename.quote
 
-(* These are the public key and fingerprint belonging to
- * Richard W.M. Jones who signs the templates on
- * http://libguestfs.org/download/builder.
- *)
-let default_fingerprint = "F777 4FB1 AD07 4A7E 8C87 67EA 9173 8F73 E1B7 68A0"
-let default_pubkey = "\
------BEGIN PGP PUBLIC KEY BLOCK-----
-Version: GnuPG v1.4.14 (GNU/Linux)
-
-mQINBE6UMMEBEADM811hfTulaF4JpkVpAI10FImyb4ArvOiu8NdcUwTFo+cyWno3
-U85B86H1Bsk/LgLTYtthSrTgsCtdxy+i5OaMjxZDIwKQ2+IYI3FCn9T3Mn28Idyh
-kLHzrO9ph0Dv0BNfrlDZhQEC53aAFe/QxN7+A49BNBV7D1VAOOCsHjxMEDzcZkCa
-oCrtXw1aNm2vkkj5ukbfukHAyLcQL7kow0qKPSVa1G4lfQP0WiG259Ydy+sUmbVb
-TGdb6MEC84PQRDuw6/ZeoV04tn7ZNtQEMOS0uiciHOGfr2hBxQf9VIPNrHg42yaL
-dOv51D99GuaxZ9E0HSoH/RwB1oXgd6rFdqVNYaBIQnnkwJANUEeGBArtIOZNCADT
-Bt8vkSDm+lLEAFS+V8CACyW/LMIrGCvLdHeqtoAv0GDVyR2GPxldYfdtEmCUMWcb
-Jlf71V9iAse2gUdoiHp5FfpGMkA5j7idKuxIws11XxRZJXXbBqiBqmVEAQ/v0m6p
-kdo0MYTHydmecLuUK2bAGhpysfX97EfTSrxfrYphYWjTfKRD9GrADeZNfuz1DbKs
-7LSqVaQJSjQrfgAwcnZLRaU0V4P5zxiz50gz1Aj3AZRL+Y3meZenzZTXcLFdnusg
-wUfhhCuL3tluMtEh6tznumyxb43WO1yLwj6J6LtveiuJN1Z+KSQ6OieZcwARAQAB
-tCVSaWNoYXJkIFcuTS4gSm9uZXMgPHJpY2hAYW5uZXhpYS5vcmc+iQI4BBMBAgAi
-BQJOlDDBAhsDBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRCRc49z4bdooHQY
-D/wJLklSZNyXIW+rG5sUbg7j9cTIF5p/lB9kI2yx6KodJp/2knKyvnmzz0gBw/OE
-HL4E4UW26oWKo+36I8wkBnuGa6UtANeITcJqFE19VpHEXHsxre64jNQnO8/w748W
-1ROW+Ry43xmrlRWKuCm4oPYUzlp0fq9ATAne8eblfG+NOs8DYuA8xZNQzFaI2kDC
-QLD4YoXLoNsP27Koga36b0KwxPFD9tyVZiu9XDH/3hMN7Nb15B66PFr+HcMmQ67G
-nUIN5ulcIwj38i40cyaTs1VRheOzTHXE/a6Q2AhMKiKqOoEjQ73/mV7cAVoPtM3o
-83Q/8aVKBH0bVRwAeV1tju6b14fqKoG0zNBEcXdlSkht6ScxJYIc/LPUxAMDwgSE
-OWshjmeRzKXypBbHn/DP8QVyM2gk5wY+mMSH7MpR0p/hgj+rFO8H9L7pC4dCog3E
-qzrYhRN+TaP6MPH3WkOwPH4d4IfQRFnHp+VPYPijKEiLrUl/o8k3DyAanAPBpJ/x
-na4wXAjlFBctOq6g+SrCUiHpwk7b2YNwGgr5Vl3GmZELzK/G8gg3uJYKQ9Bpv16t
-WWOz+IFiOFa0UULeo0QPmFAIMZiDojNsY1SwBKB3ZL1YWZezgMdQAbpze/IXoSt7
-zxWJoKH2jK7q9mvFiaY12l2YnKuCcegWVAViLxRpBnrbz7QmUmljaGFyZCBXLk0u
-IEpvbmVzIDxyam9uZXNAcmVkaGF0LmNvbT6JAjgEEwECACIFAk6UOQsCGwMGCwkI
-BwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJEJFzj3Pht2igIUYQAKomI0edLakahsUQ
-MxOZuhBbXJ4/VWF8bXYChDNPKvJp5nB7fBXujJ+39cIUM5fe2ViO6qSDpFC29imx
-F5pPbAqspZBPBkLLiZLji8R42hGarntdtTW0UWSBpq+nC5+G1psrnATI3uXGNxKQ
-R99c5HoMY7dBC2Y8TCGE64NINZ/XVh472s6IGLPn8MTn26YdRKC9BrVkCFMP2OBr
-6D4IprnyTAWAzb68ew20QmyWO+NBi9MplaDNQVl8PIOgfpyWlkgX1z9m67pcSDkw
-46hksp0yuOD1VwR4iVZ2/CmIsGRUlx41vWD6BIp9KxKyDIU1CYTRhq72dahHsl/8
-BjCndV5PO0GphqfCzmCv4DXjUwmrMTbH/GFnt5rfwcMcXUgcK0vV9vQ2SOU56Zd1
-fb27ZCFJKZc0Fu8krwFldCp/NYILf6ogUL/C1hfuCGSSuyDVY16Gg3dla1x+6zpF
-asnWQlaw8xT5LlMWvTZs5WsoSVHu7dVZWlgxINP++hlZrTz/S8l38yyQ15YFFl3W
-9M7dzkegOeDTPfx6B89WgfvfJjA/D0/FYxxWPXEtrn9DlJ4daEJqNsrvfLErz9R8
-4IQmfmhR93j+rdotner+6keC/wVByEfbW1wmXtmFKXQ6srdpj8VKRFrvkyXVgepM
-DypLgRH2v7lL2kdWhUu2y4EAgrwzuQINBE6UMMEBEADxQxMgUuDrw5GT4tqARTPI
-SSdNcUsRxRhVA8srYOyECliE+B3TwcRDFBs+MyPFJVEuX8fi4eGj/AK5t1GHerfk
-orUGlz72q4c7LLhkfZrsuJbk2dgkjvldKJnIazQJa6epGLqdsE5RlmSgwedIbtMd
-naGJBQH8aKP/Wi1+wUxsm5N3p7+R2WRx48VfpEhYB+Zf/FkFm1Ycjwh57KQ0+OHw
-ykf8VfMisxuH30tDxOCV+VptWKfOF2rDNdaNPWhij2YIjhJXRpkuRR+1PpI4jLaD
-JxcVZmG/0zucacupUN2g5OUH59ySU/totD6YMnmp3FONoyF1uIEJo6Vs30npHGkO
-XgBo3Pxt7oLJeykLPtdSLgm3cwXIYMWarVsAkKNXitQIVGpVRLeaK373VwmXFqoi
-M2SMHeawTUdOORFjpQzkknlJWM1TmUVtHHKt8Pl9+/5+wXKyt2IDdcUkMrB6K5qF
-fb7EwVhoI8ehJQK+eeDCjFwCAiwB3iV8JlyW+tEU7JuyXOQlwY1VWm/WqMD8gaRi
-rT+RFDFliZ3tQbW2pqUoZBROV5HN4tieDfwxGKCvk6Tsdb30zA9DPQp93+238bYf
-312sg9R+CD0AqxoxFG5FJu4HShcPRrPnYtRZqKRe40GDWvBEArXZprwL1qrP+Kl/
-mRrEQpxAGIoFG8HbVvD3EQARAQABiQIfBBgBAgAJBQJOlDDBAhsMAAoJEJFzj3Ph
-t2igSLQP/2uIrAY2CDr0kWBJiD3TztiHy8IdxwUpyTBTebwmAbi44/EvtJfIisrG
-YjKIEv/w0E61gO7O1JBG4+IG93W+v9fTT/e39JMyxsYqoZZHUhP11Okx5grDS5b0
-O8VXOmXVRMdVNfstRBr10HD9uNDq7ruKD18TxYTwN0GPD4gj1dbHQDR77Tr5cyBs
-6Ou5PBOH4r3qcqf/cJUSMeUUu75xLwixux6E7tD2S+t6F07wlWxntUcPtzyAHj20
-J89orUC+dT6r6MypBoI0jdJCp9JPGtR7i+fE5Gm4E5+AUSubLPtZGRY9Um2eMoS2
-DnQpGOKx1VvsixR/Kw44j2tRAvmYMS4iDKcuZU+nZ+xokAgObILj/b9n/Qe2/fXy
-CFdcgSvbm+dV1fZxsdMF/P9OU8aqdT9A9Fv5y+cDMEg4DVnhwMJTxGh/TCkw/H+A
-frHEtRc98lSQN5odpITNG17mG6JOdHM+wA57qHH0uy4+5RsbyAJahcdBcmObK/RF
-i4WZlThpbHftX5O/LH98aYQ2fJayIxv1EAjzOBOQ0MfBHI0KCJR1pysEisX28sJA
-Ic73gnJJ3BLZbqfBRgxjNMNroxC+5Tw6uPGFHa3YnuIAxxw0HcDVZ9vnTWBWFPGw
-ZvXkQ3FVJwZoLmHw47vvlVpLD/4gi1SuHWieRvZ+UdDq00E348pm
-=neBW
------END PGP PUBLIC KEY BLOCK-----
-"
-let key_imported = ref false
+type gpgkey_type =
+  | No_Key
+  | Fingerprint of string
+  | KeyFile of string
 
 type t = {
-  debug : bool;
+  verbose : bool;
   gpg : string;
   fingerprint : string;
   check_signature : bool;
+  gpghome : string;
 }
 
-let create ~debug ~gpg ?(fingerprint = default_fingerprint) ~check_signature =
+(* Import the specified key file. *)
+let import_keyfile ~gpg ~gpghome ~verbose keyfile =
+  let status_file = Filename.temp_file "vbstat" ".txt" in
+  unlink_on_exit status_file;
+  let cmd = sprintf "%s --homedir %s --status-file %s --import %s%s"
+    gpg gpghome (quote status_file) (quote keyfile)
+    (if verbose then "" else " >/dev/null 2>&1") in
+  if verbose then eprintf "%s\n%!" cmd;
+  let r = Sys.command cmd in
+  if r <> 0 then (
+    eprintf (f_"virt-builder: error: could not import public key\nUse the '-v' option and look for earlier error messages.\n");
+    exit 1
+  );
+  status_file
+
+let rec create ~verbose ~gpg ~gpgkey ~check_signature =
+  (* Create a temporary directory for gnupg. *)
+  let tmpdir = Mkdtemp.temp_dir "vb.gpghome." "" in
+  rmdir_on_exit tmpdir;
+  (* Make sure we have no check_signature=true with no actual key. *)
+  let check_signature, gpgkey =
+    match check_signature, gpgkey with
+    | true, No_Key -> false, No_Key
+    | x, y -> x, y in
+  let fingerprint =
+    if check_signature then (
+      (* Run gpg so it can setup its own home directory, failing if it
+       * cannot.
+       *)
+      let cmd = sprintf "%s --homedir %s --list-keys%s"
+        gpg tmpdir (if verbose then "" else " >/dev/null 2>&1") in
+      if verbose then eprintf "%s\n%!" cmd;
+      let r = Sys.command cmd in
+      if r <> 0 then (
+        eprintf (f_"virt-builder: error: GPG failure: could not run GPG the first time\nUse the '-v' option and look for earlier error messages.\n");
+        exit 1
+      );
+      match gpgkey with
+      | No_Key ->
+        assert false
+      | KeyFile kf ->
+        let status_file = import_keyfile gpg tmpdir verbose kf in
+        let status = read_whole_file status_file in
+        let status = string_nsplit "\n" status in
+        let fingerprint = ref "" in
+        List.iter (
+          fun line ->
+            let line = string_nsplit " " line in
+            match line with
+            | "[GNUPG:]" :: "IMPORT_OK" :: _ :: fp :: _ -> fingerprint := fp
+            | _ -> ()
+        ) status;
+        !fingerprint
+      | Fingerprint fp ->
+        let filename = Filename.temp_file "vbpubkey" ".asc" in
+        unlink_on_exit filename;
+        let cmd = sprintf "%s --yes --armor --output %s --export %s%s"
+          gpg (quote filename) (quote fp)
+          (if verbose then "" else " >/dev/null 2>&1") in
+        if verbose then eprintf "%s\n%!" cmd;
+        let r = Sys.command cmd in
+        if r <> 0 then (
+          eprintf (f_"virt-builder: error: could not export public key\nUse the '-v' option and look for earlier error messages.\n");
+          exit 1
+        );
+        ignore (import_keyfile gpg tmpdir verbose filename);
+        fp
+    ) else
+      "" in
   {
-    debug = debug;
+    verbose = verbose;
     gpg = gpg;
     fingerprint = fingerprint;
     check_signature = check_signature;
+    gpghome = tmpdir;
   }
 
 (* Compare two strings of hex digits ignoring whitespace and case. *)
-let rec equal_fingerprints fp1 fp2 =
+and equal_fingerprints fp1 fp2 =
   let len1 = String.length fp1 and len2 = String.length fp2 in
   let rec loop i j =
     if i = len1 && j = len2 then true (* match! *)
@@ -154,14 +156,14 @@ and verify_detached t filename sigfile =
   )
 
 and do_verify t args =
-  import_key t;
-
   let status_file = Filename.temp_file "vbstat" ".txt" in
+  unlink_on_exit status_file;
   let cmd =
-    sprintf "%s --verify%s --status-file %s %s"
-        t.gpg (if t.debug then "" else " -q --logger-file /dev/null")
+    sprintf "%s --homedir %s --verify%s --status-file %s %s"
+        t.gpg t.gpghome
+        (if t.verbose then "" else " -q --logger-file /dev/null")
         (quote status_file) args in
-  if t.debug then eprintf "%s\n%!" cmd;
+  if t.verbose then eprintf "%s\n%!" cmd;
   let r = Sys.command cmd in
   if r <> 0 then (
     eprintf (f_"virt-builder: error: GPG failure: could not verify digital signature of file\nTry:\n - Use the '-v' option and look for earlier error messages.\n - Delete the cache: virt-builder --delete-cache\n - Check no one has tampered with the website or your network!\n");
@@ -170,7 +172,6 @@ and do_verify t args =
 
   (* Check the fingerprint is who it should be. *)
   let status = read_whole_file status_file in
-  unlink status_file;
 
   let status = string_nsplit "\n" status in
   let fingerprint = ref "" in
@@ -183,27 +184,36 @@ and do_verify t args =
   ) status;
 
   if not (equal_fingerprints !fingerprint t.fingerprint) then (
-    eprintf (f_"virt-builder: error: fingerprint of signature does not match the expected fingerprint!\n     found fingerprint: %s\n  expected fingerprint: %s\n")
+    eprintf (f_"virt-builder: error: fingerprint of signature does not match the expected fingerprint!\n  found fingerprint: %s\n  expected fingerprint: %s\n")
       !fingerprint t.fingerprint;
     exit 1
   )
 
-(* Import the default public key, if it's the default fingerprint. *)
-and import_key t =
-  if not !key_imported && equal_fingerprints t.fingerprint default_fingerprint
-  then (
-    let filename, chan = Filename.open_temp_file "vbpubkey" ".asc" in
-    output_string chan default_pubkey;
-    close_out chan;
+type csum_t = SHA512 of string
 
-    let cmd = sprintf "%s --import %s%s"
-      t.gpg (quote filename)
-      (if t.debug then "" else " >/dev/null 2>&1") in
-    let r = Sys.command cmd in
-    if r <> 0 then (
-      eprintf (f_"virt-builder: error: could not import public key\nUse the '-v' option and look for earlier error messages.\n");
-      exit 1
-    );
-    unlink filename;
-    key_imported := true
+let verify_checksum t (SHA512 csum) filename =
+  let csum_file = Filename.temp_file "vbcsum" ".txt" in
+  unlink_on_exit csum_file;
+  let cmd = sprintf "sha512sum %s | awk '{print $1}' > %s"
+    (quote filename) (quote csum_file) in
+  if t.verbose then eprintf "%s\n%!" cmd;
+  let r = Sys.command cmd in
+  if r <> 0 then (
+    eprintf (f_"virt-builder: error: could not run sha512sum command to verify checksum\n");
+    exit 1
+  );
+
+  let csum_actual = read_whole_file csum_file in
+
+  let csum_actual =
+    let len = String.length csum_actual in
+    if len > 0 && csum_actual.[len-1] = '\n' then
+      String.sub csum_actual 0 (len-1)
+    else
+      csum_actual in
+
+  if csum <> csum_actual then (
+    eprintf (f_"virt-builder: error: checksum of template did not match the expected checksum!\n  found checksum: %s\n  expected checksum: %s\nTry:\n - Use the '-v' option and look for earlier error messages.\n - Delete the cache: virt-builder --delete-cache\n - Check no one has tampered with the website or your network!\n")
+      csum_actual csum;
+    exit 1
   )

builder/sigchecker.mli

@@ -18,7 +18,12 @@
 
 type t
 
-val create : debug:bool -> gpg:string -> ?fingerprint:string -> check_signature:bool -> t
+type gpgkey_type =
+  | No_Key
+  | Fingerprint of string
+  | KeyFile of string
+
+val create : verbose:bool -> gpg:string -> gpgkey:gpgkey_type -> check_signature:bool -> t
 
 val verify : t -> string -> unit
 (** Verify the file is signed (if check_signature is true). *)
@@ -26,3 +31,9 @@ val verify : t -> string -> unit
 val verify_detached : t -> string -> string option -> unit
 (** Verify the file is signed against the detached signature
     (if check_signature is true). *)
+
+type csum_t = SHA512 of string
+
+val verify_checksum : t -> csum_t -> string -> unit
+(** Verify the checksum of the file.  This is always verified even if
+    check_signature if false. *)

builder/sources.ml

@@ -0,0 +1,139 @@
+(* virt-builder
+ * Copyright (C) 2014 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+open Common_gettext.Gettext
+open Common_utils
+
+open Printf
+open Unix
+
+type source = {
+  name : string;
+  uri : string;
+  gpgkey : string option;
+  proxy : Downloader.proxy_mode;
+}
+
+module StringSet = Set.Make (String)
+
+let parse_conf ~prog ~verbose file =
+  if verbose then (
+    eprintf (f_"%s: trying to read %s\n") prog file;
+  );
+  let sections = Ini_reader.read_ini ~prog ~error_suffix:"[ignored]" file in
+
+  let sources = List.fold_right (
+    fun (n, fields) acc ->
+      let give_source n fields =
+        let fields = List.map (fun (k, sk, v) -> (k, sk), v) fields in
+        let uri =
+          try List.assoc ("uri", None) fields
+          with Not_found as ex ->
+            eprintf (f_"%s: no 'uri' entry for '%s' in %s, skipping it\n") prog n file;
+            raise ex in
+        let gpgkey =
+          let k =
+            try Some (URI.parse_uri (List.assoc ("gpgkey", None) fields)) with
+            | Not_found -> None
+            | Invalid_argument "URI.parse_uri" as ex ->
+              if verbose then (
+                eprintf (f_"%s: '%s' has invalid gpgkey URI\n") prog n;
+              );
+              raise ex in
+          match k with
+          | None -> None
+          | Some uri ->
+            (match uri.URI.protocol with
+            | "file" -> Some uri.URI.path
+            | _ ->
+              if verbose then (
+                eprintf (f_"%s: '%s' has non-local gpgkey URI\n") prog n;
+              );
+              None
+            ) in
+        let proxy =
+          try
+            (match (List.assoc ("proxy", None) fields) with
+            | "no" | "off" -> Downloader.UnsetProxy
+            | "system" -> Downloader.SystemProxy
+            | _ as proxy -> Downloader.ForcedProxy proxy
+            )
+          with
+            Not_found -> Downloader.SystemProxy in
+        {
+          name = n; uri = uri; gpgkey = gpgkey; proxy = proxy;
+        }
+      in
+      try (give_source n fields) :: acc
+      with Not_found | Invalid_argument _ -> acc
+  ) sections [] in
+
+  if verbose then (
+    eprintf (f_"%s: ... read %d sources\n") prog (List.length sources);
+  );
+
+  sources
+
+let merge_sources current_sources new_sources =
+  List.fold_right (
+    fun source acc ->
+      if List.exists (fun { name = n } -> n = source.name) acc then
+        acc
+      else
+        source :: acc
+  ) new_sources current_sources
+
+let filter_filenames filename =
+  Filename.check_suffix filename ".conf"
+
+let read_sources ~prog ~verbose =
+  let dirs = Paths.xdg_config_dirs ~prog in
+  let dirs =
+    match Paths.xdg_config_home ~prog with
+    | None -> dirs
+    | Some dir -> dir :: dirs in
+  let dirs = List.map (fun x -> x // "repos.d") dirs in
+  let fnseen = ref StringSet.empty in
+  List.fold_left (
+    fun acc dir ->
+      let files =
+        try List.filter filter_filenames (Array.to_list (Sys.readdir dir))
+        with Sys_error _ -> [] in
+      let files = List.filter (fun x -> StringSet.mem x !fnseen <> true) files in
+      List.fold_left (
+        fun acc file ->
+          try (
+            let s = merge_sources acc (parse_conf ~prog ~verbose (dir // file)) in
+            (* Add the current file name to the set only if its parsing
+             * was successful.
+             *)
+            fnseen := StringSet.add file !fnseen;
+            s
+          ) with
+          | Unix_error (code, fname, _) ->
+            if verbose then (
+              eprintf (f_"%s: file error: %s: %s\n") prog fname (error_message code)
+            );
+            acc
+          | Invalid_argument msg ->
+            if verbose then (
+              eprintf (f_"%s: internal error: invalid argument: %s\n") prog msg
+            );
+            acc
+      ) acc files
+  ) [] dirs

builder/sources.mli

@@ -0,0 +1,26 @@
+(* virt-builder
+ * Copyright (C) 2014 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+type source = {
+  name : string;
+  uri : string;
+  gpgkey : string option;
+  proxy : Downloader.proxy_mode;
+}
+
+val read_sources : prog:string -> verbose:bool -> source list

builder/test-config/virt-builder/repos.d/test-index.conf.in

@@ -0,0 +1,2 @@
+[test-index]
+uri=file://@abs_top_builddir@/builder/test-index

builder/test-index

@@ -0,0 +1,66 @@
+[phony-debian]
+name=Phony Debian
+arch=x86_64
+file=debian.xz
+format=raw
+size=536870912
+expand=/dev/sda2
+lvexpand=/dev/debian/root
+notes=Phony Debian look-alike used for testing.
+
+[phony-fedora]
+name=Phony Fedora
+arch=x86_64
+file=fedora.xz
+format=raw
+size=1073741824
+expand=/dev/sda2
+lvexpand=/dev/VG/Root
+notes=Phony Fedora look-alike used for testing.
+
+[phony-fedora-qcow2]
+name=Phony Fedora qcow2
+arch=x86_64
+file=fedora.qcow2.xz
+format=qcow2
+size=1073741824
+expand=/dev/sda2
+lvexpand=/dev/VG/Root
+notes=Phony Fedora look-alike used for testing.
+
+[phony-fedora-qcow2-uncompressed]
+name=Phony Fedora qcow2 uncompressed
+arch=x86_64
+file=fedora.qcow2
+format=qcow2
+size=1073741824
+expand=/dev/sda2
+lvexpand=/dev/VG/Root
+notes=Phony Fedora look-alike used for testing.
+
+[phony-fedora-no-format]
+name=Phony Fedora
+arch=x86_64
+file=fedora.qcow2.xz
+size=1073741824
+expand=/dev/sda2
+lvexpand=/dev/VG/Root
+notes=Phony Fedora look-alike used for testing.
+
+[phony-ubuntu]
+name=Phony Ubuntu
+arch=x86_64
+file=ubuntu.xz
+format=raw
+size=536870912
+expand=/dev/sda2
+notes=Phony Ubuntu look-alike used for testing.
+
+[phony-windows]
+name=Phony Windows
+arch=x86_64
+file=windows.xz
+format=raw
+size=536870912
+expand=/dev/sda2
+notes=Phony Windows look-alike used for testing.

builder/test-index.in

@@ -1,33 +0,0 @@
-[phony-debian]
-name=Phony Debian
-file=debian.xz
-format=raw
-size=536870912
-expand=/dev/sda2
-lvexpand=/dev/debian/root
-notes=Phony Debian look-alike used for testing.
-
-[phony-fedora]
-name=Phony Fedora
-file=fedora.xz
-format=raw
-size=1073741824
-expand=/dev/sda2
-lvexpand=/dev/VG/Root
-notes=Phony Fedora look-alike used for testing.
-
-[phony-ubuntu]
-name=Phony Ubuntu
-file=ubuntu.xz
-format=raw
-size=536870912
-expand=/dev/sda2
-notes=Phony Ubuntu look-alike used for testing.
-
-[phony-windows]
-name=Phony Windows
-file=windows.xz
-format=raw
-size=536870912
-expand=/dev/sda2
-notes=Phony Windows look-alike used for testing.

builder/test-virt-builder-list.sh

@@ -21,16 +21,20 @@
 export LANG=C
 set -e
 
-abs_srcdir=$(cd $srcdir && pwd)
+abs_builddir=$(pwd)
 
-export VIRT_BUILDER_SOURCE=file://$abs_srcdir/test-index
+export XDG_CONFIG_HOME=
+export XDG_CONFIG_DIRS="$abs_builddir/test-config"
 
-short_list=$(./virt-builder --no-check-signature --no-cache --list)
+short_list=$($VG ./virt-builder --no-check-signature --no-cache --list)
 
-if [ "$short_list" != "phony-debian             Phony Debian
-phony-fedora             Phony Fedora
-phony-ubuntu             Phony Ubuntu
-phony-windows            Phony Windows" ]; then
+if [ "$short_list" != "phony-debian             x86_64     Phony Debian
+phony-fedora             x86_64     Phony Fedora
+phony-fedora-qcow2       x86_64     Phony Fedora qcow2
+phony-fedora-qcow2-uncompressed x86_64     Phony Fedora qcow2 uncompressed
+phony-fedora-no-format   x86_64     Phony Fedora
+phony-ubuntu             x86_64     Phony Ubuntu
+phony-windows            x86_64     Phony Windows" ]; then
     echo "$0: unexpected --list output:"
     echo "$short_list"
     exit 1
@@ -38,36 +42,158 @@ fi
 
 long_list=$(./virt-builder --no-check-signature --no-cache --list --long)
 
-if [ "$long_list" != "Source URI: $VIRT_BUILDER_SOURCE
+if [ "$long_list" != "Source URI: file://$abs_builddir/test-index
 
 os-version:              phony-debian
 Full name:               Phony Debian
+Architecture:            x86_64
 Minimum/default size:    512.0M
 
 Notes:
- Phony Debian look-alike used for testing.
+
+Phony Debian look-alike used for testing.
 
 os-version:              phony-fedora
 Full name:               Phony Fedora
+Architecture:            x86_64
 Minimum/default size:    1.0G
 
 Notes:
- Phony Fedora look-alike used for testing.
+
+Phony Fedora look-alike used for testing.
+
+os-version:              phony-fedora-qcow2
+Full name:               Phony Fedora qcow2
+Architecture:            x86_64
+Minimum/default size:    1.0G
+
+Notes:
+
+Phony Fedora look-alike used for testing.
+
+os-version:              phony-fedora-qcow2-uncompressed
+Full name:               Phony Fedora qcow2 uncompressed
+Architecture:            x86_64
+Minimum/default size:    1.0G
+
+Notes:
+
+Phony Fedora look-alike used for testing.
+
+os-version:              phony-fedora-no-format
+Full name:               Phony Fedora
+Architecture:            x86_64
+Minimum/default size:    1.0G
+
+Notes:
+
+Phony Fedora look-alike used for testing.
 
 os-version:              phony-ubuntu
 Full name:               Phony Ubuntu
+Architecture:            x86_64
 Minimum/default size:    512.0M
 
 Notes:
- Phony Ubuntu look-alike used for testing.
+
+Phony Ubuntu look-alike used for testing.
 
 os-version:              phony-windows
 Full name:               Phony Windows
+Architecture:            x86_64
 Minimum/default size:    512.0M
 
 Notes:
- Phony Windows look-alike used for testing." ]; then
+
+Phony Windows look-alike used for testing." ]; then
     echo "$0: unexpected --list --long output:"
     echo "$long_list"
     exit 1
 fi
+
+json_list=$(./virt-builder --no-check-signature --no-cache --list --list-format json)
+
+if [ "$json_list" != "{
+  \"version\": 1,
+  \"sources\": [
+  {
+    \"uri\": \"file://$abs_builddir/test-index\"
+  }
+  ],
+  \"templates\": [
+  {
+    \"os-version\": \"phony-debian\",
+    \"full-name\": \"Phony Debian\",
+    \"arch\": \"x86_64\",
+    \"size\": 536870912,
+    \"notes\": {
+      \"C\": \"Phony Debian look-alike used for testing.\"
+    },
+    \"hidden\": false
+  },
+  {
+    \"os-version\": \"phony-fedora\",
+    \"full-name\": \"Phony Fedora\",
+    \"arch\": \"x86_64\",
+    \"size\": 1073741824,
+    \"notes\": {
+      \"C\": \"Phony Fedora look-alike used for testing.\"
+    },
+    \"hidden\": false
+  },
+  {
+    \"os-version\": \"phony-fedora-qcow2\",
+    \"full-name\": \"Phony Fedora qcow2\",
+    \"arch\": \"x86_64\",
+    \"size\": 1073741824,
+    \"notes\": {
+      \"C\": \"Phony Fedora look-alike used for testing.\"
+    },
+    \"hidden\": false
+  },
+  {
+    \"os-version\": \"phony-fedora-qcow2-uncompressed\",
+    \"full-name\": \"Phony Fedora qcow2 uncompressed\",
+    \"arch\": \"x86_64\",
+    \"size\": 1073741824,
+    \"notes\": {
+      \"C\": \"Phony Fedora look-alike used for testing.\"
+    },
+    \"hidden\": false
+  },
+  {
+    \"os-version\": \"phony-fedora-no-format\",
+    \"full-name\": \"Phony Fedora\",
+    \"arch\": \"x86_64\",
+    \"size\": 1073741824,
+    \"notes\": {
+      \"C\": \"Phony Fedora look-alike used for testing.\"
+    },
+    \"hidden\": false
+  },
+  {
+    \"os-version\": \"phony-ubuntu\",
+    \"full-name\": \"Phony Ubuntu\",
+    \"arch\": \"x86_64\",
+    \"size\": 536870912,
+    \"notes\": {
+      \"C\": \"Phony Ubuntu look-alike used for testing.\"
+    },
+    \"hidden\": false
+  },
+  {
+    \"os-version\": \"phony-windows\",
+    \"full-name\": \"Phony Windows\",
+    \"arch\": \"x86_64\",
+    \"size\": 536870912,
+    \"notes\": {
+      \"C\": \"Phony Windows look-alike used for testing.\"
+    },
+    \"hidden\": false
+  }
+  ]
+}" ]; then
+    echo "$0: unexpected --list --format json output:"
+    echo "$json_list"
+    exit 1
+fi

builder/test-virt-builder-planner.sh

@@ -0,0 +1,52 @@
+#!/bin/bash -
+# libguestfs virt-builder test script
+# Copyright (C) 2013 Red Hat Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+export LANG=C
+set -e
+
+abs_builddir=$(pwd)
+
+export XDG_CONFIG_HOME=
+export XDG_CONFIG_DIRS="$abs_builddir/test-config"
+
+if [ ! -f fedora.xz -o ! -f fedora.qcow2 -o ! -f fedora.qcow2.xz ]; then
+    echo "$0: test skipped because there is no fedora.xz, fedora.qcow2 or fedora.qcow2.xz in the build directory"
+    exit 77
+fi
+
+if [ "$(../fish/guestfish get-backend)" = "uml" ]; then
+    echo "$0: test skipped because backend is UML"
+    exit 77
+fi
+
+rm -f planner-output
+
+for input in phony-fedora phony-fedora-qcow2 phony-fedora-qcow2-uncompressed phony-fedora-no-format; do
+    for size in none 1G 1.1G 2G; do
+        for format in none raw qcow2; do
+            args="--output planner-output --no-cache --no-check-signature"
+            if [ "$size" != "none" ]; then args="$args --size $size"; fi
+            if [ "$format" != "none" ]; then args="$args --format $format"; fi
+
+            echo $VG ./virt-builder $input $args
+            $VG ./virt-builder $input $args
+        done
+    done
+done
+
+rm planner-output

builder/test-virt-builder.sh

@@ -19,9 +19,15 @@
 export LANG=C
 set -e
 
-abs_srcdir=$(cd $srcdir && pwd)
+abs_builddir=$(pwd)
 
-export VIRT_BUILDER_SOURCE=file://$abs_srcdir/test-index
+export XDG_CONFIG_HOME=
+export XDG_CONFIG_DIRS="$abs_builddir/test-config"
+
+if [ -n "$SKIP_TEST_VIRT_BUILDER_SH" ]; then
+    echo "$0: skipping test because environment variable is set."
+    exit 77
+fi
 
 if [ ! -f fedora.xz ]; then
     echo "$0: test skipped because there is no fedora.xz in the build directory"
@@ -49,16 +55,68 @@ rm -f $output
 # Note we cannot test --install, --run since the phony Fedora doesn't
 # have a real OS inside just some configuration files.  Just about
 # every other option is fair game.
-./virt-builder phony-fedora \
+$VG ./virt-builder phony-fedora \
     -v --no-cache --no-check-signature $no_network \
     -o $output --size 2G --format $format \
+    --arch x86_64 \
     --hostname test.example.com \
+    --timezone Europe/London \
     --root-password password:123456 \
+    --mkdir /etc/foo/bar/baz \
+    --write '/etc/foo/bar/baz/foo:Hello World' \
     --upload Makefile:/Makefile \
+    --upload Makefile:/etc/foo/bar/baz \
     --delete /Makefile \
+    --link /etc/foo/bar/baz/foo:/foo \
+    --link /etc/foo/bar/baz/foo:/foo1:/foo2:/foo3 \
     --firstboot Makefile --firstboot-command 'echo "hello"' \
     --firstboot-install "minicom,inkscape"
 
-# XXX Test that the modifications were made.
+# Check that some modifications were made.
+$VG ../fish/guestfish --ro -i -a $output > test.out <<EOF
+# Uploaded files
+is-file /etc/foo/bar/baz/Makefile
+cat /etc/foo/bar/baz/foo
+is-symlink /foo
+is-symlink /foo1
+is-symlink /foo2
+is-symlink /foo3
+
+echo -----
+# Hostname
+cat /etc/sysconfig/network | grep HOSTNAME=
+
+echo -----
+# Timezone
+is-file /usr/share/zoneinfo/Europe/London
+is-symlink /etc/localtime
+readlink /etc/localtime
+
+echo -----
+# Password
+is-file /etc/shadow
+cat /etc/shadow | sed -r '/^root:/!d;s,^(root:\\\$6\\\$).*,\\1,g'
+EOF
+
+if [ "$(cat test.out)" != "true
+Hello World
+true
+true
+true
+true
+-----
+HOSTNAME=test.example.com
+-----
+true
+true
+/usr/share/zoneinfo/Europe/London
+-----
+true
+root:\$6\$" ]; then
+    echo "$0: unexpected output:"
+    cat test.out
+    exit 1
+fi
 
 rm $output
+rm test.out

builder/test-virt-index-validate-bad-1

@@ -0,0 +1,8 @@
+name=Phony Fedora
+arch=x86_64
+file=fedora.xz
+format=raw
+size=1073741824
+expand=/dev/sda2
+lvexpand=/dev/VG/Root
+notes=Phony Fedora look-alike used for testing.

builder/test-virt-index-validate-good-2

@@ -0,0 +1,2 @@
+# All lines commented out.
+# [foo]

builder/test-virt-index-validate.sh

@@ -0,0 +1,24 @@
+#!/bin/bash -
+# test virt-index-validate
+# Copyright (C) 2014 Red Hat Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+export LANG=C
+set -e
+
+! $VG ./virt-index-validate test-virt-index-validate-bad-1
+$VG ./virt-index-validate test-virt-index-validate-good-1
+$VG ./virt-index-validate test-virt-index-validate-good-2

builder/test-website/virt-builder/repos.d/libguestfs.conf.in

@@ -0,0 +1,3 @@
+[libguestfs.org]
+uri=file://@abs_top_srcdir@/builder/website/index.asc
+gpgkey=file://@abs_top_srcdir@/builder/libguestfs.gpg

builder/uname-c.c

@@ -0,0 +1,55 @@
+/* virt-builder
+ * Copyright (C) 2014 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#include <config.h>
+
+#include <errno.h>
+#include <sys/utsname.h>
+
+#include <caml/alloc.h>
+#include <caml/fail.h>
+#include <caml/memory.h>
+#include <caml/mlvalues.h>
+
+#ifdef HAVE_CAML_UNIXSUPPORT_H
+#include <caml/unixsupport.h>
+#else
+#define Nothing ((value) 0)
+extern void unix_error (int errcode, char * cmdname, value arg) Noreturn;
+#endif
+
+value
+virt_builder_uname (value unit)
+{
+  CAMLparam0 ();
+  CAMLlocal1 (rv);
+  struct utsname u;
+
+  if (uname (&u) < 0)
+    unix_error (errno, (char *) "uname", Val_int (0));
+
+  rv = caml_alloc (5, 0);
+
+  Store_field (rv, 0, caml_copy_string (u.sysname));
+  Store_field (rv, 1, caml_copy_string (u.nodename));
+  Store_field (rv, 2, caml_copy_string (u.release));
+  Store_field (rv, 3, caml_copy_string (u.version));
+  Store_field (rv, 4, caml_copy_string (u.machine));
+
+  CAMLreturn (rv);
+}

builder/uname.ml

@@ -0,0 +1,27 @@
+(* virt-builder
+ * Copyright (C) 2014 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+type uname_struct = {
+  sysname : string;
+  nodename : string;
+  release : string;
+  version : string;
+  machine : string;
+}
+
+external uname : unit -> uname_struct = "virt_builder_uname"

builder/uname.mli

@@ -0,0 +1,28 @@
+(* virt-builder
+ * Copyright (C) 2014 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+type uname_struct = {
+  sysname : string;
+  nodename : string;
+  release : string;
+  version : string;
+  machine : string;
+}
+
+val uname : unit -> uname_struct
+(** [uname] Tiny wrapper to the C [uname]. *)

builder/virt-index-validate.pod

@@ -0,0 +1,90 @@
+=head1 NAME
+
+virt-index-validate - Validate virt-builder index file
+
+=head1 SYNOPSIS
+
+ virt-index-validate index
+
+=head1 DESCRIPTION
+
+L<virt-builder(1)> uses an index file to store metadata about templates
+that it knows how to use.  This index file has a specific format which
+virt-index-validate knows how to validate.
+
+Note that virt-index-validate can validate either the signed or
+unsigned index file (ie. either C<index> or C<index.asc>).  It can
+only validate a local file, not a URL.
+
+=head1 OPTIONS
+
+=over 4
+
+=item B<--compat-1.24.0>
+
+Check for compatibility with virt-builder 1.24.0.  (Using this option
+implies I<--compat-1.24.1>, so you don't need to use both.)
+
+In particular:
+
+=over 4
+
+=item *
+
+This version of virt-builder could not handle C<[...]>
+(square brackets) in field names (eg. C<checksum[sha512]=...>).
+
+=item *
+
+It required detached signatures (C<sig=...>).
+
+=back
+
+=item B<--compat-1.24.1>
+
+Check for compatibility with virt-builder E<ge> 1.24.1.
+
+In particular:
+
+=over 4
+
+=item *
+
+This version of virt-builder could not handle C<.> (period) in field
+names or C<,> (comma) in subfield names.
+
+=item *
+
+It could not handle comments appearing in the file.
+
+=back
+
+=item B<--help>
+
+Display help.
+
+=item B<-V>
+
+=item B<--version>
+
+Display version number and exit.
+
+=back
+
+=head1 EXIT STATUS
+
+This program returns 0 if the index file validates, or non-zero if
+there was an error.
+
+=head1 SEE ALSO
+
+L<virt-builder(1)>,
+L<http://libguestfs.org/>.
+
+=head1 AUTHOR
+
+Richard W.M. Jones L<http://people.redhat.com/~rjones/>
+
+=head1 COPYRIGHT
+
+Copyright (C) 2013 Red Hat Inc.

builder/website/Makefile.am

@@ -0,0 +1,44 @@
+# libguestfs virt-builder tool
+# Copyright (C) 2013 Red Hat Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+include $(top_srcdir)/subdir-rules.mk
+
+EXTRA_DIST = \
+	.gitignore \
+	compress.sh \
+	test-guest.sh \
+	validate.sh \
+	README \
+	index \
+	index.asc \
+	centos.sh \
+	debian.preseed \
+	debian.sh \
+	fedora.sh \
+	rhel.sh \
+	scientificlinux.sh \
+	ubuntu.preseed \
+	ubuntu.sh
+
+CLEANFILES = *~
+
+# Validates the index file.
+TESTS_ENVIRONMENT = $(top_builddir)/run --test
+TESTS = validate.sh
+
+check-valgrind:
+	$(MAKE) VG="$(top_builddir)/run @VG@" check

builder/website/README

@@ -25,7 +25,7 @@ directory.
 
 ie:
 
-  ./run ./builder/virt-builder fedora-20
+  ./run virt-builder fedora-20
 
 will fail unless you have downloaded fedora-20.xz here.
 

builder/website/centos.sh

@@ -0,0 +1,98 @@
+#!/bin/bash -
+# virt-builder
+# Copyright (C) 2013-2014 Red Hat Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+unset CDPATH
+export LANG=C
+set -e
+set -x
+
+if [ $# -ne 1 ]; then
+    echo "$0 VERSION"
+    exit 1
+fi
+
+version=$1
+output=centos-$version
+tmpname=tmp-$(tr -cd 'a-f0-9' < /dev/urandom | head -c 8)
+
+case $version in
+    6)
+        # We rebuild this every time there is a new 6.x release, and bump
+        # the revision in the index.
+        tree=http://mirror.bytemark.co.uk/centos/$version/os/x86_64/
+        major=6
+        ;;
+    7.*)
+        major=7
+        # XXX Need to fix the URL when 7.1 comes out.
+        tree=http://mirror.centos.org/centos-7/$major/os/x86_64/
+        ;;
+esac
+
+rm -f $output $output.old $output.xz
+
+# Generate the kickstart to a temporary file.
+ks=$(mktemp)
+cat > $ks <<'EOF'
+install
+text
+reboot
+lang en_US.UTF-8
+keyboard us
+network --bootproto dhcp
+rootpw builder
+firewall --enabled --ssh
+selinux --enforcing
+timezone --utc America/New_York
+bootloader --location=mbr --append="console=tty0 console=ttyS0,115200 rd_NO_PLYMOUTH"
+zerombr
+clearpart --all --initlabel
+part /boot --fstype=ext4 --size=512         --asprimary
+part swap                --size=1024        --asprimary
+part /     --fstype=ext4 --size=1024 --grow --asprimary
+
+# Halt the system once configuration has finished.
+poweroff
+
+%packages
+@core
+%end
+EOF
+
+# Clean up function.
+cleanup ()
+{
+    rm -f $ks
+    virsh undefine $tmpname ||:
+}
+trap cleanup INT QUIT TERM EXIT ERR
+
+virt-install \
+    --name=$tmpname \
+    --ram=2048 \
+    --cpu=host --vcpus=2 \
+    --os-type=linux --os-variant=rhel$major \
+    --initrd-inject=$ks \
+    --extra-args="ks=file:/`basename $ks` console=tty0 console=ttyS0,115200 proxy=$http_proxy" \
+    --disk $(pwd)/$output,size=6 \
+    --serial pty \
+    --location=$tree \
+    --nographics \
+    --noreboot
+
+source $(dirname "$0")/compress.sh $output

builder/website/compress.sh

@@ -0,0 +1,38 @@
+#!/bin/bash -
+# virt-builder
+# Copyright (C) 2013 Red Hat Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+# Common code which syspreps, sparsifies and compresses the templates.
+
+output=$1
+
+# Sysprep (removes logfiles and so on).
+# Note this also touches /.autorelabel so the further installation
+# changes that we make will be labelled properly at first boot.
+virt-sysprep -a $output
+
+# Sparsify.
+mv $output $output.old
+virt-sparsify $output.old $output
+rm $output.old
+
+# Compress.
+xz --best --block-size=16777216 $output
+
+# Result.  These can be copied into the index file directly.
+echo -n compressed_size= ; stat -c %s $output.xz
+echo -n checksum= ; sha512sum $output.xz | awk '{print $1}'

builder/website/debian-6.xz.sig

@@ -1,17 +0,0 @@
------BEGIN PGP SIGNATURE-----
-Version: GnuPG v1.4.14 (GNU/Linux)
-
-iQIcBAABAgAGBQJSWY5TAAoJEJFzj3Pht2igCvMQAMVm82+6OLoQjWZ5MA14KFfF
-I7/uJ5PRGDPoI+IjvSaVN3v7ZZxFSRbTdV51GaBjryxRGCVqpVF5b2sAgaFCzhSZ
-DMuVNHUSH6Tjo/IER0B1/GCvTQYu0O/xgZHY5V3JeXDhXb+99PMFUKT5OD9RQD9E
-ln41vI5wmFBxJuJKWBnQc8XI22FNFJN/yK4ZPbtd7fNSihWEmH95DNBsYlaAmYP1
-ZYH4FeOIm0wXMiqF1GnLDduwFmQgNepGwaBAGJ3TYLhwRzgCfQxX3VW7/uHr+V9M
-7P1SoBIzXztLvw2fS3ZY5/R7LIfTjEWDUkol7lTr+cqiqy6/hAPiOTDEVGP3DBzJ
-OhjspK5gvscrXMGKj9P9O5sKJvbrxGFmAzwIhpwMc3WnimwOCGzCsKGMe6Ve6eqr
-TKxpehsEacgNiBDpb7BSHNCJ6JcNwJKPzXTBRn85a3dr56K5olGbywZKRbzp8h1U
-vVDBc1h7MP0xWTkIsriNHhNThAgGhsQc0ONCIGEX6ISXvSC97GHjB8kFP1ajpHJl
-M0umRIHEneTIP0R+fIrbEAZyjr1SQW9o+vk2q2QK8GMhvGAHlEF3ubIisu1/OILU
-C3GfuqqVU6dLQSzzuiD0Edl/kc+UJGaYHShhiRR/E9z9TnW1KnTcHG9dvY2CscfB
-T+vKXH6tyXSsguuD1HiG
-=ulbl
------END PGP SIGNATURE-----

builder/website/debian-7.preseed

@@ -1,385 +0,0 @@
-#### Contents of the preconfiguration file (for wheezy)
-### Localization
-# Preseeding only locale sets language, country and locale.
-d-i debian-installer/locale string en_US
-
-# The values can also be preseeded individually for greater flexibility.
-#d-i debian-installer/language string en
-#d-i debian-installer/country string NL
-#d-i debian-installer/locale string en_GB.UTF-8
-# Optionally specify additional locales to be generated.
-#d-i localechooser/supported-locales multiselect en_US.UTF-8, nl_NL.UTF-8
-
-# Keyboard selection.
-# keymap is an alias for keyboard-configuration/xkb-keymap
-d-i keymap select us
-# d-i keyboard-configuration/toggle select No toggling
-
-### Network configuration
-# Disable network configuration entirely. This is useful for cdrom
-# installations on non-networked devices where the network questions,
-# warning and long timeouts are a nuisance.
-d-i netcfg/enable boolean true
-
-# netcfg will choose an interface that has link if possible. This makes it
-# skip displaying a list if there is more than one interface.
-d-i netcfg/choose_interface select auto
-
-# To pick a particular interface instead:
-#d-i netcfg/choose_interface select eth1
-
-# To set a different link detection timeout (default is 3 seconds).
-# Values are interpreted as seconds.
-#d-i netcfg/link_detection_timeout string 10
-
-# If you have a slow dhcp server and the installer times out waiting for
-# it, this might be useful.
-#d-i netcfg/dhcp_timeout string 60
-#d-i netcfg/dhcpv6_timeout string 60
-
-# If you prefer to configure the network manually, uncomment this line and
-# the static network configuration below.
-#d-i netcfg/disable_autoconfig boolean true
-
-# If you want the preconfiguration file to work on systems both with and
-# without a dhcp server, uncomment these lines and the static network
-# configuration below.
-#d-i netcfg/dhcp_failed note
-#d-i netcfg/dhcp_options select Configure network manually
-
-# Static network configuration.
-#
-# IPv4 example
-#d-i netcfg/get_ipaddress string 192.168.1.42
-#d-i netcfg/get_netmask string 255.255.255.0
-#d-i netcfg/get_gateway string 192.168.1.1
-#d-i netcfg/get_nameservers string 192.168.1.1
-#d-i netcfg/confirm_static boolean true
-#
-# IPv6 example
-#d-i netcfg/get_ipaddress string fc00::2
-#d-i netcfg/get_netmask string ffff:ffff:ffff:ffff::
-#d-i netcfg/get_gateway string fc00::1
-#d-i netcfg/get_nameservers string fc00::1
-#d-i netcfg/confirm_static boolean true
-
-# Any hostname and domain names assigned from dhcp take precedence over
-# values set here. However, setting the values still prevents the questions
-# from being shown, even if values come from dhcp.
-d-i netcfg/get_hostname string unassigned-hostname
-d-i netcfg/get_domain string unassigned-domain
-
-# If you want to force a hostname, regardless of what either the DHCP
-# server returns or what the reverse DNS entry for the IP is, uncomment
-# and adjust the following line.
-#d-i netcfg/hostname string somehost
-
-# Disable that annoying WEP key dialog.
-d-i netcfg/wireless_wep string
-# The wacky dhcp hostname that some ISPs use as a password of sorts.
-#d-i netcfg/dhcp_hostname string radish
-
-# If non-free firmware is needed for the network or other hardware, you can
-# configure the installer to always try to load it, without prompting. Or
-# change to false to disable asking.
-#d-i hw-detect/load_firmware boolean true
-
-### Network console
-# Use the following settings if you wish to make use of the network-console
-# component for remote installation over SSH. This only makes sense if you
-# intend to perform the remainder of the installation manually.
-#d-i anna/choose_modules string network-console
-#d-i network-console/authorized_keys_url string http://10.0.0.1/openssh-key
-#d-i network-console/password password r00tme
-#d-i network-console/password-again password r00tme
-
-### Mirror settings
-# If you select ftp, the mirror/country string does not need to be set.
-#d-i mirror/protocol string ftp
-d-i mirror/country string manual
-d-i mirror/http/hostname string ftp.uk.debian.org
-d-i mirror/http/directory string /debian
-d-i mirror/http/proxy string @CACHE@
-
-# Suite to install.
-#d-i mirror/suite string testing
-# Suite to use for loading installer components (optional).
-#d-i mirror/udeb/suite string testing
-
-### Account setup
-# Skip creation of a root account (normal user account will be able to
-# use sudo).
-#d-i passwd/root-login boolean false
-# Alternatively, to skip creation of a normal user account.
-d-i passwd/make-user boolean false
-
-# Root password, either in clear text
-d-i passwd/root-password password builder
-d-i passwd/root-password-again password builder
-# or encrypted using an MD5 hash.
-#d-i passwd/root-password-crypted password [MD5 hash]
-
-# To create a normal user account.
-#d-i passwd/user-fullname string Debian User
-#d-i passwd/username string debian
-# Normal user's password, either in clear text
-#d-i passwd/user-password password insecure
-#d-i passwd/user-password-again password insecure
-# or encrypted using an MD5 hash.
-#d-i passwd/user-password-crypted password [MD5 hash]
-# Create the first user with the specified UID instead of the default.
-#d-i passwd/user-uid string 1010
-
-# The user account will be added to some standard initial groups. To
-# override that, use this.
-#d-i passwd/user-default-groups string audio cdrom video
-
-### Clock and time zone setup
-# Controls whether or not the hardware clock is set to UTC.
-d-i clock-setup/utc boolean true
-
-# You may set this to any valid setting for $TZ; see the contents of
-# /usr/share/zoneinfo/ for valid values.
-d-i time/zone string US/Eastern
-
-# Controls whether to use NTP to set the clock during the install
-d-i clock-setup/ntp boolean true
-# NTP server to use. The default is almost always fine here.
-#d-i clock-setup/ntp-server string ntp.example.com
-
-### Partitioning
-## Partitioning example
-# If the system has free space you can choose to only partition that space.
-# This is only honoured if partman-auto/method (below) is not set.
-#d-i partman-auto/init_automatically_partition select biggest_free
-
-# Alternatively, you may specify a disk to partition. If the system has only
-# one disk the installer will default to using that, but otherwise the device
-# name must be given in traditional, non-devfs format (so e.g. /dev/hda or
-# /dev/sda, and not e.g. /dev/discs/disc0/disc).
-# For example, to use the first SCSI/SATA hard disk:
-#d-i partman-auto/disk string /dev/sda
-# In addition, you'll need to specify the method to use.
-# The presently available methods are:
-# - regular: use the usual partition types for your architecture
-# - lvm:     use LVM to partition the disk
-# - crypto:  use LVM within an encrypted partition
-d-i partman-auto/method string regular
-
-# If one of the disks that are going to be automatically partitioned
-# contains an old LVM configuration, the user will normally receive a
-# warning. This can be preseeded away...
-d-i partman-lvm/device_remove_lvm boolean true
-# The same applies to pre-existing software RAID array:
-d-i partman-md/device_remove_md boolean true
-# And the same goes for the confirmation to write the lvm partitions.
-d-i partman-lvm/confirm boolean true
-d-i partman-lvm/confirm_nooverwrite boolean true
-
-# You can choose one of the three predefined partitioning recipes:
-# - atomic: all files in one partition
-# - home:   separate /home partition
-# - multi:  separate /home, /usr, /var, and /tmp partitions
-d-i partman-auto/choose_recipe select atomic
-
-# Or provide a recipe of your own...
-# If you have a way to get a recipe file into the d-i environment, you can
-# just point at it.
-#d-i partman-auto/expert_recipe_file string /hd-media/recipe
-
-# If not, you can put an entire recipe into the preconfiguration file in one
-# (logical) line. This example creates a small /boot partition, suitable
-# swap, and uses the rest of the space for the root partition:
-#d-i partman-auto/expert_recipe string                         \
-#      boot-root ::                                            \
-#              40 50 100 ext3                                  \
-#                      $primary{ } $bootable{ }                \
-#                      method{ format } format{ }              \
-#                      use_filesystem{ } filesystem{ ext3 }    \
-#                      mountpoint{ /boot }                     \
-#              .                                               \
-#              500 10000 1000000000 ext3                       \
-#                      method{ format } format{ }              \
-#                      use_filesystem{ } filesystem{ ext3 }    \
-#                      mountpoint{ / }                         \
-#              .                                               \
-#              64 512 300% linux-swap                          \
-#                      method{ swap } format{ }                \
-#              .
-
-# The full recipe format is documented in the file partman-auto-recipe.txt
-# included in the 'debian-installer' package or available from D-I source
-# repository. This also documents how to specify settings such as file
-# system labels, volume group names and which physical devices to include
-# in a volume group.
-
-# This makes partman automatically partition without confirmation, provided
-# that you told it what to do using one of the methods above.
-d-i partman-partitioning/confirm_write_new_label boolean true
-d-i partman/choose_partition select finish
-d-i partman/confirm boolean true
-d-i partman/confirm_nooverwrite boolean true
-
-## Partitioning using RAID
-# The method should be set to "raid".
-#d-i partman-auto/method string raid
-# Specify the disks to be partitioned. They will all get the same layout,
-# so this will only work if the disks are the same size.
-#d-i partman-auto/disk string /dev/sda /dev/sdb
-
-# Next you need to specify the physical partitions that will be used. 
-#d-i partman-auto/expert_recipe string \
-#      multiraid ::                                         \
-#              1000 5000 4000 raid                          \
-#                      $primary{ } method{ raid }           \
-#              .                                            \
-#              64 512 300% raid                             \
-#                      method{ raid }                       \
-#              .                                            \
-#              500 10000 1000000000 raid                    \
-#                      method{ raid }                       \
-#              .
-
-# Last you need to specify how the previously defined partitions will be
-# used in the RAID setup. Remember to use the correct partition numbers
-# for logical partitions. RAID levels 0, 1, 5, 6 and 10 are supported;
-# devices are separated using "#".
-# Parameters are:
-# <raidtype> <devcount> <sparecount> <fstype> <mountpoint> \
-#          <devices> <sparedevices>
-
-#d-i partman-auto-raid/recipe string \
-#    1 2 0 ext3 /                    \
-#          /dev/sda1#/dev/sdb1       \
-#    .                               \
-#    1 2 0 swap -                    \
-#          /dev/sda5#/dev/sdb5       \
-#    .                               \
-#    0 2 0 ext3 /home                \
-#          /dev/sda6#/dev/sdb6       \
-#    .
-
-# For additional information see the file partman-auto-raid-recipe.txt
-# included in the 'debian-installer' package or available from D-I source
-# repository.
-
-# This makes partman automatically partition without confirmation.
-d-i partman-md/confirm boolean true
-d-i partman-partitioning/confirm_write_new_label boolean true
-d-i partman/choose_partition select finish
-d-i partman/confirm boolean true
-d-i partman/confirm_nooverwrite boolean true
-
-## Controlling how partitions are mounted
-# The default is to mount by UUID, but you can also choose "traditional" to
-# use traditional device names, or "label" to try filesystem labels before
-# falling back to UUIDs.
-#d-i partman/mount_style select uuid
-
-### Base system installation
-# Configure APT to not install recommended packages by default. Use of this
-# option can result in an incomplete system and should only be used by very
-# experienced users.
-#d-i base-installer/install-recommends boolean false
-
-# The kernel image (meta) package to be installed; "none" can be used if no
-# kernel is to be installed.
-#d-i base-installer/kernel/image string linux-image-486
-
-### Apt setup
-# You can choose to install non-free and contrib software.
-#d-i apt-setup/non-free boolean true
-#d-i apt-setup/contrib boolean true
-# Uncomment this if you don't want to use a network mirror.
-#d-i apt-setup/use_mirror boolean false
-# Select which update services to use; define the mirrors to be used.
-# Values shown below are the normal defaults.
-#d-i apt-setup/services-select multiselect security, volatile
-#d-i apt-setup/security_host string security.debian.org
-#d-i apt-setup/volatile_host string volatile.debian.org
-
-# Additional repositories, local[0-9] available
-#d-i apt-setup/local0/repository string \
-#       http://local.server/debian stable main
-#d-i apt-setup/local0/comment string local server
-# Enable deb-src lines
-#d-i apt-setup/local0/source boolean true
-# URL to the public key of the local repository; you must provide a key or
-# apt will complain about the unauthenticated repository and so the
-# sources.list line will be left commented out
-#d-i apt-setup/local0/key string http://local.server/key
-
-# By default the installer requires that repositories be authenticated
-# using a known gpg key. This setting can be used to disable that
-# authentication. Warning: Insecure, not recommended.
-#d-i debian-installer/allow_unauthenticated boolean true
-
-### Package selection
-#tasksel tasksel/first multiselect standard, web-server
-# If the desktop task is selected, install the kde and xfce desktops
-# instead of the default gnome desktop.
-#tasksel tasksel/desktop multiselect kde, xfce
-
-# Individual additional packages to install
-#d-i pkgsel/include string openssh-server build-essential
-# Whether to upgrade packages after debootstrap.
-# Allowed values: none, safe-upgrade, full-upgrade
-#d-i pkgsel/upgrade select none
-
-# Some versions of the installer can report back on what software you have
-# installed, and what software you use. The default is not to report back,
-# but sending reports helps the project determine what software is most
-# popular and include it on CDs.
-popularity-contest popularity-contest/participate boolean false
-
-### Finishing up the installation
-# During installations from serial console, the regular virtual consoles
-# (VT1-VT6) are normally disabled in /etc/inittab. Uncomment the next
-# line to prevent this.
-#d-i finish-install/keep-consoles boolean true
-
-# Avoid that last message about the install being complete.
-d-i finish-install/reboot_in_progress note
-
-# This will prevent the installer from ejecting the CD during the reboot,
-# which is useful in some situations.
-#d-i cdrom-detect/eject boolean false
-
-# This is how to make the installer shutdown when finished, but not
-# reboot into the installed system.
-#d-i debian-installer/exit/halt boolean true
-# This will power off the machine instead of just halting it.
-#d-i debian-installer/exit/poweroff boolean true
-
-### Preseeding other packages
-# Depending on what software you choose to install, or if things go wrong
-# during the installation process, it's possible that other questions may
-# be asked. You can preseed those too, of course. To get a list of every
-# possible question that could be asked during an install, do an
-# installation, and then run these commands:
-#   debconf-get-selections --installer > file
-#   debconf-get-selections >> file
-
-
-#### Advanced options
-### Running custom commands during the installation
-# d-i preseeding is inherently not secure. Nothing in the installer checks
-# for attempts at buffer overflows or other exploits of the values of a
-# preconfiguration file like this one. Only use preconfiguration files from
-# trusted locations! To drive that home, and because it's generally useful,
-# here's a way to run any shell command you'd like inside the installer,
-# automatically.
-
-# This first command is run as early as possible, just after
-# preseeding is read.
-#d-i preseed/early_command string anna-install some-udeb
-# This command is run immediately before the partitioner starts. It may be
-# useful to apply dynamic partitioner preseeding that depends on the state
-# of the disks (which may not be visible when preseed/early_command runs).
-#d-i partman/early_command \
-#       string debconf-set partman-auto/disk "$(list-devices disk | head -n1)"
-# This command is run just before the install finishes, but when there is
-# still a usable /target directory. You can chroot to /target and use it
-# directly, or use the apt-install and in-target commands to easily install
-# packages and run commands in the target system.
-#d-i preseed/late_command string apt-install zsh; in-target chsh -s /bin/zsh

builder/website/debian-7.xz.sig

@@ -1,17 +0,0 @@
------BEGIN PGP SIGNATURE-----
-Version: GnuPG v1.4.14 (GNU/Linux)
-
-iQIcBAABAgAGBQJSTbVGAAoJEJFzj3Pht2igBfsP/2EdkKhGLoFW8SxHv8rbxYtS
-VlbC5OaLhSosCerrZnna1qeKuR4i1CoDT08WfYRD5JQ4WnsaeckRWANcl9VKU1rL
-Efc1RCis815b4YbsA/acmc9pT7XxduF0GiV/NYWsFnZj+K/NI7UrgX0aolC21ZvZ
-uVnJ1Be5GICa0CXgNfwkFHb2gCBIaPhasxGaFuAspzWyQ8fVsfiDTxuO2jtJJJc/
-l00oZ0JMW0f30ux9K9PXF0rdIEX7h13RxZVLcw55Nfl389gQK5yVmyCLvcmsSZ6a
-K9O1ffPenRD5E1kpAy1xwDUz+7uUyqa5hXyxjB7qaCwIUGvF3FBIuePmK0XXPkh0
-NF73sb3DXKTJgT1t2LmKg4QprX/8b97ZfrMIpu5KRF7WnxqdGeaNK87lnZQFMxRW
-kgCXzdJ9qfilOMcAkj6omHw6Fj7NXwS8+t6fNZwCsLx6uHKRx2lllG6yxbkoT/VS
-Gs2CSHVOZ94ByaiIUTzOzonwcjy0e7eqLzqIJETcuCedLNKoNW2mpwA23dae8U9q
-x2r9l/C6z27Ofn23wJrbWM/hwrofH249N+nxALqwb3joPxtdsMFVE+GW8GF0dBl8
-KJAIponJ2FWssiL2YsDk8EfvgsQKQVWuPOMCmczl+CVpPr+sslpO0RabGOIvjyOO
-4B5d0Kq92X8Rlx7KAln8
-=DdNg
------END PGP SIGNATURE-----

builder/website/debian.sh

@@ -20,57 +20,59 @@
 # http://honk.sigxcpu.org/con/Preseeding_Debian_virtual_machines_with_virt_install.html
 # https://wiki.debian.org/DebianInstaller/Preseed
 
+unset CDPATH
+export LANG=C
 set -e
 set -x
 
+if [ $# -ne 2 ]; then
+    echo "$0 VERSION DIST"
+    exit 1
+fi
+
 # Some configuration.
-export http_proxy=http://cache.home.annexia.org:3128
-export https_proxy=$http_proxy
-export ftp_proxy=$http_proxy
-location=http://ftp.uk.debian.org/debian/dists/squeeze/main/installer-amd64
+version=$1
+dist=$2
+location=http://ftp.uk.debian.org/debian/dists/$dist/main/installer-amd64
+output=debian-$version
+tmpname=tmp-$(tr -cd 'a-f0-9' < /dev/urandom | head -c 8)
+
+rm -f $output $output.old $output.xz
 
 # Make sure it's being run from the correct directory.
-if [ ! -f debian-6.preseed ]; then
+if [ ! -f debian.preseed ]; then
     echo "You are running this script from the wrong directory."
     exit 1
 fi
 
-pwd=`pwd`
-
 # Note that the injected file must be called "/preseed.cfg" in order
 # for d-i to pick it up.
-sed -e "s,@CACHE@,$http_proxy,g" < debian-6.preseed > preseed.cfg
+sed -e "s,@CACHE@,$http_proxy,g" < debian.preseed > preseed.cfg
 
-virsh undefine tmpd6 ||:
-rm -f debian-6 debian-6.old
+# Clean up function.
+cleanup ()
+{
+    rm -f preseed.cfg
+    virsh undefine $tmpname ||:
+}
+trap cleanup INT QUIT TERM EXIT ERR
 
 virt-install \
-    --name tmpd6 \
+    --name=$tmpname \
     --ram=1024 \
-    --os-type=linux --os-variant=debiansqueeze \
-    --initrd-inject=$pwd/preseed.cfg \
+    --os-type=linux --os-variant=debian$dist \
+    --initrd-inject=$(pwd)/preseed.cfg \
     --extra-args="auto console=tty0 console=ttyS0,115200" \
-    --disk=$pwd/debian-6,size=4 \
+    --disk=$(pwd)/$output,size=4 \
+    --serial pty \
     --location=$location \
     --nographics \
     --noreboot
-# The virt-install command should exit after complete installation.
-# Remove the guest, we don't want it to be defined in libvirt.
-virsh undefine tmpd6
 
-rm preseed.cfg
+# Remove apt proxy configuration (thanks: Daniel Miranda).
+guestfish -a $output -i <<EOF
+  rm /etc/apt/apt.conf
+  touch /etc/apt/apt.conf
+EOF
 
-# Sysprep (removes logfiles and so on).
-virt-sysprep -a debian-6
-
-# Sparsify.
-mv debian-6 debian-6.old
-virt-sparsify debian-6.old debian-6
-rm debian-6.old
-
-# Compress.
-rm -f debian-6.xz
-xz --best --block-size=$((16*1024*1024)) debian-6
-
-# Result:
-ls -lh debian-6.xz
+source $(dirname "$0")/compress.sh $output

builder/website/fedora-18.sh

@@ -1,75 +0,0 @@
-#!/bin/bash -
-# virt-builder
-# Copyright (C) 2013 Red Hat Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-set -e
-set -x
-
-# Some configuration.
-export http_proxy=http://cache.home.annexia.org:3128
-export https_proxy=$http_proxy
-export ftp_proxy=$http_proxy
-tree=http://mirror.bytemark.co.uk/fedora/linux/releases/18/Fedora/x86_64/os/
-
-# Currently you have to run this script as root.
-if [ `id -u` -ne 0 ]; then
-    echo "You have to run this script as root."
-    exit 1
-fi
-
-# Make sure it's being run from the correct directory.
-if [ ! -f fedora-18.ks ]; then
-    echo "You are running this script from the wrong directory."
-    exit 1
-fi
-
-pwd=`pwd`
-
-virsh undefine tmpf18 ||:
-rm -f fedora-18 fedora-18.old
-
-virt-install \
-    --name=tmpf18 \
-    --ram 2048 \
-    --cpu=host --vcpus=2 \
-    --os-type=linux --os-variant=fedora18 \
-    --initrd-inject=$pwd/fedora-18.ks \
-    --extra-args="ks=file:/fedora-18.ks console=tty0 console=ttyS0,115200 proxy=$http_proxy" \
-    --disk $pwd/fedora-18,size=6 \
-    --location=$tree \
-    --nographics \
-    --noreboot
-# The virt-install command should exit after complete installation.
-# Remove the guest, we don't want it to be defined in libvirt.
-virsh undefine tmpf18
-
-# Sysprep (removes logfiles and so on).
-# Note this also touches /.autorelabel so the further installation
-# changes that we make will be labelled properly at first boot.
-virt-sysprep -a fedora-18
-
-# Sparsify.
-mv fedora-18 fedora-18.old
-virt-sparsify fedora-18.old fedora-18
-rm fedora-18.old
-
-# Compress.
-rm -f fedora-18.xz
-xz --best --block-size=16777216 fedora-18
-
-# Result:
-ls -lh fedora-18.xz

builder/website/fedora-18.xz.sig

@@ -1,17 +0,0 @@
------BEGIN PGP SIGNATURE-----
-Version: GnuPG v1.4.14 (GNU/Linux)
-
-iQIcBAABAgAGBQJSSt3OAAoJEJFzj3Pht2ig8x4QAKoQVfMWxXiFvV3lrHUVsjpG
-n+fDcoFJqMKeE/kMLhmlmzF+QDz1qRs3Xlypy7C8B5ii5jbFVCv6q5edeEWrVYE3
-HZOeNgUgmVJEiCSgScEXPpVMAOL/zppJ2PjAbg7CNLHQ64HweNkv6F1ePg4NCIoA
-Op5yIvzm0gUHN9ONfsJRBLlZGQRu8bdsNhdYOAr3rmdxhuuunIi/17qNrT6eaWFO
-2pnGFIyYMn9q2ReXBG8mFIkHlac9ZpT2sR7EKY3LBt0FvGz0qQrNM+9M9bQBwbIQ
-dD/IRfSUUQqjCLYLqiFvFVC/pqAAcR1G2rQ20jRLPkpSUFceFL/K5ueI2flEPEJk
-mF6WR4MsN4lX4w0iiJvpWwE0jqlLVQ1GnhoE2GDZEHgkth/4l+0pN0Jos8OKINDJ
-wB4W1Xc2aXNXaD1JAaebu+CthZNnFEXpa5TrXMFAOnBY4oQ4DgOt/ad1s9Ju4zLX
-EI+Zn0Q++l+iMyU2InXnTHoaTagqKqtngvHWGmSuK55dM8jW9HoYsjZZl0oC0lVz
-GbXQ/1t1loTBqh+crr4kcP0oKvRFT9YPASadUVThmeYlxOyhvQ0e30IBdz0Te1OR
-rqfymVGR8NsGdLW+jCKOfbLEaYShTksTnUmr+yF5w5D2uzZHqZasE7cpwPXAEhuZ
-TF7ZoSqz1utSQF9/xBnz
-=L4x6
------END PGP SIGNATURE-----

builder/website/fedora-19.sh

@@ -1,75 +0,0 @@
-#!/bin/bash -
-# virt-builder
-# Copyright (C) 2013 Red Hat Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-set -e
-set -x
-
-# Some configuration.
-export http_proxy=http://cache.home.annexia.org:3128
-export https_proxy=$http_proxy
-export ftp_proxy=$http_proxy
-tree=http://mirror.bytemark.co.uk/fedora/linux/releases/19/Fedora/x86_64/os/
-
-# Currently you have to run this script as root.
-if [ `id -u` -ne 0 ]; then
-    echo "You have to run this script as root."
-    exit 1
-fi
-
-# Make sure it's being run from the correct directory.
-if [ ! -f fedora-19.ks ]; then
-    echo "You are running this script from the wrong directory."
-    exit 1
-fi
-
-pwd=`pwd`
-
-virsh undefine tmpf19 ||:
-rm -f fedora-19 fedora-19.old
-
-virt-install \
-    --name=tmpf19 \
-    --ram 2048 \
-    --cpu=host --vcpus=2 \
-    --os-type=linux --os-variant=fedora19 \
-    --initrd-inject=$pwd/fedora-19.ks \
-    --extra-args="ks=file:/fedora-19.ks console=tty0 console=ttyS0,115200 proxy=$http_proxy" \
-    --disk $pwd/fedora-19,size=4 \
-    --location=$tree \
-    --nographics \
-    --noreboot
-# The virt-install command should exit after complete installation.
-# Remove the guest, we don't want it to be defined in libvirt.
-virsh undefine tmpf19
-
-# Sysprep (removes logfiles and so on).
-# Note this also touches /.autorelabel so the further installation
-# changes that we make will be labelled properly at first boot.
-virt-sysprep -a fedora-19
-
-# Sparsify.
-mv fedora-19 fedora-19.old
-virt-sparsify fedora-19.old fedora-19
-rm fedora-19.old
-
-# Compress.
-rm -f fedora-19.xz
-xz --best --block-size=16777216 fedora-19
-
-# Result:
-ls -lh fedora-19.xz

builder/website/fedora-19.xz.sig

@@ -1,17 +0,0 @@
------BEGIN PGP SIGNATURE-----
-Version: GnuPG v1.4.14 (GNU/Linux)
-
-iQIcBAABAgAGBQJSSt3WAAoJEJFzj3Pht2igSeMQAI42FK1YU6Fr/LreD9/BdjNH
-TUJ3L/B3ZMJwlMT+F4cX9r1uVD5udzpXdClUEcKgdwPdqzPsQvuwLSBS4fn38nZM
-mryzNeUjbcNPSPSY6bj/MNuYNlYNaWShhL6UjHBD6XYIhAs6hYp0RGcogcpgRU9Z
-0peHFyJBp+uUqCpnBDz00zJht49mgT6ovUA3cGd/vPhHrsrJUusgzzi46X3F90+H
-9eRoe6/xzCbmbiD7HUlARnMK8ec0b7H/CWsNNCd0v9KBNyW9QROkbwBMtRcWtQZW
-p9h9JGT2N3AmIqRbUbKwfWxGN6AfK+Xtl9YjJb/ugwSlw0Pif055EKNeUZjGUUzk
-V4YA9b1Z6mncyL4wxV8VD9w8en+dr5Iu6ga02r7Xr0rIHuTQyQQaKfHvNoWZFmhq
-WI2xsSBLr1EYfLEetdqUkEo2f2gpy7f6UKL1bsHi/6oqA3NaOOXiPdVlbVDNOQFe
-kXPfjz/8hnzQ/8O/zIiSiDefQObdM4DQDTk5ha/vOu4pt5XevHkusaciPNGUAhx1
-JmxEmPF6sqg2Y0xmeFZB2ab5s6VL/CfVGBF5ZFq7QmDj9eNFjrElfv+uN9NqD+ll
-YlR/g3f7vFLHSG64ez4yV/Hgmfv1+4DMGi9MNmhFF3u0W5AIxznTsQzj1KFhbvmV
-8QUyTDG7lGNqsgsSgb7W
-=l9GJ
------END PGP SIGNATURE-----

builder/website/fedora.sh

@@ -0,0 +1,96 @@
+#!/bin/bash -
+# virt-builder
+# Copyright (C) 2013 Red Hat Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+# This script was used to create the Fedora templates used by
+# virt-builder.
+
+unset CDPATH
+export LANG=C
+set -e
+set -x
+
+if [ $# -ne 1 ]; then
+    echo "$0 VERSION"
+    exit 1
+fi
+
+version=$1
+tree=http://mirror.bytemark.co.uk/fedora/linux/releases/$version/Fedora/x86_64/os/
+output=fedora-$version
+tmpname=tmp-$(tr -cd 'a-f0-9' < /dev/urandom | head -c 8)
+
+rm -f $output $output.old $output.xz
+
+# Generate the kickstart to a temporary file.
+ks=$(mktemp)
+cat > $ks <<'EOF'
+install
+text
+reboot
+lang en_US.UTF-8
+keyboard us
+network --bootproto dhcp
+rootpw builder
+firewall --enabled --ssh
+selinux --enforcing
+timezone --utc America/New_York
+bootloader --location=mbr --append="console=tty0 console=ttyS0,115200 rd_NO_PLYMOUTH"
+zerombr
+clearpart --all --initlabel
+autopart --type=plain
+
+# Halt the system once configuration has finished.
+poweroff
+
+%packages
+@core
+%end
+
+%post
+# Enable Xen domU support:
+pushd /etc/dracut.conf.d
+echo 'add_drivers+="xen:vbd xen:vif"' > virt-builder-xen-drivers.conf
+popd
+# Rerun dracut for the installed kernel (not the running kernel):
+KERNEL_VERSION=$(rpm -q kernel --qf '%{version}-%{release}.%{arch}\n')
+dracut -f /boot/initramfs-$KERNEL_VERSION.img $KERNEL_VERSION
+%end
+EOF
+
+# Clean up function.
+cleanup ()
+{
+    rm -f $ks
+    virsh undefine $tmpname ||:
+}
+trap cleanup INT QUIT TERM EXIT ERR
+
+virt-install \
+    --name=$tmpname \
+    --ram=2048 \
+    --cpu=host --vcpus=2 \
+    --os-type=linux --os-variant=fedora18 \
+    --initrd-inject=$ks \
+    --extra-args="ks=file:/`basename $ks` console=tty0 console=ttyS0,115200 proxy=$http_proxy" \
+    --disk $(pwd)/$output,size=6 \
+    --serial pty \
+    --location=$tree \
+    --nographics \
+    --noreboot
+
+source $(dirname "$0")/compress.sh $output

builder/website/index

@@ -1,17 +1,84 @@
+[centos-6]
+name=CentOS 6.5
+osinfo=centos6.5
+arch=x86_64
+file=centos-6.xz
+revision=5
+checksum=8f08505ec708b04a25bbd21816cdda8a5eff8bbf85fec3caff6d7a9a7bfba07dafa707e60772f5676c437a6d3381cd26b4e48bc82a232bc940ca9dcddcff7a85
+format=raw
+size=6442450944
+compressed_size=197139324
+expand=/dev/sda3
+notes=CentOS 6.5
+ 
+ This CentOS image contains only unmodified @Core group packages.
+ 
+ It is thus very minimal.  The kickstart and install script can be
+ found in the libguestfs source tree:
+ 
+ builder/website/centos.sh
+ 
+ Note that `virt-builder centos-6' will always install the latest
+ 6.x release.
+
+[centos-7.0]
+name=CentOS 7.0
+osinfo=centos7.0
+arch=x86_64
+file=centos-7.0.xz
+checksum=cf9ae295f633fbd04e575eeca16f372e933c70c3107c44eb06864760d04354aa94b4f356bfc9a598c138e687304a52e96777e4467e7db1ec0cb5b2d2ec61affc
+format=raw
+size=6442450944
+compressed_size=213203844
+expand=/dev/sda3
+notes=CentOS 7.0
+ 
+ This CentOS image contains only unmodified @Core group packages.
+ 
+ It is thus very minimal.  The kickstart and install script can be
+ found in the libguestfs source tree:
+ 
+ builder/website/centos.sh
+
+[cirros-0.3.1]
+name=CirrOS 0.3.1
+arch=x86_64
+file=cirros-0.3.1.xz
+checksum=096209f00eb62d5722accf3d22ca3a4ee5baaac6d7d4ce0be93b56bbd1c8ab2e3eb4f5db1deffcb570e2c3d41f4d721798a1c499675346cee9546554a4b10388
+format=raw
+size=41126400
+compressed_size=11419004
+expand=/dev/sda1
+notes=CirrOS 0.3.1
+ 
+ CirrOS is a commonly used test image, ideal because it is very
+ small and boots into a minimally usable Linux system.
+ 
+ Note this is not a real Linux distribution, and several virt-builder
+ features such as installing packages will not (and cannot) work.
+ 
+ This CirrOS image comes from https://launchpad.net/cirros
+
 [debian-6]
 name=Debian 6 (Squeeze)
 osinfo=debian6
+arch=x86_64
 file=debian-6.xz
-sig=debian-6.xz.sig
+revision=2
+checksum=bff9c28da0375fde65fa238d7a2ea644cbfad0ea3246783a2f44a98f2374850987679c3f1032a632d3c6238de8d9e43291d07a82efc1e824945000e206b9f6cc
 format=raw
 size=4294967296
-compressed_size=141531780
+compressed_size=139615908
 expand=/dev/sda1
-notes=This is a default Debian 6 (Squeeze) install.
+notes=Debian 6 (Squeeze).
+ 
+ This is a default Debian install.
+ 
  The preseed and virt-install scripts that produced this image
- can be found in the libguestfs git tree:
- libguestfs.git/builder/website/debian-6.preseed
- libguestfs.git/builder/website/debian-6.sh
+ can be found in the libguestfs source tree:
+ 
+ builder/website/debian.preseed
+ builder/website/debian.sh
  
  This image is so very minimal that it only includes an ssh
  server and no virtual consoles.  To enable virtual consoles
@@ -19,21 +86,31 @@ notes=This is a default Debian 6 (Squeeze) install.
  
  virt-builder debian-6 \
      --edit '/etc/inittab: s,^#([1-9].*respawn.*/sbin/getty.*),$1,'
+ 
+ This image does not contain SSH host keys.  To regenerate them use:
+ 
+ --firstboot-command "dpkg-reconfigure openssh-server"
 
 [debian-7]
 name=Debian 7 (Wheezy)
 osinfo=debian7
+arch=x86_64
 file=debian-7.xz
-sig=debian-7.xz.sig
+revision=2
+checksum=c980a7d878c086b14619b3b9f644ab131c1f88b87729062f2832de12a938ba9e5a511c49402a9d02b740a30876f51889d20e7d0ed4997755f542650b8485f013
 format=raw
 size=4294967296
-compressed_size=149230076
+compressed_size=150734028
 expand=/dev/sda1
-notes=This is a default Debian 7 (Wheezy) install.
+notes=Debian 7 (Wheezy).
+ 
+ This is a default Debian install.
+ 
  The preseed and virt-install scripts that produced this image
- can be found in the libguestfs git tree:
- libguestfs.git/builder/website/debian-7.preseed
- libguestfs.git/builder/website/debian-7.sh
+ can be found in the libguestfs source tree:
+ 
+ builder/website/debian.preseed
+ builder/website/debian.sh
  
  This image is so very minimal that it only includes an ssh
  server and no virtual consoles.  To enable virtual consoles
@@ -41,21 +118,29 @@ notes=This is a default Debian 7 (Wheezy) install.
  
  virt-builder debian-7 \
      --edit '/etc/inittab: s,^#([1-9].*respawn.*/sbin/getty.*),$1,'
+ 
+ This image does not contain SSH host keys.  To regenerate them use:
+ 
+ --firstboot-command "dpkg-reconfigure openssh-server"
 
 [fedora-18]
 name=Fedora® 18
 osinfo=fedora18
+arch=x86_64
 file=fedora-18.xz
-sig=fedora-18.xz.sig
+checksum=12435775193b69f6e22658aaa001d4ca9b15fd68a04b4b7e9be20b3b517e857e417dc3268a302979d4a702b20f25754025f7ae0e9fb7088419a4ca1669585e6f
 format=raw
 size=6442450944
 compressed_size=148947524
 expand=/dev/sda3
-notes=This Fedora image contains only unmodified @Core group packages.
+notes=Fedora 18.
+ 
+ This Fedora image contains only unmodified @Core group packages.
+ 
  It is thus very minimal.  The kickstart and install script can be
- found in the libguestfs git tree:
- libguestfs.git/builder/website/fedora-18.ks
- libguestfs.git/builder/website/fedora-18.sh
+ found in the libguestfs source tree:
+ 
+ builder/website/fedora.sh
  
  Fedora and the Infinity design logo are trademarks of Red Hat, Inc.
  Source and further information is available from http://fedoraproject.org/
@@ -63,65 +148,170 @@ notes=This Fedora image contains only unmodified @Core group packages.
 [fedora-19]
 name=Fedora® 19
 osinfo=fedora19
+arch=x86_64
 file=fedora-19.xz
-sig=fedora-19.xz.sig
+revision=2
+checksum=acecd8d4bca0d6a3f937e0c9386f3185c916df3eaf5f825988c73d41e946a6dc4fda06cdd74a12bc60932edd65846097547b11aca2309a61dd6d0e91ab9d16f3
 format=raw
 size=4294967296
-compressed_size=172190964
+compressed_size=169531628
 expand=/dev/sda3
-notes=This Fedora image contains only unmodified @Core group packages.
+notes=Fedora 19.
+ 
+ This Fedora image contains only unmodified @Core group packages.
+ 
  It is thus very minimal.  The kickstart and install script can be
- found in the libguestfs git tree:
- libguestfs.git/builder/website/fedora-19.ks
- libguestfs.git/builder/website/fedora-19.sh
+ found in the libguestfs source tree:
+ 
+ builder/website/fedora.sh
  
  Fedora and the Infinity design logo are trademarks of Red Hat, Inc.
  Source and further information is available from http://fedoraproject.org/
 
+[fedora-20]
+name=Fedora® 20
+osinfo=fedora20
+arch=x86_64
+file=fedora-20.xz
+revision=2
+checksum=983a1b33c34cb311ea3a283f06312d24dc81041b64ebc90e40ef2fd7587362acd1a5654b13252f9f57001870aa95495065537e730d5225b49389c1a0478cb028
+format=raw
+size=4294967296
+compressed_size=181164220
+expand=/dev/sda3
+notes=Fedora 20.
+ 
+ This Fedora image contains only unmodified @Core group packages.
+ 
+ It is thus very minimal.  The kickstart and install script can be
+ found in the libguestfs source tree:
+ 
+ builder/website/fedora.sh
+ 
+ Fedora and the Infinity design logo are trademarks of Red Hat, Inc.
+ Source and further information is available from http://fedoraproject.org/
+
+[rhel-7rc]
+name=Red Hat Enterprise Linux® 7 Release Candidate
+arch=x86_64
+file=rhel-7rc.xz
+checksum=c4252cfdf7482d4289de24f78e7fd1b9c9b6ff702d717a1c2a54a341b0565c03a5c8e95d32e1f23f7afe5ed2cf5b8e0f0ee3e16c1179e01348a54f82cb552c0f
+format=raw
+size=6442450944
+compressed_size=210202084
+expand=/dev/sda3
+notes=Red Hat Enterprise Linux® 7 Release Candidate
+ 
+ This image contains only unmodified @Core group packages.
+ 
+ It is thus very minimal.  The kickstart and install script can be
+ found in the libguestfs source tree:
+ 
+ builder/website/rhel.sh
+ 
+ Red Hat Enterprise Linux is a trademark of Red Hat, Inc.
+ Source and further information is available from ftp://ftp.redhat.com
+
+[scientificlinux-6]
+name=Scientific Linux 6.5
+osinfo=centos6.5
+arch=x86_64
+file=scientificlinux-6.xz
+revision=5
+checksum=be42b0e97d256ab5476079fe71411bebf59abe9c6ff4a7d27330aa5383fac9507472c06d11ef061d3c0e6695bec5ac9cfb7fbfd86fdbb5649e5cb535e2ac5b88
+format=raw
+size=6442450944
+compressed_size=191667880
+expand=/dev/sda3
+notes=Scientific Linux 6.5
+ 
+ This Scientific Linux image contains only unmodified @Core group packages.
+ 
+ It is thus very minimal.  The kickstart and install script can be
+ found in the libguestfs source tree:
+ 
+ builder/website/scientificlinux.sh
+ 
+ Note that `virt-builder scientificlinux-6' will always install the
+ latest 6.x release.
+
 [ubuntu-10.04]
 name=Ubuntu 10.04 (Lucid)
 osinfo=ubuntulucid
+arch=x86_64
 file=ubuntu-10.04.xz
-sig=ubuntu-10.04.xz.sig
+checksum=9e322782bc77c740ce956d57d0d89ec9ac9f2b890c474ac937f400db848df3e164683eaf4631b989f8b4771f64e0c0972452d1555a15c65714b5ceefb295fb7e
 format=raw
 size=4294967296
 compressed_size=149728112
 expand=/dev/sda1
-notes=This is a minimal Ubuntu 10.04 (Lucid) install.
+notes=Ubuntu 10.04 (Lucid).
+ 
+ This is a minimal Ubuntu 10.04 (Lucid) install.
+ 
  Only the openssh-server package is selected in tasksel.
  The preseed and virt-install scripts that produced this image
- can be found in the libguestfs git tree:
- libguestfs.git/builder/website/ubuntu-10.04.preseed
- libguestfs.git/builder/website/ubuntu-10.04.sh
+ can be found in the libguestfs source tree:
+ 
+ builder/website/ubuntu.preseed
+ builder/website/ubuntu.sh
+ 
+ This image does not contain SSH host keys.  To regenerate them use:
+ 
+ --firstboot-command "dpkg-reconfigure openssh-server"
 
 [ubuntu-12.04]
 name=Ubuntu 12.04 (Precise)
 osinfo=ubuntuprecise
+arch=x86_64
 file=ubuntu-12.04.xz
-sig=ubuntu-12.04.xz.sig
+checksum=15eab4610a44bf28423c08982cec47f6c3d43f29834791440249916ed76cc2dfeaf0405fddc8627086257bbbdaa4016c8982ad9b269f57f8f625b0e70d09e4e0
 format=raw
 size=4294967296
 compressed_size=168593316
 expand=/dev/sda1
-notes=This is a minimal Ubuntu 12.04 (Precise) install.
+notes=Ubuntu 12.04 (Precise).
+ 
+ This is a minimal Ubuntu 12.04 (Precise) install.
+ 
  Only the openssh-server package is selected in tasksel.
  The preseed and virt-install scripts that produced this image
- can be found in the libguestfs git tree:
- libguestfs.git/builder/website/ubuntu-12.04.preseed
- libguestfs.git/builder/website/ubuntu-12.04.sh
+ can be found in the libguestfs source tree:
+ 
+ builder/website/ubuntu.preseed
+ builder/website/ubuntu.sh
+ 
+ This image does not contain SSH host keys.  To regenerate them use:
+ 
+ --firstboot-command "dpkg-reconfigure openssh-server"
 
-[ubuntu-13.10]
-name=Ubuntu 13.10 (Saucy)
-osinfo=ubuntusaucy
-file=ubuntu-13.10.xz
-sig=ubuntu-13.10.xz.sig
+[ubuntu-14.04]
+name=Ubuntu 14.04 (Trusty)
+osinfo=ubuntutrusty
+arch=x86_64
+file=ubuntu-14.04.xz
+checksum=dbc5505c7cfbfb48275b09e91f781dfd7f8d521db577f0c31e0a70b13fbd18c8bcec97b1e274b28062ebc3aba70d68e64c84b9c9690ff7dfca9e81f813047e8a
 format=raw
 size=4294967296
-compressed_size=196481392
+compressed_size=206250764
 expand=/dev/sda1
-notes=This is a minimal Ubuntu 13.10 (Saucy) install.
+notes=Ubuntu 14.04 (Trusty).
+ 
+ This is a minimal Ubuntu 14.04 (Trusty) install.
+ 
  Only the openssh-server package is selected in tasksel.
  The preseed and virt-install scripts that produced this image
- can be found in the libguestfs git tree:
- libguestfs.git/builder/website/ubuntu-13.10.preseed
- libguestfs.git/builder/website/ubuntu-13.10.sh
+ can be found in the libguestfs source tree:
+ 
+ builder/website/ubuntu.preseed
+ builder/website/ubuntu.sh
+ 
+ This image does not contain SSH host keys.  To regenerate them use:
+ 
+ --firstboot-command "dpkg-reconfigure openssh-server"
+ 
+ IMPORTANT NOTE: It seems to be impossible to create an Ubuntu >= 14.04
+ image using preseed without creating a user account.  Therefore this
+ image contains a user account 'builder'.  I have disabled it, so that
+ people who don't read release notes don't get caught out, but you might
+ still wish to delete it completely.

builder/website/index.asc

@@ -1,20 +1,87 @@
 -----BEGIN PGP SIGNED MESSAGE-----
 Hash: SHA1
 
+[centos-6]
+name=CentOS 6.5
+osinfo=centos6.5
+arch=x86_64
+file=centos-6.xz
+revision=5
+checksum=8f08505ec708b04a25bbd21816cdda8a5eff8bbf85fec3caff6d7a9a7bfba07dafa707e60772f5676c437a6d3381cd26b4e48bc82a232bc940ca9dcddcff7a85
+format=raw
+size=6442450944
+compressed_size=197139324
+expand=/dev/sda3
+notes=CentOS 6.5
+ 
+ This CentOS image contains only unmodified @Core group packages.
+ 
+ It is thus very minimal.  The kickstart and install script can be
+ found in the libguestfs source tree:
+ 
+ builder/website/centos.sh
+ 
+ Note that `virt-builder centos-6' will always install the latest
+ 6.x release.
+
+[centos-7.0]
+name=CentOS 7.0
+osinfo=centos7.0
+arch=x86_64
+file=centos-7.0.xz
+checksum=cf9ae295f633fbd04e575eeca16f372e933c70c3107c44eb06864760d04354aa94b4f356bfc9a598c138e687304a52e96777e4467e7db1ec0cb5b2d2ec61affc
+format=raw
+size=6442450944
+compressed_size=213203844
+expand=/dev/sda3
+notes=CentOS 7.0
+ 
+ This CentOS image contains only unmodified @Core group packages.
+ 
+ It is thus very minimal.  The kickstart and install script can be
+ found in the libguestfs source tree:
+ 
+ builder/website/centos.sh
+
+[cirros-0.3.1]
+name=CirrOS 0.3.1
+arch=x86_64
+file=cirros-0.3.1.xz
+checksum=096209f00eb62d5722accf3d22ca3a4ee5baaac6d7d4ce0be93b56bbd1c8ab2e3eb4f5db1deffcb570e2c3d41f4d721798a1c499675346cee9546554a4b10388
+format=raw
+size=41126400
+compressed_size=11419004
+expand=/dev/sda1
+notes=CirrOS 0.3.1
+ 
+ CirrOS is a commonly used test image, ideal because it is very
+ small and boots into a minimally usable Linux system.
+ 
+ Note this is not a real Linux distribution, and several virt-builder
+ features such as installing packages will not (and cannot) work.
+ 
+ This CirrOS image comes from https://launchpad.net/cirros
+
 [debian-6]
 name=Debian 6 (Squeeze)
 osinfo=debian6
+arch=x86_64
 file=debian-6.xz
-sig=debian-6.xz.sig
+revision=2
+checksum=bff9c28da0375fde65fa238d7a2ea644cbfad0ea3246783a2f44a98f2374850987679c3f1032a632d3c6238de8d9e43291d07a82efc1e824945000e206b9f6cc
 format=raw
 size=4294967296
-compressed_size=141531780
+compressed_size=139615908
 expand=/dev/sda1
-notes=This is a default Debian 6 (Squeeze) install.
+notes=Debian 6 (Squeeze).
+ 
+ This is a default Debian install.
+ 
  The preseed and virt-install scripts that produced this image
- can be found in the libguestfs git tree:
- libguestfs.git/builder/website/debian-6.preseed
- libguestfs.git/builder/website/debian-6.sh
+ can be found in the libguestfs source tree:
+ 
+ builder/website/debian.preseed
+ builder/website/debian.sh
  
  This image is so very minimal that it only includes an ssh
  server and no virtual consoles.  To enable virtual consoles
@@ -22,21 +89,31 @@ notes=This is a default Debian 6 (Squeeze) install.
  
  virt-builder debian-6 \
      --edit '/etc/inittab: s,^#([1-9].*respawn.*/sbin/getty.*),$1,'
+ 
+ This image does not contain SSH host keys.  To regenerate them use:
+ 
+ --firstboot-command "dpkg-reconfigure openssh-server"
 
 [debian-7]
 name=Debian 7 (Wheezy)
 osinfo=debian7
+arch=x86_64
 file=debian-7.xz
-sig=debian-7.xz.sig
+revision=2
+checksum=c980a7d878c086b14619b3b9f644ab131c1f88b87729062f2832de12a938ba9e5a511c49402a9d02b740a30876f51889d20e7d0ed4997755f542650b8485f013
 format=raw
 size=4294967296
-compressed_size=149230076
+compressed_size=150734028
 expand=/dev/sda1
-notes=This is a default Debian 7 (Wheezy) install.
+notes=Debian 7 (Wheezy).
+ 
+ This is a default Debian install.
+ 
  The preseed and virt-install scripts that produced this image
- can be found in the libguestfs git tree:
- libguestfs.git/builder/website/debian-7.preseed
- libguestfs.git/builder/website/debian-7.sh
+ can be found in the libguestfs source tree:
+ 
+ builder/website/debian.preseed
+ builder/website/debian.sh
  
  This image is so very minimal that it only includes an ssh
  server and no virtual consoles.  To enable virtual consoles
@@ -44,21 +121,29 @@ notes=This is a default Debian 7 (Wheezy) install.
  
  virt-builder debian-7 \
      --edit '/etc/inittab: s,^#([1-9].*respawn.*/sbin/getty.*),$1,'
+ 
+ This image does not contain SSH host keys.  To regenerate them use:
+ 
+ --firstboot-command "dpkg-reconfigure openssh-server"
 
 [fedora-18]
 name=Fedora® 18
 osinfo=fedora18
+arch=x86_64
 file=fedora-18.xz
-sig=fedora-18.xz.sig
+checksum=12435775193b69f6e22658aaa001d4ca9b15fd68a04b4b7e9be20b3b517e857e417dc3268a302979d4a702b20f25754025f7ae0e9fb7088419a4ca1669585e6f
 format=raw
 size=6442450944
 compressed_size=148947524
 expand=/dev/sda3
-notes=This Fedora image contains only unmodified @Core group packages.
+notes=Fedora 18.
+ 
+ This Fedora image contains only unmodified @Core group packages.
+ 
  It is thus very minimal.  The kickstart and install script can be
- found in the libguestfs git tree:
- libguestfs.git/builder/website/fedora-18.ks
- libguestfs.git/builder/website/fedora-18.sh
+ found in the libguestfs source tree:
+ 
+ builder/website/fedora.sh
  
  Fedora and the Infinity design logo are trademarks of Red Hat, Inc.
  Source and further information is available from http://fedoraproject.org/
@@ -66,82 +151,187 @@ notes=This Fedora image contains only unmodified @Core group packages.
 [fedora-19]
 name=Fedora® 19
 osinfo=fedora19
+arch=x86_64
 file=fedora-19.xz
-sig=fedora-19.xz.sig
+revision=2
+checksum=acecd8d4bca0d6a3f937e0c9386f3185c916df3eaf5f825988c73d41e946a6dc4fda06cdd74a12bc60932edd65846097547b11aca2309a61dd6d0e91ab9d16f3
 format=raw
 size=4294967296
-compressed_size=172190964
+compressed_size=169531628
 expand=/dev/sda3
-notes=This Fedora image contains only unmodified @Core group packages.
+notes=Fedora 19.
+ 
+ This Fedora image contains only unmodified @Core group packages.
+ 
  It is thus very minimal.  The kickstart and install script can be
- found in the libguestfs git tree:
- libguestfs.git/builder/website/fedora-19.ks
- libguestfs.git/builder/website/fedora-19.sh
+ found in the libguestfs source tree:
+ 
+ builder/website/fedora.sh
  
  Fedora and the Infinity design logo are trademarks of Red Hat, Inc.
  Source and further information is available from http://fedoraproject.org/
 
+[fedora-20]
+name=Fedora® 20
+osinfo=fedora20
+arch=x86_64
+file=fedora-20.xz
+revision=2
+checksum=983a1b33c34cb311ea3a283f06312d24dc81041b64ebc90e40ef2fd7587362acd1a5654b13252f9f57001870aa95495065537e730d5225b49389c1a0478cb028
+format=raw
+size=4294967296
+compressed_size=181164220
+expand=/dev/sda3
+notes=Fedora 20.
+ 
+ This Fedora image contains only unmodified @Core group packages.
+ 
+ It is thus very minimal.  The kickstart and install script can be
+ found in the libguestfs source tree:
+ 
+ builder/website/fedora.sh
+ 
+ Fedora and the Infinity design logo are trademarks of Red Hat, Inc.
+ Source and further information is available from http://fedoraproject.org/
+
+[rhel-7rc]
+name=Red Hat Enterprise Linux® 7 Release Candidate
+arch=x86_64
+file=rhel-7rc.xz
+checksum=c4252cfdf7482d4289de24f78e7fd1b9c9b6ff702d717a1c2a54a341b0565c03a5c8e95d32e1f23f7afe5ed2cf5b8e0f0ee3e16c1179e01348a54f82cb552c0f
+format=raw
+size=6442450944
+compressed_size=210202084
+expand=/dev/sda3
+notes=Red Hat Enterprise Linux® 7 Release Candidate
+ 
+ This image contains only unmodified @Core group packages.
+ 
+ It is thus very minimal.  The kickstart and install script can be
+ found in the libguestfs source tree:
+ 
+ builder/website/rhel.sh
+ 
+ Red Hat Enterprise Linux is a trademark of Red Hat, Inc.
+ Source and further information is available from ftp://ftp.redhat.com
+
+[scientificlinux-6]
+name=Scientific Linux 6.5
+osinfo=centos6.5
+arch=x86_64
+file=scientificlinux-6.xz
+revision=5
+checksum=be42b0e97d256ab5476079fe71411bebf59abe9c6ff4a7d27330aa5383fac9507472c06d11ef061d3c0e6695bec5ac9cfb7fbfd86fdbb5649e5cb535e2ac5b88
+format=raw
+size=6442450944
+compressed_size=191667880
+expand=/dev/sda3
+notes=Scientific Linux 6.5
+ 
+ This Scientific Linux image contains only unmodified @Core group packages.
+ 
+ It is thus very minimal.  The kickstart and install script can be
+ found in the libguestfs source tree:
+ 
+ builder/website/scientificlinux.sh
+ 
+ Note that `virt-builder scientificlinux-6' will always install the
+ latest 6.x release.
+
 [ubuntu-10.04]
 name=Ubuntu 10.04 (Lucid)
 osinfo=ubuntulucid
+arch=x86_64
 file=ubuntu-10.04.xz
-sig=ubuntu-10.04.xz.sig
+checksum=9e322782bc77c740ce956d57d0d89ec9ac9f2b890c474ac937f400db848df3e164683eaf4631b989f8b4771f64e0c0972452d1555a15c65714b5ceefb295fb7e
 format=raw
 size=4294967296
 compressed_size=149728112
 expand=/dev/sda1
-notes=This is a minimal Ubuntu 10.04 (Lucid) install.
+notes=Ubuntu 10.04 (Lucid).
+ 
+ This is a minimal Ubuntu 10.04 (Lucid) install.
+ 
  Only the openssh-server package is selected in tasksel.
  The preseed and virt-install scripts that produced this image
- can be found in the libguestfs git tree:
- libguestfs.git/builder/website/ubuntu-10.04.preseed
- libguestfs.git/builder/website/ubuntu-10.04.sh
+ can be found in the libguestfs source tree:
+ 
+ builder/website/ubuntu.preseed
+ builder/website/ubuntu.sh
+ 
+ This image does not contain SSH host keys.  To regenerate them use:
+ 
+ --firstboot-command "dpkg-reconfigure openssh-server"
 
 [ubuntu-12.04]
 name=Ubuntu 12.04 (Precise)
 osinfo=ubuntuprecise
+arch=x86_64
 file=ubuntu-12.04.xz
-sig=ubuntu-12.04.xz.sig
+checksum=15eab4610a44bf28423c08982cec47f6c3d43f29834791440249916ed76cc2dfeaf0405fddc8627086257bbbdaa4016c8982ad9b269f57f8f625b0e70d09e4e0
 format=raw
 size=4294967296
 compressed_size=168593316
 expand=/dev/sda1
-notes=This is a minimal Ubuntu 12.04 (Precise) install.
+notes=Ubuntu 12.04 (Precise).
+ 
+ This is a minimal Ubuntu 12.04 (Precise) install.
+ 
  Only the openssh-server package is selected in tasksel.
  The preseed and virt-install scripts that produced this image
- can be found in the libguestfs git tree:
- libguestfs.git/builder/website/ubuntu-12.04.preseed
- libguestfs.git/builder/website/ubuntu-12.04.sh
+ can be found in the libguestfs source tree:
+ 
+ builder/website/ubuntu.preseed
+ builder/website/ubuntu.sh
+ 
+ This image does not contain SSH host keys.  To regenerate them use:
+ 
+ --firstboot-command "dpkg-reconfigure openssh-server"
 
-[ubuntu-13.10]
-name=Ubuntu 13.10 (Saucy)
-osinfo=ubuntusaucy
-file=ubuntu-13.10.xz
-sig=ubuntu-13.10.xz.sig
+[ubuntu-14.04]
+name=Ubuntu 14.04 (Trusty)
+osinfo=ubuntutrusty
+arch=x86_64
+file=ubuntu-14.04.xz
+checksum=dbc5505c7cfbfb48275b09e91f781dfd7f8d521db577f0c31e0a70b13fbd18c8bcec97b1e274b28062ebc3aba70d68e64c84b9c9690ff7dfca9e81f813047e8a
 format=raw
 size=4294967296
-compressed_size=196481392
+compressed_size=206250764
 expand=/dev/sda1
-notes=This is a minimal Ubuntu 13.10 (Saucy) install.
+notes=Ubuntu 14.04 (Trusty).
+ 
+ This is a minimal Ubuntu 14.04 (Trusty) install.
+ 
  Only the openssh-server package is selected in tasksel.
  The preseed and virt-install scripts that produced this image
- can be found in the libguestfs git tree:
- libguestfs.git/builder/website/ubuntu-13.10.preseed
- libguestfs.git/builder/website/ubuntu-13.10.sh
+ can be found in the libguestfs source tree:
+ 
+ builder/website/ubuntu.preseed
+ builder/website/ubuntu.sh
+ 
+ This image does not contain SSH host keys.  To regenerate them use:
+ 
+ --firstboot-command "dpkg-reconfigure openssh-server"
+ 
+ IMPORTANT NOTE: It seems to be impossible to create an Ubuntu >= 14.04
+ image using preseed without creating a user account.  Therefore this
+ image contains a user account 'builder'.  I have disabled it, so that
+ people who don't read release notes don't get caught out, but you might
+ still wish to delete it completely.
 -----BEGIN PGP SIGNATURE-----
-Version: GnuPG v1.4.14 (GNU/Linux)
+Version: GnuPG v1
 
-iQIcBAEBAgAGBQJSWbIpAAoJEJFzj3Pht2igW1gP/2+vFhng/117EB8Wja2Yx9+f
-xOhrJdJOyGXGFBzUTqVCY8vIdY0VNSJRV6UleKfXqMSNEuH91+hQKiIfTwz+aYLB
-Q9eGJmsa+86JhanKk/TS37gJGHgvfeezKBAy7At5rFmQsXYxgjqSatu3qCM/S+Qk
-F8fbdaSlAXGnfNCTCu4RAoA1HcoFSBL+5cJb6rfz5jWuw4pb+2Lb0sDWuBAjO1GI
-xfuF9+VsltZm36Wv6zHJL2VT7WA0h7FaMCz/xzbqrGZfCPnQh/rJpvSI6Q+k/Vk4
-oiwCiNYfvYuCrOFYqy0can6qCI/4/PJUU4bb+KEvPnksFF42ZGB6INGZWbSheUU2
-dGI0usyHuzJBPNQn7LXq8+/fHR7XyWUXHIVF97s6o/LJ05+ObxbiQl/4/LQh7Ls/
-ry0J6p9NdXCLazmM8R7OOthkmtsQVOh54OSYuvy69/wsPPuds01YDtQnmUN5hCt+
-Ik9qedWmOQOy2/b1ZXBg9xYuvk4wng82P8ESIvMOvg3yr8gutYbx6C333BZU6ly3
-D6/d284lJuTfzmiUNGCKWIPtNsCC3ZgjjELPz4OCT2aq7wYrfNJrV6+Kiau7sD8V
-keC7nERZaRjPHxkScLaCBwBkMbMeDKP6syDkPja8rPw8/e3IA6taqzZ6vcq1+9Nh
-Cdr5jKji1cChryROJR3U
-=pp6M
+iQIcBAEBAgAGBQJTu7WkAAoJEJFzj3Pht2igRNYQAKlLcgtJ9uMMmP70d93fT9dP
+o1Ui29qpXYGEyD5hm+39Dy86FrUYGRbXIMZZX1mS4mkPYfD4zcD6sx0anhEkfSBq
+wBGAQgMHDO1voFHJHpydmqDd3F60DRBycIZnGr9kDcpdwo/AgFcCGZWYSdBR5uSs
+X1oIwZtWRy03rKBDBoCS/XQ68JP3rw9wjQHvOewMijmpVSYcTUQybBhSNsd9Yrdd
+sulALoaJTjjWhQv8XDXI6iQSO9Q7+MWz08fLRqFv8tNkAeWSfIPchVtUHF0WvUVR
+VnVVJEc/hb+MJoYEFAU84wRsryy9frii8C5d2y89oAJB7nX+axS1smkTDxBcZPEQ
+mdAxkoZC1wFlryhgnNIQXDZBwxmIuMHUldExR8FZA8nmSh0SPWGrq9ofpZh0u8N1
+oxSoauDrzWHJuhU15gR+1K3uoidVGTXHM+A4us/flJ0qNJ8+kZuxtSgvcXzha39K
+uSmgk/APGnumKwr8VB9DaM6eC5VqKFnHKk3PttRS+G2bkdUHakcu5TWd0Fr2ISmW
+RJRwQYodd2SLH/bJ3mdR99Uv5ObkzBzq9IIqCD3squPm9Qh6cQUOaJ1bgwMCuvQN
+rx0y9K0CYBg6pPfNgEiRmtItqrJUgvgQEQBfQOQmbcXYTX089HNuTBTXLPWBLiRH
+HE/+U5EuzjFGZzUasFmn
+=bMpw
 -----END PGP SIGNATURE-----

builder/website/rhel.sh

@@ -0,0 +1,181 @@
+#!/bin/bash -
+# virt-builder
+# Copyright (C) 2013 Red Hat Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+unset CDPATH
+export LANG=C
+set -e
+set -x
+
+# Hack for RWMJ
+unset http_proxy
+
+if [ $# -ne 1 ]; then
+    echo "$0 VERSION"
+    exit 1
+fi
+
+version=$1
+output=rhel-$version
+tmpname=tmp-$(tr -cd 'a-f0-9' < /dev/urandom | head -c 8)
+guestroot=/dev/sda3
+
+case $version in
+    5.*)
+        major=5
+        minor=`echo $version | awk -F. '{print $2}'`
+        topurl=http://download.devel.redhat.com/released/RHEL-$major-Server/U$minor
+        tree=$topurl/x86_64/os/Server
+        srpms=$topurl/source/SRPMS
+        bootfs=ext2
+        rootfs=ext4
+        guestroot=/dev/sda2
+        ;;
+    6.*)
+        major=6
+        topurl=http://download.devel.redhat.com/released/RHEL-$major/$version
+        tree=$topurl/Server/x86_64/os
+        srpms=$topurl/source/SRPMS
+        optional=$topurl/Server/optional/x86_64/os
+        optionalsrpms=$topurl/Server/optional/source/SRPMS
+        bootfs=ext4
+        rootfs=ext4
+        ;;
+    7.*)
+        major=7
+        topurl=http://download.devel.redhat.com/released/RHEL-$major/$version
+        tree=$topurl/Server/x86_64/os
+        srpms=$topurl/source/SRPMS
+        optional=$topurl/Server-optional/x86_64/os
+        optionalsrpms=$topurl/Server-optional/source/tree
+        bootfs=ext4
+        rootfs=xfs
+        ;;
+    *)
+        echo "$0: version $version not supported by this script yet"
+        exit 1
+esac
+
+rm -f $output $output.old $output.xz
+
+# Generate the kickstart to a temporary file.
+ks=$(mktemp)
+cat > $ks <<'EOF'
+install
+text
+lang en_US.UTF-8
+keyboard us
+network --bootproto dhcp
+rootpw builder
+firewall --enabled --ssh
+selinux --enforcing
+timezone --utc America/New_York
+EOF
+
+if [ $major -eq 5 ]; then
+cat >> $ks <<EOF
+key --skip
+EOF
+fi
+
+cat >> $ks <<EOF
+bootloader --location=mbr --append="console=tty0 console=ttyS0,115200 rd_NO_PLYMOUTH"
+zerombr
+clearpart --all --initlabel
+part /boot --fstype=$bootfs --size=512         --asprimary
+part swap                   --size=1024        --asprimary
+part /     --fstype=$rootfs --size=1024 --grow --asprimary
+
+# Halt the system once configuration has finished.
+poweroff
+
+%packages
+@core
+EOF
+
+# RHEL 5 didn't understand the %end directive, but RHEL >= 6
+# requires it.
+if [ $major -ge 6 ]; then
+cat >> $ks <<EOF
+%end
+EOF
+fi
+
+# Yum configuration.
+yum=$(mktemp)
+cat > $yum <<EOF
+[rhel$major]
+name=RHEL $major Server
+baseurl=$tree
+enabled=1
+gpgcheck=0
+keepcache=0
+
+[rhel$major-source]
+name=RHEL $major Server Source
+baseurl=$srpms
+enabled=0
+gpgcheck=0
+keepcache=0
+EOF
+
+if [ -n "$optional" ]; then
+cat >> $yum <<EOF
+[rhel$major-optional]
+name=RHEL $major Server Optional
+baseurl=$optional
+enabled=1
+gpgcheck=0
+keepcache=0
+
+[rhel$major-optional-source]
+name=RHEL $major Server Optional
+baseurl=$optionalsrpms
+enabled=0
+gpgcheck=0
+keepcache=0
+EOF
+fi
+
+# Clean up function.
+cleanup ()
+{
+    rm -f $ks
+    rm -f $yum
+    virsh undefine $tmpname ||:
+}
+trap cleanup INT QUIT TERM EXIT ERR
+
+virt-install \
+    --name=$tmpname \
+    --ram=2048 \
+    --cpu=host --vcpus=2 \
+    --os-type=linux --os-variant=rhel$major \
+    --initrd-inject=$ks \
+    --extra-args="ks=file:/`basename $ks` console=tty0 console=ttyS0,115200" \
+    --disk $(pwd)/$output,size=6 \
+    --serial pty \
+    --location=$tree \
+    --nographics \
+    --noreboot
+
+# We have to replace yum config so it doesn't try to use RHN (it
+# won't be registered).
+guestfish --rw -a $output -m $guestroot \
+  upload $yum /etc/yum.repos.d/download.devel.redhat.com.repo
+
+source $(dirname "$0")/compress.sh $output

builder/website/scientificlinux.sh

@@ -0,0 +1,89 @@
+#!/bin/bash -
+# virt-builder
+# Copyright (C) 2013 Red Hat Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+unset CDPATH
+export LANG=C
+set -e
+set -x
+
+if [ $# -ne 1 ]; then
+    echo "$0 VERSION"
+    exit 1
+fi
+
+version=$1
+output=scientificlinux-$version
+tmpname=tmp-$(tr -cd 'a-f0-9' < /dev/urandom | head -c 8)
+major=$(echo $version | awk -F. '{print $1}')
+
+# We rebuild this every time there is a new 6.x release, and bump
+# the revision in the index.
+tree=http://www.mirrorservice.org/sites/ftp.scientificlinux.org/linux/scientific/$version/x86_64/os
+
+rm -f $output $output.old $output.xz
+
+# Generate the kickstart to a temporary file.
+ks=$(mktemp)
+cat > $ks <<'EOF'
+install
+text
+reboot
+lang en_US.UTF-8
+keyboard us
+network --bootproto dhcp
+rootpw builder
+firewall --enabled --ssh
+selinux --enforcing
+timezone --utc America/New_York
+bootloader --location=mbr --append="console=tty0 console=ttyS0,115200 rd_NO_PLYMOUTH"
+zerombr
+clearpart --all --initlabel
+part /boot --fstype=ext4 --size=512         --asprimary
+part swap                --size=1024        --asprimary
+part /     --fstype=ext4 --size=1024 --grow --asprimary
+
+# Halt the system once configuration has finished.
+poweroff
+
+%packages
+@core
+%end
+EOF
+
+# Clean up function.
+cleanup ()
+{
+    rm -f $ks
+    virsh undefine $tmpname ||:
+}
+trap cleanup INT QUIT TERM EXIT ERR
+
+virt-install \
+    --name=$tmpname \
+    --ram=2048 \
+    --cpu=host --vcpus=2 \
+    --os-type=linux --os-variant=rhel$major \
+    --initrd-inject=$ks \
+    --extra-args="ks=file:/`basename $ks` console=tty0 console=ttyS0,115200 proxy=$http_proxy" \
+    --disk $(pwd)/$output,size=6 \
+    --serial pty \
+    --location=$tree \
+    --nographics \
+    --noreboot
+
+source $(dirname "$0")/compress.sh $output

builder/website/test-guest.sh

@@ -0,0 +1,90 @@
+#!/bin/bash -
+# virt-builder
+# Copyright (C) 2013 Red Hat Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+# This is not part of the automated test suite.  It's a manual test
+# run by the maintainer which must be run on each new guest to ensure
+# that all the virt-builder features work on the new guest.
+#
+# Usage:
+# ./run builder/website/test-guest.sh os-version [extra virt-builder args]
+# Then read the instructions ...
+
+export LANG=C
+set -e
+
+if [ ! -x builder/virt-builder ]; then
+    echo "$0: running the test from the wrong directory, or libguestfs has not been built"
+    exit 1
+fi
+
+if [ $# -lt 1 ]; then
+    echo "$0: missing os-version"
+    echo "try: ./run builder/virt-builder -l"
+    exit 1
+fi
+
+osversion="$1"
+shift
+output="$osversion.img"
+
+builder/virt-builder "$osversion" \
+    --no-cache -v \
+    --size 10G \
+    --root-password password:123456 \
+    --hostname test.example.com \
+    --install scrub \
+    --edit '/etc/issue: s/(.*)/$lineno: $1/' \
+    --upload builder/virt-builder.pod:/virt-builder.pod \
+    --run-command 'echo RUN COMMAND 1 >> /run-command.log' \
+    --run-command 'echo RUN COMMAND 2 >> /run-command.log' \
+    --run-command 'echo RUN COMMAND 3 >> /run-command.log' \
+    --firstboot-command 'useradd -m -p "" rjones ; chage -d 0 rjones' \
+    --firstboot-command 'echo FIRSTBOOT COMMAND 1' \
+    --firstboot-command 'echo FIRSTBOOT COMMAND 2' \
+    --firstboot-command 'echo FIRSTBOOT COMMAND 3' \
+    "$@" |& tee "$osversion.log"
+
+# Boot the guest.
+qemu-system-x86_64 \
+    -m 1024 \
+    -drive "file=$output,format=raw,snapshot=on,if=ide" &
+
+cat <<EOF
+
+          ========================================
+The "$osversion" guest is being booted.
+The trace file is here: "$osversion.log"
+
+Checklist:
+
+ 1: Root password is 123456
+ 2: Hostname is test.example.com
+ 3: scrub package is installed
+ 4: /etc/issue has line numbers
+ 5: /virt-builder.pod exists and looks reasonable
+ 6: /run-command.log exists and has 3 lines in correct order
+ 7: /root/virt-sysprep-firstboot.log exists and has 3 entries in correct order
+ 8: rjones account exists, with no password
+ 9: rjones password must be changed at first login
+10: /home/rjones exists and is populated
+11: random-seed file was created or modified
+          ========================================
+
+EOF
+
+#rm $output

builder/website/ubuntu-10.04.sh

@@ -1,85 +0,0 @@
-#!/bin/bash -
-# libguestfs
-# Copyright (C) 2013 Red Hat Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-# The basic plan comes from:
-# http://honk.sigxcpu.org/con/Preseeding_Debian_virtual_machines_with_virt_install.html
-# https://wiki.debian.org/DebianInstaller/Preseed
-# https://help.ubuntu.com/10.04/installation-guide/i386/preseed-using.html
-
-set -e
-set -x
-
-# Some configuration.
-#export http_proxy=http://squid.corp.redhat.com:3128
-#export https_proxy=$http_proxy
-#export ftp_proxy=$http_proxy
-location=http://archive.ubuntu.net/ubuntu/dists/lucid/main/installer-amd64
-
-# Currently you have to run this script as root.
-if [ `id -u` -ne 0 ]; then
-    echo "You have to run this script as root."
-    exit 1
-fi
-
-# Make sure it's being run from the correct directory.
-if [ ! -f ubuntu-10.04.preseed ]; then
-    echo "You are running this script from the wrong directory."
-    exit 1
-fi
-
-pwd=`pwd`
-
-# Note that the injected file must be called "/preseed.cfg" in order
-# for d-i to pick it up.
-sed -e "s,@CACHE@,$http_proxy,g" < ubuntu-10.04.preseed > preseed.cfg
-
-name=tmplucid
-
-virsh undefine $name ||:
-rm -f ubuntu-10.04 ubuntu-10.04.old
-
-virt-install \
-    --name $name \
-    --ram=1024 \
-    --os-type=linux --os-variant=ubuntulucid \
-    --initrd-inject=$pwd/preseed.cfg \
-    --extra-args="auto console=tty0 console=ttyS0,115200" \
-    --disk=$pwd/ubuntu-10.04,size=4 \
-    --location=$location \
-    --nographics \
-    --noreboot
-# The virt-install command should exit after complete installation.
-# Remove the guest, we don't want it to be defined in libvirt.
-virsh undefine $name
-
-rm preseed.cfg
-
-# Sysprep (removes logfiles and so on).
-virt-sysprep -a ubuntu-10.04
-
-# Sparsify.
-mv ubuntu-10.04 ubuntu-10.04.old
-virt-sparsify ubuntu-10.04.old ubuntu-10.04
-rm ubuntu-10.04.old
-
-# Compress.
-rm -f ubuntu-10.04.xz
-xz --best --block-size=16777216 ubuntu-10.04
-
-# Result:
-ls -lh ubuntu-10.04.xz