From 9ba6717e944b6a0d3a1a44ce1637e56fe0595409 Mon Sep 17 00:00:00 2001 From: "Richard W.M. Jones" Date: Tue, 24 Sep 2013 14:00:37 +0100 Subject: [PATCH] New tool: virt-builder: For quickly building virtual machine images. On baremetal you can build and customize a new guest in under 2 minutes. For example: $ virt-builder fedora-19 \ --root-password password:test \ --install minicom \ --firstboot-command 'yum -y update' \ --firstboot-command 'useradd -m -p "" rjones ; chage -d 0 rjones' [ 0.0] Downloading: file:///home/rjones/d/libguestfs/builder/website/fedora-19.xz [ 1.0] Uncompressing: file:///home/rjones/d/libguestfs/builder/website/fedora-19.xz [ 24.0] Running virt-resize to expand the disk to 4.2G [ 77.0] Opening the new disk [ 81.0] Installing packages: minicom [ 94.0] Installing firstboot command: [001] yum -y update [ 94.0] Installing firstboot command: [002] useradd -m -p "" rjones ; chage -d 0 rjones [ 94.0] Finishing off --- .gitignore | 7 + Makefile.am | 5 +- README | 10 + bash/Makefile.am | 3 + bash/virt-resize | 9 +- builder/Makefile.am | 161 +++++ builder/builder.ml | 721 ++++++++++++++++++++++ builder/downloader.ml | 114 ++++ builder/downloader.mli | 38 ++ builder/get_kernel.ml | 121 ++++ builder/get_kernel.mli | 19 + builder/index_parser.ml | 373 ++++++++++++ builder/index_parser.mli | 35 ++ builder/list_entries.ml | 66 +++ builder/list_entries.mli | 19 + builder/sigchecker.ml | 209 +++++++ builder/sigchecker.mli | 28 + builder/test-virt-builder.sh | 20 + builder/virt-builder.pod | 986 +++++++++++++++++++++++++++++++ builder/website/.gitignore | 1 + builder/website/README | 42 ++ builder/website/fedora-18.ks | 38 ++ builder/website/fedora-18.sh | 75 +++ builder/website/fedora-18.xz.sig | 17 + builder/website/fedora-19.ks | 38 ++ builder/website/fedora-19.sh | 75 +++ builder/website/fedora-19.xz.sig | 17 + builder/website/index | 35 ++ builder/website/index.asc | 55 ++ configure.ac | 1 + fish/guestfish.pod | 1 + mllib/Makefile.am | 8 + mllib/common_utils.ml | 51 ++ po-docs/ja/Makefile.am | 1 + po-docs/podfiles | 1 + po-docs/uk/Makefile.am | 1 + po/POTFILES-ml | 7 + run.in | 14 +- src/guestfs.pod | 5 + 39 files changed, 3421 insertions(+), 6 deletions(-) create mode 100644 builder/Makefile.am create mode 100644 builder/builder.ml create mode 100644 builder/downloader.ml create mode 100644 builder/downloader.mli create mode 100644 builder/get_kernel.ml create mode 100644 builder/get_kernel.mli create mode 100644 builder/index_parser.ml create mode 100644 builder/index_parser.mli create mode 100644 builder/list_entries.ml create mode 100644 builder/list_entries.mli create mode 100644 builder/sigchecker.ml create mode 100644 builder/sigchecker.mli create mode 100755 builder/test-virt-builder.sh create mode 100644 builder/virt-builder.pod create mode 100644 builder/website/.gitignore create mode 100644 builder/website/README create mode 100644 builder/website/fedora-18.ks create mode 100755 builder/website/fedora-18.sh create mode 100644 builder/website/fedora-18.xz.sig create mode 100644 builder/website/fedora-19.ks create mode 100755 builder/website/fedora-19.sh create mode 100644 builder/website/fedora-19.xz.sig create mode 100644 builder/website/index create mode 100644 builder/website/index.asc diff --git a/.gitignore b/.gitignore index 305e266d9..55eb95ac1 100644 --- a/.gitignore +++ b/.gitignore @@ -46,6 +46,7 @@ Makefile.in /appliance/stamp-supermin /appliance/supermin.d /autom4te.cache +/bash/virt-builder /bash/virt-cat /bash/virt-df /bash/virt-edit @@ -56,6 +57,10 @@ Makefile.in /bash/virt-sysprep /bash/virt-sparsify /build-aux +/builder/.depend +/builder/stamp-virt-builder.pod +/builder/virt-builder +/builder/virt-builder.1 /cat/stamp-virt-*.pod /cat/virt-cat /cat/virt-cat.1 @@ -206,6 +211,7 @@ Makefile.in /html/libguestfs-make-fixed-appliance.1.html /html/libguestfs-test-tool.1.html /html/virt-alignment-scan.1.html +/html/virt-builder.1.html /html/virt-cat.1.html /html/virt-copy-in.1.html /html/virt-copy-out.1.html @@ -264,6 +270,7 @@ Makefile.in /mllib/common_gettext.ml /mllib/common_utils_tests /mllib/dummy +/mllib/libdir.ml /ocaml/bindtests.bc /ocaml/bindtests.opt /ocaml/bindtests.ml diff --git a/Makefile.am b/Makefile.am index 802d13553..18d7cce39 100644 --- a/Makefile.am +++ b/Makefile.am @@ -118,7 +118,7 @@ SUBDIRS += csharp # OCaml tools. Note 'mllib' contains random shared code used by # all of the OCaml tools. if HAVE_OCAML -SUBDIRS += mllib resize sparsify sysprep +SUBDIRS += mllib builder resize sparsify sysprep endif # Perl tools. @@ -206,6 +206,7 @@ HTMLFILES = \ html/libguestfs-make-fixed-appliance.1.html \ html/libguestfs-test-tool.1.html \ html/virt-alignment-scan.1.html \ + html/virt-builder.1.html \ html/virt-cat.1.html \ html/virt-copy-in.1.html \ html/virt-copy-out.1.html \ @@ -275,7 +276,7 @@ all-local: grep -v -E '/((guestfs|rc)_protocol\.c)$$' | \ LC_ALL=C sort > po/POTFILES cd $(srcdir); \ - find mllib resize sparsify sysprep -name '*.ml' | \ + find builder mllib resize sparsify sysprep -name '*.ml' | \ LC_ALL=C sort > po/POTFILES-ml # Manual pages in top level directory. diff --git a/README b/README index 8e3fe932d..5186e9d6e 100644 --- a/README +++ b/README @@ -162,6 +162,16 @@ The full requirements are described below. +--------------+-------------+---+-----------------------------------------+ | uml_mkcow | | O | For the UML backend. | +--------------+-------------+---+-----------------------------------------+ +| curl | | O | Used by virt-builder for downloads | ++--------------+-------------+---+-----------------------------------------+ +| gpg | | O | Used by virt-builder for digital | +| | | | signatures | ++--------------+-------------+---+-----------------------------------------+ +| xz | | O | Used by virt-builder for compression | ++--------------+-------------+---+-----------------------------------------+ +| nbdkit | | O | Used by virt-builder to speed up | +| | | | template xz-decompression | ++--------------+-------------+---+-----------------------------------------+ | findlib | | O | For the OCaml bindings. | +--------------+-------------+---+-----------------------------------------+ | ocaml-gettext| | O | For localizing OCaml virt-* tools. | diff --git a/bash/Makefile.am b/bash/Makefile.am index 65f38cb74..44b945bb9 100644 --- a/bash/Makefile.am +++ b/bash/Makefile.am @@ -21,6 +21,7 @@ scripts = \ guestfish \ guestmount \ virt-alignment-scan \ + virt-builder \ virt-cat \ virt-df \ virt-edit \ @@ -55,6 +56,8 @@ virt-ls: virt-sysprep: ln -sf virt-alignment-scan $@ +virt-builder: + ln -sf virt-resize $@ virt-sparsify: ln -sf virt-resize $@ diff --git a/bash/virt-resize b/bash/virt-resize index d9770dcae..cc308cc2a 100644 --- a/bash/virt-resize +++ b/bash/virt-resize @@ -1,4 +1,5 @@ -# virt-resize, virt-sparsify bash completion script -*- shell-script -*- +# virt-resize, virt-builder, virt-sparsify bash completion script +# -*- shell-script -*- # Copyright (C) 2010-2013 Red Hat Inc. # # This program is free software; you can redistribute it and/or modify @@ -33,6 +34,12 @@ _guestfs_options_only () esac } +_virt_builder () +{ + _guestfs_options_only "$(virt-builder --long-options)" +} && +complete -o default -F _virt_builder virt-builder + _virt_resize () { _guestfs_options_only "$(virt-resize --long-options)" diff --git a/builder/Makefile.am b/builder/Makefile.am new file mode 100644 index 000000000..fec0f69e3 --- /dev/null +++ b/builder/Makefile.am @@ -0,0 +1,161 @@ +# libguestfs virt-builder tool +# Copyright (C) 2013 Red Hat Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +include $(top_srcdir)/subdir-rules.mk + +EXTRA_DIST = \ + $(SOURCES) \ + virt-builder.pod \ + test-virt-builder.sh \ + website/.gitignore \ + website/README \ + website/index \ + website/index.asc \ + website/fedora-18.ks \ + website/fedora-18.sh \ + website/fedora-18.xz.sig \ + website/fedora-19.ks \ + website/fedora-19.sh \ + website/fedora-19.xz.sig + +CLEANFILES = *~ *.cmi *.cmo *.cmx *.cmxa *.o virt-builder + +# Alphabetical order. +SOURCES = \ + builder.ml \ + downloader.mli \ + downloader.ml \ + get_kernel.mli \ + get_kernel.ml \ + index_parser.mli \ + index_parser.ml \ + list_entries.mli \ + list_entries.ml \ + sigchecker.mli \ + sigchecker.ml + +if HAVE_OCAML + +# Note this list must be in dependency order. +OBJECTS = \ + $(top_builddir)/mllib/libdir.cmx \ + $(top_builddir)/mllib/common_gettext.cmx \ + $(top_builddir)/mllib/common_utils.cmx \ + $(top_builddir)/mllib/random_seed.cmx \ + $(top_builddir)/mllib/firstboot.cmx \ + $(top_builddir)/mllib/crypt-c.o \ + $(top_builddir)/mllib/crypt.cmx \ + $(top_builddir)/mllib/password.cmx \ + get_kernel.cmx \ + downloader.cmx \ + sigchecker.cmx \ + index_parser.cmx \ + list_entries.cmx \ + builder.cmx + +bin_SCRIPTS = virt-builder + +# -I $(top_builddir)/src/.libs is a hack which forces corresponding -L +# option to be passed to gcc, so we don't try linking against an +# installed copy of libguestfs. +OCAMLPACKAGES = \ + -package str,unix \ + -I $(top_builddir)/src/.libs \ + -I $(top_builddir)/ocaml \ + -I $(top_builddir)/mllib +if HAVE_OCAML_PKG_GETTEXT +OCAMLPACKAGES += -package gettext-stub +endif + +OCAMLCFLAGS = -g -warn-error CDEFLMPSUVYZX $(OCAMLPACKAGES) +OCAMLOPTFLAGS = $(OCAMLCFLAGS) + +virt-builder: $(OBJECTS) + $(OCAMLFIND) ocamlopt $(OCAMLOPTFLAGS) \ + mlguestfs.cmxa -linkpkg $^ \ + -cclib '-lncurses -lcrypt' \ + $(OCAML_GCOV_LDFLAGS) \ + -o $@ + +.mli.cmi: + $(OCAMLFIND) ocamlc $(OCAMLCFLAGS) -c $< -o $@ +.ml.cmo: + $(OCAMLFIND) ocamlc $(OCAMLCFLAGS) -c $< -o $@ +.ml.cmx: + $(OCAMLFIND) ocamlopt $(OCAMLOPTFLAGS) -c $< -o $@ + +# automake will decide we don't need C support in this file. Really +# we do, so we have to provide it ourselves. + +DEFAULT_INCLUDES = \ + -I. \ + -I$(top_builddir) \ + -I$(shell $(OCAMLC) -where) \ + -I$(top_srcdir)/src \ + -I$(top_srcdir)/fish + +.c.o: + $(CC) $(CFLAGS) $(PROF_CFLAGS) $(DEFAULT_INCLUDES) -c $< -o $@ + +# Manual pages and HTML files for the website. + +man_MANS = virt-builder.1 + +noinst_DATA = $(top_builddir)/html/virt-builder.1.html + +virt-builder.1 $(top_builddir)/html/virt-builder.1.html: stamp-virt-builder.pod + +stamp-virt-builder.pod: virt-builder.pod + $(PODWRAPPER) \ + --man virt-builder.1 \ + --html $(top_builddir)/html/virt-builder.1.html \ + --license GPLv2+ \ + $< + touch $@ + +CLEANFILES += stamp-virt-builder.pod + +# Tests. + +TESTS_ENVIRONMENT = $(top_builddir)/run --test + +if ENABLE_APPLIANCE +TESTS = test-virt-builder.sh +endif ENABLE_APPLIANCE + +check-valgrind: + $(MAKE) VG="$(top_builddir)/run @VG@" check + +# Dependencies. +depend: .depend + +.depend: $(wildcard $(abs_srcdir)/*.mli) $(wildcard $(abs_srcdir)/*.ml) + rm -f $@ $@-t + $(OCAMLFIND) ocamldep -I ../ocaml -I $(abs_srcdir) -I $(top_srcdir)/mllib $^ | \ + $(SED) 's/ *$$//' | \ + $(SED) -e :a -e '/ *\\$$/N; s/ *\\\n */ /; ta' | \ + $(SED) -e 's,$(abs_srcdir)/,$(builddir)/,g' | \ + sort > $@-t + mv $@-t $@ + +-include .depend + +endif + +DISTCLEANFILES = .depend + +.PHONY: depend docs diff --git a/builder/builder.ml b/builder/builder.ml new file mode 100644 index 000000000..9a3b13c1d --- /dev/null +++ b/builder/builder.ml @@ -0,0 +1,721 @@ +(* virt-builder + * Copyright (C) 2013 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +open Common_gettext.Gettext + +module G = Guestfs + +open Common_utils +open Password + +open Unix +open Printf + +let quote = Filename.quote + +(* Command line argument parsing. *) +let prog = Filename.basename Sys.executable_name + +let cachedir = + try Some (Sys.getenv "XDG_CACHE_HOME" // "virt-builder") + with Not_found -> + try Some (Sys.getenv "HOME" // ".cache" // "virt-builder") + with Not_found -> + None (* no cache directory *) + +let mode, arg, + attach, cache, check_signature, curl, debug, fingerprint, + firstboot, run, + format, gpg, install, list_long, network, output, + password_crypto, quiet, root_password, + size, source, upload = + let display_version () = + let g = new G.guestfs () in + let version = g#version () in + printf (f_"virt-builder %Ld.%Ld.%Ld%s\n") + version.G.major version.G.minor version.G.release version.G.extra; + exit 0 + in + + let mode = ref `Install in + let list_mode () = mode := `List in + let get_kernel_mode () = mode := `Get_kernel in + let delete_cache_mode () = mode := `Delete_cache in + + let attach = ref [] in + let attach_format = ref None in + let set_attach_format = function + | "auto" -> attach_format := None + | s -> attach_format := Some s + in + let attach_disk s = attach := (!attach_format, s) :: !attach in + + let cache = ref cachedir in + let set_cache arg = cache := Some arg in + let no_cache () = cache := None in + + let check_signature = ref true in + let curl = ref "curl" in + let debug = ref false in + + let fingerprint = + try Some (Sys.getenv "VIRT_BUILDER_FINGERPRINT") + with Not_found -> None in + let fingerprint = ref fingerprint in + let set_fingerprint fp = fingerprint := Some fp in + + let firstboot = ref [] in + let add_firstboot s = + if not (Sys.file_exists s) then ( + eprintf (f_"%s: --firstboot: %s: file not found.\n") prog s; + exit 1 + ); + firstboot := `Script s :: !firstboot + in + let add_firstboot_cmd s = firstboot := `Command s :: !firstboot in + let add_firstboot_install pkgs = + let pkgs = string_nsplit "," pkgs in + firstboot := `Packages pkgs :: !firstboot + in + + let format = ref "" in + let gpg = ref "gpg" in + + let install = ref [] in + let add_install pkgs = + let pkgs = string_nsplit "," pkgs in + install := pkgs @ !install + in + + let list_long = ref false in + let network = ref true in + let output = ref "" in + + let password_crypto : password_crypto option ref = ref None in + let set_password_crypto arg = + password_crypto := Some (password_crypto_of_string ~prog arg) + in + + let quiet = ref false in + + let root_password = ref None in + let set_root_password arg = + let pw = get_password ~prog arg in + root_password := Some pw + in + + let run = ref [] in + let add_run s = + if not (Sys.file_exists s) then ( + eprintf (f_"%s: --run: %s: file not found.\n") prog s; + exit 1 + ); + run := `Script s :: !run + in + let add_run_cmd s = run := `Command s :: !run in + + let size = ref None in + let set_size arg = size := Some (parse_size ~prog arg) in + + let source = + try Sys.getenv "VIRT_BUILDER_SOURCE" + with Not_found -> "http://libguestfs.org/download/builder/index.asc" in + let source = ref source in + + let upload = ref [] in + let add_upload arg = + let i = + try String.index arg ':' + with Not_found -> + eprintf (f_"%s: invalid --upload format, see the man page.\n") prog; + exit 1 in + let len = String.length arg in + let file = String.sub arg 0 i in + if not (Sys.file_exists file) then ( + eprintf (f_"%s: --upload: %s: file not found.\n") prog file; + exit 1 + ); + let dest = String.sub arg (i+1) (len-(i+1)) in + upload := (file, dest) :: !upload + in + + let ditto = " -\"-" in + let argspec = Arg.align [ + "--attach", Arg.String attach_disk, "iso" ^ " " ^ s_"Attach data disk/ISO during install"; + "--attach-format", Arg.String set_attach_format, + "format" ^ " " ^ s_"Set attach disk format"; + "--cache", Arg.String set_cache, "dir" ^ " " ^ s_"Set template cache dir"; + "--no-cache", Arg.Unit no_cache, " " ^ s_"Disable template cache"; + "--check-signature", Arg.Set check_signature, + " " ^ s_"Check digital signatures"; + "--check-signatures", Arg.Set check_signature, ditto; + "--no-check-signature", Arg.Clear check_signature, + " " ^ s_"Disable digital signatures"; + "--no-check-signatures", Arg.Clear check_signature, ditto; + "--curl", Arg.Set_string curl, "curl" ^ " " ^ s_"Set curl binary/command"; + "--delete-cache", Arg.Unit delete_cache_mode, + " " ^ s_"Delete the template cache"; + "--fingerprint", Arg.String set_fingerprint, + "AAAA.." ^ " " ^ s_"Fingerprint of valid signing key"; + "--firstboot", Arg.String add_firstboot, "script" ^ " " ^ s_"Run script at first guest boot"; + "--firstboot-command", Arg.String add_firstboot_cmd, "cmd+args" ^ " " ^ s_"Run command at first guest boot"; + "--firstboot-install", Arg.String add_firstboot_install, + "pkg,pkg" ^ " " ^ s_"Add package(s) to install at firstboot"; + "--format", Arg.Set_string format, "raw|qcow2" ^ " " ^ s_"Output format (default: raw)"; + "--get-kernel", Arg.Unit get_kernel_mode, + "image" ^ " " ^ s_"Get kernel from image"; + "--gpg", Arg.Set_string gpg, "gpg" ^ " " ^ s_"Set GPG binary/command"; + "--install", Arg.String add_install, "pkg,pkg" ^ " " ^ s_"Add package(s) to install"; + "-l", Arg.Unit list_mode, " " ^ s_"List available templates"; + "--list", Arg.Unit list_mode, ditto; + "--long", Arg.Set list_long, ditto; + "--long-options", Arg.Unit display_long_options, " " ^ s_"List long options"; + "--network", Arg.Set network, " " ^ s_"Enable appliance network (default)"; + "--no-network", Arg.Clear network, " " ^ s_"Disable appliance network"; + "-o", Arg.Set_string output, "file" ^ " " ^ s_"Set output filename"; + "--output", Arg.Set_string output, "file" ^ ditto; + "--password-crypto", Arg.String set_password_crypto, + "md5|sha256|sha512" ^ " " ^ s_"Set password crypto"; + "--quiet", Arg.Set quiet, " " ^ s_"No progress messages"; + "--root-password", Arg.String set_root_password, + "..." ^ " " ^ s_"Set root password"; + "--run", Arg.String add_run, "script" ^ " " ^ s_"Run script in disk image"; + "--run-command", Arg.String add_run_cmd, "cmd+args" ^ " " ^ s_"Run command in disk image"; + "--size", Arg.String set_size, "size" ^ " " ^ s_"Set output disk size"; + "--source", Arg.Set_string source, "URL" ^ " " ^ s_"Set source URL"; + "--upload", Arg.String add_upload, "file:dest" ^ " " ^ s_"Upload file to dest"; + "-v", Arg.Set debug, " " ^ s_"Enable debugging messages"; + "--verbose", Arg.Set debug, ditto; + "-V", Arg.Unit display_version, " " ^ s_"Display version and exit"; + "--version", Arg.Unit display_version, ditto; + ] in + long_options := argspec; + + let args = ref [] in + let anon_fun s = args := s :: !args in + let usage_msg = + sprintf (f_"\ +%s: build virtual machine images quickly + +A short summary of the options is given below. For detailed help please +read the man page virt-builder(1). +") + prog in + Arg.parse argspec anon_fun usage_msg; + + (* Dereference options. *) + let args = List.rev !args in + let mode = !mode in + let attach = List.rev !attach in + let cache = !cache in + let check_signature = !check_signature in + let curl = !curl in + let debug = !debug in + let fingerprint = !fingerprint in + let firstboot = List.rev !firstboot in + let run = List.rev !run in + let format = match !format with "" -> None | s -> Some s in + let gpg = !gpg in + let install = !install in + let list_long = !list_long in + let network = !network in + let output = match !output with "" -> None | s -> Some s in + let password_crypto = !password_crypto in + let quiet = !quiet in + let root_password = !root_password in + let size = !size in + let source = !source in + let upload = List.rev !upload in + + (* Check options. *) + let arg = + match mode with + | `Install -> + (match args with + | [arg] -> arg + | [] -> + eprintf (f_"%s: virt-builder os-version\nMissing 'os-version'. Use '--list' to list available template names.\n") prog; + exit 1 + | _ -> + eprintf (f_"%s: virt-builder: too many parameters, expecting 'os-version'\n") prog; + exit 1 + ) + | `List -> + (match args with + | [] -> "" + | _ -> + eprintf (f_"%s: virt-builder --list does not need any extra arguments.\n") prog; + exit 1 + ) + | `Delete_cache -> + (match args with + | [] -> "" + | _ -> + eprintf (f_"%s: virt-builder --delete-cache does not need any extra arguments.\n") prog; + exit 1 + ) + | `Get_kernel -> + (match args with + | [arg] -> arg + | [] -> + eprintf (f_"%s: virt-builder --get-kernel image\nMissing 'image' (disk image file) argument.\n") prog; + exit 1 + | _ -> + eprintf (f_"%s: virt-builder --get-kernel: too many parameters\n") prog; + exit 1 + ) in + + mode, arg, + attach, cache, check_signature, curl, debug, fingerprint, + firstboot, run, + format, gpg, install, list_long, network, output, + password_crypto, quiet, root_password, + size, source, upload + +(* Timestamped messages in ordinary, non-debug non-quiet mode. *) +let msg fs = make_message_function ~quiet fs + +(* If debugging, echo the command line arguments. *) +let () = + if debug then ( + eprintf "command line:"; + List.iter (eprintf " %s") (Array.to_list Sys.argv); + prerr_newline () + ) + +(* --get-kernel is really a different program ... *) +let () = + if mode = `Get_kernel then ( + Get_kernel.get_kernel ~debug ?format ?output arg; + exit 0 + ) + +let () = + if mode = `Delete_cache then ( + match cachedir with + | Some cachedir -> + msg "Deleting: %s" cachedir; + let cmd = sprintf "rm -rf %s" (quote cachedir) in + ignore (Sys.command cmd); + exit 0 + | None -> + eprintf (f_"%s: error: could not find cache directory\nIs $HOME set?\n") + prog; + exit 1 + ) + +(* Check various programs/dependencies are installed. *) +let have_nbdkit = + (* Check that gpg is installed. Optional as long as the user + * disables all signature checks. + *) + let cmd = sprintf "%s --help >/dev/null 2>&1" gpg in + if Sys.command cmd <> 0 then ( + if check_signature then ( + eprintf (f_"%s: gpg is not installed (or does not work)\nYou should install gpg, or use --gpg option, or use --no-check-signature.\n") prog; + exit 1 + ) + else if debug then + eprintf (f_"%s: warning: gpg program is not available\n") prog + ); + + (* Check that curl works. *) + let cmd = sprintf "%s --help >/dev/null 2>&1" curl in + if Sys.command cmd <> 0 then ( + eprintf (f_"%s: curl is not installed (or does not work)\n") prog; + exit 1 + ); + + (* Check that virt-resize works. *) + let cmd = "virt-resize --help >/dev/null 2>&1" in + if Sys.command cmd <> 0 then ( + eprintf (f_"%s: virt-resize is not installed (or does not work)\n") prog; + exit 1 + ); + + (* Find out if nbdkit + nbdkit-xz-plugin is installed (optional). *) + let cmd = + sprintf "nbdkit %s/nbdkit/plugins/nbdkit-xz-plugin.so --help >/dev/null 2>&1" + Libdir.libdir in + let have_nbdkit = Sys.command cmd = 0 in + if not have_nbdkit && debug then + eprintf (f_"%s: warning: nbdkit or nbdkit-xz-plugin is not available\n") + prog; + + have_nbdkit + +(* Create the cache directory. *) +let cache = + match cache with + | None -> None + | (Some dir) as cache -> + (try mkdir dir 0o755 with _ -> ()); + if Sys.is_directory dir then cache else None + +(* Make the downloader and signature checker abstract data types. *) +let downloader = + Downloader.create ~debug ~curl ~cache +let sigchecker = + Sigchecker.create ~debug ~gpg ?fingerprint ~check_signature + +(* Download the source (index) file. *) +let index = + Index_parser.get_index ~debug ~downloader ~sigchecker source + +(* Now we can do the --list option. *) +let () = + if mode = `List then ( + List_entries.list_entries ~list_long ~source index; + exit 0 + ) + +(* If we get here, we want to create a guest (but which one?) *) +let entry = + assert (mode = `Install); + + try List.assoc arg index + with Not_found -> + eprintf (f_"%s: cannot find os-version '%s'.\nUse --list to list available guest types.\n") + prog arg; + exit 1 + +(* Download the template, or it may be in the cache. *) +let template = + let template, delete_on_exit = + let { Index_parser.revision = revision; file_uri = file_uri } = entry in + let template = arg, revision in + msg (f_"Downloading: %s") file_uri; + Downloader.download downloader ~template file_uri in + if delete_on_exit then unlink_on_exit template; + template + +(* Check the signature of the file. *) +let () = + let sigfile = + match entry with + | { Index_parser.signature_uri = None } -> None + | { Index_parser.signature_uri = Some signature_uri } -> + let sigfile, delete_on_exit = + Downloader.download downloader signature_uri in + if delete_on_exit then unlink_on_exit sigfile; + Some sigfile in + + Sigchecker.verify_detached sigchecker template sigfile + +(* Check the --size option. *) +let headroom = 256L *^ 1024L *^ 1024L +let size = + let { Index_parser.size = default_size } = entry in + match size with + | None -> default_size +^ headroom + | Some size -> + if size < default_size +^ headroom then ( + eprintf (f_"%s: --size is too small for this disk image, minimum size is %s\n") + prog (human_size default_size); + exit 1 + ); + size + +(* Create the output file. *) +let output, format = + match output, format with + | None, None -> sprintf "%s.img" arg, "raw" + | None, Some "raw" -> sprintf "%s.img" arg, "raw" + | None, Some format -> sprintf "%s.%s" arg format, format + | Some output, None -> output, "raw" + | Some output, Some format -> output, format + +let delete_output_file = + let cmd = + sprintf "qemu-img create -f %s %s %Ld%s" + (quote format) (quote output) size + (if debug then "" else " >/dev/null 2>&1") in + let r = Sys.command cmd in + if r <> 0 then ( + eprintf (f_"%s: error: could not create output file '%s'\n") prog output; + exit 1 + ); + (* This ensures the output file will be deleted on failure, + * until we set !delete_output_file = false at the end of the build. + *) + let delete_output_file = ref true in + let delete_file () = + if !delete_output_file then + try unlink output with _ -> () + in + at_exit delete_file; + delete_output_file + +let source = + (* XXX Disable this for now because libvirt is broken: + * https://bugzilla.redhat.com/show_bug.cgi?id=1011063 + *) + if have_nbdkit && false then ( + (* If we have nbdkit, then we can use NBD to uncompress the xz + * file on the fly. + *) + let socket = Filename.temp_file "vbnbd" ".sock" in + let source = sprintf "nbd://?socket=%s" socket in + let argv = [| "nbdkit"; "-r"; "-f"; "-U"; socket; + Libdir.libdir // "nbdkit/plugins/nbdkit-xz-plugin.so"; + "file=" ^ template |] in + let pid = + match fork () with + | 0 -> (* child *) + execvp "nbdkit" argv + | pid -> pid in + (* Clean up when the program exits. *) + let clean_up () = + (try kill pid Sys.sigterm with _ -> ()); + (try unlink socket with _ -> ()) + in + at_exit clean_up; + source + ) + else ( + (* Otherwise we have to uncompress it to a temporary file. *) + let { Index_parser.file_uri = file_uri } = entry in + let tmpfile = Filename.temp_file "vbsrc" ".img" in + let cmd = sprintf "xzcat %s > %s" (quote template) (quote tmpfile) in + if debug then eprintf "%s\n%!" cmd; + msg (f_"Uncompressing: %s") file_uri; + let r = Sys.command cmd in + if r <> 0 then ( + eprintf (f_"%s: error: failed to uncompress template\n") prog; + exit 1 + ); + unlink_on_exit tmpfile; + tmpfile + ) + +(* Resize the source to the output file. *) +let () = + msg (f_"Running virt-resize to expand the disk to %s") (human_size size); + + let { Index_parser.expand = expand; lvexpand = lvexpand; format = format } = + entry in + let cmd = + sprintf "virt-resize%s%s%s%s %s %s" + (if debug then " --verbose" else " --quiet") + (match format with + | None -> "" + | Some format -> sprintf " --format %s" (quote format)) + (match expand with + | None -> "" + | Some expand -> sprintf " --expand %s" (quote expand)) + (match lvexpand with + | None -> "" + | Some lvexpand -> sprintf " --lv-expand %s" (quote lvexpand)) + (quote source) (quote output) in + if debug then eprintf "%s\n%!" cmd; + let r = Sys.command cmd in + if r <> 0 then ( + eprintf (f_"%s: error: virt-resize failed\n") prog; + exit 1 + ) + +(* Now mount the output disk so we can make changes. *) +let g = + msg (f_"Opening the new disk"); + + let g = new G.guestfs () in + if debug then g#set_trace true; + + g#set_network network; + + g#add_drive_opts ~format output; + + (* Attach ISOs, if we have any. *) + List.iter ( + fun (format, file) -> + g#add_drive_opts ?format ~readonly:true file; + ) attach; + + g#launch (); + + g + +(* Inspect the disk and mount it up. *) +let root = + match Array.to_list (g#inspect_os ()) with + | [root] -> + let mps = g#inspect_get_mountpoints root in + let cmp (a,_) (b,_) = + compare (String.length a) (String.length b) in + let mps = List.sort cmp mps in + List.iter ( + fun (mp, dev) -> + try g#mount dev mp + with Guestfs.Error msg -> eprintf (f_"%s: %s (ignored)\n") prog msg + ) mps; + root + | _ -> + eprintf (f_"%s: no guest operating systems or multiboot OS found in this disk image\nThis is a failure of the source repository. Use -v for more information.\n") prog; + exit 1 + +(* Set the random seed. *) +let () = ignore (Random_seed.set_random_seed g root) + +(* Useful wrapper for scripts. *) +let do_run cmd = + (* Add a prologue to the scripts: + * - Pass environment variables through from the host. + * - Send stdout to stderr so we capture all output in error messages. + *) + let env_vars = + filter_map ( + fun name -> + try Some (sprintf "export %s=%s" name (quote (Sys.getenv name))) + with Not_found -> None + ) [ "http_proxy"; "https_proxy"; "ftp_proxy" ] in + let env_vars = String.concat "\n" env_vars ^ "\n" in + + let cmd = sprintf "\ +exec 1>&2 +%s +%s +" env_vars cmd in + + if debug then eprintf "running: %s\n%!" cmd; + ignore (g#sh cmd) + +let guest_install_command packages = + let quoted_args = String.concat " " (List.map quote packages) in + match g#inspect_get_package_management root with + | "apt" -> + sprintf "apt-get -y install %s" quoted_args + | "pisi" -> + sprintf "pisi it %s" quoted_args + | "pacman" -> + sprintf "pacman -S %s" quoted_args + | "urpmi" -> + sprintf "urpmi %s" quoted_args + | "yum" -> + sprintf "yum -y install %s" quoted_args + | "zypper" -> + (* XXX Should we use -n option? *) + sprintf "zypper in %s" quoted_args + | "unknown" -> + eprintf (f_"%s: --install is not supported for this guest operating system\n") + prog; + exit 1 + | pm -> + eprintf (f_"%s: sorry, don't know how to use --install with the '%s' package manager\n") + prog pm; + exit 1 + +(* Install packages. *) +let () = + if install <> [] then ( + msg (f_"Installing packages: %s") (String.concat " " install); + + let cmd = guest_install_command install in + do_run cmd; + ) + +(* Root password. + * Note 'None' means that we randomize the root password. + *) +let () = + let make_random_password () = + (* Get random characters from the set [A-Za-z0-9] *) + let chars = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" in + let nr_chars = String.length chars in + + let chan = open_in "/dev/urandom" in + let buf = String.create 16 in + for i = 0 to 15 do + buf.[i] <- chars.[Char.code (input_char chan) mod nr_chars] + done; + close_in chan; + + msg "Random root password: %s [did you mean to use --root-password?]" buf; + + buf + in + + let root_password = + match root_password with + | Some pw -> pw + | None -> make_random_password () in + + match g#inspect_get_type root with + | "linux" -> + let h = Hashtbl.create 1 in + Hashtbl.replace h "root" root_password; + set_linux_passwords ~prog ?password_crypto g root h + | _ -> + () + +(* Upload files. *) +let () = + List.iter ( + fun (file, dest) -> + msg (f_"Uploading: %s") dest; + g#upload file dest + ) upload + +(* Firstboot scripts/commands/install. *) +let () = + let id = ref 0 in + List.iter ( + fun op -> + incr id; + let id = sprintf "%03d" !id in + match op with + | `Script script -> + msg (f_"Installing firstboot script: [%s] %s") id script; + let cmd = read_whole_file script in + Firstboot.add_firstboot_script g root id cmd + | `Command cmd -> + msg (f_"Installing firstboot command: [%s] %s") id cmd; + Firstboot.add_firstboot_script g root id cmd + | `Packages pkgs -> + msg (f_"Installing firstboot packages: [%s] %s") id + (String.concat " " pkgs); + let cmd = guest_install_command pkgs in + Firstboot.add_firstboot_script g root id cmd + ) firstboot + +(* Run scripts. *) +let () = + List.iter ( + function + | `Script script -> + msg (f_"Running: %s") script; + let cmd = read_whole_file script in + do_run cmd + | `Command cmd -> + msg (f_"Running: %s") cmd; + do_run cmd + ) run + +(* Unmount everything and we're done! *) +let () = + msg "Finishing off"; + + g#umount_all (); + g#shutdown (); + g#close () + +(* Now that we've finished the build, don't delete the output file on + * exit. + *) +let () = + delete_output_file := false diff --git a/builder/downloader.ml b/builder/downloader.ml new file mode 100644 index 000000000..cb76b75a6 --- /dev/null +++ b/builder/downloader.ml @@ -0,0 +1,114 @@ +(* virt-builder + * Copyright (C) 2013 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +open Common_gettext.Gettext + +open Unix +open Printf + +let quote = Filename.quote +let (//) = Filename.concat + +type uri = string +type filename = string + +type t = { + debug : bool; + curl : string; + cache : string option; (* cache directory for templates *) +} + +let create ~debug ~curl ~cache = { + debug = debug; + curl = curl; + cache = cache; +} + +let rec download t ?template uri = + match template with + | None -> (* no cache, simple download *) + (* Create a temporary name. *) + let tmpfile = Filename.temp_file "vbcache" ".txt" in + download_to t uri tmpfile; + (tmpfile, true) + + | Some (name, revision) -> + match t.cache with + | None -> + (* Not using the cache at all? *) + download t uri + + | Some cachedir -> + let filename = cachedir // sprintf "%s.%d" name revision in + + (* Is the requested template name + revision in the cache already? + * If not, download it. + *) + if not (Sys.file_exists filename) then + download_to t uri filename; + + (filename, false) + +and download_to t uri filename = + (* Get the status code first to ensure the file exists. *) + let cmd = sprintf "%s%s -g -o /dev/null -I -w '%%{http_code}' %s" + t.curl (if t.debug then "" else " -s -S") (quote uri) in + let chan = open_process_in cmd in + let status_code = input_line chan in + let stat = close_process_in chan in + (match stat with + | WEXITED 0 -> () + | WEXITED i -> + eprintf (f_"virt-builder: curl (download) command failed downloading '%s'\n") uri; + exit 1 + | WSIGNALED i -> + eprintf (f_"virt-builder: external command '%s' killed by signal %d\n") + cmd i; + exit 1 + | WSTOPPED i -> + eprintf (f_"virt-builder: external command '%s' stopped by signal %d\n") + cmd i; + exit 1 + ); + let bad_status_code = function + | "" -> true + | s when s.[0] = '4' -> true (* 4xx *) + | s when s.[0] = '5' -> true (* 5xx *) + | _ -> false + in + if bad_status_code status_code then ( + eprintf (f_"virt-builder: failed to download %s: HTTP status code %s\n") + uri status_code; + exit 1 + ); + + (* Now download the file. *) + let filename_new = filename ^ ".new" in + let cmd = sprintf "%s%s -g -o %s %s" + t.curl (if t.debug then "" else " -s -S") + (quote filename_new) (quote uri) in + if t.debug then eprintf "%s\n%!" cmd; + let r = Sys.command cmd in + if r <> 0 then ( + eprintf (f_"virt-builder: curl (download) command failed downloading '%s'\n") uri; + (try unlink filename_new with _ -> ()); + exit 1 + ); + + (* Rename the file if curl was successful. *) + rename filename_new filename diff --git a/builder/downloader.mli b/builder/downloader.mli new file mode 100644 index 000000000..ed91fc0be --- /dev/null +++ b/builder/downloader.mli @@ -0,0 +1,38 @@ +(* virt-builder + * Copyright (C) 2013 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +(** This module is a wrapper around curl, plus local caching. *) + +type uri = string +type filename = string + +type t +(** The abstract data type. *) + +val create : debug:bool -> curl:string -> cache:string option -> t +(** Create the abstract type. *) + +val download : t -> ?template:(string*int) -> uri -> (filename * bool) +(** Download the URI, returning the downloaded filename and a + temporary file flag. The temporary file flag is [true] iff + the downloaded file is temporary and should be deleted by the + caller (otherwise it's in the cache and you shouldn't delete it). + + For templates, you must supply [~template:(name, revision)]. This + causes the cache to be used (if possible). Name and revision are + used for cache control (see the man page for details). *) diff --git a/builder/get_kernel.ml b/builder/get_kernel.ml new file mode 100644 index 000000000..5138c9159 --- /dev/null +++ b/builder/get_kernel.ml @@ -0,0 +1,121 @@ +(* virt-builder + * Copyright (C) 2013 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +open Common_gettext.Gettext +open Common_utils + +module G = Guestfs + +open Printf + +let rex_numbers = Str.regexp "^\\([0-9]+\\)\\(.*\\)$" +let rex_letters = Str.regexp_case_fold "^\\([a-z]+\\)\\(.*\\)$" + +(* Originally: + * http://rwmj.wordpress.com/2013/09/13/get-kernel-and-initramfs-from-a-disk-image/ + *) +let rec get_kernel ~debug ?format ?output disk = + let g = new G.guestfs () in + if debug then g#set_trace true; + g#add_drive_opts ?format ~readonly:true disk; + g#launch (); + + let roots = g#inspect_os () in + if Array.length roots = 0 then ( + eprintf (f_"virt-builder: get-kernel: no operating system found\n"); + exit 1 + ); + if Array.length roots > 1 then ( + eprintf (f_"virt-builder: get-kernel: daual/mult-boot images are not supported by this tool\n"); + exit 1 + ); + let root = roots.(0) in + + (* Mount up the disks. *) + let mps = g#inspect_get_mountpoints root in + let cmp (a,_) (b,_) = compare (String.length a) (String.length b) in + let mps = List.sort cmp mps in + List.iter ( + fun (mp, dev) -> + try g#mount_ro dev mp + with Guestfs.Error msg -> eprintf "%s (ignored)\n" msg + ) mps; + + (* Get all kernels and initramfses. *) + let glob w = Array.to_list (g#glob_expand w) in + let kernels = glob "/boot/vmlinuz-*" in + let initrds = glob "/boot/initramfs-*" in + + (* Old RHEL: *) + let initrds = if initrds <> [] then initrds else glob "/boot/initrd-*" in + + (* Debian/Ubuntu: *) + let initrds = if initrds <> [] then initrds else glob "/boot/initrd.img-*" in + + (* Sort by version to get the latest version as first element. *) + let kernels = List.rev (List.sort compare_version kernels) in + let initrds = List.rev (List.sort compare_version initrds) in + + if kernels = [] then ( + eprintf (f_"virt-builder: no kernel found\n"); + exit 1 + ); + + (* Download the latest. *) + let outputdir = + match output with + | None -> Filename.current_dir_name + | Some dir -> dir in + let kernel_in = List.hd kernels in + let kernel_out = outputdir // Filename.basename kernel_in in + printf "download: %s -> %s\n%!" kernel_in kernel_out; + g#download kernel_in kernel_out; + + if initrds <> [] then ( + let initrd_in = List.hd initrds in + let initrd_out = outputdir // Filename.basename initrd_in in + printf "download: %s -> %s\n%!" initrd_in initrd_out; + g#download initrd_in initrd_out + ); + + (* Shutdown. *) + g#shutdown (); + g#close () + +and compare_version v1 v2 = + compare (split_version v1) (split_version v2) + +and split_version = function + | "" -> [] + | str -> + let first, rest = + if Str.string_match rex_numbers str 0 then ( + let n = Str.matched_group 1 str in + let rest = Str.matched_group 2 str in + let n = + try `Number (int_of_string n) + with Failure "int_of_string" -> `String n in + n, rest + ) + else if Str.string_match rex_letters str 0 then + `String (Str.matched_group 1 str), Str.matched_group 2 str + else ( + let len = String.length str in + `Char str.[0], String.sub str 1 (len-1) + ) in + first :: split_version rest diff --git a/builder/get_kernel.mli b/builder/get_kernel.mli new file mode 100644 index 000000000..7c48f256d --- /dev/null +++ b/builder/get_kernel.mli @@ -0,0 +1,19 @@ +(* virt-builder + * Copyright (C) 2013 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +val get_kernel : debug:bool -> ?format:string -> ?output:string -> string -> unit diff --git a/builder/index_parser.ml b/builder/index_parser.ml new file mode 100644 index 000000000..16d8dc721 --- /dev/null +++ b/builder/index_parser.ml @@ -0,0 +1,373 @@ +(* virt-builder + * Copyright (C) 2013 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +open Common_gettext.Gettext +open Common_utils + +open Printf +open Unix + +type index = (string * entry) list (* string = "os-version" *) +and entry = { + printable_name : string option; (* the name= field *) + osinfo : string option; + file_uri : string; + signature_uri : string option; + revision : int; + format : string option; + size : int64; + compressed_size : int64 option; + expand : string option; + lvexpand : string option; + notes : string option; + hidden : bool; +} + +let print_entry chan (name, { printable_name = printable_name; + file_uri = file_uri; + osinfo = osinfo; + signature_uri = signature_uri; + revision = revision; + format = format; + size = size; + compressed_size = compressed_size; + expand = expand; + lvexpand = lvexpand; + notes = notes; + hidden = hidden }) = + let fp fs = fprintf chan fs in + fp "[%s]\n" name; + (match printable_name with + | None -> () + | Some name -> fp "name=%s\n" name + ); + (match osinfo with + | None -> () + | Some id -> fp "osinfo=%s\n" id + ); + fp "file=%s\n" file_uri; + (match signature_uri with + | None -> () + | Some uri -> fp "sig=%s\n" uri + ); + fp "revision=%d\n" revision; + (match format with + | None -> () + | Some format -> fp "format=%s\n" format + ); + fp "size=%Ld\n" size; + (match compressed_size with + | None -> () + | Some size -> fp "compressed_size=%Ld\n" size + ); + (match expand with + | None -> () + | Some expand -> fp "expand=%s\n" expand + ); + (match lvexpand with + | None -> () + | Some lvexpand -> fp "lvexpand=%s\n" lvexpand + ); + (match notes with + | None -> () + | Some notes -> fp "notes=%s\n" notes + ); + if hidden then fp "hidden=true\n" + +let fieldname_rex = Str.regexp "^\\([a-z_]+\\)=\\(.*\\)$" + +let get_index ~debug ~downloader ~sigchecker source = + let rec corrupt_line line = + eprintf (f_"virt-builder: error parsing index near this line:\n\n%s\n") + line; + corrupt_file () + and corrupt_file () = + eprintf (f_"\nThe index file downloaded from '%s' is corrupt.\nYou need to ask the supplier of this file to fix it and upload a fixed version.\n") + source; + exit 1 + in + + let rec get_index () = + (* Get the index page. *) + let tmpfile, delete_tmpfile = Downloader.download downloader source in + + (* Check index file signature (also verifies it was fully + * downloaded and not corrupted in transit). + *) + Sigchecker.verify sigchecker tmpfile; + + (* Check the index page is not too huge. *) + let st = stat tmpfile in + if st.st_size > 1_000_000 then ( + eprintf (f_"virt-builder: index page '%s' is too large (size %d bytes)\n") + source st.st_size; + exit 1 + ); + + (* Load the file into memory. *) + let index = read_whole_file tmpfile in + if delete_tmpfile then + (try Unix.unlink tmpfile with _ -> ()); + + (* Split file into lines. *) + let index = string_nsplit "\n" index in + + (* If there is a signature (checked above) then remove it. *) + let index = + match index with + | "-----BEGIN PGP SIGNED MESSAGE-----" :: lines -> + (* Ignore all lines until we get to first blank. *) + let lines = dropwhile ((<>) "") lines in + (* Ignore the blank line too. *) + let lines = List.tl lines in + (* Take lines until we get to the end signature. *) + let lines = takewhile ((<>) "-----BEGIN PGP SIGNATURE-----") lines in + lines + | _ -> index in + + (* Split into sections around each /^[/ *) + let rec loop = function + | [] -> [] + | x :: xs when String.length x >= 1 && x.[0] = '[' -> + let lines = takewhile ((<>) "") xs in + let rest = dropwhile ((<>) "") xs in + if rest = [] then + [x, lines] + else ( + let rest = List.tl rest in + let rest = loop rest in + (x, lines) :: rest + ) + | x :: _ -> corrupt_line x + in + let sections = loop index in + + (* Parse the fields in each section. *) + let isspace = function ' ' | '\t' -> true | _ -> false in + let starts_space str = String.length str >= 1 && isspace str.[0] in + let rec loop = function + | [] -> [] + | x :: xs when not (starts_space x) && String.contains x '=' -> + let xs' = takewhile starts_space xs in + let ys = dropwhile starts_space xs in + (x :: xs') :: loop ys + | x :: _ -> corrupt_line x + in + let sections = List.map (fun (n, lines) -> n, loop lines) sections in + + if debug then ( + eprintf "index file (%s) after splitting:\n" source; + List.iter ( + fun (n, fields) -> + eprintf " os-version: %s\n" n; + let i = ref 0 in + List.iter ( + fun field -> + eprintf " %d: " !i; + List.iter prerr_endline field; + incr i + ) fields + ) sections + ); + + (* Now we've parsed the file into the correct sections, we + * interpret the meaning of the fields. + *) + let sections = List.map ( + fun (n, fields) -> + let len = String.length n in + if len < 3 || n.[0] <> '[' || n.[len-1] <> ']' then + corrupt_line n; + let n = String.sub n 1 (len-2) in + + let fields = List.map ( + function + | [] -> assert false (* can never happen, I think? *) + | x :: xs when Str.string_match fieldname_rex x 0 -> + let field = Str.matched_group 1 x in + let rest_of_line = Str.matched_group 2 x in + let allow_multiline = + match field with + | "name" -> false + | "osinfo" -> false + | "file" -> false + | "sig" -> false + | "revision" -> false + | "format" -> false + | "size" -> false + | "compressed_size" -> false + | "expand" -> false + | "lvexpand" -> false + | "notes" -> true + | "hidden" -> false + | _ -> + eprintf "warning: unknown field '%s' in index (ignored)\n%!" + field; + true in + let value = + if not allow_multiline then ( + if xs <> [] then ( + eprintf (f_"virt-builder: field '%s' cannot span multiple lines\n") + field; + corrupt_line (List.hd xs) + ); + rest_of_line + ) else ( + String.concat "\n" (rest_of_line :: xs) + ) in + field, value + | x :: _ -> + corrupt_line x + ) fields in + + (n, fields) + ) sections in + + (* Check for repeated os-version names. *) + let nseen = Hashtbl.create 13 in + List.iter ( + fun (n, _) -> + if Hashtbl.mem nseen n then ( + eprintf (f_"virt-builder: index is corrupt: os-version '%s' appears two or more times\n") n; + corrupt_file () + ); + Hashtbl.add nseen n true + ) sections; + + (* Check for repeated fields. *) + List.iter ( + fun (n, fields) -> + let fseen = Hashtbl.create 13 in + List.iter ( + fun (field, _) -> + if Hashtbl.mem fseen field then ( + eprintf (f_"virt-builder: index is corrupt: %s: field '%s' appears two or more times\n") n field; + corrupt_file () + ); + Hashtbl.add fseen field true + ) fields + ) sections; + + (* Turn the sections into the final index. *) + let entries = + List.map ( + fun (n, fields) -> + let printable_name = + try Some (List.assoc "name" fields) with Not_found -> None in + let osinfo = + try Some (List.assoc "osinfo" fields) with Not_found -> None in + let file_uri = + try make_absolute_uri (List.assoc "file" fields) + with Not_found -> + eprintf (f_"virt-builder: no 'file' (URI) entry for '%s'\n") n; + corrupt_file () in + let signature_uri = + try Some (make_absolute_uri (List.assoc "sig" fields)) + with Not_found -> None in + let revision = + try int_of_string (List.assoc "revision" fields) + with + | Not_found -> 1 + | Failure "int_of_string" -> + eprintf (f_"virt-builder: cannot parse 'revision' field for '%s'\n") + n; + corrupt_file () in + let format = + try Some (List.assoc "format" fields) with Not_found -> None in + let size = + try Int64.of_string (List.assoc "size" fields) + with + | Not_found -> + eprintf (f_"virt-builder: no 'size' field for '%s'\n") n; + corrupt_file () + | Failure "int_of_string" -> + eprintf (f_"virt-builder: cannot parse 'size' field for '%s'\n") + n; + corrupt_file () in + let compressed_size = + try Some (Int64.of_string (List.assoc "compressed_size" fields)) + with + | Not_found -> + None + | Failure "int_of_string" -> + eprintf (f_"virt-builder: cannot parse 'compressed_size' field for '%s'\n") + n; + corrupt_file () in + let expand = + try Some (List.assoc "expand" fields) with Not_found -> None in + let lvexpand = + try Some (List.assoc "lvexpand" fields) with Not_found -> None in + let notes = + try Some (List.assoc "notes" fields) with Not_found -> None in + let hidden = + try bool_of_string (List.assoc "hidden" fields) + with + | Not_found -> false + | Failure "bool_of_string" -> + eprintf (f_"virt-builder: cannot parse 'hidden' field for '%s'\n") + n; + corrupt_file () in + + let entry = { printable_name = printable_name; + osinfo = osinfo; + file_uri = file_uri; + signature_uri = signature_uri; + revision = revision; + format = format; + size = size; + compressed_size = compressed_size; + expand = expand; + lvexpand = lvexpand; + notes = notes; + hidden = hidden } in + n, entry + ) sections in + + if debug then ( + eprintf "index file (%s) after parsing:\n" source; + List.iter (print_entry Pervasives.stderr) entries + ); + + entries + + (* Verify same-origin policy for the file= and sig= fields. *) + and make_absolute_uri path = + if String.length path = 0 then ( + eprintf (f_"virt-builder: zero length path in the index file\n"); + corrupt_file () + ) + else if string_find path "://" >= 0 then ( + eprintf (f_"virt-builder: cannot use a URI ('%s') in the index file\n") + path; + corrupt_file () + ) + else if path.[0] = '/' then ( + eprintf (f_"virt-builder: you must use relative paths (not '%s') in the index file\n") path; + corrupt_file () + ) + else ( + (* Construct the URI. *) + try + let i = String.rindex source '/' in + String.sub source 0 (i+1) ^ path + with + Not_found -> source // path + ) + in + + get_index () diff --git a/builder/index_parser.mli b/builder/index_parser.mli new file mode 100644 index 000000000..35b34e5fd --- /dev/null +++ b/builder/index_parser.mli @@ -0,0 +1,35 @@ +(* virt-builder + * Copyright (C) 2013 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +type index = (string * entry) list (* string = "os-version" *) +and entry = { + printable_name : string option; (* the name= field *) + osinfo : string option; + file_uri : string; + signature_uri : string option; + revision : int; + format : string option; + size : int64; + compressed_size : int64 option; + expand : string option; + lvexpand : string option; + notes : string option; + hidden : bool; +} + +val get_index : debug:bool -> downloader:Downloader.t -> sigchecker:Sigchecker.t -> string -> index diff --git a/builder/list_entries.ml b/builder/list_entries.ml new file mode 100644 index 000000000..b233f0e49 --- /dev/null +++ b/builder/list_entries.ml @@ -0,0 +1,66 @@ +(* virt-builder + * Copyright (C) 2013 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +open Common_gettext.Gettext +open Common_utils + +open Printf + +let list_entries ?(list_long = false) ~source index = + if list_long then ( + printf (f_"Source URI: %s\n") source; + printf "\n" + ); + + List.iter ( + fun (name, { Index_parser.printable_name = printable_name; + size = size; + compressed_size = compressed_size; + notes = notes; + hidden = hidden }) -> + if not hidden then ( + if not list_long then ( (* Short *) + printf "%-24s" name; + (match printable_name with + | None -> () + | Some s -> printf " %s" s + ); + printf "\n" + ) + else ( (* Long *) + printf "%-24s %s\n" "os-version:" name; + (match printable_name with + | None -> () + | Some name -> printf "%-24s %s\n" (s_"Full name:") name; + ); + printf "%-24s %s\n" (s_"Minimum/default size:") (human_size size); + (match compressed_size with + | None -> () + | Some size -> + printf "%-24s %s\n" (s_"Download size:") (human_size size); + ); + (match notes with + | None -> () + | Some notes -> + printf "\n"; + printf "Notes:\n %s\n" notes + ); + printf "\n" + ) + ) + ) index diff --git a/builder/list_entries.mli b/builder/list_entries.mli new file mode 100644 index 000000000..e1d5c06db --- /dev/null +++ b/builder/list_entries.mli @@ -0,0 +1,19 @@ +(* virt-builder + * Copyright (C) 2013 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +val list_entries : ?list_long:bool -> source:string -> Index_parser.index -> unit diff --git a/builder/sigchecker.ml b/builder/sigchecker.ml new file mode 100644 index 000000000..3c14aba88 --- /dev/null +++ b/builder/sigchecker.ml @@ -0,0 +1,209 @@ +(* virt-builder + * Copyright (C) 2013 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +open Common_gettext.Gettext +open Common_utils + +open Printf +open Unix + +let quote = Filename.quote + +(* These are the public key and fingerprint belonging to + * Richard W.M. Jones who signs the templates on + * http://libguestfs.org/download/builder. + *) +let default_fingerprint = "F777 4FB1 AD07 4A7E 8C87 67EA 9173 8F73 E1B7 68A0" +let default_pubkey = "\ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1.4.14 (GNU/Linux) + +mQINBE6UMMEBEADM811hfTulaF4JpkVpAI10FImyb4ArvOiu8NdcUwTFo+cyWno3 +U85B86H1Bsk/LgLTYtthSrTgsCtdxy+i5OaMjxZDIwKQ2+IYI3FCn9T3Mn28Idyh +kLHzrO9ph0Dv0BNfrlDZhQEC53aAFe/QxN7+A49BNBV7D1VAOOCsHjxMEDzcZkCa +oCrtXw1aNm2vkkj5ukbfukHAyLcQL7kow0qKPSVa1G4lfQP0WiG259Ydy+sUmbVb +TGdb6MEC84PQRDuw6/ZeoV04tn7ZNtQEMOS0uiciHOGfr2hBxQf9VIPNrHg42yaL +dOv51D99GuaxZ9E0HSoH/RwB1oXgd6rFdqVNYaBIQnnkwJANUEeGBArtIOZNCADT +Bt8vkSDm+lLEAFS+V8CACyW/LMIrGCvLdHeqtoAv0GDVyR2GPxldYfdtEmCUMWcb +Jlf71V9iAse2gUdoiHp5FfpGMkA5j7idKuxIws11XxRZJXXbBqiBqmVEAQ/v0m6p +kdo0MYTHydmecLuUK2bAGhpysfX97EfTSrxfrYphYWjTfKRD9GrADeZNfuz1DbKs +7LSqVaQJSjQrfgAwcnZLRaU0V4P5zxiz50gz1Aj3AZRL+Y3meZenzZTXcLFdnusg +wUfhhCuL3tluMtEh6tznumyxb43WO1yLwj6J6LtveiuJN1Z+KSQ6OieZcwARAQAB +tCVSaWNoYXJkIFcuTS4gSm9uZXMgPHJpY2hAYW5uZXhpYS5vcmc+iQI4BBMBAgAi +BQJOlDDBAhsDBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRCRc49z4bdooHQY +D/wJLklSZNyXIW+rG5sUbg7j9cTIF5p/lB9kI2yx6KodJp/2knKyvnmzz0gBw/OE +HL4E4UW26oWKo+36I8wkBnuGa6UtANeITcJqFE19VpHEXHsxre64jNQnO8/w748W +1ROW+Ry43xmrlRWKuCm4oPYUzlp0fq9ATAne8eblfG+NOs8DYuA8xZNQzFaI2kDC +QLD4YoXLoNsP27Koga36b0KwxPFD9tyVZiu9XDH/3hMN7Nb15B66PFr+HcMmQ67G +nUIN5ulcIwj38i40cyaTs1VRheOzTHXE/a6Q2AhMKiKqOoEjQ73/mV7cAVoPtM3o +83Q/8aVKBH0bVRwAeV1tju6b14fqKoG0zNBEcXdlSkht6ScxJYIc/LPUxAMDwgSE +OWshjmeRzKXypBbHn/DP8QVyM2gk5wY+mMSH7MpR0p/hgj+rFO8H9L7pC4dCog3E +qzrYhRN+TaP6MPH3WkOwPH4d4IfQRFnHp+VPYPijKEiLrUl/o8k3DyAanAPBpJ/x +na4wXAjlFBctOq6g+SrCUiHpwk7b2YNwGgr5Vl3GmZELzK/G8gg3uJYKQ9Bpv16t +WWOz+IFiOFa0UULeo0QPmFAIMZiDojNsY1SwBKB3ZL1YWZezgMdQAbpze/IXoSt7 +zxWJoKH2jK7q9mvFiaY12l2YnKuCcegWVAViLxRpBnrbz7QmUmljaGFyZCBXLk0u +IEpvbmVzIDxyam9uZXNAcmVkaGF0LmNvbT6JAjgEEwECACIFAk6UOQsCGwMGCwkI +BwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJEJFzj3Pht2igIUYQAKomI0edLakahsUQ +MxOZuhBbXJ4/VWF8bXYChDNPKvJp5nB7fBXujJ+39cIUM5fe2ViO6qSDpFC29imx +F5pPbAqspZBPBkLLiZLji8R42hGarntdtTW0UWSBpq+nC5+G1psrnATI3uXGNxKQ +R99c5HoMY7dBC2Y8TCGE64NINZ/XVh472s6IGLPn8MTn26YdRKC9BrVkCFMP2OBr +6D4IprnyTAWAzb68ew20QmyWO+NBi9MplaDNQVl8PIOgfpyWlkgX1z9m67pcSDkw +46hksp0yuOD1VwR4iVZ2/CmIsGRUlx41vWD6BIp9KxKyDIU1CYTRhq72dahHsl/8 +BjCndV5PO0GphqfCzmCv4DXjUwmrMTbH/GFnt5rfwcMcXUgcK0vV9vQ2SOU56Zd1 +fb27ZCFJKZc0Fu8krwFldCp/NYILf6ogUL/C1hfuCGSSuyDVY16Gg3dla1x+6zpF +asnWQlaw8xT5LlMWvTZs5WsoSVHu7dVZWlgxINP++hlZrTz/S8l38yyQ15YFFl3W +9M7dzkegOeDTPfx6B89WgfvfJjA/D0/FYxxWPXEtrn9DlJ4daEJqNsrvfLErz9R8 +4IQmfmhR93j+rdotner+6keC/wVByEfbW1wmXtmFKXQ6srdpj8VKRFrvkyXVgepM +DypLgRH2v7lL2kdWhUu2y4EAgrwzuQINBE6UMMEBEADxQxMgUuDrw5GT4tqARTPI +SSdNcUsRxRhVA8srYOyECliE+B3TwcRDFBs+MyPFJVEuX8fi4eGj/AK5t1GHerfk +orUGlz72q4c7LLhkfZrsuJbk2dgkjvldKJnIazQJa6epGLqdsE5RlmSgwedIbtMd +naGJBQH8aKP/Wi1+wUxsm5N3p7+R2WRx48VfpEhYB+Zf/FkFm1Ycjwh57KQ0+OHw +ykf8VfMisxuH30tDxOCV+VptWKfOF2rDNdaNPWhij2YIjhJXRpkuRR+1PpI4jLaD +JxcVZmG/0zucacupUN2g5OUH59ySU/totD6YMnmp3FONoyF1uIEJo6Vs30npHGkO +XgBo3Pxt7oLJeykLPtdSLgm3cwXIYMWarVsAkKNXitQIVGpVRLeaK373VwmXFqoi +M2SMHeawTUdOORFjpQzkknlJWM1TmUVtHHKt8Pl9+/5+wXKyt2IDdcUkMrB6K5qF +fb7EwVhoI8ehJQK+eeDCjFwCAiwB3iV8JlyW+tEU7JuyXOQlwY1VWm/WqMD8gaRi +rT+RFDFliZ3tQbW2pqUoZBROV5HN4tieDfwxGKCvk6Tsdb30zA9DPQp93+238bYf +312sg9R+CD0AqxoxFG5FJu4HShcPRrPnYtRZqKRe40GDWvBEArXZprwL1qrP+Kl/ +mRrEQpxAGIoFG8HbVvD3EQARAQABiQIfBBgBAgAJBQJOlDDBAhsMAAoJEJFzj3Ph +t2igSLQP/2uIrAY2CDr0kWBJiD3TztiHy8IdxwUpyTBTebwmAbi44/EvtJfIisrG +YjKIEv/w0E61gO7O1JBG4+IG93W+v9fTT/e39JMyxsYqoZZHUhP11Okx5grDS5b0 +O8VXOmXVRMdVNfstRBr10HD9uNDq7ruKD18TxYTwN0GPD4gj1dbHQDR77Tr5cyBs +6Ou5PBOH4r3qcqf/cJUSMeUUu75xLwixux6E7tD2S+t6F07wlWxntUcPtzyAHj20 +J89orUC+dT6r6MypBoI0jdJCp9JPGtR7i+fE5Gm4E5+AUSubLPtZGRY9Um2eMoS2 +DnQpGOKx1VvsixR/Kw44j2tRAvmYMS4iDKcuZU+nZ+xokAgObILj/b9n/Qe2/fXy +CFdcgSvbm+dV1fZxsdMF/P9OU8aqdT9A9Fv5y+cDMEg4DVnhwMJTxGh/TCkw/H+A +frHEtRc98lSQN5odpITNG17mG6JOdHM+wA57qHH0uy4+5RsbyAJahcdBcmObK/RF +i4WZlThpbHftX5O/LH98aYQ2fJayIxv1EAjzOBOQ0MfBHI0KCJR1pysEisX28sJA +Ic73gnJJ3BLZbqfBRgxjNMNroxC+5Tw6uPGFHa3YnuIAxxw0HcDVZ9vnTWBWFPGw +ZvXkQ3FVJwZoLmHw47vvlVpLD/4gi1SuHWieRvZ+UdDq00E348pm +=neBW +-----END PGP PUBLIC KEY BLOCK----- +" +let key_imported = ref false + +type t = { + debug : bool; + gpg : string; + fingerprint : string; + check_signature : bool; +} + +let create ~debug ~gpg ?(fingerprint = default_fingerprint) ~check_signature = + { + debug = debug; + gpg = gpg; + fingerprint = fingerprint; + check_signature = check_signature; + } + +(* Compare two strings of hex digits ignoring whitespace and case. *) +let rec equal_fingerprints fp1 fp2 = + let len1 = String.length fp1 and len2 = String.length fp2 in + let rec loop i j = + if i = len1 && j = len2 then true (* match! *) + else if i = len1 || j = len2 then false (* no match - different lengths *) + else ( + let x1 = getxdigit fp1.[i] and x2 = getxdigit fp2.[j] in + match x1, x2 with + | Some x1, Some x2 when x1 = x2 -> loop (i+1) (j+1) + | Some x1, Some x2 -> false (* no match - different content *) + | Some _, None -> loop i (j+1) + | None, Some _ -> loop (i+1) j + | None, None -> loop (i+1) (j+1) + ) + in + loop 0 0 + +and getxdigit = function + | '0'..'9' as c -> Some (Char.code c - Char.code '0') + | 'a'..'f' as c -> Some (Char.code c - Char.code 'a') + | 'A'..'F' as c -> Some (Char.code c - Char.code 'A') + | _ -> None + +let rec verify t filename = + if t.check_signature then ( + let args = quote filename in + do_verify t args + ) + +and verify_detached t filename sigfile = + if t.check_signature then ( + match sigfile with + | None -> + eprintf (f_"virt-builder: error: there is no detached signature file\nThis probably means the index file is missing a sig=... line.\nYou can use --no-check-signature to ignore this error, but that means\nyou are susceptible to man-in-the-middle attacks.\n"); + exit 1 + | Some sigfile -> + let args = sprintf "%s %s" (quote sigfile) (quote filename) in + do_verify t args + ) + +and do_verify t args = + import_key t; + + let status_file = Filename.temp_file "vbstat" ".txt" in + let cmd = + sprintf "%s --verify%s --status-file %s %s" + t.gpg (if t.debug then "" else " -q --logger-file /dev/null") + (quote status_file) args in + if t.debug then eprintf "%s\n%!" cmd; + let r = Sys.command cmd in + if r <> 0 then ( + eprintf (f_"virt-builder: error: GPG failure: could not verify digital signature of file\nTry:\n - Use the '-v' option and look for earlier error messages.\n - Delete the cache: virt-builder --delete-cache\n - Check no one has tampered with the website or your network!\n"); + exit 1 + ); + + (* Check the fingerprint is who it should be. *) + let status = read_whole_file status_file in + unlink status_file; + + let status = string_nsplit "\n" status in + let fingerprint = ref "" in + List.iter ( + fun line -> + let line = string_nsplit " " line in + match line with + | "[GNUPG:]" :: "VALIDSIG" :: fp :: _ -> fingerprint := fp + | _ -> () + ) status; + + if not (equal_fingerprints !fingerprint t.fingerprint) then ( + eprintf (f_"virt-builder: error: fingerprint of signature does not match the expected fingerprint!\n found fingerprint: %s\n expected fingerprint: %s\n") + !fingerprint t.fingerprint; + exit 1 + ) + +(* Import the default public key, if it's the default fingerprint. *) +and import_key t = + if not !key_imported && equal_fingerprints t.fingerprint default_fingerprint + then ( + let filename, chan = Filename.open_temp_file "vbpubkey" ".asc" in + output_string chan default_pubkey; + close_out chan; + + let cmd = sprintf "%s --import %s%s" + t.gpg (quote filename) + (if t.debug then "" else " >/dev/null 2>&1") in + let r = Sys.command cmd in + if r <> 0 then ( + eprintf (f_"virt-builder: error: could not import public key\nUse the '-v' option and look for earlier error messages.\n"); + exit 1 + ); + unlink filename; + key_imported := true + ) diff --git a/builder/sigchecker.mli b/builder/sigchecker.mli new file mode 100644 index 000000000..dd930add7 --- /dev/null +++ b/builder/sigchecker.mli @@ -0,0 +1,28 @@ +(* virt-builder + * Copyright (C) 2013 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + *) + +type t + +val create : debug:bool -> gpg:string -> ?fingerprint:string -> check_signature:bool -> t + +val verify : t -> string -> unit +(** Verify the file is signed (if check_signature is true). *) + +val verify_detached : t -> string -> string option -> unit +(** Verify the file is signed against the detached signature + (if check_signature is true). *) diff --git a/builder/test-virt-builder.sh b/builder/test-virt-builder.sh new file mode 100755 index 000000000..1a1964225 --- /dev/null +++ b/builder/test-virt-builder.sh @@ -0,0 +1,20 @@ +#!/bin/bash - +# libguestfs virt-builder test script +# Copyright (C) 2013 Red Hat Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +export LANG=C +set -e diff --git a/builder/virt-builder.pod b/builder/virt-builder.pod new file mode 100644 index 000000000..d4314269a --- /dev/null +++ b/builder/virt-builder.pod @@ -0,0 +1,986 @@ +=encoding utf8 + +=head1 NAME + +virt-builder - Build virtual machine images quickly + +=head1 SYNOPSIS + + virt-builder [-o|--output DISKIMAGE] [--size SIZE] [--format raw|qcow2] + [--attach ISOFILE] + [--install PKG,[PKG...]] + [--root-password ...] + [--upload FILE:DEST] + [--run SCRIPT] [--run-command 'CMD ARGS ...'] + [--firstboot SCRIPT] [--firstboot-command 'CMD ARGS ...'] + [--firstboot-install PKG,[PKG...]] + os-version + + virt-builder -l|--list [--long] + + virt-builder --delete-cache + + virt-builder --get-kernel DISKIMAGE + [--format raw|qcow2] [--output OUTPUTDIR] + +=head1 DESCRIPTION + +Virt-builder is a tool for quickly building new virtual machines. You +can build a variety of VMs for local or cloud use, usually within a +few minutes or less. Virt-builder also has many ways to customize +these VMs. Everything is run from the command line and nothing +requires root privileges, so automation and scripting is simple. + +Note that virt-builder does not install guests from scratch. It takes +cleanly prepared, digitally signed OSes and customizes them. This +approach is used because it is much faster, but if you need to do +fresh installs you may want to look at L and +L. + +The easiest way to get started is by looking at the examples in the +next section. + +=head1 EXAMPLES + +=head2 List the virtual machines available + + virt-builder --list + +will list out the operating systems available to install. A selection +of freely redistributable OSes is available as standard. You can add +your own too (see below). + +=head2 Build a virtual machine + + virt-builder fedora-20 + +will build a Fedora 20 image. This will have all default +configuration (minimal size, no user accounts, random root password, +only the bare minimum installed software, etc.). + +Note you I. + +The first time this runs it has to download the template over the +network, but this gets cached (see L). + +The name of the output file is derived from the template name, so +above it will be C. You can change the output filename +using the I<-o> option: + + virt-builder fedora-20 -o mydisk.img + +You can also use the I<-o> option to write to existing devices or +logical volumes. + + virt-builder fedora-20 --format qcow2 + +As above, but write the output in qcow2 format to C. + + virt-builder fedora-20 --size 20G + +As above, but the output size will be 20 GB. The guest OS is resized +as it is copied to the output (automatically, using +L). + +=head2 Setting the root password + + virt-builder fedora-20 --root-password file:/tmp/rootpw + +Create a Fedora 20 image. The root password is taken from the file +C. + +Note if you I set I<--root-password> then the guest is given +a I root password. + +You can also create user accounts. See L below. + +=head2 Installing software + +To install packages from the ordinary (guest) software repository +(eg. yum or apt): + + virt-builder fedora-20 --install "inkscape,+Xfce Desktop" + +C<+> is used to install groups of packages (see L). + +=head2 Customizing the installation + +There are four options that let you run shell scripts to customize the +installation. They are: I<--run>/I<--run-command>, which run a shell +script or command while the disk image is being generated and lets you +add or edit files that go into the disk image. And +I<--firstboot>/I<--firstboot-command>, which let you add +scripts/commands that are run the first time the guest boots. + +For example: + + cat <<'EOF' > /tmp/yum-update.sh + yum -y update + EOF + + virt-builder fedora-20 --firstboot /tmp/yum-update.sh + +or simply: + + virt-builder fedora-20 --firstboot-command 'yum -y update' + +which makes the C command run once the first time the +guest boots. + +Or: + + cat <<'EOF' > /tmp/no-gpg-sigs.sh + sed -i 's/gpgcheck=1/gpgcheck=0/' /etc/yum.conf + EOF + + virt-builder fedora-20 --run /tmp/no-gpg-sigs.sh + +which edits C inside the disk image (during disk image +creation, long before boot). + +You can combine these options, and have multiple of either or both +sets of scripts. + +=head1 OPTIONS + +=over 4 + +=item B<--help> + +Display help. + +=item B<--attach> ISOFILE + +During the customization phase, the given disk is attached to the +libguestfs appliance. This is used to provide extra software +repositories or other data for customization. + +You probably want to ensure the volume(s) or filesystems in the +attached disks are labelled (or an ISO volume name) so that you can +mount them by label in your run-scripts: + + mkdir /tmp/mount + mount LABEL=EXTRA /tmp/mount + +You can have multiple I<--attach> options, and the format can be any +disk format (not just an ISO). + +See also: I<--run>, +L, +L. + +=item B<--attach-format> FORMAT + +Specify the disk format for the next I<--attach> option. The +C is usually C or C. Use C for ISOs. + +=item B<--cache> DIR + +=item B<--no-cache> + +I<--cache> DIR sets the directory to use/check for cached template +files. If not set, defaults to either +C<$XDG_CACHE_HOME/virt-builder/> or C<$HOME/.cache/virt-builder/>. + +I<--no-cache> disables template caching. + +=item B<--check-signature> + +=item B<--no-check-signature> + +Check/don't check the digital signature of the OS template. The +default is to check the signature and exit if it is not correct. +Using I<--no-check-signature> bypasses this check. + +See also I<--fingerprint>. + +=item B<--curl> CURL + +Specify an alternate L binary. You can also use this to add +curl parameters, for example to disable https certificate checks: + + virt-builder --curl "curl --insecure" [...] + +=item B<--delete-cache> + +Delete the template cache. See L. + +=item B<--fingerprint> 'AAAA BBBB ...' + +Check that the digital signature is signed by the key with the given +fingerprint. (The fingerprint is a long string, usually written as 10 +groups of 4 hexadecimal digits). + +If signature checking is enabled and the I<--fingerprint> option is +not given, then this checks the download was signed by +S (which is +S key). + +You can also set the C environment variable. + +=item B<--firstboot> SCRIPT + +=item B<--firstboot-command> 'CMD ARGS ...' + +Install C