v2v: ova: don't extract files from OVA if it's not needed

We don't have to always extract all files from the OVA archive. The OVA,
as defined in the standard, is plain tar. We can work directly over the
tar archive if we use correct 'offset' and 'size' options when defining
the backing file for QEMU. This puts much lower requirement on available
disk space.

Since the virt-v2v behaviour for OVA input now depends on QEMU version
available this affects some of the tests. Expected result of the
affected also has to depend on the QEMU used thus such tests will have
two *.expected files.

Signed-off-by: Tomáš Golembiovský <tgolembi@redhat.com>
This commit is contained in:
Tomáš Golembiovský
2017-02-04 15:10:46 +01:00
committed by Richard W.M. Jones
parent d3c93e3320
commit 8f91d3a9b0
12 changed files with 421 additions and 24 deletions

View File

@@ -54,3 +54,22 @@ do_sha256 ()
;;
esac
}
# Returns 0 if QEMU version is greater or equal to the arguments
qemu_is_version() {
if [ $# -ne 2 ] ; then
echo "Usage: $0 <major_version> <minor_version>" >&2
return 3
fi
[[ "$(qemu-img --version)" =~ 'qemu-img version '([0-9]+)\.([0-9]+) ]] || return 2
QMAJ=${BASH_REMATCH[1]}
QMIN=${BASH_REMATCH[2]}
if [ \( $QMAJ -gt $1 \) -o \( $QMAJ -eq $1 -a $QMIN -ge $2 \) ] ; then
return 0
fi
return 1
}

View File

@@ -260,6 +260,7 @@ TESTS_ENVIRONMENT = $(top_builddir)/run --test
TESTS = \
test-v2v-docs.sh \
test-v2v-i-ova-tar.sh \
test-v2v-i-ova-formats.sh \
test-v2v-i-ova-gz.sh \
test-v2v-i-ova-subfolders.sh \
@@ -359,6 +360,7 @@ EXTRA_DIST += \
test-v2v-i-ova-subfolders.expected \
test-v2v-i-ova-subfolders.ovf \
test-v2v-i-ova-subfolders.sh \
test-v2v-i-ova-tar.sh \
test-v2v-i-ova-two-disks.expected \
test-v2v-i-ova-two-disks.ovf \
test-v2v-i-ova-two-disks.sh \

View File

@@ -39,17 +39,23 @@ object
method source () =
let untar ?(format = "") file outdir =
let cmd = [ "tar"; sprintf "-x%sf" format; file; "-C"; outdir ] in
(* Untar part or all files from tar archive. If [paths] is specified it is
* a list of paths in the tar archive.
*)
let untar ?(format = "") ?paths file outdir =
let cmd =
[ "tar"; sprintf "-x%sf" format; file; "-C"; outdir ]
@ match paths with None -> [] | Some p -> p in
if run_command cmd <> 0 then
error (f_"error unpacking %s, see earlier error messages") ova in
error (f_"error unpacking %s, see earlier error messages") ova
in
(* Extract ova file. *)
let exploded =
let exploded, partial =
(* The spec allows a directory to be specified as an ova. This
* is also pretty convenient.
*)
if is_directory ova then ova
if is_directory ova then ova, false
else (
let uncompress_head zcat file =
let cmd = sprintf "%s %s" zcat (quote file) in
@@ -67,11 +73,35 @@ object
tmpfile in
(* Untar only ovf and manifest from the archive *)
let untar_metadata ova outdir =
let files =
external_command (sprintf "tar -tf %s" (Filename.quote ova)) in
let files =
filter_map (fun f ->
if Filename.check_suffix f ".ovf" ||
Filename.check_suffix f ".mf" then
Some f
else None
) files in
untar ~paths:files ova outdir
in
match detect_file_type ova with
| `Tar ->
(* Normal ovas are tar file (not compressed). *)
untar ova tmpdir;
tmpdir
let qmajor, qminor = qemu_img_version () in
if qmajor > 2 || (qmajor == 2 && qminor >= 8) then (
(* If QEMU is 2.8 or newer we don't have to extract everything.
* We can access disks inside the tar archive directly.
*)
untar_metadata ova tmpdir;
tmpdir, true
) else (
untar ova tmpdir;
tmpdir, false
)
| `Zip ->
(* However, although not permitted by the spec, people ship
* zip files as ova too.
@@ -81,7 +111,7 @@ object
[ "-j"; "-d"; tmpdir; ova ] in
if run_command cmd <> 0 then
error (f_"error unpacking %s, see earlier error messages") ova;
tmpdir
tmpdir, false
| (`GZip|`XZ) as format ->
let zcat, tar_fmt =
match format with
@@ -94,7 +124,7 @@ object
(match tmpfiletype with
| `Tar ->
untar ~format:tar_fmt ova tmpdir;
tmpdir
tmpdir, false
| `Zip | `GZip | `XZ | `Unknown ->
error (f_"%s: unsupported file format\n\nFormats which we currently understand for '-i ova' are: tar (uncompressed, compress with gzip or xz), zip") ova
)
@@ -152,6 +182,7 @@ object
fun mf ->
debug "processing manifest %s" mf;
let mf_folder = Filename.dirname mf in
let mf_subfolder = subdirectory exploded mf_folder in
let chan = open_in mf in
let rec loop () =
let line = input_line chan in
@@ -160,7 +191,11 @@ object
let disk = Str.matched_group 2 line in
let expected = Str.matched_group 3 line in
let csum = Checksums.of_string mode expected in
try Checksums.verify_checksum csum (mf_folder // disk)
try
if partial then
Checksums.verify_checksum csum ~tar:ova (mf_subfolder // disk)
else
Checksums.verify_checksum csum (mf_folder // disk)
with Checksums.Mismatched_checksum (_, actual) ->
error (f_"checksum of disk %s does not match manifest %s (actual %s(%s) = %s, expected %s(%s) = %s)")
disk mf mode disk actual mode disk expected;
@@ -283,9 +318,26 @@ object
| Some "gzip" -> true
| Some s -> error (f_"unsupported compression in OVF: %s") s in
(* Does the file exist and is it readable? *)
let filename = ovf_folder // filename in
Unix.access filename [Unix.R_OK];
let partial =
if compressed && partial then (
(* We cannot access compressed disk inside the tar;
* we have to extract it.
*)
untar ~paths:[(subdirectory exploded ovf_folder) // filename]
ova tmpdir;
false
)
else
partial in
let filename =
if partial then
(subdirectory exploded ovf_folder) // filename
else (
(* Does the file exist and is it readable? *)
Unix.access (ovf_folder // filename) [Unix.R_OK];
ovf_folder // filename
) in
(* The spec allows the file to be gzip-compressed, in which case
* we must uncompress it into the tmpdir.
@@ -302,9 +354,39 @@ object
)
else filename in
let qemu_uri =
if not partial then (
filename
)
else (
let offset, size =
try find_file_in_tar ova filename
with
| Not_found ->
error (f_"file '%s' not found in the ova") filename
| Failure msg -> error (f_"%s") msg in
(* QEMU requires size aligned to 512 bytes. This is safe because
* tar also works with 512 byte blocks.
*)
let size = roundup64 size 512L in
let doc = [
"file", JSON.Dict [
"driver", JSON.String "raw";
"offset", JSON.Int64 offset;
"size", JSON.Int64 size;
"file", JSON.Dict [
"filename", JSON.String ova]
]
] in
let uri =
sprintf "json:%s" (JSON.string_of_doc ~fmt:JSON.Compact doc) in
debug "json: %s" uri;
uri
) in
let disk = {
s_disk_id = i;
s_qemu_uri = filename;
s_qemu_uri = qemu_uri;
s_format = Some "vmdk";
s_controller = controller;
} in

View File

@@ -22,7 +22,7 @@ unset CDPATH
export LANG=C
set -e
formats="tar zip tar-gz tar-xz"
formats="zip tar-gz tar-xz"
if [ -n "$SKIP_TEST_V2V_I_OVA_FORMATS_SH" ]; then
echo "$0: test skipped because environment variable is set"
@@ -63,9 +63,6 @@ cp ../test-v2v-i-ova-formats.ovf .
for format in $formats; do
case "$format" in
tar)
tar -cf test-$format.ova test-v2v-i-ova-formats.ovf disk1.vmdk disk1.mf
;;
zip)
zip -r test test-v2v-i-ova-formats.ovf disk1.vmdk disk1.mf
mv test.zip test-$format.ova

View File

@@ -0,0 +1,18 @@
Source guest information (--print-source option):
source name: 2K8R2EESP1_2_Medium
hypervisor type: vmware
memory: 1073741824 (bytes)
nr vCPUs: 1
CPU features:
firmware: uefi
display:
video:
sound:
disks:
json:{ "file": { "driver": "raw", "offset": 2048, "size": 10240, "file": { "filename": "test.ova" } } } (vmdk) [scsi]
removable media:
CD-ROM [ide] in slot 0
NICs:
Network "Network adapter 1"

View File

@@ -56,10 +56,17 @@ popd
# normalize the output.
$VG virt-v2v --debug-gc --quiet \
-i ova $d/test.ova \
--print-source |
sed 's,[^ \t]*\(subfolder/disk.*\.vmdk\),\1,' > $d/source
--print-source > $d/source
# Check the parsed source is what we expect.
diff -u test-v2v-i-ova-subfolders.expected $d/source
if qemu_is_version 2 8 ; then
# normalize the output
sed -i -e "s,\"$d/,\"," $d/source
diff -u test-v2v-i-ova-subfolders.expected2 $d/source
else
# normalize the output
sed -i -e 's,[^ \t]*\(subfolder/disk.*\.vmdk\),\1,' $d/source
diff -u test-v2v-i-ova-subfolders.expected $d/source
fi
rm -rf $d

View File

@@ -0,0 +1,18 @@
Source guest information (--print-source option):
source name: 2K8R2EESP1_2_Medium
hypervisor type: vmware
memory: 1073741824 (bytes)
nr vCPUs: 1
CPU features:
firmware: uefi
display:
video:
sound:
disks:
disk1.vmdk (vmdk) [scsi]
removable media:
CD-ROM [ide] in slot 0
NICs:
Network "Network adapter 1"

View File

@@ -0,0 +1,18 @@
Source guest information (--print-source option):
source name: 2K8R2EESP1_2_Medium
hypervisor type: vmware
memory: 1073741824 (bytes)
nr vCPUs: 1
CPU features:
firmware: uefi
display:
video:
sound:
disks:
json:{ "file": { "driver": "raw", "offset": 9216, "size": 10240, "file": { "filename": "test-tar.ova" } } } (vmdk) [scsi]
removable media:
CD-ROM [ide] in slot 0
NICs:
Network "Network adapter 1"

138
v2v/test-v2v-i-ova-tar.ovf Normal file
View File

@@ -0,0 +1,138 @@
<?xml version="1.0" encoding="UTF-8"?>
<Envelope vmw:buildId="build-1750787" xmlns="http://schemas.dmtf.org/ovf/envelope/1" xmlns:cim="http://schemas.dmtf.org/wbem/wscim/1/common" xmlns:ovf="http://schemas.dmtf.org/ovf/envelope/1" xmlns:rasd="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData" xmlns:vmw="http://www.vmware.com/schema/ovf" xmlns:vssd="http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_VirtualSystemSettingData" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<References>
<File ovf:href="disk1.vmdk" ovf:id="file1" ovf:size="7804077568"/>
</References>
<DiskSection>
<Info>Virtual disk information</Info>
<Disk ovf:capacity="50" ovf:capacityAllocationUnits="byte * 2^30" ovf:diskId="vmdisk1" ovf:fileRef="file1" ovf:format="http://www.vmware.com/interfaces/specifications/vmdk.html#streamOptimized" ovf:populatedSize="18975752192"/>
</DiskSection>
<NetworkSection>
<Info>The list of logical networks</Info>
<Network ovf:name="PG-VLAN60">
<Description>The PG-VLAN60 network</Description>
</Network>
</NetworkSection>
<VirtualSystem ovf:id="2K8R2EESP1_2_Medium">
<Info>A virtual machine</Info>
<Name>2K8R2EESP1_2_Medium</Name>
<OperatingSystemSection ovf:id="103" vmw:osType="windows7Server64Guest">
<Info>The kind of installed guest operating system</Info>
<Description>Microsoft Windows Server 2008 R2 (64-bit)</Description>
</OperatingSystemSection>
<VirtualHardwareSection>
<Info>Virtual hardware requirements</Info>
<System>
<vssd:ElementName>Virtual Hardware Family</vssd:ElementName>
<vssd:InstanceID>0</vssd:InstanceID>
<vssd:VirtualSystemIdentifier>2K8R2EESP1_2_Medium</vssd:VirtualSystemIdentifier>
<vssd:VirtualSystemType>vmx-10</vssd:VirtualSystemType>
</System>
<Item>
<rasd:AllocationUnits>hertz * 10^6</rasd:AllocationUnits>
<rasd:Description>Number of Virtual CPUs</rasd:Description>
<rasd:ElementName>1 virtual CPU(s)</rasd:ElementName>
<rasd:InstanceID>1</rasd:InstanceID>
<rasd:ResourceType>3</rasd:ResourceType>
<rasd:VirtualQuantity>1</rasd:VirtualQuantity>
</Item>
<Item>
<rasd:AllocationUnits>byte * 2^20</rasd:AllocationUnits>
<rasd:Description>Memory Size</rasd:Description>
<rasd:ElementName>1024MB of memory</rasd:ElementName>
<rasd:InstanceID>2</rasd:InstanceID>
<rasd:ResourceType>4</rasd:ResourceType>
<rasd:VirtualQuantity>1024</rasd:VirtualQuantity>
</Item>
<Item>
<rasd:Address>0</rasd:Address>
<rasd:Description>SCSI Controller</rasd:Description>
<rasd:ElementName>SCSI controller 0</rasd:ElementName>
<rasd:InstanceID>3</rasd:InstanceID>
<rasd:ResourceSubType>lsilogicsas</rasd:ResourceSubType>
<rasd:ResourceType>6</rasd:ResourceType>
<vmw:Config ovf:required="false" vmw:key="slotInfo.pciSlotNumber" vmw:value="160"/>
</Item>
<Item>
<rasd:Address>1</rasd:Address>
<rasd:Description>IDE Controller</rasd:Description>
<rasd:ElementName>IDE 1</rasd:ElementName>
<rasd:InstanceID>4</rasd:InstanceID>
<rasd:ResourceType>5</rasd:ResourceType>
</Item>
<Item>
<rasd:Address>0</rasd:Address>
<rasd:Description>IDE Controller</rasd:Description>
<rasd:ElementName>IDE 0</rasd:ElementName>
<rasd:InstanceID>5</rasd:InstanceID>
<rasd:ResourceType>5</rasd:ResourceType>
</Item>
<Item ovf:required="false">
<rasd:AutomaticAllocation>false</rasd:AutomaticAllocation>
<rasd:ElementName>Video card</rasd:ElementName>
<rasd:InstanceID>6</rasd:InstanceID>
<rasd:ResourceType>24</rasd:ResourceType>
<vmw:Config ovf:required="false" vmw:key="enable3DSupport" vmw:value="false"/>
<vmw:Config ovf:required="false" vmw:key="use3dRenderer" vmw:value="automatic"/>
<vmw:Config ovf:required="false" vmw:key="useAutoDetect" vmw:value="true"/>
<vmw:Config ovf:required="false" vmw:key="videoRamSizeInKB" vmw:value="4096"/>
</Item>
<Item ovf:required="false">
<rasd:AutomaticAllocation>false</rasd:AutomaticAllocation>
<rasd:ElementName>VMCI device</rasd:ElementName>
<rasd:InstanceID>7</rasd:InstanceID>
<rasd:ResourceSubType>vmware.vmci</rasd:ResourceSubType>
<rasd:ResourceType>1</rasd:ResourceType>
<vmw:Config ovf:required="false" vmw:key="allowUnrestrictedCommunication" vmw:value="false"/>
<vmw:Config ovf:required="false" vmw:key="slotInfo.pciSlotNumber" vmw:value="32"/>
</Item>
<Item ovf:required="false">
<rasd:AddressOnParent>0</rasd:AddressOnParent>
<rasd:AutomaticAllocation>false</rasd:AutomaticAllocation>
<rasd:ElementName>CD/DVD drive 1</rasd:ElementName>
<rasd:InstanceID>8</rasd:InstanceID>
<rasd:Parent>4</rasd:Parent>
<rasd:ResourceSubType>vmware.cdrom.atapi</rasd:ResourceSubType>
<rasd:ResourceType>15</rasd:ResourceType>
</Item>
<Item>
<rasd:AddressOnParent>0</rasd:AddressOnParent>
<rasd:ElementName>Hard disk 1</rasd:ElementName>
<rasd:HostResource>ovf:/disk/vmdisk1</rasd:HostResource>
<rasd:InstanceID>9</rasd:InstanceID>
<rasd:Parent>3</rasd:Parent>
<rasd:ResourceType>17</rasd:ResourceType>
<vmw:Config ovf:required="false" vmw:key="backing.writeThrough" vmw:value="false"/>
</Item>
<Item>
<rasd:AddressOnParent>7</rasd:AddressOnParent>
<rasd:AutomaticAllocation>true</rasd:AutomaticAllocation>
<rasd:Connection>PG-VLAN60</rasd:Connection>
<rasd:Description>E1000 ethernet adapter on &quot;PG-VLAN60&quot;</rasd:Description>
<rasd:ElementName>Network adapter 1</rasd:ElementName>
<rasd:InstanceID>11</rasd:InstanceID>
<rasd:ResourceSubType>E1000</rasd:ResourceSubType>
<rasd:ResourceType>10</rasd:ResourceType>
<vmw:Config ovf:required="false" vmw:key="slotInfo.pciSlotNumber" vmw:value="33"/>
<vmw:Config ovf:required="false" vmw:key="wakeOnLanEnabled" vmw:value="true"/>
</Item>
<vmw:Config ovf:required="false" vmw:key="cpuHotAddEnabled" vmw:value="false"/>
<vmw:Config ovf:required="false" vmw:key="cpuHotRemoveEnabled" vmw:value="false"/>
<vmw:Config ovf:required="false" vmw:key="firmware" vmw:value="efi"/>
<vmw:Config ovf:required="false" vmw:key="virtualICH7MPresent" vmw:value="false"/>
<vmw:Config ovf:required="false" vmw:key="virtualSMCPresent" vmw:value="false"/>
<vmw:Config ovf:required="false" vmw:key="memoryHotAddEnabled" vmw:value="false"/>
<vmw:Config ovf:required="false" vmw:key="nestedHVEnabled" vmw:value="false"/>
<vmw:Config ovf:required="false" vmw:key="powerOpInfo.powerOffType" vmw:value="soft"/>
<vmw:Config ovf:required="false" vmw:key="powerOpInfo.resetType" vmw:value="soft"/>
<vmw:Config ovf:required="false" vmw:key="powerOpInfo.standbyAction" vmw:value="checkpoint"/>
<vmw:Config ovf:required="false" vmw:key="powerOpInfo.suspendType" vmw:value="hard"/>
<vmw:Config ovf:required="false" vmw:key="tools.afterPowerOn" vmw:value="true"/>
<vmw:Config ovf:required="false" vmw:key="tools.afterResume" vmw:value="true"/>
<vmw:Config ovf:required="false" vmw:key="tools.beforeGuestShutdown" vmw:value="true"/>
<vmw:Config ovf:required="false" vmw:key="tools.beforeGuestStandby" vmw:value="true"/>
<vmw:Config ovf:required="false" vmw:key="tools.syncTimeWithHost" vmw:value="false"/>
<vmw:Config ovf:required="false" vmw:key="tools.toolsUpgradePolicy" vmw:value="upgradeAtPowerCycle"/>
</VirtualHardwareSection>
</VirtualSystem>
</Envelope>

72
v2v/test-v2v-i-ova-tar.sh Executable file
View File

@@ -0,0 +1,72 @@
#!/bin/bash -
# libguestfs virt-v2v test script
# Copyright (C) 2014-2016 Red Hat Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
# Test -i ova option with ova file compressed in different ways
unset CDPATH
export LANG=C
set -e
if [ -n "$SKIP_TEST_V2V_I_OVA_FORMATS_SH" ]; then
echo "$0: test skipped because environment variable is set"
exit 77
fi
if [ "$(guestfish get-backend)" = "uml" ]; then
echo "$0: test skipped because UML backend does not support network"
exit 77
fi
export VIRT_TOOLS_DATA_DIR="$srcdir/../test-data/fake-virt-tools"
. $srcdir/../test-data/test-utils.sh
d=test-v2v-i-ova-tar.d
rm -rf $d
mkdir $d
pushd $d
# Create a phony OVA. This is only a test of source parsing, not
# conversion, so the contents of the disks doesn't matter.
truncate -s 10k disk1.vmdk
sha=`do_sha1 disk1.vmdk`
echo -e "SHA1(disk1.vmdk)= $sha\r" > disk1.mf
cp ../test-v2v-i-ova-tar.ovf .
tar -cf test-tar.ova test-v2v-i-ova-tar.ovf disk1.vmdk disk1.mf
popd
# Run virt-v2v but only as far as the --print-source stage
$VG virt-v2v --debug-gc --quiet \
-i ova $d/test-tar.ova \
--print-source > $d/source
# Check the parsed source is what we expect.
if qemu_is_version 2 8 ; then
# normalize the output
sed -i -e "s,\"$d/,\"," $d/source
diff -u test-v2v-i-ova-tar.expected2 $d/source
else
# normalize the output
sed -i -e 's,[^ \t]*\(disk.*.vmdk\),\1,' $d/source
diff -u test-v2v-i-ova-tar.expected $d/source
fi
rm -rf $d

View File

@@ -0,0 +1,19 @@
Source guest information (--print-source option):
source name: 2K8R2EESP1_2_Medium
hypervisor type: vmware
memory: 1073741824 (bytes)
nr vCPUs: 1
CPU features:
firmware: bios
display:
video:
sound:
disks:
json:{ "file": { "driver": "raw", "offset": 9728, "size": 10240, "file": { "filename": "test.ova" } } } (vmdk) [scsi]
json:{ "file": { "driver": "raw", "offset": 21504, "size": 102400, "file": { "filename": "test.ova" } } } (vmdk) [scsi]
removable media:
CD-ROM [ide] in slot 0
NICs:
Network "Network adapter 1"

View File

@@ -60,10 +60,17 @@ popd
# normalize the output.
$VG virt-v2v --debug-gc --quiet \
-i ova $d/test.ova \
--print-source |
sed 's,[^ \t]*\(disk.*.vmdk\),\1,' > $d/source
--print-source > $d/source
# Check the parsed source is what we expect.
diff -u test-v2v-i-ova-two-disks.expected $d/source
if qemu_is_version 2 8 ; then
# normalize the output
sed -i -e "s,\"$d/,\"," $d/source
diff -u test-v2v-i-ova-two-disks.expected2 $d/source
else
# normalize the output
sed -i -e 's,[^ \t]*\(disk.*.vmdk\),\1,' $d/source
diff -u test-v2v-i-ova-two-disks.expected $d/source
fi
rm -rf $d