v2v: add Var_expander

This helper module provides a facility to replace %{FOO}-like variables
in text strings with user-provided content.

(cherry picked from commit a27748d700)

.gitignore

@@ -693,6 +693,7 @@ Makefile.in
 /v2v/uefi.ml
 /v2v/uefi.mli
 /v2v/v2v_unit_tests
+/v2v/var_expander_tests
 /v2v/virt-v2v
 /v2v/virt-v2v.1
 /v2v/virt-v2v-copy-to-local

v2v/Makefile.am

@@ -98,6 +98,7 @@ SOURCES_MLI = \
 	utils.mli \
 	v2v.mli \
 	vCenter.mli \
+	var_expander.mli \
 	windows.mli \
 	windows_virtio.mli
 
@@ -106,6 +107,7 @@ SOURCES_ML = \
 	types.ml \
 	uefi.ml \
 	utils.ml \
+	var_expander.ml \
 	python_script.ml \
 	name_from_disk.ml \
 	vCenter.ml \
@@ -442,7 +444,7 @@ TESTS += \
 endif
 
 if HAVE_OCAML_PKG_OUNIT
-TESTS += v2v_unit_tests
+TESTS += v2v_unit_tests var_expander_tests
 endif
 
 if ENABLE_APPLIANCE
@@ -651,7 +653,7 @@ EXTRA_DIST += \
 # Unit tests.
 check_PROGRAMS =
 if HAVE_OCAML_PKG_OUNIT
-check_PROGRAMS += v2v_unit_tests
+check_PROGRAMS += v2v_unit_tests var_expander_tests
 endif
 
 v2v_unit_tests_BOBJECTS = \
@@ -671,13 +673,28 @@ v2v_unit_tests_SOURCES = $(virt_v2v_SOURCES)
 v2v_unit_tests_CPPFLAGS = $(virt_v2v_CPPFLAGS)
 v2v_unit_tests_CFLAGS = $(virt_v2v_CFLAGS)
 
+var_expander_tests_BOBJECTS = \
+	var_expander.cmo \
+	var_expander_tests.cmo
+var_expander_tests_XOBJECTS = $(var_expander_tests_BOBJECTS:.cmo=.cmx)
+
+var_expander_tests_SOURCES = dummy.c
+var_expander_tests_CPPFLAGS = $(virt_v2v_CPPFLAGS)
+var_expander_tests_CFLAGS = $(virt_v2v_CFLAGS)
+
 if !HAVE_OCAMLOPT
 # Can't call this v2v_unit_tests_OBJECTS because automake gets confused.
 v2v_unit_tests_THEOBJECTS = $(v2v_unit_tests_BOBJECTS)
 v2v_unit_tests.cmo: OCAMLPACKAGES += -package oUnit
+
+var_expander_tests_THEOBJECTS = $(var_expander_tests_BOBJECTS)
+var_expander_tests.cmo: OCAMLPACKAGES += -package oUnit
 else
 v2v_unit_tests_THEOBJECTS = $(v2v_unit_tests_XOBJECTS)
 v2v_unit_tests.cmx: OCAMLPACKAGES += -package oUnit
+
+var_expander_tests_THEOBJECTS = $(var_expander_tests_XOBJECTS)
+var_expander_tests.cmx: OCAMLPACKAGES += -package oUnit
 endif
 
 v2v_unit_tests_DEPENDENCIES = \
@@ -696,6 +713,17 @@ v2v_unit_tests_LINK = \
 	  $(OCAMLLINKFLAGS) \
 	  $(v2v_unit_tests_THEOBJECTS) -o $@
 
+var_expander_tests_DEPENDENCIES = \
+	$(var_expander_tests_THEOBJECTS) \
+	../common/mlpcre/mlpcre.$(MLARCHIVE) \
+	$(top_srcdir)/ocaml-link.sh
+var_expander_tests_LINK = \
+	$(top_srcdir)/ocaml-link.sh -cclib '$(OCAMLCLIBS)' -- \
+	  $(OCAMLFIND) $(BEST) $(OCAMLFLAGS) \
+	  $(OCAMLPACKAGES) -package oUnit \
+	  $(OCAMLLINKFLAGS) \
+	  $(var_expander_tests_THEOBJECTS) -o $@
+
 # Dependencies.
 .depend: \
 	$(srcdir)/*.mli \

v2v/dummy.c

@@ -0,0 +1,2 @@
+/* Dummy source, to be used for OCaml-based tools with no C sources. */
+enum { foo = 1 };

v2v/var_expander.ml

@@ -0,0 +1,72 @@
+(* virt-v2v
+ * Copyright (C) 2019 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+open Std_utils
+
+exception Invalid_variable of string
+
+let var_re = PCRE.compile "(^|[^%])%{([^}]+)}"
+
+let check_variable var =
+  String.iter (
+    function
+    | '0'..'9'
+    | 'a'..'z'
+    | 'A'..'Z'
+    | '_'
+    | '-' -> ()
+    | _ -> raise (Invalid_variable var)
+  ) var
+
+let scan_variables str =
+  let res = ref [] in
+  let offset = ref 0 in
+  while PCRE.matches ~offset:!offset var_re str; do
+    let var = PCRE.sub 2 in
+    check_variable var;
+    let _, end_ = PCRE.subi 0 in
+    List.push_back res var;
+    offset := end_
+  done;
+  List.remove_duplicates !res
+
+let replace_fn str fn =
+  let res = ref str in
+  let offset = ref 0 in
+  while PCRE.matches ~offset:!offset var_re !res; do
+    let var = PCRE.sub 2 in
+    check_variable var;
+    let start_, end_ = PCRE.subi 0 in
+    match fn var with
+    | None ->
+      offset := end_
+    | Some text ->
+      let prefix_len =
+        let prefix_start, prefix_end = PCRE.subi 1 in
+        prefix_end - prefix_start in
+      res := (String.sub !res 0 (start_ + prefix_len)) ^ text ^ (String.sub !res end_ (String.length !res - end_));
+      offset := start_ + prefix_len + String.length text
+  done;
+  !res
+
+let replace_list str lst =
+  let fn var =
+    try Some (List.assoc var lst)
+    with Not_found -> None
+  in
+  replace_fn str fn

v2v/var_expander.mli

@@ -0,0 +1,82 @@
+(* virt-v2v
+ * Copyright (C) 2019 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+(** Simple variable expander.
+
+    This module provides the support to expand variables in strings,
+    specified in the form of [%{name}].
+
+    For example:
+
+{v
+let str = "variable-%{INDEX} in %{INDEX} replaced %{INDEX} times"
+let index = ref 0
+let fn = function
+  | "INDEX" ->
+    incr index;
+    Some (string_of_int !index)
+  | _ -> None
+in
+let str = Var_expander.replace_fn str fn
+(* now str is "variable-1 in 2 replaced 3 times" *)
+v}
+
+    The names of variables can contain only ASCII letters (uppercase,
+    and lowercase), digits, underscores, and dashes.
+
+    The replacement is done in a single pass: this means that if a
+    variable is replaced with the text of a variable, that new text
+    is kept as is in the final output.  In practice:
+
+{v
+let str = "%{VAR}"
+let str = Var_expander.replace_list str [("VAR", "%{VAR}")]
+(* now str is "%{VAR}" *)
+v}
+*)
+
+exception Invalid_variable of string
+(** Invalid variable name error.
+
+    In case a variable contains characters not allowed, then this
+    exception with the actual unacceptable variable. *)
+
+val scan_variables : string -> string list
+(** Scan the pattern string for all the variables available.
+
+    This can raise {!Invalid_variable} in case there are invalid
+    variable names. *)
+
+val replace_fn : string -> (string -> string option) -> string
+(** Replaces a string expanding all the variables.
+
+    The replacement function specify how a variable is replaced;
+    if [None] is returned, then that variable is not replaced.
+
+    This can raise {!Invalid_variable} in case there are invalid
+    variable names. *)
+
+val replace_list : string -> (string * string) list -> string
+(** Replaces a string expanding all the variables.
+
+    The replacement list specify how a variable is replaced;
+    if it is not specified in the list, then that variable is not
+    replaced.
+
+    This can raise {!Invalid_variable} in case there are invalid
+    variable names. *)

v2v/var_expander_tests.ml

@@ -0,0 +1,113 @@
+(* virt-v2v
+ * Copyright (C) 2019 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+open Printf
+open OUnit
+
+open Std_utils
+
+let assert_equal_string = assert_equal ~printer:identity
+let assert_equal_stringlist = assert_equal ~printer:(fun x -> "(" ^ (String.escaped (String.concat "," x)) ^ ")")
+
+let replace_none_fn _ = None
+let replace_empty_fn _ = Some ""
+
+let test_no_replacement () =
+  assert_equal_string "" (Var_expander.replace_fn "" replace_none_fn);
+  assert_equal_string "x" (Var_expander.replace_fn "x" replace_none_fn);
+  assert_equal_string "%{}" (Var_expander.replace_fn "%{}" replace_none_fn);
+  assert_equal_string "%{EMPTY}" (Var_expander.replace_fn "%{EMPTY}" replace_none_fn);
+  assert_equal_string "%{EMPTY} %{no}" (Var_expander.replace_fn "%{EMPTY} %{no}" replace_none_fn);
+  assert_equal_string "a %{EMPTY} b" (Var_expander.replace_fn "a %{EMPTY} b" replace_none_fn);
+  ()
+
+let test_replacements () =
+  assert_equal_string "" (Var_expander.replace_fn "%{EMPTY}" replace_empty_fn);
+  assert_equal_string "x " (Var_expander.replace_fn "x %{EMPTY}" replace_empty_fn);
+  assert_equal_string "xy" (Var_expander.replace_fn "x%{EMPTY}y" replace_empty_fn);
+  assert_equal_string "x<->y" (Var_expander.replace_fn "x%{FOO}y" (function | "FOO" -> Some "<->" | _ -> None));
+  assert_equal_string "a x b" (Var_expander.replace_fn "a %{FOO} b" (function | "FOO" -> Some "x" | _ -> None));
+  assert_equal_string "%{FOO} x" (Var_expander.replace_fn "%{FOO} %{BAR}" (function | "BAR" -> Some "x" | _ -> None));
+  assert_equal_string "%{FOO}" (Var_expander.replace_fn "%{BAR}" (function | "BAR" -> Some "%{FOO}" | _ -> None));
+  assert_equal_string "%{FOO} x" (Var_expander.replace_fn "%{BAR} %{FOO}" (function | "BAR" -> Some "%{FOO}" | "FOO" -> Some "x" | _ -> None));
+  begin
+    let str = "%{INDEX}, %{INDEX}, %{INDEX}" in
+    let index = ref 0 in
+    let fn = function
+      | "INDEX" ->
+        incr index;
+        Some (string_of_int !index)
+      | _ -> None
+    in
+    assert_equal_string "1, 2, 3" (Var_expander.replace_fn str fn)
+  end;
+  ()
+
+let test_escape () =
+  assert_equal_string "%%{FOO}" (Var_expander.replace_fn "%%{FOO}" replace_empty_fn);
+  assert_equal_string "x %%{FOO} x" (Var_expander.replace_fn "%{FOO} %%{FOO} %{FOO}" (function | "FOO" -> Some "x" | _ -> None));
+  ()
+
+let test_list () =
+  assert_equal_string "x %{NONE}" (Var_expander.replace_list "%{FOO} %{NONE}" [("FOO", "x")]);
+  ()
+
+let test_scan_variables () =
+  let assert_invalid_variable var =
+    let str = "%{" ^ var ^ "}" in
+    assert_raises (Var_expander.Invalid_variable var)
+                  (fun () -> Var_expander.scan_variables str)
+  in
+  assert_equal_stringlist [] (Var_expander.scan_variables "");
+  assert_equal_stringlist [] (Var_expander.scan_variables "foo");
+  assert_equal_stringlist ["FOO"] (Var_expander.scan_variables "%{FOO}");
+  assert_equal_stringlist ["FOO"; "BAR"] (Var_expander.scan_variables "%{FOO} %{BAR}");
+  assert_equal_stringlist ["FOO"; "BAR"] (Var_expander.scan_variables "%{FOO} %{BAR} %{FOO}");
+  assert_equal_stringlist ["FOO"; "BAR"] (Var_expander.scan_variables "%{FOO} %%{ESCAPED} %{BAR}");
+  assert_invalid_variable "FOO/BAR";
+  ()
+
+let test_errors () =
+  let assert_invalid_variable var =
+    let str = "%{" ^ var ^ "}" in
+    assert_raises (Var_expander.Invalid_variable var)
+                  (fun () -> Var_expander.replace_fn str replace_none_fn)
+  in
+  assert_invalid_variable "FOO/BAR";
+  assert_invalid_variable "FOO:BAR";
+  assert_invalid_variable "FOO(BAR";
+  assert_invalid_variable "FOO)BAR";
+  assert_invalid_variable "FOO@BAR";
+  ()
+
+(* Suites declaration. *)
+let suite =
+  TestList ([
+    "basic" >::: [
+      "no_replacement" >:: test_no_replacement;
+      "replacements" >:: test_replacements;
+      "escape" >:: test_escape;
+      "list" >:: test_list;
+      "scan_variables" >:: test_scan_variables;
+      "errors" >:: test_errors;
+    ];
+  ])
+
+let () =
+  ignore (run_test_tt_main suite);
+  Printf.fprintf stderr "\n"