tools: Introduce a mini-library for estimating max threads based on free memory.

This mini-library runs the 'free -m' command and greps the output to
estimate the max. number of libguestfs appliances we could run in
parallel in the remaining free memory.

df/estimate-max-threads.c

@@ -0,0 +1,78 @@
+/* libguestfs
+ * Copyright (C) 2013 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#include <config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <error.h>
+#include <errno.h>
+#include <libintl.h>
+
+#include "progname.h"
+
+#include "guestfs.h"
+#include "guestfs-internal-frontend.h"
+#include "estimate-max-threads.h"
+
+static char *read_line_from (const char *cmd);
+
+/* The actual overhead is likely much smaller than this, but err on
+ * the safe side.
+ */
+#define MBYTES_PER_THREAD 650
+
+size_t
+estimate_max_threads (void)
+{
+  CLEANUP_FREE char *mbytes_str = NULL;
+  size_t mbytes;
+
+  /* Choose the number of threads based on the amount of free memory. */
+  mbytes_str = read_line_from ("LANG=C free -m | "
+                               "grep 'buffers/cache' | awk '{print $NF}'");
+  if (mbytes_str == NULL)
+    return 1;
+
+  if (sscanf (mbytes_str, "%zu", &mbytes) != 1)
+    return 1;
+
+  return MAX (1, mbytes / MBYTES_PER_THREAD);
+}
+
+/* Run external command and read the first line of output. */
+static char *
+read_line_from (const char *cmd)
+{
+  FILE *pp;
+  char *ret = NULL;
+  size_t allocsize;
+
+  pp = popen (cmd, "r");
+  if (pp == NULL)
+    error (EXIT_FAILURE, errno, "%s: external command failed", cmd);
+
+  if (getline (&ret, &allocsize, pp) == -1)
+    error (EXIT_FAILURE, errno, "could not read line from external command");
+
+  if (pclose (pp) == -1)
+    error (EXIT_FAILURE, errno, "pclose");
+
+  return ret;
+}

df/estimate-max-threads.h

@@ -0,0 +1,28 @@
+/* libguestfs
+ * Copyright (C) 2013 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#ifndef GUESTFS_ESTIMATE_MAX_THREADS_H_
+#define GUESTFS_ESTIMATE_MAX_THREADS_H_
+
+/* This function uses the output of 'free -m' to estimate how many
+ * libguestfs appliances could be safely started in parallel.  Note
+ * that it always returns >= 1.
+ */
+extern size_t estimate_max_threads (void);
+
+#endif /* GUESTFS_ESTIMATE_MAX_THREADS_H_ */

po/POTFILES

@@ -99,6 +99,7 @@ daemon/zero.c
 daemon/zerofree.c
 df/df.c
 df/domains.c
+df/estimate-max-threads.c
 df/main.c
 df/output.c
 edit/virt-edit.c

tests/mount-local/Makefile.am

@@ -25,11 +25,15 @@ TESTS_ENVIRONMENT = $(top_builddir)/run --test $(VG)
 
 check_PROGRAMS = $(TESTS)
 
-test_parallel_mount_local_SOURCES = test-parallel-mount-local.c
+test_parallel_mount_local_SOURCES = \
+	test-parallel-mount-local.c \
+	$(top_srcdir)/df/estimate-max-threads.c \
+	$(top_srcdir)/df/estimate-max-threads.h
 test_parallel_mount_local_CPPFLAGS = \
 	-DGUESTFS_WARN_DEPRECATED=1 \
 	-I$(top_srcdir)/gnulib/lib -I$(top_builddir)/gnulib/lib \
-	-I$(top_srcdir)/src -I$(top_builddir)/src
+	-I$(top_srcdir)/src -I$(top_builddir)/src \
+	-I$(top_srcdir)/df
 test_parallel_mount_local_CFLAGS = \
 	-pthread \
 	$(WARN_CFLAGS) $(WERROR_CFLAGS) \
@@ -37,7 +41,10 @@ test_parallel_mount_local_CFLAGS = \
 	$(FUSE_CFLAGS)
 test_parallel_mount_local_LDADD = \
 	$(FUSE_LIBS) \
+	$(top_builddir)/src/libutils.la \
 	$(top_builddir)/src/libguestfs.la \
+	$(LIBXML2_LIBS) \
+	$(LIBVIRT_LIBS) \
 	$(top_builddir)/gnulib/lib/libgnu.la
 
 check-valgrind:

tests/mount-local/test-parallel-mount-local.c

@@ -39,14 +39,13 @@
 
 #include "guestfs.h"
 #include "guestfs-internal-frontend.h"
+#include "estimate-max-threads.h"
 
 #include "ignore-value.h"
 
 #define TOTAL_TIME 60           /* Seconds, excluding launch. */
 #define DEBUG 1                 /* Print overview debugging messages. */
-#define MIN_THREADS 2
 #define MAX_THREADS 12
-#define MBYTES_PER_THREAD 900
 
 struct thread_state {
   pthread_t thread;             /* Thread handle. */
@@ -60,7 +59,6 @@ static size_t nr_threads;
 static void *start_thread (void *) __attribute__((noreturn));
 static void test_mountpoint (const char *mp);
 static void cleanup_thread_state (void);
-static char *read_line_from (const char *cmd);
 static int unmount (const char *mp, unsigned flags);
 #define UNMOUNT_SILENT 1
 #define UNMOUNT_RMDIR  2
@@ -81,8 +79,8 @@ catch_sigint (int signal)
 int
 main (int argc, char *argv[])
 {
-  size_t i, mbytes;
-  char *skip, *mbytes_s;
+  size_t i;
+  char *skip;
   struct sigaction sa;
   int fd, r, errors = 0;
   void *status;
@@ -110,20 +108,7 @@ main (int argc, char *argv[])
   }
 
   /* Choose the number of threads based on the amount of free memory. */
-  mbytes_s = read_line_from ("LANG=C free -m | "
-                             "grep 'buffers/cache' | awk '{print $NF}'");
-  if (!mbytes_s)
-    nr_threads = MIN_THREADS; /* default */
-  else {
-    if (sscanf (mbytes_s, "%zu", &mbytes) != 1)
-      error (EXIT_FAILURE, 0, "expecting integer but got \"%s\"", mbytes_s);
-    free (mbytes_s);
-    nr_threads = mbytes / MBYTES_PER_THREAD;
-    if (nr_threads < MIN_THREADS)
-      nr_threads = MIN_THREADS;
-    else if (nr_threads > MAX_THREADS)
-      nr_threads = MAX_THREADS;
-  }
+  nr_threads = MIN (MAX_THREADS, estimate_max_threads ());
 
   memset (&sa, 0, sizeof sa);
   sa.sa_handler = catch_sigint;
@@ -459,24 +444,3 @@ cleanup_thread_state (void)
     }
   }
 }
-
-/* Run external command and read the first line of output. */
-static char *
-read_line_from (const char *cmd)
-{
-  FILE *pp;
-  char *ret = NULL;
-  size_t allocsize;
-
-  pp = popen (cmd, "r");
-  if (pp == NULL)
-    error (EXIT_FAILURE, errno, "%s: external command failed", cmd);
-
-  if (getline (&ret, &allocsize, pp) == -1)
-    error (EXIT_FAILURE, errno, "could not read line from external command");
-
-  if (pclose (pp) == -1)
-    error (EXIT_FAILURE, errno, "pclose");
-
-  return ret;
-}