From 6a81e8d666a8f276833fa2e6a32a6047b2fd5790 Mon Sep 17 00:00:00 2001
From: Jake Hunsaker <jhunsake@redhat.com>
Date: Mon, 20 Apr 2020 13:17:32 -0400
Subject: [collector] Passthru --threads, replace with --jobs for collect

Replaces the previous `--threads` option in collect with a `--jobs`
option to determine the number of concurrent collections to run.

Uses `--threads` as a passthru option now if the node supports it
(sos-3.6+), in an effort to unify option meanings between components.

Signed-off-by: Jake Hunsaker <jhunsake@redhat.com>
---
 man/en/sos-collect.1      | 21 +++++++++++++++------
 sos/collector/__init__.py |  9 ++++++---
 sos/collector/sosnode.py  |  6 ++++++
 3 files changed, 27 insertions(+), 9 deletions(-)

diff --git a/man/en/sos-collect.1 b/man/en/sos-collect.1
index 42c7e8bd..7d6f8b87 100644
--- a/man/en/sos-collect.1
+++ b/man/en/sos-collect.1
@@ -15,6 +15,7 @@ sos collect \- Collect sosreports from multiple (cluster) nodes
     [--encrypt-key KEY]\fR
     [--encrypt-pass PASS]\fR
     [\-\-group GROUP]
+    [\-j|\-\-jobs JOBS]
     [\-\-save\-group GROUP]
     [\-\-nopasswd-sudo]
     [\-k PLUGIN_OPTION]
@@ -166,6 +167,15 @@ with the settings for cluster-type, master, and the node list as discovered by c
 Note that this means regexes are not directly saved to host groups, but the results of matching against
 those regexes are.
 .TP
+\fB\-j\fR JOBS \fB\-\-jobs\fR JOBS
+Specify the number of concurrent node collections that should be run.
+
+If the number of nodes enumerated exceeds the number of JOBS, then sos collect
+will start collecting from the first X number of nodes and then continue to iterate
+through the remaining nodes as sosreport collection finishes.
+
+Defaults to 4.
+.TP
 \fB\-\-nopasswd-sudo\fR
 Use this option when connecting as a non-root user that has passwordless sudo
 configured.
@@ -269,19 +279,18 @@ option cannot be removed from the sosreport command as it is required to run
 sosreport non-interactively for sos collect to function.
 .TP
 \fB\-t\fR THREADS \fB\-\-threads\fR THREADS
-Specify the number of threads to use for concurrent collection of sosreports.
+Report option. Specify the number of collection threads to run.
 
-If the number of nodes enumerated exceeds the number of threads, then sos collect
-will start collecting from the first X number of nodes and then continue to iterate
-through the remaining nodes as sosreport collection finishes.
+The report process on each node will run THREADS number of plugins concurrently
+during the collection process.
 
 Defaults to 4.
 .TP
 \fB\-\-timeout\fR TIMEOUT
 Timeout for sosreport generation on each node, in seconds.
 
-Note that sosreports are collected in parallel, so this can also be considered to be
-approximately the same as a timeout for the entire collection process. 
+Note that sosreports are collected in parallel, so you can approximate the total
+runtime of sos collect via timeout*(number of nodes/jobs).
 
 Default is 180 seconds.
 .TP
diff --git a/sos/collector/__init__.py b/sos/collector/__init__.py
index f2f8fd23..177ccc0c 100644
--- a/sos/collector/__init__.py
+++ b/sos/collector/__init__.py
@@ -61,6 +61,7 @@ class SoSCollector(SoSComponent):
         'group': None,
         'save_group': '',
         'image': '',
+        'jobs': 4,
         'ssh_key': '',
         'nopasswd_sudo': False,
         'plugin_options': [],
@@ -287,6 +288,8 @@ class SoSCollector(SoSComponent):
                                  help=('Specify the container image to use for'
                                        ' containerized hosts.'))
         collect_grp.add_argument('-i', '--ssh-key', help='Specify an ssh key')
+        collect_grp.add_argument('-j', '--jobs', default=4, type=int,
+                                 help='Number of concurrent nodes to collect')
         collect_grp.add_argument('-l', '--list-options', action="store_true",
                                  help='List options available for profiles')
         collect_grp.add_argument('--label',
@@ -1004,7 +1007,7 @@ this utility or remote systems that it connects to.
             nodes = _nodes
 
         try:
-            pool = ThreadPoolExecutor(self.opts.threads)
+            pool = ThreadPoolExecutor(self.opts.jobs)
             pool.map(self._connect_to_node, nodes, chunksize=1)
             pool.shutdown(wait=True)
 
@@ -1029,9 +1032,9 @@ this utility or remote systems that it connects to.
             self.ui_log.info("\nBeginning collection of sosreports from %s "
                              "nodes, collecting a maximum of %s "
                              "concurrently\n"
-                             % (self.report_num, self.opts.threads))
+                             % (self.report_num, self.opts.jobs))
 
-            pool = ThreadPoolExecutor(self.opts.threads)
+            pool = ThreadPoolExecutor(self.opts.jobs)
             pool.map(self._collect, self.client_list, chunksize=1)
             pool.shutdown(wait=True)
         except KeyboardInterrupt:
diff --git a/sos/collector/sosnode.py b/sos/collector/sosnode.py
index 08952cc7..14c11ab5 100644
--- a/sos/collector/sosnode.py
+++ b/sos/collector/sosnode.py
@@ -599,6 +599,12 @@ class SosNode():
 
         sos_opts = []
 
+        # sos-3.6 added --threads
+        if self.check_sos_version('3.6'):
+            # 4 threads is the project's default
+            if self.opts.threads != 4:
+                sos_opts.append('--threads=%s' % quote(str(self.opts.threads)))
+
         # sos-3.7 added options
         if self.check_sos_version('3.7'):
             if self.opts.plugin_timeout:
-- 
cgit