[sos] Add SoSCleaner Component

Adds a new component/subcommand `SoSCleaner`, accessible via `sos clean`
or `sos mask`.

This component is intended to bring similar functionality that is
available in the standalone `soscleaner` utility directly into the sos
project. It is designed to take either an untarr'ed sos directory, an sos
archive, or an archive of sos archives and obfuscate sensitive network
information (and optionally other data) from the report(s) that are
typically not able to be sanely scrubbed via the `postproc()` method of
plugins.

As of this first commit, users may execute `sos clean|mask $archive`
directly from the command line and expect to have an obfuscated archive
generated. Note that this obfuscated archive does NOT replace the
original archive on disk. Currently there is support for IPv4 IP
addresses, MAC addresses, and 64-bit IPv6 MAC addresses.

Future commits will aim to provide hooks for this functionality into
both `report` and `collect`. Additionally, more parsers will be added to
handle more types of data that needs consistent obfuscation.

Closes: #1987
Closes: #311

Signed-off-by: Jake Hunsaker <jhunsake@redhat.com>

14 files changed, 1249 insertions, 54 deletions

diff --git a/sos/__init__.py b/sos/__init__.py
index 6c249d86..e11dd103 100644
--- a/sos/__init__.py
+++ b/sos/__init__.py
@@ -57,8 +57,10 @@ class SoS():
         # of shorthand names to accept in place of the full subcommand
         # if no aliases are desired, pass an empty list
         import sos.report
+        import sos.cleaner
         self._components = {
-            'report': (sos.report.SoSReport, ['rep'])
+            'report': (sos.report.SoSReport, ['rep']),
+            'clean': (sos.cleaner.SoSCleaner, ['cleaner', 'mask'])
         }
         # some distros do not want pexpect as a default dep, so try to load
         # collector here, and if it fails add an entry that implies it is at
@@ -81,8 +83,13 @@ class SoS():
         # build the top-level parser
         _com_string = ''
         for com in self._components:
-            _com_string += ("\t%s\t\t\t%s\n"
-                            % (com, self._components[com][0].desc))
+            aliases = self._components[com][1]
+            aliases.insert(0, com)
+            _com = ', '.join(aliases)
+            desc = self._components[com][0].desc
+            _com_string += (
+                "\t{com:<30}{desc}\n".format(com=_com, desc=desc)
+            )
         usage_string = ("%(prog)s <component> [options]\n\n"
                         "Available components:\n")
         usage_string = usage_string + _com_string
@@ -92,6 +99,7 @@ class SoS():
         # set the component subparsers
         self.subparsers = self.parser.add_subparsers(
             dest='component',
+            metavar='component',
             help='sos component to run'
         )
         self.subparsers.required = True
@@ -115,6 +123,8 @@ class SoS():
         """Adds the options shared across components to the parser
         """
         global_grp = parser.add_argument_group('Global Options')
+        global_grp.add_argument("--batch", default=False, action="store_true",
+                                help="Do not prompt interactively")
         global_grp.add_argument("--config-file", type=str, action="store",
                                 dest="config_file", default="/etc/sos.conf",
                                 help="specify alternate configuration file")
@@ -137,6 +147,19 @@ class SoS():
                                 dest="verbosity", default=0,
                                 help="increase verbosity")
 
+        global_grp.add_argument('-z', '--compression-type',
+                                dest="compression_type",
+                                choices=['auto', 'gzip', 'xz'],
+                                help="compression technology to use")
+
+        # Group to make tarball encryption (via GPG/password) exclusive
+        encrypt_grp = global_grp.add_mutually_exclusive_group()
+        encrypt_grp.add_argument("--encrypt-key",
+                                 help="Encrypt the archive using a GPG "
+                                      "key-pair")
+        encrypt_grp.add_argument("--encrypt-pass",
+                                 help="Encrypt the archive using a password")
+
     def _init_component(self):
         """Determine which component has been requested by the user, and then
         initialize that component.
diff --git a/sos/cleaner/__init__.py b/sos/cleaner/__init__.py
new file mode 100644
index 00000000..2545e581
--- /dev/null
+++ b/sos/cleaner/__init__.py
@@ -0,0 +1,511 @@
+# Copyright 2020 Red Hat, Inc. Jake Hunsaker <jhunsake@redhat.com>
+
+# This file is part of the sos project: https://github.com/sosreport/sos
+#
+# This copyrighted material is made available to anyone wishing to use,
+# modify, copy, or redistribute it subject to the terms and conditions of
+# version 2 of the GNU General Public License.
+#
+# See the LICENSE file in the source distribution for further information.
+
+import json
+import logging
+import os
+import re
+import shutil
+import tarfile
+import tempfile
+
+from concurrent.futures import ThreadPoolExecutor
+from pwd import getpwuid
+from sos import __version__
+from sos.component import SoSComponent
+from sos.cleaner.parsers.ip_parser import SoSIPParser
+from sos.cleaner.parsers.mac_parser import SoSMacParser
+from sos.cleaner.obfuscation_archive import SoSObfuscationArchive
+from sos.utilities import get_human_readable
+from textwrap import fill
+
+
+class SoSCleaner(SoSComponent):
+    """Take an sos report, or collection of sos reports, and scrub them of
+    potentially sensitive data such as IP addresses, hostnames, MAC addresses,
+    etc.. that are not obfuscated by individual plugins
+    """
+
+    desc = "Obfuscate sensitive networking information in a report"
+
+    arg_defaults = {
+        'jobs': 4,
+        'map_file': '/etc/sos/cleaner/mapping',
+        'no_update': False,
+        'target': ''
+    }
+
+    def __init__(self, parser=None, args=None, cmdline=None, in_place=False):
+        if parser is not None and args is not None and cmdline is not None:
+            # we are running `sos clean` directly
+            super(SoSCleaner, self).__init__(parser, args, cmdline)
+            self.from_cmdline = True
+        else:
+            # we are being hooked by either SoSReport or SoSCollector, don't
+            # re-init everything
+            self.soslog = logging.getLogger('sos')
+            self.ui_log = logging.getLogger('sos_ui')
+            self.from_cmdline = False
+
+        self.validate_map_file()
+        os.umask(0o77)
+        self.in_place = in_place
+
+        self.parsers = [
+            SoSIPParser(self.opts.map_file),
+            SoSMacParser(self.opts.map_file)
+        ]
+
+        self.log_info("Cleaner initialized. From cmdline: %s"
+                      % self.from_cmdline)
+
+    def _fmt_log_msg(self, msg, caller=None):
+        return "[cleaner%s] %s" % (":%s" % caller if caller else '', msg)
+
+    def log_debug(self, msg, caller=None):
+        self.soslog.debug(self._fmt_log_msg(msg, caller))
+
+    def log_info(self, msg, caller=None):
+        self.soslog.info(self._fmt_log_msg(msg, caller))
+
+    def log_error(self, msg, caller=None):
+        self.soslog.error(self._fmt_log_msg(msg, caller))
+
+    def _fmt_msg(self, msg):
+        width = 80
+        _fmt = ''
+        for line in msg.splitlines():
+            _fmt = _fmt + fill(line, width, replace_whitespace=False) + '\n'
+        return _fmt
+
+    def validate_map_file(self):
+        """Verifies that the map file exists and has usable content.
+
+        If the provided map file does not exist, or it is empty, we will print
+        a warning and continue on with cleaning building a fresh map
+        """
+        default_map = '/etc/sos/cleaner/default_mapping'
+        if not os.path.exists(self.opts.map_file):
+            if self.opts.map_file != default_map:
+                self.log_error(
+                    "Map file %s does not exist, will not load any obfuscation"
+                    " matches" % self.opts.map_file)
+        if os.path.isdir(self.opts.map_file):
+            self.log_error(
+                "Requested map file %s is a directory. Ignoring `--map` option"
+                " and using %s" % default_map)
+            self.opts.map_file = default_map
+
+    def print_disclaimer(self):
+        """When we are directly running `sos clean`, rather than hooking into
+        SoSCleaner via report or collect, print a disclaimer banner
+        """
+        msg = self._fmt_msg("""\
+This command will attempt to obfuscate information that is generally \
+considered to be potentially sensitive. Such information includes IP \
+addresses, MAC addresses, domain names, and any user-provided keywords.
+
+Note that this utility provides a best-effort approach to data obfuscation, \
+but it does not guarantee that such obfuscation provides complete coverage of \
+all such data in the archive, or that any obfuscation is provided to data that\
+ does not fit the description above.
+
+Users should review any resulting data and/or archives generated or processed \
+by this utility for remaining sensitive content before being passed to a \
+third party.
+""")
+        self.ui_log.info("\nsos clean (version %s)\n" % __version__)
+        self.ui_log.info(msg)
+        if not self.opts.batch:
+            try:
+                input("\nPress ENTER to continue, or CTRL-C to quit.\n")
+            except KeyboardInterrupt:
+                self.ui_log.info("\nExiting on user cancel")
+                self._exit(130)
+
+    @classmethod
+    def add_parser_options(cls, parser):
+        parser.usage = 'sos clean|mask TARGET [options]'
+        clean_grp = parser.add_argument_group(
+            'Cleaner/Masking Options',
+            'These options control how data obfuscation is performed'
+        )
+        clean_grp.add_argument('target',
+                               help='The directory or archive to obfuscate')
+        clean_grp.add_argument('-j', '--jobs', default=4, type=int,
+                               help='Number of concurrent archives to clean')
+        clean_grp.add_argument('--map', dest='map_file',
+                               default='/etc/sos/cleaner/default_mapping',
+                               help=('Provide a previously generated mapping '
+                                     'file for obfuscation'))
+        clean_grp.add_argument('--no-update', dest='no_update', default=False,
+                               action='store_true',
+                               help='Do not update the --map file with new '
+                                    'mappings from this run')
+
+    def set_target_path(self, path):
+        """For use by report and collect to set the TARGET option appropriately
+        so that execute() can be called just as if we were running `sos clean`
+        directly from the cmdline.
+        """
+        self.opts.target = path
+
+    def inspect_target_archive(self):
+        """The target path is not a directory, so inspect it for being an
+        archive or an archive of archives.
+
+        In the event the target path is not an archive, abort.
+        """
+        if not tarfile.is_tarfile(self.opts.target):
+            self.ui_log.error(
+                "Invalid target: must be directory or tar archive"
+            )
+            self._exit(1)
+
+        archive = tarfile.open(self.opts.target)
+        self.arc_name = self.opts.target.split('/')[-1].split('.')[:-2][0]
+
+        try:
+            archive.getmember(os.path.join(self.arc_name, 'logs'))
+        except Exception:
+            # this is not a sos archive
+            self.ui_log.error("Invalid target: not an sos archive")
+            self._exit(1)
+
+        # see if there are archives within this archive
+        nested_archives = []
+        for _file in archive.getmembers():
+            if (re.match('sosreport-.*.tar', _file.name.split('/')[-1]) and not
+                    _file.name.endswith('.md5')):
+                nested_archives.append(_file.name.split('/')[-1])
+
+        if nested_archives:
+            self.log_info("Found nested archive(s), extracting top level")
+            nested_path = self.extract_archive(archive)
+            for arc_file in os.listdir(nested_path):
+                if re.match('sosreport.*.tar.*', arc_file):
+                    self.report_paths.append(os.path.join(nested_path,
+                                                          arc_file))
+            # add the toplevel extracted archive
+            self.report_paths.append(nested_path)
+        else:
+            self.report_paths.append(self.opts.target)
+
+        archive.close()
+
+    def extract_archive(self, archive):
+        """Extract an archive into our tmpdir so that we may inspect it or
+        iterate through its contents for obfuscation
+
+        Positional arguments:
+
+            :param archive:     An open TarFile object for the archive
+
+        """
+        if not isinstance(archive, tarfile.TarFile):
+            archive = tarfile.open(archive)
+        path = os.path.join(self.tmpdir, 'cleaner')
+        archive.extractall(path)
+        return os.path.join(path, archive.name.split('/')[-1].split('.tar')[0])
+
+    def execute(self):
+        """SoSCleaner will begin by inspecting the TARGET option to determine
+        if it is a directory, archive, or archive of archives.
+
+        In the case of a directory, the default behavior will be to edit the
+        data in place. For an archive will we unpack the archive, iterate
+        over the contents, and then repack the archive. In the case of an
+        archive of archives, such as one from SoSCollector, each archive will
+        be unpacked, cleaned, and repacked and the final top-level archive will
+        then be repacked as well.
+        """
+        if self.from_cmdline:
+            self.print_disclaimer()
+        self.report_paths = []
+        if not os.path.exists(self.opts.target):
+            self.ui_log.error("Invalid target: no such file or directory %s"
+                              % self.opts.target)
+            self._exit(1)
+        if os.path.isdir(self.opts.target):
+            for _file in os.listdir(self.opts.target):
+                if _file == 'sos_logs':
+                    self.report_paths.append(self.opts.target)
+                if re.match('sosreport.*.tar.*', _file):
+                    self.report_paths.append(_file)
+            if not self.report_paths:
+                self.ui_log.error("Invalid target: not an sos directory")
+                self._exit(1)
+        else:
+            self.inspect_target_archive()
+
+        if not self.report_paths:
+            self.ui_log.error("No valid sos archives or directories found\n")
+            self._exit(1)
+
+        # we have at least one valid target to obfuscate
+        self.completed_reports = []
+        self.obfuscate_report_paths()
+
+        if not self.completed_reports:
+            if self.in_place:
+                return None
+            self.ui_log.info("No reports obfuscated, aborting...\n")
+            self._exit(1)
+
+        self.ui_log.info("\nSuccessfully obfuscated %s report(s)\n"
+                         % len(self.completed_reports))
+
+        _map = self.compile_mapping_dict()
+        map_path = self.write_map_for_archive(_map)
+        self.write_map_for_config(_map)
+
+        if self.in_place:
+            return map_path
+
+        final_path = None
+        self.hash_name = self.policy.get_preferred_hash_name()
+        if len(self.completed_reports) > 1:
+            # we have an archive of archives, so repack the obfuscated tarball
+            arc_name = self.arc_name + '-obfuscated'
+            self.setup_archive(name=arc_name)
+            for arc in self.completed_reports:
+                if arc.is_tarfile:
+                    arc_dest = arc.final_archive_path.split('/')[-1]
+                    self.archive.add_file(arc.final_archive_path,
+                                          dest=arc_dest)
+                    checksum = self.get_new_checksum(arc)
+                    if checksum is not None:
+                        dname = "checksums/%s.%s" % (arc_dest, self.hash_name)
+                        self.archive.add_string(checksum, dest=dname)
+                else:
+                    for dirname, dirs, files in os.walk(arc.archive_path):
+                        for filename in files:
+                            if filename.startswith('sosreport'):
+                                continue
+                            fname = os.path.join(dirname, filename)
+                            dnm = fname.split(arc.archive_name)[-1].lstrip('/')
+                            self.archive.add_file(fname, dest=dnm)
+            arc_path = self.archive.finalize(self.opts.compression_type)
+        else:
+            arc = self.completed_reports[0]
+            arc_path = arc.final_archive_path
+            checksum = self.get_new_checksum(arc)
+            if checksum is not None:
+                chksum_name = "%s.%s" % (arc_path.split('/')[-1],
+                                         self.hash_name)
+                with open(os.path.join(self.sys_tmp, chksum_name), 'w') as cf:
+                    cf.write(checksum)
+
+        final_path = os.path.join(self.sys_tmp, arc_path.split('/')[-1])
+        shutil.move(arc_path, final_path)
+        arcstat = os.stat(final_path)
+
+        # logging will have been shutdown at this point
+        print("A mapping of obfuscated elements is available at\n\t%s"
+              % map_path)
+
+        print("\nThe obfuscated archive is available at\n\t%s\n" % final_path)
+        print("\tSize\t%s" % get_human_readable(arcstat.st_size))
+        print("\tOwner\t%s\n" % getpwuid(arcstat.st_uid).pw_name)
+
+        print("Please send the obfuscated archive to your support "
+              "representative and keep the mapping file private")
+
+        self.cleanup()
+
+    def compile_mapping_dict(self):
+        """Build a dict that contains each parser's map as a key, with the
+        contents as that key's value. This will then be written to disk in the
+        same directory as the obfuscated report so that sysadmins have a way
+        to 'decode' the obfuscation locally
+        """
+        _map = {}
+        for parser in self.parsers:
+            _map[parser.map_file_key] = {}
+            _map[parser.map_file_key].update(parser.mapping.dataset)
+
+        return _map
+
+    def write_map_to_file(self, _map, path):
+        """Write the mapping to a file on disk that is in the same location as
+        the final archive(s).
+        """
+        with open(path, 'w') as mf:
+            mf.write(json.dumps(_map, indent=4))
+        return path
+
+    def write_map_for_archive(self, _map):
+        try:
+            map_path = self.obfuscate_string(
+                os.path.join(self.sys_tmp, "%s_private_map" % self.arc_name)
+            )
+            return self.write_map_to_file(_map, map_path)
+        except Exception as err:
+            self.log_error("Could not write private map file: %s" % err)
+            return None
+
+    def write_map_for_config(self, _map):
+        """Write the mapping to the config file so that subsequent runs are
+        able to provide the same consistent mapping
+        """
+        if self.opts.map_file and not self.opts.no_update:
+            try:
+                self.write_map_to_file(_map, self.opts.map_file)
+                self.log_debug("Wrote mapping to %s" % self.opts.map_file)
+            except Exception as err:
+                self.log_error("Could not update mapping config file: %s"
+                               % err)
+
+    def get_new_checksum(self, archive):
+        """Get a new checksum for each archive"""
+        checksum = archive.generate_checksum(self.hash_name)
+        if checksum:
+            return checksum + '\n'
+        return None
+
+    def obfuscate_report_paths(self):
+        """Perform the obfuscation for each archive or sos directory discovered
+        during setup.
+
+        Each archive is handled in a separate thread, up to self.opts.jobs will
+        be obfuscated concurrently.
+        """
+        try:
+            pool = ThreadPoolExecutor(self.opts.jobs)
+            pool.map(self.obfuscate_report, self.report_paths, chunksize=1)
+            pool.shutdown(wait=True)
+        except KeyboardInterrupt:
+            self.ui_log.info("Exiting on user cancel")
+            os._exit(130)
+
+    def obfuscate_report(self, report):
+        """Individually handle each archive or directory we've discovered by
+        running through each file therein.
+
+        Positional arguments:
+
+            :param report str:      Filepath to the directory or archive
+        """
+        try:
+            if not os.access(report, os.W_OK):
+                self.log_info("Insufficient permissions on %s" % report)
+                self.report_msg(report, "Insufficient permissions")
+                return
+
+            archive = SoSObfuscationArchive(report, self.tmpdir)
+            archive.extract()
+            self.prep_maps_from_archive(archive)
+            archive.report_msg("Beginning obfuscation...")
+
+            file_list = archive.get_file_list()
+            for fname in file_list:
+                short_name = fname.split(archive.archive_name)[1]
+                if archive.should_skip_file(short_name):
+                    continue
+                try:
+                    count = self.obfuscate_file(fname)
+                    if count:
+                        archive.update_sub_count(short_name, count)
+                except Exception as err:
+                    self.log_debug("Unable to parse file %s: %s"
+                                   % (short_name, err))
+
+            # if the archive was already a tarball, repack it
+            method = archive.get_compression()
+            if method:
+                archive.report_msg("Re-compressing...")
+                try:
+                    cmd = self.policy.get_cmd_for_compress_method(
+                        method,
+                        self.opts.threads
+                    )
+                    archive.compress(cmd)
+                except Exception as err:
+                    self.log_debug("Archive %s failed to compress: %s"
+                                   % (archive.archive_name, err))
+                    archive.report_msg("Failed to re-compress archive: %s"
+                                       % err)
+                    return
+
+            self.completed_reports.append(archive)
+            archive.report_msg("Obfuscation completed")
+
+        except Exception as err:
+            self.ui_log.info("Exception while processing %s: %s"
+                             % (report, err))
+            os._exit(1)
+
+    def prep_maps_from_archive(self, archive):
+        """Open specific files from an archive and try to load those values
+        into our mappings before iterating through the entire archive.
+
+        Positional arguments:
+
+            :param archive SoSObfuscationArchive:   An open archive object
+        """
+        for parser in self.parsers:
+            self.obfuscate_file(archive.get_file_path(parser.prep_map_file))
+
+    def obfuscate_file(self, filename, short_name=None, arc_name=None):
+        """Obfuscate and individual file, line by line.
+
+        Lines processed, even if no substitutions occur, are then written to a
+        temp file without our own tmpdir. Once the file has been completely
+        iterated through, if there have been substitutions then the temp file
+        overwrites the original file. If there are no substitutions, then the
+        original file is left in place.
+
+        Positional arguments:
+
+            :param filename str:        Filename relative to the extracted
+                                        archive root
+        """
+        if not filename:
+            # the requested file doesn't exist in the archive
+            return
+        self.log_debug("Obfuscating %s" % filename)
+        subs = 0
+        tfile = tempfile.NamedTemporaryFile(mode='w', dir=self.tmpdir)
+        with open(filename, 'r') as fname:
+            for line in fname:
+                if not line.strip() or line.startswith('#'):
+                    continue
+                try:
+                    line, count = self.obfuscate_line(line)
+                    subs += count
+                    tfile.write(line)
+                except Exception as err:
+                    self.log_debug("Unable to obfuscate %s: %s"
+                                   % (filename, err))
+        tfile.seek(0)
+        if subs:
+            shutil.copy(tfile.name, filename)
+        tfile.close()
+        return subs
+
+    def obfuscate_line(self, line):
+        """Run a line through each of the obfuscation parsers, keeping a
+        cumulative total of substitutions done on that particular line.
+
+        Positional arguments:
+
+            :param line str:        The raw line as read from the file being
+                                    processed
+
+        Returns the fully obfuscated line and the number of substitutions made
+        """
+        count = 0
+        for parser in self.parsers:
+            try:
+                line, _count = parser.parse_line(line)
+                count += _count
+            except Exception as err:
+                self.log_debug("failed to parse line: %s" % err, parser.name)
+        return line, count
diff --git a/sos/cleaner/mappings/__init__.py b/sos/cleaner/mappings/__init__.py
new file mode 100644
index 00000000..212aeb9f
--- /dev/null
+++ b/sos/cleaner/mappings/__init__.py
@@ -0,0 +1,84 @@
+# Copyright 2020 Red Hat, Inc. Jake Hunsaker <jhunsake@redhat.com>
+
+# This file is part of the sos project: https://github.com/sosreport/sos
+#
+# This copyrighted material is made available to anyone wishing to use,
+# modify, copy, or redistribute it subject to the terms and conditions of
+# version 2 of the GNU General Public License.
+#
+# See the LICENSE file in the source distribution for further information.
+
+import re
+
+from collections import OrderedDict
+from threading import Lock
+
+
+class SoSMap():
+    """Standardized way to store items with their obfuscated counterparts.
+
+    Each type of sanitization that SoSCleaner supports should have a
+    corresponding SoSMap() object, to allow for easy retrieval of obfuscated
+    items.
+    """
+
+    ignore_list = []
+
+    def __init__(self):
+        self.dataset = OrderedDict()
+        self.lock = Lock()
+
+    def ignore_item(self, item):
+        """Some items need to be completely ignored, for example link-local or
+        loopback addresses should not be obfuscated
+        """
+        for skip in self.ignore_matches:
+            if re.match(skip, item):
+                return True
+
+    def item_in_dataset_values(self, item):
+        return item in self.dataset.values()
+
+    def add(self, item):
+        """Add a particular item to the map, generating an obfuscated pair
+        for it.
+
+        Positional arguments:
+
+            :param item:        The plaintext object to obfuscate
+        """
+        with self.lock:
+            self.dataset[item] = self.sanitize_item(item)
+            return self.dataset[item]
+
+    def sanitize_item(self, item):
+        """Perform the obfuscation relevant to the item being added to the map.
+
+        This should be overridden by each type of map that subclasses SoSMap
+
+        Positional arguments:
+
+            :param item:        The plaintext object to obfuscate
+        """
+        return item
+
+    def get(self, item):
+        """Retrieve an item's obfuscated counterpart from the map. If the item
+        does not yet exist in the map, add it by generating one on the fly
+        """
+        if self.ignore_item(item) or self.item_in_dataset_values(item):
+            return item
+        if item not in self.dataset:
+            return self.add(item)
+        return self.dataset[item]
+
+    def conf_update(self, map_dict):
+        """Update the map using information from a previous run to ensure that
+        we have consistent obfuscation between reports
+
+        Positional arguments:
+
+            :param map_dict:    A dict of mappings with the form of
+                                {clean_entry: 'obfuscated_entry'}
+        """
+        self.dataset.update(map_dict)
diff --git a/sos/cleaner/mappings/ip_map.py b/sos/cleaner/mappings/ip_map.py
new file mode 100644
index 00000000..70519538
--- /dev/null
+++ b/sos/cleaner/mappings/ip_map.py
@@ -0,0 +1,203 @@
+# Copyright 2020 Red Hat, Inc. Jake Hunsaker <jhunsake@redhat.com>
+
+# This file is part of the sos project: https://github.com/sosreport/sos
+#
+# This copyrighted material is made available to anyone wishing to use,
+# modify, copy, or redistribute it subject to the terms and conditions of
+# version 2 of the GNU General Public License.
+#
+# See the LICENSE file in the source distribution for further information.
+
+import ipaddress
+import random
+
+from sos.cleaner.mappings import SoSMap
+
+
+class SoSIPMap(SoSMap):
+    """A mapping store for IP addresses
+
+    Each IP address added to this map is chcked for subnet membership. If that
+    subnet already exists in the map, then IP addresses are deterministically
+    generated sequentially within that subnet. For example, if a given IP is
+    matched to subnet 192.168.1.0/24 then 192.168.1 may be obfuscated to
+    100.11.12.0/24. Each IP address in the original 192.168.1.0/24 subnet
+    will then be assigned an address in 100.11.12.0/24 sequentially, such as
+    100.11.12.1, 100.11.12.2, etc...
+
+
+    Internally, the ipaddress library is used to manipulate the address objects
+    however, when retrieved by SoSCleaner any values will be strings.
+    """
+
+    ignore_matches = [
+        '127.*',
+        '::1',
+        '0\.(.*)?',
+        '1\.(.*)?',
+        '8.8.8.8',
+        '8.8.4.4',
+        '169.254.*',
+        '255.*'
+    ]
+
+    _networks = {}
+    network_first_octet = 100
+    skip_network_octets = ['127', '169', '172', '192']
+
+    def ip_in_dataset(self, ipaddr):
+        """There are multiple ways in which an ip address could be handed to us
+        in a way where we're matching against a previously obfuscated address.
+
+        Here, match the ip address to any of the obfuscated addresses we've
+        already created
+        """
+        for _ip in self.dataset.values():
+            if str(ipaddr).split('/')[0] == _ip.split('/')[0]:
+                return True
+        return False
+
+    def get(self, ipaddr):
+        """Ensure that when requesting an obfuscated address, we return a str
+        object instead of an IPv(4|6)Address object
+        """
+        filt_start = ('/', '=', ']', ')')
+        if ipaddr.startswith(filt_start):
+            ipaddr = ipaddr.lstrip(''.join(filt_start))
+
+        if ipaddr in self.dataset.keys():
+            return self.dataset[ipaddr]
+
+        if self.ignore_item(ipaddr) or self.ip_in_dataset(ipaddr):
+            return ipaddr
+
+        # it's not in there, but let's make sure we haven't previously added
+        # an address with a CIDR notation and we're now looking for it without
+        # that notation
+        if '/' not in ipaddr:
+            for key in self.dataset.keys():
+                if key.startswith(ipaddr):
+                    return self.dataset[key].split('/')[0]
+
+        # fallback to the default map behavior of adding it fresh
+        return self.add(ipaddr)
+
+    def set_ip_cidr_from_existing_subnet(self, addr):
+        """Determine if a given address is in a subnet of an already obfuscated
+        network and if it is, then set the address' network to the network
+        object we're tracking. This allows us to match ip addresses with or
+        without a CIDR notation and maintain proper network relationships.
+        """
+        nets = []
+        for net in self._networks:
+            if addr.ip == net.broadcast_address:
+                addr.network = net
+                return
+            if addr.ip in net:
+                nets.append(net)
+        # assign the address to the smallest network that was matched. This is
+        # necessary due to certain files specifying addresses that cause the
+        # ipaddress library to create artificially huge subnets that will
+        # include the actual subnets used by the system
+        if nets:
+            nets.sort(key=lambda n: n.prefixlen, reverse=True)
+            addr.network = nets[0]
+
+    def sanitize_item(self, item):
+        """Given an IP address, sanitize it to an obfuscated network or host
+        address as appropriate
+        """
+
+        try:
+            addr = ipaddress.ip_interface(item)
+        except ValueError:
+            # not an IP, add it to the skip list to avoid flooding logs
+            self.ignore_matches.append(item)
+            raise
+        network = addr.network
+
+        if str(network.netmask) == '255.255.255.255':
+            # check to see if this IP is in a subnet of an already obfuscated
+            # network and if it has, replace the default /32 netmask that
+            # ipaddress applies to no CIDR-notated addresses
+            self.set_ip_cidr_from_existing_subnet(addr)
+            return self.sanitize_ipaddr(addr)
+        else:
+            # we have a CIDR notation, so generate an obfuscated network
+            # address and then generate an IP address within that network's
+            # range
+            self.sanitize_network(network)
+            return self.sanitize_ipaddr(addr)
+
+    def sanitize_network(self, network):
+        """Obfuscate the network address provided, and if there are host bits
+        in the address then obfuscate those as well
+        """
+        # check if the address is in a network we've already encountered
+        if network not in self._networks:
+            self._new_obfuscated_network(network)
+
+    def sanitize_ipaddr(self, addr):
+        """Obfuscate the IP address within the known obfuscated network
+        """
+        # get the obfuscated network object
+        if addr.network in self._networks:
+            _obf_network = self._networks[addr.network]
+
+            # if the plain address is the broadcast address for it's own
+            # network, then assign the broadcast address for the obfuscated
+            # network
+            if addr.ip == addr.network.broadcast_address:
+                return str(_obf_network.broadcast_address)
+
+            # otherwise within that obfuscated network grab the next available
+            # address from it
+            for _ip in _obf_network.hosts():
+                if not self.ip_in_dataset(_ip):
+                    # the ipaddress module does not assign the network's
+                    # netmask to hosts in the hosts() generator for some reason
+                    return "%s/%s" % (str(_ip), _obf_network.prefixlen)
+
+        # ip is a single ip address without the netmask
+        return self._new_obfuscated_single_address()
+
+    def _new_obfuscated_single_address(self):
+        def _gen_address():
+            _octets = []
+            for i in range(0, 4):
+                _octets.append(random.randint(11, 99))
+            return "%s.%s.%s.%s" % tuple(_octets)
+
+        _addr = _gen_address()
+        if _addr in self.dataset.values():
+            return self._new_obfuscated_single_address()
+        return _addr
+
+    def _new_obfuscated_network(self, network):
+        """Generate an obfuscated network address for the network address given
+        which will allow us to maintain network relationships without divulging
+        actual network details
+
+        Positional arguments:
+
+            :param network:     An ipaddress.IPv{4|6)Network object
+        """
+        _obf_network = None
+
+        if isinstance(network, ipaddress.IPv4Network):
+            if self.network_first_octet in self.skip_network_octets:
+                self.network_first_octet += 1
+            _obf_address = "%s.0.0.0" % self.network_first_octet
+            _obf_mask = network.with_netmask.split('/')[1]
+            _obf_network = ipaddress.IPv4Network(
+                "%s/%s" % (_obf_address, _obf_mask)
+            )
+            self.network_first_octet += 1
+
+        if isinstance(network, ipaddress.IPv6Network):
+            # TODO: define this
+            pass
+
+        if _obf_network:
+            self._networks[network] = _obf_network
+            self.dataset[str(network)] = str(_obf_network)
diff --git a/sos/cleaner/mappings/mac_map.py b/sos/cleaner/mappings/mac_map.py
new file mode 100644
index 00000000..4b9ea7ef
--- /dev/null
+++ b/sos/cleaner/mappings/mac_map.py
@@ -0,0 +1,78 @@
+# Copyright 2020 Red Hat, Inc. Jake Hunsaker <jhunsake@redhat.com>
+
+# This file is part of the sos project: https://github.com/sosreport/sos
+#
+# This copyrighted material is made available to anyone wishing to use,
+# modify, copy, or redistribute it subject to the terms and conditions of
+# version 2 of the GNU General Public License.
+#
+# See the LICENSE file in the source distribution for further information.
+
+import random
+import re
+
+from sos.cleaner.mappings import SoSMap
+
+
+class SoSMacMap(SoSMap):
+    """Mapping store for MAC addresses
+
+    MAC addresses added to this map will be broken into two halves, vendor and
+    device like how MAC addresses are normally crafted. For the vendor hextets,
+    obfuscation will take the form of 53:4f:53, or 'SOS' in hex. The following
+    device hextets will be randomized, for example a MAC address of
+    '60:55:cb:4b:c9:27' may be obfuscated into '53:4f:53:79:ac:29' or similar
+
+    This map supports both 48-bit and 64-bit MAC addresses.
+
+    48-bit address may take the form of either:
+
+            MM:MM:MM:SS:SS:SS
+            MM-MM-MM-SS-SS-SS
+
+    For 64-bit addresses, the identifier injected by IPv6 standards is
+    used in obfuscated returns. These addresses may take either of these forms:
+
+        MM:MM:MM:FF:FE:SS:SS:SS
+        MMMM:MMFF:FESS:SSSS
+
+    All mapped mac addresses are converted to lower case.
+    Dash delimited styles will be converted to colon-delimited style.
+    """
+
+    ignore_matches = [
+        'ff:ff:ff:ff:ff:ff',
+        '00:00:00:00:00:00'
+    ]
+
+    mac_template = '53:4f:53:%s:%s:%s'
+    mac6_template = '53:4f:53:ff:fe:%s:%s:%s'
+    mac6_quad_template = '534f:53ff:fe%s:%s%s'
+
+    def add(self, item):
+        item = item.replace('-', ':').lower().strip('=.,').strip()
+        return super(SoSMacMap, self).add(item)
+
+    def get(self, item):
+        item = item.replace('-', ':').lower().strip('=.,').strip()
+        return super(SoSMacMap, self).get(item)
+
+    def sanitize_item(self, item):
+        """Randomize the device hextets, and append those to our 'vendor'
+        hextet
+        """
+        hexdigits = "0123456789abdcef"
+        hextets = []
+        for i in range(0, 3):
+            hextets.append(''.join(random.choice(hexdigits) for x in range(2)))
+
+        hextets = tuple(hextets)
+        # match 64-bit IPv6 MAC addresses matching MM:MM:MM:FF:FE:SS:SS:SS
+        if re.match('(([0-9a-fA-F]{2}:){7}[0-9a-fA-F]{2})', item):
+            return self.mac6_template % hextets
+        # match 64-bit IPv6 MAC addresses matching MMMM:MMFF:FESS:SSSS
+        if re.match('(([0-9a-fA-F]{4}:){3}([0-9a-fA-F]){4})', item):
+            return self.mac6_quad_template % hextets
+        # match 48-bit IPv4 MAC addresses
+        if re.match('([0-9a-fA-F]:?){12}', item):
+            return self.mac_template % hextets
diff --git a/sos/cleaner/obfuscation_archive.py b/sos/cleaner/obfuscation_archive.py
new file mode 100644
index 00000000..fc2db4db
--- /dev/null
+++ b/sos/cleaner/obfuscation_archive.py
@@ -0,0 +1,196 @@
+# Copyright 2020 Red Hat, Inc. Jake Hunsaker <jhunsake@redhat.com>
+
+# This file is part of the sos project: https://github.com/sosreport/sos
+#
+# This copyrighted material is made available to anyone wishing to use,
+# modify, copy, or redistribute it subject to the terms and conditions of
+# version 2 of the GNU General Public License.
+#
+# See the LICENSE file in the source distribution for further information.
+
+import hashlib
+import logging
+import os
+import tarfile
+import re
+
+from sos.utilities import sos_get_command_output
+
+
+class SoSObfuscationArchive():
+    """A representation of an extracted archive or an sos archive build
+    directory which is used by SoSCleaner.
+
+    Each archive that needs to be obfuscated is loaded into an instance of this
+    class. All report-level operations should be contained within this class.
+    """
+
+    file_sub_list = []
+    total_sub_count = 0
+
+    def __init__(self, archive_path, tmpdir):
+        self.archive_path = archive_path
+        self.final_archive_path = self.archive_path
+        self.tmpdir = tmpdir
+        self.archive_name = self.archive_path.split('/')[-1].split('.tar')[0]
+        self.soslog = logging.getLogger('sos')
+        self.ui_log = logging.getLogger('sos_ui')
+        self.skip_list = self._load_skip_list()
+        self.log_info("Loaded %s as an archive" % self.archive_path)
+
+    def report_msg(self, msg):
+        """Helper to easily format ui messages on a per-report basis"""
+        self.ui_log.info("{:<50} {}".format(self.archive_name + ' :', msg))
+
+    def _fmt_log_msg(self, msg):
+        return "[cleaner:%s] %s" % (self.archive_name, msg)
+
+    def log_debug(self, msg):
+        self.soslog.debug(self._fmt_log_msg(msg))
+
+    def log_info(self, msg):
+        self.soslog.info(self._fmt_log_msg(msg))
+
+    def _load_skip_list(self):
+        """Provide a list of files and file regexes to skip obfuscation on
+
+        Returns: list of files and file regexes
+        """
+        return [
+            '/installed-debs',
+            '/installed-rpms',
+            '/sos_commands/dpkg',
+            '/sos_commands/python/pip_list',
+            '/sos_commands/rpm',
+            '/sos_commands/yum/.*list.*',
+            '/sos_commands/snappy/snap_list_--all',
+            '/sos_commands/snappy/snap_--version',
+            '/sos_commands/vulkan/vulkaninfo',
+            '/sys/firmware',
+            '/sys/fs',
+            '/sys/kernel/debug',
+            '/sys/module',
+            '/var/log/.*dnf.*',
+            '.*.tar.*',  # TODO: support archive unpacking
+            '.*.gz'
+        ]
+
+    @property
+    def is_tarfile(self):
+        try:
+            return tarfile.is_tarfile(self.archive_path)
+        except Exception:
+            return False
+
+    def extract(self):
+        if self.is_tarfile:
+            self.report_msg("Extracting...")
+            self.extracted_path = self.extract_self()
+        else:
+            self.extracted_path = self.archive_path
+
+    def get_compression(self):
+        """Return the compression type used by the archive, if any. This is
+        then used by SoSCleaner to generate a policy-derived compression
+        command to repack the archive
+        """
+        if self.is_tarfile:
+            if self.archive_path.endswith('xz'):
+                return 'xz'
+            return 'gzip'
+        return None
+
+    def build_tar_file(self):
+        """Pack the extracted archive as a tarfile to then be re-compressed
+        """
+        self.tarpath = self.extracted_path + '-obfuscated.tar'
+        self.log_debug("building tar file %s" % self.tarpath)
+        tar = tarfile.open(self.tarpath, mode="w")
+        tar.add(self.extracted_path,
+                arcname=os.path.split(self.archive_name)[1])
+        tar.close()
+
+    def compress(self, cmd):
+        """Execute the compression command, and set the appropriate final
+        archive path for later reference by SoSCleaner on a per-archive basis
+        """
+        self.build_tar_file()
+        exec_cmd = "%s %s" % (cmd, self.tarpath)
+        res = sos_get_command_output(exec_cmd, timeout=0, stderr=True)
+        if res['status'] == 0:
+            self.final_archive_path = self.tarpath + '.' + exec_cmd[0:2]
+        else:
+            err = res['output'].split(':')[-1]
+            self.log_debug("Exception while compressing archive: %s" % err)
+            raise Exception(err)
+
+    def generate_checksum(self, hash_name):
+        """Calculate a new checksum for the obfuscated archive, as the previous
+        checksum will no longer be valid
+        """
+        try:
+            hash_size = 1024**2  # Hash 1MiB of content at a time.
+            archive_fp = open(self.final_archive_path, 'rb')
+            digest = hashlib.new(hash_name)
+            while True:
+                hashdata = archive_fp.read(hash_size)
+                if not hashdata:
+                    break
+                digest.update(hashdata)
+            archive_fp.close()
+            return digest.hexdigest()
+        except Exception as err:
+            self.log_debug("Could not generate new checksum: %s" % err)
+        return None
+
+    def extract_self(self):
+        """Extract an archive into our tmpdir so that we may inspect it or
+        iterate through its contents for obfuscation
+        """
+        archive = tarfile.open(self.archive_path)
+        path = os.path.join(self.tmpdir, 'cleaner')
+        archive.extractall(path)
+        archive.close()
+        return os.path.join(path, archive.name.split('/')[-1].split('.tar')[0])
+
+    def get_file_list(self):
+        """Return a list of all files within the archive"""
+        self.file_list = []
+        for dirname, dirs, files in os.walk(self.extracted_path):
+            for filename in files:
+                self.file_list.append(os.path.join(dirname, filename))
+        return self.file_list
+
+    def update_sub_count(self, fname, count):
+        """Called when a file has finished being parsed and used to track
+        total substitutions made and number of files that had changes made
+        """
+        self.file_sub_list.append(fname)
+        self.total_sub_count += count
+
+    def get_file_path(self, fname):
+        """Return the filepath of a specific file within the archive so that
+        it may be selectively inspected if it exists
+        """
+        _path = os.path.join(self.extracted_path, fname.lstrip('/'))
+        return _path if os.path.exists(_path) else ''
+
+    def should_skip_file(self, filename):
+        """Checks the provided filename against a list of filepaths to not
+        perform obfuscation on, as defined in self.skip_list
+
+        Positional arguments:
+
+            :param filename str:        Filename relative to the extracted
+                                        archive root
+        """
+        if filename in self.file_sub_list:
+            return True
+
+        if not os.path.isfile(self.get_file_path(filename)):
+            return True
+
+        for _skip in self.skip_list:
+            if filename.startswith(_skip) or re.match(_skip, filename):
+                return True
+        return False
diff --git a/sos/cleaner/parsers/__init__.py b/sos/cleaner/parsers/__init__.py
new file mode 100644
index 00000000..960ebba2
--- /dev/null
+++ b/sos/cleaner/parsers/__init__.py
@@ -0,0 +1,73 @@
+# Copyright 2020 Red Hat, Inc. Jake Hunsaker <jhunsake@redhat.com>
+
+# This file is part of the sos project: https://github.com/sosreport/sos
+#
+# This copyrighted material is made available to anyone wishing to use,
+# modify, copy, or redistribute it subject to the terms and conditions of
+# version 2 of the GNU General Public License.
+#
+# See the LICENSE file in the source distribution for further information.
+
+import json
+import re
+
+
+class SoSCleanerParser():
+    """Parsers are used to build objects that will take a line as input,
+    parse it for a particular pattern (E.G. IP addresses) and then make any
+    necessary subtitutions by referencing the SoSMap() associated with the
+    parser.
+
+    Ideally a new parser subclass will only need to set the class level attrs
+    in order to be fully functional.
+
+        :attr name str:                 The parser name, used in logging errors
+
+        :attr regex_patterns list:      A list of regex patterns to iterate
+                                        over for every line processed
+        :attr mapping SoSMap:           A SoSMap used by the parser to store
+                                        and obfuscate pattern matches
+        :attr map_file_key str:         The key in the map_file to read when
+                                        loading previous obfuscation matches
+        :attr prep_map_file str:        Filename to attempt to read from an
+                                        archive to pre-seed the map with
+                                        matches. E.G. ip_addr for loading IP
+                                        addresses into the SoSIPMap.
+    """
+
+    name = 'Undefined Parser'
+    regex_patterns = []
+    map_file_key = 'unset'
+    prep_map_file = 'unset'
+
+    def __init__(self, conf_file=None):
+        # attempt to load previous run data into the mapping for the parser
+        if conf_file:
+            try:
+                with open(conf_file, 'r') as map_file:
+                    _default_mappings = json.load(map_file)
+                if self.map_file_key in _default_mappings:
+                    self.mapping.conf_update(
+                        _default_mappings[self.map_file_key]
+                    )
+            except IOError:
+                pass
+
+    def parse_line(self, line):
+        """This will be called for every line in every file we process, so that
+        every parser has a chance to scrub everything.
+        """
+        count = 0
+        for pattern in self.regex_patterns:
+            matches = [m[0] for m in re.findall(pattern, line, re.I)]
+            if matches:
+                count += len(matches)
+                for match in matches:
+                    new_match = self.mapping.get(match.strip())
+                    line = line.replace(match.strip(), new_match)
+        return line, count
+
+    def get_map_contents(self):
+        """Return the contents of the mapping used by the parser
+        """
+        return self.mapping.dataset
diff --git a/sos/cleaner/parsers/ip_parser.py b/sos/cleaner/parsers/ip_parser.py
new file mode 100644
index 00000000..05dcfb7b
--- /dev/null
+++ b/sos/cleaner/parsers/ip_parser.py
@@ -0,0 +1,28 @@
+# Copyright 2020 Red Hat, Inc. Jake Hunsaker <jhunsake@redhat.com>
+
+# This file is part of the sos project: https://github.com/sosreport/sos
+#
+# This copyrighted material is made available to anyone wishing to use,
+# modify, copy, or redistribute it subject to the terms and conditions of
+# version 2 of the GNU General Public License.
+#
+# See the LICENSE file in the source distribution for further information.
+
+from sos.cleaner.parsers import SoSCleanerParser
+from sos.cleaner.mappings.ip_map import SoSIPMap
+
+
+class SoSIPParser(SoSCleanerParser):
+    """Handles parsing for IP addresses"""
+
+    name = 'IP Parser'
+    regex_patterns = [
+        # IPv4 with or without CIDR
+        r'((?<!(-|\.|\d))([0-9]{1,3}\.){3}([0-9]){1,3}(\/([0-9]{1,2}))?)'
+    ]
+    map_file_key = 'ip_map'
+    prep_map_file = 'sos_commands/networking/ip_-o_addr'
+
+    def __init__(self, conf_file=None):
+        self.mapping = SoSIPMap()
+        super(SoSIPParser, self).__init__(conf_file)
diff --git a/sos/cleaner/parsers/mac_parser.py b/sos/cleaner/parsers/mac_parser.py
new file mode 100644
index 00000000..6b4be905
--- /dev/null
+++ b/sos/cleaner/parsers/mac_parser.py
@@ -0,0 +1,31 @@
+# Copyright 2020 Red Hat, Inc. Jake Hunsaker <jhunsake@redhat.com>
+
+# This file is part of the sos project: https://github.com/sosreport/sos
+#
+# This copyrighted material is made available to anyone wishing to use,
+# modify, copy, or redistribute it subject to the terms and conditions of
+# version 2 of the GNU General Public License.
+#
+# See the LICENSE file in the source distribution for further information.
+
+from sos.cleaner.parsers import SoSCleanerParser
+from sos.cleaner.mappings.mac_map import SoSMacMap
+
+
+class SoSMacParser(SoSCleanerParser):
+    """Handles parsing for MAC addresses"""
+
+    name = 'MAC Parser'
+    regex_patterns = [
+        # IPv6
+        r'(([^:|-])([0-9a-fA-F]{2}(:|-)){7}[0-9a-fA-F]{2}(\s|$))',
+        r'(([^:|-])([0-9a-fA-F]{4}(:|-)){3}[0-9a-fA-F]{4}(\s|$))',
+        # IPv4, avoiding matching a substring within IPv6 addresses
+        r'(([^:|-])([0-9a-fA-F]{2}([:-])){5}([0-9a-fA-F]){2}(\s|$))'
+    ]
+    map_file_key = 'mac_map'
+    prep_map_file = 'sos_commands/networking/ip_-d_address'
+
+    def __init__(self, conf_file=None):
+        self.mapping = SoSMacMap()
+        super(SoSMacParser, self).__init__(conf_file)
diff --git a/sos/collector/__init__.py b/sos/collector/__init__.py
index 1bb491e9..3bbc271d 100644
--- a/sos/collector/__init__.py
+++ b/sos/collector/__init__.py
@@ -50,7 +50,6 @@ class SoSCollector(SoSComponent):
         'all_logs': False,
         'allow_system_changes': False,
         'become_root': False,
-        'batch': False,
         'case_id': False,
         'cluster_type': None,
         'cluster_options': [],
@@ -84,8 +83,7 @@ class SoSCollector(SoSComponent):
         'sos_opt_line': '',
         'ssh_user': 'root',
         'timeout': 600,
-        'verify': False,
-        'compression_type': 'auto'
+        'verify': False
     }
 
     def __init__(self, parser, parsed_args, cmdline_args):
@@ -278,8 +276,6 @@ class SoSCollector(SoSComponent):
         collect_grp.add_argument('-b', '--become', action='store_true',
                                  dest='become_root',
                                  help='Become root on the remote nodes')
-        collect_grp.add_argument('--batch', action='store_true',
-                                 help='Do not prompt interactively')
         collect_grp.add_argument('--case-id', help='Specify case number')
         collect_grp.add_argument('--cluster-type',
                                  help='Specify a type of cluster profile')
@@ -331,18 +327,6 @@ class SoSCollector(SoSComponent):
                                  help='Specify an SSH user. Default root')
         collect_grp.add_argument('--timeout', type=int, required=False,
                                  help='Timeout for sosreport on each node.')
-        collect_grp.add_argument('-z', '--compression-type',
-                                 dest="compression",
-                                 choices=['auto', 'gzip', 'xz'],
-                                 help="compression technology to use")
-
-        # Group to make tarball encryption (via GPG/password) exclusive
-        encrypt_grp = collect_grp.add_mutually_exclusive_group()
-        encrypt_grp.add_argument("--encrypt-key",
-                                 help=("Encrypt the archive using a GPG "
-                                       "key-pair"))
-        encrypt_grp.add_argument("--encrypt-pass",
-                                 help="Encrypt the archive using a password")
 
     def _check_for_control_persist(self):
         """Checks to see if the local system supported SSH ControlPersist.
@@ -795,7 +779,7 @@ class SoSCollector(SoSComponent):
         if self.opts.chroot:
             sos_opts.append('-c %s' % quote(self.opts.chroot))
         if self.opts.compression_type != 'auto':
-            sos_opts.append('-z %s' % (quote(self.opts.compression)))
+            sos_opts.append('-z %s' % (quote(self.opts.compression_type)))
         self.sos_cmd = self.sos_cmd + ' '.join(sos_opts)
         self.log_debug("Initial sos cmd set to %s" % self.sos_cmd)
         self.commons['sos_cmd'] = self.sos_cmd
diff --git a/sos/component.py b/sos/component.py
index f966340a..9d80bc61 100644
--- a/sos/component.py
+++ b/sos/component.py
@@ -51,8 +51,12 @@ class SoSComponent():
     load_policy = True
 
     _arg_defaults = {
+        "batch": False,
+        "compression_type": 'auto',
         "config_file": '/etc/sos.conf',
         "debug": False,
+        "encrypt_key": None,
+        "encrypt_pass": None,
         "quiet": False,
         "threads": 4,
         "tmp_dir": '',
diff --git a/sos/policies/__init__.py b/sos/policies/__init__.py
index 9b20b63d..7936590b 100644
--- a/sos/policies/__init__.py
+++ b/sos/policies/__init__.py
@@ -14,7 +14,8 @@ from sos.utilities import (ImporterHelper,
                            import_module,
                            is_executable,
                            shell_out,
-                           sos_get_command_output)
+                           sos_get_command_output,
+                           get_human_readable)
 from sos.report.plugins import IndependentPlugin, ExperimentalPlugin
 from sos.options import SoSOptions
 from sos import _sos as _
@@ -29,17 +30,6 @@ try:
 except ImportError:
     REQUESTS_LOADED = False
 
-
-def get_human_readable(size, precision=2):
-    # Credit to Pavan Gupta https://stackoverflow.com/questions/5194057/
-    suffixes = ['B', 'KiB', 'MiB', 'GiB', 'TiB']
-    suffixindex = 0
-    while size > 1024 and suffixindex < 4:
-        suffixindex += 1
-        size = size/1024.0
-    return "%.*f%s" % (precision, size, suffixes[suffixindex])
-
-
 def import_policy(name):
     policy_fqname = "sos.policies.%s" % name
     try:
diff --git a/sos/report/__init__.py b/sos/report/__init__.py
index 04c2749e..fa0c4573 100644
--- a/sos/report/__init__.py
+++ b/sos/report/__init__.py
@@ -75,7 +75,6 @@ class SoSReport(SoSComponent):
     arg_defaults = {
         'alloptions': False,
         'all_logs': False,
-        'batch': False,
         'build': False,
         'case_id': '',
         'chroot': 'auto',
@@ -100,7 +99,6 @@ class SoSReport(SoSComponent):
         'profiles': [],
         'since': None,
         'verify': False,
-        'compression_type': 'auto',
         'allow_system_changes': False,
         'upload': False,
         'upload_url': None,
@@ -108,9 +106,7 @@ class SoSReport(SoSComponent):
         'upload_user': None,
         'upload_pass': None,
         'add_preset': '',
-        'del_preset': '',
-        'encrypt_key': None,
-        'encrypt_pass': None
+        'del_preset': ''
     }
 
     def __init__(self, parser, args, cmdline):
@@ -184,9 +180,6 @@ class SoSReport(SoSComponent):
                             help="Escapes archived files older than date. "
                                  "This will also affect --all-logs. "
                                  "Format: YYYYMMDD[HHMMSS]")
-        parser.add_argument("--batch", action="store_true",
-                            dest="batch", default=False,
-                            help="batch mode - do not prompt interactively")
         parser.add_argument("--build", action="store_true",
                             dest="build", default=False,
                             help="preserve the temporary directory and do not "
@@ -254,11 +247,6 @@ class SoSReport(SoSComponent):
         parser.add_argument("--verify", action="store_true",
                             dest="verify", default=False,
                             help="perform data verification during collection")
-        parser.add_argument("-z", "--compression-type",
-                            dest="compression_type",
-                            default='auto',
-                            help="compression technology to use [auto, "
-                                 "gzip, xz] (default=auto)")
         parser.add_argument("--allow-system-changes", action="store_true",
                             dest="allow_system_changes", default=False,
                             help="Run commands even if they can change the "
@@ -281,14 +269,6 @@ class SoSReport(SoSComponent):
         preset_grp.add_argument("--del-preset", type=str, action="store",
                                 help="Delete the named command line preset")
 
-        # Group to make tarball encryption (via GPG/password) exclusive
-        encrypt_grp = parser.add_mutually_exclusive_group()
-        encrypt_grp.add_argument("--encrypt-key",
-                                 help="Encrypt the archive using a GPG "
-                                      "key-pair")
-        encrypt_grp.add_argument("--encrypt-pass",
-                                 help="Encrypt the archive using a password")
-
     def print_header(self):
         print("\n%s\n" % _("sosreport (version %s)" % (__version__,)))
 
diff --git a/sos/utilities.py b/sos/utilities.py
index 562dde75..8c36b27a 100644
--- a/sos/utilities.py
+++ b/sos/utilities.py
@@ -201,6 +201,16 @@ def shell_out(cmd, timeout=30, chroot=None, runat=None):
                                   chroot=chroot, chdir=runat)['output']
 
 
+def get_human_readable(size, precision=2):
+    # Credit to Pavan Gupta https://stackoverflow.com/questions/5194057/
+    suffixes = ['B', 'KiB', 'MiB', 'GiB', 'TiB']
+    suffixindex = 0
+    while size > 1024 and suffixindex < 4:
+        suffixindex += 1
+        size = size/1024.0
+    return "%.*f%s" % (precision, size, suffixes[suffixindex])
+
+
 class AsyncReader(threading.Thread):
     """Used to limit command output to a given size without deadlocking
     sos.