diff options
author | Jake Hunsaker <jhunsake@redhat.com> | 2019-12-23 15:20:20 -0500 |
---|---|---|
committer | Bryn M. Reeves <bmr@redhat.com> | 2020-02-14 19:27:10 +0000 |
commit | 609beba4eadf480c4c0d7053bcc0659abbe974ea (patch) | |
tree | 7dc1f39aa6917e8ff35ccf7dc0dffc705f51d4e9 | |
parent | 58031c8779750e6162628bdb9730d4f23c283ce3 (diff) | |
download | sos-609beba4eadf480c4c0d7053bcc0659abbe974ea.tar.gz |
[sosreport|Policy] Add ability to upload archive
Adds the option to automatically upload a generated sosreport archive to
a provided location such as HTTPS or FTP servers.
Using --upload will use policy-defined defaults (if available) to upload
to a specific vendor's defined location.
Otherwise, users may supply --upload-url, --upload-directory,
--upload-user, and --upload-pass to specify a custom location/server,
directory on that server, and authentication credentials.
Resolves: #1911
Signed-off-by: Jake Hunsaker <jhunsake@redhat.com>
-rw-r--r-- | sos/__init__.py | 6 | ||||
-rw-r--r-- | sos/policies/__init__.py | 278 | ||||
-rw-r--r-- | sos/sosreport.py | 23 |
3 files changed, 306 insertions, 1 deletions
diff --git a/sos/__init__.py b/sos/__init__.py index c9f20e59..ddee2ca6 100644 --- a/sos/__init__.py +++ b/sos/__init__.py @@ -57,6 +57,7 @@ _arg_names = [ 'list_profiles', 'log_size', 'noplugins', 'noreport', 'no_env_vars', 'no_postproc', 'note', 'onlyplugins', 'plugin_timeout', 'plugopts', 'preset', 'profiles', 'quiet', 'since', 'sysroot', 'threads', 'tmp_dir', + 'upload', 'upload_url', 'upload_directory', 'upload_user', 'upload_pass', 'verbosity', 'verify' ] @@ -194,6 +195,11 @@ class SoSOptions(object): self.sysroot = None self.threads = 4 self.tmp_dir = "" + self.upload = False + self.upload_url = "" + self.upload_directory = "" + self.upload_user = "" + self.upload_pass = "" self.verbosity = _arg_defaults["verbosity"] self.verify = False self._nondefault = set() diff --git a/sos/policies/__init__.py b/sos/policies/__init__.py index e91c79f2..e6cf5845 100644 --- a/sos/policies/__init__.py +++ b/sos/policies/__init__.py @@ -24,6 +24,12 @@ from six.moves import input PRESETS_PATH = "/var/lib/sos/presets" +try: + import requests + REQUESTS_LOADED = True +except ImportError: + REQUESTS_LOADED = False + def get_human_readable(size, precision=2): # Credit to Pavan Gupta https://stackoverflow.com/questions/5194057/ @@ -843,8 +849,21 @@ class LinuxPolicy(Policy): vendor = "None" PATH = "/bin:/sbin:/usr/bin:/usr/sbin" init = None - + # _ prefixed class attrs are used for storing any vendor-defined defaults + # the non-prefixed attrs are used by the upload methods, and will be set + # to the cmdline/config file values, if provided. If not provided, then + # those attrs will be set to the _ prefixed values as a fallback. + # TL;DR Use _upload_* for policy default values, use upload_* when wanting + # to actual use the value in a method/override + _upload_url = None + _upload_directory = '/' + _upload_user = None + _upload_password = None + _use_https_streaming = False _preferred_hash_name = None + upload_url = None + upload_user = None + upload_password = None def __init__(self, sysroot=None, init=None): super(LinuxPolicy, self).__init__(sysroot=sysroot) @@ -917,12 +936,25 @@ class LinuxPolicy(Policy): cmdline_opts = self.commons['cmdlineopts'] caseid = cmdline_opts.case_id if cmdline_opts.case_id else "" + # Set the cmdline settings to the class attrs that are referenced later + # The policy default '_' prefixed versions of these are untouched to + # allow fallback + self.upload_url = cmdline_opts.upload_url + self.upload_user = cmdline_opts.upload_user + self.upload_directory = cmdline_opts.upload_directory + self.upload_password = cmdline_opts.upload_pass + if not cmdline_opts.batch and not \ cmdline_opts.quiet: try: self.case_id = input(_("Please enter the case id " "that you are generating this " "report for [%s]: ") % caseid) + # Policies will need to handle the prompts for user information + if cmdline_opts.upload or self.upload_url: + self.prompt_for_upload_user() + self.prompt_for_upload_password() + self._print() except KeyboardInterrupt: self._print() @@ -933,5 +965,249 @@ class LinuxPolicy(Policy): return + def prompt_for_upload_user(self): + """Should be overridden by policies to determine if a user needs to + be provided or not + """ + if not self.upload_user and not self._upload_user: + msg = "Please provide upload user for %s: " % self.get_upload_url() + self.upload_user = input(_(msg)) + + def prompt_for_upload_password(self): + """Should be overridden by policies to determine if a password needs to + be provided for upload or not + """ + if ((not self.upload_password and not self._upload_password) and + self.upload_user): + msg = ( + "Please provide the upload password for %s: " + % self.upload_user + ) + self.upload_password = getpass(msg) + + def upload_archive(self, archive): + """Entry point for sos attempts to upload the generated archive to a + policy or user specified location. + + Curerntly there is support for HTTPS, SFTP, and FTP. HTTPS uploads are + preferred for policy-defined defaults. + + Policies that need to override uploading methods should override the + respective upload_https(), upload_sftp(), and/or upload_ftp() methods + and should NOT override this method. + + In order to enable this for a policy, that policy needs to implement + the following: + + Required: + Class Attrs: + _upload_url The default location to use. Note + these MUST include protocol header + _upload_user Default username, if any else None + _upload_password Default password, if any else None + _use_https_streaming Set to True if the HTTPS endpoint + supports streaming data + + Optional: + Class Attrs: + _upload_directory Default FTP server directory, if any + + Methods: + prompt_for_upload_user() Determines if sos should prompt + for a username or not. + get_upload_user() Determines if the default or a + different username should be used + get_upload_https_auth() Format authentication data for + HTTPS uploads + get_upload_url_string() If you want your policy to print + a string other than the default URL + for your vendor/distro, override + this method + + """ + self.upload_archive = archive + self.upload_url = self.get_upload_url() + if not self.upload_url: + raise Exception("No upload destination provided by policy or by " + "--upload-url") + upload_func = self._determine_upload_type() + print(_("Attempting upload to %s" % self.get_upload_url_string())) + return upload_func() + + def _determine_upload_type(self): + """Based on the url provided, determine what type of upload to attempt. + + Note that this requires users to provide a FQDN address, such as + https://myvendor.com/api or ftp://myvendor.com instead of + myvendor.com/api or myvendor.com + """ + prots = { + 'ftp': self.upload_ftp, + 'sftp': self.upload_sftp, + 'https': self.upload_https + } + if '://' not in self.upload_url: + raise Exception("Must provide protocol in upload URL") + prot, url = self.upload_url.split('://') + if prot not in prots.keys(): + raise Exception("Unsupported or unrecognized protocol: %s" % prot) + return prots[prot] + + def get_upload_https_auth(self, user=None, password=None): + """Formats the user/password credentials using basic auth + """ + if not user: + user = self.get_upload_user() + if not password: + password = self.get_upload_password() + + return requests.auth.HTTPBasicAuth(user, password) + + def get_upload_url(self): + """Helper function to determine if we should use the policy default + upload url or one provided by the user + """ + return self.upload_url or self._upload_url + + def get_upload_url_string(self): + """Used by distro policies to potentially change the string used to + report upload location from the URL to a more human-friendly string + """ + return self.get_upload_url() + + def get_upload_user(self): + """Helper function to determine if we should use the policy default + upload user or one provided by the user + """ + return self.upload_user or self._upload_user + + def get_upload_password(self): + """Helper function to determine if we should use the policy default + upload password or one provided by the user + """ + return self.upload_password or self._upload_password + + def upload_sftp(self): + """Attempts to upload the archive to an SFTP location. + + Due to the lack of well maintained, secure, and generally widespread + python libraries for SFTP, sos will shell-out to the system's local ssh + installation in order to handle these uploads. + + Do not override this method with one that uses python-paramiko, as the + upstream sos team will reject any PR that includes that dependency. + """ + raise NotImplementedError("SFTP support is not yet implemented") + + def _upload_https_streaming(self, archive): + """If upload_https() needs to use requests.put(), this method is used + to provide streaming functionality + + Policies should override this method instead of the base upload_https() + + Positional arguments: + :param archive: The open archive file object + """ + return requests.put(self.get_upload_url(), data=archive, + auth=self.get_upload_https_auth()) + + def _get_upload_headers(self): + """Define any needed headers to be passed with the POST request here + """ + return {} + + def _upload_https_no_stream(self, archive): + """If upload_https() needs to use requests.post(), this method is used + to provide non-streaming functionality + + Policies should override this method instead of the base upload_https() + + Positional arguments: + :param archive: The open archive file object + """ + files = { + 'file': (archive.name.split('/')[-1], archive, + self._get_upload_headers()) + } + return requests.post(self.get_upload_url(), files=files, + auth=self.get_upload_https_auth()) + + def upload_https(self): + """Attempts to upload the archive to an HTTPS location. + + Policies may define whether this upload attempt should use streaming + or non-streaming data by setting the `use_https_streaming` class + attr to True + """ + if not REQUESTS_LOADED: + raise Exception("Unable to upload due to missing python requests " + "library") + + with open(self.upload_archive, 'rb') as arc: + if not self._use_https_streaming: + r = self._upload_https_no_stream(arc) + else: + r = self._upload_https_streaming(arc) + if r.status_code != 201: + if r.status_code == 401: + raise Exception( + "Authentication failed: invalid user credentials" + ) + raise Exception("POST request returned %s: %s" + % (r.status_code, r.reason)) + return True + + def upload_ftp(self, url=None, directory=None, user=None, password=None): + """Attempts to upload the archive to either the policy defined or user + provided FTP location. + """ + try: + import ftplib + import socket + except ImportError: + # socket is part of the standard library, should only fail here on + # ftplib + raise Exception("missing python ftplib library") + + if not url: + url = self.get_upload_url() + if url is None: + raise Exception("no FTP server specified by policy, use --upload-" + "url to specify a location") + + url = url.replace('ftp://', '') + + if not user: + user = self.get_upload_user() + + if not password: + password = self.get_upload_password() + + if not directory: + directory = self._upload_directory + + try: + session = ftplib.FTP(url, user, password) + session.cwd(directory) + except socket.gaierror: + raise Exception("unable to connect to %s" % url) + except ftplib.error_perm as err: + errno = err.split()[0] + if errno == 503: + raise Exception("could not login as '%s'" % user) + if errno == 550: + raise Exception("could not set upload directory to %s" + % directory) + + try: + with open(self.upload_archive, 'rb') as _arcfile: + session.storbinary( + "STOR %s" % self.upload_archive.split('/')[-1], + _arcfile + ) + session.quit() + return True + except IOError: + raise Exception("could not open archive file") # vim: set et ts=4 sw=4 : diff --git a/sos/sosreport.py b/sos/sosreport.py index 71e7725c..46ed4a2f 100644 --- a/sos/sosreport.py +++ b/sos/sosreport.py @@ -245,6 +245,17 @@ def _get_parser(): help="Run commands even if they can change the " "system (e.g. load kernel modules)") + parser.add_argument("--upload", action="store_true", default=False, + help="Upload the archive to a policy-default location") + parser.add_argument("--upload-url", default=None, + help="Upload the archive to the specified server") + parser.add_argument("--upload-directory", default=None, + help="Specify the directory to upload the archive to") + parser.add_argument("--upload-user", default=None, + help="Username to authenticate to upload server with") + parser.add_argument("--upload-pass", default=None, + help="Password to authenticate to upload server with") + # Group to make add/del preset exclusive preset_grp = parser.add_mutually_exclusive_group() preset_grp.add_argument("--add-preset", type=str, action="store", @@ -1312,6 +1323,18 @@ class SoSReport(object): else: self.policy.display_results(archive, directory, checksum) + if self.opts.upload or self.opts.upload_url: + if not self.opts.build: + try: + self.policy.upload_archive(archive) + self.ui_log.info(_("Uploaded archive successfully")) + except Exception as err: + self.ui_log.error("Upload attempt failed: %s" % err) + else: + msg = ("Unable to upload archive when using --build as no " + "archive is created.") + self.ui_log.error(msg) + # clean up logging.shutdown() if self.tempfile_util: |