[travis] Move to single test script and use travis job matrix

Move all the sosreport runs to a simple bash script:
tests/simple.sh so it can be called multiple times.
This also makes it possible to run locally.

The travis job matrix let's us have 2 Ubuntu native sos
runs against installed python, while letting us to also
continue to do the old style python runs.  Ideally,
we can utilize this to do Fedora/CentOS/RHEL jobs in the
future.

Python 3.4 testing was also dropped - as it seems to not
be as well used (many test failures due to not being available).
Python 3.8 testing was added.
pycodestyle will only run once with the bionic version.

Resolves: #1896

Signed-off-by: Bryan Quigley <bryan.quigley@canonical.com>
Signed-off-by: Jake Hunsaker <jhunsake@redhat.com>

2 files changed, 172 insertions, 28 deletions

diff --git a/.travis.yml b/.travis.yml
index 86f5168b..6d7912c8 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,14 +1,64 @@
-language: python
-python:
-  - "2.7"
-  - "3.4"
-  - "3.5"
-  - "3.6"
-  - "3.7"
-  - "nightly"
-matrix:
-  allow_failures:
-    - python: "nightly"
+jobs:
+  include:
+    - name: "18.04 pycodestyle and native run (py3.6)"
+      os: linux
+      dist: bionic
+      language: shell
+      install: sudo apt-get update; sudo apt install pycodestyle;
+      script:
+        - "pycodestyle sos"
+        - "sudo ./tests/simple.sh"
+    - name: "16.04 native run (py3.5)"
+      os: linux
+      dist: xenial
+      language: shell
+      script: "sudo ./tests/simple.sh"
+    - name: "nosetests and travis Python 2.7"
+      os: linux
+      dist: bionic
+      language: python
+      python: "2.7"
+      install: pip install -r requirements.txt; python setup.py install;
+      script:
+        - "nosetests -v --with-cover --cover-package=sos --cover-html"
+        - "sudo ./tests/simple.sh ~/virtualenv/python$TRAVIS_PYTHON_VERSION/bin/python"
+    - name: "nosetests and travis Python 3.5"
+      os: linux
+      dist: bionic
+      language: python
+      python: "3.5"
+      install: pip install -r requirements.txt; python setup.py install;
+      script:
+        - "nosetests -v --with-cover --cover-package=sos --cover-html"
+        - "sudo ./tests/simple.sh ~/virtualenv/python$TRAVIS_PYTHON_VERSION/bin/python"
+    - name: "nosetests and travis Python 3.6"
+      os: linux
+      dist: bionic
+      language: python
+      python: "3.6"
+      install: pip install -r requirements.txt; python setup.py install;
+      script:
+        - "nosetests -v --with-cover --cover-package=sos --cover-html"
+        - "sudo ./tests/simple.sh ~/virtualenv/python$TRAVIS_PYTHON_VERSION/bin/python"
+    - name: "nosetests and travis Python 3.7"
+      os: linux
+      dist: bionic
+      language: python
+      python: "3.7"
+      install: pip install -r requirements.txt; python setup.py install;
+      script:
+        - "nosetests -v --with-cover --cover-package=sos --cover-html"
+        - "sudo ./tests/simple.sh ~/virtualenv/python$TRAVIS_PYTHON_VERSION/bin/python"
+    - name: "nosetests and travis Python 3.8"
+      os: linux
+      dist: bionic
+      language: python
+      python: "3.8"
+      install: pip install -r requirements.txt; python setup.py install;
+      script:
+        - "nosetests -v --with-cover --cover-package=sos --cover-html"
+        - "sudo ./tests/simple.sh ~/virtualenv/python$TRAVIS_PYTHON_VERSION/bin/python"
+
 notifications:
   email:
     sos-devel@redhat.com
@@ -16,22 +66,5 @@ notifications:
     channels:
       - "us.freenode.net#sosreport"
     on_success: change
-dist: xenial
-install:
-  - "pip install -r requirements.txt"
-  - "python setup.py install"
-script:
-  - "pycodestyle sos"
-  - "pycodestyle --version"
-  - "nosetests -v --with-cover --cover-package=sos --cover-html"
-  - "sudo ~/virtualenv/python$TRAVIS_PYTHON_VERSION/bin/python ./sosreport --help"
-  - "sudo ~/virtualenv/python$TRAVIS_PYTHON_VERSION/bin/python ./sosreport -l --config-file=sos.conf"
-  - "sudo ~/virtualenv/python$TRAVIS_PYTHON_VERSION/bin/python ./sosreport --batch --config-file=sos.conf 2> errors | tee batch_output"
-  - "sudo ~/virtualenv/python$TRAVIS_PYTHON_VERSION/bin/python ./sosreport --batch --all-logs --config-file=sos.conf 2> errors | tee batch_output"
-  - "sudo ~/virtualenv/python$TRAVIS_PYTHON_VERSION/bin/python ./sosreport --batch --since=20191007 --config-file=sos.conf 2> errors | tee batch_output"
-  - "sudo ~/virtualenv/python$TRAVIS_PYTHON_VERSION/bin/python ./sosreport --batch --all-logs --since=20191007 --config-file=sos.conf 2> errors | tee batch_output"
-  - "[[ ! -s errors ]]"
-  - "cat errors"
-  - "sudo ~/virtualenv/python$TRAVIS_PYTHON_VERSION/bin/python ./sosreport --batch --build --config-file=sos.conf"
 git:
   depth: 5
diff --git a/tests/simple.sh b/tests/simple.sh
new file mode 100755
index 00000000..6f4fc466
--- /dev/null
+++ b/tests/simple.sh
@@ -0,0 +1,111 @@
+#/bin/bash
+# A quick port of the travis tests to bash, requires root
+# TODO
+# * look into using a framework..
+# * why --dry-run fails?
+# * why --experimental fails?
+# * https://github.com/sosreport/sos/issues/1921
+# * make it better validate archives and contents
+
+PYTHON=${1:-/usr/bin/python3}
+SOSPATH=${2:-./sosreport}
+
+NUMOFFAILURES=0
+
+run_expecting_sucess () {
+    #$1 - is command options
+    #$2 - kind of check to do, so far only extract
+    FAIL=false
+    # Make sure clean
+    rm -f stderr stdout /tmp/sosreport*.tar.*
+    rm -rf /tmp/sosreport_test/
+
+    echo "######### RUNNING $1 #########"
+    $PYTHON $SOSPATH $1 2> stderr 1> stdout
+
+    if [ $? -eq 0 ]; then
+      echo "### Success"
+    else
+      echo "!!! FAILED !!!"
+      FAIL=true
+    fi
+
+    if [ -s stderr ]; then
+       FAIL=true
+       echo "!!! FAILED !!!"
+       echo "### start stderr"
+       cat stderr
+       echo "### end stderr"
+    fi
+
+    echo "### start stdout"
+    cat stdout
+    echo "### end stdout"
+
+    if [ "extract" = "$2" ]; then
+        echo "### start extraction"
+        rm -f /tmp/sosreport*md5
+        mkdir /tmp/sosreport_test/
+        tar xfa /tmp/sosreport*.tar* -C /tmp/sosreport_test --strip-components=1
+        if [ -s /tmp/sosreport_test/sos_logs/*errors.txt ]; then
+            FAIL=true
+            echo "!!! FAILED !!!"
+            echo "#### *errors.txt output"
+            ls -alh /tmp/sosreport_test/sos_logs/
+            cat /tmp/sosreport_test/sos_logs/*errors.txt
+        fi
+        echo "### stop extraction"
+    fi
+
+    echo "######### DONE WITH $1 #########"
+
+    if $FAIL; then
+      NUMOFFAILURES=$(($NUMOFFAILURES + 1))
+      return 1
+    else
+      return 0
+    fi
+}
+
+# If /etc/sos.conf doesn't exist let's just make it..
+if [ -f /etc/sos.conf ]; then
+   echo "/etc/sos.conf already exists"
+else
+   echo "Creating /etc/sos.conf"
+   touch /etc/sos.conf
+fi
+
+# Runs not generating sosreports
+run_expecting_sucess " -l"
+run_expecting_sucess " --list-presets"
+run_expecting_sucess " --list-profiles"
+
+# Test generating sosreports, 3 (new) options at a time
+# Trying to do --batch   (1 label/archive/report/verbosity change)   (other changes)
+run_expecting_sucess " --batch   --build   --no-env-vars "  # Only --build test
+run_expecting_sucess " --batch   --no-report   -o hardware " extract
+run_expecting_sucess " --batch   --label TEST   -a  -c never" extract
+run_expecting_sucess " --batch   --debug  --log-size 0  -c always" extract
+
+# Workaround Travis bug (requires -n lxd)
+if [ $TRAVIS = true ]; then
+    run_expecting_sucess " --batch   -z xz   --log-size 1 -n lxd" extract
+else
+    run_expecting_sucess " --batch   -z xz   --log-size 1" extract
+fi
+
+run_expecting_sucess " --batch   -z gzip" extract
+run_expecting_sucess " --batch   -z bzip2   -t 1 -n hardware" extract
+run_expecting_sucess " --batch   --quiet    -e opencl -k kernel.with-timer" extract
+run_expecting_sucess " --batch   --case-id 10101   --all-logs --since=20191007" extract
+run_expecting_sucess " --batch   --verbose   --no-postproc" extract
+
+if [ $NUMOFFAILURES -gt 0 ]; then
+  echo "FAILED $NUMOFFAILURES"
+  exit 1
+else
+  echo "Everything worked!"
+  exit 0
+fi
+
+# vim: set et ts=4 sw=4 :