[debian] Packaging and testing for debian pkgs

Initial start of building and testing the deb package

This will improve the overall testing of sos over the Ubuntu releases,
and also helps with the basis of thenadding further stagetwo testing in
the future. This will help to pick up issues automatically rather than
the manual testing.

Add ubuntu-latest for deb CI build and testing. Update the GCE images
for all ubuntu images for CI testing

Co-authored-by: Nikhil Kshirsagar <nikhil.kshirsagar@canonical.com>
Signed-off-by: Nikhil Kshirsagar <nikhil.kshirsagar@canonical.com>
Signed-off-by: Arif Ali <arif.ali@canonical.com>

15 files changed, 426 insertions, 9 deletions

diff --git a/.cirrus.yml b/.cirrus.yml
index acff8433..0971c84f 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -8,6 +8,7 @@ env:
 
     DEBIAN_NAME: "debian-11"
 
+    UBUNTU_LATEST_NAME: "ubuntu-23.10"
     UBUNTU_NAME: "ubuntu-22.04"
     UBUNTU_PRIOR_NAME: "ubuntu-20.04"
     UBUNTU_PRIOR2_NAME: "ubuntu-18.04"
@@ -27,10 +28,11 @@ env:
     DEBIAN_IMAGE_NAME: "debian-11-bullseye-v20230809"
     FEDORA_IMAGE_NAME: "fedora-cloud-base-gcp-38-1-6-x86-64"
     FEDORA_PRIOR_IMAGE_NAME: "fedora-cloud-base-gcp-37-1-7-x86-64"
-    UBUNTU_IMAGE_NAME: "ubuntu-2204-jammy-v20230727"
-    UBUNTU_PRIOR_IMAGE_NAME: "ubuntu-2004-focal-v20230724"
+    UBUNTU_LATEST_IMAGE_NAME: "ubuntu-2310-mantic-amd64-v20231031"
+    UBUNTU_IMAGE_NAME: "ubuntu-2204-jammy-v20231030"
+    UBUNTU_PRIOR_IMAGE_NAME: "ubuntu-2004-focal-v20231101"
     UBUNTU_PRIOR2_IMAGE_NAME: "ubuntu-1804-bionic-v20230605"
-    UBUNTU_SNAP_IMAGE_NAME: "ubuntu-2204-jammy-v20230727"
+    UBUNTU_SNAP_IMAGE_NAME: "ubuntu-2204-jammy-v20231030"
 
     # Curl-command prefix for downloading task artifacts, simply add the
     # the url-encoded task name, artifact name, and path as a suffix.
@@ -115,6 +117,27 @@ rpm_build_task:
         path: ./sos_${BUILD_NAME}.rpm
         type: application/octet-stream
 
+# Make sure a user can manually build a deb from the checkout
+deb_build_task:
+    alias: "deb_build"
+    name: "deb Build From Checkout"
+    gce_instance:
+        image_project: "${UBUNTU_PROJECT}"
+        image_name: "${UBUNTU_LATEST_IMAGE_NAME}"
+        type: e2-medium
+    setup_script: |
+        apt update --allow-releaseinfo-change
+        apt -y install devscripts equivs python3-pip
+        mk-build-deps
+        apt -y install ./sosreport-build-deps*.deb
+        pip3 install avocado-framework==94.0 --break-system-packages
+    main_script: |
+        dpkg-buildpackage -b -us -uc -rfakeroot -m --build-by="noreply@canonical.com"
+    prep_artifacts_script: mv ../*.deb ./sos_cirrus.deb
+    packages_artifacts:
+        path: ./sos_cirrus.deb
+        type: application/octet-stream
+
 # Make sure a user can manually build a snap from the checkout
 snap_build_task:
     alias: "snap_build"
@@ -146,6 +169,7 @@ report_stageone_task:
     depends_on:
         - rpm_build
         - snap_build
+        - deb_build
     gce_instance: *standardvm
     matrix:
         - env: *centos9
@@ -156,24 +180,38 @@ report_stageone_task:
             PROJECT: ${UBUNTU_PROJECT}
             BUILD_NAME: ${UBUNTU_NAME}
             VM_IMAGE_NAME: ${UBUNTU_IMAGE_NAME}
+            PKG: "snap"
         - env: &ubuntuprior
             PROJECT: ${UBUNTU_PROJECT}
             BUILD_NAME: ${UBUNTU_PRIOR_NAME}
             VM_IMAGE_NAME: ${UBUNTU_PRIOR_IMAGE_NAME}
+            PKG: "snap"
         - env: &ubuntuprior2
             PROJECT: ${UBUNTU_PROJECT}
             BUILD_NAME: ${UBUNTU_PRIOR2_NAME}
             VM_IMAGE_NAME: ${UBUNTU_PRIOR2_IMAGE_NAME}
+            PKG: "snap"
+        - env: &ubuntu-latest
+            PROJECT: ${UBUNTU_PROJECT}
+            BUILD_NAME: ${UBUNTU_LATEST_NAME}
+            VM_IMAGE_NAME: ${UBUNTU_LATEST_IMAGE_NAME}
+            PKG: "deb"
     setup_script: &setup |
         if [ $(command -v apt) ]; then
-            echo "$ARTCURL/snap%20Build%20From%20Checkout/packages/sosreport_test_amd64.snap"
-            $ARTCURL/snap%20Build%20From%20Checkout/packages/sosreport_test_amd64.snap
             apt -y purge sosreport
             apt update --allow-releaseinfo-change
             apt -y install python3-pip snapd
-            systemctl start snapd
-            snap install ./sosreport_test_amd64.snap --classic --dangerous
-            snap alias sosreport.sos sos
+            if [ ${PKG} == "snap" ] ; then
+                echo "$ARTCURL/snap%20Build%20From%20Checkout/packages/sosreport_test_amd64.snap"
+                $ARTCURL/snap%20Build%20From%20Checkout/packages/sosreport_test_amd64.snap
+                systemctl start snapd
+                snap install ./sosreport_test_amd64.snap --classic --dangerous
+                snap alias sosreport.sos sos
+            elif [ ${PKG} == "deb" ]; then
+                echo "$ARTCURL/deb%20Build%20From%20Checkout/packages/sos_cirrus.deb"
+                $ARTCURL/deb%20Build%20From%20Checkout/packages/sos_cirrus.deb
+                apt -y install ./sos_cirrus.deb
+            fi
         fi
         if [ $(command -v dnf) ]; then
             echo "$ARTCURL/rpm%20Build%20From%20Checkout%20-%20${BUILD_NAME}/packages/sos_${BUILD_NAME}.rpm"
@@ -182,7 +220,9 @@ report_stageone_task:
             dnf -y install python3-pip ethtool
             dnf -y install ./sos_${BUILD_NAME}.rpm
         fi
-        pip3 install avocado-framework==94.0
+        PIP_EXTRA=""
+        [[ $(pip3 install --help | grep break-system) ]] && PIP_EXTRA="--break-system-packages"
+        pip3 install avocado-framework==94.0 ${PIP_EXTRA}
     # run the unittests separately as they require a different PYTHONPATH in
     # order for the imports to work properly under avocado
     unittest_script: PYTHONPATH=. avocado run tests/unittests/
@@ -205,6 +245,7 @@ report_stagetwo_task:
         - env: *centos8
         - env: *fedora
         - env: *ubuntu
+        - env: *ubuntu-latest
     setup_script: *setup
     install_pexpect_script: |
         if [ $(command -v apt) ]; then
diff --git a/.gitignore b/.gitignore
index 318b19e6..cbe9c613 100644
--- a/.gitignore
+++ b/.gitignore
@@ -22,3 +22,9 @@ docs/_build
 
 # Pycharm
 .idea/
+
+# debian files
+debian/sosreport*
+debian/files
+debian/.debhelper
+debian/debhelper-build-stamp
diff --git a/debian/changelog b/debian/changelog
new file mode 100644
index 00000000..d6aadef4
--- /dev/null
+++ b/debian/changelog
@@ -0,0 +1,5 @@
+sosreport (4.6.0-0ubuntu1) mantic; urgency=medium
+
+  * New 4.6.0 upstream.
+
+ -- Nikhil Kshirsagar <nikhil.kshirsagar@canonical.com>  Thu, 17 Aug 2023 08:17:20 +0000
diff --git a/debian/control b/debian/control
new file mode 100644
index 00000000..6542e97d
--- /dev/null
+++ b/debian/control
@@ -0,0 +1,28 @@
+Source: sosreport
+Maintainer: Nikhil Kshirsagar <nikhil.kshirsagar@canonical.com>
+Section: admin
+Priority: optional
+Standards-Version: 4.5.1
+Build-Depends:
+ debhelper-compat (= 13),
+ dh-python,
+ gettext,
+ python3-all,
+ python3-coverage,
+ python3-nose,
+ python3-setuptools,
+ python3-sphinx,
+ python3-magic,
+ python3-pexpect,
+Homepage: https://github.com/sosreport/sos
+Vcs-Browser: https://salsa.debian.org/sosreport-team/sosreport
+Vcs-Git: https://salsa.debian.org/sosreport-team/sosreport.git
+
+Package: sosreport
+Architecture: any
+Depends: ${python3:Depends}, ${misc:Depends}, python3-pexpect, python3-magic
+Description: Set of tools to gather troubleshooting data from a system
+ Sos is a set of tools that gathers information about system
+ hardware and configuration. The information can then be used for
+ diagnostic purposes and debugging. Sos is commonly used to help
+ support technicians and developers.
diff --git a/debian/copyright b/debian/copyright
new file mode 100644
index 00000000..921c8937
--- /dev/null
+++ b/debian/copyright
@@ -0,0 +1,23 @@
+Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: sosreport
+Upstream-Contact: Jake Hunsaker <jacob.r.hunsaker@gmail.com>
+Source: https://github.com/sosreport/sos
+
+License: GPL-2+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or (at
+ your option) any later version.
+ .
+ This program is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ General Public License for more details.
+ .
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301,
+ USA.
+ .
+ On Debian systems, the complete text of the GNU General Public
+ License, version 2, can be found in /usr/share/common-licenses/GPL-2.
diff --git a/debian/dirs b/debian/dirs
new file mode 100644
index 00000000..78757791
--- /dev/null
+++ b/debian/dirs
@@ -0,0 +1,5 @@
+/etc/sos
+/etc/sos/cleaner
+/etc/sos/extras.d
+/etc/sos/groups.d
+/etc/sos/presets.d
diff --git a/debian/docs b/debian/docs
new file mode 100644
index 00000000..475d728a
--- /dev/null
+++ b/debian/docs
@@ -0,0 +1 @@
+README.md AUTHORS
diff --git a/debian/install b/debian/install
new file mode 100644
index 00000000..a54b1940
--- /dev/null
+++ b/debian/install
@@ -0,0 +1 @@
+sos.conf etc/sos/
diff --git a/debian/patches/0001-debian-change-tmp-dir-location.patch b/debian/patches/0001-debian-change-tmp-dir-location.patch
new file mode 100644
index 00000000..29ddefa7
--- /dev/null
+++ b/debian/patches/0001-debian-change-tmp-dir-location.patch
@@ -0,0 +1,11 @@
+Description: Change default location to /tmp
+--- a/sos.conf
++++ b/sos.conf
+@@ -7,6 +7,7 @@
+ #verify = yes
+ #batch = yes
+ #log-size = 15
++tmp-dir = /tmp
+ 
+ [report]
+ # Options that will apply to any `sos report` run should be listed here.
diff --git a/debian/patches/series b/debian/patches/series
new file mode 100644
index 00000000..3d9fdd41
--- /dev/null
+++ b/debian/patches/series
@@ -0,0 +1 @@
+0001-debian-change-tmp-dir-location.patch
diff --git a/debian/rules b/debian/rules
new file mode 100755
index 00000000..79b75aad
--- /dev/null
+++ b/debian/rules
@@ -0,0 +1,15 @@
+#!/usr/bin/make -f
+
+export PYBUILD_NAME=sosreport
+
+%:
+	dh $@ --with python3 --buildsystem=pybuild
+
+override_dh_install:
+	# Move config file to the right location.
+	mv debian/sosreport/usr/config/sos.conf debian/sosreport/etc/sos/sos.conf
+	# Remove unnecessary unused dir.
+	rm -rf debian/sosreport/usr/config
+
+override_dh_auto_test:
+	nosetests3 -v --with-cover --cover-package=sos tests/unittests
diff --git a/debian/source/format b/debian/source/format
new file mode 100644
index 00000000..163aaf8d
--- /dev/null
+++ b/debian/source/format
@@ -0,0 +1 @@
+3.0 (quilt)
diff --git a/debian/tests/control b/debian/tests/control
new file mode 100644
index 00000000..77909a08
--- /dev/null
+++ b/debian/tests/control
@@ -0,0 +1,3 @@
+Tests: simple.sh
+Depends: sosreport
+Restrictions: needs-root, allow-stderr, isolation-machine
diff --git a/debian/tests/simple.sh b/debian/tests/simple.sh
new file mode 100755
index 00000000..8fc9b39e
--- /dev/null
+++ b/debian/tests/simple.sh
@@ -0,0 +1,272 @@
+#!/bin/bash
+# This file is part of the sos project: https://github.com/sosreport/sos
+#
+# This copyrighted material is made available to anyone wishing to use,
+# modify, copy, or redistribute it subject to the terms and conditions of
+# version 2 of the GNU General Public License.
+#
+# See the LICENSE file in the source distribution for further information.
+# A quick port of the travis tests to bash, requires root
+# TODO
+# * look into using a framework..
+# * why --dry-run fails?
+# * why --experimental fails?
+# * make it better validate archives and contents
+
+PYTHON=${1:-/usr/bin/python3}
+SOSPATH=${2:-./bin/sos report --batch --tmp-dir=/tmp }
+
+NUMOFFAILURES=0
+summary="\nSummary\n"
+FAIL_LIST=""
+
+run_expecting_success () {
+    #$1 - is command options
+    #$2 - kind of check to do, so far only extract
+    FAIL=false
+    # Make sure clean
+    rm -f /dev/shm/stderr /dev/shm/stdout /tmp/sosreport*.tar.*
+    rm -rf /tmp/sosreport_test/
+    
+    start=`date +%s`
+    echo "######### RUNNING $1 #########"
+    $PYTHON $SOSPATH $1 2> /dev/shm/stderr 1> /dev/shm/stdout
+    
+    if [ $? -eq 0 ]; then
+      echo "### Success"
+    else
+      echo "!!! FAILED !!!"
+      add_failure "$1 failed during execution"
+    fi
+
+    end=`date +%s`
+    runtime=$((end-start))
+    echo "#### Sos Total time (seconds):" $runtime
+
+    if [ -s /dev/shm/stderr ]; then
+       add_failure "test generated stderr output, see above"
+       echo "### start stderr"
+       cat /dev/shm/stderr
+       echo "### end stderr"
+    fi
+    
+    echo "### start stdout"
+    cat /dev/shm/stdout
+    echo "### end stdout"
+
+    if [ "extract" = "$2" ]; then
+        echo "### start extraction"
+        rm -f /tmp/sosreport*sha256
+        mkdir /tmp/sosreport_test/
+        tar xfa /tmp/sosreport*.tar* -C /tmp/sosreport_test --strip-components=1
+        if [ -s /tmp/sosreport_test/sos_logs/*errors.txt ]; then
+            FAIL=true
+            echo "!!! FAILED !!!"
+            add_failure "Test $1 generated errors"
+            echo "#### *errors.txt output"
+            ls -alh /tmp/sosreport_test/sos_logs/
+            cat /tmp/sosreport_test/sos_logs/*errors.txt
+        fi
+        echo "### stop extraction"
+    fi
+
+    echo "######### DONE WITH $1 #########"
+
+    if $FAIL; then
+      NUMOFFAILURES=$(($NUMOFFAILURES + 1))
+      return 1
+    else
+      return 0
+    fi
+}
+
+update_summary () {
+    size="$(grep Size /dev/shm/stdout)"
+    size="$(echo "${size:-"Size 0.00MiB"}")"
+    summary="${summary} \n failures ${FAIL} \t time ${runtime} \t ${size} \t ${1} "
+}
+
+update_failures () {
+    if $FAIL; then
+      NUMOFFAILURES=$(($NUMOFFAILURES + 1))
+    fi
+}
+
+add_failure () {
+    FAIL=true
+    echo "!!! TEST FAILED: $1 !!!"
+    FAIL_LIST="${FAIL_LIST}\n \t ${FUNCNAME[1]}: \t\t ${1}"
+}
+
+# Test a no frills run with verbosity and make sure the expected items exist
+test_normal_report () {
+    cmd="-vvv"
+    # get a list of initial kmods loaded
+    kmods=( $(lsmod | cut -f1 -d ' ' | sort) )
+    run_expecting_success "$cmd" extract
+    if [ $? -eq 0 ]; then
+        if [ ! -f /tmp/sosreport_test/sos_reports/sos.html ]; then
+            add_failure "did not generate html reports"
+        fi
+        if [ ! -f /tmp/sosreport_test/sos_reports/manifest.json ]; then
+            add_failure "did not generate manifest.json"
+        fi
+        if [ ! -f /tmp/sosreport_test/free ]; then
+            add_failure "did not create free symlink in archive root"
+        fi
+        if [ ! "$(grep "DEBUG" /tmp/sosreport_test/sos_logs/sos.log)" ]; then
+            add_failure "did not find debug logging when using -vvv"
+        fi
+        # new list, see if we added any
+        new_kmods=( $(lsmod | cut -f1 -d ' ' | sort) )
+        if [ "$(printf '%s\n' "${kmods[@]}" "${new_kmods[@]}" | sort | uniq -u)" ]; then
+            add_failure "new kernel modules loaded during execution"
+            echo "$(printf '%s\n' "${kmods[@]}" "${new_kmods[@]}" | sort | uniq -u)"
+        fi
+        update_failures
+    update_summary "$cmd"
+    fi
+}
+
+# Test for correctly skipping html generation, and label setting
+test_noreport_label_only () {
+    cmd="--no-report --label TEST -o hardware"
+    run_expecting_success "$cmd" extract
+    if [ $? -eq 0 ]; then
+        if [ -f /tmp/sosreport_test/sos_reports/sos.html ]; then
+            add_failure "html report generated when --no-report used"
+        fi
+        if [ ! $(grep /tmp/sosreport-*TEST* /dev/shm/stdout) ]; then
+            add_failure "no label set on archive"
+        fi
+        count=$(find /tmp/sosreport_test/sos_commands/* -type d | wc -l)
+        if [[ "$count" -gt 1 ]]; then
+            add_failure "more than one plugin ran when using -o hardware"
+        fi
+        update_failures
+    fi
+    update_summary "$cmd"
+}
+
+# test using mask
+test_mask () {
+    cmd="--mask"
+    run_expecting_success "$cmd" extract
+    if [ $? -eq 0 ]; then
+        if [ ! $(grep host0 /tmp/sosreport_test/hostname) ]; then
+            add_failure "hostname not obfuscated with --mask"
+        fi
+        # we don't yet support binary obfuscation, so skip binary matches
+        if [ "$(grep -rI `hostname` /tmp/sosreport_test/*)" ]; then
+            add_failure "hostname not obfuscated in all places"
+            echo "$(grep -rI `hostname` /tmp/sosreport_test/*)"
+        fi
+        # only tests first interface
+        mac_addr=$(cat /sys/class/net/$(ip route show default | awk '/default/ {print $5}')/address)
+        if [ "$(grep -rI $mac_addr /tmp/sosreport_test/*)" ]; then
+            add_failure "MAC address not obfuscated in all places"
+            echo "$(grep -rI $mac_addr /tmp/sosreport_test/*)"
+        fi
+        # only tests first interface
+        ip_addr=$(ip route show default | awk '/default/ {print $3}')
+        if [ "$(grep -rI $ip_addr /tmp/sosreport_test/*)" ]; then
+            add_failure "IP address not obfuscated in all places"
+            echo "$(grep -rI $ip_addr /tmp/sosreport_test/*)"
+        fi
+        update_failures
+    fi
+    update_summary "$cmd"
+}
+
+# test log-size, env vars, and compression type
+test_logsize_env_gzip () {
+    cmd="--log-size 0 --no-env-vars -z gzip"
+    run_expecting_success "$cmd" extract
+    if [ $? -eq 0 ]; then
+        if [ -f /tmp/sosreport_test/environment ]; then
+            add_failure "env vars captured when using --no-env-vars"
+        fi
+        if [ ! $(grep /tmp/sosreport*.gz /dev/shm/stdout) ]; then
+            add_failure "archive was not gzip compressed using -z gzip"
+        fi
+        update_failures
+    fi
+    update_summary "$cmd"
+}
+
+# test plugin enablement, plugopts and at the same time ensure our list option parsing is working
+test_enable_opts_postproc () {
+    cmd="-e opencl -v -k kernel.with-timer,libraries.ldconfigv --no-postproc"
+    run_expecting_success "$cmd" extract
+    if [ $? -eq 0 ]; then
+        if [ ! "$(grep "opencl" /dev/shm/stdout)" ]; then
+            add_failure "force enabled plugin opencl did not run"
+        fi
+        if [ ! -f /tmp/sosreport_test/proc/timer* ]; then
+            add_failure "/proc/timer* not captured when using -k kernel.with-timer"
+        fi
+        if [ ! -f /tmp/sosreport_test/sos_commands/libraries/ldconfig_-v* ]; then
+            add_failure "ldconfig -v not captured when using -k libraries.ldconfigv"
+        fi
+        if [ "$(grep "substituting" /tmp/sosreport_test/sos_logs/sos.log)" ]; then
+            add_failure "post-processing ran while using --no-post-proc"
+        fi
+
+        update_failures
+    update_summary "$cmd"
+    fi
+}
+
+# test if --build and --threads work properly
+test_build_threads () {
+    cmd="--build -t1 -o host,kernel,filesys,hardware,date,logs"
+    run_expecting_success "$cmd"
+    if [ $? -eq 0 ]; then
+        if [ ! "$(grep "Your sosreport build tree" /dev/shm/stdout)" ]; then
+            add_failure "did not save the build tree"
+        fi
+        if [ $(grep "Finishing plugins" /dev/shm/stdout) ]; then
+            add_failure "did not limit threads when using --threads 1"
+        fi
+        update_failures
+        update_summary "$cmd"
+    fi
+}
+
+# If /etc/sos/sos.conf doesn't exist let's just make it
+if [ -f /etc/sos/sos.conf ]; then
+   echo "/etc/sos/sos.conf already exists"
+else
+   echo "Creating /etc/sos.conf"
+   mkdir /etc/sos
+   touch /etc/sos/sos.conf
+fi
+
+
+# Runs not generating sosreports
+run_expecting_success " -l"; update_summary "List plugins"
+run_expecting_success " --list-presets"; update_summary "List presets"
+run_expecting_success " --list-profiles"; update_summary "List profiles"
+
+# Runs generating sosreports
+# TODO:
+# - find a way to test if --since is working
+test_build_threads
+test_normal_report
+test_enable_opts_postproc
+test_noreport_label_only
+test_logsize_env_gzip
+test_mask
+
+echo -e $summary
+
+if [ $NUMOFFAILURES -gt 0 ]; then
+  echo -e "\nTests Failed: $NUMOFFAILURES\nFailures within each test:"
+  echo -e $FAIL_LIST
+  exit 1
+else
+  echo "Everything worked!"
+  exit 0
+fi
+
+# vim: set et ts=4 sw=4 :
diff --git a/debian/watch b/debian/watch
new file mode 100644
index 00000000..fa4ded0d
--- /dev/null
+++ b/debian/watch
@@ -0,0 +1,4 @@
+version=4
+  opts="filenamemangle=s%(?:.*?)?v?(\d[\d.]*)\.tar\.gz%sos-$1.tar.gz%" \
+      https://github.com/sosreport/sos/tags \
+      (?:.*?/)?v?(\d[\d.]*)\.tar\.gz debian uupdate