3 files changed, 59 insertions, 0 deletions
diff --git a/tests/cleaner_tests/unicode_open/sos-test-unicode.txt b/tests/cleaner_tests/unicode_open/sos-test-unicode.txt
new file mode 100644
index 00000000..7dc00099
--- /dev/null
+++ b/tests/cleaner_tests/unicode_open/sos-test-unicode.txt
@@ -0,0 +1,5 @@
+This is a line with no unicode in it.
+This line has some in it æßøĄ.
+If opened normally, the last line may cause errors.
+So this file is used to test opening with errors='replace'.
+This line has the address 192.168.1.1 in it to ensure our cleaner tests are actually processing this file.
diff --git a/tests/cleaner_tests/unicode_open/unicode_in_file.py b/tests/cleaner_tests/unicode_open/unicode_in_file.py
new file mode 100644
index 00000000..01a980f5
--- /dev/null
+++ b/tests/cleaner_tests/unicode_open/unicode_in_file.py
@@ -0,0 +1,34 @@
+# This file is part of the sos project: https://github.com/sosreport/sos
+#
+# This copyrighted material is made available to anyone wishing to use,
+# modify, copy, or redistribute it subject to the terms and conditions of
+# version 2 of the GNU General Public License.
+#
+# See the LICENSE file in the source distribution for further information.
+
+from sos_tests import StageTwoReportTest
+
+
+class UnicodeOpenTest(StageTwoReportTest):
+    """This test ensures that we can safely open files that have embedded
+    unicode in them, and that those files do not trigger an exception that
+    leaves them uncleaned.
+
+    :avocado: tags=stagetwo
+    """
+
+    sos_cmd = '--clean -o unicode_test,networking,host'
+    files = [('sos-test-unicode.txt', '/tmp/sos-test-unicode.txt')]
+    install_plugins = ['unicode_test']
+
+    def test_file_was_collected(self):
+        self.assertFileCollected('/tmp/sos-test-unicode.txt')
+
+    def test_file_was_opened(self):
+        # if this fails, then we hit an exception when opening the file
+        self.assertSosLogContains('Obfuscating tmp/sos-test-unicode.txt')
+        self.assertSosLogNotContains('.*Unable to parse.*')
+
+    def test_obfuscation_complete(self):
+        # make sure that we didn't stop processing the file after the unicode
+        self.assertFileNotHasContent('tmp/sos-test-unicode.txt', '192.168.1.1')
diff --git a/tests/cleaner_tests/unicode_open/unicode_test.py b/tests/cleaner_tests/unicode_open/unicode_test.py
new file mode 100644
index 00000000..541dfb5c
--- /dev/null
+++ b/tests/cleaner_tests/unicode_open/unicode_test.py
@@ -0,0 +1,20 @@
+# This file is part of the sos project: https://github.com/sosreport/sos
+#
+# This copyrighted material is made available to anyone wishing to use,
+# modify, copy, or redistribute it subject to the terms and conditions of
+# version 2 of the GNU General Public License.
+#
+# See the LICENSE file in the source distribution for further information.
+
+from sos.report.plugins import Plugin, IndependentPlugin
+
+
+class UnicodeTest(Plugin, IndependentPlugin):
+    """Fake plugin to test the handling of a file with embedded unicode
+    """
+
+    plugin_name = 'unicode_test'
+    short_desc = 'Fake plugin to test unicode file handling'
+
+    def setup(self):
+        self.add_copy_spec('/tmp/sos-test-unicode.txt')