Merge pull request #131 from A-Hilaly/github-import

github: support for partial import and refactor into iterator/importer

1 files changed, 31 insertions, 0 deletions

diff --git a/util/text/transform.go b/util/text/transform.go
new file mode 100644
index 00000000..59dc4e03
--- /dev/null
+++ b/util/text/transform.go
@@ -0,0 +1,31 @@
+package text
+
+import (
+	"strings"
+	"unicode"
+
+	"golang.org/x/text/runes"
+	"golang.org/x/text/transform"
+)
+
+func Cleanup(text string) (string, error) {
+	// windows new line, Github, really ?
+	text = strings.Replace(text, "\r\n", "\n", -1)
+
+	// remove all unicode control characters except
+	// '\n', '\r' and '\t'
+	t := runes.Remove(runes.Predicate(func(r rune) bool {
+		switch r {
+		case '\r', '\n', '\t':
+			return false
+		}
+		return unicode.IsControl(r)
+	}))
+	sanitized, _, err := transform.String(t, text)
+	if err != nil {
+		return "", err
+	}
+
+	// trim extra new line not displayed in the github UI but still present in the data
+	return strings.TrimSpace(sanitized), nil
+}