From 2e17f371758ad25a3674d65ef0e8e32a4660e6d4 Mon Sep 17 00:00:00 2001 From: Amine Hilaly Date: Sun, 5 May 2019 17:48:49 +0200 Subject: Add unicode control characters test case Move `cleanupText` to utils/text/transform.go `text.Cleanup`: removing unicode control characters except for those allowed by `text.Safe` Add golang.org/x/text dependencies fix text.Cleanup Fix import panic --- bridge/github/import.go | 46 +++++++++++++++++++++++++++++--------------- bridge/github/import_test.go | 44 ++++++++++++++++++++++++++---------------- 2 files changed, 58 insertions(+), 32 deletions(-) (limited to 'bridge/github') diff --git a/bridge/github/import.go b/bridge/github/import.go index 9b9b790e..0c5468d8 100644 --- a/bridge/github/import.go +++ b/bridge/github/import.go @@ -3,7 +3,6 @@ package github import ( "context" "fmt" - "strings" "time" "github.com/MichaelMure/git-bug/bridge/core" @@ -11,6 +10,7 @@ import ( "github.com/MichaelMure/git-bug/cache" "github.com/MichaelMure/git-bug/identity" "github.com/MichaelMure/git-bug/util/git" + "github.com/MichaelMure/git-bug/util/text" "github.com/shurcooL/githubv4" ) @@ -112,12 +112,17 @@ func (gi *githubImporter) ensureIssue(repo *cache.RepoCache, issue issueTimeline // if issueEdits is empty if len(issueEdits) == 0 { if err == bug.ErrBugNotExist { + cleanText, err := text.Cleanup(string(issue.Body)) + if err != nil { + return nil, err + } + // create bug b, err = repo.NewBugRaw( author, issue.CreatedAt.Unix(), issue.Title, - cleanupText(string(issue.Body)), + cleanText, nil, map[string]string{ keyGithubId: parseId(issue.Id), @@ -136,6 +141,11 @@ func (gi *githubImporter) ensureIssue(repo *cache.RepoCache, issue issueTimeline continue } + cleanText, err := text.Cleanup(string(*edit.Diff)) + if err != nil { + return nil, err + } + // if the bug doesn't exist if b == nil { // we create the bug as soon as we have a legit first edition @@ -143,7 +153,7 @@ func (gi *githubImporter) ensureIssue(repo *cache.RepoCache, issue issueTimeline author, issue.CreatedAt.Unix(), issue.Title, - cleanupText(string(*edit.Diff)), + cleanText, nil, map[string]string{ keyGithubId: parseId(issue.Id), @@ -301,12 +311,16 @@ func (gi *githubImporter) ensureTimelineComment(repo *cache.RepoCache, b *cache. // if comment doesn't exist if err == cache.ErrNoMatchingOp { + cleanText, err := text.Cleanup(string(item.Body)) + if err != nil { + return err + } // add comment operation op, err := b.AddCommentRaw( author, item.CreatedAt.Unix(), - cleanupText(string(item.Body)), + cleanText, nil, map[string]string{ keyGithubId: parseId(item.Id), @@ -338,10 +352,15 @@ func (gi *githubImporter) ensureTimelineComment(repo *cache.RepoCache, b *cache. // create comment when target is empty if target == "" { + cleanText, err := text.Cleanup(string(*edit.Diff)) + if err != nil { + return err + } + op, err := b.AddCommentRaw( editor, edit.CreatedAt.Unix(), - cleanupText(string(*edit.Diff)), + cleanText, nil, map[string]string{ keyGithubId: parseId(item.Id), @@ -395,12 +414,17 @@ func (gi *githubImporter) ensureCommentEdit(repo *cache.RepoCache, b *cache.BugC case edit.DeletedAt == nil: + cleanText, err := text.Cleanup(string(*edit.Diff)) + if err != nil { + return err + } + // comment edition - _, err := b.EditCommentRaw( + _, err = b.EditCommentRaw( editor, edit.CreatedAt.Unix(), target, - cleanupText(string(*edit.Diff)), + cleanText, map[string]string{ keyGithubId: parseId(edit.Id), }, @@ -505,14 +529,6 @@ func parseId(id githubv4.ID) string { return fmt.Sprintf("%v", id) } -func cleanupText(text string) string { - // windows new line, Github, really ? - text = strings.Replace(text, "\r\n", "\n", -1) - - // trim extra new line not displayed in the github UI but still present in the data - return strings.TrimSpace(text) -} - func reverseEdits(edits []userContentEdit) []userContentEdit { for i, j := 0, len(edits)-1; i < j; i, j = i+1, j-1 { edits[i], edits[j] = edits[j], edits[i] diff --git a/bridge/github/import_test.go b/bridge/github/import_test.go index 967f50ee..48283b7a 100644 --- a/bridge/github/import_test.go +++ b/bridge/github/import_test.go @@ -31,7 +31,8 @@ func Test_Importer(t *testing.T) { Operations: []bug.Operation{ bug.NewCreateOp(author, 0, "simple issue", "initial comment", nil), bug.NewAddCommentOp(author, 0, "first comment", nil), - bug.NewAddCommentOp(author, 0, "second comment", nil)}, + bug.NewAddCommentOp(author, 0, "second comment", nil), + }, }, }, { @@ -112,6 +113,15 @@ func Test_Importer(t *testing.T) { }, }, }, + { + name: "unicode control characters", + url: "https://github.com/MichaelMure/git-bug-test-github-bridge/issues/10", + bug: &bug.Snapshot{ + Operations: []bug.Operation{ + bug.NewCreateOp(author, 0, "unicode control characters", "u0000: \nu0001: \nu0002: \nu0003: \nu0004: \nu0005: \nu0006: \nu0007: \nu0008: \nu0009: \t\nu0010: \nu0011: \nu0012: \nu0013: \nu0014: \nu0015: \nu0016: \nu0017: \nu0018: \nu0019:", nil), + }, + }, + }, } repo := test.CreateRepo(false) @@ -142,7 +152,7 @@ func Test_Importer(t *testing.T) { fmt.Printf("test repository imported in %f seconds\n", time.Since(start).Seconds()) - require.Len(t, backend.AllBugsIds(), 8) + require.Len(t, backend.AllBugsIds(), 9) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -157,26 +167,26 @@ func Test_Importer(t *testing.T) { switch op.(type) { case *bug.CreateOperation: - assert.Equal(t, ops[i].(*bug.CreateOperation).Title, op.(*bug.CreateOperation).Title) - assert.Equal(t, ops[i].(*bug.CreateOperation).Message, op.(*bug.CreateOperation).Message) - assert.Equal(t, ops[i].(*bug.CreateOperation).Author.Name(), op.(*bug.CreateOperation).Author.Name()) + assert.Equal(t, op.(*bug.CreateOperation).Title, ops[i].(*bug.CreateOperation).Title) + assert.Equal(t, op.(*bug.CreateOperation).Message, ops[i].(*bug.CreateOperation).Message) + assert.Equal(t, op.(*bug.CreateOperation).Author.Name(), ops[i].(*bug.CreateOperation).Author.Name()) case *bug.SetStatusOperation: - assert.Equal(t, ops[i].(*bug.SetStatusOperation).Status, op.(*bug.SetStatusOperation).Status) - assert.Equal(t, ops[i].(*bug.SetStatusOperation).Author.Name(), op.(*bug.SetStatusOperation).Author.Name()) + assert.Equal(t, op.(*bug.SetStatusOperation).Status, ops[i].(*bug.SetStatusOperation).Status) + assert.Equal(t, op.(*bug.SetStatusOperation).Author.Name(), ops[i].(*bug.SetStatusOperation).Author.Name()) case *bug.SetTitleOperation: - assert.Equal(t, ops[i].(*bug.SetTitleOperation).Was, op.(*bug.SetTitleOperation).Was) - assert.Equal(t, ops[i].(*bug.SetTitleOperation).Title, op.(*bug.SetTitleOperation).Title) - assert.Equal(t, ops[i].(*bug.SetTitleOperation).Author.Name(), op.(*bug.SetTitleOperation).Author.Name()) + assert.Equal(t, op.(*bug.SetTitleOperation).Was, ops[i].(*bug.SetTitleOperation).Was) + assert.Equal(t, op.(*bug.SetTitleOperation).Title, ops[i].(*bug.SetTitleOperation).Title) + assert.Equal(t, op.(*bug.SetTitleOperation).Author.Name(), ops[i].(*bug.SetTitleOperation).Author.Name()) case *bug.LabelChangeOperation: - assert.ElementsMatch(t, ops[i].(*bug.LabelChangeOperation).Added, op.(*bug.LabelChangeOperation).Added) - assert.ElementsMatch(t, ops[i].(*bug.LabelChangeOperation).Removed, op.(*bug.LabelChangeOperation).Removed) - assert.Equal(t, ops[i].(*bug.LabelChangeOperation).Author.Name(), op.(*bug.LabelChangeOperation).Author.Name()) + assert.ElementsMatch(t, op.(*bug.LabelChangeOperation).Added, ops[i].(*bug.LabelChangeOperation).Added) + assert.ElementsMatch(t, op.(*bug.LabelChangeOperation).Removed, ops[i].(*bug.LabelChangeOperation).Removed) + assert.Equal(t, op.(*bug.LabelChangeOperation).Author.Name(), ops[i].(*bug.LabelChangeOperation).Author.Name()) case *bug.AddCommentOperation: - assert.Equal(t, ops[i].(*bug.AddCommentOperation).Message, op.(*bug.AddCommentOperation).Message) - assert.Equal(t, ops[i].(*bug.AddCommentOperation).Author.Name(), op.(*bug.AddCommentOperation).Author.Name()) + assert.Equal(t, op.(*bug.AddCommentOperation).Message, ops[i].(*bug.AddCommentOperation).Message) + assert.Equal(t, op.(*bug.AddCommentOperation).Author.Name(), ops[i].(*bug.AddCommentOperation).Author.Name()) case *bug.EditCommentOperation: - assert.Equal(t, ops[i].(*bug.EditCommentOperation).Message, op.(*bug.EditCommentOperation).Message) - assert.Equal(t, ops[i].(*bug.EditCommentOperation).Author.Name(), op.(*bug.EditCommentOperation).Author.Name()) + assert.Equal(t, op.(*bug.EditCommentOperation).Message, ops[i].(*bug.EditCommentOperation).Message) + assert.Equal(t, op.(*bug.EditCommentOperation).Author.Name(), ops[i].(*bug.EditCommentOperation).Author.Name()) default: panic("Unknown operation type") -- cgit