Refactor some parts of github.go

Add additional qa tasks to Makefile
Add vendor to gitignore
remotes/1698823321490190475/tmp_refs/heads/update-golang
Jonas Franz 5 years ago
parent 000ed0ee7e
commit f758e29097
  1. 2
      .gitignore
  2. 9
      Makefile
  3. 214
      migrations/github.go
  4. 4
      migrations/github_test.go

2
.gitignore vendored

@ -16,3 +16,5 @@
.glide/ .glide/
data/ data/
vendor/

@ -74,12 +74,19 @@ lint:
fi fi
golint -set_exit_status $(go list ./...) golint -set_exit_status $(go list ./...)
.PHONY: ineffassign
ineffassign:
@hash ineffassign > /dev/null 2>&1; if [ $$? -ne 0 ]; then \
$(GO) get -u github.com/gordonklaus/ineffassign; \
fi
ineffassign .
.PHONY: vet .PHONY: vet
vet: vet:
go vet ./... go vet ./...
.PHONY: test .PHONY: test
test: lint vet test: lint vet ineffassign
go test -tags web -cover ./... go test -tags web -cover ./...
.PHONY: coverage .PHONY: coverage

@ -53,124 +53,136 @@ func (fm *FetchMigratory) MigrateFromGitHub() error {
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName), "repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
}).Info("git repository migrated") }).Info("git repository migrated")
if fm.Options.Issues || fm.Options.PullRequests { if fm.Options.Issues || fm.Options.PullRequests {
var commentsChan chan *[]*github.IssueComment if err := fm.MigrateIssuesFromGitHub(); err != nil {
if fm.Options.Comments {
commentsChan = fm.fetchCommentsAsync()
}
issues, err := fm.FetchIssues()
if err != nil {
fm.Status.Stage = Failed
fm.Status.FatalError = err
fm.Logger.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
}).Errorf("migration failed: %v", fm.Status.FatalError)
return err return err
} }
fm.Status.Stage = Migrating }
fm.Status.Issues = int64(len(issues)) if fm.Status.FatalError != nil {
migratedIssues := make(map[int]*gitea.Issue) fm.Status.Stage = Failed
for _, issue := range issues { fm.Logger.WithFields(logrus.Fields{
if (!issue.IsPullRequest() || fm.Options.PullRequests) && "repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
(issue.IsPullRequest() || fm.Options.Issues) { }).Errorf("migration failed: %v", fm.Status.FatalError)
migratedIssues[issue.GetNumber()], err = fm.Issue(issue) return nil
if err != nil { }
fm.Status.IssuesError++ fm.Status.Stage = Finished
fm.Logger.WithFields(logrus.Fields{ fm.Logger.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName), "repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
"issue": issue.GetNumber(), }).Info("migration successful")
}).Warnf("error while migrating: %v", err) return nil
continue }
}
fm.Status.IssuesMigrated++ // MigrateIssuesFromGitHub migrates all issues from GitHub to Gitea
func (fm *FetchMigratory) MigrateIssuesFromGitHub() error {
var commentsChan chan *[]*github.IssueComment
if fm.Options.Comments {
commentsChan = fm.fetchCommentsAsync()
}
issues, err := fm.FetchIssues()
if err != nil {
fm.Status.Stage = Failed
fm.Status.FatalError = err
fm.Logger.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
}).Errorf("migration failed: %v", fm.Status.FatalError)
return err
}
fm.Status.Stage = Migrating
fm.Status.Issues = int64(len(issues))
migratedIssues := make(map[int]*gitea.Issue)
for _, issue := range issues {
if (!issue.IsPullRequest() || fm.Options.PullRequests) &&
(issue.IsPullRequest() || fm.Options.Issues) {
migratedIssues[issue.GetNumber()], err = fm.Issue(issue)
if err != nil {
fm.Status.IssuesError++
fm.Logger.WithFields(logrus.Fields{ fm.Logger.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName), "repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
"issue": issue.GetNumber(), "issue": issue.GetNumber(),
}).Info("issue migrated") }).Warnf("error while migrating: %v", err)
} else { continue
fm.Status.Issues--
} }
fm.Status.IssuesMigrated++
fm.Logger.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
"issue": issue.GetNumber(),
}).Info("issue migrated")
} else {
fm.Status.Issues--
} }
if fm.Options.Comments { }
var comments []*github.IssueComment if fm.Options.Comments {
if cmts := <-commentsChan; cmts == nil { return fm.MigrateCommentsFromGitHub()
fm.Status.Stage = Failed }
err := fmt.Errorf("error while fetching issue comments") return nil
fm.Logger.WithFields(logrus.Fields{ }
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
}).Errorf("migration failed: %v", fm.Status.FatalError) func (fm *FetchMigratory) MigrateCommentsFromGitHub() error {
return err var comments []*github.IssueComment
var cmts *[]*github.IssueComment
if cmts = <-commentsChan; cmts == nil {
fm.Status.Stage = Failed
err := fmt.Errorf("error while fetching issue comments")
fm.Logger.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
}).Errorf("migration failed: %v", fm.Status.FatalError)
return err
}
comments = *cmts
if err != nil {
fm.Status.Stage = Failed
fm.Status.FatalError = err
fm.Logger.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
}).Errorf("migration failed: %v", fm.Status.FatalError)
return err
}
fm.Status.Comments = int64(len(comments))
commentsByIssue := make(map[*gitea.Issue][]*github.IssueComment, len(migratedIssues))
for _, comment := range comments {
issueIndex, err := getIssueIndexFromHTMLURL(comment.GetHTMLURL())
if err != nil {
fm.Status.CommentsError++
fm.Logger.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
"issue": issueIndex,
"comment": comment.GetID(),
}).Warnf("error while migrating comment: %v", err)
continue
}
if issue, ok := migratedIssues[issueIndex]; ok && issue != nil {
if list, ok := commentsByIssue[issue]; !ok && list != nil {
commentsByIssue[issue] = []*github.IssueComment{comment}
} else { } else {
comments = *cmts commentsByIssue[issue] = append(list, comment)
} }
if err != nil { } else {
fm.Status.Stage = Failed fm.Status.CommentsError++
fm.Status.FatalError = err continue
fm.Logger.WithFields(logrus.Fields{ }
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName), }
}).Errorf("migration failed: %v", fm.Status.FatalError) wg := sync.WaitGroup{}
return err for issue, comms := range commentsByIssue {
} wg.Add(1)
fm.Status.Comments = int64(len(comments)) go func(i *gitea.Issue, cs []*github.IssueComment) {
commentsByIssue := make(map[*gitea.Issue][]*github.IssueComment, len(migratedIssues)) for _, comm := range cs {
for _, comment := range comments { if _, err := fm.IssueComment(i, comm); err != nil {
issueIndex, err := getIssueIndexFromHTMLURL(comment.GetHTMLURL())
if err != nil {
fm.Status.CommentsError++ fm.Status.CommentsError++
fm.Logger.WithFields(logrus.Fields{ fm.Logger.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName), "repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
"issue": issueIndex, "comment": comm.GetID(),
"comment": comment.GetID(),
}).Warnf("error while migrating comment: %v", err) }).Warnf("error while migrating comment: %v", err)
continue continue
} }
if issue, ok := migratedIssues[issueIndex]; ok && issue != nil { fm.Status.CommentsMigrated++
if list, ok := commentsByIssue[issue]; !ok && list != nil { fm.Logger.WithFields(logrus.Fields{
commentsByIssue[issue] = []*github.IssueComment{comment} "repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
} else { "comment": comm.GetID(),
commentsByIssue[issue] = append(list, comment) }).Info("comment migrated")
}
} else {
fm.Status.CommentsError++
continue
}
}
wg := sync.WaitGroup{}
for issue, comms := range commentsByIssue {
wg.Add(1)
go func(i *gitea.Issue, cs []*github.IssueComment) {
for _, comm := range cs {
if _, err := fm.IssueComment(i, comm); err != nil {
fm.Status.CommentsError++
fm.Logger.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
"comment": comm.GetID(),
}).Warnf("error while migrating comment: %v", err)
continue
}
fm.Status.CommentsMigrated++
fm.Logger.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
"comment": comm.GetID(),
}).Info("comment migrated")
}
wg.Done()
}(issue, comms)
} }
wg.Wait() wg.Done()
} }(issue, comms)
} }
if fm.Status.FatalError != nil { wg.Wait()
fm.Status.Stage = Failed
fm.Logger.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
}).Errorf("migration failed: %v", fm.Status.FatalError)
return nil
}
fm.Status.Stage = Finished
fm.Logger.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
}).Info("migration successful")
return nil
} }
var issueIndexRegex = regexp.MustCompile(`/(issues|pull)/([0-9]+)#`) var issueIndexRegex = regexp.MustCompile(`/(issues|pull)/([0-9]+)#`)

@ -17,7 +17,7 @@ func TestGetIssueIndexFromHTMLURLAlt(t *testing.T) {
res, err := getIssueIndexFromHTMLURLAlt("https://github.com/octocat/Hello-World/issues/1347#issuecomment-1") res, err := getIssueIndexFromHTMLURLAlt("https://github.com/octocat/Hello-World/issues/1347#issuecomment-1")
assert.NoError(t, err) assert.NoError(t, err)
assert.Equal(t, 1347, res) assert.Equal(t, 1347, res)
res, err = getIssueIndexFromHTMLURLAlt("https://github.com/oment-1") _, err = getIssueIndexFromHTMLURLAlt("https://github.com/oment-1")
assert.Error(t, err) assert.Error(t, err)
} }
@ -31,7 +31,7 @@ func TestGetIssueIndexFromHTMLURL(t *testing.T) {
res, err := getIssueIndexFromHTMLURL("https://github.com/octocat/Hello-World/issues/1347#issuecomment-1") res, err := getIssueIndexFromHTMLURL("https://github.com/octocat/Hello-World/issues/1347#issuecomment-1")
assert.NoError(t, err) assert.NoError(t, err)
assert.Equal(t, 1347, res) assert.Equal(t, 1347, res)
res, err = getIssueIndexFromHTMLURL("https://github.com/oment-1") _, err = getIssueIndexFromHTMLURL("https://github.com/oment-1")
assert.Error(t, err) assert.Error(t, err)
} }

Loading…
Cancel
Save