Browse Source

Refactor some parts of github.go

Add additional qa tasks to Makefile
Add vendor to gitignore
pull/31/head
Jonas Franz 1 year ago
parent
commit
f758e29097
4 changed files with 125 additions and 104 deletions
  1. +2
    -0
      .gitignore
  2. +8
    -1
      Makefile
  3. +113
    -101
      migrations/github.go
  4. +2
    -2
      migrations/github_test.go

+ 2
- 0
.gitignore View File

@@ -16,3 +16,5 @@
.glide/

data/

vendor/

+ 8
- 1
Makefile View File

@@ -74,12 +74,19 @@ lint:
fi
golint -set_exit_status $(go list ./...)

.PHONY: ineffassign
ineffassign:
@hash ineffassign > /dev/null 2>&1; if [ $$? -ne 0 ]; then \
$(GO) get -u github.com/gordonklaus/ineffassign; \
fi
ineffassign .

.PHONY: vet
vet:
go vet ./...

.PHONY: test
test: lint vet
test: lint vet ineffassign
go test -tags web -cover ./...

.PHONY: coverage


+ 113
- 101
migrations/github.go View File

@@ -53,124 +53,136 @@ func (fm *FetchMigratory) MigrateFromGitHub() error {
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
}).Info("git repository migrated")
if fm.Options.Issues || fm.Options.PullRequests {
var commentsChan chan *[]*github.IssueComment
if fm.Options.Comments {
commentsChan = fm.fetchCommentsAsync()
}
issues, err := fm.FetchIssues()
if err != nil {
fm.Status.Stage = Failed
fm.Status.FatalError = err
fm.Logger.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
}).Errorf("migration failed: %v", fm.Status.FatalError)
if err := fm.MigrateIssuesFromGitHub(); err != nil {
return err
}
fm.Status.Stage = Migrating
fm.Status.Issues = int64(len(issues))
migratedIssues := make(map[int]*gitea.Issue)
for _, issue := range issues {
if (!issue.IsPullRequest() || fm.Options.PullRequests) &&
(issue.IsPullRequest() || fm.Options.Issues) {
migratedIssues[issue.GetNumber()], err = fm.Issue(issue)
if err != nil {
fm.Status.IssuesError++
fm.Logger.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
"issue": issue.GetNumber(),
}).Warnf("error while migrating: %v", err)
continue
}
fm.Status.IssuesMigrated++
}
if fm.Status.FatalError != nil {
fm.Status.Stage = Failed
fm.Logger.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
}).Errorf("migration failed: %v", fm.Status.FatalError)
return nil
}
fm.Status.Stage = Finished
fm.Logger.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
}).Info("migration successful")
return nil
}

// MigrateIssuesFromGitHub migrates all issues from GitHub to Gitea
func (fm *FetchMigratory) MigrateIssuesFromGitHub() error {
var commentsChan chan *[]*github.IssueComment
if fm.Options.Comments {
commentsChan = fm.fetchCommentsAsync()
}
issues, err := fm.FetchIssues()
if err != nil {
fm.Status.Stage = Failed
fm.Status.FatalError = err
fm.Logger.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
}).Errorf("migration failed: %v", fm.Status.FatalError)
return err
}
fm.Status.Stage = Migrating
fm.Status.Issues = int64(len(issues))
migratedIssues := make(map[int]*gitea.Issue)
for _, issue := range issues {
if (!issue.IsPullRequest() || fm.Options.PullRequests) &&
(issue.IsPullRequest() || fm.Options.Issues) {
migratedIssues[issue.GetNumber()], err = fm.Issue(issue)
if err != nil {
fm.Status.IssuesError++
fm.Logger.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
"issue": issue.GetNumber(),
}).Info("issue migrated")
} else {
fm.Status.Issues--
}).Warnf("error while migrating: %v", err)
continue
}
fm.Status.IssuesMigrated++
fm.Logger.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
"issue": issue.GetNumber(),
}).Info("issue migrated")
} else {
fm.Status.Issues--
}
if fm.Options.Comments {
var comments []*github.IssueComment
if cmts := <-commentsChan; cmts == nil {
fm.Status.Stage = Failed
err := fmt.Errorf("error while fetching issue comments")
fm.Logger.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
}).Errorf("migration failed: %v", fm.Status.FatalError)
return err
}
if fm.Options.Comments {
return fm.MigrateCommentsFromGitHub()
}
return nil
}

func (fm *FetchMigratory) MigrateCommentsFromGitHub() error {
var comments []*github.IssueComment
var cmts *[]*github.IssueComment
if cmts = <-commentsChan; cmts == nil {
fm.Status.Stage = Failed
err := fmt.Errorf("error while fetching issue comments")
fm.Logger.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
}).Errorf("migration failed: %v", fm.Status.FatalError)
return err
}
comments = *cmts
if err != nil {
fm.Status.Stage = Failed
fm.Status.FatalError = err
fm.Logger.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
}).Errorf("migration failed: %v", fm.Status.FatalError)
return err
}
fm.Status.Comments = int64(len(comments))
commentsByIssue := make(map[*gitea.Issue][]*github.IssueComment, len(migratedIssues))
for _, comment := range comments {
issueIndex, err := getIssueIndexFromHTMLURL(comment.GetHTMLURL())
if err != nil {
fm.Status.CommentsError++
fm.Logger.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
"issue": issueIndex,
"comment": comment.GetID(),
}).Warnf("error while migrating comment: %v", err)
continue
}
if issue, ok := migratedIssues[issueIndex]; ok && issue != nil {
if list, ok := commentsByIssue[issue]; !ok && list != nil {
commentsByIssue[issue] = []*github.IssueComment{comment}
} else {
comments = *cmts
commentsByIssue[issue] = append(list, comment)
}
if err != nil {
fm.Status.Stage = Failed
fm.Status.FatalError = err
fm.Logger.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
}).Errorf("migration failed: %v", fm.Status.FatalError)
return err
}
fm.Status.Comments = int64(len(comments))
commentsByIssue := make(map[*gitea.Issue][]*github.IssueComment, len(migratedIssues))
for _, comment := range comments {
issueIndex, err := getIssueIndexFromHTMLURL(comment.GetHTMLURL())
if err != nil {
} else {
fm.Status.CommentsError++
continue
}
}
wg := sync.WaitGroup{}
for issue, comms := range commentsByIssue {
wg.Add(1)
go func(i *gitea.Issue, cs []*github.IssueComment) {
for _, comm := range cs {
if _, err := fm.IssueComment(i, comm); err != nil {
fm.Status.CommentsError++
fm.Logger.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
"issue": issueIndex,
"comment": comment.GetID(),
"comment": comm.GetID(),
}).Warnf("error while migrating comment: %v", err)
continue
}
if issue, ok := migratedIssues[issueIndex]; ok && issue != nil {
if list, ok := commentsByIssue[issue]; !ok && list != nil {
commentsByIssue[issue] = []*github.IssueComment{comment}
} else {
commentsByIssue[issue] = append(list, comment)
}
} else {
fm.Status.CommentsError++
continue
}
}
wg := sync.WaitGroup{}
for issue, comms := range commentsByIssue {
wg.Add(1)
go func(i *gitea.Issue, cs []*github.IssueComment) {
for _, comm := range cs {
if _, err := fm.IssueComment(i, comm); err != nil {
fm.Status.CommentsError++
fm.Logger.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
"comment": comm.GetID(),
}).Warnf("error while migrating comment: %v", err)
continue
}
fm.Status.CommentsMigrated++
fm.Logger.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
"comment": comm.GetID(),
}).Info("comment migrated")
}
wg.Done()
}(issue, comms)
fm.Status.CommentsMigrated++
fm.Logger.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
"comment": comm.GetID(),
}).Info("comment migrated")
}
wg.Wait()
}
wg.Done()
}(issue, comms)
}
if fm.Status.FatalError != nil {
fm.Status.Stage = Failed
fm.Logger.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
}).Errorf("migration failed: %v", fm.Status.FatalError)
return nil
}
fm.Status.Stage = Finished
fm.Logger.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
}).Info("migration successful")
return nil
wg.Wait()
}

var issueIndexRegex = regexp.MustCompile(`/(issues|pull)/([0-9]+)#`)


+ 2
- 2
migrations/github_test.go View File

@@ -17,7 +17,7 @@ func TestGetIssueIndexFromHTMLURLAlt(t *testing.T) {
res, err := getIssueIndexFromHTMLURLAlt("https://github.com/octocat/Hello-World/issues/1347#issuecomment-1")
assert.NoError(t, err)
assert.Equal(t, 1347, res)
res, err = getIssueIndexFromHTMLURLAlt("https://github.com/oment-1")
_, err = getIssueIndexFromHTMLURLAlt("https://github.com/oment-1")
assert.Error(t, err)
}

@@ -31,7 +31,7 @@ func TestGetIssueIndexFromHTMLURL(t *testing.T) {
res, err := getIssueIndexFromHTMLURL("https://github.com/octocat/Hello-World/issues/1347#issuecomment-1")
assert.NoError(t, err)
assert.Equal(t, 1347, res)
res, err = getIssueIndexFromHTMLURL("https://github.com/oment-1")
_, err = getIssueIndexFromHTMLURL("https://github.com/oment-1")
assert.Error(t, err)
}



Loading…
Cancel
Save