You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
268 lines
7.8 KiB
268 lines
7.8 KiB
package migrations
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"fmt"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
|
|
"code.gitea.io/sdk/gitea"
|
|
"github.com/google/go-github/github"
|
|
"github.com/sirupsen/logrus"
|
|
)
|
|
|
|
// FetchMigratory adds GitHub fetching functions to migratory
|
|
type FetchMigratory struct {
|
|
Migratory
|
|
GHClient *github.Client
|
|
RepoOwner string
|
|
RepoName string
|
|
Logger *logrus.Logger
|
|
LogOutput *bytes.Buffer
|
|
}
|
|
|
|
func (fm *FetchMigratory) ctx() context.Context {
|
|
return context.Background()
|
|
}
|
|
|
|
// MigrateFromGitHub migrates RepoOwner/RepoName from GitHub to Gitea
|
|
func (fm *FetchMigratory) MigrateFromGitHub() error {
|
|
fm.Status = &MigratoryStatus{
|
|
Stage: Importing,
|
|
}
|
|
|
|
fm.Logger.WithFields(logrus.Fields{
|
|
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
|
|
}).Info("migrating git repository")
|
|
ghRepo, _, err := fm.GHClient.Repositories.Get(fm.ctx(), fm.RepoOwner, fm.RepoName)
|
|
if err != nil {
|
|
fm.Status.Stage = Failed
|
|
fm.Status.FatalError = err
|
|
return fmt.Errorf("GHClient Repostiories Get: %v", err)
|
|
}
|
|
fm.repository, err = fm.Repository(ghRepo)
|
|
if err != nil {
|
|
fm.Status.Stage = Failed
|
|
fm.Status.FatalError = err
|
|
return fmt.Errorf("Repository migration: %v", err)
|
|
}
|
|
fm.Logger.WithFields(logrus.Fields{
|
|
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
|
|
}).Info("git repository migrated")
|
|
if fm.Options.Issues || fm.Options.PullRequests {
|
|
var commentsChan chan *[]*github.IssueComment
|
|
if fm.Options.Comments {
|
|
commentsChan = fm.fetchCommentsAsync()
|
|
}
|
|
issues, err := fm.FetchIssues()
|
|
if err != nil {
|
|
fm.Status.Stage = Failed
|
|
fm.Status.FatalError = err
|
|
fm.Logger.WithFields(logrus.Fields{
|
|
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
|
|
}).Errorf("migration failed: %v", fm.Status.FatalError)
|
|
return err
|
|
}
|
|
fm.Status.Stage = Migrating
|
|
fm.Status.Issues = int64(len(issues))
|
|
migratedIssues := make(map[int]*gitea.Issue)
|
|
for _, issue := range issues {
|
|
if (!issue.IsPullRequest() || fm.Options.PullRequests) &&
|
|
(issue.IsPullRequest() || fm.Options.Issues) {
|
|
migratedIssues[issue.GetNumber()], err = fm.Issue(issue)
|
|
if err != nil {
|
|
fm.Status.IssuesError++
|
|
fm.Logger.WithFields(logrus.Fields{
|
|
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
|
|
"issue": issue.GetNumber(),
|
|
}).Warnf("error while migrating: %v", err)
|
|
continue
|
|
}
|
|
fm.Status.IssuesMigrated++
|
|
fm.Logger.WithFields(logrus.Fields{
|
|
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
|
|
"issue": issue.GetNumber(),
|
|
}).Info("issue migrated")
|
|
} else {
|
|
fm.Status.Issues--
|
|
}
|
|
}
|
|
if fm.Options.Comments {
|
|
var comments []*github.IssueComment
|
|
if cmts := <-commentsChan; cmts == nil {
|
|
fm.Status.Stage = Failed
|
|
err := fmt.Errorf("error while fetching issue comments")
|
|
fm.Logger.WithFields(logrus.Fields{
|
|
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
|
|
}).Errorf("migration failed: %v", fm.Status.FatalError)
|
|
return err
|
|
} else {
|
|
comments = *cmts
|
|
}
|
|
|
|
if err != nil {
|
|
fm.Status.Stage = Failed
|
|
fm.Status.FatalError = err
|
|
fm.Logger.WithFields(logrus.Fields{
|
|
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
|
|
}).Errorf("migration failed: %v", fm.Status.FatalError)
|
|
return err
|
|
}
|
|
fm.Status.Comments = int64(len(comments))
|
|
commentsByIssue := make(map[*gitea.Issue][]*github.IssueComment, len(migratedIssues))
|
|
for _, comment := range comments {
|
|
issueIndex, err := getIssueIndexFromHTMLURL(comment.GetHTMLURL())
|
|
if err != nil {
|
|
fm.Status.CommentsError++
|
|
fm.Logger.WithFields(logrus.Fields{
|
|
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
|
|
"issue": issueIndex,
|
|
"comment": comment.GetID(),
|
|
}).Warnf("error while migrating comment: %v", err)
|
|
continue
|
|
}
|
|
if issue, ok := migratedIssues[issueIndex]; ok && issue != nil {
|
|
if list, ok := commentsByIssue[issue]; !ok && list != nil {
|
|
commentsByIssue[issue] = []*github.IssueComment{comment}
|
|
} else {
|
|
commentsByIssue[issue] = append(list, comment)
|
|
}
|
|
} else {
|
|
fm.Status.CommentsError++
|
|
continue
|
|
}
|
|
}
|
|
wg := sync.WaitGroup{}
|
|
for issue, comms := range commentsByIssue {
|
|
wg.Add(1)
|
|
go func(i *gitea.Issue, cs []*github.IssueComment) {
|
|
for _, comm := range cs {
|
|
if _, err := fm.IssueComment(i, comm); err != nil {
|
|
fm.Status.CommentsError++
|
|
fm.Logger.WithFields(logrus.Fields{
|
|
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
|
|
"comment": comm.GetID(),
|
|
}).Warnf("error while migrating comment: %v", err)
|
|
continue
|
|
}
|
|
fm.Status.CommentsMigrated++
|
|
fm.Logger.WithFields(logrus.Fields{
|
|
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
|
|
"comment": comm.GetID(),
|
|
}).Info("comment migrated")
|
|
}
|
|
wg.Done()
|
|
}(issue, comms)
|
|
}
|
|
wg.Wait()
|
|
}
|
|
}
|
|
if fm.Status.FatalError != nil {
|
|
fm.Status.Stage = Failed
|
|
fm.Logger.WithFields(logrus.Fields{
|
|
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
|
|
}).Errorf("migration failed: %v", fm.Status.FatalError)
|
|
return nil
|
|
}
|
|
fm.Status.Stage = Finished
|
|
fm.Logger.WithFields(logrus.Fields{
|
|
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
|
|
}).Info("migration successful")
|
|
return nil
|
|
}
|
|
|
|
var issueIndexRegex = regexp.MustCompile(`/(issues|pull)/([0-9]+)#`)
|
|
|
|
func getIssueIndexFromHTMLURL(htmlURL string) (int, error) {
|
|
// Alt is 4 times faster but more error prune
|
|
if res, err := getIssueIndexFromHTMLURLAlt(htmlURL); err == nil {
|
|
return res, nil
|
|
}
|
|
matches := issueIndexRegex.FindStringSubmatch(htmlURL)
|
|
if len(matches) < 3 {
|
|
return 0, fmt.Errorf("cannot parse issue id from HTML URL: %s", htmlURL)
|
|
}
|
|
return strconv.Atoi(matches[2])
|
|
}
|
|
func getIssueIndexFromHTMLURLAlt(htmlURL string) (int, error) {
|
|
res := strings.Split(htmlURL, "/issues/")
|
|
if len(res) != 2 {
|
|
res = strings.Split(htmlURL, "/pull/")
|
|
}
|
|
if len(res) != 2 {
|
|
return 0, fmt.Errorf("invalid HTMLURL: %s", htmlURL)
|
|
}
|
|
number := res[1]
|
|
number = strings.Split(number, "#")[0]
|
|
return strconv.Atoi(number)
|
|
}
|
|
|
|
// FetchIssues fetches all issues from GitHub
|
|
func (fm *FetchMigratory) FetchIssues() ([]*github.Issue, error) {
|
|
opt := &github.IssueListByRepoOptions{
|
|
Sort: "created",
|
|
Direction: "asc",
|
|
State: "all",
|
|
ListOptions: github.ListOptions{
|
|
PerPage: 100,
|
|
},
|
|
}
|
|
var allIssues = make([]*github.Issue, 0)
|
|
for {
|
|
issues, resp, err := fm.GHClient.Issues.ListByRepo(fm.ctx(), fm.RepoOwner, fm.RepoName, opt)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error while listing repos: %v", err)
|
|
}
|
|
allIssues = append(allIssues, issues...)
|
|
if resp.NextPage == 0 {
|
|
break
|
|
}
|
|
opt.Page = resp.NextPage
|
|
}
|
|
return allIssues, nil
|
|
}
|
|
|
|
// FetchComments fetches all comments from GitHub
|
|
func (fm *FetchMigratory) FetchComments() ([]*github.IssueComment, error) {
|
|
var allComments = make([]*github.IssueComment, 0)
|
|
opt := &github.IssueListCommentsOptions{
|
|
Sort: "created",
|
|
Direction: "asc",
|
|
ListOptions: github.ListOptions{
|
|
PerPage: 100,
|
|
},
|
|
}
|
|
for {
|
|
comments, resp, err := fm.GHClient.Issues.ListComments(fm.ctx(), fm.RepoOwner, fm.RepoName, 0, opt)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error while listing repos: %v", err)
|
|
}
|
|
allComments = append(allComments, comments...)
|
|
if resp.NextPage == 0 {
|
|
break
|
|
}
|
|
opt.Page = resp.NextPage
|
|
}
|
|
return allComments, nil
|
|
}
|
|
|
|
func (fm *FetchMigratory) fetchCommentsAsync() chan *[]*github.IssueComment {
|
|
ret := make(chan *[]*github.IssueComment, 1)
|
|
go func(f *FetchMigratory) {
|
|
comments, err := f.FetchComments()
|
|
if err != nil {
|
|
f.Status.FatalError = err
|
|
ret <- nil
|
|
fm.Logger.WithFields(logrus.Fields{
|
|
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
|
|
}).Errorf("fetching comments failed: %v", fm.Status.FatalError)
|
|
return
|
|
}
|
|
f.Status.Comments = int64(len(comments))
|
|
ret <- &comments
|
|
}(fm)
|
|
return ret
|
|
}
|
|
|