Add migration for repos, issues, labels, milestones and comments

Signed-off-by: Jonas Franz <info@jonasfranz.software>

Gopkg.lock

@@ -0,0 +1,55 @@
+# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
+
+
+[[projects]]
+  branch = "master"
+  name = "code.gitea.io/sdk"
+  packages = ["gitea"]
+  revision = "39c609e903992e25deca0e7aa2c5304fd680530f"
+
+[[projects]]
+  name = "github.com/google/go-github"
+  packages = ["github"]
+  revision = "e48060a28fac52d0f1cb758bc8b87c07bac4a87d"
+  version = "v15.0.0"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/google/go-querystring"
+  packages = ["query"]
+  revision = "53e6ce116135b80d037921a7fdd5138cf32d7a8a"
+
+[[projects]]
+  name = "github.com/mattn/go-isatty"
+  packages = ["."]
+  revision = "0360b2af4f38e8d38c7fce2a9f4e702702d73a39"
+  version = "v0.0.3"
+
+[[projects]]
+  name = "github.com/urfave/cli"
+  packages = ["."]
+  revision = "cfb38830724cc34fedffe9a2a29fb54fa9169cd1"
+  version = "v1.20.0"
+
+[[projects]]
+  name = "github.com/vbauerster/mpb"
+  packages = [
+    ".",
+    "cwriter",
+    "decor"
+  ]
+  revision = "e227edc706423683a577a93c78beb0ce9d211b00"
+  version = "v3.1.1"
+
+[[projects]]
+  branch = "master"
+  name = "golang.org/x/sys"
+  packages = ["unix"]
+  revision = "01acb38716e021ed1fc03a602bdb5838e1358c5e"
+
+[solve-meta]
+  analyzer-name = "dep"
+  analyzer-version = 1
+  inputs-digest = "746c57ee0419dfecdb375bb45e31e55a9347b27d051af544d96397418e2d797d"
+  solver-name = "gps-cdcl"
+  solver-version = 1

Gopkg.toml

@@ -0,0 +1,46 @@
+# Gopkg.toml example
+#
+# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md
+# for detailed Gopkg.toml documentation.
+#
+# required = ["github.com/user/thing/cmd/thing"]
+# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
+#
+# [[constraint]]
+#   name = "github.com/user/project"
+#   version = "1.0.0"
+#
+# [[constraint]]
+#   name = "github.com/user/project2"
+#   branch = "dev"
+#   source = "github.com/myfork/project2"
+#
+# [[override]]
+#   name = "github.com/x/y"
+#   version = "2.4.0"
+#
+# [prune]
+#   non-go = false
+#   go-tests = true
+#   unused-packages = true
+
+
+[prune]
+  go-tests = true
+  unused-packages = true
+
+[[constraint]]
+  branch = "master"
+  name = "code.gitea.io/sdk"
+
+[[constraint]]
+  name = "github.com/urfave/cli"
+  version = "1.20.0"
+
+[[constraint]]
+  name = "github.com/google/go-github"
+  version = "15.0.0"
+
+[[constraint]]
+  name = "github.com/vbauerster/mpb"
+  version = "3.1.1"

cmd/migrate.go

@@ -0,0 +1,125 @@
+package cmd
+
+import (
+	"code.gitea.io/sdk/gitea"
+	"context"
+	"fmt"
+	"git.jonasfranz.software/JonasFranzDEV/gitea-github-migrator/migrations"
+	"github.com/google/go-github/github"
+	"github.com/urfave/cli"
+	"golang.org/x/oauth2"
+	"strings"
+)
+
+var CmdMigrate = cli.Command{
+	Name:   "migrate",
+	Usage:  "migrates a github to a gitea repository",
+	Action: runMigrate,
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:   "gh-repo",
+			Usage:  "GitHub Repository",
+			Value:  "username/reponame",
+			EnvVar: "GH_REPOSITORY",
+		},
+		cli.IntFlag{
+			Name:   "owner",
+			Usage:  "Owner ID",
+			EnvVar: "OWNER_ID",
+			Value:  0,
+		},
+		cli.StringFlag{
+			Name:   "token",
+			Usage:  "Gitea Token",
+			EnvVar: "GITEA_TOKEN",
+		},
+		cli.StringFlag{
+			Name:   "url",
+			Usage:  "Gitea URL",
+			EnvVar: "GITEA_URL",
+		},
+		cli.BoolFlag{
+			Name:   "private",
+			Usage:  "should new repository be private",
+			EnvVar: "GITEA_PRIVATE",
+		},
+	},
+}
+
+func runMigrate(ctx *cli.Context) error {
+	m := migrations.Migratory{
+		Client:     gitea.NewClient(ctx.String("url"), ctx.String("token")),
+		Private:    ctx.Bool("private"),
+		NewOwnerID: ctx.Int("owner"),
+	}
+	c := context.Background()
+
+	ts := oauth2.StaticTokenSource(
+		&oauth2.Token{AccessToken: "246947e59029a71cb568e290cc6b28adeda514df"},
+	)
+	tc := oauth2.NewClient(c, ts)
+	gc := github.NewClient(tc)
+
+	username := strings.Split(ctx.String("gh-repo"), "/")[0]
+	repo := strings.Split(ctx.String("gh-repo"), "/")[1]
+
+	//p := mpb.New()
+
+	fmt.Printf("Fetching repository %s/%s...\n", username, repo)
+	gr, _, err := gc.Repositories.Get(c, username, repo)
+	if err != nil {
+		return err
+	}
+	fmt.Printf("Migrating repository %s/%s...\n", username, repo)
+	if mr, err := m.Repository(gr); err != nil {
+		return err
+	} else {
+		fmt.Printf("Repository migrated to %s/%s\n", mr.Owner.UserName, mr.Name)
+	}
+
+	fmt.Println("Fetching issues...")
+	opt := &github.IssueListByRepoOptions{
+		Sort:      "created",
+		Direction: "asc",
+		State:     "all",
+		ListOptions: github.ListOptions{
+			PerPage: 100,
+		},
+	}
+	var allIssues = make([]*github.Issue, 0)
+	for {
+		issues, resp, err := gc.Issues.ListByRepo(c, username, repo, opt)
+		if err != nil {
+			return err
+		}
+		allIssues = append(allIssues, issues...)
+		if resp.NextPage == 0 {
+			break
+		}
+		opt.Page = resp.NextPage
+	}
+	fmt.Println("Migrating issues...")
+	//bar := p.AddBar(int64(len(issues)))
+	for _, gi := range allIssues {
+		fmt.Printf("Migrating #%d...\n", *gi.Number)
+		issue, err := m.Issue(gi)
+		if err != nil {
+			return fmt.Errorf("migrating issue[id: %d]: %v", *gi.ID, err)
+		}
+		comments, _, err := gc.Issues.ListComments(c, username, repo, gi.GetNumber(), nil)
+		if err != nil {
+			return fmt.Errorf("fetching issue[id: %d] comments: %v", *gi.ID, err)
+		}
+		for _, gc := range comments {
+			fmt.Printf("-> %d...", gc.ID)
+			if _, err := m.IssueComment(issue, gc); err != nil {
+				return fmt.Errorf("migrating issue comment [issue: %d, comment: %d]: %v", *gi.ID, gc.ID, err)
+			}
+			fmt.Print("Done!\n")
+		}
+		fmt.Printf("Migrated #%d...\n", *gi.Number)
+		//bar.Increment()
+
+	}
+	return nil
+}

main.go

@@ -0,0 +1,22 @@
+package main
+
+import (
+	"os"
+
+	"github.com/urfave/cli"
+
+	"git.jonasfranz.software/JonasFranzDEV/gitea-github-migrator/cmd"
+)
+
+func main() {
+	app := cli.NewApp()
+	app.Name = "gg-migrator"
+	app.Usage = "GitHub to Gitea migrator for repositories"
+	app.Description = `Migrate your GitHub repositories including issues to Gitea`
+	app.Commands = cli.Commands{
+		cmd.CmdMigrate,
+	}
+	if err := app.Run(os.Args); err != nil {
+		panic(err)
+	}
+}

migrations/issue.go

@@ -0,0 +1,111 @@
+package migrations
+
+import (
+	"code.gitea.io/sdk/gitea"
+	"fmt"
+	"github.com/google/go-github/github"
+)
+
+func (m *Migratory) Issue(gi *github.Issue) (*gitea.Issue, error) {
+	if m.migratedMilestones == nil {
+		m.migratedMilestones = make(map[int64]int64)
+	}
+	if m.migratedLabels == nil {
+		m.migratedLabels = make(map[int64]int64)
+	}
+
+	// Migrate milestone if it is not already migrated
+	milestone := int64(0)
+	if gi.Milestone != nil {
+		// Lookup if milestone is already migrated
+		if migratedMilestone, ok := m.migratedMilestones[*gi.Milestone.ID]; ok {
+			milestone = migratedMilestone
+		} else if ms, err := m.Milestone(gi.Milestone); err != nil {
+			return nil, err
+		} else {
+			milestone = ms.ID
+		}
+	}
+	// Migrate labels
+	labels, err := m.labels(gi.Labels)
+	if err != nil {
+		return nil, err
+	}
+
+	return m.Client.CreateIssue(m.repository.Owner.UserName, m.repository.Name,
+		gitea.CreateIssueOption{
+			Title:     gi.GetTitle(),
+			Body:      fmt.Sprintf("Author: @%s Posted at: %s\n\n\n%s", *gi.User.Login, gi.GetCreatedAt().Format("02.01.2006 15:04"), gi.GetBody()),
+			Closed:    *gi.State == "closed",
+			Milestone: milestone,
+			Labels:    labels,
+		})
+}
+
+func (m *Migratory) labels(gls []github.Label) (results []int64, err error) {
+	for _, gl := range gls {
+		if migratedLabel, ok := m.migratedLabels[*gl.ID]; ok {
+			results = append(results, migratedLabel)
+		} else {
+			var newLabel *gitea.Label
+			if newLabel, err = m.Label(&gl); err != nil {
+				return nil, err
+			}
+			m.migratedLabels[*gl.ID] = newLabel.ID
+			results = append(results, newLabel.ID)
+		}
+	}
+	return
+}
+
+func (m *Migratory) Label(gl *github.Label) (*gitea.Label, error) {
+	return m.Client.CreateLabel(m.repository.Owner.UserName, m.repository.Name,
+		gitea.CreateLabelOption{
+			Name:  gl.GetName(),
+			Color: fmt.Sprintf("#%s", gl.GetColor()),
+		})
+}
+
+func (m *Migratory) Milestone(gm *github.Milestone) (*gitea.Milestone, error) {
+	ms, err := m.Client.CreateMilestone(m.repository.Owner.UserName, m.repository.Name,
+		gitea.CreateMilestoneOption{
+			Title:       gm.GetTitle(),
+			Description: gm.GetDescription(),
+			Deadline:    gm.DueOn,
+		})
+	if err != nil {
+		return nil, err
+	}
+	m.migratedMilestones[*gm.ID] = ms.ID
+	if gm.State != nil && *gm.State != "open" {
+		return m.Client.EditMilestone(m.repository.Owner.UserName, m.repository.Name,
+			ms.ID, gitea.EditMilestoneOption{
+				State: githubStateToGiteaState(gm.State),
+			})
+	}
+	return ms, err
+}
+
+func githubStateToGiteaState(ghstate *string) *string {
+	if ghstate == nil {
+		return ghstate
+	}
+	switch *ghstate {
+	case "open":
+	case "closed":
+		return ghstate
+	case "all":
+		open := "open"
+		return &open
+	}
+	return nil
+}
+
+func (m *Migratory) IssueComment(issue *gitea.Issue, gic *github.IssueComment) (*gitea.Comment, error) {
+	return m.Client.CreateIssueComment(m.repository.Owner.UserName,
+		m.repository.Name,
+		issue.Index,
+		gitea.CreateIssueCommentOption{
+			Body: fmt.Sprintf("Author: @%s Posted at: %s\n\n\n%s", *gic.User.Login, gic.GetCreatedAt().Format("02.01.2006 15:04"), gic.GetBody()),
+		})
+}

migrations/migratory.go

@@ -0,0 +1,18 @@
+package migrations
+
+import "code.gitea.io/sdk/gitea"
+
+type Migratory struct {
+	Client       *gitea.Client
+	AuthUsername string
+	AuthPassword string
+
+	Private    bool
+	NewOwnerID int
+
+	repository *gitea.Repository
+	// key: github milestone id | value: gitea milestone id
+	migratedMilestones map[int64]int64
+	// key: github label id | value: gitea label id
+	migratedLabels map[int64]int64
+}

migrations/repo.go

@@ -0,0 +1,22 @@
+package migrations
+
+import (
+	"fmt"
+
+	"code.gitea.io/sdk/gitea"
+	"github.com/google/go-github/github"
+)
+
+func (m *Migratory) Repository(gr *github.Repository) (*gitea.Repository, error) {
+	var err error
+	m.repository, err = m.Client.MigrateRepo(gitea.MigrateRepoOption{
+		Description:  *gr.Description,
+		AuthPassword: m.AuthPassword,
+		AuthUsername: m.AuthUsername,
+		CloneAddr:    gr.GetCloneURL(),
+		RepoName:     gr.GetName(),
+		UID:          m.NewOwnerID,
+		Private:      m.Private,
+	})
+	return m.repository, err
+}