Add migrate-all

Add documentation
Add only-repo flag
Cleaned up dependencies

Signed-off-by: Jonas Franz <info@jonasfranz.software>

Gopkg.lock

@@ -7,6 +7,12 @@
   packages = ["gitea"]
   revision = "39c609e903992e25deca0e7aa2c5304fd680530f"
 
+[[projects]]
+  name = "github.com/golang/protobuf"
+  packages = ["proto"]
+  revision = "925541529c1fa6821df4e44ce2723319eb2be768"
+  version = "v1.0.0"
+
 [[projects]]
   name = "github.com/google/go-github"
   packages = ["github"]
@@ -19,12 +25,6 @@
   packages = ["query"]
   revision = "53e6ce116135b80d037921a7fdd5138cf32d7a8a"
 
-[[projects]]
-  name = "github.com/mattn/go-isatty"
-  packages = ["."]
-  revision = "0360b2af4f38e8d38c7fce2a9f4e702702d73a39"
-  version = "v0.0.3"
-
 [[projects]]
   name = "github.com/urfave/cli"
   packages = ["."]
@@ -32,24 +32,40 @@
   version = "v1.20.0"
 
 [[projects]]
-  name = "github.com/vbauerster/mpb"
+  branch = "master"
+  name = "golang.org/x/net"
   packages = [
-    ".",
+    "context",
-    "cwriter",
+    "context/ctxhttp"
-    "decor"
   ]
-  revision = "e227edc706423683a577a93c78beb0ce9d211b00"
+  revision = "24dd3780ca4f75fed9f321890729414a4b5d3f13"
-  version = "v3.1.1"
 
 [[projects]]
   branch = "master"
-  name = "golang.org/x/sys"
+  name = "golang.org/x/oauth2"
-  packages = ["unix"]
+  packages = [
-  revision = "01acb38716e021ed1fc03a602bdb5838e1358c5e"
+    ".",
+    "internal"
+  ]
+  revision = "fdc9e635145ae97e6c2cb777c48305600cf515cb"
+
+[[projects]]
+  name = "google.golang.org/appengine"
+  packages = [
+    "internal",
+    "internal/base",
+    "internal/datastore",
+    "internal/log",
+    "internal/remote_api",
+    "internal/urlfetch",
+    "urlfetch"
+  ]
+  revision = "150dc57a1b433e64154302bdc40b6bb8aefa313a"
+  version = "v1.0.0"
 
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "746c57ee0419dfecdb375bb45e31e55a9347b27d051af544d96397418e2d797d"
+  inputs-digest = "8d6cfc8199a1dde1c7b34a3771993afc73f7952a4b261f24939a63d0dce83b2e"
   solver-name = "gps-cdcl"
   solver-version = 1

Gopkg.toml

@@ -39,8 +39,4 @@
 
 [[constraint]]
   name = "github.com/google/go-github"
-  version = "15.0.0"
+  version = "15.0.0"
-
-[[constraint]]
-  name = "github.com/vbauerster/mpb"
-  version = "3.1.1"

README.md

@@ -1,3 +1,64 @@
 # gitea-github-migrator
 
-A tool to migrate GitHub Repositories to Gitea including all issues
+A tool to migrate [GitHub](https://github.com) Repositories to [Gitea](https://gitea.io) including all issues, labels, milestones
+and comments.
+
+## Features
+Migrates:
+
+- [x] Repositories
+- [x] Issues
+- [x] Labels
+- [x] Milestones
+- [x] Comments
+- [ ] Users
+- [ ] Pull Requests
+- [ ] Statuses
+
+## Installation
+
+```bash
+go get git.jonasfranz.software/JonasFranzDEV/gitea-github-migrator
+cd $GOPATH/src/git.jonasfranz.software/JonasFranzDEV/gitea-github-migrator
+dep ensure
+go install
+```
+
+## Usage
+
+Migrate one repository:
+```bash
+gitea-github-migrator migrate \
+    --gh-repo owner/reponame \
+    --gh-token GITHUB_TOKEN \
+    --url http://gitea-url.tdl \
+    --token GITEA_TOKEN \
+    --owner 1
+```
+`gh-token` is only required if you have more then 50 issues / repositories.
+
+Migrate all repositories:
+```bash
+gitea-github-migrator migrate-all \
+    --gh-user username \
+    --gh-token GITHUB_TOKEN \
+    --url http://gitea-url.tdl \
+    --token GITEA_TOKEN \
+    --owner 1
+```
+
+Migrate all repository without issues etc. (classic):
+```bash
+gitea-github-migrator migrate-all \
+    --gh-user username \
+    --gh-token GITHUB_TOKEN \
+    --url http://gitea-url.tdl \
+    --token GITEA_TOKEN \
+    --owner 1
+    --only-repo
+```
+
+## Problems
+* This migrator does not work with Gitea instances utilizing a SQLite database.
+* Comments / Issues will be added by token's user (information about date and author will be added)
+* Current Date is utilized for creation date (information about date is added in a comment)

cmd/migrate-all.go

@@ -0,0 +1,83 @@
+package cmd
+
+import (
+	"code.gitea.io/sdk/gitea"
+	"context"
+	"fmt"
+	"git.jonasfranz.software/JonasFranzDEV/gitea-github-migrator/migrations"
+	"github.com/google/go-github/github"
+	"github.com/urfave/cli"
+	"golang.org/x/oauth2"
+	"sync"
+)
+
+var CmdMigrateAll = cli.Command{
+	Name:   "migrate-all",
+	Usage:  "migrates all repositories of an user from github to a gitea repository",
+	Action: runMigrateAll,
+	Flags: append(migrateFlags,
+		cli.StringFlag{
+			Name:   "gh-user",
+			EnvVar: "GH_USER",
+			Usage:  "GitHub Username",
+		},
+	),
+}
+
+func runMigrateAll(ctx *cli.Context) error {
+	m := &migrations.Migratory{
+		Client:     gitea.NewClient(ctx.String("url"), ctx.String("token")),
+		Private:    ctx.Bool("private"),
+		NewOwnerID: ctx.Int("owner"),
+	}
+	c := context.Background()
+
+	var gc *github.Client
+	if ctx.IsSet("gh-token") {
+		ts := oauth2.StaticTokenSource(
+			&oauth2.Token{AccessToken: ctx.String("gh-token")},
+		)
+		tc := oauth2.NewClient(c, ts)
+		gc = github.NewClient(tc)
+	} else {
+		gc = github.NewClient(nil)
+	}
+
+	opt := &github.RepositoryListOptions{
+		ListOptions: github.ListOptions{PerPage: 100},
+	}
+	// get all pages of results
+	var allRepos []*github.Repository
+	for {
+		repos, resp, err := gc.Repositories.List(c, ctx.String("gh-user"), opt)
+		if err != nil {
+			return err
+		}
+		allRepos = append(allRepos, repos...)
+		if resp.NextPage == 0 {
+			break
+		}
+		opt.Page = resp.NextPage
+	}
+	errs := make(chan error, 1)
+
+	var wg sync.WaitGroup
+	wg.Add(len(allRepos))
+	for _, repo := range allRepos {
+		go func(r *github.Repository) {
+			defer wg.Done()
+			errs <- migrate(gc, c, m, r.Owner.GetLogin(), r.GetName(), ctx.Bool("only-repo"))
+		}(repo)
+	}
+
+	go func() {
+		for i := range errs {
+			if i != nil {
+				fmt.Printf("error: %v", i)
+			}
+		}
+	}()
+
+	wg.Wait()
+	return nil
+}

cmd/migrate.go

@@ -11,60 +11,79 @@ import (
 	"strings"
 )
 
+var migrateFlags = []cli.Flag{
+	cli.IntFlag{
+		Name:   "owner",
+		Usage:  "Owner ID",
+		EnvVar: "OWNER_ID",
+		Value:  0,
+	},
+	cli.StringFlag{
+		Name:   "token",
+		Usage:  "Gitea Token",
+		EnvVar: "GITEA_TOKEN",
+	},
+	cli.StringFlag{
+		Name:   "gh-token",
+		Usage:  "GitHub Token (optional)",
+		EnvVar: "GITHUB_TOKEN",
+	},
+	cli.StringFlag{
+		Name:   "url",
+		Usage:  "Gitea URL",
+		EnvVar: "GITEA_URL",
+	},
+	cli.BoolFlag{
+		Name:   "private",
+		Usage:  "should new repository be private",
+		EnvVar: "GITEA_PRIVATE",
+	},
+	cli.BoolFlag{
+		Name:   "only-repo",
+		Usage:  "skip issues etc. and only migrate repo",
+		EnvVar: "ONLY_REPO",
+	},
+}
+
 var CmdMigrate = cli.Command{
 	Name:   "migrate",
 	Usage:  "migrates a github to a gitea repository",
 	Action: runMigrate,
-	Flags: []cli.Flag{
+	Flags: append(migrateFlags,
 		cli.StringFlag{
 			Name:   "gh-repo",
 			Usage:  "GitHub Repository",
 			Value:  "username/reponame",
 			EnvVar: "GH_REPOSITORY",
 		},
-		cli.IntFlag{
+	),
-			Name:   "owner",
-			Usage:  "Owner ID",
-			EnvVar: "OWNER_ID",
-			Value:  0,
-		},
-		cli.StringFlag{
-			Name:   "token",
-			Usage:  "Gitea Token",
-			EnvVar: "GITEA_TOKEN",
-		},
-		cli.StringFlag{
-			Name:   "url",
-			Usage:  "Gitea URL",
-			EnvVar: "GITEA_URL",
-		},
-		cli.BoolFlag{
-			Name:   "private",
-			Usage:  "should new repository be private",
-			EnvVar: "GITEA_PRIVATE",
-		},
-	},
 }
 
 func runMigrate(ctx *cli.Context) error {
-	m := migrations.Migratory{
+	m := &migrations.Migratory{
 		Client:     gitea.NewClient(ctx.String("url"), ctx.String("token")),
 		Private:    ctx.Bool("private"),
 		NewOwnerID: ctx.Int("owner"),
 	}
 	c := context.Background()
-
+	var gc *github.Client
-	ts := oauth2.StaticTokenSource(
+	if ctx.IsSet("gh-token") {
-		&oauth2.Token{AccessToken: "246947e59029a71cb568e290cc6b28adeda514df"},
+		ts := oauth2.StaticTokenSource(
-	)
+			&oauth2.Token{AccessToken: ctx.String("gh-token")},
-	tc := oauth2.NewClient(c, ts)
+		)
-	gc := github.NewClient(tc)
+		tc := oauth2.NewClient(c, ts)
+		gc = github.NewClient(tc)
+	} else {
+		gc = github.NewClient(nil)
+	}
 
 	username := strings.Split(ctx.String("gh-repo"), "/")[0]
 	repo := strings.Split(ctx.String("gh-repo"), "/")[1]
 
-	//p := mpb.New()
+	return migrate(gc, c, m, username, repo, ctx.Bool("only-repo"))
+}
 
+func migrate(gc *github.Client, c context.Context, m *migrations.Migratory, username, repo string, onlyRepo bool) error {
 	fmt.Printf("Fetching repository %s/%s...\n", username, repo)
 	gr, _, err := gc.Repositories.Get(c, username, repo)
 	if err != nil {
@@ -76,6 +95,9 @@ func runMigrate(ctx *cli.Context) error {
 	} else {
 		fmt.Printf("Repository migrated to %s/%s\n", mr.Owner.UserName, mr.Name)
 	}
+	if onlyRepo {
+		return nil
+	}
 
 	fmt.Println("Fetching issues...")
 	opt := &github.IssueListByRepoOptions{

main.go

@@ -15,6 +15,7 @@ func main() {
 	app.Description = `Migrate your GitHub repositories including issues to Gitea`
 	app.Commands = cli.Commands{
 		cmd.CmdMigrate,
+		cmd.CmdMigrateAll,
 	}
 	if err := app.Run(os.Args); err != nil {
 		panic(err)

migrations/repo.go

@@ -8,7 +8,7 @@ import (
 func (m *Migratory) Repository(gr *github.Repository) (*gitea.Repository, error) {
 	var err error
 	m.repository, err = m.Client.MigrateRepo(gitea.MigrateRepoOption{
-		Description:  *gr.Description,
+		Description:  gr.GetDescription(),
 		AuthPassword: m.AuthPassword,
 		AuthUsername: m.AuthUsername,
 		CloneAddr:    gr.GetCloneURL(),