Add logging support

Signed-off-by: Jonas Franz <info@jonasfranz.software>

Gopkg.lock

@@ -91,6 +91,12 @@
   revision = "792786c7400a136282c1664665ae0a8db921c6c2"
   version = "v1.0.0"
 
+[[projects]]
+  name = "github.com/sirupsen/logrus"
+  packages = ["."]
+  revision = "3e01752db0189b9157070a0e1668a620f9a85da2"
+  version = "v1.0.6"
+
 [[projects]]
   name = "github.com/stretchr/testify"
   packages = ["assert"]
@@ -106,7 +112,10 @@
 [[projects]]
   branch = "master"
   name = "golang.org/x/crypto"
-  packages = ["pbkdf2"]
+  packages = [
+    "pbkdf2",
+    "ssh/terminal"
+  ]
   revision = "a8fb68e7206f8c78be19b432c58eb52a6aa34462"
 
 [[projects]]
@@ -128,6 +137,15 @@
   ]
   revision = "1e0a3fa8ba9a5c9eb35c271780101fdaf1b205d7"
 
+[[projects]]
+  branch = "master"
+  name = "golang.org/x/sys"
+  packages = [
+    "unix",
+    "windows"
+  ]
+  revision = "ac767d655b305d4e9612f5f6e33120b9176c4ad4"
+
 [[projects]]
   name = "google.golang.org/appengine"
   packages = [
@@ -163,6 +181,6 @@
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "d73409e3342f71d7a737dd452af78159ec22f8ae128c1b009d7189b3e87f27fb"
+  inputs-digest = "0c4f08e8c96a1b866394a858e61289c637d938c3476b14446e483acd212ba06f"
   solver-name = "gps-cdcl"
   solver-version = 1

Gopkg.toml

@@ -40,3 +40,6 @@
 [[constraint]]
   branch = "master"
   name = "github.com/go-macaron/binding"
+[[constraint]]
+  name = "github.com/sirupsen/logrus"
+  version = "1.0.6"

Makefile

@@ -21,6 +21,9 @@ all:
 docker-binary:
 	go build -ldflags "$(LDFLAGS)" -o gitea-github-migrator
 
+.PHONY: docker-binary-web
+	go build -ldflags "$(LDFLAGS)" -tags web -o gitea-github-migrator
+
 .PHONY: generate-release-file
 generate-release-file:
 	echo $(VERSION) > .version
@@ -30,7 +33,7 @@ release:
 	@hash gox > /dev/null 2>&1; if [ $$? -ne 0 ]; then \
 		$(GO) get -u github.com/mitchellh/gox; \
 	fi
-	gox -ldflags "$(LDFLAGS)" -output "releases/gitea-github-migrator_{{.OS}}_{{.Arch}}"
+	gox -ldflags "$(LDFLAGS)" -tags web -output "releases/gitea-github-migrator_{{.OS}}_{{.Arch}}"
 
 .PHONY: lint
 lint:
@@ -45,5 +48,5 @@ vet:
 
 .PHONY: test
 test: lint vet
-	go test -cover ./...
+	go test -tags web -cover ./...
 	

cmd/migrate-all.go

@@ -7,6 +7,7 @@ import (
 	"code.gitea.io/sdk/gitea"
 	"git.jonasfranz.software/JonasFranzDEV/gitea-github-migrator/migrations"
 	"github.com/google/go-github/github"
+	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 	"golang.org/x/oauth2"
 )
@@ -26,6 +27,7 @@ var CmdMigrateAll = cli.Command{
 }
 
 func runMigrateAll(ctx *cli.Context) error {
+	logrus.SetLevel(logrus.InfoLevel)
 	onlyRepos := ctx.Bool("only-repo")
 	var gc *github.Client
 	if ctx.IsSet("gh-token") {

cmd/migrate.go

@@ -7,6 +7,7 @@ import (
 	"code.gitea.io/sdk/gitea"
 	"git.jonasfranz.software/JonasFranzDEV/gitea-github-migrator/migrations"
 	"github.com/google/go-github/github"
+	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 	"golang.org/x/oauth2"
 )
@@ -38,7 +39,7 @@ func runMigrate(ctx *cli.Context) error {
 	} else {
 		gc = github.NewClient(nil)
 	}
-
+	logrus.SetLevel(logrus.InfoLevel)
 	job := migrations.NewJob(&migrations.Options{
 		Private:    ctx.Bool("private"),
 		NewOwnerID: ctx.Int("owner"),
@@ -65,67 +66,3 @@ func runMigrate(ctx *cli.Context) error {
 	}
 	return nil
 }
-
-// Deprecated: Please use Job or FetchMigratory instead
-func migrate(c context.Context, gc *github.Client, m *migrations.Migratory, username, repo string, onlyRepo bool) error {
-	fmt.Printf("Fetching repository %s/%s...\n", username, repo)
-	gr, _, err := gc.Repositories.Get(c, username, repo)
-	if err != nil {
-		return fmt.Errorf("error while fetching repo[%s/%s]: %v", username, repo, err)
-	}
-
-	fmt.Printf("Migrating repository %s/%s...\n", username, repo)
-	var mr *gitea.Repository
-	if mr, err = m.Repository(gr); err != nil {
-		return fmt.Errorf("error while migrating repo[%s/%s]: %v", username, repo, err)
-	}
-	fmt.Printf("Repository migrated to %s/%s\n", mr.Owner.UserName, mr.Name)
-
-	if onlyRepo {
-		return nil
-	}
-
-	fmt.Println("Fetching issues...")
-	opt := &github.IssueListByRepoOptions{
-		Sort:      "created",
-		Direction: "asc",
-		State:     "all",
-		ListOptions: github.ListOptions{
-			PerPage: 100,
-		},
-	}
-	var allIssues = make([]*github.Issue, 0)
-	for {
-		issues, resp, err := gc.Issues.ListByRepo(c, username, repo, opt)
-		if err != nil {
-			return fmt.Errorf("error while listing repos: %v", err)
-		}
-		allIssues = append(allIssues, issues...)
-		if resp.NextPage == 0 {
-			break
-		}
-		opt.Page = resp.NextPage
-	}
-	fmt.Println("Migrating issues...")
-	for _, gi := range allIssues {
-		fmt.Printf("Migrating #%d...\n", *gi.Number)
-		issue, err := m.Issue(gi)
-		if err != nil {
-			return fmt.Errorf("migrating issue[id: %d]: %v", *gi.ID, err)
-		}
-		comments, _, err := gc.Issues.ListComments(c, username, repo, gi.GetNumber(), nil)
-		if err != nil {
-			return fmt.Errorf("fetching issue[id: %d] comments: %v", *gi.ID, err)
-		}
-		for _, gc := range comments {
-			fmt.Printf("-> %d...", gc.ID)
-			if _, err := m.IssueComment(issue, gc); err != nil {
-				return fmt.Errorf("migrating issue comment [issue: %d, comment: %d]: %v", *gi.ID, gc.ID, err)
-			}
-			fmt.Print("Done!\n")
-		}
-		fmt.Printf("Migrated #%d...\n", *gi.Number)
-
-	}
-	return nil
-}

cmd/web.go

@@ -1,3 +1,5 @@
+// +build web
+
 package cmd
 
 import (
@@ -7,6 +9,7 @@ import (
 	"git.jonasfranz.software/JonasFranzDEV/gitea-github-migrator/config"
 	"git.jonasfranz.software/JonasFranzDEV/gitea-github-migrator/web"
 	"github.com/jinzhu/configor"
+	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
@@ -23,7 +26,15 @@ func runWeb(_ *cli.Context) error {
 	}
 	r := web.InitRoutes()
 
-	fmt.Println("Server is running...")
+	hostname := config.Config.Web.Host
-	// TODO add port / host to config
+	if len(hostname) == 0 {
-	return http.ListenAndServe("0.0.0.0:4000", r)
+		hostname = "0.0.0.0"
+	}
+	port := config.Config.Web.Port
+	if port == 0 {
+		port = 4000
+	}
+	logrus.Infof("Server is running at http://%s:%d", hostname, port)
+	logrus.SetLevel(logrus.PanicLevel)
+	return http.ListenAndServe(fmt.Sprintf("%s:%d", hostname, port), r)
 }

commands.go

@@ -0,0 +1,13 @@
+// +build !web
+
+package main
+
+import (
+	"git.jonasfranz.software/JonasFranzDEV/gitea-github-migrator/cmd"
+	"github.com/urfave/cli"
+)
+
+var cmds = cli.Commands{
+	cmd.CmdMigrate,
+	cmd.CmdMigrateAll,
+}

commands_web.go

@@ -0,0 +1,14 @@
+// +build web
+
+package main
+
+import (
+	"git.jonasfranz.software/JonasFranzDEV/gitea-github-migrator/cmd"
+	"github.com/urfave/cli"
+)
+
+var cmds = cli.Commands{
+	cmd.CmdMigrate,
+	cmd.CmdMigrateAll,
+	cmd.CmdWeb,
+}

config/config.go

@@ -6,5 +6,9 @@ var Config = struct {
 		ClientID     string `required:"true" yaml:"client_id"`
 		ClientSecret string `required:"true" yaml:"client_secret"`
 	}
+	Web struct {
+		Host string `yaml:"host"`
+		Port int    `yaml:"port"`
+	}
 	Secret string `required:"true"`
 }{}

main.go

@@ -6,8 +6,6 @@ import (
 	"os"
 
 	"github.com/urfave/cli"
-
-	"git.jonasfranz.software/JonasFranzDEV/gitea-github-migrator/cmd"
 )
 
 var (
@@ -21,11 +19,7 @@ func main() {
 	app.Version = fmt.Sprintf("%s+%s", version, build)
 	app.Usage = "GitHub to Gitea migrator for repositories"
 	app.Description = `Migrate your GitHub repositories including issues to Gitea`
-	app.Commands = cli.Commands{
+	app.Commands = cmds
-		cmd.CmdMigrate,
-		cmd.CmdMigrateAll,
-		cmd.CmdWeb,
-	}
 	if err := app.Run(os.Args); err != nil {
 		panic(err)
 	}

migrations/github.go

@@ -10,6 +10,7 @@ import (
 
 	"code.gitea.io/sdk/gitea"
 	"github.com/google/go-github/github"
+	"github.com/sirupsen/logrus"
 )
 
 // FetchMigratory adds GitHub fetching functions to migratory
@@ -29,6 +30,9 @@ func (fm *FetchMigratory) MigrateFromGitHub() error {
 	fm.Status = &MigratoryStatus{
 		Stage: Importing,
 	}
+	logrus.WithFields(logrus.Fields{
+		"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
+	}).Info("migrating git repository")
 	ghRepo, _, err := fm.GHClient.Repositories.Get(fm.ctx(), fm.RepoOwner, fm.RepoName)
 	if err != nil {
 		fm.Status.Stage = Failed
@@ -41,6 +45,9 @@ func (fm *FetchMigratory) MigrateFromGitHub() error {
 		fm.Status.FatalError = err
 		return fmt.Errorf("Repository migration: %v", err)
 	}
+	logrus.WithFields(logrus.Fields{
+		"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
+	}).Info("git repository migrated")
 	if fm.Options.Issues || fm.Options.PullRequests {
 		var commentsChan chan *[]*github.IssueComment
 		if fm.Options.Comments {
@@ -50,6 +57,9 @@ func (fm *FetchMigratory) MigrateFromGitHub() error {
 		if err != nil {
 			fm.Status.Stage = Failed
 			fm.Status.FatalError = err
+			logrus.WithFields(logrus.Fields{
+				"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
+			}).Fatalf("migration failed: %v", fm.Status.FatalError)
 			return err
 		}
 		fm.Status.Stage = Migrating
@@ -61,10 +71,17 @@ func (fm *FetchMigratory) MigrateFromGitHub() error {
 				migratedIssues[issue.GetNumber()], err = fm.Issue(issue)
 				if err != nil {
 					fm.Status.IssuesError++
-					// TODO log errors
+					logrus.WithFields(logrus.Fields{
+						"repo":  fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
+						"issue": issue.GetNumber(),
+					}).Warnf("error while migrating: %v", err)
 					continue
 				}
 				fm.Status.IssuesMigrated++
+				logrus.WithFields(logrus.Fields{
+					"repo":  fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
+					"issue": issue.GetNumber(),
+				}).Info("issue migrated")
 			} else {
 				fm.Status.Issues--
 			}
@@ -73,6 +90,10 @@ func (fm *FetchMigratory) MigrateFromGitHub() error {
 			var comments []*github.IssueComment
 			if cmts := <-commentsChan; cmts == nil {
 				fm.Status.Stage = Failed
+				err := fmt.Errorf("error while fetching issue comments")
+				logrus.WithFields(logrus.Fields{
+					"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
+				}).Fatalf("migration failed: %v", fm.Status.FatalError)
 				return err
 			} else {
 				comments = *cmts
@@ -81,6 +102,9 @@ func (fm *FetchMigratory) MigrateFromGitHub() error {
 			if err != nil {
 				fm.Status.Stage = Failed
 				fm.Status.FatalError = err
+				logrus.WithFields(logrus.Fields{
+					"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
+				}).Fatalf("migration failed: %v", fm.Status.FatalError)
 				return err
 			}
 			fm.Status.Comments = int64(len(comments))
@@ -89,6 +113,11 @@ func (fm *FetchMigratory) MigrateFromGitHub() error {
 				issueIndex, err := getIssueIndexFromHTMLURL(comment.GetHTMLURL())
 				if err != nil {
 					fm.Status.CommentsError++
+					logrus.WithFields(logrus.Fields{
+						"repo":    fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
+						"issue":   issueIndex,
+						"comment": comment.GetID(),
+					}).Warnf("error while migrating comment: %v", err)
 					continue
 				}
 				if issue, ok := migratedIssues[issueIndex]; ok && issue != nil {
@@ -109,10 +138,19 @@ func (fm *FetchMigratory) MigrateFromGitHub() error {
 					for _, comm := range cs {
 						if _, err := fm.IssueComment(i, comm); err != nil {
 							fm.Status.CommentsError++
+							logrus.WithFields(logrus.Fields{
+								"repo":    fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
+								"comment": comm.GetID(),
+							}).Warnf("error while migrating comment: %v", err)
 							continue
 						}
 						fm.Status.CommentsMigrated++
+						logrus.WithFields(logrus.Fields{
+							"repo":    fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
+							"comment": comm.GetID(),
+						}).Info("comment migrated")
 					}
+					wg.Done()
 				}(issue, comms)
 			}
 			wg.Wait()
@@ -120,9 +158,15 @@ func (fm *FetchMigratory) MigrateFromGitHub() error {
 	}
 	if fm.Status.FatalError != nil {
 		fm.Status.Stage = Failed
+		logrus.WithFields(logrus.Fields{
+			"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
+		}).Fatalf("migration failed: %v", fm.Status.FatalError)
 		return nil
 	}
 	fm.Status.Stage = Finished
+	logrus.WithFields(logrus.Fields{
+		"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
+	}).Info("migration successful")
 	return nil
 }
 
@@ -208,6 +252,9 @@ func (fm *FetchMigratory) fetchCommentsAsync() chan *[]*github.IssueComment {
 		if err != nil {
 			f.Status.FatalError = err
 			ret <- nil
+			logrus.WithFields(logrus.Fields{
+				"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
+			}).Fatalf("fetching comments failed: %v", fm.Status.FatalError)
 			return
 		}
 		f.Status.Comments = int64(len(comments))

migrations/github_test.go

@@ -2,6 +2,9 @@ package migrations
 
 import (
 	"testing"
+
+	"github.com/google/go-github/github"
+	"github.com/stretchr/testify/assert"
 )
 
 func BenchmarkGetIssueIndexFromHTMLURLAlt(b *testing.B) {
@@ -15,3 +18,22 @@ func BenchmarkGetIssueIndexFromHTMLURL(b *testing.B) {
 		getIssueIndexFromHTMLURL("https://github.com/octocat/Hello-World/issues/1347#issuecomment-1")
 	}
 }
+
+var testFMig = &FetchMigratory{
+	Migratory: *DemoMigratory,
+	GHClient:  github.NewClient(nil),
+	RepoOwner: "JonasFranzDEV",
+	RepoName:  "test",
+}
+
+func TestFetchMigratory_FetchIssues(t *testing.T) {
+	issues, err := testFMig.FetchIssues()
+	assert.NoError(t, err)
+	assert.True(t, len(issues) > 0, "at least one issue found")
+}
+
+func TestFetchMigratory_FetchComments(t *testing.T) {
+	comments, err := testFMig.FetchIssues()
+	assert.NoError(t, err)
+	assert.True(t, len(comments) > 0, "at least one comment found")
+}

web/context/context.go

@@ -38,13 +38,13 @@ type User struct {
 var runningJobs = make(map[string]*migrations.Job)
 
 // GetCurrentJob returns the current job of the user
-// Bug(JonasFranzDEV): prevents saleability (FIXME)
+// Bug(JonasFranzDEV): prevents scalability (FIXME)
 func (ctx *Context) GetCurrentJob() *migrations.Job {
 	return runningJobs[ctx.Session.ID()]
 }
 
 // SetCurrentJob sets the current job of the user
-// Bug(JonasFranzDEV): prevents saleability (FIXME)
+// Bug(JonasFranzDEV): prevents scalability (FIXME)
 func (ctx *Context) SetCurrentJob(job *migrations.Job) {
 	runningJobs[ctx.Session.ID()] = job
 }

web/migration/repos.go

@@ -1,21 +1,25 @@
 package migration
 
 import (
+	bgctx "context"
 	"regexp"
 	"strings"
 
 	"git.jonasfranz.software/JonasFranzDEV/gitea-github-migrator/migrations"
 	"git.jonasfranz.software/JonasFranzDEV/gitea-github-migrator/web/context"
 	"github.com/google/go-github/github"
-
-	bgctx "context"
 )
 
 const repoRegex = "^[A-Za-z0-9-.]+/[A-Za-z0-9-.]+$"
 
 // ListRepos shows all available repos of the signed in user
 func ListRepos(ctx *context.Context) {
-	repos, _, err := ctx.Client.Repositories.List(bgctx.Background(), ctx.User.Username, &github.RepositoryListOptions{})
+	repos, _, err := ctx.Client.Repositories.List(bgctx.Background(), ctx.User.Username, &github.RepositoryListOptions{
+		ListOptions: github.ListOptions{
+			PerPage: 100,
+		},
+	})
+
 	if err != nil {
 		ctx.Handle(500, "list repositories", err)
 		return