From 7114f1018b7c2ab48e009e4f066c2006d90964cb Mon Sep 17 00:00:00 2001 From: Jonas Franz Date: Mon, 23 Jul 2018 12:03:56 +0200 Subject: [PATCH] Add logging support Signed-off-by: Jonas Franz --- Gopkg.lock | 22 +++++++++++-- Gopkg.toml | 5 ++- Makefile | 7 ++-- cmd/migrate-all.go | 2 ++ cmd/migrate.go | 67 ++------------------------------------- cmd/web.go | 17 ++++++++-- commands.go | 13 ++++++++ commands_web.go | 14 ++++++++ config/config.go | 4 +++ main.go | 8 +---- migrations/github.go | 49 +++++++++++++++++++++++++++- migrations/github_test.go | 22 +++++++++++++ web/context/context.go | 4 +-- web/migration/repos.go | 10 ++++-- 14 files changed, 158 insertions(+), 86 deletions(-) create mode 100644 commands.go create mode 100644 commands_web.go diff --git a/Gopkg.lock b/Gopkg.lock index 60fcdcc..045e4aa 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -91,6 +91,12 @@ revision = "792786c7400a136282c1664665ae0a8db921c6c2" version = "v1.0.0" +[[projects]] + name = "github.com/sirupsen/logrus" + packages = ["."] + revision = "3e01752db0189b9157070a0e1668a620f9a85da2" + version = "v1.0.6" + [[projects]] name = "github.com/stretchr/testify" packages = ["assert"] @@ -106,7 +112,10 @@ [[projects]] branch = "master" name = "golang.org/x/crypto" - packages = ["pbkdf2"] + packages = [ + "pbkdf2", + "ssh/terminal" + ] revision = "a8fb68e7206f8c78be19b432c58eb52a6aa34462" [[projects]] @@ -128,6 +137,15 @@ ] revision = "1e0a3fa8ba9a5c9eb35c271780101fdaf1b205d7" +[[projects]] + branch = "master" + name = "golang.org/x/sys" + packages = [ + "unix", + "windows" + ] + revision = "ac767d655b305d4e9612f5f6e33120b9176c4ad4" + [[projects]] name = "google.golang.org/appengine" packages = [ @@ -163,6 +181,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "d73409e3342f71d7a737dd452af78159ec22f8ae128c1b009d7189b3e87f27fb" + inputs-digest = "0c4f08e8c96a1b866394a858e61289c637d938c3476b14446e483acd212ba06f" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 1389f5f..427343f 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -39,4 +39,7 @@ [[constraint]] branch = "master" - name = "github.com/go-macaron/binding" \ No newline at end of file + name = "github.com/go-macaron/binding" +[[constraint]] + name = "github.com/sirupsen/logrus" + version = "1.0.6" diff --git a/Makefile b/Makefile index afd337f..18315b2 100644 --- a/Makefile +++ b/Makefile @@ -21,6 +21,9 @@ all: docker-binary: go build -ldflags "$(LDFLAGS)" -o gitea-github-migrator +.PHONY: docker-binary-web + go build -ldflags "$(LDFLAGS)" -tags web -o gitea-github-migrator + .PHONY: generate-release-file generate-release-file: echo $(VERSION) > .version @@ -30,7 +33,7 @@ release: @hash gox > /dev/null 2>&1; if [ $$? -ne 0 ]; then \ $(GO) get -u github.com/mitchellh/gox; \ fi - gox -ldflags "$(LDFLAGS)" -output "releases/gitea-github-migrator_{{.OS}}_{{.Arch}}" + gox -ldflags "$(LDFLAGS)" -tags web -output "releases/gitea-github-migrator_{{.OS}}_{{.Arch}}" .PHONY: lint lint: @@ -45,5 +48,5 @@ vet: .PHONY: test test: lint vet - go test -cover ./... + go test -tags web -cover ./... \ No newline at end of file diff --git a/cmd/migrate-all.go b/cmd/migrate-all.go index c314f9a..40712a7 100644 --- a/cmd/migrate-all.go +++ b/cmd/migrate-all.go @@ -7,6 +7,7 @@ import ( "code.gitea.io/sdk/gitea" "git.jonasfranz.software/JonasFranzDEV/gitea-github-migrator/migrations" "github.com/google/go-github/github" + "github.com/sirupsen/logrus" "github.com/urfave/cli" "golang.org/x/oauth2" ) @@ -26,6 +27,7 @@ var CmdMigrateAll = cli.Command{ } func runMigrateAll(ctx *cli.Context) error { + logrus.SetLevel(logrus.InfoLevel) onlyRepos := ctx.Bool("only-repo") var gc *github.Client if ctx.IsSet("gh-token") { diff --git a/cmd/migrate.go b/cmd/migrate.go index 5b55b9b..e33fe56 100644 --- a/cmd/migrate.go +++ b/cmd/migrate.go @@ -7,6 +7,7 @@ import ( "code.gitea.io/sdk/gitea" "git.jonasfranz.software/JonasFranzDEV/gitea-github-migrator/migrations" "github.com/google/go-github/github" + "github.com/sirupsen/logrus" "github.com/urfave/cli" "golang.org/x/oauth2" ) @@ -38,7 +39,7 @@ func runMigrate(ctx *cli.Context) error { } else { gc = github.NewClient(nil) } - + logrus.SetLevel(logrus.InfoLevel) job := migrations.NewJob(&migrations.Options{ Private: ctx.Bool("private"), NewOwnerID: ctx.Int("owner"), @@ -65,67 +66,3 @@ func runMigrate(ctx *cli.Context) error { } return nil } - -// Deprecated: Please use Job or FetchMigratory instead -func migrate(c context.Context, gc *github.Client, m *migrations.Migratory, username, repo string, onlyRepo bool) error { - fmt.Printf("Fetching repository %s/%s...\n", username, repo) - gr, _, err := gc.Repositories.Get(c, username, repo) - if err != nil { - return fmt.Errorf("error while fetching repo[%s/%s]: %v", username, repo, err) - } - - fmt.Printf("Migrating repository %s/%s...\n", username, repo) - var mr *gitea.Repository - if mr, err = m.Repository(gr); err != nil { - return fmt.Errorf("error while migrating repo[%s/%s]: %v", username, repo, err) - } - fmt.Printf("Repository migrated to %s/%s\n", mr.Owner.UserName, mr.Name) - - if onlyRepo { - return nil - } - - fmt.Println("Fetching issues...") - opt := &github.IssueListByRepoOptions{ - Sort: "created", - Direction: "asc", - State: "all", - ListOptions: github.ListOptions{ - PerPage: 100, - }, - } - var allIssues = make([]*github.Issue, 0) - for { - issues, resp, err := gc.Issues.ListByRepo(c, username, repo, opt) - if err != nil { - return fmt.Errorf("error while listing repos: %v", err) - } - allIssues = append(allIssues, issues...) - if resp.NextPage == 0 { - break - } - opt.Page = resp.NextPage - } - fmt.Println("Migrating issues...") - for _, gi := range allIssues { - fmt.Printf("Migrating #%d...\n", *gi.Number) - issue, err := m.Issue(gi) - if err != nil { - return fmt.Errorf("migrating issue[id: %d]: %v", *gi.ID, err) - } - comments, _, err := gc.Issues.ListComments(c, username, repo, gi.GetNumber(), nil) - if err != nil { - return fmt.Errorf("fetching issue[id: %d] comments: %v", *gi.ID, err) - } - for _, gc := range comments { - fmt.Printf("-> %d...", gc.ID) - if _, err := m.IssueComment(issue, gc); err != nil { - return fmt.Errorf("migrating issue comment [issue: %d, comment: %d]: %v", *gi.ID, gc.ID, err) - } - fmt.Print("Done!\n") - } - fmt.Printf("Migrated #%d...\n", *gi.Number) - - } - return nil -} diff --git a/cmd/web.go b/cmd/web.go index 34bfc5b..611804c 100644 --- a/cmd/web.go +++ b/cmd/web.go @@ -1,3 +1,5 @@ +// +build web + package cmd import ( @@ -7,6 +9,7 @@ import ( "git.jonasfranz.software/JonasFranzDEV/gitea-github-migrator/config" "git.jonasfranz.software/JonasFranzDEV/gitea-github-migrator/web" "github.com/jinzhu/configor" + "github.com/sirupsen/logrus" "github.com/urfave/cli" ) @@ -23,7 +26,15 @@ func runWeb(_ *cli.Context) error { } r := web.InitRoutes() - fmt.Println("Server is running...") - // TODO add port / host to config - return http.ListenAndServe("0.0.0.0:4000", r) + hostname := config.Config.Web.Host + if len(hostname) == 0 { + hostname = "0.0.0.0" + } + port := config.Config.Web.Port + if port == 0 { + port = 4000 + } + logrus.Infof("Server is running at http://%s:%d", hostname, port) + logrus.SetLevel(logrus.PanicLevel) + return http.ListenAndServe(fmt.Sprintf("%s:%d", hostname, port), r) } diff --git a/commands.go b/commands.go new file mode 100644 index 0000000..bb93b9b --- /dev/null +++ b/commands.go @@ -0,0 +1,13 @@ +// +build !web + +package main + +import ( + "git.jonasfranz.software/JonasFranzDEV/gitea-github-migrator/cmd" + "github.com/urfave/cli" +) + +var cmds = cli.Commands{ + cmd.CmdMigrate, + cmd.CmdMigrateAll, +} diff --git a/commands_web.go b/commands_web.go new file mode 100644 index 0000000..1f7460f --- /dev/null +++ b/commands_web.go @@ -0,0 +1,14 @@ +// +build web + +package main + +import ( + "git.jonasfranz.software/JonasFranzDEV/gitea-github-migrator/cmd" + "github.com/urfave/cli" +) + +var cmds = cli.Commands{ + cmd.CmdMigrate, + cmd.CmdMigrateAll, + cmd.CmdWeb, +} diff --git a/config/config.go b/config/config.go index 2a2e8b3..2945105 100644 --- a/config/config.go +++ b/config/config.go @@ -6,5 +6,9 @@ var Config = struct { ClientID string `required:"true" yaml:"client_id"` ClientSecret string `required:"true" yaml:"client_secret"` } + Web struct { + Host string `yaml:"host"` + Port int `yaml:"port"` + } Secret string `required:"true"` }{} diff --git a/main.go b/main.go index 96e6707..ae6a25b 100644 --- a/main.go +++ b/main.go @@ -6,8 +6,6 @@ import ( "os" "github.com/urfave/cli" - - "git.jonasfranz.software/JonasFranzDEV/gitea-github-migrator/cmd" ) var ( @@ -21,11 +19,7 @@ func main() { app.Version = fmt.Sprintf("%s+%s", version, build) app.Usage = "GitHub to Gitea migrator for repositories" app.Description = `Migrate your GitHub repositories including issues to Gitea` - app.Commands = cli.Commands{ - cmd.CmdMigrate, - cmd.CmdMigrateAll, - cmd.CmdWeb, - } + app.Commands = cmds if err := app.Run(os.Args); err != nil { panic(err) } diff --git a/migrations/github.go b/migrations/github.go index 72431fd..45a7513 100644 --- a/migrations/github.go +++ b/migrations/github.go @@ -10,6 +10,7 @@ import ( "code.gitea.io/sdk/gitea" "github.com/google/go-github/github" + "github.com/sirupsen/logrus" ) // FetchMigratory adds GitHub fetching functions to migratory @@ -29,6 +30,9 @@ func (fm *FetchMigratory) MigrateFromGitHub() error { fm.Status = &MigratoryStatus{ Stage: Importing, } + logrus.WithFields(logrus.Fields{ + "repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName), + }).Info("migrating git repository") ghRepo, _, err := fm.GHClient.Repositories.Get(fm.ctx(), fm.RepoOwner, fm.RepoName) if err != nil { fm.Status.Stage = Failed @@ -41,6 +45,9 @@ func (fm *FetchMigratory) MigrateFromGitHub() error { fm.Status.FatalError = err return fmt.Errorf("Repository migration: %v", err) } + logrus.WithFields(logrus.Fields{ + "repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName), + }).Info("git repository migrated") if fm.Options.Issues || fm.Options.PullRequests { var commentsChan chan *[]*github.IssueComment if fm.Options.Comments { @@ -50,6 +57,9 @@ func (fm *FetchMigratory) MigrateFromGitHub() error { if err != nil { fm.Status.Stage = Failed fm.Status.FatalError = err + logrus.WithFields(logrus.Fields{ + "repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName), + }).Fatalf("migration failed: %v", fm.Status.FatalError) return err } fm.Status.Stage = Migrating @@ -61,10 +71,17 @@ func (fm *FetchMigratory) MigrateFromGitHub() error { migratedIssues[issue.GetNumber()], err = fm.Issue(issue) if err != nil { fm.Status.IssuesError++ - // TODO log errors + logrus.WithFields(logrus.Fields{ + "repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName), + "issue": issue.GetNumber(), + }).Warnf("error while migrating: %v", err) continue } fm.Status.IssuesMigrated++ + logrus.WithFields(logrus.Fields{ + "repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName), + "issue": issue.GetNumber(), + }).Info("issue migrated") } else { fm.Status.Issues-- } @@ -73,6 +90,10 @@ func (fm *FetchMigratory) MigrateFromGitHub() error { var comments []*github.IssueComment if cmts := <-commentsChan; cmts == nil { fm.Status.Stage = Failed + err := fmt.Errorf("error while fetching issue comments") + logrus.WithFields(logrus.Fields{ + "repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName), + }).Fatalf("migration failed: %v", fm.Status.FatalError) return err } else { comments = *cmts @@ -81,6 +102,9 @@ func (fm *FetchMigratory) MigrateFromGitHub() error { if err != nil { fm.Status.Stage = Failed fm.Status.FatalError = err + logrus.WithFields(logrus.Fields{ + "repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName), + }).Fatalf("migration failed: %v", fm.Status.FatalError) return err } fm.Status.Comments = int64(len(comments)) @@ -89,6 +113,11 @@ func (fm *FetchMigratory) MigrateFromGitHub() error { issueIndex, err := getIssueIndexFromHTMLURL(comment.GetHTMLURL()) if err != nil { fm.Status.CommentsError++ + logrus.WithFields(logrus.Fields{ + "repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName), + "issue": issueIndex, + "comment": comment.GetID(), + }).Warnf("error while migrating comment: %v", err) continue } if issue, ok := migratedIssues[issueIndex]; ok && issue != nil { @@ -109,10 +138,19 @@ func (fm *FetchMigratory) MigrateFromGitHub() error { for _, comm := range cs { if _, err := fm.IssueComment(i, comm); err != nil { fm.Status.CommentsError++ + logrus.WithFields(logrus.Fields{ + "repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName), + "comment": comm.GetID(), + }).Warnf("error while migrating comment: %v", err) continue } fm.Status.CommentsMigrated++ + logrus.WithFields(logrus.Fields{ + "repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName), + "comment": comm.GetID(), + }).Info("comment migrated") } + wg.Done() }(issue, comms) } wg.Wait() @@ -120,9 +158,15 @@ func (fm *FetchMigratory) MigrateFromGitHub() error { } if fm.Status.FatalError != nil { fm.Status.Stage = Failed + logrus.WithFields(logrus.Fields{ + "repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName), + }).Fatalf("migration failed: %v", fm.Status.FatalError) return nil } fm.Status.Stage = Finished + logrus.WithFields(logrus.Fields{ + "repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName), + }).Info("migration successful") return nil } @@ -208,6 +252,9 @@ func (fm *FetchMigratory) fetchCommentsAsync() chan *[]*github.IssueComment { if err != nil { f.Status.FatalError = err ret <- nil + logrus.WithFields(logrus.Fields{ + "repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName), + }).Fatalf("fetching comments failed: %v", fm.Status.FatalError) return } f.Status.Comments = int64(len(comments)) diff --git a/migrations/github_test.go b/migrations/github_test.go index c0a736f..5191e8e 100644 --- a/migrations/github_test.go +++ b/migrations/github_test.go @@ -2,6 +2,9 @@ package migrations import ( "testing" + + "github.com/google/go-github/github" + "github.com/stretchr/testify/assert" ) func BenchmarkGetIssueIndexFromHTMLURLAlt(b *testing.B) { @@ -15,3 +18,22 @@ func BenchmarkGetIssueIndexFromHTMLURL(b *testing.B) { getIssueIndexFromHTMLURL("https://github.com/octocat/Hello-World/issues/1347#issuecomment-1") } } + +var testFMig = &FetchMigratory{ + Migratory: *DemoMigratory, + GHClient: github.NewClient(nil), + RepoOwner: "JonasFranzDEV", + RepoName: "test", +} + +func TestFetchMigratory_FetchIssues(t *testing.T) { + issues, err := testFMig.FetchIssues() + assert.NoError(t, err) + assert.True(t, len(issues) > 0, "at least one issue found") +} + +func TestFetchMigratory_FetchComments(t *testing.T) { + comments, err := testFMig.FetchIssues() + assert.NoError(t, err) + assert.True(t, len(comments) > 0, "at least one comment found") +} diff --git a/web/context/context.go b/web/context/context.go index 19fdaca..0d8379d 100644 --- a/web/context/context.go +++ b/web/context/context.go @@ -38,13 +38,13 @@ type User struct { var runningJobs = make(map[string]*migrations.Job) // GetCurrentJob returns the current job of the user -// Bug(JonasFranzDEV): prevents saleability (FIXME) +// Bug(JonasFranzDEV): prevents scalability (FIXME) func (ctx *Context) GetCurrentJob() *migrations.Job { return runningJobs[ctx.Session.ID()] } // SetCurrentJob sets the current job of the user -// Bug(JonasFranzDEV): prevents saleability (FIXME) +// Bug(JonasFranzDEV): prevents scalability (FIXME) func (ctx *Context) SetCurrentJob(job *migrations.Job) { runningJobs[ctx.Session.ID()] = job } diff --git a/web/migration/repos.go b/web/migration/repos.go index 761b3d9..73f36f0 100644 --- a/web/migration/repos.go +++ b/web/migration/repos.go @@ -1,21 +1,25 @@ package migration import ( + bgctx "context" "regexp" "strings" "git.jonasfranz.software/JonasFranzDEV/gitea-github-migrator/migrations" "git.jonasfranz.software/JonasFranzDEV/gitea-github-migrator/web/context" "github.com/google/go-github/github" - - bgctx "context" ) const repoRegex = "^[A-Za-z0-9-.]+/[A-Za-z0-9-.]+$" // ListRepos shows all available repos of the signed in user func ListRepos(ctx *context.Context) { - repos, _, err := ctx.Client.Repositories.List(bgctx.Background(), ctx.User.Username, &github.RepositoryListOptions{}) + repos, _, err := ctx.Client.Repositories.List(bgctx.Background(), ctx.User.Username, &github.RepositoryListOptions{ + ListOptions: github.ListOptions{ + PerPage: 100, + }, + }) + if err != nil { ctx.Handle(500, "list repositories", err) return