From f2c99b3ae2f015cea75e345082be4044e1916471 Mon Sep 17 00:00:00 2001 From: Jonas Franz Date: Tue, 11 Sep 2018 19:51:10 +0000 Subject: [PATCH] Add web log (#27) --- cmd/web.go | 1 - main.go | 4 ++-- migrations/github.go | 40 +++++++++++++++++++---------------- migrations/job.go | 27 ++++++++++++++++++----- migrations/migratory.go | 1 + web/public/js/repos-status.js | 4 ++++ web/templates/base/head.tmpl | 2 +- web/templates/migration.tmpl | 15 ++++++++++++- 8 files changed, 66 insertions(+), 28 deletions(-) diff --git a/cmd/web.go b/cmd/web.go index 43fd810..f8d30cb 100644 --- a/cmd/web.go +++ b/cmd/web.go @@ -43,6 +43,5 @@ func runWeb(ctx *cli.Context) error { port = 4000 } logrus.Infof("Server is running at http://%s:%d", hostname, port) - logrus.SetLevel(logrus.PanicLevel) return http.ListenAndServe(fmt.Sprintf("%s:%d", hostname, port), r) } diff --git a/main.go b/main.go index ae6a25b..c5983d8 100644 --- a/main.go +++ b/main.go @@ -1,10 +1,10 @@ -//go:generate swagger generate spec -i ./swagger.yml -o ./swagger.json package main import ( "fmt" "os" + "github.com/sirupsen/logrus" "github.com/urfave/cli" ) @@ -21,6 +21,6 @@ func main() { app.Description = `Migrate your GitHub repositories including issues to Gitea` app.Commands = cmds if err := app.Run(os.Args); err != nil { - panic(err) + logrus.Panic(err) } } diff --git a/migrations/github.go b/migrations/github.go index 45a7513..fac894d 100644 --- a/migrations/github.go +++ b/migrations/github.go @@ -1,6 +1,7 @@ package migrations import ( + "bytes" "context" "fmt" "regexp" @@ -19,6 +20,8 @@ type FetchMigratory struct { GHClient *github.Client RepoOwner string RepoName string + Logger *logrus.Logger + LogOutput *bytes.Buffer } func (fm *FetchMigratory) ctx() context.Context { @@ -30,7 +33,8 @@ func (fm *FetchMigratory) MigrateFromGitHub() error { fm.Status = &MigratoryStatus{ Stage: Importing, } - logrus.WithFields(logrus.Fields{ + + fm.Logger.WithFields(logrus.Fields{ "repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName), }).Info("migrating git repository") ghRepo, _, err := fm.GHClient.Repositories.Get(fm.ctx(), fm.RepoOwner, fm.RepoName) @@ -45,7 +49,7 @@ func (fm *FetchMigratory) MigrateFromGitHub() error { fm.Status.FatalError = err return fmt.Errorf("Repository migration: %v", err) } - logrus.WithFields(logrus.Fields{ + fm.Logger.WithFields(logrus.Fields{ "repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName), }).Info("git repository migrated") if fm.Options.Issues || fm.Options.PullRequests { @@ -57,9 +61,9 @@ func (fm *FetchMigratory) MigrateFromGitHub() error { if err != nil { fm.Status.Stage = Failed fm.Status.FatalError = err - logrus.WithFields(logrus.Fields{ + fm.Logger.WithFields(logrus.Fields{ "repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName), - }).Fatalf("migration failed: %v", fm.Status.FatalError) + }).Errorf("migration failed: %v", fm.Status.FatalError) return err } fm.Status.Stage = Migrating @@ -71,14 +75,14 @@ func (fm *FetchMigratory) MigrateFromGitHub() error { migratedIssues[issue.GetNumber()], err = fm.Issue(issue) if err != nil { fm.Status.IssuesError++ - logrus.WithFields(logrus.Fields{ + fm.Logger.WithFields(logrus.Fields{ "repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName), "issue": issue.GetNumber(), }).Warnf("error while migrating: %v", err) continue } fm.Status.IssuesMigrated++ - logrus.WithFields(logrus.Fields{ + fm.Logger.WithFields(logrus.Fields{ "repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName), "issue": issue.GetNumber(), }).Info("issue migrated") @@ -91,9 +95,9 @@ func (fm *FetchMigratory) MigrateFromGitHub() error { if cmts := <-commentsChan; cmts == nil { fm.Status.Stage = Failed err := fmt.Errorf("error while fetching issue comments") - logrus.WithFields(logrus.Fields{ + fm.Logger.WithFields(logrus.Fields{ "repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName), - }).Fatalf("migration failed: %v", fm.Status.FatalError) + }).Errorf("migration failed: %v", fm.Status.FatalError) return err } else { comments = *cmts @@ -102,9 +106,9 @@ func (fm *FetchMigratory) MigrateFromGitHub() error { if err != nil { fm.Status.Stage = Failed fm.Status.FatalError = err - logrus.WithFields(logrus.Fields{ + fm.Logger.WithFields(logrus.Fields{ "repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName), - }).Fatalf("migration failed: %v", fm.Status.FatalError) + }).Errorf("migration failed: %v", fm.Status.FatalError) return err } fm.Status.Comments = int64(len(comments)) @@ -113,7 +117,7 @@ func (fm *FetchMigratory) MigrateFromGitHub() error { issueIndex, err := getIssueIndexFromHTMLURL(comment.GetHTMLURL()) if err != nil { fm.Status.CommentsError++ - logrus.WithFields(logrus.Fields{ + fm.Logger.WithFields(logrus.Fields{ "repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName), "issue": issueIndex, "comment": comment.GetID(), @@ -138,14 +142,14 @@ func (fm *FetchMigratory) MigrateFromGitHub() error { for _, comm := range cs { if _, err := fm.IssueComment(i, comm); err != nil { fm.Status.CommentsError++ - logrus.WithFields(logrus.Fields{ + fm.Logger.WithFields(logrus.Fields{ "repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName), "comment": comm.GetID(), }).Warnf("error while migrating comment: %v", err) continue } fm.Status.CommentsMigrated++ - logrus.WithFields(logrus.Fields{ + fm.Logger.WithFields(logrus.Fields{ "repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName), "comment": comm.GetID(), }).Info("comment migrated") @@ -158,13 +162,13 @@ func (fm *FetchMigratory) MigrateFromGitHub() error { } if fm.Status.FatalError != nil { fm.Status.Stage = Failed - logrus.WithFields(logrus.Fields{ + fm.Logger.WithFields(logrus.Fields{ "repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName), - }).Fatalf("migration failed: %v", fm.Status.FatalError) + }).Errorf("migration failed: %v", fm.Status.FatalError) return nil } fm.Status.Stage = Finished - logrus.WithFields(logrus.Fields{ + fm.Logger.WithFields(logrus.Fields{ "repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName), }).Info("migration successful") return nil @@ -252,9 +256,9 @@ func (fm *FetchMigratory) fetchCommentsAsync() chan *[]*github.IssueComment { if err != nil { f.Status.FatalError = err ret <- nil - logrus.WithFields(logrus.Fields{ + fm.Logger.WithFields(logrus.Fields{ "repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName), - }).Fatalf("fetching comments failed: %v", fm.Status.FatalError) + }).Errorf("fetching comments failed: %v", fm.Status.FatalError) return } f.Status.Comments = int64(len(comments)) diff --git a/migrations/job.go b/migrations/job.go index 3145075..19a9c96 100644 --- a/migrations/job.go +++ b/migrations/job.go @@ -1,11 +1,13 @@ package migrations import ( + "bytes" "fmt" "strings" "code.gitea.io/sdk/gitea" "github.com/google/go-github/github" + "github.com/sirupsen/logrus" ) // Job manages all migrations of a "migartion job" @@ -14,8 +16,9 @@ type Job struct { Options *Options Client *gitea.Client GHClient *github.Client + UseStdErr bool - migratories map[string]*Migratory + migratories map[string]*FetchMigratory } // JobReport represents the current status of a Job @@ -41,6 +44,7 @@ func (job *Job) StatusReport() *JobReport { } for _, repo := range job.Repositories { if migratory, ok := job.migratories[repo]; ok { + migratory.Status.Log = migratory.LogOutput.String() switch migratory.Status.Stage { case Finished: report.Finished[repo] = migratory.Status @@ -70,10 +74,10 @@ func (job *Job) StartMigration() chan error { close(errs) } } - job.migratories = make(map[string]*Migratory, pendingRepos) + job.migratories = make(map[string]*FetchMigratory, pendingRepos) for _, repo := range job.Repositories { mig, err := job.initFetchMigratory(repo) - job.migratories[repo] = &mig.Migratory + job.migratories[repo] = mig if err != nil { mig.Status = &MigratoryStatus{ Stage: Failed, @@ -97,7 +101,7 @@ func (job *Job) initFetchMigratory(repo string) (*FetchMigratory, error) { if len(res) != 2 { return nil, fmt.Errorf("invalid repo name: %s", repo) } - return &FetchMigratory{ + fm := &FetchMigratory{ Migratory: Migratory{ Client: job.Client, Options: *job.Options, @@ -105,7 +109,20 @@ func (job *Job) initFetchMigratory(repo string) (*FetchMigratory, error) { RepoName: res[1], RepoOwner: res[0], GHClient: job.GHClient, - }, nil + Logger: logrus.New(), + LogOutput: new(bytes.Buffer), + } + if !job.UseStdErr { + fm.Logger.Formatter = &logrus.TextFormatter{ + DisableColors: true, + DisableLevelTruncation: true, + DisableTimestamp: true, + } + fm.Logger.SetOutput(fm.LogOutput) + } else { + fm.LogOutput = nil + } + return fm, nil } // Finished indicates if the job is finished or not diff --git a/migrations/migratory.go b/migrations/migratory.go index 9902734..9c5f10f 100644 --- a/migrations/migratory.go +++ b/migrations/migratory.go @@ -44,4 +44,5 @@ type MigratoryStatus struct { // FatalError should only be used if stage == failed; indicates which fatal error occurred FatalError error + Log string `json:"log"` } diff --git a/web/public/js/repos-status.js b/web/public/js/repos-status.js index 2098822..2acb8a6 100644 --- a/web/public/js/repos-status.js +++ b/web/public/js/repos-status.js @@ -51,6 +51,8 @@ function handleData(data) { } content.find(".comment-progress").progress('set progress', report.migrated_comments + report.failed_comments); content.find(".issue-progress").progress('set progress', report.migrated_issues + report.failed_issues); + content.find('.log-content').val(report.log); + content.find('.log-content').scrollTop(content.find('.log-content')[0].scrollHeight) }); forEach(data.finished, function (repo, report) { var content = handleNonPending(repo, report); @@ -64,6 +66,7 @@ function handleData(data) { content.find(".issue-progress").progress('set progress', report.migrated_issues + report.failed_issues); content.find(".issue-progress").progress('complete'); content.find(".comment-progress").progress('complete'); + content.find('.log-content').val(report.log); }); } function forEach(object, callback) { @@ -78,6 +81,7 @@ function handleNonPending(repo, report) { var content = contentFromRepo(repo); if(!content.hasClass("non-pending")) { content.html(renderNonPending().html()); + content.find(".accordion").accordion(); content.find(".issue-progress").progress({ text: { active : 'Migrated {value} of {total} issues', diff --git a/web/templates/base/head.tmpl b/web/templates/base/head.tmpl index 06189d8..1447c2d 100644 --- a/web/templates/base/head.tmpl +++ b/web/templates/base/head.tmpl @@ -1,5 +1,5 @@ - + diff --git a/web/templates/migration.tmpl b/web/templates/migration.tmpl index 0f3baef..a4384c1 100644 --- a/web/templates/migration.tmpl +++ b/web/templates/migration.tmpl @@ -14,7 +14,7 @@
Migrating repositories
-
+
{{range .StatusReport.Pending}}
@@ -43,6 +43,19 @@
Migrating comments

0 migration(s) of comments failed

+
+
+ + View Log +
+
+
+
+ +
+
+
+