Add web log (#27)

cmd/web.go

@@ -43,6 +43,5 @@ func runWeb(ctx *cli.Context) error {
 		port = 4000
 	}
 	logrus.Infof("Server is running at http://%s:%d", hostname, port)
-	logrus.SetLevel(logrus.PanicLevel)
 	return http.ListenAndServe(fmt.Sprintf("%s:%d", hostname, port), r)
 }

main.go

@@ -1,10 +1,10 @@
-//go:generate swagger generate spec -i ./swagger.yml -o ./swagger.json
 package main
 
 import (
 	"fmt"
 	"os"
 
+	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
@@ -21,6 +21,6 @@ func main() {
 	app.Description = `Migrate your GitHub repositories including issues to Gitea`
 	app.Commands = cmds
 	if err := app.Run(os.Args); err != nil {
-		panic(err)
+		logrus.Panic(err)
 	}
 }

migrations/github.go

@@ -1,6 +1,7 @@
 package migrations
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"regexp"
@@ -19,6 +20,8 @@ type FetchMigratory struct {
 	GHClient  *github.Client
 	RepoOwner string
 	RepoName  string
+	Logger    *logrus.Logger
+	LogOutput *bytes.Buffer
 }
 
 func (fm *FetchMigratory) ctx() context.Context {
@@ -30,7 +33,8 @@ func (fm *FetchMigratory) MigrateFromGitHub() error {
 	fm.Status = &MigratoryStatus{
 		Stage: Importing,
 	}
-	logrus.WithFields(logrus.Fields{
+
+	fm.Logger.WithFields(logrus.Fields{
 		"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
 	}).Info("migrating git repository")
 	ghRepo, _, err := fm.GHClient.Repositories.Get(fm.ctx(), fm.RepoOwner, fm.RepoName)
@@ -45,7 +49,7 @@ func (fm *FetchMigratory) MigrateFromGitHub() error {
 		fm.Status.FatalError = err
 		return fmt.Errorf("Repository migration: %v", err)
 	}
-	logrus.WithFields(logrus.Fields{
+	fm.Logger.WithFields(logrus.Fields{
 		"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
 	}).Info("git repository migrated")
 	if fm.Options.Issues || fm.Options.PullRequests {
@@ -57,9 +61,9 @@ func (fm *FetchMigratory) MigrateFromGitHub() error {
 		if err != nil {
 			fm.Status.Stage = Failed
 			fm.Status.FatalError = err
-			logrus.WithFields(logrus.Fields{
+			fm.Logger.WithFields(logrus.Fields{
 				"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
-			}).Fatalf("migration failed: %v", fm.Status.FatalError)
+			}).Errorf("migration failed: %v", fm.Status.FatalError)
 			return err
 		}
 		fm.Status.Stage = Migrating
@@ -71,14 +75,14 @@ func (fm *FetchMigratory) MigrateFromGitHub() error {
 				migratedIssues[issue.GetNumber()], err = fm.Issue(issue)
 				if err != nil {
 					fm.Status.IssuesError++
-					logrus.WithFields(logrus.Fields{
+					fm.Logger.WithFields(logrus.Fields{
 						"repo":  fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
 						"issue": issue.GetNumber(),
 					}).Warnf("error while migrating: %v", err)
 					continue
 				}
 				fm.Status.IssuesMigrated++
-				logrus.WithFields(logrus.Fields{
+				fm.Logger.WithFields(logrus.Fields{
 					"repo":  fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
 					"issue": issue.GetNumber(),
 				}).Info("issue migrated")
@@ -91,9 +95,9 @@ func (fm *FetchMigratory) MigrateFromGitHub() error {
 			if cmts := <-commentsChan; cmts == nil {
 				fm.Status.Stage = Failed
 				err := fmt.Errorf("error while fetching issue comments")
-				logrus.WithFields(logrus.Fields{
+				fm.Logger.WithFields(logrus.Fields{
 					"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
-				}).Fatalf("migration failed: %v", fm.Status.FatalError)
+				}).Errorf("migration failed: %v", fm.Status.FatalError)
 				return err
 			} else {
 				comments = *cmts
@@ -102,9 +106,9 @@ func (fm *FetchMigratory) MigrateFromGitHub() error {
 			if err != nil {
 				fm.Status.Stage = Failed
 				fm.Status.FatalError = err
-				logrus.WithFields(logrus.Fields{
+				fm.Logger.WithFields(logrus.Fields{
 					"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
-				}).Fatalf("migration failed: %v", fm.Status.FatalError)
+				}).Errorf("migration failed: %v", fm.Status.FatalError)
 				return err
 			}
 			fm.Status.Comments = int64(len(comments))
@@ -113,7 +117,7 @@ func (fm *FetchMigratory) MigrateFromGitHub() error {
 				issueIndex, err := getIssueIndexFromHTMLURL(comment.GetHTMLURL())
 				if err != nil {
 					fm.Status.CommentsError++
-					logrus.WithFields(logrus.Fields{
+					fm.Logger.WithFields(logrus.Fields{
 						"repo":    fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
 						"issue":   issueIndex,
 						"comment": comment.GetID(),
@@ -138,14 +142,14 @@ func (fm *FetchMigratory) MigrateFromGitHub() error {
 					for _, comm := range cs {
 						if _, err := fm.IssueComment(i, comm); err != nil {
 							fm.Status.CommentsError++
-							logrus.WithFields(logrus.Fields{
+							fm.Logger.WithFields(logrus.Fields{
 								"repo":    fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
 								"comment": comm.GetID(),
 							}).Warnf("error while migrating comment: %v", err)
 							continue
 						}
 						fm.Status.CommentsMigrated++
-						logrus.WithFields(logrus.Fields{
+						fm.Logger.WithFields(logrus.Fields{
 							"repo":    fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
 							"comment": comm.GetID(),
 						}).Info("comment migrated")
@@ -158,13 +162,13 @@ func (fm *FetchMigratory) MigrateFromGitHub() error {
 	}
 	if fm.Status.FatalError != nil {
 		fm.Status.Stage = Failed
-		logrus.WithFields(logrus.Fields{
+		fm.Logger.WithFields(logrus.Fields{
 			"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
-		}).Fatalf("migration failed: %v", fm.Status.FatalError)
+		}).Errorf("migration failed: %v", fm.Status.FatalError)
 		return nil
 	}
 	fm.Status.Stage = Finished
-	logrus.WithFields(logrus.Fields{
+	fm.Logger.WithFields(logrus.Fields{
 		"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
 	}).Info("migration successful")
 	return nil
@@ -252,9 +256,9 @@ func (fm *FetchMigratory) fetchCommentsAsync() chan *[]*github.IssueComment {
 		if err != nil {
 			f.Status.FatalError = err
 			ret <- nil
-			logrus.WithFields(logrus.Fields{
+			fm.Logger.WithFields(logrus.Fields{
 				"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
-			}).Fatalf("fetching comments failed: %v", fm.Status.FatalError)
+			}).Errorf("fetching comments failed: %v", fm.Status.FatalError)
 			return
 		}
 		f.Status.Comments = int64(len(comments))

migrations/job.go

@@ -1,11 +1,13 @@
 package migrations
 
 import (
+	"bytes"
 	"fmt"
 	"strings"
 
 	"code.gitea.io/sdk/gitea"
 	"github.com/google/go-github/github"
+	"github.com/sirupsen/logrus"
 )
 
 // Job manages all migrations of a "migartion job"
@@ -14,8 +16,9 @@ type Job struct {
 	Options      *Options
 	Client       *gitea.Client
 	GHClient     *github.Client
+	UseStdErr    bool
 
-	migratories map[string]*Migratory
+	migratories map[string]*FetchMigratory
 }
 
 // JobReport represents the current status of a Job
@@ -41,6 +44,7 @@ func (job *Job) StatusReport() *JobReport {
 	}
 	for _, repo := range job.Repositories {
 		if migratory, ok := job.migratories[repo]; ok {
+			migratory.Status.Log = migratory.LogOutput.String()
 			switch migratory.Status.Stage {
 			case Finished:
 				report.Finished[repo] = migratory.Status
@@ -70,10 +74,10 @@ func (job *Job) StartMigration() chan error {
 			close(errs)
 		}
 	}
-	job.migratories = make(map[string]*Migratory, pendingRepos)
+	job.migratories = make(map[string]*FetchMigratory, pendingRepos)
 	for _, repo := range job.Repositories {
 		mig, err := job.initFetchMigratory(repo)
-		job.migratories[repo] = &mig.Migratory
+		job.migratories[repo] = mig
 		if err != nil {
 			mig.Status = &MigratoryStatus{
 				Stage:      Failed,
@@ -97,7 +101,7 @@ func (job *Job) initFetchMigratory(repo string) (*FetchMigratory, error) {
 	if len(res) != 2 {
 		return nil, fmt.Errorf("invalid repo name: %s", repo)
 	}
-	return &FetchMigratory{
+	fm := &FetchMigratory{
 		Migratory: Migratory{
 			Client:  job.Client,
 			Options: *job.Options,
@@ -105,7 +109,20 @@ func (job *Job) initFetchMigratory(repo string) (*FetchMigratory, error) {
 		RepoName:  res[1],
 		RepoOwner: res[0],
 		GHClient:  job.GHClient,
-	}, nil
+		Logger:    logrus.New(),
+		LogOutput: new(bytes.Buffer),
+	}
+	if !job.UseStdErr {
+		fm.Logger.Formatter = &logrus.TextFormatter{
+			DisableColors:          true,
+			DisableLevelTruncation: true,
+			DisableTimestamp:       true,
+		}
+		fm.Logger.SetOutput(fm.LogOutput)
+	} else {
+		fm.LogOutput = nil
+	}
+	return fm, nil
 }
 
 // Finished indicates if the job is finished or not

migrations/migratory.go

@@ -44,4 +44,5 @@ type MigratoryStatus struct {
 
 	// FatalError should only be used if stage == failed; indicates which fatal error occurred
 	FatalError error
+	Log        string `json:"log"`
 }

web/public/js/repos-status.js

@@ -51,6 +51,8 @@ function handleData(data) {
         }
         content.find(".comment-progress").progress('set progress', report.migrated_comments + report.failed_comments);
         content.find(".issue-progress").progress('set progress', report.migrated_issues + report.failed_issues);
+        content.find('.log-content').val(report.log);
+        content.find('.log-content').scrollTop(content.find('.log-content')[0].scrollHeight)
     });
     forEach(data.finished, function (repo, report) {
         var content = handleNonPending(repo, report);
@@ -64,6 +66,7 @@ function handleData(data) {
         content.find(".issue-progress").progress('set progress', report.migrated_issues + report.failed_issues);
         content.find(".issue-progress").progress('complete');
         content.find(".comment-progress").progress('complete');
+        content.find('.log-content').val(report.log);
     });
 }
 function forEach(object, callback) {
@@ -78,6 +81,7 @@ function handleNonPending(repo, report) {
     var content = contentFromRepo(repo);
     if(!content.hasClass("non-pending")) {
         content.html(renderNonPending().html());
+        content.find(".accordion").accordion();
         content.find(".issue-progress").progress({
             text: {
                 active  : 'Migrated {value} of {total} issues',

web/templates/base/head.tmpl

@@ -1,5 +1,5 @@
 <!DOCTYPE html>
-<html>
+<html lang="en">
 <head>
     <!-- Standard Meta -->
     <meta charset="utf-8" />

web/templates/migration.tmpl

@@ -14,7 +14,7 @@
             <div class="bar"></div>
             <div class="label">Migrating repositories</div>
         </div>
-        <div class="ui three stackable cards" id="migration-list">
+        <div class="ui two stackable cards" id="migration-list">
         {{range .StatusReport.Pending}}
             <div class="ui repo-card card" data-repo="{{.}}" data-status="pending">
                 <div class="content">
@@ -43,6 +43,19 @@
         <div class="label">Migrating comments</div>
     </div>
     <p><b class="failed-comments">0</b> migration(s) of comments failed</p>
+    <div class="ui accordion" onload="$(this).accordion();">
+        <div class="title">
+            <i class="dropdown icon"></i>
+            View Log
+        </div>
+        <div class="content">
+            <div class="ui form">
+                <div class="ui field">
+                    <textarea title="Log content" class="log-content" readonly></textarea>
+                </div>
+            </div>
+        </div>
+    </div>
 </div>
 <div id="content-failed" style="display: none;">
     <div class="ui negative message">