Browse Source

Add web log (#27)

pull/31/head
Jonas Franz 2 years ago
committed by Gitea
parent
commit
f2c99b3ae2
8 changed files with 66 additions and 28 deletions
  1. +0
    -1
      cmd/web.go
  2. +2
    -2
      main.go
  3. +22
    -18
      migrations/github.go
  4. +22
    -5
      migrations/job.go
  5. +1
    -0
      migrations/migratory.go
  6. +4
    -0
      web/public/js/repos-status.js
  7. +1
    -1
      web/templates/base/head.tmpl
  8. +14
    -1
      web/templates/migration.tmpl

+ 0
- 1
cmd/web.go View File

@@ -43,6 +43,5 @@ func runWeb(ctx *cli.Context) error {
port = 4000
}
logrus.Infof("Server is running at http://%s:%d", hostname, port)
logrus.SetLevel(logrus.PanicLevel)
return http.ListenAndServe(fmt.Sprintf("%s:%d", hostname, port), r)
}

+ 2
- 2
main.go View File

@@ -1,10 +1,10 @@
//go:generate swagger generate spec -i ./swagger.yml -o ./swagger.json
package main

import (
"fmt"
"os"

"github.com/sirupsen/logrus"
"github.com/urfave/cli"
)

@@ -21,6 +21,6 @@ func main() {
app.Description = `Migrate your GitHub repositories including issues to Gitea`
app.Commands = cmds
if err := app.Run(os.Args); err != nil {
panic(err)
logrus.Panic(err)
}
}

+ 22
- 18
migrations/github.go View File

@@ -1,6 +1,7 @@
package migrations

import (
"bytes"
"context"
"fmt"
"regexp"
@@ -19,6 +20,8 @@ type FetchMigratory struct {
GHClient *github.Client
RepoOwner string
RepoName string
Logger *logrus.Logger
LogOutput *bytes.Buffer
}

func (fm *FetchMigratory) ctx() context.Context {
@@ -30,7 +33,8 @@ func (fm *FetchMigratory) MigrateFromGitHub() error {
fm.Status = &MigratoryStatus{
Stage: Importing,
}
logrus.WithFields(logrus.Fields{

fm.Logger.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
}).Info("migrating git repository")
ghRepo, _, err := fm.GHClient.Repositories.Get(fm.ctx(), fm.RepoOwner, fm.RepoName)
@@ -45,7 +49,7 @@ func (fm *FetchMigratory) MigrateFromGitHub() error {
fm.Status.FatalError = err
return fmt.Errorf("Repository migration: %v", err)
}
logrus.WithFields(logrus.Fields{
fm.Logger.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
}).Info("git repository migrated")
if fm.Options.Issues || fm.Options.PullRequests {
@@ -57,9 +61,9 @@ func (fm *FetchMigratory) MigrateFromGitHub() error {
if err != nil {
fm.Status.Stage = Failed
fm.Status.FatalError = err
logrus.WithFields(logrus.Fields{
fm.Logger.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
}).Fatalf("migration failed: %v", fm.Status.FatalError)
}).Errorf("migration failed: %v", fm.Status.FatalError)
return err
}
fm.Status.Stage = Migrating
@@ -71,14 +75,14 @@ func (fm *FetchMigratory) MigrateFromGitHub() error {
migratedIssues[issue.GetNumber()], err = fm.Issue(issue)
if err != nil {
fm.Status.IssuesError++
logrus.WithFields(logrus.Fields{
fm.Logger.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
"issue": issue.GetNumber(),
}).Warnf("error while migrating: %v", err)
continue
}
fm.Status.IssuesMigrated++
logrus.WithFields(logrus.Fields{
fm.Logger.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
"issue": issue.GetNumber(),
}).Info("issue migrated")
@@ -91,9 +95,9 @@ func (fm *FetchMigratory) MigrateFromGitHub() error {
if cmts := <-commentsChan; cmts == nil {
fm.Status.Stage = Failed
err := fmt.Errorf("error while fetching issue comments")
logrus.WithFields(logrus.Fields{
fm.Logger.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
}).Fatalf("migration failed: %v", fm.Status.FatalError)
}).Errorf("migration failed: %v", fm.Status.FatalError)
return err
} else {
comments = *cmts
@@ -102,9 +106,9 @@ func (fm *FetchMigratory) MigrateFromGitHub() error {
if err != nil {
fm.Status.Stage = Failed
fm.Status.FatalError = err
logrus.WithFields(logrus.Fields{
fm.Logger.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
}).Fatalf("migration failed: %v", fm.Status.FatalError)
}).Errorf("migration failed: %v", fm.Status.FatalError)
return err
}
fm.Status.Comments = int64(len(comments))
@@ -113,7 +117,7 @@ func (fm *FetchMigratory) MigrateFromGitHub() error {
issueIndex, err := getIssueIndexFromHTMLURL(comment.GetHTMLURL())
if err != nil {
fm.Status.CommentsError++
logrus.WithFields(logrus.Fields{
fm.Logger.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
"issue": issueIndex,
"comment": comment.GetID(),
@@ -138,14 +142,14 @@ func (fm *FetchMigratory) MigrateFromGitHub() error {
for _, comm := range cs {
if _, err := fm.IssueComment(i, comm); err != nil {
fm.Status.CommentsError++
logrus.WithFields(logrus.Fields{
fm.Logger.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
"comment": comm.GetID(),
}).Warnf("error while migrating comment: %v", err)
continue
}
fm.Status.CommentsMigrated++
logrus.WithFields(logrus.Fields{
fm.Logger.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
"comment": comm.GetID(),
}).Info("comment migrated")
@@ -158,13 +162,13 @@ func (fm *FetchMigratory) MigrateFromGitHub() error {
}
if fm.Status.FatalError != nil {
fm.Status.Stage = Failed
logrus.WithFields(logrus.Fields{
fm.Logger.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
}).Fatalf("migration failed: %v", fm.Status.FatalError)
}).Errorf("migration failed: %v", fm.Status.FatalError)
return nil
}
fm.Status.Stage = Finished
logrus.WithFields(logrus.Fields{
fm.Logger.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
}).Info("migration successful")
return nil
@@ -252,9 +256,9 @@ func (fm *FetchMigratory) fetchCommentsAsync() chan *[]*github.IssueComment {
if err != nil {
f.Status.FatalError = err
ret <- nil
logrus.WithFields(logrus.Fields{
fm.Logger.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
}).Fatalf("fetching comments failed: %v", fm.Status.FatalError)
}).Errorf("fetching comments failed: %v", fm.Status.FatalError)
return
}
f.Status.Comments = int64(len(comments))


+ 22
- 5
migrations/job.go View File

@@ -1,11 +1,13 @@
package migrations

import (
"bytes"
"fmt"
"strings"

"code.gitea.io/sdk/gitea"
"github.com/google/go-github/github"
"github.com/sirupsen/logrus"
)

// Job manages all migrations of a "migartion job"
@@ -14,8 +16,9 @@ type Job struct {
Options *Options
Client *gitea.Client
GHClient *github.Client
UseStdErr bool

migratories map[string]*Migratory
migratories map[string]*FetchMigratory
}

// JobReport represents the current status of a Job
@@ -41,6 +44,7 @@ func (job *Job) StatusReport() *JobReport {
}
for _, repo := range job.Repositories {
if migratory, ok := job.migratories[repo]; ok {
migratory.Status.Log = migratory.LogOutput.String()
switch migratory.Status.Stage {
case Finished:
report.Finished[repo] = migratory.Status
@@ -70,10 +74,10 @@ func (job *Job) StartMigration() chan error {
close(errs)
}
}
job.migratories = make(map[string]*Migratory, pendingRepos)
job.migratories = make(map[string]*FetchMigratory, pendingRepos)
for _, repo := range job.Repositories {
mig, err := job.initFetchMigratory(repo)
job.migratories[repo] = &mig.Migratory
job.migratories[repo] = mig
if err != nil {
mig.Status = &MigratoryStatus{
Stage: Failed,
@@ -97,7 +101,7 @@ func (job *Job) initFetchMigratory(repo string) (*FetchMigratory, error) {
if len(res) != 2 {
return nil, fmt.Errorf("invalid repo name: %s", repo)
}
return &FetchMigratory{
fm := &FetchMigratory{
Migratory: Migratory{
Client: job.Client,
Options: *job.Options,
@@ -105,7 +109,20 @@ func (job *Job) initFetchMigratory(repo string) (*FetchMigratory, error) {
RepoName: res[1],
RepoOwner: res[0],
GHClient: job.GHClient,
}, nil
Logger: logrus.New(),
LogOutput: new(bytes.Buffer),
}
if !job.UseStdErr {
fm.Logger.Formatter = &logrus.TextFormatter{
DisableColors: true,
DisableLevelTruncation: true,
DisableTimestamp: true,
}
fm.Logger.SetOutput(fm.LogOutput)
} else {
fm.LogOutput = nil
}
return fm, nil
}

// Finished indicates if the job is finished or not


+ 1
- 0
migrations/migratory.go View File

@@ -44,4 +44,5 @@ type MigratoryStatus struct {

// FatalError should only be used if stage == failed; indicates which fatal error occurred
FatalError error
Log string `json:"log"`
}

+ 4
- 0
web/public/js/repos-status.js View File

@@ -51,6 +51,8 @@ function handleData(data) {
}
content.find(".comment-progress").progress('set progress', report.migrated_comments + report.failed_comments);
content.find(".issue-progress").progress('set progress', report.migrated_issues + report.failed_issues);
content.find('.log-content').val(report.log);
content.find('.log-content').scrollTop(content.find('.log-content')[0].scrollHeight)
});
forEach(data.finished, function (repo, report) {
var content = handleNonPending(repo, report);
@@ -64,6 +66,7 @@ function handleData(data) {
content.find(".issue-progress").progress('set progress', report.migrated_issues + report.failed_issues);
content.find(".issue-progress").progress('complete');
content.find(".comment-progress").progress('complete');
content.find('.log-content').val(report.log);
});
}
function forEach(object, callback) {
@@ -78,6 +81,7 @@ function handleNonPending(repo, report) {
var content = contentFromRepo(repo);
if(!content.hasClass("non-pending")) {
content.html(renderNonPending().html());
content.find(".accordion").accordion();
content.find(".issue-progress").progress({
text: {
active : 'Migrated {value} of {total} issues',


+ 1
- 1
web/templates/base/head.tmpl View File

@@ -1,5 +1,5 @@
<!DOCTYPE html>
<html>
<html lang="en">
<head>
<!-- Standard Meta -->
<meta charset="utf-8" />


+ 14
- 1
web/templates/migration.tmpl View File

@@ -14,7 +14,7 @@
<div class="bar"></div>
<div class="label">Migrating repositories</div>
</div>
<div class="ui three stackable cards" id="migration-list">
<div class="ui two stackable cards" id="migration-list">
{{range .StatusReport.Pending}}
<div class="ui repo-card card" data-repo="{{.}}" data-status="pending">
<div class="content">
@@ -43,6 +43,19 @@
<div class="label">Migrating comments</div>
</div>
<p><b class="failed-comments">0</b> migration(s) of comments failed</p>
<div class="ui accordion" onload="$(this).accordion();">
<div class="title">
<i class="dropdown icon"></i>
View Log
</div>
<div class="content">
<div class="ui form">
<div class="ui field">
<textarea title="Log content" class="log-content" readonly></textarea>
</div>
</div>
</div>
</div>
</div>
<div id="content-failed" style="display: none;">
<div class="ui negative message">


Loading…
Cancel
Save