Browse Source

Web interface implementation (#18)

master
Jonas Franz 2 years ago
committed by Gitea
parent
commit
2872647d3d
51 changed files with 2344 additions and 176 deletions
  1. +22
    -4
      .drone.yml
  2. +2
    -0
      .gitignore
  3. +4
    -1
      Dockerfile
  4. +25
    -0
      Dockerfile.web
  5. +104
    -7
      Gopkg.lock
  6. +27
    -0
      Gopkg.toml
  7. +1
    -1
      LICENSE
  8. +40
    -5
      Makefile
  9. +69
    -5
      README.md
  10. +28
    -33
      cmd/migrate-all.go
  11. +21
    -75
      cmd/migrate.go
  12. +0
    -23
      cmd/migrate_test.go
  13. +48
    -0
      cmd/web.go
  14. +13
    -0
      commands.go
  15. +14
    -0
      commands_web.go
  16. +8
    -0
      config.example.yml
  17. +13
    -0
      config/config.go
  18. +2
    -6
      main.go
  19. +264
    -0
      migrations/github.go
  20. +55
    -0
      migrations/github_test.go
  21. +9
    -5
      migrations/issue.go
  22. +2
    -2
      migrations/issue_test.go
  23. +114
    -0
      migrations/job.go
  24. +66
    -0
      migrations/job_test.go
  25. +33
    -5
      migrations/migratory.go
  26. +32
    -0
      migrations/options.go
  27. +18
    -4
      migrations/utils.go
  28. +50
    -0
      web/auth/gitea.go
  29. +68
    -0
      web/auth/github.go
  30. +115
    -0
      web/context/context.go
  31. +58
    -0
      web/fs.go
  32. +62
    -0
      web/migration/repos.go
  33. +14
    -0
      web/migration/status.go
  34. +364
    -0
      web/public/css/semantic.min.css
  35. +4
    -0
      web/public/js/jquery-3.1.1.min.js
  36. +121
    -0
      web/public/js/repos-status.js
  37. +30
    -0
      web/public/js/select-repos.js
  38. +11
    -0
      web/public/js/semantic.min.js
  39. +68
    -0
      web/router.go
  40. +2
    -0
      web/templates/base/footer.tmpl
  41. +34
    -0
      web/templates/base/head.tmpl
  42. +32
    -0
      web/templates/dashboard.tmpl
  43. +67
    -0
      web/templates/login_gitea.tmpl
  44. +22
    -0
      web/templates/login_github.tmpl
  45. +54
    -0
      web/templates/migration.tmpl
  46. +1
    -0
      web/templates/modules/username.tmpl
  47. +77
    -0
      web/templates/repos.tmpl
  48. +14
    -0
      web/templates/status/403.tmpl
  49. +14
    -0
      web/templates/status/404.tmpl
  50. +14
    -0
      web/templates/status/500.tmpl
  51. +14
    -0
      web/templates/status/unknown_error.tmpl

+ 22
- 4
.drone.yml View File

@@ -17,13 +17,15 @@ pipeline:
commands:
- go get -u github.com/golang/dep/cmd/dep
- dep ensure
- go get -u github.com/gobuffalo/packr/...
- packr -z
test:
image: golang:1.10
pull: true
environment:
GOPATH: /go
commands:
- make test
- make test build
static:
image: golang:1.10
pull: true
@@ -40,9 +42,16 @@ pipeline:
environment:
GOPATH: /go
commands:
- make generate-release-file release
- make generate-release-file release
when:
event: [ tag ]
clean:
image: golang:1.10
pull: true
environment:
GOPATH: /go
commands:
- packr clean
gitea:
image: plugins/gitea-release:latest
pull: true
@@ -61,11 +70,20 @@ pipeline:
image: plugins/docker:17.12
secrets: [ docker_username, docker_password ]
pull: true
repo: jonasfranz/gitea-github-migrator
repo: ggmigrator/cli
default_tags: true
when:
event: [ push, tag ]

docker-web:
image: plugins/docker:17.12
secrets: [ docker_username, docker_password ]
pull: true
dockerfile: Dockerfile.web
repo: ggmigrator/web
default_tags: true
when:
event: [ push, tag ]
branch: [ master ]
s3:
image: plugins/s3:1
pull: true


+ 2
- 0
.gitignore View File

@@ -10,7 +10,9 @@

# Output of the go coverage tool, specifically when used with LiteIDE
*.out
*-packr.go

# Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736
.glide/

data/

+ 4
- 1
Dockerfile View File

@@ -1,8 +1,11 @@
#Build stage
FROM golang:1.10-alpine3.7 AS build-env


ARG VERSION

#Build deps
RUN apk --no-cache add build-base git
RUN apk --no-cache add build-base git ca-certificates

#Setup repo
COPY . ${GOPATH}/src/git.jonasfranz.software/JonasFranzDEV/gitea-github-migrator


+ 25
- 0
Dockerfile.web View File

@@ -0,0 +1,25 @@
#Build stage
FROM golang:1.10-alpine3.7 AS build-env

ARG VERSION

#Build deps
RUN apk --no-cache add build-base git ca-certificates

#Setup repo
COPY . ${GOPATH}/src/git.jonasfranz.software/JonasFranzDEV/gitea-github-migrator
WORKDIR ${GOPATH}/src/git.jonasfranz.software/JonasFranzDEV/gitea-github-migrator

RUN make clean docker-binary-web

FROM alpine:3.7
LABEL maintainer="info@jonasfranz.software"

COPY --from=build-env /go/src/git.jonasfranz.software/JonasFranzDEV/gitea-github-migrator/gitea-github-migrator /usr/local/bin/gitea-github-migrator

VOLUME "/data"
VOLUME "/usr/local/bin/data"

COPY --from=build-env /go/src/git.jonasfranz.software/JonasFranzDEV/gitea-github-migrator/config.example.yml /data/config.example.yml

ENTRYPOINT ["/usr/local/bin/gitea-github-migrator", "web", "-c", "/data/config.yml"]

+ 104
- 7
Gopkg.lock View File

@@ -7,17 +7,59 @@
packages = ["gitea"]
revision = "79a281c4e34ae44cf96a23f0283729a074a6c2a0"

[[projects]]
name = "github.com/BurntSushi/toml"
packages = ["."]
revision = "b26d9c308763d68093482582cea63d69be07a0f0"
version = "v0.3.0"

[[projects]]
name = "github.com/Unknwon/com"
packages = ["."]
revision = "7677a1d7c1137cd3dd5ba7a076d0c898a1ef4520"
version = "master"

[[projects]]
branch = "master"
name = "github.com/adam-hanna/randomstrings"
packages = ["."]
revision = "88fd7c52a2c704c5d530718c1be454292a806e2b"

[[projects]]
name = "github.com/davecgh/go-spew"
packages = ["spew"]
revision = "346938d642f2ec3594ed81d874461961cd0faa76"
version = "v1.1.0"

[[projects]]
branch = "master"
name = "github.com/go-macaron/binding"
packages = ["."]
revision = "ac54ee249c27dca7e76fad851a4a04b73bd1b183"

[[projects]]
branch = "master"
name = "github.com/go-macaron/inject"
packages = ["."]
revision = "d8a0b8677191f4380287cfebd08e462217bac7ad"

[[projects]]
branch = "master"
name = "github.com/go-macaron/session"
packages = ["."]
revision = "b8e286a0dba8f4999042d6b258daf51b31d08938"

[[projects]]
name = "github.com/gobuffalo/packr"
packages = ["."]
revision = "bd47f2894846e32edcf9aa37290fef76c327883f"
version = "v1.11.1"

[[projects]]
name = "github.com/golang/protobuf"
packages = ["proto"]
revision = "925541529c1fa6821df4e44ce2723319eb2be768"
version = "v1.0.0"
revision = "b4deda0973fb4c70b50d226b1af49f3da59f5265"
version = "v1.1.0"

[[projects]]
name = "github.com/google/go-github"
@@ -31,12 +73,30 @@
packages = ["query"]
revision = "53e6ce116135b80d037921a7fdd5138cf32d7a8a"

[[projects]]
branch = "master"
name = "github.com/jinzhu/configor"
packages = ["."]
revision = "4edaf76fe18865e36de734d887f4c01bd05707af"

[[projects]]
name = "github.com/pkg/errors"
packages = ["."]
revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
version = "v0.8.0"

[[projects]]
name = "github.com/pmezard/go-difflib"
packages = ["difflib"]
revision = "792786c7400a136282c1664665ae0a8db921c6c2"
version = "v1.0.0"

[[projects]]
name = "github.com/sirupsen/logrus"
packages = ["."]
revision = "3e01752db0189b9157070a0e1668a620f9a85da2"
version = "v1.0.6"

[[projects]]
name = "github.com/stretchr/testify"
packages = ["assert"]
@@ -49,6 +109,15 @@
revision = "cfb38830724cc34fedffe9a2a29fb54fa9169cd1"
version = "v1.20.0"

[[projects]]
branch = "master"
name = "golang.org/x/crypto"
packages = [
"pbkdf2",
"ssh/terminal"
]
revision = "a8fb68e7206f8c78be19b432c58eb52a6aa34462"

[[projects]]
branch = "master"
name = "golang.org/x/net"
@@ -56,16 +125,26 @@
"context",
"context/ctxhttp"
]
revision = "24dd3780ca4f75fed9f321890729414a4b5d3f13"
revision = "db08ff08e8622530d9ed3a0e8ac279f6d4c02196"

[[projects]]
branch = "master"
name = "golang.org/x/oauth2"
packages = [
".",
"github",
"internal"
]
revision = "fdc9e635145ae97e6c2cb777c48305600cf515cb"
revision = "1e0a3fa8ba9a5c9eb35c271780101fdaf1b205d7"

[[projects]]
branch = "master"
name = "golang.org/x/sys"
packages = [
"unix",
"windows"
]
revision = "ac767d655b305d4e9612f5f6e33120b9176c4ad4"

[[projects]]
name = "google.golang.org/appengine"
@@ -78,12 +157,30 @@
"internal/urlfetch",
"urlfetch"
]
revision = "150dc57a1b433e64154302bdc40b6bb8aefa313a"
version = "v1.0.0"
revision = "b1f26356af11148e710935ed1ac8a7f5702c7612"
version = "v1.1.0"

[[projects]]
name = "gopkg.in/ini.v1"
packages = ["."]
revision = "06f5f3d67269ccec1fe5fe4134ba6e982984f7f5"
version = "v1.37.0"

[[projects]]
name = "gopkg.in/macaron.v1"
packages = ["."]
revision = "c1be95e6d21e769e44e1ec33cec9da5837861c10"
version = "v1.3.1"

[[projects]]
name = "gopkg.in/yaml.v2"
packages = ["."]
revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183"
version = "v2.2.1"

[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
inputs-digest = "a63a8945bd36ecc14dfbb7f9894543855f9e9e2e493dfdb77c06b4a12aada8a4"
inputs-digest = "0c4f08e8c96a1b866394a858e61289c637d938c3476b14446e483acd212ba06f"
solver-name = "gps-cdcl"
solver-version = 1

+ 27
- 0
Gopkg.toml View File

@@ -16,3 +16,30 @@
[[constraint]]
name = "github.com/stretchr/testify"
version = "1.2.2"

[[constraint]]
branch = "master"
name = "github.com/jinzhu/configor"

[[constraint]]
name = "github.com/gobuffalo/packr"
version = "1.11.1"

[[constraint]]
branch = "master"
name = "github.com/adam-hanna/randomstrings"

[[constraint]]
name = "gopkg.in/macaron.v1"
version = "1.3.1"

[[constraint]]
branch = "master"
name = "github.com/go-macaron/session"

[[constraint]]
branch = "master"
name = "github.com/go-macaron/binding"
[[constraint]]
name = "github.com/sirupsen/logrus"
version = "1.0.6"

+ 1
- 1
LICENSE View File

@@ -1,5 +1,5 @@
MIT License
Copyright (c) <year> <copyright holders>
Copyright (c) 2018 Jonas Franz
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

+ 40
- 5
Makefile View File

@@ -17,20 +17,55 @@ LDFLAGS := -X main.version=$(VERSION) -X main.build=$(DRONE_BUILD_NUMBER)
.PHONY: all
all:

.PHONY: docker-binary
docker-binary:
.PHONY: build
build:
go build -ldflags "$(LDFLAGS)" -o gitea-github-migrator

.PHONY: build-binary-web
build-binary-web:
go build -ldflags "$(LDFLAGS)" -tags web -o gitea-github-migrator

.PHONY: build-web
build-web: packr build-binary-web packr-clean

.PHONY: packr
packr:
@hash packr > /dev/null 2>&1; if [ $$? -ne 0 ]; then \
$(GO) get -u github.com/gobuffalo/packr/...; \
fi
packr -z

.PHONY: packr-clean
packr-clean:
@hash packr > /dev/null 2>&1; if [ $$? -ne 0 ]; then \
$(GO) get -u github.com/gobuffalo/packr/...; \
fi
packr clean

.PHONY: clean
clean: packr-clean
go clean ./...

.PHONY: docker-binary
docker-binary: build

.PHONY: docker-binary-web
docker-binary-web: build-web


.PHONY: generate-release-file
generate-release-file:
echo $(VERSION) > .version

.PHONY: release
release:
release: packr release-builds packr-clean

.PHONY: release-builds
release-builds:
@hash gox > /dev/null 2>&1; if [ $$? -ne 0 ]; then \
$(GO) get -u github.com/mitchellh/gox; \
fi
gox -ldflags "$(LDFLAGS)" -output "releases/gitea-github-migrator_{{.OS}}_{{.Arch}}"
gox -ldflags "$(LDFLAGS)" -tags web -output "releases/gitea-github-migrator_{{.OS}}_{{.Arch}}"

.PHONY: lint
lint:
@@ -45,5 +80,5 @@ vet:

.PHONY: test
test: lint vet
go test -cover ./...
go test -tags web -cover ./...

+ 69
- 5
README.md View File

@@ -1,12 +1,15 @@
# gitea-github-migrator

[![Build Status](https://drone.jonasfranz.software/api/badges/JonasFranzDEV/gitea-github-migrator/status.svg)](https://drone.jonasfranz.software/JonasFranzDEV/gitea-github-migrator)
[![Latest Release](https://img.shields.io/badge/dynamic/json.svg?label=release&url=https%3A%2F%2Fgit.jonasfranz.software%2Fapi%2Fv1%2Frepos%2FJonasFranzDEV%2Fgitea-github-migrator%2Freleases&query=%24%5B0%5D.tag_name)](https://git.jonasfranz.software/JonasFranzDEV/gitea-github-migrator/releases)
[![Docker Pulls](https://img.shields.io/docker/pulls/jonasfranz/gitea-github-migrator.svg)](https://hub.docker.com/r/jonasfranz/gitea-github-migrator/)
[![Docker Pulls](https://img.shields.io/docker/pulls/ggmigrator/cli.svg)](https://hub.docker.com/r/ggmigrator/cli/)
[![Docker Pulls](https://img.shields.io/docker/pulls/ggmigrator/web.svg)](https://hub.docker.com/r/ggmigrator/web/)

A tool to migrate [GitHub](https://github.com) Repositories to [Gitea](https://gitea.io) including all issues, labels, milestones
and comments.

## Features

Migrates:

- [x] Repositories
@@ -26,8 +29,11 @@ Migrates:
go get git.jonasfranz.software/JonasFranzDEV/gitea-github-migrator
cd $GOPATH/src/git.jonasfranz.software/JonasFranzDEV/gitea-github-migrator
dep ensure
go install
make build
```
#### Web Support

Run `make web-build` instead of `make build` to include web support.

### From Binary
We provide binaries of master builds and all releases at our [minio storage server](https://storage.h.jonasfranz.software/minio/gitea-github-migrator/dist/).
@@ -35,9 +41,30 @@ We provide binaries of master builds and all releases at our [minio storage serv
Additionally we provide them for every release as release attachment under [releases](https://git.jonasfranz.software/JonasFranzDEV/gitea-github-migrator/releases).

You don't need any dependencies except the binary to run the migrator.

These binaries include web support by default.

### From Docker image

We provide a [cli docker image](https://hub.docker.com/r/ggmigrator/cli/):

For master builds:
```docker
docker run ggmigrator/cli:latest
```

For release builds:
```docker
docker run ggmigrator/cli:0.0.10
```


## Usage

### Command line

Migrate one repository:

```bash
gitea-github-migrator migrate \
--gh-repo owner/reponame \
@@ -46,9 +73,11 @@ gitea-github-migrator migrate \
--token GITEA_TOKEN \
--owner 1
```

`gh-token` is only required if you have more than 50 issues / repositories.

Migrate all repositories:

```bash
gitea-github-migrator migrate-all \
--gh-user username \
@@ -59,6 +88,7 @@ gitea-github-migrator migrate-all \
```

Migrate all repositories without issues etc. (classic):

```bash
gitea-github-migrator migrate-all \
--gh-user username \
@@ -69,7 +99,41 @@ gitea-github-migrator migrate-all \
--only-repo
```

### Web interface

Since 0.1.0 gitea-github-migrator comes with an integrated web interface.
Follow these steps to get the web interface running:

1. Download or build a web-capable binary of the gitea-github-migrator. The builds on our storage server are build with web support included.
If you build from source, please follow [web support](#web-support).
2. Create `config.yml` file and change the properties according to your wishes. Please keep in mind that
you have to create a GitHub OAuth application to make the web interface work.
3. Run `./gitea-github-migrator web`
4. Visit `http://localhost:4000`

#### Docker

We're providing a docker image with web support. To start the web service please run:
```docker
docker run ggmigrator/web -p 4000:4000 -v data/:/data
```
Place your `config.yml` into `data/config.yml`.

#### Config
Example:
```yaml
# GitHub contains the OAuth2 application data obtainable from GitHub
GitHub:
client_id: GITHUB_OAUTH_CLIENT_ID
client_secret: GITHUB_OAUTH_SECRET
# Web contains the configuration for the integrated web server
Web:
port: 4000
host: 0.0.0.0
```

## Problems
* This migration tool does not work with Gitea instances using a SQLite database.
* Comments / Issues will be added in the name of the user to whom belongs the token (information about the original date and author will be added)
* The current date is used for creation date (information about the actual date is added in a comment)

- This migration tool does not work with Gitea instances using a SQLite database.
- Comments / Issues will be added in the name of the user to whom belongs the token (information about the original date and author will be added)
- The current date is used for creation date (information about the actual date is added in a comment)

+ 28
- 33
cmd/migrate-all.go View File

@@ -3,11 +3,11 @@ package cmd
import (
"context"
"fmt"
"sync"

"code.gitea.io/sdk/gitea"
"git.jonasfranz.software/JonasFranzDEV/gitea-github-migrator/migrations"
"github.com/google/go-github/github"
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
"golang.org/x/oauth2"
)
@@ -27,26 +27,14 @@ var CmdMigrateAll = cli.Command{
}

func runMigrateAll(ctx *cli.Context) error {
m := &migrations.Migratory{
Client: gitea.NewClient(ctx.String("url"), ctx.String("token")),
Private: ctx.Bool("private"),
NewOwnerID: ctx.Int64("owner"),
}
if m.NewOwnerID == 0 {
usr, err := m.Client.GetMyUserInfo()
if err != nil {
return fmt.Errorf("cannot fetch user info about current user: %v", err)
}
m.NewOwnerID = usr.ID
}
c := context.Background()

logrus.SetLevel(logrus.InfoLevel)
onlyRepos := ctx.Bool("only-repo")
var gc *github.Client
if ctx.IsSet("gh-token") {
ts := oauth2.StaticTokenSource(
&oauth2.Token{AccessToken: ctx.String("gh-token")},
)
tc := oauth2.NewClient(c, ts)
tc := oauth2.NewClient(context.Background(), ts)
gc = github.NewClient(tc)
} else {
gc = github.NewClient(nil)
@@ -58,7 +46,7 @@ func runMigrateAll(ctx *cli.Context) error {
// get all pages of results
var allRepos []*github.Repository
for {
repos, resp, err := gc.Repositories.List(c, ctx.String("gh-user"), opt)
repos, resp, err := gc.Repositories.List(context.Background(), ctx.String("gh-user"), opt)
if err != nil {
return err
}
@@ -68,25 +56,32 @@ func runMigrateAll(ctx *cli.Context) error {
}
opt.Page = resp.NextPage
}
errs := make(chan error, 1)
job := migrations.NewJob(&migrations.Options{
Private: ctx.Bool("private"),
NewOwnerID: ctx.Int("owner"),

var wg sync.WaitGroup
wg.Add(len(allRepos))
Comments: !onlyRepos,
Issues: !onlyRepos,
Labels: !onlyRepos,
Milestones: !onlyRepos,
PullRequests: !onlyRepos,
Strategy: migrations.Classic,
}, gitea.NewClient(ctx.String("url"), ctx.String("token")), gc)
if job.Options.NewOwnerID == 0 {
usr, err := job.Client.GetMyUserInfo()
if err != nil {
return fmt.Errorf("cannot fetch user info about current user: %v", err)
}
job.Options.NewOwnerID = int(usr.ID)
}
for _, repo := range allRepos {
go func(r *github.Repository) {
defer wg.Done()
errs <- migrate(c, gc, m, r.Owner.GetLogin(), r.GetName(), ctx.Bool("only-repo"))
}(repo)
job.Repositories = append(job.Repositories, repo.GetFullName())
}

go func() {
for i := range errs {
if i != nil {
fmt.Printf("error: %v", i)
}
errs := job.StartMigration()
for i := range errs {
if i != nil {
fmt.Printf("error: %v\n", i)
}
}()

wg.Wait()
}
return nil
}

+ 21
- 75
cmd/migrate.go View File

@@ -3,11 +3,11 @@ package cmd
import (
"context"
"fmt"
"strings"

"code.gitea.io/sdk/gitea"
"git.jonasfranz.software/JonasFranzDEV/gitea-github-migrator/migrations"
"github.com/google/go-github/github"
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
"golang.org/x/oauth2"
)
@@ -28,95 +28,41 @@ var CmdMigrate = cli.Command{
}

func runMigrate(ctx *cli.Context) error {
m := &migrations.Migratory{
Client: gitea.NewClient(ctx.String("url"), ctx.String("token")),
Private: ctx.Bool("private"),
NewOwnerID: ctx.Int64("owner"),
}
if m.NewOwnerID == 0 {
usr, err := m.Client.GetMyUserInfo()
if err != nil {
return fmt.Errorf("cannot fetch user info about current user: %v", err)
}
m.NewOwnerID = usr.ID
}
c := context.Background()
onlyRepos := ctx.Bool("only-repo")
var gc *github.Client
if ctx.IsSet("gh-token") {
ts := oauth2.StaticTokenSource(
&oauth2.Token{AccessToken: ctx.String("gh-token")},
)
tc := oauth2.NewClient(c, ts)
tc := oauth2.NewClient(context.Background(), ts)
gc = github.NewClient(tc)
} else {
gc = github.NewClient(nil)
}
logrus.SetLevel(logrus.InfoLevel)
job := migrations.NewJob(&migrations.Options{
Private: ctx.Bool("private"),
NewOwnerID: ctx.Int("owner"),

username := strings.Split(ctx.String("gh-repo"), "/")[0]
repo := strings.Split(ctx.String("gh-repo"), "/")[1]

return migrate(c, gc, m, username, repo, ctx.Bool("only-repo"))
}

func migrate(c context.Context, gc *github.Client, m *migrations.Migratory, username, repo string, onlyRepo bool) error {
fmt.Printf("Fetching repository %s/%s...\n", username, repo)
gr, _, err := gc.Repositories.Get(c, username, repo)
if err != nil {
return fmt.Errorf("error while fetching repo[%s/%s]: %v", username, repo, err)
}

fmt.Printf("Migrating repository %s/%s...\n", username, repo)
var mr *gitea.Repository
if mr, err = m.Repository(gr); err != nil {
return fmt.Errorf("error while migrating repo[%s/%s]: %v", username, repo, err)
}
fmt.Printf("Repository migrated to %s/%s\n", mr.Owner.UserName, mr.Name)

if onlyRepo {
return nil
}

fmt.Println("Fetching issues...")
opt := &github.IssueListByRepoOptions{
Sort: "created",
Direction: "asc",
State: "all",
ListOptions: github.ListOptions{
PerPage: 100,
},
}
var allIssues = make([]*github.Issue, 0)
for {
issues, resp, err := gc.Issues.ListByRepo(c, username, repo, opt)
Comments: !onlyRepos,
Issues: !onlyRepos,
Labels: !onlyRepos,
Milestones: !onlyRepos,
PullRequests: !onlyRepos,
Strategy: migrations.Classic,
}, gitea.NewClient(ctx.String("url"), ctx.String("token")), gc, ctx.String("gh-repo"))
if job.Options.NewOwnerID == 0 {
usr, err := job.Client.GetMyUserInfo()
if err != nil {
return fmt.Errorf("error while listing repos: %v", err)
}
allIssues = append(allIssues, issues...)
if resp.NextPage == 0 {
break
return fmt.Errorf("cannot fetch user info about current user: %v", err)
}
opt.Page = resp.NextPage
job.Options.NewOwnerID = int(usr.ID)
}
fmt.Println("Migrating issues...")
for _, gi := range allIssues {
fmt.Printf("Migrating #%d...\n", *gi.Number)
issue, err := m.Issue(gi)
if err != nil {
return fmt.Errorf("migrating issue[id: %d]: %v", *gi.ID, err)
}
comments, _, err := gc.Issues.ListComments(c, username, repo, gi.GetNumber(), nil)
errs := job.StartMigration()
for err := range errs {
if err != nil {
return fmt.Errorf("fetching issue[id: %d] comments: %v", *gi.ID, err)
}
for _, gc := range comments {
fmt.Printf("-> %d...", gc.ID)
if _, err := m.IssueComment(issue, gc); err != nil {
return fmt.Errorf("migrating issue comment [issue: %d, comment: %d]: %v", *gi.ID, gc.ID, err)
}
fmt.Print("Done!\n")
return err
}
fmt.Printf("Migrated #%d...\n", *gi.Number)

}
return nil
}

+ 0
- 23
cmd/migrate_test.go View File

@@ -1,23 +0,0 @@
package cmd

import (
"context"
"testing"

"git.jonasfranz.software/JonasFranzDEV/gitea-github-migrator/migrations"
"github.com/google/go-github/github"
"github.com/stretchr/testify/assert"
)

// Test_migrate is an integration tests for migrate command
// using repo JonasFranzDEV/test
func Test_migrate(t *testing.T) {
assert.NoError(t, migrate(
context.Background(),
github.NewClient(nil),
migrations.DemoMigratory,
"JonasFranzDEV",
"test",
false,
))
}

+ 48
- 0
cmd/web.go View File

@@ -0,0 +1,48 @@
// +build web

package cmd

import (
"fmt"
"net/http"

"git.jonasfranz.software/JonasFranzDEV/gitea-github-migrator/config"
"git.jonasfranz.software/JonasFranzDEV/gitea-github-migrator/web"
"github.com/jinzhu/configor"
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
)

// CmdWeb stars the web interface
var CmdWeb = cli.Command{
Name: "web",
Usage: "Starts the web interface",
Action: runWeb,
Flags: []cli.Flag{
cli.StringFlag{
Name: "c,config",
Usage: "config file",
Value: "config.yml",
EnvVar: "MIGRATOR_CONFIG",
},
},
}

func runWeb(ctx *cli.Context) error {
if err := configor.New(&configor.Config{ErrorOnUnmatchedKeys: true}).Load(&config.Config, ctx.String("config")); err != nil {
return err
}
r := web.InitRoutes()

hostname := config.Config.Web.Host
if len(hostname) == 0 {
hostname = "0.0.0.0"
}
port := config.Config.Web.Port
if port == 0 {
port = 4000
}
logrus.Infof("Server is running at http://%s:%d", hostname, port)
logrus.SetLevel(logrus.PanicLevel)
return http.ListenAndServe(fmt.Sprintf("%s:%d", hostname, port), r)
}

+ 13
- 0
commands.go View File

@@ -0,0 +1,13 @@
// +build !web

package main

import (
"git.jonasfranz.software/JonasFranzDEV/gitea-github-migrator/cmd"
"github.com/urfave/cli"
)

var cmds = cli.Commands{
cmd.CmdMigrate,
cmd.CmdMigrateAll,
}

+ 14
- 0
commands_web.go View File

@@ -0,0 +1,14 @@
// +build web

package main

import (
"git.jonasfranz.software/JonasFranzDEV/gitea-github-migrator/cmd"
"github.com/urfave/cli"
)

var cmds = cli.Commands{
cmd.CmdMigrate,
cmd.CmdMigrateAll,
cmd.CmdWeb,
}

+ 8
- 0
config.example.yml View File

@@ -0,0 +1,8 @@
# GitHub contains the OAuth2 application data obtainable from GitHub
GitHub:
client_id: GITHUB_OAUTH_CLIENT_ID
client_secret: GITHUB_OAUTH_SECRET
# Web contains the configuration for the integrated web server
Web:
port: 4000
host: 0.0.0.0

+ 13
- 0
config/config.go View File

@@ -0,0 +1,13 @@
package config

// Config holds all configurations needed for web interface
var Config = struct {
GitHub struct {
ClientID string `required:"true" yaml:"client_id"`
ClientSecret string `required:"true" yaml:"client_secret"`
}
Web struct {
Host string `yaml:"host"`
Port int `yaml:"port"`
}
}{}

+ 2
- 6
main.go View File

@@ -1,3 +1,4 @@
//go:generate swagger generate spec -i ./swagger.yml -o ./swagger.json
package main

import (
@@ -5,8 +6,6 @@ import (
"os"

"github.com/urfave/cli"

"git.jonasfranz.software/JonasFranzDEV/gitea-github-migrator/cmd"
)

var (
@@ -20,10 +19,7 @@ func main() {
app.Version = fmt.Sprintf("%s+%s", version, build)
app.Usage = "GitHub to Gitea migrator for repositories"
app.Description = `Migrate your GitHub repositories including issues to Gitea`
app.Commands = cli.Commands{
cmd.CmdMigrate,
cmd.CmdMigrateAll,
}
app.Commands = cmds
if err := app.Run(os.Args); err != nil {
panic(err)
}


+ 264
- 0
migrations/github.go View File

@@ -0,0 +1,264 @@
package migrations

import (
"context"
"fmt"
"regexp"
"strconv"
"strings"
"sync"

"code.gitea.io/sdk/gitea"
"github.com/google/go-github/github"
"github.com/sirupsen/logrus"
)

// FetchMigratory adds GitHub fetching functions to migratory
type FetchMigratory struct {
Migratory
GHClient *github.Client
RepoOwner string
RepoName string
}

func (fm *FetchMigratory) ctx() context.Context {
return context.Background()
}

// MigrateFromGitHub migrates RepoOwner/RepoName from GitHub to Gitea
func (fm *FetchMigratory) MigrateFromGitHub() error {
fm.Status = &MigratoryStatus{
Stage: Importing,
}
logrus.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
}).Info("migrating git repository")
ghRepo, _, err := fm.GHClient.Repositories.Get(fm.ctx(), fm.RepoOwner, fm.RepoName)
if err != nil {
fm.Status.Stage = Failed
fm.Status.FatalError = err
return fmt.Errorf("GHClient Repostiories Get: %v", err)
}
fm.repository, err = fm.Repository(ghRepo)
if err != nil {
fm.Status.Stage = Failed
fm.Status.FatalError = err
return fmt.Errorf("Repository migration: %v", err)
}
logrus.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
}).Info("git repository migrated")
if fm.Options.Issues || fm.Options.PullRequests {
var commentsChan chan *[]*github.IssueComment
if fm.Options.Comments {
commentsChan = fm.fetchCommentsAsync()
}
issues, err := fm.FetchIssues()
if err != nil {
fm.Status.Stage = Failed
fm.Status.FatalError = err
logrus.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
}).Fatalf("migration failed: %v", fm.Status.FatalError)
return err
}
fm.Status.Stage = Migrating
fm.Status.Issues = int64(len(issues))
migratedIssues := make(map[int]*gitea.Issue)
for _, issue := range issues {
if (!issue.IsPullRequest() || fm.Options.PullRequests) &&
(issue.IsPullRequest() || fm.Options.Issues) {
migratedIssues[issue.GetNumber()], err = fm.Issue(issue)
if err != nil {
fm.Status.IssuesError++
logrus.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
"issue": issue.GetNumber(),
}).Warnf("error while migrating: %v", err)
continue
}
fm.Status.IssuesMigrated++
logrus.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
"issue": issue.GetNumber(),
}).Info("issue migrated")
} else {
fm.Status.Issues--
}
}
if fm.Options.Comments {
var comments []*github.IssueComment
if cmts := <-commentsChan; cmts == nil {
fm.Status.Stage = Failed
err := fmt.Errorf("error while fetching issue comments")
logrus.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
}).Fatalf("migration failed: %v", fm.Status.FatalError)
return err
} else {
comments = *cmts
}

if err != nil {
fm.Status.Stage = Failed
fm.Status.FatalError = err
logrus.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
}).Fatalf("migration failed: %v", fm.Status.FatalError)
return err
}
fm.Status.Comments = int64(len(comments))
commentsByIssue := make(map[*gitea.Issue][]*github.IssueComment, len(migratedIssues))
for _, comment := range comments {
issueIndex, err := getIssueIndexFromHTMLURL(comment.GetHTMLURL())
if err != nil {
fm.Status.CommentsError++
logrus.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
"issue": issueIndex,
"comment": comment.GetID(),
}).Warnf("error while migrating comment: %v", err)
continue
}
if issue, ok := migratedIssues[issueIndex]; ok && issue != nil {
if list, ok := commentsByIssue[issue]; !ok && list != nil {
commentsByIssue[issue] = []*github.IssueComment{comment}
} else {
commentsByIssue[issue] = append(list, comment)
}
} else {
fm.Status.CommentsError++
continue
}
}
wg := sync.WaitGroup{}
for issue, comms := range commentsByIssue {
wg.Add(1)
go func(i *gitea.Issue, cs []*github.IssueComment) {
for _, comm := range cs {
if _, err := fm.IssueComment(i, comm); err != nil {
fm.Status.CommentsError++
logrus.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
"comment": comm.GetID(),
}).Warnf("error while migrating comment: %v", err)
continue
}
fm.Status.CommentsMigrated++
logrus.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
"comment": comm.GetID(),
}).Info("comment migrated")
}
wg.Done()
}(issue, comms)
}
wg.Wait()
}
}
if fm.Status.FatalError != nil {
fm.Status.Stage = Failed
logrus.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
}).Fatalf("migration failed: %v", fm.Status.FatalError)
return nil
}
fm.Status.Stage = Finished
logrus.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
}).Info("migration successful")
return nil
}

var issueIndexRegex = regexp.MustCompile(`/(issues|pull)/([0-9]+)#`)

func getIssueIndexFromHTMLURL(htmlURL string) (int, error) {
// Alt is 4 times faster but more error prune
if res, err := getIssueIndexFromHTMLURLAlt(htmlURL); err == nil {
return res, nil
}
matches := issueIndexRegex.FindStringSubmatch(htmlURL)
if len(matches) < 3 {
return 0, fmt.Errorf("cannot parse issue id from HTML URL: %s", htmlURL)
}
return strconv.Atoi(matches[2])
}
func getIssueIndexFromHTMLURLAlt(htmlURL string) (int, error) {
res := strings.Split(htmlURL, "/issues/")
if len(res) != 2 {
res = strings.Split(htmlURL, "/pull/")
}
if len(res) != 2 {
return 0, fmt.Errorf("invalid HTMLURL: %s", htmlURL)
}
number := res[1]
number = strings.Split(number, "#")[0]
return strconv.Atoi(number)
}

// FetchIssues fetches all issues from GitHub
func (fm *FetchMigratory) FetchIssues() ([]*github.Issue, error) {
opt := &github.IssueListByRepoOptions{
Sort: "created",
Direction: "asc",
State: "all",
ListOptions: github.ListOptions{
PerPage: 100,
},
}
var allIssues = make([]*github.Issue, 0)
for {
issues, resp, err := fm.GHClient.Issues.ListByRepo(fm.ctx(), fm.RepoOwner, fm.RepoName, opt)
if err != nil {
return nil, fmt.Errorf("error while listing repos: %v", err)
}
allIssues = append(allIssues, issues...)
if resp.NextPage == 0 {
break
}
opt.Page = resp.NextPage
}
return allIssues, nil
}

// FetchComments fetches all comments from GitHub
func (fm *FetchMigratory) FetchComments() ([]*github.IssueComment, error) {
var allComments = make([]*github.IssueComment, 0)
opt := &github.IssueListCommentsOptions{
Sort: "created",
Direction: "asc",
ListOptions: github.ListOptions{
PerPage: 100,
},
}
for {
comments, resp, err := fm.GHClient.Issues.ListComments(fm.ctx(), fm.RepoOwner, fm.RepoName, 0, opt)
if err != nil {
return nil, fmt.Errorf("error while listing repos: %v", err)
}
allComments = append(allComments, comments...)
if resp.NextPage == 0 {
break
}
opt.Page = resp.NextPage
}
return allComments, nil
}

func (fm *FetchMigratory) fetchCommentsAsync() chan *[]*github.IssueComment {
ret := make(chan *[]*github.IssueComment, 1)
go func(f *FetchMigratory) {
comments, err := f.FetchComments()
if err != nil {
f.Status.FatalError = err
ret <- nil
logrus.WithFields(logrus.Fields{
"repo": fmt.Sprintf("%s/%s", fm.RepoOwner, fm.RepoName),
}).Fatalf("fetching comments failed: %v", fm.Status.FatalError)
return
}
f.Status.Comments = int64(len(comments))
ret <- &comments
}(fm)
return ret
}

+ 55
- 0
migrations/github_test.go View File

@@ -0,0 +1,55 @@
package migrations

import (
"testing"

"github.com/google/go-github/github"
"github.com/stretchr/testify/assert"
)

func BenchmarkGetIssueIndexFromHTMLURLAlt(b *testing.B) {
for i := 0; i <= b.N; i++ {
getIssueIndexFromHTMLURLAlt("https://github.com/octocat/Hello-World/issues/1347#issuecomment-1")
}
}

func TestGetIssueIndexFromHTMLURLAlt(t *testing.T) {
res, err := getIssueIndexFromHTMLURLAlt("https://github.com/octocat/Hello-World/issues/1347#issuecomment-1")
assert.NoError(t, err)
assert.Equal(t, 1347, res)
res, err = getIssueIndexFromHTMLURLAlt("https://github.com/oment-1")
assert.Error(t, err)
}

func BenchmarkGetIssueIndexFromHTMLURL(b *testing.B) {
for i := 0; i <= b.N; i++ {
getIssueIndexFromHTMLURL("https://github.com/octocat/Hello-World/issues/1347#issuecomment-1")
}
}

func TestGetIssueIndexFromHTMLURL(t *testing.T) {
res, err := getIssueIndexFromHTMLURL("https://github.com/octocat/Hello-World/issues/1347#issuecomment-1")
assert.NoError(t, err)
assert.Equal(t, 1347, res)
res, err = getIssueIndexFromHTMLURL("https://github.com/oment-1")
assert.Error(t, err)
}

var testFMig = &FetchMigratory{
Migratory: *DemoMigratory,
GHClient: github.NewClient(nil),
RepoOwner: "JonasFranzDEV",
RepoName: "test",
}

func TestFetchMigratory_FetchIssues(t *testing.T) {
issues, err := testFMig.FetchIssues()
assert.NoError(t, err)
assert.True(t, len(issues) > 0, "at least one issue found")
}

func TestFetchMigratory_FetchComments(t *testing.T) {
comments, err := testFMig.FetchIssues()
assert.NoError(t, err)
assert.True(t, len(comments) > 0, "at least one comment found")
}

+ 9
- 5
migrations/issue.go View File

@@ -18,7 +18,7 @@ func (m *Migratory) Issue(gi *github.Issue) (*gitea.Issue, error) {

// Migrate milestone if it is not already migrated
milestone := int64(0)
if gi.Milestone != nil {
if gi.Milestone != nil && m.Options.Milestones {
// Lookup if milestone is already migrated
if migratedMilestone, ok := m.migratedMilestones[*gi.Milestone.ID]; ok {
milestone = migratedMilestone
@@ -28,10 +28,14 @@ func (m *Migratory) Issue(gi *github.Issue) (*gitea.Issue, error) {
milestone = ms.ID
}
}
// Migrate labels
labels, err := m.labels(gi.Labels)
if err != nil {
return nil, err
var labels = make([]int64, 0)
var err error
if m.Options.Labels {
// Migrate labels
labels, err = m.labels(gi.Labels)
if err != nil {
return nil, err
}
}

return m.Client.CreateIssue(m.repository.Owner.UserName, m.repository.Name,


+ 2
- 2
migrations/issue_test.go View File

@@ -28,7 +28,7 @@ func TestMigratory_Label(t *testing.T) {
Name: github.String("testlabel"),
Color: github.String("123456"),
})
assert.NoError(t, err)
assertNoError(t, err)
assert.Equal(t, "123456", res.Color)
assert.Equal(t, "testlabel", res.Name)
}
@@ -41,7 +41,7 @@ func TestMigratory_Milestone(t *testing.T) {
Title: github.String("TEST"),
DueOn: &demoTime,
})
assert.NoError(t, err)
assertNoError(t, err)
assert.Equal(t, "TEST", res.Title)
assert.Equal(t, "test milestone", res.Description)
assert.Equal(t, demoTime.Unix(), res.Deadline.Unix())


+ 114
- 0
migrations/job.go View File

@@ -0,0 +1,114 @@
package migrations

import (
"fmt"
"strings"

"code.gitea.io/sdk/gitea"
"github.com/google/go-github/github"
)

// Job manages all migrations of a "migartion job"
type Job struct {
Repositories []string
Options *Options
Client *gitea.Client
GHClient *github.Client

migratories map[string]*Migratory
}

// JobReport represents the current status of a Job
type JobReport struct {
Pending []string `json:"pending"`
Running map[string]*MigratoryStatus `json:"running"`
Finished map[string]*MigratoryStatus `json:"finished"`
Failed map[string]string `json:"failed"`
}

// NewJob returns an instance of initialized instance of Job
func NewJob(options *Options, client *gitea.Client, githubClient *github.Client, repos ...string) *Job {
return &Job{Repositories: repos, Options: options, Client: client, GHClient: githubClient}
}

// StatusReport generates a JobReport indicating which state the job is
func (job *Job) StatusReport() *JobReport {
report := &JobReport{
Pending: make([]string, 0),
Finished: make(map[string]*MigratoryStatus),
Running: make(map[string]*MigratoryStatus),
Failed: make(map[string]string),
}
for _, repo := range job.Repositories {
if migratory, ok := job.migratories[repo]; ok {
switch migratory.Status.Stage {
case Finished:
report.Finished[repo] = migratory.Status
case Importing:
case Migrating:
report.Running[repo] = migratory.Status
case Failed:
report.Failed[repo] = migratory.Status.FatalError.Error()
default:
report.Pending = append(report.Pending, repo)
fmt.Printf("unknown status %d\n", migratory.Status.Stage)
}
} else {
report.Pending = append(report.Pending, repo)
}
}
return report
}

// StartMigration migrates all repos from Repositories
func (job *Job) StartMigration() chan error {
errs := make(chan error, len(job.Repositories))
var pendingRepos = len(job.Repositories)
autoclose := func() {
pendingRepos--
if pendingRepos <= 0 {
close(errs)
}
}
job.migratories = make(map[string]*Migratory, pendingRepos)
for _, repo := range job.Repositories {
mig, err := job.initFetchMigratory(repo)
job.migratories[repo] = &mig.Migratory
if err != nil {
mig.Status = &MigratoryStatus{
Stage: Failed,
FatalError: err,
}
errs <- err
autoclose()
continue
}
go func() {
err := mig.MigrateFromGitHub()
errs <- err
autoclose()
}()
}
return errs
}

func (job *Job) initFetchMigratory(repo string) (*FetchMigratory, error) {
res := strings.Split(repo, "/")
if len(res) != 2 {
return nil, fmt.Errorf("invalid repo name: %s", repo)
}
return &FetchMigratory{
Migratory: Migratory{
Client: job.Client,
Options: *job.Options,
},
RepoName: res[1],
RepoOwner: res[0],
GHClient: job.GHClient,
}, nil
}

// Finished indicates if the job is finished or not
func (job *Job) Finished() bool {
return (len(job.StatusReport().Failed) + len(job.StatusReport().Finished)) >= len(job.Repositories)
}

+ 66
- 0
migrations/job_test.go View File

@@ -0,0 +1,66 @@
package migrations

import (
"fmt"
"testing"

"github.com/stretchr/testify/assert"
)

func TestJob_StatusReport(t *testing.T) {
jobWithStatus := func(status *MigratoryStatus) *Job {
return &Job{
migratories: map[string]*Migratory{
"test/test": {
Status: status,
},
},
Repositories: []string{
"test/test",
},
}
}
// Pending
pendingJob := &Job{
Repositories: []string{
"test/test",
},
}
report := pendingJob.StatusReport()
assert.Len(t, report.Pending, 1)
assert.Equal(t, report.Pending[0], "test/test")
assert.Len(t, report.Failed, 0)
assert.Len(t, report.Running, 0)
assert.Len(t, report.Finished, 0)

// Finished
report = jobWithStatus(&MigratoryStatus{
Stage: Finished,
}).StatusReport()
assert.Len(t, report.Pending, 0)
assert.Len(t, report.Failed, 0)
assert.Len(t, report.Running, 0)
assert.Len(t, report.Finished, 1)
assert.Equal(t, Finished, report.Finished["test/test"].Stage)

// Failed
report = jobWithStatus(&MigratoryStatus{
Stage: Failed,
FatalError: fmt.Errorf("test"),
}).StatusReport()
assert.Len(t, report.Failed, 1)
assert.Equal(t, "test", report.Failed["test/test"])
assert.Len(t, report.Pending, 0)
assert.Len(t, report.Running, 0)
assert.Len(t, report.Finished, 0)

// Running
report = jobWithStatus(&MigratoryStatus{
Stage: Migrating,
}).StatusReport()
assert.Len(t, report.Running, 1)
assert.Equal(t, Migrating, report.Running["test/test"].Stage)
assert.Len(t, report.Pending, 0)
assert.Len(t, report.Failed, 0)
assert.Len(t, report.Finished, 0)
}

+ 33
- 5
migrations/migratory.go View File

@@ -2,14 +2,26 @@ package migrations

import "code.gitea.io/sdk/gitea"

// MigratoryStage represents the actual step in the process
type MigratoryStage int

const (
// Importing imports the repo to Gitea
Importing MigratoryStage = iota
// Migrating migrates issues, etc to Gitea
Migrating
// Finished means that everything is migrated successfully
Finished
// Failed is only entered if a fatal error occurs
Failed
)

// Migratory is the context for migrating things from GitHub to Gitea
type Migratory struct {
Client *gitea.Client
AuthUsername string
AuthPassword string
Options
Client *gitea.Client

Private bool
NewOwnerID int64
Status *MigratoryStatus

repository *gitea.Repository
// key: github milestone id | value: gitea milestone id
@@ -17,3 +29,19 @@ type Migratory struct {
// key: github label id | value: gitea label id
migratedLabels map[int64]int64
}

// MigratoryStatus represents the actual state of a migratory
type MigratoryStatus struct {
Stage MigratoryStage `json:"stage"`

Issues int64 `json:"total_issues"`
IssuesMigrated int64 `json:"migrated_issues"`
IssuesError int64 `json:"failed_issues"`

Comments int64 `json:"total_comments"`
CommentsError int64 `json:"failed_comments"`
CommentsMigrated int64 `json:"migrated_comments"`

// FatalError should only be used if stage == failed; indicates which fatal error occurred
FatalError error
}

+ 32
- 0
migrations/options.go View File

@@ -0,0 +1,32 @@
package migrations

// Options defines the way a repository gets migrated
type Options struct {
Issues bool
Milestones bool
Labels bool
Comments bool
PullRequests bool

AuthUsername string
AuthPassword string

Private bool
NewOwnerID int

Strategy Strategy
}

// Strategy represents the procedure of migration.
type Strategy int

const (
// Classic works for all Gitea versions and creates comments by the user migrating the repository. This does not require
// admin permissions. The issue "number" is also assinged by Gitea and could be different to the GitHub issue "number".
// Creation date of comments, issues, milestones, etc. will be the date of creation.
Classic Strategy = iota
// Advanced works for all Gitea versions 1.6+ and utilizes the Gitea Migration API which allows the tool to create comments
// with Ghost Users. Creation date and issue numbers will be the same like GitHub. It requires admin permissions for repo
// (creation date, issue number) and/or
Advanced
)

+ 18
- 4
migrations/utils.go View File

@@ -1,24 +1,38 @@
package migrations

import (
"strings"
"testing"
"time"

"code.gitea.io/sdk/gitea"
"github.com/stretchr/testify/assert"
)

// DemoMigratory is been used for testing
var DemoMigratory = &Migratory{
AuthUsername: "demo",
AuthPassword: "demo",
Client: gitea.NewClient("http://gitea:3000", "8bffa364d5a4b2f18421426da0baf6ccddd16d6b"),
Options: Options{
AuthUsername: "demo",
AuthPassword: "demo",
NewOwnerID: 1,
},
Client: gitea.NewClient("http://gitea:3000", "8bffa364d5a4b2f18421426da0baf6ccddd16d6b"),
repository: &gitea.Repository{
Name: "demo",
Owner: &gitea.User{
UserName: "demo",
},
},
NewOwnerID: 1,
migratedMilestones: make(map[int64]int64),
migratedLabels: make(map[int64]int64),
}

var demoTime = time.Date(2018, 01, 01, 01, 01, 01, 01, time.UTC)

func assertNoError(t *testing.T, err error) {
if err != nil && strings.Contains(err.Error(), "lookup gitea") {
t.Skip("gitea instance is not running")
} else {
assert.NoError(t, err)
}
}

+ 50
- 0
web/auth/gitea.go View File

@@ -0,0 +1,50 @@
package auth

import (
"code.gitea.io/sdk/gitea"
"git.jonasfranz.software/JonasFranzDEV/gitea-github-migrator/web/context"
)

// GiteaLoginForm represents the data required for logging in into gitea
type GiteaLoginForm struct {
Username string `form:"username"`
Password string `form:"password"`
AccessToken string `form:"access-token"`
GiteaURL string `form:"gitea-url"`
Type string `form:"use" binding:"Required;In(token,password)"`
}

// LoginToGitea handles the POST request for signing in with a Gitea account
func LoginToGitea(ctx *context.Context, form GiteaLoginForm) {
var token string
if form.Type == "password" {
client := gitea.NewClient(form.GiteaURL, "")
tkn, err := client.CreateAccessToken(form.Username, form.Password, gitea.CreateAccessTokenOption{
Name: "gitea-github-migrator",
})
if err != nil {
ctx.Flash.Error("Cannot create access token please check your credentials!")
ctx.Redirect("/")
return
}
token = tkn.Sha1
} else {
token = form.AccessToken
}
client := gitea.NewClient(form.GiteaURL, token)
usr, err := client.GetMyUserInfo()
if err != nil {
ctx.Flash.Error("Invalid Gitea credentials.")
ctx.Redirect("/")
return
}
ctx.Session.Set("gitea_user", &context.User{
ID: usr.ID,
Username: usr.UserName,
Token: token,
AvatarURL: usr.AvatarURL,
})
ctx.Session.Set("gitea", form.GiteaURL)
ctx.Redirect("/")
return
}

+ 68
- 0
web/auth/github.go View File

@@ -0,0 +1,68 @@
package auth

import (
"context"
"net/http"

"git.jonasfranz.software/JonasFranzDEV/gitea-github-migrator/config"
webcontext "git.jonasfranz.software/JonasFranzDEV/gitea-github-migrator/web/context"
"github.com/adam-hanna/randomstrings"
"github.com/go-macaron/session"
"github.com/google/go-github/github"
"golang.org/x/oauth2"
githubauth "golang.org/x/oauth2/github"
)

var (
githubOAuthConfig *oauth2.Config
)

// InitGitHubOAuthConfig loads values from config into githubOAuthConfig
func InitGitHubOAuthConfig() {
githubOAuthConfig = &oauth2.Config{
ClientID: config.Config.GitHub.ClientID,
ClientSecret: config.Config.GitHub.ClientSecret,
Scopes: []string{"repo"},
Endpoint: githubauth.Endpoint,
}
}

// RedirectToGitHub returns the redirect URL for github
func RedirectToGitHub(ctx *webcontext.Context, session session.Store) {
state, err := randomstrings.GenerateRandomString(64)
if err != nil {
return
}
session.Set("state", state)
ctx.Redirect(githubOAuthConfig.AuthCodeURL(state), http.StatusTemporaryRedirect)
}

// CallbackFromGitHub handles the callback from the GitHub OAuth provider
func CallbackFromGitHub(ctx *webcontext.Context, session session.Store) {
bg := context.Background()
var state string
var ok bool
if state, ok = session.Get("state").(string); state == "" || !ok || state != ctx.Query("state") {
ctx.Handle(400, "invalid session", nil)
return
}
token, err := githubOAuthConfig.Exchange(bg, ctx.Query("code"))
if err != nil {
ctx.Handle(403, "access denied", err)
return
}
tc := oauth2.NewClient(bg, oauth2.StaticTokenSource(token))
client := github.NewClient(tc)
user, _, err := client.Users.Get(bg, "")
if err != nil {
ctx.Handle(403, "access denied", err)
return
}
session.Set("user", &webcontext.User{
ID: user.GetID(),
AvatarURL: *user.AvatarURL,
Username: user.GetLogin(),
Token: token.AccessToken,
})
ctx.Redirect("/")
}

+ 115
- 0
web/context/context.go View File

@@ -0,0 +1,115 @@
package context

import (
bgctx "context"
"fmt"
"strings"

"code.gitea.io/sdk/gitea"
"git.jonasfranz.software/JonasFranzDEV/gitea-github-migrator/config"
"git.jonasfranz.software/JonasFranzDEV/gitea-github-migrator/migrations"
"github.com/go-macaron/session"
"github.com/google/go-github/github"
"github.com/sirupsen/logrus"
"golang.org/x/oauth2"
"gopkg.in/macaron.v1"
)

// Context represents context of a request.
type Context struct {
*macaron.Context
Flash *session.Flash
Session session.Store

Client *github.Client
GiteaClient *gitea.Client
User *User //GitHub user
GiteaUser *User
Link string // current request URL
}

// User is an abstraction of a Gitea or GitHub user, saving the required information
type User struct {
ID int64
Username string
AvatarURL string
Token string
}

var runningJobs = make(map[string]*migrations.Job)

// GetCurrentJob returns the current job of the user
// Bug(JonasFranzDEV): prevents scalability (FIXME)
func (ctx *Context) GetCurrentJob() *migrations.Job {
return runningJobs[ctx.Session.ID()]
}

// SetCurrentJob sets the current job of the user
// Bug(JonasFranzDEV): prevents scalability (FIXME)
func (ctx *Context) SetCurrentJob(job *migrations.Job) {
runningJobs[ctx.Session.ID()] = job
}

// Handle displays the corresponding error message
func (ctx *Context) Handle(status int, title string, err error) {
if err != nil {
if macaron.Env != macaron.PROD {
ctx.Data["ErrorMsg"] = err
}
}
logrus.Warnf("Handle: %v", err)
ctx.Data["ErrTitle"] = title

switch status {
case 403:
ctx.Data["Title"] = "Access denied"
case 404:
ctx.Data["Title"] = "Page not found"
case 500:
ctx.Data["Title"] = "Internal Server Error"
default:
ctx.Context.HTML(status, "status/unknown_error")
return
}
ctx.Context.HTML(status, fmt.Sprintf("status/%d", status))
}

// Contexter injects context.Context into macaron
func Contexter() macaron.Handler {
return func(c *macaron.Context, sess session.Store, f *session.Flash) {
ctx := &Context{
Context: c,
Flash: f,
Session: sess,
Link: c.Req.URL.String(),
}
c.Data["Link"] = ctx.Link
if ctx.Req.Method == "POST" && strings.Contains(ctx.Req.Header.Get("Content-Type"), "multipart/form-data") {
if err := ctx.Req.ParseMultipartForm(5242880); err != nil &&
strings.Contains(err.Error(), "EOF") {
ctx.Handle(500, "ParseMultipartForm", err)
}
}
ctx.Data["Config"] = config.Config
usr := sess.Get("user")
if usr != nil {
ctx.User = usr.(*User)
ctx.Data["User"] = ctx.User
}
giteaUsr := sess.Get("gitea_user")
if giteaUsr != nil {
ctx.GiteaUser = giteaUsr.(*User)
ctx.Data["GiteaUser"] = ctx.GiteaUser
}
if ctx.User != nil && ctx.User.Token != "" {
tc := oauth2.NewClient(bgctx.Background(), oauth2.StaticTokenSource(&oauth2.Token{AccessToken: ctx.User.Token}))
ctx.Client = github.NewClient(tc)
} else {
ctx.Client = github.NewClient(nil)
}
if giteaURL, ok := sess.Get("gitea").(string); ok && giteaURL != "" && ctx.GiteaUser != nil && ctx.GiteaUser.Token != "" {
ctx.GiteaClient = gitea.NewClient(giteaURL, ctx.GiteaUser.Token)
}
c.Map(ctx)
}
}

+ 58
- 0
web/fs.go View File

@@ -0,0 +1,58 @@
package web

import (
"bytes"
"io"
"path"
"strings"

"github.com/gobuffalo/packr"
"gopkg.in/macaron.v1"
)

// BundledFS implements ServeFileSystem for packr.Box
type BundledFS struct {
packr.Box
}

// Exists returns true if filepath exists
func (fs *BundledFS) Exists(prefix string, filepath string) bool {
if p := strings.TrimPrefix(filepath, prefix); len(p) < len(filepath) {
return fs.Has(p)
}
return false
}

// ListFiles returns all files in FS
func (fs *BundledFS) ListFiles() (files []macaron.TemplateFile) {
for _, filename := range fs.List() {
files = append(files, &BundledFile{fs: fs, FileName: filename})
}
return files
}

// Get returns the content of filename
func (fs *BundledFS) Get(filename string) (io.Reader, error) {
return bytes.NewReader(fs.Bytes(filename)), nil
}

// BundledFile represents a file in a BundledFS
type BundledFile struct {
fs *BundledFS
FileName string
}

// Name represents the name of the file
func (b *BundledFile) Name() string {
return strings.TrimSuffix(b.FileName, path.Ext(b.FileName))
}

// Data returns the content of file
func (b *BundledFile) Data() []byte {
return b.fs.Bytes(b.FileName)
}

// Ext returns the file extension
func (b *BundledFile) Ext() string {
return path.Ext(b.FileName)
}

+ 62
- 0
web/migration/repos.go View File

@@ -0,0 +1,62 @@
package migration

import (
bgctx "context"
"regexp"
"strings"

"git.jonasfranz.software/JonasFranzDEV/gitea-github-migrator/migrations"
"git.jonasfranz.software/JonasFranzDEV/gitea-github-migrator/web/context"
"github.com/google/go-github/github"
)

const repoRegex = "^[A-Za-z0-9-.]+/[A-Za-z0-9-.]+$"

// ListRepos shows all available repos of the signed in user
func ListRepos(ctx *context.Context) {
repos, _, err := ctx.Client.Repositories.List(bgctx.Background(), ctx.User.Username, &github.RepositoryListOptions{
ListOptions: github.ListOptions{
PerPage: 100,
},
})

if err != nil {
ctx.Handle(500, "list repositories", err)
return
}
ctx.Data["Repos"] = repos
ctx.HTML(200, "repos")
}

// ListReposPost handles the form submission of ListRepos
func ListReposPost(ctx *context.Context) {
if err := ctx.Req.ParseForm(); err != nil {
ctx.Handle(500, "parse form", err)
return
}
// TODO implement migration options
job := migrations.NewJob(&migrations.Options{
Labels: true,
Comments: true,
Issues: true,
Milestones: true,
PullRequests: true,
Strategy: migrations.Classic,
NewOwnerID: int(ctx.GiteaUser.ID), // TODO implement user/org selection
}, ctx.GiteaClient, ctx.Client)
for repo, val := range ctx.Req.Form {
activated := strings.Join(val, "")
if activated != "on" {
continue
}
// Validate repo format (reponame/owner)
if matched, err := regexp.MatchString(repoRegex, repo); err != nil || !matched {
continue
}
job.Repositories = append(job.Repositories, repo)
}
go job.StartMigration()
ctx.SetCurrentJob(job)
ctx.Data["StatusReport"] = job.StatusReport()
ctx.HTML(200, "migration")
}

+ 14
- 0
web/migration/status.go View File

@@ -0,0 +1,14 @@
package migration

import (
"git.jonasfranz.software/JonasFranzDEV/gitea-github-migrator/web/context"
)

// StatusReport returns the status of the current Job in JSON
func StatusReport(ctx *context.Context) {
if job := ctx.GetCurrentJob(); job != nil {
ctx.JSON(200, job.StatusReport())
return
}
ctx.Status(404)
}

+ 364
- 0
web/public/css/semantic.min.css
File diff suppressed because it is too large
View File


+ 4
- 0
web/public/js/jquery-3.1.1.min.js
File diff suppressed because it is too large
View File


+ 121
- 0
web/public/js/repos-status.js View File

@@ -0,0 +1,121 @@
var done = false;

function update() {
$.getJSON("/status", function(data) {
handleData(data);
}).always(function() {
if(!done){
window.setTimeout(update, 1000);
}
});
}
$(function() {
$(".repo-progress").progress({
text: {
active : 'Migrated or failed {value} of {total} repositories',
success : '{total} repositories migrated or failed!'
},
total: $(".repo-progress").data("total"),
value: 0
});
});

function handleData(data) {
if(Object.keys(data.finished).length + Object.keys(data.failed).length === $(".repo-progress").progress('get total')) {
$(".repo-progress").progress('complete');
done = true;
} else {
$(".repo-progress").progress('set progress', Object.keys(data.finished).length + Object.keys(data.failed).length);
}
data.pending.forEach(function(repo) {
var content = contentFromRepo(repo);
if (!content.hasClass("pending")) {
content.html(renderPending().html());
content.addClass("pending");
}
});
forEach(data.failed, function (repo, errormsg) {
var content = contentFromRepo(repo);
if (!content.hasClass("failed")) {
content.html(renderFailed(errormsg).html());
content.addClass("failed");
}
});
forEach(data.running, function (repo, report) {
var content = handleNonPending(repo, report);
if (content.find(".comment-progress").progress('get total') !== report.total_comments) {
content.find(".comment-progress").progress('set total', report.total_comments)
}
if (content.find(".issue-progress").progress('get total') !== report.total_issues) {
content.find(".issue-progress").progress('set total', report.total_issues)
}
content.find(".comment-progress").progress('set progress', report.migrated_comments + report.failed_comments);
content.find(".issue-progress").progress('set progress', report.migrated_issues + report.failed_issues);
});
forEach(data.finished, function (repo, report) {
var content = handleNonPending(repo, report);
if (content.find(".comment-progress").progress('get total') !== report.total_comments) {
content.find(".comment-progress").progress('set total', report.total_comments)
}
if (content.find(".issue-progress").progress('get total') !== report.total_issues) {
content.find(".issue-progress").progress('set total', report.total_issues)
}
content.find(".comment-progress").progress('set progress', report.migrated_comments + report.failed_comments);
content.find(".issue-progress").progress('set progress', report.migrated_issues + report.failed_issues);
content.find(".issue-progress").progress('complete');
content.find(".comment-progress").progress('complete');
});
}
function forEach(object, callback) {
for(var prop in object) {
if(object.hasOwnProperty(prop)) {
callback(prop, object[prop]);
}