diff --git a/cmd/backup.go b/cmd/backup.go new file mode 100644 index 000000000..dd8838663 --- /dev/null +++ b/cmd/backup.go @@ -0,0 +1,194 @@ +// Copyright 2017 the Gitea Authors. All rights reserved. +// Copyright 2017 The Gogs Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package cmd + +import ( + "fmt" + "io/ioutil" + "os" + "path" + "runtime/debug" + "time" + + "github.com/Unknwon/cae/zip" + "github.com/Unknwon/com" + "github.com/urfave/cli" + "gopkg.in/ini.v1" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" +) + +// Backup files and database +var Backup = cli.Command{ + Name: "backup", + Usage: "Backup files and database", + Description: `Backup dumps and compresses all related files and database into zip file, + which can be used for migrating Gitea to another server. The output format is meant to be + portable among all supported database engines.`, + Action: runBackup, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "config, c", + Value: "custom/conf/app.ini", + Usage: "Custom configuration `FILE` path", + }, + cli.StringFlag{ + Name: "tempdir, t", + Value: os.TempDir(), + Usage: "Temporary directory `PATH`", + }, + cli.StringFlag{ + Name: "target", + Value: "./", + Usage: "Target directory `PATH` to save backup archive", + }, + cli.BoolFlag{ + Name: "verbose, v", + Usage: "Show process details", + }, + cli.BoolTFlag{ + Name: "db", + Usage: "Backup the database (default: true)", + }, + cli.BoolTFlag{ + Name: "repos", + Usage: "Backup repositories (default: true)", + }, + cli.BoolTFlag{ + Name: "data", + Usage: "Backup attachments and avatars (default: true)", + }, + cli.BoolTFlag{ + Name: "custom", + Usage: "Backup custom files (default: true)", + }, + }, +} + +const archiveRootDir = "gitea-backup" + +func runBackup(c *cli.Context) error { + zip.Verbose = c.Bool("verbose") + if c.IsSet("config") { + setting.CustomConf = c.String("config") + } + setting.NewContext() + models.LoadConfigs() + if err := models.SetEngine(); err != nil { + return err + } + + // Setup temp-dir + tmpDir := c.String("tempdir") + if !com.IsExist(tmpDir) { + log.Fatal(0, "'--tempdir' does not exist: %s", tmpDir) + } + rootDir, err := ioutil.TempDir(tmpDir, "gitea-backup-") + if err != nil { + log.Fatal(0, "Fail to create backup root directory '%s': %v", rootDir, err) + } + defer func(rootDir string) { + os.RemoveAll(rootDir) + }(rootDir) + log.Info("Backup root directory: %s", rootDir) + + // Metadata + metaFile := path.Join(rootDir, "metadata.ini") + metadata := ini.Empty() + metadata.Section("").Key("VERSION").SetValue("1") + metadata.Section("").Key("DATE_TIME").SetValue(time.Now().String()) + metadata.Section("").Key("GITEA_VERSION").SetValue(setting.AppVer) + if err = metadata.SaveTo(metaFile); err != nil { + log.Fatal(0, "Fail to save metadata '%s': %v", metaFile, err) + } + + // Create ZIP-file + archiveName := path.Join(c.String("target"), fmt.Sprintf("gitea-backup-%d.zip", time.Now().Unix())) + log.Info("Packing backup files to: %s", archiveName) + + z, err := zip.Create(archiveName) + if err != nil { + log.Fatal(0, "Fail to create backup archive '%s': %v", archiveName, err) + } + defer func(archiveName string) { + if r := recover(); r != nil { + var ok bool + err, ok = r.(error) + if !ok { + err = fmt.Errorf("pkg: %v", r) + } + debug.PrintStack() + log.Info("Removing partial backup-file %s\n", archiveName) + os.Remove(archiveName) + log.Fatal(9, "%v\n", err) + } + }(archiveName) + + // Add metadata-file + if err = z.AddFile(archiveRootDir+"/metadata.ini", metaFile); err != nil { + log.Fatal(0, "Fail to include 'metadata.ini': %v", err) + } + + // Database + if c.Bool("db") { + log.Info("Backing up database") + dbDir := path.Join(rootDir, "db") + if err = models.DumpDatabase(dbDir); err != nil { + log.Fatal(0, "Fail to dump database: %v", err) + } + if err = z.AddDir(archiveRootDir+"/db", dbDir); err != nil { + log.Fatal(0, "Fail to include 'db': %v", err) + } + } + + // Custom files + if c.Bool("custom") { + log.Info("Backing up custom files") + if err = z.AddDir(archiveRootDir+"/custom", setting.CustomPath); err != nil { + log.Fatal(0, "Fail to include 'custom': %v", err) + } + } + + // Data files + if c.Bool("data") { + log.Info("Backing up attachments and avatars") + for _, dir := range []string{"attachments", "avatars"} { + dirPath := path.Join(setting.AppDataPath, dir) + if !com.IsDir(dirPath) { + continue + } + + if err = z.AddDir(path.Join(archiveRootDir+"/data", dir), dirPath); err != nil { + log.Fatal(0, "Fail to include 'data': %v", err) + } + } + } + + // Repositories + if c.Bool("repos") { + log.Info("Backing up repositories") + reposDump := path.Join(rootDir, "repositories.zip") + log.Info("Dumping repositories in '%s'", setting.RepoRootPath) + if err = zip.PackTo(setting.RepoRootPath, reposDump, true); err != nil { + log.Fatal(0, "Fail to dump repositories: %v", err) + } + log.Info("Repositories dumped to: %s", reposDump) + + if err = z.AddFile(archiveRootDir+"/repositories.zip", reposDump); err != nil { + log.Fatal(0, "Fail to include 'repositories.zip': %v", err) + } + } + + if err = z.Close(); err != nil { + log.Fatal(0, "Fail to save backup archive '%s': %v", archiveName, err) + } + + os.RemoveAll(rootDir) + log.Info("Backup succeed! Archive is located at: %s", archiveName) + return nil +} diff --git a/cmd/dump.go b/cmd/dump.go index 98478c17b..661156f87 100644 --- a/cmd/dump.go +++ b/cmd/dump.go @@ -8,13 +8,13 @@ package cmd import ( "fmt" "io/ioutil" - "log" "os" "path" "path/filepath" "time" "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "github.com/Unknwon/cae/zip" @@ -22,10 +22,10 @@ import ( "github.com/urfave/cli" ) -// CmdDump represents the available dump sub-command. -var CmdDump = cli.Command{ +// Dump represents the available dump sub-command. +var Dump = cli.Command{ Name: "dump", - Usage: "Dump Gitea files and database", + Usage: "DEPRICATED! Dump Gitea files and database", Description: `Dump compresses all related files and database into zip file. It can be used for backup and capture Gitea server image to send to maintainer`, Action: runDump, @@ -59,65 +59,66 @@ func runDump(ctx *cli.Context) error { setting.NewServices() // cannot access session settings otherwise models.LoadConfigs() - err := models.SetEngine() - if err != nil { + log.Info("") + + if err := models.SetEngine(); err != nil { return err } tmpDir := ctx.String("tempdir") if _, err := os.Stat(tmpDir); os.IsNotExist(err) { - log.Fatalf("Path does not exist: %s", tmpDir) + log.Fatal(4, "Path does not exist: %s", tmpDir) } TmpWorkDir, err := ioutil.TempDir(tmpDir, "gitea-dump-") if err != nil { - log.Fatalf("Failed to create tmp work directory: %v", err) + log.Fatal(4, "Failed to create tmp work directory: %v", err) } - log.Printf("Creating tmp work dir: %s", TmpWorkDir) + log.Info("Creating tmp work dir: %s", TmpWorkDir) reposDump := path.Join(TmpWorkDir, "gitea-repo.zip") dbDump := path.Join(TmpWorkDir, "gitea-db.sql") - log.Printf("Dumping local repositories...%s", setting.RepoRootPath) + log.Info("Dumping local repositories...%s", setting.RepoRootPath) zip.Verbose = ctx.Bool("verbose") - if err := zip.PackTo(setting.RepoRootPath, reposDump, true); err != nil { - log.Fatalf("Failed to dump local repositories: %v", err) + if err = zip.PackTo(setting.RepoRootPath, reposDump, true); err != nil { + log.Fatal(4, "Failed to dump local repositories: %v", err) } targetDBType := ctx.String("database") if len(targetDBType) > 0 && targetDBType != models.DbCfg.Type { - log.Printf("Dumping database %s => %s...", models.DbCfg.Type, targetDBType) + log.Info("Dumping database %s => %s...", models.DbCfg.Type, targetDBType) } else { - log.Printf("Dumping database...") + log.Info("Dumping database...") } - if err := models.DumpDatabase(dbDump, targetDBType); err != nil { - log.Fatalf("Failed to dump database: %v", err) + if err = models.DumpDatabaseOld(dbDump, targetDBType); err != nil { + log.Fatal(4, "Failed to dump database: %v", err) } fileName := fmt.Sprintf("gitea-dump-%d.zip", time.Now().Unix()) - log.Printf("Packing dump files...") + log.Info("Packing dump files...") z, err := zip.Create(fileName) if err != nil { - log.Fatalf("Failed to create %s: %v", fileName, err) + log.Fatal(4, "Failed to create %s: %v", fileName, err) } - if err := z.AddFile("gitea-repo.zip", reposDump); err != nil { - log.Fatalf("Failed to include gitea-repo.zip: %v", err) + if err = z.AddFile("gitea-repo.zip", reposDump); err != nil { + log.Fatal(4, "Failed to include gitea-repo.zip: %v", err) } - if err := z.AddFile("gitea-db.sql", dbDump); err != nil { - log.Fatalf("Failed to include gitea-db.sql: %v", err) + if err = z.AddFile("gitea-db.sql", dbDump); err != nil { + log.Fatal(4, "Failed to include gitea-db.sql: %v", err) } customDir, err := os.Stat(setting.CustomPath) if err == nil && customDir.IsDir() { - if err := z.AddDir("custom", setting.CustomPath); err != nil { - log.Fatalf("Failed to include custom: %v", err) + if err = z.AddDir("custom", setting.CustomPath); err != nil { + log.Fatal(4, "Failed to include custom: %v", err) } } else { - log.Printf("Custom dir %s doesn't exist, skipped", setting.CustomPath) + log.Info("Custom dir %s doesn't exist, skipped", setting.CustomPath) } if com.IsExist(setting.AppDataPath) { - log.Printf("Packing data directory...%s", setting.AppDataPath) + log.Info("Packing data directory...%s", setting.AppDataPath) var sessionAbsPath string if setting.SessionConfig.Provider == "file" { @@ -126,30 +127,30 @@ func runDump(ctx *cli.Context) error { } sessionAbsPath, _ = filepath.Abs(setting.SessionConfig.ProviderConfig) } - if err := zipAddDirectoryExclude(z, "data", setting.AppDataPath, sessionAbsPath); err != nil { - log.Fatalf("Failed to include data directory: %v", err) + if err = zipAddDirectoryExclude(z, "data", setting.AppDataPath, sessionAbsPath); err != nil { + log.Fatal(4, "Failed to include data directory: %v", err) } } - if err := z.AddDir("log", setting.LogRootPath); err != nil { - log.Fatalf("Failed to include log: %v", err) + if err = z.AddDir("log", setting.LogRootPath); err != nil { + log.Fatal(4, "Failed to include log: %v", err) } if err = z.Close(); err != nil { _ = os.Remove(fileName) - log.Fatalf("Failed to save %s: %v", fileName, err) + log.Fatal(4, "Failed to save %s: %v", fileName, err) } if err := os.Chmod(fileName, 0600); err != nil { - log.Printf("Can't change file access permissions mask to 0600: %v", err) + log.Info("Can't change file access permissions mask to 0600: %v", err) } - log.Printf("Removing tmp work dir: %s", TmpWorkDir) + log.Info("Removing tmp work dir: %s", TmpWorkDir) if err := os.RemoveAll(TmpWorkDir); err != nil { - log.Fatalf("Failed to remove %s: %v", TmpWorkDir, err) + log.Fatal(4, "Failed to remove %s: %v", TmpWorkDir, err) } - log.Printf("Finish dumping in file %s", fileName) + log.Info("Finish dumping in file %s", fileName) return nil } diff --git a/cmd/restore.go b/cmd/restore.go new file mode 100644 index 000000000..e883617a1 --- /dev/null +++ b/cmd/restore.go @@ -0,0 +1,155 @@ +// Copyright 2017 The Gogs Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package cmd + +import ( + "os" + "path" + + "github.com/Unknwon/cae/zip" + "github.com/Unknwon/com" + "github.com/mcuadros/go-version" + "github.com/urfave/cli" + "gopkg.in/ini.v1" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" +) + +var Restore = cli.Command{ + Name: "restore", + Usage: "Restore files and database from backup", + Description: `Restore imports all related files and database from a backup archive. +The backup version must lower or equal to current Gitea version. You can also import +backup from other database engines, which is useful for database migrating. + +If corresponding files or database tables are not presented in the archive, they will +be skipped and remian unchanged.`, + Action: runRestore, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "config, c", + Value: "custom/conf/app.ini", + Usage: "Custom configuration file path", + }, + cli.StringFlag{ + Name: "tempdir, t", + Value: os.TempDir(), + Usage: "Temporary directory path", + }, + cli.StringFlag{ + Name: "from", + Value: "", + Usage: "Path to backup archive", + }, + cli.BoolFlag{ + Name: "verbose, v", + Usage: "Show process details", + }, + cli.BoolFlag{ + Name: "no-repos", + Usage: "Don't restore repositiries", + }, + cli.BoolFlag{ + Name: "no-metadata", + Usage: "Don't restore metadata (such as attachments and avatars)", + }, + cli.BoolFlag{ + Name: "no-custom", + Usage: "Don't restore custom files", + }, + }, +} + +func runRestore(c *cli.Context) error { + zip.Verbose = c.Bool("verbose") + + tmpDir := c.String("tempdir") + if !com.IsExist(tmpDir) { + log.Fatal(0, "'--tempdir' does not exist: %s", tmpDir) + } + + log.Info("Restore backup from: %s", c.String("from")) + if err := zip.ExtractTo(c.String("from"), tmpDir); err != nil { + log.Fatal(0, "Fail to extract backup archive: %v", err) + } + archivePath := path.Join(tmpDir, archiveRootDir) + + // Check backup version + metaFile := path.Join(archivePath, "metadata.ini") + if !com.IsExist(metaFile) { + log.Fatal(0, "File 'metadata.ini' is missing") + } + metadata, err := ini.Load(metaFile) + if err != nil { + log.Fatal(0, "Fail to load metadata '%s': %v", metaFile, err) + } + backupVersion := metadata.Section("").Key("GITEA_VERSION").MustString("999.0") + if version.Compare(setting.AppVer, backupVersion, "<") { + log.Fatal(0, "Current Gitea version is lower than backup version: %s < %s", setting.AppVer, backupVersion) + } + + // If config file is not present in backup, user must set this file via flag. + // Otherwise, it's optional to set config file flag. + configFile := path.Join(archivePath, "custom/conf/app.ini") + if c.IsSet("config") { + setting.CustomConf = c.String("config") + } else if !com.IsExist(configFile) { + log.Fatal(0, "'--config' is not specified and custom config file is not found in backup") + } else { + setting.CustomConf = configFile + } + setting.NewContext() + models.LoadConfigs() + models.SetEngine() + + // Database + dbDir := path.Join(archivePath, "db") + if err = models.ImportDatabase(dbDir); err != nil { + log.Fatal(0, "Fail to import database: %v", err) + } + + // Custom files + if !c.Bool("no-custom") { + if com.IsExist(setting.CustomPath) { + if err = os.Rename(setting.CustomPath, setting.CustomPath+".bak"); err != nil { + log.Fatal(0, "Fail to backup current 'custom': %v", err) + } + } + if err = os.Rename(path.Join(archivePath, "custom"), setting.CustomPath); err != nil { + log.Fatal(0, "Fail to import 'custom': %v", err) + } + } + + // Data files + if !c.Bool("no-metadata") { + for _, dir := range []string{"attachments", "avatars"} { + dirPath := path.Join(setting.AppDataPath, dir) + if com.IsExist(dirPath) { + if err = os.Rename(dirPath, dirPath+".bak"); err != nil { + log.Fatal(0, "Fail to backup current 'data': %v", err) + } + } + if err = os.Rename(path.Join(archivePath, "data", dir), dirPath); err != nil { + log.Fatal(0, "Fail to import 'data': %v", err) + } + } + } + + // Repositories + if !c.Bool("no-repos") { + reposPath := path.Join(archivePath, "repositories.zip") + if !c.Bool("exclude-repos") && !c.Bool("database-only") && com.IsExist(reposPath) { + if err := zip.ExtractTo(reposPath, path.Dir(setting.RepoRootPath)); err != nil { + log.Fatal(0, "Fail to extract 'repositories.zip': %v", err) + } + } + } + + os.RemoveAll(path.Join(tmpDir, archiveRootDir)) + log.Info("Restore succeed!") + return nil +} diff --git a/main.go b/main.go index 383dbc209..f321066b2 100644 --- a/main.go +++ b/main.go @@ -36,9 +36,11 @@ func main() { cmd.CmdWeb, cmd.CmdServ, cmd.CmdHook, - cmd.CmdDump, cmd.CmdCert, cmd.CmdAdmin, + cmd.Backup, + cmd.Restore, + cmd.Dump, } app.Flags = append(app.Flags, []cli.Flag{}...) err := app.Run(os.Args) diff --git a/models/models.go b/models/models.go index 02499b6eb..de3ee8e3a 100644 --- a/models/models.go +++ b/models/models.go @@ -5,7 +5,9 @@ package models import ( + "bufio" "database/sql" + "encoding/json" "errors" "fmt" "net/url" @@ -13,20 +15,21 @@ import ( "path" "strings" - // Needed for the MySQL driver - _ "github.com/go-sql-driver/mysql" + "code.gitea.io/gitea/models/migrations" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + + "github.com/Unknwon/com" "github.com/go-xorm/core" "github.com/go-xorm/xorm" + // Needed for the MySQL driver + _ "github.com/go-sql-driver/mysql" // Needed for the Postgresql driver _ "github.com/lib/pq" // Needed for the MSSSQL driver _ "github.com/denisenkom/go-mssqldb" - - "code.gitea.io/gitea/models/migrations" - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/setting" ) // Engine represents a xorm engine or session. @@ -327,14 +330,104 @@ func Ping() error { return x.Ping() } -// DumpDatabase dumps all data from database according the special database SQL syntax to file system. -func DumpDatabase(filePath string, dbType string) error { +// Version describes the version table. Should have only one row with id==1 +type Version struct { + ID int64 `xorm:"pk autoincr"` + Version int64 +} + +// DumpDatabaseOld dumps all data from database according the special database SQL syntax to file system. +// NOTE: DEPRICATED in favour of DumpDatabase +func DumpDatabaseOld(filePath string, dbType string) error { var tbs []*core.Table for _, t := range tables { tbs = append(tbs, x.TableInfo(t).Table) + if len(dbType) > 0 { + return x.DumpTablesToFile(tbs, filePath, core.DbType(dbType)) + } + } + return nil +} + +// DumpDatabase dumps all data from database to file system in JSON format. +func DumpDatabase(dirPath string) (err error) { + os.MkdirAll(dirPath, os.ModePerm) + // Purposely create a local variable to not modify global variable + internalTables := append(tables, new(Version)) + for _, table := range internalTables { + tableName := strings.TrimPrefix(fmt.Sprintf("%T", table), "*models.") + tableFile := path.Join(dirPath, tableName+".json") + f, err := os.Create(tableFile) + if err != nil { + return fmt.Errorf("fail to create JSON file: %v", err) + } + + if err = x.Asc("id").Iterate(table, func(idx int, bean interface{}) (err error) { + enc := json.NewEncoder(f) + return enc.Encode(bean) + }); err != nil { + f.Close() + return fmt.Errorf("fail to dump table '%s': %v", tableName, err) + } + f.Close() } - if len(dbType) > 0 { - return x.DumpTablesToFile(tbs, filePath, core.DbType(dbType)) + return nil +} + +// ImportDatabase imports data from backup archive. +func ImportDatabase(dirPath string) (err error) { + // Purposely create a local variable to not modify global variable + internalTables := append(tables, new(Version)) + for _, table := range internalTables { + tableName := strings.TrimPrefix(fmt.Sprintf("%T", table), "*models.") + tableFile := path.Join(dirPath, tableName+".json") + if !com.IsExist(tableFile) { + continue + } + + if err = x.DropTables(table); err != nil { + return fmt.Errorf("fail to drop table '%s': %v", tableName, err) + } else if err = x.Sync2(table); err != nil { + return fmt.Errorf("fail to sync table '%s': %v", tableName, err) + } + + f, err := os.Open(tableFile) + if err != nil { + return fmt.Errorf("fail to open JSON file: %v", err) + } + scanner := bufio.NewScanner(f) + for scanner.Scan() { + switch bean := table.(type) { + case *LoginSource: + meta := make(map[string]interface{}) + if err = json.Unmarshal(scanner.Bytes(), &meta); err != nil { + return fmt.Errorf("fail to unmarshal to map: %v", err) + } + + tp := LoginType(com.StrTo(com.ToStr(meta["Type"])).MustInt64()) + switch tp { + case LoginLDAP, LoginDLDAP: + bean.Cfg = new(LDAPConfig) + case LoginSMTP: + bean.Cfg = new(SMTPConfig) + case LoginPAM: + bean.Cfg = new(PAMConfig) + case LoginOAuth2: + bean.Cfg = new(OAuth2Config) + default: + return fmt.Errorf("unrecognized login source type:: %v", tp) + } + table = bean + } + + if err = json.Unmarshal(scanner.Bytes(), table); err != nil { + return fmt.Errorf("fail to unmarshal to struct: %v", err) + } + + if _, err = x.Insert(table); err != nil { + return fmt.Errorf("fail to insert strcut: %v", err) + } + } } - return x.DumpTablesToFile(tbs, filePath) + return nil }