e759794408
Also convert a comment into a warning in the logs when the deletion of an artifact cannot find the file in the destination storage. The case were an error happens while deleting the file is not covered as it would require to mock the storage.Copy function.
267 lines
8 KiB
Go
267 lines
8 KiB
Go
// Copyright 2020 The Gitea Authors. All rights reserved.
|
|
// SPDX-License-Identifier: MIT
|
|
|
|
package cmd
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io/fs"
|
|
"strings"
|
|
|
|
actions_model "code.gitea.io/gitea/models/actions"
|
|
"code.gitea.io/gitea/models/db"
|
|
git_model "code.gitea.io/gitea/models/git"
|
|
"code.gitea.io/gitea/models/migrations"
|
|
packages_model "code.gitea.io/gitea/models/packages"
|
|
repo_model "code.gitea.io/gitea/models/repo"
|
|
user_model "code.gitea.io/gitea/models/user"
|
|
"code.gitea.io/gitea/modules/log"
|
|
packages_module "code.gitea.io/gitea/modules/packages"
|
|
"code.gitea.io/gitea/modules/setting"
|
|
"code.gitea.io/gitea/modules/storage"
|
|
|
|
"github.com/urfave/cli/v2"
|
|
)
|
|
|
|
// CmdMigrateStorage represents the available migrate storage sub-command.
|
|
var CmdMigrateStorage = &cli.Command{
|
|
Name: "migrate-storage",
|
|
Usage: "Migrate the storage",
|
|
Description: "Copies stored files from storage configured in app.ini to parameter-configured storage",
|
|
Action: runMigrateStorage,
|
|
Flags: []cli.Flag{
|
|
&cli.StringFlag{
|
|
Name: "type",
|
|
Aliases: []string{"t"},
|
|
Value: "",
|
|
Usage: "Type of stored files to copy. Allowed types: 'attachments', 'lfs', 'avatars', 'repo-avatars', 'repo-archivers', 'packages', 'actions-log', 'actions-artifacts'",
|
|
},
|
|
&cli.StringFlag{
|
|
Name: "storage",
|
|
Aliases: []string{"s"},
|
|
Value: "",
|
|
Usage: "New storage type: local (default) or minio",
|
|
},
|
|
&cli.StringFlag{
|
|
Name: "path",
|
|
Aliases: []string{"p"},
|
|
Value: "",
|
|
Usage: "New storage placement if store is local (leave blank for default)",
|
|
},
|
|
&cli.StringFlag{
|
|
Name: "minio-endpoint",
|
|
Value: "",
|
|
Usage: "Minio storage endpoint",
|
|
},
|
|
&cli.StringFlag{
|
|
Name: "minio-access-key-id",
|
|
Value: "",
|
|
Usage: "Minio storage accessKeyID",
|
|
},
|
|
&cli.StringFlag{
|
|
Name: "minio-secret-access-key",
|
|
Value: "",
|
|
Usage: "Minio storage secretAccessKey",
|
|
},
|
|
&cli.StringFlag{
|
|
Name: "minio-bucket",
|
|
Value: "",
|
|
Usage: "Minio storage bucket",
|
|
},
|
|
&cli.StringFlag{
|
|
Name: "minio-location",
|
|
Value: "",
|
|
Usage: "Minio storage location to create bucket",
|
|
},
|
|
&cli.StringFlag{
|
|
Name: "minio-base-path",
|
|
Value: "",
|
|
Usage: "Minio storage base path on the bucket",
|
|
},
|
|
&cli.BoolFlag{
|
|
Name: "minio-use-ssl",
|
|
Usage: "Enable SSL for minio",
|
|
},
|
|
&cli.BoolFlag{
|
|
Name: "minio-insecure-skip-verify",
|
|
Usage: "Skip SSL verification",
|
|
},
|
|
&cli.StringFlag{
|
|
Name: "minio-checksum-algorithm",
|
|
Value: "",
|
|
Usage: "Minio checksum algorithm (default/md5)",
|
|
},
|
|
},
|
|
}
|
|
|
|
func migrateAttachments(ctx context.Context, dstStorage storage.ObjectStorage) error {
|
|
return db.Iterate(ctx, nil, func(ctx context.Context, attach *repo_model.Attachment) error {
|
|
_, err := storage.Copy(dstStorage, attach.RelativePath(), storage.Attachments, attach.RelativePath())
|
|
return err
|
|
})
|
|
}
|
|
|
|
func migrateLFS(ctx context.Context, dstStorage storage.ObjectStorage) error {
|
|
return db.Iterate(ctx, nil, func(ctx context.Context, mo *git_model.LFSMetaObject) error {
|
|
_, err := storage.Copy(dstStorage, mo.RelativePath(), storage.LFS, mo.RelativePath())
|
|
return err
|
|
})
|
|
}
|
|
|
|
func migrateAvatars(ctx context.Context, dstStorage storage.ObjectStorage) error {
|
|
return db.Iterate(ctx, nil, func(ctx context.Context, user *user_model.User) error {
|
|
if user.CustomAvatarRelativePath() == "" {
|
|
return nil
|
|
}
|
|
_, err := storage.Copy(dstStorage, user.CustomAvatarRelativePath(), storage.Avatars, user.CustomAvatarRelativePath())
|
|
return err
|
|
})
|
|
}
|
|
|
|
func migrateRepoAvatars(ctx context.Context, dstStorage storage.ObjectStorage) error {
|
|
return db.Iterate(ctx, nil, func(ctx context.Context, repo *repo_model.Repository) error {
|
|
if repo.CustomAvatarRelativePath() == "" {
|
|
return nil
|
|
}
|
|
_, err := storage.Copy(dstStorage, repo.CustomAvatarRelativePath(), storage.RepoAvatars, repo.CustomAvatarRelativePath())
|
|
return err
|
|
})
|
|
}
|
|
|
|
func migrateRepoArchivers(ctx context.Context, dstStorage storage.ObjectStorage) error {
|
|
return db.Iterate(ctx, nil, func(ctx context.Context, archiver *repo_model.RepoArchiver) error {
|
|
p := archiver.RelativePath()
|
|
_, err := storage.Copy(dstStorage, p, storage.RepoArchives, p)
|
|
return err
|
|
})
|
|
}
|
|
|
|
func migratePackages(ctx context.Context, dstStorage storage.ObjectStorage) error {
|
|
return db.Iterate(ctx, nil, func(ctx context.Context, pb *packages_model.PackageBlob) error {
|
|
p := packages_module.KeyToRelativePath(packages_module.BlobHash256Key(pb.HashSHA256))
|
|
_, err := storage.Copy(dstStorage, p, storage.Packages, p)
|
|
return err
|
|
})
|
|
}
|
|
|
|
func migrateActionsLog(ctx context.Context, dstStorage storage.ObjectStorage) error {
|
|
return db.Iterate(ctx, nil, func(ctx context.Context, task *actions_model.ActionTask) error {
|
|
if task.LogExpired {
|
|
// the log has been cleared
|
|
return nil
|
|
}
|
|
if !task.LogInStorage {
|
|
// running tasks store logs in DBFS
|
|
return nil
|
|
}
|
|
p := task.LogFilename
|
|
_, err := storage.Copy(dstStorage, p, storage.Actions, p)
|
|
return err
|
|
})
|
|
}
|
|
|
|
func migrateActionsArtifacts(ctx context.Context, dstStorage storage.ObjectStorage) error {
|
|
return db.Iterate(ctx, nil, func(ctx context.Context, artifact *actions_model.ActionArtifact) error {
|
|
if artifact.Status == int64(actions_model.ArtifactStatusExpired) {
|
|
return nil
|
|
}
|
|
|
|
_, err := storage.Copy(dstStorage, artifact.StoragePath, storage.ActionsArtifacts, artifact.StoragePath)
|
|
if err != nil {
|
|
if errors.Is(err, fs.ErrNotExist) {
|
|
log.Warn("ignored: actions artifact %s exists in the database but not in storage", artifact.StoragePath)
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
})
|
|
}
|
|
|
|
func runMigrateStorage(ctx *cli.Context) error {
|
|
stdCtx, cancel := installSignals()
|
|
defer cancel()
|
|
|
|
if err := initDB(stdCtx); err != nil {
|
|
return err
|
|
}
|
|
|
|
log.Info("AppPath: %s", setting.AppPath)
|
|
log.Info("AppWorkPath: %s", setting.AppWorkPath)
|
|
log.Info("Custom path: %s", setting.CustomPath)
|
|
log.Info("Log path: %s", setting.Log.RootPath)
|
|
log.Info("Configuration file: %s", setting.CustomConf)
|
|
|
|
if err := db.InitEngineWithMigration(context.Background(), migrations.Migrate); err != nil {
|
|
log.Fatal("Failed to initialize ORM engine: %v", err)
|
|
return err
|
|
}
|
|
|
|
if err := storage.Init(); err != nil {
|
|
return err
|
|
}
|
|
|
|
var dstStorage storage.ObjectStorage
|
|
var err error
|
|
switch strings.ToLower(ctx.String("storage")) {
|
|
case "":
|
|
fallthrough
|
|
case string(setting.LocalStorageType):
|
|
p := ctx.String("path")
|
|
if p == "" {
|
|
log.Fatal("Path must be given when storage is local")
|
|
return nil
|
|
}
|
|
dstStorage, err = storage.NewLocalStorage(
|
|
stdCtx,
|
|
&setting.Storage{
|
|
Path: p,
|
|
})
|
|
case string(setting.MinioStorageType):
|
|
dstStorage, err = storage.NewMinioStorage(
|
|
stdCtx,
|
|
&setting.Storage{
|
|
MinioConfig: setting.MinioStorageConfig{
|
|
Endpoint: ctx.String("minio-endpoint"),
|
|
AccessKeyID: ctx.String("minio-access-key-id"),
|
|
SecretAccessKey: ctx.String("minio-secret-access-key"),
|
|
Bucket: ctx.String("minio-bucket"),
|
|
Location: ctx.String("minio-location"),
|
|
BasePath: ctx.String("minio-base-path"),
|
|
UseSSL: ctx.Bool("minio-use-ssl"),
|
|
InsecureSkipVerify: ctx.Bool("minio-insecure-skip-verify"),
|
|
ChecksumAlgorithm: ctx.String("minio-checksum-algorithm"),
|
|
},
|
|
})
|
|
default:
|
|
return fmt.Errorf("unsupported storage type: %s", ctx.String("storage"))
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
migratedMethods := map[string]func(context.Context, storage.ObjectStorage) error{
|
|
"attachments": migrateAttachments,
|
|
"lfs": migrateLFS,
|
|
"avatars": migrateAvatars,
|
|
"repo-avatars": migrateRepoAvatars,
|
|
"repo-archivers": migrateRepoArchivers,
|
|
"packages": migratePackages,
|
|
"actions-log": migrateActionsLog,
|
|
"actions-artifacts": migrateActionsArtifacts,
|
|
}
|
|
|
|
tp := strings.ToLower(ctx.String("type"))
|
|
if m, ok := migratedMethods[tp]; ok {
|
|
if err := m(stdCtx, dstStorage); err != nil {
|
|
return err
|
|
}
|
|
log.Info("%s files have successfully been copied to the new storage.", tp)
|
|
return nil
|
|
}
|
|
|
|
return fmt.Errorf("unsupported storage: %s", ctx.String("type"))
|
|
}
|