Skip to content

Add Status Updates whilst Gitea migrations are occurring #15076

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 11 commits into from
Jun 16, 2021
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions models/migrations/migrations.go
Original file line number Diff line number Diff line change
@@ -317,6 +317,8 @@ var migrations = []Migration{
NewMigration("Add issue resource index table", addIssueResourceIndexTable),
// v183 -> v184
NewMigration("Create PushMirror table", createPushMirrorTable),
// v184 -> v185
NewMigration("Rename Task errors to message", renameTaskErrorsToMessage),
}

// GetCurrentDBVersion returns the current db version
47 changes: 47 additions & 0 deletions models/migrations/v184.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package migrations

import (
"fmt"

"code.gitea.io/gitea/modules/setting"

"xorm.io/xorm"
)

func renameTaskErrorsToMessage(x *xorm.Engine) error {
type Task struct {
Errors string `xorm:"TEXT"` // if task failed, saved the error reason
Type int
Status int `xorm:"index"`
}

sess := x.NewSession()
defer sess.Close()
if err := sess.Begin(); err != nil {
return err
}

if err := sess.Sync2(new(Task)); err != nil {
return fmt.Errorf("error on Sync2: %v", err)
}

switch {
case setting.Database.UseMySQL:
if _, err := sess.Exec("ALTER TABLE `task` CHANGE errors message text"); err != nil {
return err
}
case setting.Database.UseMSSQL:
if _, err := sess.Exec("sp_rename 'task.errors', 'message', 'COLUMN'"); err != nil {
return err
}
default:
if _, err := sess.Exec("ALTER TABLE `task` RENAME COLUMN errors TO message"); err != nil {
return err
}
}
return sess.Commit()
}
8 changes: 7 additions & 1 deletion models/task.go
Original file line number Diff line number Diff line change
@@ -32,10 +32,16 @@ type Task struct {
StartTime timeutil.TimeStamp
EndTime timeutil.TimeStamp
PayloadContent string `xorm:"TEXT"`
Errors string `xorm:"TEXT"` // if task failed, saved the error reason
Message string `xorm:"TEXT"` // if task failed, saved the error reason
Created timeutil.TimeStamp `xorm:"created"`
}

// TranslatableMessage represents JSON struct that can be translated with a Locale
type TranslatableMessage struct {
Format string
Args []interface{} `json:"omitempty"`
}

// LoadRepo loads repository of the task
func (task *Task) LoadRepo() error {
return task.loadRepo(x)
11 changes: 11 additions & 0 deletions modules/migrations/base/messenger.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package base

// Messenger is a formatting function similar to i18n.Tr
type Messenger func(key string, args ...interface{})

// NilMessenger represents an empty formatting function
func NilMessenger(string, ...interface{}) {}
4 changes: 2 additions & 2 deletions modules/migrations/dump.go
Original file line number Diff line number Diff line change
@@ -555,7 +555,7 @@ func DumpRepository(ctx context.Context, baseDir, ownerName string, opts base.Mi
return err
}

if err := migrateRepository(downloader, uploader, opts); err != nil {
if err := migrateRepository(downloader, uploader, opts, nil); err != nil {
if err1 := uploader.Rollback(); err1 != nil {
log.Error("rollback failed: %v", err1)
}
@@ -620,7 +620,7 @@ func RestoreRepository(ctx context.Context, baseDir string, ownerName, repoName
}
updateOptionsUnits(&migrateOpts, units)

if err = migrateRepository(downloader, uploader, migrateOpts); err != nil {
if err = migrateRepository(downloader, uploader, migrateOpts, nil); err != nil {
if err1 := uploader.Rollback(); err1 != nil {
log.Error("rollback failed: %v", err1)
}
2 changes: 1 addition & 1 deletion modules/migrations/gitea_uploader_test.go
Original file line number Diff line number Diff line change
@@ -47,7 +47,7 @@ func TestGiteaUploadRepo(t *testing.T) {
PullRequests: true,
Private: true,
Mirror: false,
})
}, nil)
assert.NoError(t, err)

repo := models.AssertExistsAndLoadBean(t, &models.Repository{OwnerID: user.ID, Name: repoName}).(*models.Repository)
17 changes: 14 additions & 3 deletions modules/migrations/migrate.go
Original file line number Diff line number Diff line change
@@ -99,7 +99,7 @@ func IsMigrateURLAllowed(remoteURL string, doer *models.User) error {
}

// MigrateRepository migrate repository according MigrateOptions
func MigrateRepository(ctx context.Context, doer *models.User, ownerName string, opts base.MigrateOptions) (*models.Repository, error) {
func MigrateRepository(ctx context.Context, doer *models.User, ownerName string, opts base.MigrateOptions, messenger base.Messenger) (*models.Repository, error) {
err := IsMigrateURLAllowed(opts.CloneAddr, doer)
if err != nil {
return nil, err
@@ -118,7 +118,7 @@ func MigrateRepository(ctx context.Context, doer *models.User, ownerName string,
var uploader = NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName)
uploader.gitServiceType = opts.GitServiceType

if err := migrateRepository(downloader, uploader, opts); err != nil {
if err := migrateRepository(downloader, uploader, opts, messenger); err != nil {
if err1 := uploader.Rollback(); err1 != nil {
log.Error("rollback failed: %v", err1)
}
@@ -167,7 +167,11 @@ func newDownloader(ctx context.Context, ownerName string, opts base.MigrateOptio
// migrateRepository will download information and then upload it to Uploader, this is a simple
// process for small repository. For a big repository, save all the data to disk
// before upload is better
func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts base.MigrateOptions) error {
func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts base.MigrateOptions, messenger base.Messenger) error {
if messenger == nil {
messenger = base.NilMessenger
}

repo, err := downloader.GetRepoInfo()
if err != nil {
if !base.IsErrNotSupported(err) {
@@ -185,12 +189,14 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
}

log.Trace("migrating git data from %s", repo.CloneURL)
messenger("repo.migrate.migrating_git")
if err = uploader.CreateRepo(repo, opts); err != nil {
return err
}
defer uploader.Close()

log.Trace("migrating topics")
messenger("repo.migrate.migrating_topics")
topics, err := downloader.GetTopics()
if err != nil {
if !base.IsErrNotSupported(err) {
@@ -206,6 +212,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts

if opts.Milestones {
log.Trace("migrating milestones")
messenger("repo.migrate.migrating_milestones")
milestones, err := downloader.GetMilestones()
if err != nil {
if !base.IsErrNotSupported(err) {
@@ -229,6 +236,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts

if opts.Labels {
log.Trace("migrating labels")
messenger("repo.migrate.migrating_labels")
labels, err := downloader.GetLabels()
if err != nil {
if !base.IsErrNotSupported(err) {
@@ -252,6 +260,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts

if opts.Releases {
log.Trace("migrating releases")
messenger("repo.migrate.migrating_releases")
releases, err := downloader.GetReleases()
if err != nil {
if !base.IsErrNotSupported(err) {
@@ -285,6 +294,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts

if opts.Issues {
log.Trace("migrating issues and comments")
messenger("repo.migrate.migrating_issues")
var issueBatchSize = uploader.MaxBatchInsertSize("issue")

for i := 1; ; i++ {
@@ -339,6 +349,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts

if opts.PullRequests {
log.Trace("migrating pull requests and comments")
messenger("repo.migrate.migrating_pulls")
var prBatchSize = uploader.MaxBatchInsertSize("pullrequest")
for i := 1; ; i++ {
prs, isEnd, err := downloader.GetPullRequests(i, prBatchSize)
14 changes: 12 additions & 2 deletions modules/task/migrate.go
Original file line number Diff line number Diff line change
@@ -20,6 +20,7 @@ import (
"code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/timeutil"
"code.gitea.io/gitea/modules/util"
jsoniter "github.com/json-iterator/go"
)

func handleCreateError(owner *models.User, err error) error {
@@ -56,7 +57,7 @@ func runMigrateTask(t *models.Task) (err error) {

t.EndTime = timeutil.TimeStampNow()
t.Status = structs.TaskStatusFailed
t.Errors = err.Error()
t.Message = err.Error()
t.RepoID = 0
if err := t.UpdateCols("status", "errors", "repo_id", "end_time"); err != nil {
log.Error("Task UpdateCols failed: %v", err)
@@ -106,7 +107,16 @@ func runMigrateTask(t *models.Task) (err error) {
return
}

repo, err = migrations.MigrateRepository(ctx, t.Doer, t.Owner.Name, *opts)
repo, err = migrations.MigrateRepository(ctx, t.Doer, t.Owner.Name, *opts, func(format string, args ...interface{}) {
message := models.TranslatableMessage{
Format: format,
Args: args,
}
json := jsoniter.ConfigCompatibleWithStandardLibrary
bs, _ := json.Marshal(message)
t.Message = string(bs)
_ = t.UpdateCols("message")
})
if err == nil {
log.Trace("Repository migrated [%d]: %s/%s", repo.ID, t.Owner.Name, repo.Name)
return
8 changes: 8 additions & 0 deletions options/locale/locale_en-US.ini
Original file line number Diff line number Diff line change
@@ -824,11 +824,19 @@ migrated_from_fake = Migrated From %[1]s
migrate.migrate = Migrate From %s
migrate.migrating = Migrating from <b>%s</b> ...
migrate.migrating_failed = Migrating from <b>%s</b> failed.
migrate.migrating_failed.error = Error: %s
migrate.github.description = Migrating data from Github.com or Github Enterprise.
migrate.git.description = Migrating or Mirroring git data from Git services
migrate.gitlab.description = Migrating data from GitLab.com or Self-Hosted gitlab server.
migrate.gitea.description = Migrating data from Gitea.com or Self-Hosted Gitea server.
migrate.gogs.description = Migrating data from notabug.org or other Self-Hosted Gogs server.
migrate.migrating_git = Migrating Git Data
migrate.migrating_topics = Migrating Topics
migrate.migrating_milestones = Migrating Milestones
migrate.migrating_labels = Migrating Labels
migrate.migrating_releases = Migrating Releases
migrate.migrating_issues = Migrating Issues
migrate.migrating_pulls = Migrating Pull Requests

mirror_from = mirror of
forked_from = forked from
2 changes: 1 addition & 1 deletion routers/api/v1/repo/migrate.go
Original file line number Diff line number Diff line change
@@ -199,7 +199,7 @@ func Migrate(ctx *context.APIContext) {
}
}()

if _, err = migrations.MigrateRepository(graceful.GetManager().HammerContext(), ctx.User, repoOwner.Name, opts); err != nil {
if _, err = migrations.MigrateRepository(graceful.GetManager().HammerContext(), ctx.User, repoOwner.Name, opts, nil); err != nil {
handleMigrateError(ctx, repoOwner, remoteAddr, err)
return
}
18 changes: 17 additions & 1 deletion routers/web/user/task.go
Original file line number Diff line number Diff line change
@@ -9,6 +9,7 @@ import (

"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/context"
jsoniter "github.com/json-iterator/go"
)

// TaskStatus returns task's status
@@ -21,9 +22,24 @@ func TaskStatus(ctx *context.Context) {
return
}

message := task.Message

if task.Message != "" && task.Message[0] == '{' {
// assume message is actually a translatable string
json := jsoniter.ConfigCompatibleWithStandardLibrary
var translatableMessage models.TranslatableMessage
if err := json.Unmarshal([]byte(message), &translatableMessage); err != nil {
translatableMessage = models.TranslatableMessage{
Format: "migrate.migrating_failed.error",
Args: []interface{}{task.Message},
}
}
message = ctx.Tr(translatableMessage.Format, translatableMessage.Args...)
}

ctx.JSON(http.StatusOK, map[string]interface{}{
"status": task.Status,
"err": task.Errors,
"message": message,
"repo-id": task.RepoID,
"repo-name": opts.RepoName,
"start": task.StartTime,
1 change: 1 addition & 0 deletions templates/repo/migrate/migrating.tmpl
Original file line number Diff line number Diff line change
@@ -22,6 +22,7 @@
<div class="sixteen wide center aligned centered column">
<div id="repo_migrating_progress">
<p>{{.i18n.Tr "repo.migrate.migrating" .CloneAddr | Safe}}</p>
<p id="repo_migrating_progress_message"></p>
</div>
<div id="repo_migrating_failed" hidden>
<p>{{.i18n.Tr "repo.migrate.migrating_failed" .CloneAddr | Safe}}</p>
7 changes: 6 additions & 1 deletion web_src/js/index.js
Original file line number Diff line number Diff line change
@@ -202,6 +202,7 @@ function initRepoStatusChecker() {
const migrating = $('#repo_migrating');
$('#repo_migrating_failed').hide();
$('#repo_migrating_failed_image').hide();
$('#repo_migrating_progress_message').hide();
if (migrating) {
const task = migrating.attr('task');
if (typeof task === 'undefined') {
@@ -223,9 +224,13 @@ function initRepoStatusChecker() {
$('#repo_migrating').hide();
$('#repo_migrating_failed').show();
$('#repo_migrating_failed_image').show();
$('#repo_migrating_failed_error').text(xhr.responseJSON.err);
$('#repo_migrating_failed_error').text(xhr.responseJSON.message);
return;
}
if (xhr.responseJSON.message) {
$('#repo_migrating_progress_message').show();
$('#repo_migrating_progress_message').text(xhr.responseJSON.message);
}
setTimeout(() => {
initRepoStatusChecker();
}, 2000);