refactor: prepare to decouple the "model migration" package and "models" package (#38533)

Migrations should never use model structs directly, because the model
structs can be different in different releases. e.g. if one migration uses
"User" model, it works in the early releases, then one day, when the
User model changes, the migration breaks because it will use the
new (incorrect) User model, it should only use the old User model.

The same to "modules/structs".

---------

Signed-off-by: wxiaoguang <wxiaoguang@gmail.com>
Co-authored-by: delvh <dev.lh@web.de>
This commit is contained in:
wxiaoguang
2026-07-20 03:42:02 +00:00
committed by GitHub
co-authored by delvh
parent 775e3bdb34
commit fff32e9469
378 changed files with 708 additions and 721 deletions
+14
View File
@@ -0,0 +1,14 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_15
import (
"testing"
"gitea.dev/modelmigration/migrationtest"
)
func TestMain(m *testing.M) {
migrationtest.MainTest(m)
}
+15
View File
@@ -0,0 +1,15 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_15
import "gitea.dev/modelmigration/base"
func AddLFSMirrorColumns(x base.EngineMigration) error {
type Mirror struct {
LFS bool `xorm:"lfs_enabled NOT NULL DEFAULT false"`
LFSEndpoint string `xorm:"lfs_endpoint TEXT"`
}
return x.Sync(new(Mirror))
}
+27
View File
@@ -0,0 +1,27 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_15
import (
"gitea.dev/modelmigration/base"
"xorm.io/xorm/schemas"
)
func ConvertAvatarURLToText(x base.EngineMigration) error {
dbType := x.Dialect().URI().DBType
if dbType == schemas.SQLITE { // For SQLITE, varchar or char will always be represented as TEXT
return nil
}
// Some oauth2 providers may give very long avatar urls (i.e. Google)
return base.ModifyColumn(x, "external_login_user", &schemas.Column{
Name: "avatar_url",
SQLType: schemas.SQLType{
Name: schemas.Text,
},
Nullable: true,
DefaultIsEmpty: true,
})
}
+121
View File
@@ -0,0 +1,121 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_15
import (
"gitea.dev/modelmigration/base"
"gitea.dev/modules/json"
"gitea.dev/modules/util"
"xorm.io/builder"
)
func DeleteMigrationCredentials(x base.EngineMigration) (err error) {
// Task represents a task
type Task struct {
ID int64
DoerID int64 `xorm:"index"` // operator
OwnerID int64 `xorm:"index"` // repo owner id, when creating, the repoID maybe zero
RepoID int64 `xorm:"index"`
Type int
Status int `xorm:"index"`
StartTime int64
EndTime int64
PayloadContent string `xorm:"TEXT"`
Errors string `xorm:"TEXT"` // if task failed, saved the error reason
Created int64 `xorm:"created"`
}
const TaskTypeMigrateRepo = 0
const TaskStatusStopped = 2
const batchSize = 100
// only match migration tasks, that are not pending or running
cond := builder.Eq{
"type": TaskTypeMigrateRepo,
}.And(builder.Gte{
"status": TaskStatusStopped,
})
sess := x.NewSession()
defer sess.Close()
for start := 0; ; start += batchSize {
tasks := make([]*Task, 0, batchSize)
if err := sess.Limit(batchSize, start).Where(cond, 0).Find(&tasks); err != nil {
return err
}
if len(tasks) == 0 {
break
}
if err := sess.Begin(); err != nil {
return err
}
for _, t := range tasks {
if t.PayloadContent, err = removeCredentials(t.PayloadContent); err != nil {
return err
}
if _, err := sess.ID(t.ID).Cols("payload_content").Update(t); err != nil {
return err
}
}
if err := sess.Commit(); err != nil {
return err
}
}
return err
}
func removeCredentials(payload string) (string, error) {
// MigrateOptions defines the way a repository gets migrated
// this is for internal usage by migrations module and func who interact with it
type MigrateOptions struct {
// required: true
CloneAddr string `json:"clone_addr" binding:"Required"`
CloneAddrEncrypted string `json:"clone_addr_encrypted,omitempty"`
AuthUsername string `json:"auth_username"`
AuthPassword string `json:"-"`
AuthPasswordEncrypted string `json:"auth_password_encrypted,omitempty"`
AuthToken string `json:"-"`
AuthTokenEncrypted string `json:"auth_token_encrypted,omitempty"`
// required: true
UID int `json:"uid" binding:"Required"`
// required: true
RepoName string `json:"repo_name" binding:"Required"`
Mirror bool `json:"mirror"`
LFS bool `json:"lfs"`
LFSEndpoint string `json:"lfs_endpoint"`
Private bool `json:"private"`
Description string `json:"description"`
OriginalURL string
GitServiceType int
Wiki bool
Issues bool
Milestones bool
Labels bool
Releases bool
Comments bool
PullRequests bool
ReleaseAssets bool
MigrateToRepoID int64
MirrorInterval string `json:"mirror_interval"`
}
var opts MigrateOptions
err := json.Unmarshal([]byte(payload), &opts)
if err != nil {
return "", err
}
opts.AuthPassword = ""
opts.AuthToken = ""
opts.CloneAddr = util.SanitizeCredentialURLs(opts.CloneAddr)
confBytes, err := json.Marshal(opts)
if err != nil {
return "", err
}
return string(confBytes), nil
}
+91
View File
@@ -0,0 +1,91 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_15
import (
"strings"
"gitea.dev/modelmigration/base"
)
func AddPrimaryEmail2EmailAddress(x base.EngineMigration) error {
type User struct {
ID int64 `xorm:"pk autoincr"`
Email string `xorm:"NOT NULL"`
IsActive bool `xorm:"INDEX"` // Activate primary email
}
type EmailAddress1 struct {
ID int64 `xorm:"pk autoincr"`
UID int64 `xorm:"INDEX NOT NULL"`
Email string `xorm:"UNIQUE NOT NULL"`
LowerEmail string
IsActivated bool
IsPrimary bool `xorm:"DEFAULT(false) NOT NULL"`
}
// Add lower_email and is_primary columns
if err := x.Table("email_address").Sync(new(EmailAddress1)); err != nil {
return err
}
if _, err := x.Exec("UPDATE email_address SET lower_email=LOWER(email), is_primary=?", false); err != nil {
return err
}
type EmailAddress struct {
ID int64 `xorm:"pk autoincr"`
UID int64 `xorm:"INDEX NOT NULL"`
Email string `xorm:"UNIQUE NOT NULL"`
LowerEmail string `xorm:"UNIQUE NOT NULL"`
IsActivated bool
IsPrimary bool `xorm:"DEFAULT(false) NOT NULL"`
}
// change lower_email as unique
if err := x.Sync(new(EmailAddress)); err != nil {
return err
}
sess := x.NewSession()
defer sess.Close()
const batchSize = 100
for start := 0; ; start += batchSize {
users := make([]*User, 0, batchSize)
if err := sess.Limit(batchSize, start).Find(&users); err != nil {
return err
}
if len(users) == 0 {
break
}
for _, user := range users {
exist, err := sess.Where("email=?", user.Email).Table("email_address").Exist()
if err != nil {
return err
}
if !exist {
if _, err := sess.Insert(&EmailAddress{
UID: user.ID,
Email: user.Email,
LowerEmail: strings.ToLower(user.Email),
IsActivated: user.IsActive,
IsPrimary: true,
}); err != nil {
return err
}
} else {
if _, err := sess.Where("email=?", user.Email).Cols("is_primary").Update(&EmailAddress{
IsPrimary: true,
}); err != nil {
return err
}
}
}
}
return nil
}
+55
View File
@@ -0,0 +1,55 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_15
import (
"strings"
"testing"
"gitea.dev/modelmigration/migrationtest"
"github.com/stretchr/testify/assert"
)
func Test_AddPrimaryEmail2EmailAddress(t *testing.T) {
type User struct {
ID int64
Email string
IsActive bool
}
// Prepare and load the testing database
x, deferable := migrationtest.PrepareTestEnv(t, 0, new(User))
if x == nil || t.Failed() {
defer deferable()
return
}
defer deferable()
err := AddPrimaryEmail2EmailAddress(x)
assert.NoError(t, err)
type EmailAddress struct {
ID int64 `xorm:"pk autoincr"`
UID int64 `xorm:"INDEX NOT NULL"`
Email string `xorm:"UNIQUE NOT NULL"`
LowerEmail string `xorm:"UNIQUE NOT NULL"`
IsActivated bool
IsPrimary bool `xorm:"DEFAULT(false) NOT NULL"`
}
users := make([]User, 0, 20)
err = x.Find(&users)
assert.NoError(t, err)
for _, user := range users {
var emailAddress EmailAddress
has, err := x.Where("lower_email=?", strings.ToLower(user.Email)).Get(&emailAddress)
assert.NoError(t, err)
assert.True(t, has)
assert.True(t, emailAddress.IsPrimary)
assert.Equal(t, user.IsActive, emailAddress.IsActivated)
assert.Equal(t, user.ID, emailAddress.UID)
}
}
+39
View File
@@ -0,0 +1,39 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_15
import "gitea.dev/modelmigration/base"
func AddIssueResourceIndexTable(x base.EngineMigration) error {
type ResourceIndex struct {
GroupID int64 `xorm:"pk"`
MaxIndex int64 `xorm:"index"`
}
sess := x.NewSession()
defer sess.Close()
if err := sess.Begin(); err != nil {
return err
}
if err := sess.Table("issue_index").Sync(new(ResourceIndex)); err != nil {
return err
}
// Remove data we're goint to rebuild
if _, err := sess.Table("issue_index").Where("1=1").Delete(&ResourceIndex{}); err != nil {
return err
}
// Create current data for all repositories with issues and PRs
if _, err := sess.Exec("INSERT INTO issue_index (group_id, max_index) " +
"SELECT max_data.repo_id, max_data.max_index " +
"FROM ( SELECT issue.repo_id AS repo_id, max(issue.`index`) AS max_index " +
"FROM issue GROUP BY issue.repo_id) AS max_data"); err != nil {
return err
}
return sess.Commit()
}
+60
View File
@@ -0,0 +1,60 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_15
import (
"testing"
"gitea.dev/modelmigration/migrationtest"
"github.com/stretchr/testify/assert"
)
func Test_AddIssueResourceIndexTable(t *testing.T) {
// Create the models used in the migration
type Issue struct {
ID int64 `xorm:"pk autoincr"`
RepoID int64 `xorm:"UNIQUE(s)"`
Index int64 `xorm:"UNIQUE(s)"`
}
// Prepare and load the testing database
x, deferable := migrationtest.PrepareTestEnv(t, 0, new(Issue))
if x == nil || t.Failed() {
defer deferable()
return
}
defer deferable()
// Run the migration
if err := AddIssueResourceIndexTable(x); err != nil {
assert.NoError(t, err)
return
}
type ResourceIndex struct {
GroupID int64 `xorm:"pk"`
MaxIndex int64 `xorm:"index"`
}
start := 0
const batchSize = 1000
for {
indexes := make([]ResourceIndex, 0, batchSize)
err := x.Table("issue_index").Limit(batchSize, start).Find(&indexes)
assert.NoError(t, err)
for _, idx := range indexes {
var maxIndex int
has, err := x.SQL("SELECT max(`index`) FROM issue WHERE repo_id = ?", idx.GroupID).Get(&maxIndex)
assert.NoError(t, err)
assert.True(t, has)
assert.EqualValues(t, maxIndex, idx.MaxIndex)
}
if len(indexes) < batchSize {
break
}
start += len(indexes)
}
}
+37
View File
@@ -0,0 +1,37 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_15
import (
"fmt"
"time"
"gitea.dev/modelmigration/base"
"gitea.dev/modules/timeutil"
)
func CreatePushMirrorTable(x base.EngineMigration) error {
type PushMirror struct {
ID int64 `xorm:"pk autoincr"`
RepoID int64 `xorm:"INDEX"`
RemoteName string
Interval time.Duration
CreatedUnix timeutil.TimeStamp `xorm:"created"`
LastUpdateUnix timeutil.TimeStamp `xorm:"INDEX last_update"`
LastError string `xorm:"text"`
}
sess := x.NewSession()
defer sess.Close()
if err := sess.Begin(); err != nil {
return err
}
if err := sess.Sync(new(PushMirror)); err != nil {
return fmt.Errorf("Sync: %w", err)
}
return sess.Commit()
}
+69
View File
@@ -0,0 +1,69 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_15
import (
"context"
"fmt"
"gitea.dev/modelmigration/base"
"gitea.dev/modules/setting"
)
func RenameTaskErrorsToMessage(x base.EngineMigration) error {
type Task struct {
Errors string `xorm:"TEXT"` // if task failed, saved the error reason
Type int
Status int `xorm:"index"`
}
// This migration maybe rerun so that we should check if it has been run
messageExist, err := x.Dialect().IsColumnExist(x.DB(), context.Background(), "task", "message")
if err != nil {
return err
}
if messageExist {
errorsExist, err := x.Dialect().IsColumnExist(x.DB(), context.Background(), "task", "errors")
if err != nil {
return err
}
if !errorsExist {
return nil
}
}
sess := x.NewSession()
defer sess.Close()
if err := sess.Begin(); err != nil {
return err
}
if err := sess.Sync(new(Task)); err != nil {
return fmt.Errorf("error on Sync: %w", err)
}
if messageExist {
// if both errors and message exist, drop message at first
if err := base.DropTableColumns(sess, "task", "message"); err != nil {
return err
}
}
switch {
case setting.Database.Type.IsMySQL():
if _, err := sess.Exec("ALTER TABLE `task` CHANGE errors message text"); err != nil {
return err
}
case setting.Database.Type.IsMSSQL():
if _, err := sess.Exec("sp_rename 'task.errors', 'message', 'COLUMN'"); err != nil {
return err
}
default:
if _, err := sess.Exec("ALTER TABLE `task` RENAME COLUMN errors TO message"); err != nil {
return err
}
}
return sess.Commit()
}
+19
View File
@@ -0,0 +1,19 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_15
import "gitea.dev/modelmigration/base"
func AddRepoArchiver(x base.EngineMigration) error {
// RepoArchiver represents all archivers
type RepoArchiver struct {
ID int64 `xorm:"pk autoincr"`
RepoID int64 `xorm:"index unique(s)"`
Type int `xorm:"unique(s)"`
Status int
CommitID string `xorm:"VARCHAR(40) unique(s)"`
CreatedUnix int64 `xorm:"INDEX NOT NULL created"`
}
return x.Sync(new(RepoArchiver))
}
+24
View File
@@ -0,0 +1,24 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_15
import (
"gitea.dev/modelmigration/base"
"gitea.dev/modules/timeutil"
)
func CreateProtectedTagTable(x base.EngineMigration) error {
type ProtectedTag struct {
ID int64 `xorm:"pk autoincr"`
RepoID int64
NamePattern string
AllowlistUserIDs []int64 `xorm:"JSON TEXT"`
AllowlistTeamIDs []int64 `xorm:"JSON TEXT"`
CreatedUnix timeutil.TimeStamp `xorm:"created"`
UpdatedUnix timeutil.TimeStamp `xorm:"updated"`
}
return x.Sync(new(ProtectedTag))
}
+45
View File
@@ -0,0 +1,45 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_15
import (
"gitea.dev/modelmigration/base"
)
func DropWebhookColumns(x base.EngineMigration) error {
// Make sure the columns exist before dropping them
type Webhook struct {
Signature string `xorm:"TEXT"`
IsSSL bool `xorm:"is_ssl"`
}
if err := x.Sync(new(Webhook)); err != nil {
return err
}
type HookTask struct {
Typ string `xorm:"VARCHAR(16) index"`
URL string `xorm:"TEXT"`
Signature string `xorm:"TEXT"`
HTTPMethod string `xorm:"http_method"`
ContentType int
IsSSL bool
}
if err := x.Sync(new(HookTask)); err != nil {
return err
}
sess := x.NewSession()
defer sess.Close()
if err := sess.Begin(); err != nil {
return err
}
if err := base.DropTableColumns(sess, "webhook", "signature", "is_ssl"); err != nil {
return err
}
if err := base.DropTableColumns(sess, "hook_task", "typ", "url", "signature", "http_method", "content_type", "is_ssl"); err != nil {
return err
}
return sess.Commit()
}
+14
View File
@@ -0,0 +1,14 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_15
import "gitea.dev/modelmigration/base"
func AddKeyIsVerified(x base.EngineMigration) error {
type GPGKey struct {
Verified bool `xorm:"NOT NULL DEFAULT false"`
}
return x.Sync(new(GPGKey))
}