refactor: prepare to decouple the "model migration" package and "models" package (#38533)

Migrations should never use model structs directly, because the model
structs can be different in different releases. e.g. if one migration uses
"User" model, it works in the early releases, then one day, when the
User model changes, the migration breaks because it will use the
new (incorrect) User model, it should only use the old User model.

The same to "modules/structs".

---------

Signed-off-by: wxiaoguang <wxiaoguang@gmail.com>
Co-authored-by: delvh <dev.lh@web.de>
This commit is contained in:
wxiaoguang
2026-07-20 03:42:02 +00:00
committed by GitHub
co-authored by delvh
parent 775e3bdb34
commit fff32e9469
378 changed files with 708 additions and 721 deletions
+14
View File
@@ -0,0 +1,14 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_17
import (
"testing"
"gitea.dev/modelmigration/migrationtest"
)
func TestMain(m *testing.M) {
migrationtest.MainTest(m)
}
+10
View File
@@ -0,0 +1,10 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_17
import "gitea.dev/modelmigration/base"
func CreateForeignReferenceTable(_ base.EngineMigration) error {
return nil // This table was dropped in v1_19/v237.go
}
+92
View File
@@ -0,0 +1,92 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_17
import (
"gitea.dev/modelmigration/base"
"gitea.dev/modules/timeutil"
)
func AddPackageTables(x base.EngineMigration) error {
type Package struct {
ID int64 `xorm:"pk autoincr"`
OwnerID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"`
RepoID int64 `xorm:"INDEX"`
Type string `xorm:"UNIQUE(s) INDEX NOT NULL"`
Name string `xorm:"NOT NULL"`
LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"`
SemverCompatible bool `xorm:"NOT NULL DEFAULT false"`
}
if err := x.Sync(new(Package)); err != nil {
return err
}
type PackageVersion struct {
ID int64 `xorm:"pk autoincr"`
PackageID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"`
CreatorID int64 `xorm:"NOT NULL DEFAULT 0"`
Version string `xorm:"NOT NULL"`
LowerVersion string `xorm:"UNIQUE(s) INDEX NOT NULL"`
CreatedUnix timeutil.TimeStamp `xorm:"created INDEX NOT NULL"`
IsInternal bool `xorm:"INDEX NOT NULL DEFAULT false"`
MetadataJSON string `xorm:"metadata_json TEXT"`
DownloadCount int64 `xorm:"NOT NULL DEFAULT 0"`
}
if err := x.Sync(new(PackageVersion)); err != nil {
return err
}
type PackageProperty struct {
ID int64 `xorm:"pk autoincr"`
RefType int64 `xorm:"INDEX NOT NULL"`
RefID int64 `xorm:"INDEX NOT NULL"`
Name string `xorm:"INDEX NOT NULL"`
Value string `xorm:"TEXT NOT NULL"`
}
if err := x.Sync(new(PackageProperty)); err != nil {
return err
}
type PackageFile struct {
ID int64 `xorm:"pk autoincr"`
VersionID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"`
BlobID int64 `xorm:"INDEX NOT NULL"`
Name string `xorm:"NOT NULL"`
LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"`
CompositeKey string `xorm:"UNIQUE(s) INDEX"`
IsLead bool `xorm:"NOT NULL DEFAULT false"`
CreatedUnix timeutil.TimeStamp `xorm:"created INDEX NOT NULL"`
}
if err := x.Sync(new(PackageFile)); err != nil {
return err
}
type PackageBlob struct {
ID int64 `xorm:"pk autoincr"`
Size int64 `xorm:"NOT NULL DEFAULT 0"`
HashMD5 string `xorm:"hash_md5 char(32) UNIQUE(md5) INDEX NOT NULL"`
HashSHA1 string `xorm:"hash_sha1 char(40) UNIQUE(sha1) INDEX NOT NULL"`
HashSHA256 string `xorm:"hash_sha256 char(64) UNIQUE(sha256) INDEX NOT NULL"`
HashSHA512 string `xorm:"hash_sha512 char(128) UNIQUE(sha512) INDEX NOT NULL"`
CreatedUnix timeutil.TimeStamp `xorm:"created INDEX NOT NULL"`
}
if err := x.Sync(new(PackageBlob)); err != nil {
return err
}
type PackageBlobUpload struct {
ID string `xorm:"pk"`
BytesReceived int64 `xorm:"NOT NULL DEFAULT 0"`
HashStateBytes []byte `xorm:"BLOB"`
CreatedUnix timeutil.TimeStamp `xorm:"created NOT NULL"`
UpdatedUnix timeutil.TimeStamp `xorm:"updated INDEX NOT NULL"`
}
return x.Sync(new(PackageBlobUpload))
}
+15
View File
@@ -0,0 +1,15 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_17
import "gitea.dev/modelmigration/base"
func AddAllowMaintainerEdit(x base.EngineMigration) error {
// PullRequest represents relation between pull request and repositories.
type PullRequest struct {
AllowMaintainerEdit bool `xorm:"NOT NULL DEFAULT false"`
}
return x.Sync(new(PullRequest))
}
+20
View File
@@ -0,0 +1,20 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_17
import "gitea.dev/modelmigration/base"
func AddAutoMergeTable(x base.EngineMigration) error {
type MergeStyle string
type PullAutoMerge struct {
ID int64 `xorm:"pk autoincr"`
PullID int64 `xorm:"UNIQUE"`
DoerID int64 `xorm:"NOT NULL"`
MergeStyle MergeStyle `xorm:"varchar(30)"`
Message string `xorm:"LONGTEXT"`
CreatedUnix int64 `xorm:"created"`
}
return x.Sync(&PullAutoMerge{})
}
+23
View File
@@ -0,0 +1,23 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_17
import (
"gitea.dev/modelmigration/base"
"gitea.dev/models/pull"
"gitea.dev/modules/timeutil"
)
func AddReviewViewedFiles(x base.EngineMigration) error {
type ReviewState struct {
ID int64 `xorm:"pk autoincr"`
UserID int64 `xorm:"NOT NULL UNIQUE(pull_commit_user)"`
PullID int64 `xorm:"NOT NULL INDEX UNIQUE(pull_commit_user) DEFAULT 0"`
CommitSHA string `xorm:"NOT NULL VARCHAR(40) UNIQUE(pull_commit_user)"`
UpdatedFiles map[string]pull.ViewedState `xorm:"NOT NULL LONGTEXT JSON"`
UpdatedUnix timeutil.TimeStamp `xorm:"updated"`
}
return x.Sync(new(ReviewState))
}
+7
View File
@@ -0,0 +1,7 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_17
// This migration added non-ideal indices to the action table which on larger datasets slowed things down
// it has been superseded by v218.go
+24
View File
@@ -0,0 +1,24 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_17
import (
"gitea.dev/modelmigration/base"
"gitea.dev/modules/setting"
)
func AlterHookTaskTextFieldsToLongText(x base.EngineMigration) error {
sess := x.NewSession()
defer sess.Close()
if err := sess.Begin(); err != nil {
return err
}
if setting.Database.Type.IsMySQL() {
if _, err := sess.Exec("ALTER TABLE `hook_task` CHANGE `payload_content` `payload_content` LONGTEXT, CHANGE `request_content` `request_content` LONGTEXT, change `response_content` `response_content` LONGTEXT"); err != nil {
return err
}
}
return sess.Commit()
}
+52
View File
@@ -0,0 +1,52 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_17
import (
"gitea.dev/modelmigration/base"
"gitea.dev/modules/setting"
"gitea.dev/modules/timeutil"
"xorm.io/xorm/schemas"
)
type improveActionTableIndicesAction struct {
ID int64 `xorm:"pk autoincr"`
UserID int64 // Receiver user id.
OpType int
ActUserID int64 // Action user id.
RepoID int64
CommentID int64 `xorm:"INDEX"`
IsDeleted bool `xorm:"NOT NULL DEFAULT false"`
RefName string
IsPrivate bool `xorm:"NOT NULL DEFAULT false"`
Content string `xorm:"TEXT"`
CreatedUnix timeutil.TimeStamp `xorm:"created"`
}
// TableName sets the name of this table
func (*improveActionTableIndicesAction) TableName() string {
return "action"
}
// TableIndices implements xorm's TableIndices interface
func (*improveActionTableIndicesAction) TableIndices() []*schemas.Index {
repoIndex := schemas.NewIndex("r_u_d", schemas.IndexType)
repoIndex.AddColumn("repo_id", "user_id", "is_deleted")
actUserIndex := schemas.NewIndex("au_r_c_u_d", schemas.IndexType)
actUserIndex.AddColumn("act_user_id", "repo_id", "created_unix", "user_id", "is_deleted")
indices := []*schemas.Index{actUserIndex, repoIndex}
if setting.Database.Type.IsPostgreSQL() {
cudIndex := schemas.NewIndex("c_u_d", schemas.IndexType)
cudIndex.AddColumn("created_unix", "user_id", "is_deleted")
indices = append(indices, cudIndex)
}
return indices
}
func ImproveActionTableIndices(x base.EngineMigration) error {
return x.Sync(&improveActionTableIndicesAction{})
}
+29
View File
@@ -0,0 +1,29 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_17
import (
"time"
"gitea.dev/modelmigration/base"
"gitea.dev/models/repo"
"gitea.dev/modules/timeutil"
)
func AddSyncOnCommitColForPushMirror(x base.EngineMigration) error {
type PushMirror struct {
ID int64 `xorm:"pk autoincr"`
RepoID int64 `xorm:"INDEX"`
Repo *repo.Repository `xorm:"-"`
RemoteName string
SyncOnCommit bool `xorm:"NOT NULL DEFAULT true"`
Interval time.Duration
CreatedUnix timeutil.TimeStamp `xorm:"created"`
LastUpdateUnix timeutil.TimeStamp `xorm:"INDEX last_update"`
LastError string `xorm:"text"`
}
return x.Sync(new(PushMirror))
}
+27
View File
@@ -0,0 +1,27 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_17
import (
"gitea.dev/modelmigration/base"
packages_model "gitea.dev/models/packages"
container_module "gitea.dev/modules/packages/container"
"xorm.io/xorm/schemas"
)
func AddContainerRepositoryProperty(x base.EngineMigration) (err error) {
switch x.Dialect().URI().DBType {
case schemas.SQLITE:
_, err = x.Exec("INSERT INTO package_property (ref_type, ref_id, name, value) SELECT ?, p.id, ?, u.lower_name || '/' || p.lower_name FROM package p JOIN `user` u ON p.owner_id = u.id WHERE p.type = ?",
packages_model.PropertyTypePackage, container_module.PropertyRepository, packages_model.TypeContainer)
case schemas.MSSQL:
_, err = x.Exec("INSERT INTO package_property (ref_type, ref_id, name, value) SELECT ?, p.id, ?, u.lower_name + '/' + p.lower_name FROM package p JOIN `user` u ON p.owner_id = u.id WHERE p.type = ?",
packages_model.PropertyTypePackage, container_module.PropertyRepository, packages_model.TypeContainer)
default:
_, err = x.Exec("INSERT INTO package_property (ref_type, ref_id, name, value) SELECT ?, p.id, ?, CONCAT(u.lower_name, '/', p.lower_name) FROM package p JOIN `user` u ON p.owner_id = u.id WHERE p.type = ?",
packages_model.PropertyTypePackage, container_module.PropertyRepository, packages_model.TypeContainer)
}
return err
}
+73
View File
@@ -0,0 +1,73 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_17
import (
"encoding/base32"
"fmt"
"gitea.dev/modelmigration/base"
"gitea.dev/modules/timeutil"
)
func StoreWebauthnCredentialIDAsBytes(x base.EngineMigration) error {
// Create webauthnCredential table
type webauthnCredential struct {
ID int64 `xorm:"pk autoincr"`
Name string
LowerName string `xorm:"unique(s)"`
UserID int64 `xorm:"INDEX unique(s)"`
CredentialID string `xorm:"INDEX VARCHAR(410)"`
// Note the lack of INDEX here - these will be created once the column is renamed in v223.go
CredentialIDBytes []byte `xorm:"VARBINARY(1024)"` // CredentialID is at most 1023 bytes as per spec released 20 July 2022
PublicKey []byte
AttestationType string
AAGUID []byte
SignCount uint32 `xorm:"BIGINT"`
CloneWarning bool
CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"`
UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"`
}
if err := x.Sync(&webauthnCredential{}); err != nil {
return err
}
var start int
creds := make([]*webauthnCredential, 0, 50)
for {
err := x.Select("id, credential_id").OrderBy("id").Limit(50, start).Find(&creds)
if err != nil {
return err
}
err = func() error {
sess := x.NewSession()
defer sess.Close()
if err := sess.Begin(); err != nil {
return fmt.Errorf("unable to allow start session. Error: %w", err)
}
for _, cred := range creds {
cred.CredentialIDBytes, err = base32.HexEncoding.DecodeString(cred.CredentialID)
if err != nil {
return fmt.Errorf("unable to parse credential id %s for credential[%d]: %w", cred.CredentialID, cred.ID, err)
}
count, err := sess.ID(cred.ID).Cols("credential_id_bytes").Update(cred)
if count != 1 || err != nil {
return fmt.Errorf("unable to update credential id bytes for credential[%d]: %d,%w", cred.ID, count, err)
}
}
return sess.Commit()
}()
if err != nil {
return err
}
if len(creds) < 50 {
break
}
start += 50
creds = creds[:0]
}
return nil
}
+66
View File
@@ -0,0 +1,66 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_17
import (
"encoding/base32"
"testing"
"gitea.dev/modelmigration/migrationtest"
"github.com/stretchr/testify/assert"
)
func Test_StoreWebauthnCredentialIDAsBytes(t *testing.T) {
// Create webauthnCredential table
type WebauthnCredential struct {
ID int64 `xorm:"pk autoincr"`
Name string
LowerName string `xorm:"unique(s)"`
UserID int64 `xorm:"INDEX unique(s)"`
CredentialID string `xorm:"INDEX VARCHAR(410)"`
PublicKey []byte
AttestationType string
AAGUID []byte
SignCount uint32 `xorm:"BIGINT"`
CloneWarning bool
}
type ExpectedWebauthnCredential struct {
ID int64 `xorm:"pk autoincr"`
CredentialID string // CredentialID is at most 1023 bytes as per spec released 20 July 2022
}
type ConvertedWebauthnCredential struct {
ID int64 `xorm:"pk autoincr"`
CredentialIDBytes []byte `xorm:"VARBINARY(1024)"` // CredentialID is at most 1023 bytes as per spec released 20 July 2022
}
// Prepare and load the testing database
x, deferable := migrationtest.PrepareTestEnv(t, 0, new(WebauthnCredential), new(ExpectedWebauthnCredential))
defer deferable()
if x == nil || t.Failed() {
return
}
if err := StoreWebauthnCredentialIDAsBytes(x); err != nil {
assert.NoError(t, err)
return
}
expected := []ExpectedWebauthnCredential{}
if err := x.Table("expected_webauthn_credential").Asc("id").Find(&expected); !assert.NoError(t, err) {
return
}
got := []ConvertedWebauthnCredential{}
if err := x.Table("webauthn_credential").Select("id, credential_id_bytes").Asc("id").Find(&got); !assert.NoError(t, err) {
return
}
for i, e := range expected {
credIDBytes, _ := base32.HexEncoding.DecodeString(e.CredentialID)
assert.Equal(t, credIDBytes, got[i].CredentialIDBytes)
}
}
+63
View File
@@ -0,0 +1,63 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_17
import (
"context"
"errors"
"fmt"
"gitea.dev/modelmigration/base"
"gitea.dev/modules/timeutil"
)
func DropOldCredentialIDColumn(x base.EngineMigration) error {
// This migration maybe rerun so that we should check if it has been run
credentialIDExist, err := x.Dialect().IsColumnExist(x.DB(), context.Background(), "webauthn_credential", "credential_id")
if err != nil {
return err
}
if !credentialIDExist {
// Column is already non-extant
return nil
}
credentialIDBytesExists, err := x.Dialect().IsColumnExist(x.DB(), context.Background(), "webauthn_credential", "credential_id_bytes")
if err != nil {
return err
}
if !credentialIDBytesExists {
// looks like 221 hasn't properly run
return errors.New("webauthn_credential does not have a credential_id_bytes column... it is not safe to run this migration")
}
// Create webauthnCredential table
type webauthnCredential struct {
ID int64 `xorm:"pk autoincr"`
Name string
LowerName string `xorm:"unique(s)"`
UserID int64 `xorm:"INDEX unique(s)"`
CredentialID string `xorm:"INDEX VARCHAR(410)"`
// Note the lack of the INDEX on CredentialIDBytes - we will add this in v223.go
CredentialIDBytes []byte `xorm:"VARBINARY(1024)"` // CredentialID is at most 1023 bytes as per spec released 20 July 2022
PublicKey []byte
AttestationType string
AAGUID []byte
SignCount uint32 `xorm:"BIGINT"`
CloneWarning bool
CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"`
UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"`
}
if err := x.Sync(&webauthnCredential{}); err != nil {
return err
}
// Drop the old credential ID
sess := x.NewSession()
defer sess.Close()
if err := base.DropTableColumns(sess, "webauthn_credential", "credential_id"); err != nil {
return fmt.Errorf("unable to drop old credentialID column: %w", err)
}
return sess.Commit()
}
+101
View File
@@ -0,0 +1,101 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_17
import (
"context"
"fmt"
"gitea.dev/modelmigration/base"
"gitea.dev/modules/setting"
"gitea.dev/modules/timeutil"
)
func RenameCredentialIDBytes(x base.EngineMigration) error {
// This migration maybe rerun so that we should check if it has been run
credentialIDExist, err := x.Dialect().IsColumnExist(x.DB(), context.Background(), "webauthn_credential", "credential_id")
if err != nil {
return err
}
if credentialIDExist {
credentialIDBytesExists, err := x.Dialect().IsColumnExist(x.DB(), context.Background(), "webauthn_credential", "credential_id_bytes")
if err != nil {
return err
}
if !credentialIDBytesExists {
return nil
}
}
err = func() error {
// webauthnCredential table
type webauthnCredential struct {
ID int64 `xorm:"pk autoincr"`
Name string
LowerName string `xorm:"unique(s)"`
UserID int64 `xorm:"INDEX unique(s)"`
// Note the lack of INDEX here
CredentialIDBytes []byte `xorm:"VARBINARY(1024)"` // CredentialID is at most 1023 bytes as per spec released 20 July 2022
PublicKey []byte
AttestationType string
AAGUID []byte
SignCount uint32 `xorm:"BIGINT"`
CloneWarning bool
CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"`
UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"`
}
sess := x.NewSession()
defer sess.Close()
if err := sess.Begin(); err != nil {
return err
}
if err := sess.Sync(new(webauthnCredential)); err != nil {
return fmt.Errorf("error on Sync: %w", err)
}
if credentialIDExist {
// if both errors and message exist, drop message at first
if err := base.DropTableColumns(sess, "webauthn_credential", "credential_id"); err != nil {
return err
}
}
switch {
case setting.Database.Type.IsMySQL():
if _, err := sess.Exec("ALTER TABLE `webauthn_credential` CHANGE credential_id_bytes credential_id VARBINARY(1024)"); err != nil {
return err
}
case setting.Database.Type.IsMSSQL():
if _, err := sess.Exec("sp_rename 'webauthn_credential.credential_id_bytes', 'credential_id', 'COLUMN'"); err != nil {
return err
}
default:
if _, err := sess.Exec("ALTER TABLE `webauthn_credential` RENAME COLUMN credential_id_bytes TO credential_id"); err != nil {
return err
}
}
return sess.Commit()
}()
if err != nil {
return err
}
// Create webauthnCredential table
type webauthnCredential struct {
ID int64 `xorm:"pk autoincr"`
Name string
LowerName string `xorm:"unique(s)"`
UserID int64 `xorm:"INDEX unique(s)"`
CredentialID []byte `xorm:"INDEX VARBINARY(1024)"` // CredentialID is at most 1023 bytes as per spec released 20 July 2022
PublicKey []byte
AttestationType string
AAGUID []byte
SignCount uint32 `xorm:"BIGINT"`
CloneWarning bool
CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"`
UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"`
}
return x.Sync(&webauthnCredential{})
}