ynkmr/gitea
1
0
Fork 0
mirror of https://github.com/go-gitea/gitea.git synced 2026-09-08 14:03:24 +09:00
Code Issues Projects Releases Wiki Activity

refactor: prepare to decouple the "model migration" package and "models" package (#38533)

Browse Source 
Migrations should never use model structs directly, because the model
structs can be different in different releases. e.g. if one migration uses
"User" model, it works in the early releases, then one day, when the
User model changes, the migration breaks because it will use the
new (incorrect) User model, it should only use the old User model.

The same to "modules/structs".

---------

Signed-off-by: wxiaoguang <wxiaoguang@gmail.com>
Co-authored-by: delvh <dev.lh@web.de>
This commit is contained in:
wxiaoguang
2026-07-20 03:42:02 +00:00
committed by GitHub
co-authored by delvh
parent 775e3bdb34
commit fff32e9469
378 changed files with 708 additions and 721 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modelmigration/v1_19/main_test.go
+14
View File
@@ -0,0 +1,14 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_19
import (
"testing"
"gitea.dev/modelmigration/migrationtest"
)
func TestMain(m *testing.M) {
migrationtest.MainTest(m)
}
modelmigration/v1_19/v231.go
+16
View File
@@ -0,0 +1,16 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_19
import "gitea.dev/modelmigration/base"
func AddIndexForHookTask(x base.EngineMigration) error {
type HookTask struct {
ID int64 `xorm:"pk autoincr"`
HookID int64 `xorm:"index"`
UUID string `xorm:"unique"`
}
return x.Sync(new(HookTask))
}
modelmigration/v1_19/v232.go
+24
View File
@@ -0,0 +1,24 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_19
import (
"gitea.dev/modelmigration/base"
"gitea.dev/modules/setting"
)
func AlterPackageVersionMetadataToLongText(x base.EngineMigration) error {
sess := x.NewSession()
defer sess.Close()
if err := sess.Begin(); err != nil {
return err
}
if setting.Database.Type.IsMySQL() {
if _, err := sess.Exec("ALTER TABLE `package_version` MODIFY COLUMN `metadata_json` LONGTEXT"); err != nil {
return err
}
}
return sess.Commit()
}
modelmigration/v1_19/v233.go
+180
View File
@@ -0,0 +1,180 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_19
import (
"fmt"
"gitea.dev/modelmigration/base"
"gitea.dev/modules/json"
"gitea.dev/modules/secret"
"gitea.dev/modules/setting"
"xorm.io/builder"
)
func batchProcess[T any](x base.EngineMigration, buf []T, query func(limit, start int) base.Session, process func(base.Session, T) error) error {
size := cap(buf)
start := 0
for {
err := query(size, start).Find(&buf)
if err != nil {
return err
}
if len(buf) == 0 {
return nil
}
err = func() error {
sess := x.NewSession()
defer sess.Close()
if err := sess.Begin(); err != nil {
return fmt.Errorf("unable to allow start session. Error: %w", err)
}
for _, record := range buf {
if err := process(sess, record); err != nil {
return err
}
}
return sess.Commit()
}()
if err != nil {
return err
}
if len(buf) < size {
return nil
}
start += size
buf = buf[:0]
}
}
func AddHeaderAuthorizationEncryptedColWebhook(x base.EngineMigration) error {
// Add the column to the table
type Webhook struct {
ID int64 `xorm:"pk autoincr"`
Type string `xorm:"VARCHAR(16) 'type'"`
Meta string `xorm:"TEXT"` // store hook-specific attributes
// HeaderAuthorizationEncrypted should be accessed using HeaderAuthorization() and SetHeaderAuthorization()
HeaderAuthorizationEncrypted string `xorm:"TEXT"`
}
err := x.Sync(new(Webhook))
if err != nil {
return err
}
// Migrate the matrix webhooks
type MatrixMeta struct {
HomeserverURL string `json:"homeserver_url"`
Room string `json:"room_id"`
MessageType int `json:"message_type"`
}
type MatrixMetaWithAccessToken struct {
MatrixMeta
AccessToken string `json:"access_token"`
}
err = batchProcess(x,
make([]*Webhook, 0, 50),
func(limit, start int) base.Session {
return x.Where("type=?", "matrix").OrderBy("id").Limit(limit, start)
},
func(sess base.Session, hook *Webhook) error {
// retrieve token from meta
var withToken MatrixMetaWithAccessToken
err := json.Unmarshal([]byte(hook.Meta), &withToken)
if err != nil {
return fmt.Errorf("unable to unmarshal matrix meta for webhook[id=%d]: %w", hook.ID, err)
}
if withToken.AccessToken == "" {
return nil
}
// encrypt token
authorization := "Bearer " + withToken.AccessToken
hook.HeaderAuthorizationEncrypted, err = secret.EncryptSecret(setting.SecretKey, authorization)
if err != nil {
return fmt.Errorf("unable to encrypt access token for webhook[id=%d]: %w", hook.ID, err)
}
// remove token from meta
withoutToken, err := json.Marshal(withToken.MatrixMeta)
if err != nil {
return fmt.Errorf("unable to marshal matrix meta for webhook[id=%d]: %w", hook.ID, err)
}
hook.Meta = string(withoutToken)
// save in database
count, err := sess.ID(hook.ID).Cols("meta", "header_authorization_encrypted").Update(hook)
if count != 1 || err != nil {
return fmt.Errorf("unable to update header_authorization_encrypted for webhook[id=%d]: %d,%w", hook.ID, count, err)
}
return nil
})
if err != nil {
return err
}
// Remove access_token from HookTask
type HookTask struct {
ID int64 `xorm:"pk autoincr"`
HookID int64
PayloadContent string `xorm:"LONGTEXT"`
}
type MatrixPayloadSafe struct {
Body string `json:"body"`
MsgType string `json:"msgtype"`
Format string `json:"format"`
FormattedBody string `json:"formatted_body"`
Commits json.Value `json:"io.gitea.commits,omitempty"`
}
type MatrixPayloadUnsafe struct {
MatrixPayloadSafe
AccessToken string `json:"access_token"`
}
err = batchProcess(x,
make([]*HookTask, 0, 50),
func(limit, start int) base.Session {
return x.Where(builder.And(
builder.In("hook_id", builder.Select("id").From("webhook").Where(builder.Eq{"type": "matrix"})),
builder.Like{"payload_content", "access_token"},
)).OrderBy("id").Limit(limit, 0) // ignore the provided "start", since other payload were already converted and don't contain 'payload_content' anymore
},
func(sess base.Session, hookTask *HookTask) error {
// retrieve token from payload_content
var withToken MatrixPayloadUnsafe
err := json.Unmarshal([]byte(hookTask.PayloadContent), &withToken)
if err != nil {
return fmt.Errorf("unable to unmarshal payload_content for hook_task[id=%d]: %w", hookTask.ID, err)
}
if withToken.AccessToken == "" {
return nil
}
// remove token from payload_content
withoutToken, err := json.Marshal(withToken.MatrixPayloadSafe)
if err != nil {
return fmt.Errorf("unable to marshal payload_content for hook_task[id=%d]: %w", hookTask.ID, err)
}
hookTask.PayloadContent = string(withoutToken)
// save in database
count, err := sess.ID(hookTask.ID).Cols("payload_content").Update(hookTask)
if count != 1 || err != nil {
return fmt.Errorf("unable to update payload_content for hook_task[id=%d]: %d,%w", hookTask.ID, count, err)
}
return nil
})
if err != nil {
return err
}
return nil
}
modelmigration/v1_19/v233_test.go
+87
View File
@@ -0,0 +1,87 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_19
import (
"testing"
"gitea.dev/modelmigration/migrationtest"
"gitea.dev/modules/json"
"gitea.dev/modules/secret"
"gitea.dev/modules/setting"
webhook_module "gitea.dev/modules/webhook"
"github.com/stretchr/testify/assert"
)
func Test_AddHeaderAuthorizationEncryptedColWebhook(t *testing.T) {
// Create Webhook table
type Webhook struct {
ID int64 `xorm:"pk autoincr"`
Type webhook_module.HookType `xorm:"VARCHAR(16) 'type'"`
Meta string `xorm:"TEXT"` // store hook-specific attributes
// HeaderAuthorizationEncrypted should be accessed using HeaderAuthorization() and SetHeaderAuthorization()
HeaderAuthorizationEncrypted string `xorm:"TEXT"`
}
type ExpectedWebhook struct {
ID int64 `xorm:"pk autoincr"`
Meta string
HeaderAuthorization string
}
type HookTask struct {
ID int64 `xorm:"pk autoincr"`
HookID int64
PayloadContent string `xorm:"LONGTEXT"`
}
// Prepare and load the testing database
x, deferable := migrationtest.PrepareTestEnv(t, 0, new(Webhook), new(ExpectedWebhook), new(HookTask))
defer deferable()
if x == nil || t.Failed() {
return
}
if err := AddHeaderAuthorizationEncryptedColWebhook(x); err != nil {
assert.NoError(t, err)
return
}
expected := []ExpectedWebhook{}
if err := x.Table("expected_webhook").Asc("id").Find(&expected); !assert.NoError(t, err) {
return
}
got := []Webhook{}
if err := x.Table("webhook").Select("id, meta, header_authorization_encrypted").Asc("id").Find(&got); !assert.NoError(t, err) {
return
}
for i, e := range expected {
assert.Equal(t, e.Meta, got[i].Meta)
if e.HeaderAuthorization == "" {
assert.Empty(t, got[i].HeaderAuthorizationEncrypted)
} else {
cipherhex := got[i].HeaderAuthorizationEncrypted
cleartext, err := secret.DecryptSecret(setting.SecretKey, cipherhex)
assert.NoError(t, err)
assert.Equal(t, e.HeaderAuthorization, cleartext)
}
}
// ensure that no hook_task has some remaining "access_token"
hookTasks := []HookTask{}
if err := x.Table("hook_task").Select("id, payload_content").Asc("id").Find(&hookTasks); !assert.NoError(t, err) {
return
}
for _, h := range hookTasks {
var m map[string]any
err := json.Unmarshal([]byte(h.PayloadContent), &m)
assert.NoError(t, err)
assert.Nil(t, m["access_token"])
}
}
modelmigration/v1_19/v234.go
+27
View File
@@ -0,0 +1,27 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_19
import (
"gitea.dev/modelmigration/base"
"gitea.dev/modules/timeutil"
)
func CreatePackageCleanupRuleTable(x base.EngineMigration) error {
type PackageCleanupRule struct {
ID int64 `xorm:"pk autoincr"`
Enabled bool `xorm:"INDEX NOT NULL DEFAULT false"`
OwnerID int64 `xorm:"UNIQUE(s) INDEX NOT NULL DEFAULT 0"`
Type string `xorm:"UNIQUE(s) INDEX NOT NULL"`
KeepCount int `xorm:"NOT NULL DEFAULT 0"`
KeepPattern string `xorm:"NOT NULL DEFAULT ''"`
RemoveDays int `xorm:"NOT NULL DEFAULT 0"`
RemovePattern string `xorm:"NOT NULL DEFAULT ''"`
MatchFullName bool `xorm:"NOT NULL DEFAULT false"`
CreatedUnix timeutil.TimeStamp `xorm:"created NOT NULL DEFAULT 0"`
UpdatedUnix timeutil.TimeStamp `xorm:"updated NOT NULL DEFAULT 0"`
}
return x.Sync(new(PackageCleanupRule))
}
modelmigration/v1_19/v235.go
+14
View File
@@ -0,0 +1,14 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_19
import "gitea.dev/modelmigration/base"
func AddIndexForAccessToken(x base.EngineMigration) error {
type AccessToken struct {
TokenLastEight string `xorm:"INDEX token_last_eight"`
}
return x.Sync(new(AccessToken))
}
modelmigration/v1_19/v236.go
+22
View File
@@ -0,0 +1,22 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_19
import (
"gitea.dev/modelmigration/base"
"gitea.dev/modules/timeutil"
)
func CreateSecretsTable(x base.EngineMigration) error {
type Secret struct {
ID int64
OwnerID int64 `xorm:"INDEX UNIQUE(owner_repo_name) NOT NULL"`
RepoID int64 `xorm:"INDEX UNIQUE(owner_repo_name) NOT NULL DEFAULT 0"`
Name string `xorm:"UNIQUE(owner_repo_name) NOT NULL"`
Data string `xorm:"LONGTEXT"`
CreatedUnix timeutil.TimeStamp `xorm:"created NOT NULL"`
}
return x.Sync(new(Secret))
}
modelmigration/v1_19/v237.go
+13
View File
@@ -0,0 +1,13 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_19
import "gitea.dev/modelmigration/base"
func DropForeignReferenceTable(x base.EngineMigration) error {
// Drop the table introduced in `v211`, it's considered badly designed and doesn't look like to be used.
// See: https://github.com/go-gitea/gitea/issues/21086#issuecomment-1318217453
type ForeignReference struct{}
return x.DropTables(new(ForeignReference))
}
modelmigration/v1_19/v238.go
+26
View File
@@ -0,0 +1,26 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_19
import (
"gitea.dev/modelmigration/base"
"gitea.dev/modules/timeutil"
)
// AddUpdatedUnixToLFSMetaObject adds an updated column to the LFSMetaObject to allow for garbage collection
func AddUpdatedUnixToLFSMetaObject(x base.EngineMigration) error {
// Drop the table introduced in `v211`, it's considered badly designed and doesn't look like to be used.
// See: https://github.com/go-gitea/gitea/issues/21086#issuecomment-1318217453
// LFSMetaObject stores metadata for LFS tracked files.
type LFSMetaObject struct {
ID int64 `xorm:"pk autoincr"`
Oid string `json:"oid" xorm:"UNIQUE(s) INDEX NOT NULL"`
Size int64 `json:"size" xorm:"NOT NULL"`
RepositoryID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"`
CreatedUnix timeutil.TimeStamp `xorm:"created"`
UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"`
}
return x.Sync(new(LFSMetaObject))
}
modelmigration/v1_19/v239.go
+20
View File
@@ -0,0 +1,20 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_19
import "gitea.dev/modelmigration/base"
func AddScopeForAccessTokens(x base.EngineMigration) error {
type AccessToken struct {
Scope string
}
if err := x.Sync(new(AccessToken)); err != nil {
return err
}
// all previous tokens have `all` and `sudo` scopes
_, err := x.Exec("UPDATE access_token SET scope = ? WHERE scope IS NULL OR scope = ''", "all,sudo")
return err
}
modelmigration/v1_19/v240.go
+175
View File
@@ -0,0 +1,175 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_19
import (
"gitea.dev/modelmigration/base"
"gitea.dev/models/db"
"gitea.dev/modules/timeutil"
)
func AddActionsTables(x base.EngineMigration) error {
type ActionRunner struct {
ID int64
UUID string `xorm:"CHAR(36) UNIQUE"`
Name string `xorm:"VARCHAR(255)"`
OwnerID int64 `xorm:"index"` // org level runner, 0 means system
RepoID int64 `xorm:"index"` // repo level runner, if orgid also is zero, then it's a global
Description string `xorm:"TEXT"`
Base int // 0 native 1 docker 2 virtual machine
RepoRange string // glob match which repositories could use this runner
Token string `xorm:"-"`
TokenHash string `xorm:"UNIQUE"` // sha256 of token
TokenSalt string
// TokenLastEight string `xorm:"token_last_eight"` // it's unnecessary because we don't find runners by token
LastOnline timeutil.TimeStamp `xorm:"index"`
LastActive timeutil.TimeStamp `xorm:"index"`
// Store OS and Artch.
AgentLabels []string
// Store custom labes use defined.
CustomLabels []string
Created timeutil.TimeStamp `xorm:"created"`
Updated timeutil.TimeStamp `xorm:"updated"`
Deleted timeutil.TimeStamp `xorm:"deleted"`
}
type ActionRunnerToken struct {
ID int64
Token string `xorm:"UNIQUE"`
OwnerID int64 `xorm:"index"` // org level runner, 0 means system
RepoID int64 `xorm:"index"` // repo level runner, if orgid also is zero, then it's a global
IsActive bool
Created timeutil.TimeStamp `xorm:"created"`
Updated timeutil.TimeStamp `xorm:"updated"`
Deleted timeutil.TimeStamp `xorm:"deleted"`
}
type ActionRun struct {
ID int64
Title string
RepoID int64 `xorm:"index unique(repo_index)"`
OwnerID int64 `xorm:"index"`
WorkflowID string `xorm:"index"` // the name of workflow file
Index int64 `xorm:"index unique(repo_index)"` // a unique number for each run of a repository
TriggerUserID int64
Ref string
CommitSHA string
Event string
IsForkPullRequest bool
EventPayload string `xorm:"LONGTEXT"`
Status int `xorm:"index"`
Started timeutil.TimeStamp
Stopped timeutil.TimeStamp
Created timeutil.TimeStamp `xorm:"created"`
Updated timeutil.TimeStamp `xorm:"updated"`
}
type ActionRunJob struct {
ID int64
RunID int64 `xorm:"index"`
RepoID int64 `xorm:"index"`
OwnerID int64 `xorm:"index"`
CommitSHA string `xorm:"index"`
IsForkPullRequest bool
Name string `xorm:"VARCHAR(255)"`
Attempt int64
WorkflowPayload []byte
JobID string `xorm:"VARCHAR(255)"` // job id in workflow, not job's id
Needs []string `xorm:"JSON TEXT"`
RunsOn []string `xorm:"JSON TEXT"`
TaskID int64 // the latest task of the job
Status int `xorm:"index"`
Started timeutil.TimeStamp
Stopped timeutil.TimeStamp
Created timeutil.TimeStamp `xorm:"created"`
Updated timeutil.TimeStamp `xorm:"updated index"`
}
type Repository struct {
NumActionRuns int `xorm:"NOT NULL DEFAULT 0"`
NumClosedActionRuns int `xorm:"NOT NULL DEFAULT 0"`
}
type ActionRunIndex db.ResourceIndex
type ActionTask struct {
ID int64
JobID int64
Attempt int64
RunnerID int64 `xorm:"index"`
Status int `xorm:"index"`
Started timeutil.TimeStamp `xorm:"index"`
Stopped timeutil.TimeStamp
RepoID int64 `xorm:"index"`
OwnerID int64 `xorm:"index"`
CommitSHA string `xorm:"index"`
IsForkPullRequest bool
TokenHash string `xorm:"UNIQUE"` // sha256 of token
TokenSalt string
TokenLastEight string `xorm:"index token_last_eight"`
LogFilename string // file name of log
LogInStorage bool // read log from database or from storage
LogLength int64 // lines count
LogSize int64 // blob size
LogIndexes []int64 `xorm:"LONGBLOB"` // line number to offset
LogExpired bool // files that are too old will be deleted
Created timeutil.TimeStamp `xorm:"created"`
Updated timeutil.TimeStamp `xorm:"updated index"`
}
type ActionTaskStep struct {
ID int64
Name string `xorm:"VARCHAR(255)"`
TaskID int64 `xorm:"index unique(task_index)"`
Index int64 `xorm:"index unique(task_index)"`
RepoID int64 `xorm:"index"`
Status int `xorm:"index"`
LogIndex int64
LogLength int64
Started timeutil.TimeStamp
Stopped timeutil.TimeStamp
Created timeutil.TimeStamp `xorm:"created"`
Updated timeutil.TimeStamp `xorm:"updated"`
}
type dbfsMeta struct {
ID int64 `xorm:"pk autoincr"`
FullPath string `xorm:"VARCHAR(500) UNIQUE NOT NULL"`
BlockSize int64 `xorm:"BIGINT NOT NULL"`
FileSize int64 `xorm:"BIGINT NOT NULL"`
CreateTimestamp int64 `xorm:"BIGINT NOT NULL"`
ModifyTimestamp int64 `xorm:"BIGINT NOT NULL"`
}
type dbfsData struct {
ID int64 `xorm:"pk autoincr"`
Revision int64 `xorm:"BIGINT NOT NULL"`
MetaID int64 `xorm:"BIGINT index(meta_offset) NOT NULL"`
BlobOffset int64 `xorm:"BIGINT index(meta_offset) NOT NULL"`
BlobSize int64 `xorm:"BIGINT NOT NULL"`
BlobData []byte `xorm:"BLOB NOT NULL"`
}
return x.Sync(
new(ActionRunner),
new(ActionRunnerToken),
new(ActionRun),
new(ActionRunJob),
new(Repository),
new(ActionRunIndex),
new(ActionTask),
new(ActionTaskStep),
new(dbfsMeta),
new(dbfsData),
)
}
modelmigration/v1_19/v241.go
+15
View File
@@ -0,0 +1,15 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_19
import "gitea.dev/modelmigration/base"
// AddCardTypeToProjectTable: add CardType column, setting existing rows to CardTypeTextOnly
func AddCardTypeToProjectTable(x base.EngineMigration) error {
type Project struct {
CardType int `xorm:"NOT NULL DEFAULT 0"`
}
return x.Sync(new(Project))
}
modelmigration/v1_19/v242.go
+25
View File
@@ -0,0 +1,25 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_19
import (
"gitea.dev/modelmigration/base"
"gitea.dev/modules/setting"
)
// AlterPublicGPGKeyImportContentFieldToMediumText: set GPGKeyImport Content field to MEDIUMTEXT
func AlterPublicGPGKeyImportContentFieldToMediumText(x base.EngineMigration) error {
sess := x.NewSession()
defer sess.Close()
if err := sess.Begin(); err != nil {
return err
}
if setting.Database.Type.IsMySQL() {
if _, err := sess.Exec("ALTER TABLE `gpg_key_import` CHANGE `content` `content` MEDIUMTEXT"); err != nil {
return err
}
}
return sess.Commit()
}
modelmigration/v1_19/v243.go
+14
View File
@@ -0,0 +1,14 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_19
import "gitea.dev/modelmigration/base"
func AddExclusiveLabel(x base.EngineMigration) error {
type Label struct {
Exclusive bool
}
return x.Sync(new(Label))
}