mirror of
https://github.com/go-gitea/gitea.git
synced 2026-09-09 06:23:24 +09:00
fix(storage): fix Azure Blob dump failing with file does not exist (#38814)
## Issue Gitea fails to dump LFS (and other object-storage) files when Azure Blob Storage is configured as the storage backend. The dump reports: Failed to dump LFS objects: /file/path: copying contents: file does not exist This happens with any non-empty base path (the default for LFS storage), which is why the user could only work around it by using `--skip-*` flags. The root cause is in `AzureBlobStorage.IterateObjects()`: Azure's list API already returns each blob's name including the configured base path, but the code was building the read client by running that name through the base-path-prepending helper a second time. This doubled the base path (e.g. `gitea-lfs/gitea-lfs/aa/bb/hash`), pointing at a blob that doesn't exist. `Stat()` still succeeded because it doesn't touch the network, so the failure only surfaced when the dumper actually tried to read the object's contents. ## Solution Add `getBlobClientByFullName()`, which builds a blob client from a name that is already fully qualified, without re-applying `buildAzureBlobPath()`. `IterateObjects()` now uses it for names obtained from Azure's list API. `getBlobClient()` (used by `Open`, `Stat`, `Delete`, `ServeDirectURL`, which take relative paths) is unchanged in behavior. Also add `TestAzureBlobStorageDumpArchive`, a regression test that drives the real dump path (`IterateObjects` → `Stat` → `dump.Dumper.AddFileByReader` → `mholt/archives` zip writer) against a **non-empty** `BasePath`, and verifies the produced archive contains the object with the correct content. The existing Azure tests use an empty `BasePath` and never read object content via `IterateObjects`, which is why they didn't catch this. Fixes #35476 --------- Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
This commit is contained in:
@@ -10,82 +10,45 @@ import (
|
||||
|
||||
"gitea.dev/modules/setting"
|
||||
"gitea.dev/modules/test"
|
||||
"gitea.dev/modules/util"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestAzureBlobStorage(t *testing.T) {
|
||||
func prepareAzureStorageConfig(t *testing.T, basePath ...string) *setting.Storage {
|
||||
endpoint := test.ExternalServiceHTTP(t, "TEST_AZURESTORAGE_ENDPOINT", "http://devstoreaccount1.azurite.local:10000")
|
||||
storageType := setting.AzureBlobStorageType
|
||||
config := &setting.Storage{
|
||||
return &setting.Storage{
|
||||
AzureBlobConfig: setting.AzureBlobStorageConfig{
|
||||
// https://learn.microsoft.com/azure/storage/common/storage-use-azurite?tabs=visual-studio-code#ip-style-url
|
||||
Endpoint: endpoint,
|
||||
// https://learn.microsoft.com/azure/storage/common/storage-use-azurite?tabs=visual-studio-code#well-known-storage-account-and-key
|
||||
AccountName: "devstoreaccount1",
|
||||
AccountKey: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==",
|
||||
Container: "test",
|
||||
Container: "test-container",
|
||||
BasePath: util.OptionalArg(basePath),
|
||||
},
|
||||
}
|
||||
table := []struct {
|
||||
name string
|
||||
test func(t *testing.T, typStr Type, cfg *setting.Storage)
|
||||
}{
|
||||
{
|
||||
name: "iterator",
|
||||
test: testStorageIterator,
|
||||
},
|
||||
{
|
||||
name: "testBlobStorageURLContentTypeAndDisposition",
|
||||
test: testBlobStorageURLContentTypeAndDisposition,
|
||||
},
|
||||
}
|
||||
for _, entry := range table {
|
||||
t.Run(entry.name, func(t *testing.T) {
|
||||
entry.test(t, storageType, config)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAzureBlobStoragePath(t *testing.T) {
|
||||
m := &AzureBlobStorage{cfg: &setting.AzureBlobStorageConfig{BasePath: ""}}
|
||||
assert.Empty(t, m.buildAzureBlobPath("/"))
|
||||
assert.Empty(t, m.buildAzureBlobPath("."))
|
||||
assert.Equal(t, "a", m.buildAzureBlobPath("/a"))
|
||||
assert.Equal(t, "a/b", m.buildAzureBlobPath("/a/b/"))
|
||||
|
||||
m = &AzureBlobStorage{cfg: &setting.AzureBlobStorageConfig{BasePath: "/"}}
|
||||
assert.Empty(t, m.buildAzureBlobPath("/"))
|
||||
assert.Empty(t, m.buildAzureBlobPath("."))
|
||||
assert.Equal(t, "a", m.buildAzureBlobPath("/a"))
|
||||
assert.Equal(t, "a/b", m.buildAzureBlobPath("/a/b/"))
|
||||
|
||||
m = &AzureBlobStorage{cfg: &setting.AzureBlobStorageConfig{BasePath: "/base"}}
|
||||
assert.Equal(t, "base", m.buildAzureBlobPath("/"))
|
||||
assert.Equal(t, "base", m.buildAzureBlobPath("."))
|
||||
assert.Equal(t, "base/a", m.buildAzureBlobPath("/a"))
|
||||
assert.Equal(t, "base/a/b", m.buildAzureBlobPath("/a/b/"))
|
||||
|
||||
m = &AzureBlobStorage{cfg: &setting.AzureBlobStorageConfig{BasePath: "/base/"}}
|
||||
assert.Equal(t, "base", m.buildAzureBlobPath("/"))
|
||||
assert.Equal(t, "base", m.buildAzureBlobPath("."))
|
||||
assert.Equal(t, "base/a", m.buildAzureBlobPath("/a"))
|
||||
assert.Equal(t, "base/a/b", m.buildAzureBlobPath("/a/b/"))
|
||||
func TestAzureBlobStorage(t *testing.T) {
|
||||
t.Run("NoBasePath", func(t *testing.T) {
|
||||
config := prepareAzureStorageConfig(t)
|
||||
objStore, err := NewStorage(setting.AzureBlobStorageType, config)
|
||||
require.NoError(t, err)
|
||||
testStorageGeneral(t, objStore)
|
||||
})
|
||||
t.Run("WithBasePath", func(t *testing.T) {
|
||||
config := prepareAzureStorageConfig(t, "test-base-path")
|
||||
objStore, err := NewStorage(setting.AzureBlobStorageType, config)
|
||||
require.NoError(t, err)
|
||||
testStorageGeneral(t, objStore)
|
||||
})
|
||||
}
|
||||
|
||||
func Test_azureBlobObject(t *testing.T) {
|
||||
endpoint := test.ExternalServiceHTTP(t, "TEST_AZURESTORAGE_ENDPOINT", "http://devstoreaccount1.azurite.local:10000")
|
||||
s, err := NewStorage(setting.AzureBlobStorageType, &setting.Storage{
|
||||
AzureBlobConfig: setting.AzureBlobStorageConfig{
|
||||
// https://learn.microsoft.com/azure/storage/common/storage-use-azurite?tabs=visual-studio-code#ip-style-url
|
||||
Endpoint: endpoint,
|
||||
// https://learn.microsoft.com/azure/storage/common/storage-use-azurite?tabs=visual-studio-code#well-known-storage-account-and-key
|
||||
AccountName: "devstoreaccount1",
|
||||
AccountKey: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==",
|
||||
Container: "test",
|
||||
},
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
s, err := NewStorage(setting.AzureBlobStorageType, prepareAzureStorageConfig(t))
|
||||
require.NoError(t, err)
|
||||
|
||||
data := "Q2xTckt6Y1hDOWh0"
|
||||
_, err = s.Save("test.txt", strings.NewReader(data), int64(len(data)))
|
||||
|
||||
Reference in New Issue
Block a user