mirror of
https://github.com/go-gitea/gitea.git
synced 2026-09-08 22:13:26 +09:00
fix(storage): fix Azure Blob dump failing with file does not exist (#38814)
## Issue Gitea fails to dump LFS (and other object-storage) files when Azure Blob Storage is configured as the storage backend. The dump reports: Failed to dump LFS objects: /file/path: copying contents: file does not exist This happens with any non-empty base path (the default for LFS storage), which is why the user could only work around it by using `--skip-*` flags. The root cause is in `AzureBlobStorage.IterateObjects()`: Azure's list API already returns each blob's name including the configured base path, but the code was building the read client by running that name through the base-path-prepending helper a second time. This doubled the base path (e.g. `gitea-lfs/gitea-lfs/aa/bb/hash`), pointing at a blob that doesn't exist. `Stat()` still succeeded because it doesn't touch the network, so the failure only surfaced when the dumper actually tried to read the object's contents. ## Solution Add `getBlobClientByFullName()`, which builds a blob client from a name that is already fully qualified, without re-applying `buildAzureBlobPath()`. `IterateObjects()` now uses it for names obtained from Azure's list API. `getBlobClient()` (used by `Open`, `Stat`, `Delete`, `ServeDirectURL`, which take relative paths) is unchanged in behavior. Also add `TestAzureBlobStorageDumpArchive`, a regression test that drives the real dump path (`IterateObjects` → `Stat` → `dump.Dumper.AddFileByReader` → `mholt/archives` zip writer) against a **non-empty** `BasePath`, and verifies the produced archive contains the object with the correct content. The existing Azure tests use an empty `BasePath` and never read object content via `IterateObjects`, which is why they didn't catch this. Fixes #35476 --------- Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
This commit is contained in:
+18
-23
@@ -158,20 +158,11 @@ func NewMinioStorage(ctx context.Context, cfg *setting.Storage) (ObjectStorage,
|
||||
}
|
||||
|
||||
func (m *MinioStorage) buildMinioPath(p string) string {
|
||||
p = strings.TrimPrefix(util.PathJoinRelX(m.basePath, p), "/") // object store doesn't use slash for root path
|
||||
if p == "." {
|
||||
p = "" // object store doesn't use dot as relative path
|
||||
}
|
||||
return p
|
||||
return buildObjectStorePath(m.basePath, p)
|
||||
}
|
||||
|
||||
func (m *MinioStorage) buildMinioDirPrefix(p string) string {
|
||||
// ending slash is required for avoiding matching like "foo/" and "foobar/" with prefix "foo"
|
||||
p = m.buildMinioPath(p) + "/"
|
||||
if p == "/" {
|
||||
p = "" // object store doesn't use slash for root path
|
||||
}
|
||||
return p
|
||||
return buildObjectStorePathPrefix(m.basePath, p)
|
||||
}
|
||||
|
||||
func buildMinioCredentials(config setting.MinioStorageConfig) *credentials.Credentials {
|
||||
@@ -312,22 +303,26 @@ func (m *MinioStorage) ServeDirectURL(storePath, name, method string, opt *Serve
|
||||
return u, convertMinioErr(err)
|
||||
}
|
||||
|
||||
// IterateObjects iterates across the objects in the miniostorage
|
||||
func (m *MinioStorage) IterateObjects(dirName string, fn func(path string, obj Object) error) error {
|
||||
opts := minio.GetObjectOptions{}
|
||||
// FIXME: this loop is not right and causes resource leaking, see the comment of ListObjects
|
||||
for mObjInfo := range m.client.ListObjects(m.ctx, m.bucket, minio.ListObjectsOptions{
|
||||
Prefix: m.buildMinioDirPrefix(dirName),
|
||||
Recursive: true,
|
||||
}) {
|
||||
object, err := m.client.GetObject(m.ctx, m.bucket, mObjInfo.Key, opts)
|
||||
basePrefix := m.buildMinioDirPrefix("")
|
||||
dirPrefix := m.buildMinioDirPrefix(dirName)
|
||||
callback := func(object *minio.Object, objPath string) error {
|
||||
defer object.Close()
|
||||
return fn(objPath, &minioObject{object})
|
||||
}
|
||||
|
||||
ctxList, ctxListCancel := context.WithCancel(m.ctx)
|
||||
defer ctxListCancel() // ListObjectsIter: make sure to cancel the passed context, without that you might leak coroutines
|
||||
|
||||
getOpts := minio.GetObjectOptions{}
|
||||
listOpts := minio.ListObjectsOptions{Prefix: dirPrefix, Recursive: true}
|
||||
for mObjInfo := range m.client.ListObjectsIter(ctxList, m.bucket, listOpts) {
|
||||
object, err := m.client.GetObject(m.ctx, m.bucket, mObjInfo.Key, getOpts)
|
||||
if err != nil {
|
||||
return convertMinioErr(err)
|
||||
}
|
||||
if err := func(object *minio.Object, fn func(path string, obj Object) error) error {
|
||||
defer object.Close()
|
||||
return fn(strings.TrimPrefix(mObjInfo.Key, m.basePath), &minioObject{object})
|
||||
}(object, fn); err != nil {
|
||||
objPath := strings.TrimPrefix(mObjInfo.Key, basePrefix)
|
||||
if err := callback(object, objPath); err != nil {
|
||||
return convertMinioErr(err)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user