fix(storage): fix Azure Blob dump failing with file does not exist (#38814)

## Issue

Gitea fails to dump LFS (and other object-storage) files when Azure Blob
Storage is configured as the storage backend. The dump reports:

Failed to dump LFS objects: /file/path: copying contents: file does not
exist

This happens with any non-empty base path (the default for LFS storage),
which is why the user could only work around it by using `--skip-*`
flags.

The root cause is in `AzureBlobStorage.IterateObjects()`: Azure's list
API already returns each blob's name including the configured base path,
but the code was building the read client by running that name through
the base-path-prepending helper a second time. This doubled the base
path (e.g. `gitea-lfs/gitea-lfs/aa/bb/hash`), pointing at a blob that
doesn't exist. `Stat()` still succeeded because it doesn't touch the
network, so the failure only surfaced when the dumper actually tried to
read the object's contents.

## Solution

Add `getBlobClientByFullName()`, which builds a blob client from a name
that is already fully qualified, without re-applying
`buildAzureBlobPath()`. `IterateObjects()` now uses it for names
obtained from Azure's list API. `getBlobClient()` (used by `Open`,
`Stat`, `Delete`, `ServeDirectURL`, which take relative paths) is
unchanged in behavior.

Also add `TestAzureBlobStorageDumpArchive`, a regression test that
drives the real dump path (`IterateObjects` → `Stat` →
`dump.Dumper.AddFileByReader` → `mholt/archives` zip writer) against a
**non-empty** `BasePath`, and verifies the produced archive contains the
object with the correct content. The existing Azure tests use an empty
`BasePath` and never read object content via `IterateObjects`, which is
why they didn't catch this.

Fixes #35476

---------

Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>
This commit is contained in:
Mitrahsoft
2026-08-07 16:35:30 +00:00
committed by GitHub
co-authored by wxiaoguang
parent 4c382cea59
commit 7733f1953f
7 changed files with 198 additions and 226 deletions
+34 -41
View File
@@ -32,21 +32,21 @@ var _ Object = &azureBlobObject{}
type azureBlobObject struct {
blobClient *blob.Client
Context context.Context
Name string
Size int64
ModTime *time.Time
ctx context.Context
name string
size int64
modTime *time.Time
offset int64
}
func (a *azureBlobObject) Read(p []byte) (int, error) {
// TODO: improve the performance, we can implement another interface, maybe implement io.WriteTo
if a.offset >= a.Size {
if a.offset >= a.size {
return 0, io.EOF
}
count := min(int64(len(p)), a.Size-a.offset)
count := min(int64(len(p)), a.size-a.offset)
res, err := a.blobClient.DownloadBuffer(a.Context, p, &blob.DownloadBufferOptions{
res, err := a.blobClient.DownloadBuffer(a.ctx, p, &blob.DownloadBufferOptions{
Range: blob.HTTPRange{
Offset: a.offset,
Count: count,
@@ -71,12 +71,12 @@ func (a *azureBlobObject) Seek(offset int64, whence int) (int64, error) {
case io.SeekCurrent:
offset += a.offset
case io.SeekEnd:
offset = a.Size + offset
offset = a.size + offset
default:
return 0, errors.New("Seek: invalid whence")
}
if offset > a.Size {
if offset > a.size {
return 0, errors.New("Seek: invalid offset")
} else if offset < 0 {
return 0, errors.New("Seek: invalid offset")
@@ -87,15 +87,14 @@ func (a *azureBlobObject) Seek(offset int64, whence int) (int64, error) {
func (a *azureBlobObject) Stat() (os.FileInfo, error) {
return &azureBlobFileInfo{
a.Name,
a.Size,
*a.ModTime,
a.name,
a.size,
*a.modTime,
}, nil
}
var _ ObjectStorage = &AzureBlobStorage{}
// AzureStorage returns a azure blob storage
type AzureBlobStorage struct {
cfg *setting.AzureBlobStorageConfig
ctx context.Context
@@ -150,11 +149,7 @@ func NewAzureBlobStorage(ctx context.Context, cfg *setting.Storage) (ObjectStora
}
func (a *AzureBlobStorage) buildAzureBlobPath(p string) string {
p = util.PathJoinRelX(a.cfg.BasePath, p)
if p == "." || p == "/" {
p = "" // azure uses prefix, so path should be empty as relative path
}
return p
return buildObjectStorePath(a.cfg.BasePath, p)
}
func (a *AzureBlobStorage) getObjectNameFromPath(path string) string {
@@ -170,11 +165,11 @@ func (a *AzureBlobStorage) Open(path string) (Object, error) {
return nil, convertAzureBlobErr(err)
}
return &azureBlobObject{
Context: a.ctx,
ctx: a.ctx,
blobClient: blobClient,
Name: a.getObjectNameFromPath(path),
Size: *res.ContentLength,
ModTime: res.LastModified,
name: a.getObjectNameFromPath(path),
size: *res.ContentLength,
modTime: res.LastModified,
}, nil
}
@@ -302,33 +297,32 @@ func (a *AzureBlobStorage) ServeDirectURL(storePath, name, method string, reqPar
return url.Parse(u)
}
// IterateObjects iterates across the objects in the azureblobstorage
func (a *AzureBlobStorage) IterateObjects(dirName string, fn func(path string, obj Object) error) error {
dirName = a.buildAzureBlobPath(dirName)
if dirName != "" {
dirName += "/"
}
basePrefix := buildObjectStorePathPrefix(a.cfg.BasePath, "")
dirPrefix := buildObjectStorePathPrefix(a.cfg.BasePath, dirName)
pager := a.client.NewListBlobsFlatPager(a.cfg.Container, &container.ListBlobsFlatOptions{
Prefix: &dirName,
Prefix: &dirPrefix,
})
callback := func(object *azureBlobObject, objPath string) error {
defer object.Close()
return fn(objPath, object)
}
for pager.More() {
resp, err := pager.NextPage(a.ctx)
if err != nil {
return convertAzureBlobErr(err)
}
for _, object := range resp.Segment.BlobItems {
blobClient := a.getBlobClient(*object.Name)
object := &azureBlobObject{
Context: a.ctx,
blobClient: blobClient,
Name: *object.Name,
Size: *object.Properties.ContentLength,
ModTime: object.Properties.LastModified,
for _, azureObj := range resp.Segment.BlobItems {
objPath := strings.TrimPrefix(*azureObj.Name, basePrefix)
objWrap := &azureBlobObject{
ctx: a.ctx,
blobClient: a.getBlobClient(objPath),
name: *azureObj.Name,
size: *azureObj.Properties.ContentLength,
modTime: azureObj.Properties.LastModified,
}
if err := func(object *azureBlobObject, fn func(path string, obj Object) error) error {
defer object.Close()
return fn(strings.TrimPrefix(object.Name, a.cfg.BasePath), object)
}(object, fn); err != nil {
if err := callback(objWrap, objPath); err != nil {
return convertAzureBlobErr(err)
}
}
@@ -336,7 +330,6 @@ func (a *AzureBlobStorage) IterateObjects(dirName string, fn func(path string, o
return nil
}
// Delete delete a file
func (a *AzureBlobStorage) getBlobClient(path string) *blob.Client {
return a.client.ServiceClient().NewContainerClient(a.cfg.Container).NewBlobClient(a.buildAzureBlobPath(path))
}