fix(storage): fix Azure Blob dump failing with file does not exist (#38814) (#38828)

backport #38814

modules/storage/azureblob.go

@@ -304,12 +304,10 @@ func (a *AzureBlobStorage) ServeDirectURL(storePath, name, method string, reqPar
 
 // IterateObjects iterates across the objects in the azureblobstorage
 func (a *AzureBlobStorage) IterateObjects(dirName string, fn func(path string, obj Object) error) error {
-	dirName = a.buildAzureBlobPath(dirName)
+	basePrefix := buildObjectStorePathPrefix(a.cfg.BasePath, "")
-	if dirName != "" {
+	dirPrefix := buildObjectStorePathPrefix(a.cfg.BasePath, dirName)
-		dirName += "/"
-	}
 	pager := a.client.NewListBlobsFlatPager(a.cfg.Container, &container.ListBlobsFlatOptions{
-		Prefix: &dirName,
+		Prefix: &dirPrefix,
 	})
 	for pager.More() {
 		resp, err := pager.NextPage(a.ctx)
@@ -317,7 +315,8 @@ func (a *AzureBlobStorage) IterateObjects(dirName string, fn func(path string, o
 			return convertAzureBlobErr(err)
 		}
 		for _, object := range resp.Segment.BlobItems {
-			blobClient := a.getBlobClient(*object.Name)
+			objPath := strings.TrimPrefix(*object.Name, basePrefix)
+			blobClient := a.getBlobClient(objPath)
 			object := &azureBlobObject{
 				Context:    a.ctx,
 				blobClient: blobClient,
@@ -327,7 +326,7 @@ func (a *AzureBlobStorage) IterateObjects(dirName string, fn func(path string, o
 			}
 			if err := func(object *azureBlobObject, fn func(path string, obj Object) error) error {
 				defer object.Close()
-				return fn(strings.TrimPrefix(object.Name, a.cfg.BasePath), object)
+				return fn(objPath, object)
 			}(object, fn); err != nil {
 				return convertAzureBlobErr(err)
 			}
@@ -336,7 +335,6 @@ func (a *AzureBlobStorage) IterateObjects(dirName string, fn func(path string, o
 	return nil
 }
 
-// Delete delete a file
 func (a *AzureBlobStorage) getBlobClient(path string) *blob.Client {
 	return a.client.ServiceClient().NewContainerClient(a.cfg.Container).NewBlobClient(a.buildAzureBlobPath(path))
 }

modules/storage/azureblob_test.go

@@ -27,24 +27,16 @@ func TestAzureBlobStorage(t *testing.T) {
 			Container:   "test",
 		},
 	}
-	table := []struct {
+	t.Run("Iterator", func(t *testing.T) {
-		name string
+		testStorageIterator(t, storageType, config)
-		test func(t *testing.T, typStr Type, cfg *setting.Storage)
+	})
-	}{
+	t.Run("BlobStorageURLContentTypeAndDisposition", func(t *testing.T) {
-		{
+		testBlobStorageURLContentTypeAndDisposition(t, storageType, config)
-			name: "iterator",
+	})
-			test: testStorageIterator,
+	t.Run("IteratorWithBasePath", func(t *testing.T) {
-		},
+		config.AzureBlobConfig.BasePath = "test-base-path"
-		{
+		testStorageIterator(t, storageType, config)
-			name: "testBlobStorageURLContentTypeAndDisposition",
+	})
-			test: testBlobStorageURLContentTypeAndDisposition,
-		},
-	}
-	for _, entry := range table {
-		t.Run(entry.name, func(t *testing.T) {
-			entry.test(t, storageType, config)
-		})
-	}
 }
 
 func TestAzureBlobStoragePath(t *testing.T) {

modules/storage/storage.go

@@ -11,11 +11,13 @@ import (
 	"net/url"
 	"os"
 	"path"
+	"strings"
 
 	"gitea.dev/modules/httplib"
 	"gitea.dev/modules/log"
 	"gitea.dev/modules/public"
 	"gitea.dev/modules/setting"
+	"gitea.dev/modules/util"
 )
 
 // ErrURLNotSupported represents url is not supported
@@ -139,6 +141,23 @@ func SaveFrom(objStorage ObjectStorage, path string, callback func(w io.Writer)
 	return err
 }
 
+func buildObjectStorePath(base, p string) string {
+	p = strings.TrimPrefix(util.PathJoinRelX(base, p), "/") // object store doesn't use slash for root path
+	if p == "." {
+		p = "" // object store doesn't use dot as relative path
+	}
+	return p
+}
+
+func buildObjectStorePathPrefix(base, p string) string {
+	// ending slash is required for avoiding matching like "foo/" and "foobar/" with prefix "foo"
+	p = buildObjectStorePath(base, p) + "/"
+	if p == "/" {
+		p = "" // object store doesn't use slash for root path
+	}
+	return p
+}
+
 var (
 	// Attachments represents attachments storage
 	Attachments ObjectStorage = uninitializedStorage

modules/storage/storage_test.go

@@ -4,6 +4,7 @@
 package storage
 
 import (
+	"io"
 	"net/http"
 	"strings"
 	"testing"
@@ -31,6 +32,11 @@ func testStorageIterator(t *testing.T, typStr Type, cfg *setting.Storage) {
 		_, err = l.Save(f[0], strings.NewReader(f[1]), -1)
 		assert.NoError(t, err)
 	}
+	defer func() {
+		for _, f := range testFiles {
+			_ = l.Delete(f[0])
+		}
+	}()
 
 	expectedList := map[string][]string{
 		"a":           {"a/1.txt"},
@@ -43,7 +49,9 @@ func testStorageIterator(t *testing.T, typStr Type, cfg *setting.Storage) {
 	for dir, expected := range expectedList {
 		count := 0
 		err = l.IterateObjects(dir, func(path string, f Object) error {
-			defer f.Close()
+			content, err := io.ReadAll(f)
+			assert.NoError(t, err)
+			assert.NotEmpty(t, content)
 			assert.Contains(t, expected, path)
 			count++
 			return nil