refactor: replace Azure Blob SDK with a REST client (#39315)

Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>

go.mod

@@ -16,8 +16,6 @@ require (
 	gitea.dev/sdk v1.2.0
 	github.com/42wim/httpsig v1.2.4
 	github.com/42wim/sshsig v0.0.0-20260317195500-b9f38cf0d432
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.7.0
 	github.com/Azure/go-ntlmssp v0.1.1
 	github.com/Necoro/html2text v0.0.0-20250804200300-7bf1ce1c7347
 	github.com/ProtonMail/go-crypto v1.4.1
@@ -74,7 +72,7 @@ require (
 	github.com/meilisearch/meilisearch-go v0.36.3
 	github.com/mholt/archives v0.1.5
 	github.com/microcosm-cc/bluemonday v1.0.27
-	github.com/microsoft/go-mssqldb v1.10.0
+	github.com/microsoft/go-mssqldb v1.11.0
 	github.com/minio/minio-go/v7 v7.3.0
 	github.com/msteinert/pam/v2 v2.1.0
 	github.com/niklasfasching/go-org v1.9.1
@@ -120,7 +118,8 @@ require (
 	cloud.google.com/go/compute/metadata v0.9.0 // indirect
 	dario.cat/mergo v1.0.2 // indirect
 	filippo.io/edwards25519 v1.2.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0 // indirect
 	github.com/DataDog/zstd v1.5.7 // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
 	github.com/RoaringBitmap/roaring/v2 v2.18.2 // indirect
@@ -276,13 +275,3 @@ ignore (
 	./vendor
 	./web_src
 )
-
-// When doing "go get -u ./...", Golang will try to update all dependencies
-// But not all latest versions of dependencies are compatible with other packages or our codebase, so we need to pin some dependencies to specific versions
-// Need to regularly maintain this list to try to update them to latest versions, especially the TODO ones
-
-replace github.com/Azure/azure-sdk-for-go/sdk/azcore => github.com/Azure/azure-sdk-for-go/sdk/azcore v1.19.0 // v1.21.0+ uses API version unsupported by Azurite in CI
-
-replace github.com/Azure/azure-sdk-for-go/sdk/storage/azblob => github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2 // v1.6.4+ uses API version unsupported by Azurite in CI
-
-replace github.com/microsoft/go-mssqldb => github.com/microsoft/go-mssqldb v1.9.7 // downgraded with Azure SDK

go.sum

@@ -30,24 +30,20 @@ github.com/42wim/httpsig v1.2.4 h1:mI5bH0nm4xn7K18fo1K3okNDRq8CCJ0KbBYWyA6r8lU=
 github.com/42wim/httpsig v1.2.4/go.mod h1:yKsYfSyTBEohkPik224QPFylmzEBtda/kjyIAJjh3ps=
 github.com/42wim/sshsig v0.0.0-20260317195500-b9f38cf0d432 h1:3Fcz1QzlS7Jv4FT2KI3cHNSZL+KPN3dXxurn9f3YL/Y=
 github.com/42wim/sshsig v0.0.0-20260317195500-b9f38cf0d432/go.mod h1:BLWe6Nol65Xxncvaw07yYMxiyk02We1lBrbRYsMYsjE=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.19.0 h1:ci6Yd6nysBRLEodoziB6ah1+YOzZbZk+NYneoA6q+6E=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1 h1:zvXfGJCWvywnCA814d8ZiVyt+fm9nnTE8xSb99zRyfo=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.19.0/go.mod h1:QyVsSSN64v5TGltphKLQ2sQxe4OBQg0J1eKRcVBnfgE=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1/go.mod h1:iptorS+VYKFL2N6PnebpS91dubG35eAOEERnT4PJbQU=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpzme37xbCDdNTxU7O9eb5+LB4=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0 h1:CU4+EJeJi3TKYWEcYuSdWsjzw0nVsK/H0MSQOiPcymU=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1/go.mod h1:IYus9qsFobWIc2YVwe/WPjcnyCkPKtnHAqUYeebc8z0=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.0/go.mod h1:q0+UTSRvShwUCrR/s5HtyInYphN7Wvxb7snFM3u+SLA=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6XuRW0nU7hgg4zlmZZa+a9q4=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0/go.mod h1:7dCRMLwisfRH3dBupKeNCioWYUZ4SS09Z14H+7i8ZoY=
-github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig=
+github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.5.0 h1:MaKvxE6D0KkjOg6Wd9M00iqP5PR0kUxCfiezes4JweM=
-github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA=
+github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.5.0/go.mod h1:i2h9fsTFKZorh8RdV2IcSUf/Qj98GlTkrTvUbX/s8as=
-github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.3.1 h1:Wgf5rZba3YZqeTNJPtvqZoBu1sBN/L4sry+u2U3Y75w=
+github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0 h1:nCYfgcSyHZXJI8J0IWE5MsCGlb2xp9fJiXyxWgmOFg4=
-github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.3.1/go.mod h1:xxCBG/f/4Vbmh2XQJBsOmNdxWUY5j/s27jujKPbQf14=
+github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.2.0/go.mod h1:ucUjca2JtSZboY8IoUqyQyuuXvwbMBVwFOm0vdQPNhA=
-github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.1.1 h1:bFWuoEKg+gImo7pvkiQEFAc8ocibADgXeiLAxWhWmkI=
-github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.1.1/go.mod h1:Vih/3yc6yac2JzU4hzpaDupBJP0Flaia9rXXrU8xyww=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2 h1:FwladfywkNirM+FZYLBR2kBz5C8Tg0fw5w5Y7meRXWI=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2/go.mod h1:vv5Ad0RrIoT1lJFdWBZwt4mB1+j+V8DUroixmKDTCdk=
 github.com/Azure/go-ntlmssp v0.1.1 h1:l+FM/EEMb0U9QZE7mKNEDw5Mu3mFiaa2GKOoTSsNDPw=
 github.com/Azure/go-ntlmssp v0.1.1/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9DiVTbhpk=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgvJqCH0sFfrBUTnUJSBrBf7++ypk+twtRs=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0 h1:Nljr4q1GRA/5vCrMONS+g4u4LRHNgOXVSh3O43J2CnI=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk=
+github.com/AzureAD/microsoft-authentication-library-for-go v1.8.0/go.mod h1:Y33QHnf0FfdVewFFISOGe20mkZbxX4H839o955/PoeI=
 github.com/DataDog/zstd v1.5.7 h1:ybO8RBeh29qrxIhCA9E8gKY6xfONU9T6G6aP9DTKfLE=
 github.com/DataDog/zstd v1.5.7/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
 github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
@@ -475,8 +471,8 @@ github.com/mholt/archives v0.1.5 h1:Fh2hl1j7VEhc6DZs2DLMgiBNChUux154a1G+2esNvzQ=
 github.com/mholt/archives v0.1.5/go.mod h1:3TPMmBLPsgszL+1As5zECTuKwKvIfj6YcwWPpeTAXF4=
 github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk=
 github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA=
-github.com/microsoft/go-mssqldb v1.9.7 h1:I+JEk79gYsc6bdVzDHFSSYE9dtNa7dxRwJ0WQbt6i8w=
+github.com/microsoft/go-mssqldb v1.11.0 h1:YbDqolEjGH9hBfvKzONTf5/dbl9RKXmizMJE93lVxNs=
-github.com/microsoft/go-mssqldb v1.9.7/go.mod h1:yYMPDufyoF2vVuVCUGtZARr06DKFIhMrluTcgWlXpr4=
+github.com/microsoft/go-mssqldb v1.11.0/go.mod h1:goQLDOPlMN/l1REhnNPElMoY/yX+fUWn1+7UoFJPH9Y=
 github.com/miekg/dns v1.1.72 h1:vhmr+TF2A3tuoGNkLDFK9zi36F2LS+hKTRW0Uf8kbzI=
 github.com/miekg/dns v1.1.72/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs=
 github.com/mikelolasagasti/xz v1.0.1 h1:Q2F2jX0RYJUG3+WsM+FJknv+6eVjsjXNDV0KJXZzkD0=

modules/setting/storage.go

@@ -59,7 +59,6 @@ func (cfg *MinioStorageConfig) ToShadow() {
 	}
 }
 
-// MinioStorageConfig represents the configuration for a minio storage
 type AzureBlobStorageConfig struct {
 	Endpoint    string `ini:"AZURE_BLOB_ENDPOINT" json:",omitempty"`
 	AccountName string `ini:"AZURE_BLOB_ACCOUNT_NAME" json:",omitempty"`

modules/storage/azureblob.go

@@ -4,15 +4,26 @@
 package storage
 
 import (
+	"bytes"
+	"cmp"
 	"context"
+	"crypto/hmac"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/xml"
 	"errors"
 	"fmt"
 	"io"
 	"io/fs"
+	"maps"
+	"net"
 	"net/http"
 	"net/url"
 	"os"
 	"path"
+	"slices"
+	"strconv"
 	"strings"
 	"time"
 
@@ -20,50 +31,71 @@ import (
 	"gitea.dev/modules/setting"
 	"gitea.dev/modules/util"
 
-	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
+	"golang.org/x/sync/errgroup"
-	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
-	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
-	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
-	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
-	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
-	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas"
 )
 
-var _ Object = &azureBlobObject{}
+const azureBlobAPIVersion = "2025-11-05" // must not exceed the Azurite version used in CI
+
+type azureBlobError string
+
+func (e azureBlobError) Error() string { return string(e) }
 
 type azureBlobObject struct {
-	blobClient *blob.Client
+	storage *AzureBlobStorage
-	ctx        context.Context
+	blobURL *url.URL
-	name       string
+	info    *objectFileInfo
-	size       int64
+	etag    string
-	modTime    *time.Time
+	offset  int64
-	offset     int64
+	closed  bool
+
+	respBody io.ReadCloser
+}
+
+func (a *azureBlobObject) resetRespBody() {
+	// close resp, the next attempt reopens at the current offset
+	if a.respBody != nil {
+		_ = a.respBody.Close()
+		a.respBody = nil
+	}
 }
 
 func (a *azureBlobObject) Read(p []byte) (int, error) {
-	// TODO: improve the performance, we can implement another interface, maybe implement io.WriteTo
+	if a.closed {
-	if a.offset >= a.size {
+		return 0, fs.ErrClosed
+	}
+	if a.offset >= a.info.size {
 		return 0, io.EOF
 	}
-	count := min(int64(len(p)), a.size-a.offset)
+	var lastErr error
-
+	for range 4 {
-	res, err := a.blobClient.DownloadBuffer(a.ctx, p, &blob.DownloadBufferOptions{
+		if a.respBody == nil {
-		Range: blob.HTTPRange{
+			reqHeader := http.Header{"X-Ms-Range": {fmt.Sprintf("bytes=%d-", a.offset)}, "If-Match": {a.etag}}
-			Offset: a.offset,
+			_, body, err := a.storage.do(a.storage.ctx, http.MethodGet, a.blobURL, reqHeader, nil)
-			Count:  count,
+			if err != nil {
-		},
+				return 0, err
-	})
+			}
-	if err != nil {
+			a.respBody = body
-		return 0, convertAzureBlobErr(err)
+		}
+		n, err := io.ReadFull(a.respBody, p[:min(int64(len(p)), a.info.size-a.offset)])
+		a.offset += int64(n)
+		if err != nil {
+			a.resetRespBody()
+		}
+		if err == nil || n > 0 {
+			return n, nil
+		}
+		lastErr = err
 	}
-	a.offset += res
+	return 0, util.Iif(lastErr == io.EOF, io.ErrUnexpectedEOF, lastErr)
-
-	return int(res), nil
 }
 
-func (a *azureBlobObject) Close() error {
+func (a *azureBlobObject) Close() (err error) {
-	a.offset = 0
+	a.closed = true
-	return nil
+	if a.respBody != nil {
+		err = a.respBody.Close()
+		a.respBody = nil
+	}
+	return err
 }
 
 func (a *azureBlobObject) Seek(offset int64, whence int) (int64, error) {
@@ -72,267 +104,311 @@ func (a *azureBlobObject) Seek(offset int64, whence int) (int64, error) {
 	case io.SeekCurrent:
 		offset += a.offset
 	case io.SeekEnd:
-		offset = a.size + offset
+		offset = a.info.size + offset
 	default:
-		return 0, errors.New("Seek: invalid whence")
+		return 0, errors.New("seek: invalid whence")
 	}
 
-	if offset > a.size {
+	if offset < 0 || offset > a.info.size {
-		return 0, errors.New("Seek: invalid offset")
+		return 0, errors.New("seek: invalid offset")
-	} else if offset < 0 {
-		return 0, errors.New("Seek: invalid offset")
 	}
+	a.resetRespBody()
 	a.offset = offset
 	return a.offset, nil
 }
 
 func (a *azureBlobObject) Stat() (os.FileInfo, error) {
-	return &azureBlobFileInfo{
+	return a.info, nil
-		a.name,
-		a.size,
-		*a.modTime,
-	}, nil
 }
 
-var _ ObjectStorage = &AzureBlobStorage{}
-
 type AzureBlobStorage struct {
-	cfg        *setting.AzureBlobStorageConfig
+	cfg         *setting.AzureBlobStorageConfig
-	ctx        context.Context
+	ctx         context.Context
-	credential *azblob.SharedKeyCredential
+	client      *http.Client
-	client     *azblob.Client
+	endpoint    *url.URL
+	key         []byte
+	blockSize   int
+	concurrency int
+	retryDelay  time.Duration
 }
 
-func convertAzureBlobErr(err error) error {
-	if err == nil {
-		return nil
-	}
-
-	if bloberror.HasCode(err, bloberror.BlobNotFound) {
-		return fs.ErrNotExist
-	}
-	var respErr *azcore.ResponseError
-	if !errors.As(err, &respErr) {
-		return err
-	}
-	return fmt.Errorf("%s", respErr.ErrorCode)
-}
-
-// NewAzureBlobStorage returns a azure blob storage
 func NewAzureBlobStorage(ctx context.Context, cfg *setting.Storage) (ObjectStorage, error) {
 	config := cfg.AzureBlobConfig
 
 	log.Info("Creating Azure Blob storage at %s:%s with base path %s", config.Endpoint, config.Container, config.BasePath)
 
-	cred, err := azblob.NewSharedKeyCredential(config.AccountName, config.AccountKey)
+	key, err := base64.StdEncoding.DecodeString(config.AccountKey)
 	if err != nil {
-		return nil, convertAzureBlobErr(err)
+		return nil, fmt.Errorf("invalid azure blob account key: %w", err)
 	}
-	client, err := azblob.NewClientWithSharedKeyCredential(config.Endpoint, cred, &azblob.ClientOptions{})
+	endpoint, err := url.Parse(config.Endpoint)
 	if err != nil {
-		return nil, convertAzureBlobErr(err)
+		return nil, err
 	}
 
-	_, err = client.CreateContainer(ctx, config.Container, &container.CreateOptions{})
+	transport := &http.Transport{
-	if err != nil {
+		Proxy:               http.ProxyFromEnvironment,
-		// Check to see if we already own this container (which happens if you run this twice)
+		DialContext:         (&net.Dialer{Timeout: 10 * time.Second}).DialContext,
-		if !bloberror.HasCode(err, bloberror.ContainerAlreadyExists) {
+		TLSHandshakeTimeout: 10 * time.Second,
-			return nil, convertMinioErr(err)
+		IdleConnTimeout:     90 * time.Second,
+		MaxIdleConnsPerHost: 10,
+	}
+	a := &AzureBlobStorage{
+		cfg:         &config,
+		ctx:         ctx,
+		client:      &http.Client{Transport: transport},
+		endpoint:    endpoint,
+		key:         key,
+		blockSize:   4 * 1024 * 1024,
+		concurrency: 4,
+		retryDelay:  200 * time.Millisecond,
+	}
+	_, _, err = a.do(ctx, http.MethodPut, a.url(config.Container, url.Values{"restype": {"container"}}), nil, nil)
+	if err != nil && !errors.Is(err, azureBlobError("ContainerAlreadyExists")) {
+		return nil, err
+	}
+	return a, nil
+}
+
+func (a *AzureBlobStorage) url(name string, query url.Values) *url.URL {
+	u := *a.endpoint
+	u.Path = strings.TrimSuffix(u.Path, "/") + "/" + name
+	u.RawQuery = strings.ReplaceAll(query.Encode(), "+", "%20") // Azure doesn't decode "+" as space
+	return &u
+}
+
+func (a *AzureBlobStorage) blobName(p string) string {
+	return a.cfg.Container + "/" + buildObjectStorePath(a.cfg.BasePath, p)
+}
+
+func (a *AzureBlobStorage) signString(s string) string {
+	mac := hmac.New(sha256.New, a.key)
+	_, _ = mac.Write([]byte(s))
+	return base64.StdEncoding.EncodeToString(mac.Sum(nil))
+}
+
+// https://learn.microsoft.com/rest/api/storageservices/authorize-with-shared-key
+func (a *AzureBlobStorage) signRequest(req *http.Request) string {
+	lines := []string{
+		req.Method,
+		req.Header.Get("Content-Encoding"),
+		req.Header.Get("Content-Language"),
+		util.Iif(req.ContentLength > 0, strconv.FormatInt(req.ContentLength, 10), ""),
+	}
+	for _, name := range []string{
+		"Content-MD5", "Content-Type", "Date",
+		"If-Modified-Since", "If-Match", "If-None-Match", "If-Unmodified-Since", "Range",
+	} {
+		lines = append(lines, req.Header.Get(name))
+	}
+	msHeaders := map[string]string{}
+	for key, values := range req.Header {
+		if key = strings.ToLower(key); strings.HasPrefix(key, "x-ms-") {
+			msHeaders[key] = key + ":" + strings.Join(values, ",")
 		}
 	}
-
+	for _, key := range slices.Sorted(maps.Keys(msHeaders)) {
-	return &AzureBlobStorage{
+		lines = append(lines, msHeaders[key])
-		cfg:        &config,
-		ctx:        ctx,
-		credential: cred,
-		client:     client,
-	}, nil
-}
-
-func (a *AzureBlobStorage) buildAzureBlobPath(p string) string {
-	return buildObjectStorePath(a.cfg.BasePath, p)
-}
-
-func (a *AzureBlobStorage) getObjectNameFromPath(path string) string {
-	s := strings.Split(path, "/")
-	return s[len(s)-1]
-}
-
-// Open opens a file
-func (a *AzureBlobStorage) Open(path string) (Object, error) {
-	blobClient := a.getBlobClient(path)
-	res, err := blobClient.GetProperties(a.ctx, &blob.GetPropertiesOptions{})
-	if err != nil {
-		return nil, convertAzureBlobErr(err)
 	}
-	return &azureBlobObject{
+	lines = append(lines, "/"+a.cfg.AccountName+req.URL.EscapedPath()) // encoded as sent, not decoded
-		ctx:        a.ctx,
+	query := req.URL.Query()
-		blobClient: blobClient,
+	for _, key := range slices.Sorted(maps.Keys(query)) {
-		name:       a.getObjectNameFromPath(path),
+		slices.Sort(query[key])
-		size:       *res.ContentLength,
+		lines = append(lines, strings.ToLower(key)+":"+strings.Join(query[key], ","))
-		modTime:    res.LastModified,
-	}, nil
-}
-
-// Save saves a file to azure blob storage
-func (a *AzureBlobStorage) Save(path string, r io.Reader, size int64) (int64, error) {
-	rd := util.NewCountingReader(r)
-	_, err := a.client.UploadStream(
-		a.ctx,
-		a.cfg.Container,
-		a.buildAzureBlobPath(path),
-		rd,
-		// TODO: support set block size and concurrency
-		&blockblob.UploadStreamOptions{},
-	)
-	if err != nil {
-		return 0, convertAzureBlobErr(err)
 	}
-	return int64(rd.Count()), nil
+	return a.signString(strings.Join(lines, "\n"))
 }
 
-type azureBlobFileInfo struct {
+// only GET returns the body, the caller closes it
-	name    string
+func (a *AzureBlobStorage) do(ctx context.Context, method string, u *url.URL, header http.Header, body []byte) (http.Header, io.ReadCloser, error) {
-	size    int64
+	const maxDelay = 3 * time.Second
-	modTime time.Time
+	for retry := 0; ; retry++ {
-}
+		req, err := http.NewRequestWithContext(ctx, method, u.String(), bytes.NewReader(body))
-
-func (a azureBlobFileInfo) Name() string {
-	return path.Base(a.name)
-}
-
-func (a azureBlobFileInfo) Size() int64 {
-	return a.size
-}
-
-func (a azureBlobFileInfo) ModTime() time.Time {
-	return a.modTime
-}
-
-func (a azureBlobFileInfo) IsDir() bool {
-	return strings.HasSuffix(a.name, "/")
-}
-
-func (a azureBlobFileInfo) Mode() os.FileMode {
-	return os.ModePerm
-}
-
-func (a azureBlobFileInfo) Sys() any {
-	return nil
-}
-
-// Stat returns the stat information of the object
-func (a *AzureBlobStorage) Stat(path string) (os.FileInfo, error) {
-	blobClient := a.getBlobClient(path)
-	res, err := blobClient.GetProperties(a.ctx, &blob.GetPropertiesOptions{})
-	if err != nil {
-		return nil, convertAzureBlobErr(err)
-	}
-	s := strings.Split(path, "/")
-	return &azureBlobFileInfo{
-		s[len(s)-1],
-		*res.ContentLength,
-		*res.LastModified,
-	}, nil
-}
-
-// Delete delete a file
-func (a *AzureBlobStorage) Delete(path string) error {
-	blobClient := a.getBlobClient(path)
-	_, err := blobClient.Delete(a.ctx, nil)
-	return convertAzureBlobErr(err)
-}
-
-func (a *AzureBlobStorage) getSasURL(b *blob.Client, template sas.BlobSignatureValues) (string, error) {
-	urlParts, err := blob.ParseURL(b.URL())
-	if err != nil {
-		return "", err
-	}
-
-	var t time.Time
-	if urlParts.Snapshot == "" {
-		t = time.Time{}
-	} else {
-		t, err = time.Parse(blob.SnapshotTimeFormat, urlParts.Snapshot)
 		if err != nil {
-			return "", err
+			return nil, nil, err
 		}
+		maps.Copy(req.Header, header)
+		req.Header.Set("x-ms-date", time.Now().UTC().Format(http.TimeFormat))
+		req.Header.Set("x-ms-version", azureBlobAPIVersion)
+		req.Header.Set("Authorization", "SharedKey "+a.cfg.AccountName+":"+a.signRequest(req))
+
+		resp, err := a.client.Do(req)
+		if retry < 3 && (err != nil || slices.Contains([]int{408, 429, 500, 502, 503, 504}, resp.StatusCode)) {
+			if err == nil {
+				_ = resp.Body.Close()
+			}
+			select {
+			case <-ctx.Done():
+				return nil, nil, ctx.Err()
+			case <-time.After(min(a.retryDelay<<retry, maxDelay)):
+			}
+			continue
+		}
+		if err != nil {
+			return nil, nil, err
+		}
+		if resp.StatusCode < http.StatusBadRequest {
+			if method == http.MethodGet {
+				return resp.Header, resp.Body, nil
+			}
+			_ = resp.Body.Close()
+			return resp.Header, nil, nil
+		}
+		var errBody struct{ Message, AuthenticationErrorDetail string }
+		_ = xml.NewDecoder(resp.Body).Decode(&errBody)
+		_ = resp.Body.Close()
+		code := cmp.Or(resp.Header.Get("x-ms-error-code"), resp.Status)
+		if code == "BlobNotFound" {
+			return nil, nil, fs.ErrNotExist
+		}
+		if errBody.Message == "" {
+			return nil, nil, azureBlobError(code)
+		}
+		return nil, nil, fmt.Errorf("%w: %s", azureBlobError(code), strings.TrimSpace(errBody.Message+"\n"+errBody.AuthenticationErrorDetail))
 	}
-
-	template.ContainerName = urlParts.ContainerName
-	template.BlobName = urlParts.BlobName
-	template.SnapshotTime = t
-	template.Version = sas.Version
-
-	qps, err := template.SignWithSharedKey(a.credential)
-	if err != nil {
-		return "", err
-	}
-
-	endpoint := b.URL() + "?" + qps.Encode()
-
-	return endpoint, nil
 }
 
-func (a *AzureBlobStorage) ServeDirectURL(storePath, name, method string, reqParams *ServeDirectOptions) (*url.URL, error) {
+func (a *AzureBlobStorage) Open(path string) (Object, error) {
-	blobClient := a.getBlobClient(storePath)
+	obj, err := a.stat(path)
-
-	startTime := time.Now().UTC()
-
-	param := prepareServeDirectOptions(reqParams, name)
-
-	u, err := a.getSasURL(blobClient, sas.BlobSignatureValues{
-		Permissions: (&sas.BlobPermissions{
-			Read:  method == http.MethodGet || method == http.MethodHead,
-			Write: method == http.MethodPut,
-		}).String(),
-		StartTime:          startTime,
-		ExpiryTime:         startTime.Add(5 * time.Minute),
-		ContentDisposition: param.ContentDisposition,
-		ContentType:        param.ContentType,
-	})
 	if err != nil {
-		return nil, convertAzureBlobErr(err)
+		return nil, err
+	}
+	return obj, nil
+}
+
+func (a *AzureBlobStorage) Save(path string, r io.Reader, _ int64) (int64, error) {
+	name := a.blobName(path)
+	block := make([]byte, a.blockSize)
+	n, err := util.ReadAtMost(r, block)
+	if err != nil {
+		return 0, err
+	}
+	if n < a.blockSize {
+		_, _, err := a.do(a.ctx, http.MethodPut, a.url(name, nil), http.Header{"X-Ms-Blob-Type": {"BlockBlob"}}, block[:n])
+		return int64(n), err
 	}
 
-	return url.Parse(u)
+	g, ctx := errgroup.WithContext(a.ctx)
+	g.SetLimit(a.concurrency)
+	blockList := bytes.NewBufferString(`<?xml version="1.0" encoding="utf-8"?><BlockList>`)
+	idPrefix := rand.Text()
+	var total int64
+	for blockNum := 0; n > 0 && ctx.Err() == nil; blockNum++ {
+		id := base64.StdEncoding.EncodeToString(fmt.Appendf(nil, "%s%038d", idPrefix, blockNum)) // 64 bytes like the old SDK, Azure rejects mixed ID lengths
+		blockList.WriteString("<Latest>" + id + "</Latest>")
+		total += int64(n)
+		data := block[:n]
+		g.Go(func() error {
+			_, _, err := a.do(ctx, http.MethodPut, a.url(name, url.Values{"comp": {"block"}, "blockid": {id}}), nil, data)
+			return err
+		})
+		if n < a.blockSize {
+			break
+		}
+		block = make([]byte, a.blockSize)
+		if n, err = util.ReadAtMost(r, block); err != nil {
+			break
+		}
+	}
+	if err = errors.Join(g.Wait(), err); err != nil {
+		return 0, err
+	}
+	blockList.WriteString("</BlockList>")
+	_, _, err = a.do(a.ctx, http.MethodPut, a.url(name, url.Values{"comp": {"blocklist"}}), nil, blockList.Bytes())
+	return total, err
+}
+
+func (a *AzureBlobStorage) stat(p string) (*azureBlobObject, error) {
+	blobURL := a.url(a.blobName(p), nil)
+	header, _, err := a.do(a.ctx, http.MethodHead, blobURL, nil, nil)
+	if err != nil {
+		return nil, err
+	}
+	size, sizeErr := strconv.ParseInt(header.Get("Content-Length"), 10, 64)
+	modTime, timeErr := http.ParseTime(header.Get("Last-Modified"))
+	return &azureBlobObject{storage: a, blobURL: blobURL, info: &objectFileInfo{path.Base(p), size, modTime}, etag: header.Get("ETag")}, errors.Join(sizeErr, timeErr)
+}
+
+func (a *AzureBlobStorage) Stat(path string) (os.FileInfo, error) {
+	obj, err := a.stat(path)
+	if err != nil {
+		return nil, err // not obj.info, a nil pointer would be a non-nil os.FileInfo
+	}
+	return obj.info, nil
+}
+
+func (a *AzureBlobStorage) Delete(path string) error {
+	_, _, err := a.do(a.ctx, http.MethodDelete, a.url(a.blobName(path), nil), nil, nil)
+	return err
+}
+
+// https://learn.microsoft.com/rest/api/storageservices/create-service-sas
+func (a *AzureBlobStorage) ServeDirectURL(storePath, name, method string, reqParams *ServeDirectOptions) (*url.URL, error) {
+	permissions := util.Iif(method == http.MethodPut, "w", "r")
+	param := prepareServeDirectOptions(reqParams, name)
+	now := time.Now().UTC()
+	start, expiry := now.Add(-15*time.Minute).Format(time.RFC3339), now.Add(5*time.Minute).Format(time.RFC3339) // SAS expiration policies require a start, backdated for clock skew
+	canonicalName := "/blob/" + a.cfg.AccountName + "/" + a.blobName(storePath)
+	signature := a.signString(strings.Join([]string{permissions, start, expiry, canonicalName, "", "", "", azureBlobAPIVersion, "b", "", "", "", param.ContentDisposition, "", "", param.ContentType}, "\n"))
+
+	query := url.Values{"sv": {azureBlobAPIVersion}, "st": {start}, "se": {expiry}, "sr": {"b"}, "sp": {permissions}, "sig": {signature}}
+	if param.ContentDisposition != "" {
+		query.Set("rscd", param.ContentDisposition)
+	}
+	if param.ContentType != "" {
+		query.Set("rsct", param.ContentType)
+	}
+	return a.url(a.blobName(storePath), query), nil
 }
 
 func (a *AzureBlobStorage) IterateObjects(dirName string, fn func(path string, obj Object) error) error {
 	basePrefix := buildObjectStorePathPrefix(a.cfg.BasePath, "")
-	dirPrefix := buildObjectStorePathPrefix(a.cfg.BasePath, dirName)
+	query := url.Values{"restype": {"container"}, "comp": {"list"}, "prefix": {buildObjectStorePathPrefix(a.cfg.BasePath, dirName)}}
-	pager := a.client.NewListBlobsFlatPager(a.cfg.Container, &container.ListBlobsFlatOptions{
+	for {
-		Prefix: &dirPrefix,
+		_, body, err := a.do(a.ctx, http.MethodGet, a.url(a.cfg.Container, query), nil, nil)
-	})
-
-	callback := func(object *azureBlobObject, objPath string) error {
-		defer object.Close()
-		return fn(objPath, object)
-	}
-	for pager.More() {
-		resp, err := pager.NextPage(a.ctx)
 		if err != nil {
-			return convertAzureBlobErr(err)
+			return err
 		}
-		for _, azureObj := range resp.Segment.BlobItems {
+		var result struct {
-			objPath := strings.TrimPrefix(*azureObj.Name, basePrefix)
+			Blobs []struct {
-			objWrap := &azureBlobObject{
+				Name          string `xml:"Name"`
-				ctx:        a.ctx,
+				ContentLength int64  `xml:"Properties>Content-Length"`
-				blobClient: a.getBlobClient(objPath),
+				LastModified  string `xml:"Properties>Last-Modified"`
-				name:       *azureObj.Name,
+				Etag          string `xml:"Properties>Etag"`
-				size:       *azureObj.Properties.ContentLength,
+				ResourceType  string `xml:"Properties>ResourceType"`
-				modTime:    azureObj.Properties.LastModified,
+			} `xml:"Blobs>Blob"`
+			NextMarker string `xml:"NextMarker"`
+		}
+		err = xml.NewDecoder(body).Decode(&result)
+		_ = body.Close()
+		if err != nil {
+			return err
+		}
+		for _, blob := range result.Blobs {
+			if blob.ResourceType == "directory" { // listed by hierarchical namespace accounts
+				continue
 			}
-			if err := callback(objWrap, objPath); err != nil {
+			modTime, err := http.ParseTime(blob.LastModified)
-				return convertAzureBlobErr(err)
+			if err != nil {
+				return err
+			}
+			object := &azureBlobObject{
+				storage: a,
+				blobURL: a.url(a.cfg.Container+"/"+blob.Name, nil),
+				info:    &objectFileInfo{path.Base(blob.Name), blob.ContentLength, modTime},
+				etag:    blob.Etag,
+			}
+			err = fn(strings.TrimPrefix(blob.Name, basePrefix), object)
+			_ = object.Close()
+			if err != nil {
+				return err
 			}
 		}
+		if result.NextMarker == "" {
+			return nil
+		}
+		query.Set("marker", result.NextMarker)
 	}
-	return nil
-}
-
-func (a *AzureBlobStorage) getBlobClient(path string) *blob.Client {
-	return a.client.ServiceClient().NewContainerClient(a.cfg.Container).NewBlobClient(a.buildAzureBlobPath(path))
 }
 
 func init() {

modules/storage/azureblob_test.go

@@ -5,72 +5,121 @@ package storage
 
 import (
 	"io"
+	"io/fs"
+	"net/http"
+	"os"
 	"strings"
+	"sync/atomic"
 	"testing"
+	"testing/iotest"
 
 	"gitea.dev/modules/setting"
 	"gitea.dev/modules/test"
-	"gitea.dev/modules/util"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
-func prepareAzureStorageConfig(t *testing.T, basePath ...string) *setting.Storage {
+type azureBlobFaultTransport struct {
+	failed, truncated atomic.Bool
+}
+
+func (t *azureBlobFaultTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	if t.failed.CompareAndSwap(false, true) {
+		return &http.Response{StatusCode: http.StatusServiceUnavailable, Body: http.NoBody}, nil
+	}
+	resp, err := http.DefaultTransport.RoundTrip(req)
+	if err == nil && req.Method == http.MethodGet && t.truncated.CompareAndSwap(false, true) {
+		resp.Body = struct {
+			io.Reader
+			io.Closer
+		}{io.MultiReader(io.LimitReader(resp.Body, 2), iotest.ErrReader(io.ErrUnexpectedEOF)), resp.Body}
+	}
+	return resp, err
+}
+
+func newAzureBlobTestStorage(t *testing.T, basePath string) *AzureBlobStorage {
 	endpoint := test.ExternalServiceHTTP(t, "TEST_AZURESTORAGE_ENDPOINT", "http://devstoreaccount1.azurite.local:10000")
-	return &setting.Storage{
+	objStore, err := NewStorage(setting.AzureBlobStorageType, &setting.Storage{
 		AzureBlobConfig: setting.AzureBlobStorageConfig{
-			// https://learn.microsoft.com/azure/storage/common/storage-use-azurite?tabs=visual-studio-code#ip-style-url
+			Endpoint:    endpoint,
-			Endpoint: endpoint,
-			// https://learn.microsoft.com/azure/storage/common/storage-use-azurite?tabs=visual-studio-code#well-known-storage-account-and-key
 			AccountName: "devstoreaccount1",
 			AccountKey:  "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==",
 			Container:   "test-container",
-			BasePath:    util.OptionalArg(basePath),
+			BasePath:    basePath,
 		},
-	}
+	})
+	require.NoError(t, err)
+	s, ok := objStore.(*AzureBlobStorage)
+	require.True(t, ok)
+	return s
 }
 
 func TestAzureBlobStorage(t *testing.T) {
-	t.Run("NoBasePath", func(t *testing.T) {
+	t.Run("General", func(t *testing.T) { testStorageGeneral(t, newAzureBlobTestStorage(t, "")) })
-		config := prepareAzureStorageConfig(t)
+	t.Run("GeneralWithBasePath", func(t *testing.T) { testStorageGeneral(t, newAzureBlobTestStorage(t, "test-base-path")) })
-		objStore, err := NewStorage(setting.AzureBlobStorageType, config)
-		require.NoError(t, err)
-		testStorageGeneral(t, objStore)
-	})
-	t.Run("WithBasePath", func(t *testing.T) {
-		config := prepareAzureStorageConfig(t, "test-base-path")
-		objStore, err := NewStorage(setting.AzureBlobStorageType, config)
-		require.NoError(t, err)
-		testStorageGeneral(t, objStore)
-	})
-}
-
-func Test_azureBlobObject(t *testing.T) {
-	s, err := NewStorage(setting.AzureBlobStorageType, prepareAzureStorageConfig(t))
-	require.NoError(t, err)
 
+	s := newAzureBlobTestStorage(t, "")
+	s.blockSize, s.concurrency, s.retryDelay = 4, 2, 0
+	transport := &azureBlobFaultTransport{}
+	s.client.Transport = transport
 	data := "Q2xTckt6Y1hDOWh0"
-	_, err = s.Save("test.txt", strings.NewReader(data), int64(len(data)))
+
-	assert.NoError(t, err)
+	t.Run("SaveBlocksWithRetryAndRejectTruncatedInput", func(t *testing.T) {
-	obj, err := s.Open("test.txt")
+		written, err := s.Save("test.txt", strings.NewReader(data), -1)
-	assert.NoError(t, err)
+		require.NoError(t, err)
-	offset, err := obj.Seek(2, io.SeekStart)
+		assert.EqualValues(t, len(data), written)
-	assert.NoError(t, err)
+		assert.True(t, transport.failed.Load())
-	assert.EqualValues(t, 2, offset)
+		_, err = s.Save("truncated.txt", io.MultiReader(strings.NewReader(data), iotest.ErrReader(io.ErrUnexpectedEOF)), -1)
-	buf1 := make([]byte, 3)
+		assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
-	read, err := obj.Read(buf1)
+	})
-	assert.NoError(t, err)
+
-	assert.Equal(t, 3, read)
+	t.Run("ReadResumesSeeksAndFailsOnChangedBlob", func(t *testing.T) {
-	assert.Equal(t, data[2:5], string(buf1))
+		obj, err := s.Open("test.txt")
-	offset, err = obj.Seek(-5, io.SeekEnd)
+		require.NoError(t, err)
-	assert.NoError(t, err)
+		defer obj.Close()
-	assert.EqualValues(t, len(data)-5, offset)
+		buf := make([]byte, 4)
-	buf2 := make([]byte, 4)
+		_, err = io.ReadFull(obj, buf)
-	read, err = obj.Read(buf2)
+		require.NoError(t, err)
-	assert.NoError(t, err)
+		assert.Equal(t, data[:4], string(buf))
-	assert.Equal(t, 4, read)
+		assert.True(t, transport.truncated.Load())
-	assert.Equal(t, data[11:15], string(buf2))
+		_, err = obj.Seek(-5, io.SeekEnd)
-	assert.NoError(t, obj.Close())
+		require.NoError(t, err)
-	assert.NoError(t, s.Delete("test.txt"))
+		_, err = io.ReadFull(obj, buf)
+		require.NoError(t, err)
+		assert.Equal(t, data[11:15], string(buf))
+
+		_, err = s.Save("test.txt", strings.NewReader("changed"), -1)
+		require.NoError(t, err)
+		_, err = obj.Seek(0, io.SeekStart)
+		require.NoError(t, err)
+		_, err = io.ReadAll(obj)
+		assert.ErrorIs(t, err, azureBlobError("ConditionNotMet"))
+	})
+
+	t.Run("ServeDirectURLAllowsPut", func(t *testing.T) {
+		u, err := s.ServeDirectURL("direct.txt", "direct.txt", http.MethodPut, nil)
+		require.NoError(t, err)
+		req, err := http.NewRequestWithContext(t.Context(), http.MethodPut, u.String(), strings.NewReader("direct"))
+		require.NoError(t, err)
+		req.Header.Set("x-ms-blob-type", "BlockBlob")
+		resp, err := http.DefaultClient.Do(req)
+		require.NoError(t, err)
+		_ = resp.Body.Close()
+		assert.Equal(t, http.StatusCreated, resp.StatusCode)
+		obj, err := s.Open("direct.txt")
+		require.NoError(t, err)
+		defer obj.Close()
+		content, err := io.ReadAll(obj)
+		require.NoError(t, err)
+		assert.Equal(t, "direct", string(content))
+	})
+
+	t.Run("StatAfterDeleteReturnsNilInfo", func(t *testing.T) {
+		assert.NoError(t, s.Delete("test.txt"))
+		assert.NoError(t, s.Delete("direct.txt"))
+		info, err := s.Stat("test.txt")
+		assert.ErrorIs(t, err, fs.ErrNotExist)
+		assert.Equal(t, os.FileInfo(nil), info)
+	})
 }

modules/storage/minio.go

@@ -39,7 +39,7 @@ func (m *minioObject) Stat() (os.FileInfo, error) {
 		return nil, convertMinioErr(err)
 	}
 
-	return &minioFileInfo{oi}, nil
+	return &objectFileInfo{path.Base(oi.Key), oi.Size, oi.LastModified}, nil
 }
 
 // minio reports a missing key on the first Read, ReadAt or Seek rather than on Open, so all
@@ -234,46 +234,18 @@ func (m *MinioStorage) Save(path string, r io.Reader, size int64) (int64, error)
 	return uploadInfo.Size, nil
 }
 
-type minioFileInfo struct {
-	minio.ObjectInfo
-}
-
-func (m minioFileInfo) Name() string {
-	return path.Base(m.ObjectInfo.Key)
-}
-
-func (m minioFileInfo) Size() int64 {
-	return m.ObjectInfo.Size
-}
-
-func (m minioFileInfo) ModTime() time.Time {
-	return m.LastModified
-}
-
-func (m minioFileInfo) IsDir() bool {
-	return strings.HasSuffix(m.ObjectInfo.Key, "/")
-}
-
-func (m minioFileInfo) Mode() os.FileMode {
-	return os.ModePerm
-}
-
-func (m minioFileInfo) Sys() any {
-	return nil
-}
-
 // Stat returns the stat information of the object
-func (m *MinioStorage) Stat(path string) (os.FileInfo, error) {
+func (m *MinioStorage) Stat(p string) (os.FileInfo, error) {
 	info, err := m.client.StatObject(
 		m.ctx,
 		m.bucket,
-		m.buildMinioPath(path),
+		m.buildMinioPath(p),
 		minio.StatObjectOptions{},
 	)
 	if err != nil {
 		return nil, convertMinioErr(err)
 	}
-	return &minioFileInfo{info}, nil
+	return &objectFileInfo{path.Base(info.Key), info.Size, info.LastModified}, nil
 }
 
 // Delete delete a file

modules/storage/minio_test.go

@@ -10,13 +10,12 @@ import (
 
 	"gitea.dev/modules/setting"
 	"gitea.dev/modules/test"
-	"gitea.dev/modules/util"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
-func prepareMinioStorageConfig(t *testing.T, basePath ...string) *setting.Storage {
+func prepareMinioStorageConfig(t *testing.T, basePath string) *setting.Storage {
 	return &setting.Storage{
 		MinioConfig: setting.MinioStorageConfig{
 			Endpoint:        test.ExternalServiceHTTP(t, "TEST_MINIO_ENDPOINT", "minio:9000"),
@@ -24,28 +23,21 @@ func prepareMinioStorageConfig(t *testing.T, basePath ...string) *setting.Storag
 			SecretAccessKey: "12345678",
 			Bucket:          "gitea",
 			Location:        "us-east-1",
-			BasePath:        util.OptionalArg(basePath),
+			BasePath:        basePath,
 		},
 	}
 }
 
 func TestMinioStorage(t *testing.T) {
-	t.Run("NoBasePath", func(t *testing.T) {
+	for _, basePath := range []string{"", "test-base-path"} {
-		config := prepareMinioStorageConfig(t)
+		objStore, err := NewStorage(setting.MinioStorageType, prepareMinioStorageConfig(t, basePath))
-		objStore, err := NewStorage(setting.MinioStorageType, config)
 		require.NoError(t, err)
 		testStorageGeneral(t, objStore)
-	})
+	}
-	t.Run("WithBasePath", func(t *testing.T) {
-		config := prepareMinioStorageConfig(t, "test-base-path")
-		objStore, err := NewStorage(setting.MinioStorageType, config)
-		require.NoError(t, err)
-		testStorageGeneral(t, objStore)
-	})
 }
 
 func TestS3StorageBadRequest(t *testing.T) {
-	cfg := prepareMinioStorageConfig(t)
+	cfg := prepareMinioStorageConfig(t, "")
 	cfg.MinioConfig.SecretAccessKey = "invalid-secret"
 	_, err := NewStorage(setting.MinioStorageType, cfg)
 	assert.ErrorContains(t, err, "ObjectStorage.BucketExists: endpoint="+cfg.MinioConfig.Endpoint)

modules/storage/storage.go

@@ -12,6 +12,7 @@ import (
 	"os"
 	"path"
 	"strings"
+	"time"
 
 	"gitea.dev/modules/httplib"
 	"gitea.dev/modules/log"
@@ -42,6 +43,19 @@ type Object interface {
 	Stat() (os.FileInfo, error)
 }
 
+type objectFileInfo struct {
+	name    string
+	size    int64
+	modTime time.Time
+}
+
+func (o *objectFileInfo) Name() string       { return o.name } // base name, no dir path
+func (o *objectFileInfo) Size() int64        { return o.size }
+func (o *objectFileInfo) ModTime() time.Time { return o.modTime }
+func (o *objectFileInfo) IsDir() bool        { return false } // object storage doesn't support dir
+func (o *objectFileInfo) Mode() os.FileMode  { return os.ModePerm }
+func (o *objectFileInfo) Sys() any           { return nil }
+
 // ServeDirectOptions customizes HTTP headers for a generated signed URL.
 type ServeDirectOptions struct {
 	// Overrides the automatically detected MIME type.

modules/storage/storage_test.go

@@ -6,6 +6,7 @@ package storage
 import (
 	"io"
 	"net/http"
+	"path"
 	"strings"
 	"testing"
 
@@ -47,7 +48,7 @@ func TestObjectStoragePath(t *testing.T) {
 	assert.Equal(t, "base/a/", buildObjectStorePathPrefix(base, "/a/"))
 }
 
-func testStorageIterator(t *testing.T, objStore ObjectStorage) {
+func testStorageAccess(t *testing.T, objStore ObjectStorage) {
 	testFiles := [][]string{
 		{"a/1.txt", "a1"},
 		{"/a/1.txt", "aa1"}, // same as above, but with leading slash that will be trim
@@ -57,39 +58,64 @@ func testStorageIterator(t *testing.T, objStore ObjectStorage) {
 		{"b/3.txt", "b3"},
 		{"b/x 4.txt", "bx4"},
 	}
-	for _, f := range testFiles {
-		_, err := objStore.Save(f[0], strings.NewReader(f[1]), -1)
-		assert.NoError(t, err)
-	}
 	defer func() {
 		for _, f := range testFiles {
 			_ = objStore.Delete(f[0])
 		}
 	}()
 
-	expectedList := map[string][]string{
+	t.Run("ReadWrite", func(t *testing.T) {
-		"a":           {"a/1.txt"},
+		for _, it := range testFiles {
-		"a/":          {"a/1.txt"},
+			fp, content := it[0], it[1]
-		"/a/":         {"a/1.txt"},
+			_, err := objStore.Save(fp, strings.NewReader(content), -1)
-		"b":           {"b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt"},
-		"":            {"a/1.txt", "b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt", "ab/1.txt"},
-		"/":           {"a/1.txt", "b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt", "ab/1.txt"},
-		".":           {"a/1.txt", "b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt", "ab/1.txt"},
-		"a/b/../../a": {"a/1.txt"},
-	}
-	for dir, expected := range expectedList {
-		count := 0
-		err := objStore.IterateObjects(dir, func(path string, f Object) error {
-			content, err := io.ReadAll(f)
 			assert.NoError(t, err)
-			assert.NotEmpty(t, content)
+
-			assert.Contains(t, expected, path)
+			stat, err := objStore.Stat(fp)
-			count++
+			assert.NoError(t, err)
-			return nil
+			assert.Equal(t, path.Base(fp), stat.Name())
-		})
+
-		assert.NoError(t, err)
+			f, err := objStore.Open(fp)
-		assert.Len(t, expected, count)
+			assert.NoError(t, err)
-	}
+			stat, err = f.Stat()
+			assert.NoError(t, err)
+			assert.Equal(t, path.Base(fp), stat.Name())
+
+			contentBytes, err := io.ReadAll(f)
+			assert.NoError(t, err)
+			assert.Equal(t, content, string(contentBytes))
+			_ = f.Close()
+		}
+	})
+
+	t.Run("Iterator", func(t *testing.T) {
+		expectedList := map[string][]string{
+			"a":           {"a/1.txt"},
+			"a/":          {"a/1.txt"},
+			"/a/":         {"a/1.txt"},
+			"b":           {"b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt"},
+			"":            {"a/1.txt", "b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt", "ab/1.txt"},
+			"/":           {"a/1.txt", "b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt", "ab/1.txt"},
+			".":           {"a/1.txt", "b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt", "ab/1.txt"},
+			"a/b/../../a": {"a/1.txt"},
+		}
+		for dir, expected := range expectedList {
+			count := 0
+			err := objStore.IterateObjects(dir, func(p string, f Object) error {
+				content, err := io.ReadAll(f)
+				assert.NoError(t, err)
+				assert.NotEmpty(t, content)
+				assert.Contains(t, expected, p)
+				count++
+
+				stat, err := f.Stat()
+				assert.NoError(t, err)
+				assert.Equal(t, path.Base(p), stat.Name())
+				return nil
+			})
+			assert.NoError(t, err)
+			assert.Len(t, expected, count)
+		}
+	})
 }
 
 func testStorageURLContentTypeAndDisposition(t *testing.T, objStore ObjectStorage) {
@@ -139,7 +165,7 @@ func testStorageURLContentTypeAndDisposition(t *testing.T, objStore ObjectStorag
 }
 
 func testStorageGeneral(t *testing.T, objStore ObjectStorage) {
-	t.Run("StorageIterator", func(t *testing.T) { testStorageIterator(t, objStore) })
+	t.Run("StorageAccess", func(t *testing.T) { testStorageAccess(t, objStore) })
 
 	if _, ok := objStore.(*LocalStorage); ok {
 		t.Skipf("Skipping tests for local storage")

modules/util/io.go

@@ -17,17 +17,18 @@ func (NopCloser) Close() error { return nil }
 
 // ReadAtMost reads at most len(buf) bytes from r into buf.
 // It returns the number of bytes copied. n is only less than len(buf) if r provides fewer bytes.
-// If EOF or ErrUnexpectedEOF occurs while reading, err will be nil.
+// If EOF occurs while reading, err will be nil.
 func ReadAtMost(r io.Reader, buf []byte) (n int, err error) {
-	n, err = io.ReadFull(r, buf)
+	for n < len(buf) && err == nil {
-	if err == io.EOF || err == io.ErrUnexpectedEOF {
+		var read int
-		err = nil
+		read, err = r.Read(buf[n:])
+		n += read
 	}
-	return n, err
+	return n, Iif(err == io.EOF, nil, err)
 }
 
 // ReadWithLimit reads at most "limit" bytes from r into buf.
-// If EOF or ErrUnexpectedEOF occurs while reading, err will be nil.
+// If EOF occurs while reading, err will be nil.
 func ReadWithLimit(r io.Reader, n int) (buf []byte, err error) {
 	return readWithLimit(r, 4*1024, n)
 }
@@ -49,12 +50,11 @@ func readWithLimit(r io.Reader, batch, limit int) ([]byte, error) {
 		if res.Len()+batch > limit {
 			bufTmp = bufFix[:limit-res.Len()]
 		}
-		n, err := io.ReadFull(r, bufTmp)
+		n, err := ReadAtMost(r, bufTmp)
-		if err == io.EOF || err == io.ErrUnexpectedEOF {
+		if err != nil {
-			eof = true
-		} else if err != nil {
 			return nil, err
 		}
+		eof = n != len(bufTmp)
 		if _, err = res.Write(bufTmp[:n]); err != nil {
 			return nil, err
 		}
@@ -82,24 +82,3 @@ func IsEmptyReader(r io.Reader) (err error) {
 		}
 	}
 }
-
-type CountingReader struct {
-	io.Reader
-	n int
-}
-
-var _ io.Reader = &CountingReader{}
-
-func (w *CountingReader) Count() int {
-	return w.n
-}
-
-func (w *CountingReader) Read(p []byte) (int, error) {
-	n, err := w.Reader.Read(p)
-	w.n += n
-	return n, err
-}
-
-func NewCountingReader(rd io.Reader) *CountingReader {
-	return &CountingReader{Reader: rd}
-}

renovate.json5

@@ -86,18 +86,6 @@
       "matchPackageNames": ["typescript"],
       "allowedVersions": "^6", // typescript-eslint is not yet compatible with typescript 7
     },
-    {
-      "matchPackageNames": ["github.com/Azure/azure-sdk-for-go/sdk/azcore"],
-      "allowedVersions": "<1.21.0", // v1.21.0+ uses API version unsupported by Azurite in CI
-    },
-    {
-      "matchPackageNames": ["github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"],
-      "allowedVersions": "<1.6.4", // v1.6.4+ uses API version unsupported by Azurite in CI
-    },
-    {
-      "matchPackageNames": ["github.com/microsoft/go-mssqldb"],
-      "allowedVersions": "<=1.9.7", // downgraded with Azure SDK
-    },
     {
       "matchPackageNames": ["github.com/yuin/goldmark"],
       "allowedVersions": "<2.0.0", // v2 is a rewrite, the markdown renderer and extensions need a migration