fix: restore missing blob file when re-publishing a package (#39239)

Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>

services/packages/packages.go

@@ -264,6 +264,42 @@ func NewPackageBlob(hsr packages_module.HashedSizeReader) *packages_model.Packag
 	}
 }
 
+func GetOrSavePackageBlob(ctx context.Context, contentStore *packages_module.ContentStore, blob *packages_model.PackageBlob, data packages_module.HashedSizeReader) (_ *packages_model.PackageBlob, _ bool, retErr error) {
+	if blob.Size != data.Size() {
+		return nil, false, fmt.Errorf("size mismatch: blob size %d, data size %d", blob.Size, data.Size())
+	}
+	pb, existsInDatabase, err := packages_model.GetOrInsertBlob(ctx, blob)
+	if err != nil {
+		return nil, false, fmt.Errorf("unable to get or insert blob: %w", err)
+	}
+
+	defer func() {
+		if retErr != nil && !existsInDatabase {
+			if errDelete := packages_model.DeleteBlobByID(ctx, pb.ID); errDelete != nil {
+				log.Error("unable to delete blob from database after failed save in content store: %v", errDelete)
+			}
+		}
+	}()
+
+	var objSize optional.Option[int64]
+	storeKey := packages_module.BlobHash256Key(pb.HashSHA256)
+	if existsInDatabase {
+		// check if the blob file actually is valid in the content store
+		objSize, err = contentStore.OptionalSize(storeKey)
+		if err != nil {
+			return nil, false, fmt.Errorf("unable to check object size in content store: %w", err)
+		}
+	}
+	if objSize.ValueOrDefault(-1) != blob.Size {
+		if err := contentStore.Save(storeKey, data, data.Size()); err != nil {
+			return nil, false, fmt.Errorf("unable to save object in content store: %w", err)
+		}
+	}
+	// existsInDatabase controls the "roll back", if other errors happen later,
+	// the "non-existing (newly created)" blob will be deleted from the content store, but not if it already existed in the database.
+	return pb, existsInDatabase, nil
+}
+
 func addFileToPackageVersion(ctx context.Context, pv *packages_model.PackageVersion, pvi *PackageInfo, pfci *PackageFileCreationInfo) (*packages_model.PackageFile, *packages_model.PackageBlob, bool, error) {
 	if err := CheckSizeQuotaExceeded(ctx, pfci.Creator, pvi.Owner, pvi.PackageType, pfci.Data.Size()); err != nil {
 		return nil, nil, false, err
@@ -272,21 +308,13 @@ func addFileToPackageVersion(ctx context.Context, pv *packages_model.PackageVers
 	return addFileToPackageVersionUnchecked(ctx, pv, pfci)
 }
 
-func addFileToPackageVersionUnchecked(ctx context.Context, pv *packages_model.PackageVersion, pfci *PackageFileCreationInfo) (*packages_model.PackageFile, *packages_model.PackageBlob, bool, error) {
+func addFileToPackageVersionUnchecked(ctx context.Context, pv *packages_model.PackageVersion, pfci *PackageFileCreationInfo) (_ *packages_model.PackageFile, _ *packages_model.PackageBlob, created bool, _ error) {
 	log.Trace("Adding package file: %v, %s", pv.ID, pfci.Filename)
 
-	pb, exists, err := packages_model.GetOrInsertBlob(ctx, NewPackageBlob(pfci.Data))
+	pb, exists, err := GetOrSavePackageBlob(ctx, packages_module.NewContentStore(), NewPackageBlob(pfci.Data), pfci.Data)
 	if err != nil {
-		log.Error("Error inserting package blob: %v", err)
 		return nil, nil, false, err
 	}
-	if !exists {
-		contentStore := packages_module.NewContentStore()
-		if err := contentStore.Save(packages_module.BlobHash256Key(pb.HashSHA256), pfci.Data, pfci.Data.Size()); err != nil {
-			log.Error("Error saving package blob in content store: %v", err)
-			return nil, nil, false, err
-		}
-	}
 
 	if pfci.OverwriteExisting {
 		pf, err := packages_model.GetFileForVersionByName(ctx, pv.ID, pfci.Filename, pfci.CompositeKey)

services/packages/packages_test.go

@@ -4,6 +4,11 @@
 package packages
 
 import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"io"
+	"net/http"
 	"testing"
 
 	"gitea.dev/models/db"
@@ -13,6 +18,8 @@ import (
 	unit_model "gitea.dev/models/unit"
 	"gitea.dev/models/unittest"
 	user_model "gitea.dev/models/user"
+	packages_module "gitea.dev/modules/packages"
+	"gitea.dev/modules/test"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -22,6 +29,72 @@ func TestMain(m *testing.M) {
 	unittest.MainTest(m)
 }
 
+func TestCreatePackageAndAddFileRestoresMissingBlobFile(t *testing.T) {
+	assert.NoError(t, unittest.PrepareTestDatabase())
+	user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2})
+
+	uploadPackage := func(t *testing.T, user *user_model.User, name, filename string, data []byte) (*packages_model.PackageFile, error) {
+		buf, err := packages_module.CreateHashedBufferFromReader(bytes.NewReader(data))
+		require.NoError(t, err)
+		_, pf, err := CreatePackageAndAddFile(t.Context(),
+			&PackageCreationInfo{
+				Owner:            user,
+				PackageType:      packages_model.TypeNuGet,
+				Name:             name,
+				Version:          "1.0.0",
+				SemverCompatible: true,
+				Creator:          user,
+			},
+			&PackageFileCreationInfo{
+				Filename: filename,
+				Creator:  user,
+				Data:     buf,
+				IsLead:   true,
+			})
+		return pf, err
+	}
+
+	// This test data is from https://github.com/go-gitea/gitea/issues/39215, it doesn't really matter, actually.
+	// The key point is that if the blob object is missing in the content storage, it must be restored when uploaded again.
+	pkgData := test.WriteZipArchive(map[string]string{
+		"package.nuspec":         "<package><metadata><id>nuget.repro</id><version>1.0.0</version></metadata></package>",
+		"lib/netstandard2.0/_._": "",
+	}).Bytes()
+	pkgDataSum := sha256.Sum256(pkgData)
+	key := packages_module.BlobHash256Key(hex.EncodeToString(pkgDataSum[:]))
+	contentStore := packages_module.NewContentStore()
+
+	// The initial upload writes the blob row and its file
+	pf1, err := uploadPackage(t, user, "nuget.repro", "nuget.repro.1.0.0.nupkg", pkgData)
+	require.NoError(t, err)
+	sz, err := contentStore.OptionalSize(key)
+	assert.NoError(t, err)
+	assert.EqualValues(t, len(pkgData), sz.ValueOrDefault(-1))
+
+	// Simulate the storage inconsistency: the blob row survives but its file is missing
+	require.NoError(t, contentStore.Delete(key))
+	sz, err = contentStore.OptionalSize(key)
+	assert.NoError(t, err)
+	assert.EqualValues(t, -1, sz.ValueOrDefault(-1))
+
+	// Publishing a package with identical content must restore the blob file
+	pf2, err := uploadPackage(t, user, "nuget.repro-copy", "nuget.repro-copy.1.0.0.nupkg", pkgData)
+	require.NoError(t, err)
+	sz, err = contentStore.OptionalSize(key)
+	assert.NoError(t, err)
+	assert.EqualValues(t, len(pkgData), sz.ValueOrDefault(-1))
+
+	// The blob file must be present and both packages must be downloadable
+	for _, pf := range []*packages_model.PackageFile{pf1, pf2} {
+		s, _, _, err := OpenFileForDownload(t.Context(), pf, http.MethodGet)
+		require.NoError(t, err)
+		respData, err := io.ReadAll(s)
+		require.NoError(t, err)
+		assert.NoError(t, s.Close())
+		assert.Equal(t, pkgData, respData)
+	}
+}
+
 func TestUnlinkFromRepositoryRequiresTargetRepoAdmin(t *testing.T) {
 	assert.NoError(t, unittest.PrepareTestDatabase())
 	repo := &repo_model.Repository{OwnerID: 3, OwnerName: "org3", Name: "package-repo", LowerName: "package-repo", IsPrivate: true}