fix(lfs): failed upload deletes a concurrent upload's meta object (#38693)

UploadHandler creates the LFS meta object only as the last step of
uploadOrVerify, after the content is already in the store, so a request
that errors has never created a row of its own. The removal on the error
path was a real compensating action when it was added in #14726, where
the meta object was created before contentStore.Put, but #16865 moved
creation after the Put and left the removal behind.

Since then it can only ever delete a row created by a different request:
a stalled git-lfs PUT that fails after its own retry has already
succeeded wipes the winner's meta object, leaving the content in the
store unreachable and eligible for orphan cleanup.

Drop the removal and add a regression test asserting that a failing
upload keeps a pre-existing meta object.

Fixes: #38424
Assisted-by: Claude Code:claude-opus-5
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>

services/lfs/server.go

@@ -11,6 +11,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/fs"
 	"net/http"
 	"net/url"
 	"regexp"
@@ -319,28 +320,25 @@ func UploadHandler(ctx *context.Context) {
 		return
 	}
 
-	contentStore := lfs_module.NewContentStore()
-	exists, err := contentStore.Exists(p)
-	if err != nil {
-		log.Error("Unable to check if LFS OID[%s] exist. Error: %v", p.Oid, err)
-		writeStatus(ctx, http.StatusInternalServerError)
-		return
-	}
-
 	uploadOrVerify := func() error {
-		if exists {
+		contentStore := lfs_module.NewContentStore()
+		stat, err := contentStore.Stat(p)
+		if stat != nil {
 			// The bytes already exist in the content store. Only skip proof of
 			// possession when the object is already linked to *this* repo; never
 			// trust cross-repo access (ctx.Doer is the repo owner for deploy keys),
 			// which would let a caller link an object it cannot produce.
 			meta, err := git_model.GetLFSMetaObjectByOid(ctx, repository.ID, p.Oid)
-			if err != nil && err != git_model.ErrLFSObjectNotExist {
+			if err != nil && !errors.Is(err, util.ErrNotExist) {
 				log.Error("Unable to get LFS MetaObject [%s]. Error: %v", p.Oid, err)
 				return err
 			}
 			if meta == nil {
-				// The file exists but is not linked to this repo.
+				// The file exists but is not linked to this repo, or the file is being uploaded.
 				// The upload gets verified by hashing and size comparison to prove access to it.
+				// Keep in mind: here the file might be incomplete due to concurrent uploading, so the verification might fail.
+				// ATTENTION: it's impossible to handle corrupted file on server-side at the moment,
+				// we don't know whether a file is really corrupted, or it is being uploaded.
 				hash := sha256.New()
 				written, err := io.Copy(hash, ctx.Req.Body)
 				if err != nil {
@@ -355,11 +353,17 @@ func UploadHandler(ctx *context.Context) {
 					return lfs_module.ErrHashMismatch
 				}
 			}
-		} else if err := contentStore.Put(p, ctx.Req.Body); err != nil {
-			log.Error("Error putting LFS MetaObject [%s] into content store. Error: %v", p.Oid, err)
+		} else if errors.Is(err, fs.ErrNotExist) {
+			// not exist, store it into the store
+			if err := contentStore.Put(p, ctx.Req.Body); err != nil {
+				log.Error("Error putting LFS MetaObject [%s] into content store. Error: %v", p.Oid, err)
+				return err
+			}
+		} else {
+			log.Error("Unable to check if LFS OID[%s] stat. Error: %v", p.Oid, err)
 			return err
 		}
-		_, err := git_model.NewLFSMetaObject(ctx, repository.ID, p)
+		_, err = git_model.NewLFSMetaObject(ctx, repository.ID, p)
 		return err
 	}
 
@@ -372,9 +376,12 @@ func UploadHandler(ctx *context.Context) {
 			log.Error("Error whilst uploadOrVerify LFS OID[%s]: %v", p.Oid, err)
 			writeStatus(ctx, http.StatusInternalServerError)
 		}
-		if _, err = git_model.RemoveLFSMetaObjectByOid(ctx, repository.ID, p.Oid); err != nil {
-			log.Error("Error whilst removing MetaObject for LFS OID[%s]: %v", p.Oid, err)
-		}
+		// Do not remove the LFS MetaObject here: this request only creates it after the content is verified and stored,
+		// an invalid request should not remove the existing correct record.
+		// If two requests are loading (the file is incomplete):
+		// * one will keep writing the file content
+		// * one will fail the verification because it reads an incomplete file, the failure should be just ignore
+		// In the end, the first one will complete the upload and insert a LFS MetaObject record.
 		return
 	}
 

services/repository/lfs.go

@@ -101,7 +101,7 @@ func GarbageCollectLFSMetaObjectsForRepo(ctx context.Context, repo *repo_model.R
 				return nil
 			}
 
-			if err := store.Delete(metaObject.RelativePath()); err != nil {
+			if err := store.ObjectStorage.Delete(metaObject.RelativePath()); err != nil {
 				log.Error("Unable to remove lfs metaobject %s from store: %v", metaObject.Oid, err)
 			}
 			deleted++