fix(process): reap entire process group on cmd.Cancel (#39143)

Signed-off-by: Royce Remer <royceremer@gmail.com>
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>

modules/process/command.go

@@ -0,0 +1,54 @@
+// Copyright 2026 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package process
+
+import (
+	"bytes"
+	"context"
+	"os/exec"
+)
+
+type Cmd struct {
+	*exec.Cmd
+
+	onCancelUserFunc func() error
+	termGraceful     bool
+}
+
+func (c *Cmd) WithOnCancelGracefully(userFunc func() error) *Cmd {
+	c.termGraceful, c.onCancelUserFunc = true, userFunc
+	return c
+}
+
+func (c *Cmd) WithOnCancelForceKill(userFunc func() error) *Cmd {
+	c.termGraceful, c.onCancelUserFunc = false, userFunc
+	return c
+}
+
+func (c *Cmd) WithDir(dir string) *Cmd {
+	c.Cmd.Dir = dir
+	return c
+}
+
+func (c *Cmd) OutputString() (string, string, error) {
+	stdout, stderr := &bytes.Buffer{}, &bytes.Buffer{}
+	c.Cmd.Stdout = stdout
+	c.Cmd.Stderr = stderr
+	err := c.Cmd.Run()
+	return stdout.String(), stderr.String(), err
+}
+
+// CommandContext returns a wrapped exec.Cmd which kills the process group when the context is canceled.
+// By default, it uses graceful termination (SIGTERM) on Unix-like systems to make the subprocesses have chances
+// to clean up (e.g. remove temporary files or lock files).
+func CommandContext(ctx context.Context, name string, arg ...string) *Cmd {
+	c := &Cmd{Cmd: exec.CommandContext(ctx, name, arg...)} //nolint:forbidigo // wrap it
+	setSysProcAttribute(c.Cmd)
+	c.Cmd.Cancel = c.onCancel
+
+	// Unlike exec.CommandContext, we use graceful termination by default to avoid corrupting data or leaving lock files behind.
+	// If some processes don't respond to SIGTERM, can switch to WithOnCancelForceKill (SIGKILL) to force kill them.
+	c.termGraceful = true
+	return c
+}

modules/process/command_unix.go

@@ -0,0 +1,32 @@
+// Copyright 2026 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+//go:build unix
+
+package process
+
+import (
+	"os/exec"
+	"syscall"
+
+	"gitea.dev/modules/util"
+)
+
+func setSysProcAttribute(cmd *exec.Cmd) {
+	// When Gitea runs SubProcessA -> SubProcessB and SubProcessA gets killed by context cancel,
+	// use process group to make sure the sub processes can be killed and reaped instead of leaving defunct(zombie) processes.
+	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
+}
+
+func (c *Cmd) onCancel() error {
+	if c.onCancelUserFunc != nil {
+		if err := c.onCancelUserFunc(); err != nil {
+			return err
+		}
+	}
+	sig := util.Iif(c.termGraceful, syscall.SIGTERM, syscall.SIGKILL)
+	// kill the whole process group
+	// ATTENTION: do not access PID after Wait or in other goroutine, it will just cause PID reuse data-race.
+	// There is no easy solution to implement "first SIGTERM then SIGKILL" in a safe way, only one signal can be sent to the process group.
+	return syscall.Kill(-c.Process.Pid, sig)
+}

modules/process/command_unix_test.go

@@ -0,0 +1,67 @@
+// Copyright 2026 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+//go:build unix
+
+package process
+
+import (
+	"bufio"
+	"context"
+	"os"
+	"strconv"
+	"strings"
+	"syscall"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCommandContextCancelKillProcessGroup(t *testing.T) {
+	ctx, cancel := context.WithCancel(t.Context())
+	defer cancel()
+
+	// When executing external commands, there can be multiple subprocesses involved.
+	// e.g.: Gitea ->  "git upload-pack" -> "git pack-objects".
+	// If the context is canceled (HTTP client disconnects), all subprocesses must terminate.
+
+	// Spawn a shell that itself spawns a long-lived background process and
+	// prints its PID — mimicking git upload-pack spawning git pack-objects.
+
+	r, w, err := os.Pipe()
+	require.NoError(t, err)
+
+	cmd := CommandContext(ctx, "sh", "-c", "sleep 600 & echo $!; wait")
+	cmd.Stdout = w
+	require.NoError(t, cmd.Start())
+	_ = w.Close() // parent keeps only the read end
+
+	t.Cleanup(func() {
+		// make sure our test doesn't leave a zombie process even if test fails
+		_ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
+		_ = cmd.Wait()
+	})
+
+	// Block until the shell prints the grandchild PID.
+	scanner := bufio.NewScanner(r)
+	require.True(t, scanner.Scan(), "expected grandchild PID on stdout")
+	grandchildPID, err := strconv.Atoi(strings.TrimSpace(scanner.Text()))
+	require.NoError(t, err)
+	_ = r.Close()
+
+	// Sanity: grandchild must be alive before we cancel.
+	grandchild, err := os.FindProcess(grandchildPID)
+	require.NoError(t, err)
+	require.NoError(t, grandchild.Signal(syscall.Signal(0)), "grandchild should be alive before cancel")
+
+	// Cancel the context
+	cancel()
+	_ = cmd.Wait()
+
+	// Subprocess should not exist after context cancel (killed by process group)
+	assert.Eventually(t, func() bool {
+		return grandchild.Signal(syscall.Signal(0)) != nil
+	}, 5*time.Second, 10*time.Millisecond)
+}

modules/process/command_windows.go

@@ -0,0 +1,19 @@
+// Copyright 2026 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package process
+
+import "os/exec"
+
+// There is no graceful way to kill a process on Windows at the moment
+
+func setSysProcAttribute(cmd *exec.Cmd) {}
+
+func (c *Cmd) onCancel() error {
+	if c.onCancelUserFunc != nil {
+		if err := c.onCancelUserFunc(); err != nil {
+			return err
+		}
+	}
+	return c.Process.Kill()
+}

modules/process/error.go

@@ -1,25 +0,0 @@
-// Copyright 2022 The Gitea Authors. All rights reserved.
-// SPDX-License-Identifier: MIT
-
-package process
-
-import "fmt"
-
-// Error is a wrapped error describing the error results of Process Execution
-type Error struct {
-	PID         IDType
-	Description string
-	Err         error
-	CtxErr      error
-	Stdout      string
-	Stderr      string
-}
-
-func (err *Error) Error() string {
-	return fmt.Sprintf("exec(%s:%s) failed: %v(%v) stdout: %s stderr: %s", err.PID, err.Description, err.Err, err.CtxErr, err.Stdout, err.Stderr)
-}
-
-// Unwrap implements the unwrappable implicit interface for go1.13 Unwrap()
-func (err *Error) Unwrap() error {
-	return err.Err
-}

modules/process/manager.go

@@ -23,9 +23,6 @@ import (
 var (
 	manager     *Manager
 	managerInit sync.Once
-
-	// DefaultContext is the default context to run processing commands in
-	DefaultContext = context.Background()
 )
 
 type (

modules/process/manager_exec.go

@@ -1,79 +0,0 @@
-// Copyright 2022 The Gitea Authors. All rights reserved.
-// SPDX-License-Identifier: MIT
-
-package process
-
-import (
-	"bytes"
-	"context"
-	"io"
-	"os/exec"
-	"time"
-)
-
-// Exec a command and use the default timeout.
-func (pm *Manager) Exec(desc, cmdName string, args ...string) (string, string, error) {
-	return pm.ExecDir(DefaultContext, -1, "", desc, cmdName, args...)
-}
-
-// ExecTimeout a command and use a specific timeout duration.
-func (pm *Manager) ExecTimeout(timeout time.Duration, desc, cmdName string, args ...string) (string, string, error) {
-	return pm.ExecDir(DefaultContext, timeout, "", desc, cmdName, args...)
-}
-
-// ExecDir a command and use the default timeout.
-func (pm *Manager) ExecDir(ctx context.Context, timeout time.Duration, dir, desc, cmdName string, args ...string) (string, string, error) {
-	return pm.ExecDirEnv(ctx, timeout, dir, desc, nil, cmdName, args...)
-}
-
-// ExecDirEnv runs a command in given path and environment variables, and waits for its completion
-// up to the given timeout (or DefaultTimeout if -1 is given).
-// Returns its complete stdout and stderr
-// outputs and an error, if any (including timeout)
-func (pm *Manager) ExecDirEnv(ctx context.Context, timeout time.Duration, dir, desc string, env []string, cmdName string, args ...string) (string, string, error) {
-	return pm.ExecDirEnvStdIn(ctx, timeout, dir, desc, env, nil, cmdName, args...)
-}
-
-// ExecDirEnvStdIn runs a command in given path and environment variables with provided stdIN, and waits for its completion
-// up to the given timeout (or DefaultTimeout if timeout <= 0 is given).
-// Returns its complete stdout and stderr
-// outputs and an error, if any (including timeout)
-func (pm *Manager) ExecDirEnvStdIn(ctx context.Context, timeout time.Duration, dir, desc string, env []string, stdIn io.Reader, cmdName string, args ...string) (string, string, error) {
-	if timeout <= 0 {
-		timeout = 60 * time.Second
-	}
-
-	stdOut := new(bytes.Buffer)
-	stdErr := new(bytes.Buffer)
-
-	ctx, _, finished := pm.AddContextTimeout(ctx, timeout, desc)
-	defer finished()
-
-	cmd := exec.CommandContext(ctx, cmdName, args...)
-	cmd.Dir = dir
-	cmd.Env = env
-	cmd.Stdout = stdOut
-	cmd.Stderr = stdErr
-	if stdIn != nil {
-		cmd.Stdin = stdIn
-	}
-	SetSysProcAttribute(cmd)
-
-	if err := cmd.Start(); err != nil {
-		return "", "", err
-	}
-
-	err := cmd.Wait()
-	if err != nil {
-		err = &Error{
-			PID:         GetPID(ctx),
-			Description: desc,
-			Err:         err,
-			CtxErr:      ctx.Err(),
-			Stdout:      stdOut.String(),
-			Stderr:      stdErr.String(),
-		}
-	}
-
-	return stdOut.String(), stdErr.String(), err
-}

modules/process/manager_test.go

@@ -6,7 +6,6 @@ package process
 import (
 	"context"
 	"testing"
-	"time"
 
 	"github.com/stretchr/testify/assert"
 )
@@ -87,25 +86,3 @@ func TestManager_Remove(t *testing.T) {
 	_, exists := pm.processMap[GetPID(p2Ctx)]
 	assert.False(t, exists, "PID %d is in the list but shouldn't", GetPID(p2Ctx))
 }
-
-func TestExecTimeoutNever(t *testing.T) {
-	// TODO Investigate how to improve the time elapsed per round.
-	maxLoops := 10
-	for i := 1; i < maxLoops; i++ {
-		_, stderr, err := GetManager().ExecTimeout(5*time.Second, "ExecTimeout", "git", "--version")
-		if err != nil {
-			t.Fatalf("git --version: %v(%s)", err, stderr)
-		}
-	}
-}
-
-func TestExecTimeoutAlways(t *testing.T) {
-	maxLoops := 100
-	for i := 1; i < maxLoops; i++ {
-		_, stderr, err := GetManager().ExecTimeout(100*time.Microsecond, "ExecTimeout", "sleep", "5")
-		// TODO Simplify logging and errors to get precise error type. E.g. checking "if err != context.DeadlineExceeded".
-		if err == nil {
-			t.Fatalf("sleep 5 secs: %v(%s)", err, stderr)
-		}
-	}
-}

modules/process/manager_unix.go

@@ -1,17 +0,0 @@
-// Copyright 2022 The Gitea Authors. All rights reserved.
-// SPDX-License-Identifier: MIT
-
-//go:build !windows
-
-package process
-
-import (
-	"os/exec"
-	"syscall"
-)
-
-// SetSysProcAttribute sets the common SysProcAttrs for commands
-func SetSysProcAttribute(cmd *exec.Cmd) {
-	// When Gitea runs SubProcessA -> SubProcessB and SubProcessA gets killed by context timeout, use setpgid to make sure the sub processes can be reaped instead of leaving defunct(zombie) processes.
-	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
-}

modules/process/manager_windows.go

@@ -1,15 +0,0 @@
-// Copyright 2022 The Gitea Authors. All rights reserved.
-// SPDX-License-Identifier: MIT
-
-//go:build windows
-
-package process
-
-import (
-	"os/exec"
-)
-
-// SetSysProcAttribute sets the common SysProcAttrs for commands
-func SetSysProcAttribute(cmd *exec.Cmd) {
-	// Do nothing
-}