feat: execute post run cleanup when workflow is cancelled (#37275)

## Fixes #36983

## Summary
1. Add transitional `Cancelling` status (between `Running` and
`Cancelled`); cancel flow marks active tasks `Cancelling`, runner
finalizes to `Cancelled` on terminal result.
2. Taskless jobs cancel directly (no runner to finalize).
3. Runner-protocol responses map `Cancelling` → `RESULT_CANCELLED`.
4. Run/job aggregation treats `Cancelling` as active.
5. Status mapping/aggregation tests + en-US locale added.

**Problem**
When a workflow was cancelled from the UI, jobs were marked cancelled
immediately, which could skip post-run cleanup behavior.

## Solution
Use a transitional status path:
Running → Cancelling → Cancelled
This allows runner finalization and cleanup path execution before final
terminal state.

**Testing**

> 1. go test -tags "sqlite sqlite_unlock_notify" ./models/actions -run
"TestAggregateJobStatus|TestStatusAsResult|TestStatusFromResult"
> 2. go run
github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.11.4 run
./models/actions/... ./routers/api/actions/runner/...

## Related
- act_runner: https://gitea.com/gitea/act_runner/pulls/825 —
independent; this PR's capability gate keeps legacy runners on the
immediate-cancel path. The new flow activates only for runners that
advertise the `cancelling` capability.

Co-authored-by: Nicolas <bircni@icloud.com>
Co-authored-by: silverwind <me@silverwind.io>
Co-authored-by: Claude (Opus 4.7) <noreply@anthropic.com>
Co-authored-by: Zettat123 <zettat123@gmail.com>
Co-authored-by: Giteabot <teabot@gitea.io>
This commit is contained in:
Kalash Thakare ☯︎
2026-05-17 08:41:39 +02:00
committed by GitHub
co-authored by Nicolas silverwind Claude Zettat123 Giteabot
parent ae9b34897f
commit e7af84df72
31 changed files with 786 additions and 111 deletions
+30 -42
View File
@@ -19,20 +19,30 @@ import (
webhook_module "code.gitea.io/gitea/modules/webhook"
)
// StopZombieTasks stops the task which have running status, but haven't been updated for a long time
// StopZombieTasks stops tasks in running/cancelling status that haven't been updated for a long time
func StopZombieTasks(ctx context.Context) error {
return stopTasks(ctx, actions_model.FindTaskOptions{
Status: actions_model.StatusRunning,
return stopTasksByStatuses(ctx, actions_model.FindTaskOptions{
UpdatedBefore: timeutil.TimeStamp(time.Now().Add(-setting.Actions.ZombieTaskTimeout).Unix()),
})
}, actions_model.StatusRunning, actions_model.StatusCancelling)
}
// StopEndlessTasks stops the tasks which have running status and continuous updates, but don't end for a long time
// StopEndlessTasks stops tasks in running/cancelling status with continuous updates that don't end for a long time
func StopEndlessTasks(ctx context.Context) error {
return stopTasks(ctx, actions_model.FindTaskOptions{
Status: actions_model.StatusRunning,
return stopTasksByStatuses(ctx, actions_model.FindTaskOptions{
StartedBefore: timeutil.TimeStamp(time.Now().Add(-setting.Actions.EndlessTaskTimeout).Unix()),
})
}, actions_model.StatusRunning, actions_model.StatusCancelling)
}
func stopTasksByStatuses(ctx context.Context, opts actions_model.FindTaskOptions, statuses ...actions_model.Status) error {
for _, status := range statuses {
optsByStatus := opts
optsByStatus.Status = status
if err := stopTasks(ctx, optsByStatus); err != nil {
return err
}
}
return nil
}
func CancelPreviousJobs(ctx context.Context, repoID int64, ref, workflowID string, event webhook_module.HookEventType) error {
@@ -59,7 +69,7 @@ func shouldBlockJobByConcurrency(ctx context.Context, job *actions_model.ActionR
return false, nil
}
attempts, jobs, err := actions_model.GetConcurrentRunAttemptsAndJobs(ctx, job.RepoID, job.ConcurrencyGroup, []actions_model.Status{actions_model.StatusRunning})
attempts, jobs, err := actions_model.GetConcurrentRunAttemptsAndJobs(ctx, job.RepoID, job.ConcurrencyGroup, []actions_model.Status{actions_model.StatusRunning, actions_model.StatusCancelling})
if err != nil {
return false, fmt.Errorf("GetConcurrentRunAttemptsAndJobs: %w", err)
}
@@ -89,7 +99,7 @@ func shouldBlockRunByConcurrency(ctx context.Context, attempt *actions_model.Act
return false, nil
}
attempts, jobs, err := actions_model.GetConcurrentRunAttemptsAndJobs(ctx, attempt.RepoID, attempt.ConcurrencyGroup, []actions_model.Status{actions_model.StatusRunning})
attempts, jobs, err := actions_model.GetConcurrentRunAttemptsAndJobs(ctx, attempt.RepoID, attempt.ConcurrencyGroup, []actions_model.Status{actions_model.StatusRunning, actions_model.StatusCancelling})
if err != nil {
return false, fmt.Errorf("find concurrent runs and jobs: %w", err)
}
@@ -123,7 +133,11 @@ func stopTasks(ctx context.Context, opts actions_model.FindTaskOptions) error {
jobs := make([]*actions_model.ActionRunJob, 0, len(tasks))
for _, task := range tasks {
if err := db.WithTx(ctx, func(ctx context.Context) error {
if err := actions_model.StopTask(ctx, task.ID, actions_model.StatusFailure); err != nil {
stopStatus := actions_model.StatusFailure
if task.Status == actions_model.StatusCancelling {
stopStatus = actions_model.StatusCancelled
}
if err := actions_model.StopTask(ctx, task.ID, stopStatus); err != nil {
return err
}
if err := task.LoadJob(ctx); err != nil {
@@ -157,44 +171,18 @@ func stopTasks(ctx context.Context, opts actions_model.FindTaskOptions) error {
// CancelAbandonedJobs cancels jobs that have not been picked by any runner for a long time
func CancelAbandonedJobs(ctx context.Context) error {
jobs, err := db.Find[actions_model.ActionRunJob](ctx, actions_model.FindRunJobOptions{
abandonedJobs, err := db.Find[actions_model.ActionRunJob](ctx, actions_model.FindRunJobOptions{
Statuses: []actions_model.Status{actions_model.StatusWaiting, actions_model.StatusBlocked},
UpdatedBefore: timeutil.TimeStampNow().AddDuration(-setting.Actions.AbandonedJobTimeout),
})
if err != nil {
log.Warn("find abandoned tasks: %v", err)
log.Warn("find abandoned jobs: %v", err)
return err
}
now := timeutil.TimeStampNow()
updatedJobs := []*actions_model.ActionRunJob{}
for _, job := range jobs {
job.Status = actions_model.StatusCancelled
job.Stopped = now
updated := false
if err := db.WithTx(ctx, func(ctx context.Context) error {
n, err := actions_model.UpdateRunJob(ctx, job, nil, "status", "stopped")
if err != nil {
return err
}
if err := job.LoadAttributes(ctx); err != nil {
return err
}
updated = n > 0
return nil
}); err != nil {
log.Warn("cancel abandoned job %v: %v", job.ID, err)
// go on
}
if job.Run == nil || job.Run.Repo == nil {
continue // error occurs during loading attributes, the following code that depends on "Run.Repo" will fail, so ignore and skip
}
if updated {
CreateCommitStatusForRunJobs(ctx, job.Run, job)
updatedJobs = append(updatedJobs, job)
}
updatedJobs, err := actions_model.CancelJobs(ctx, abandonedJobs)
if err != nil {
log.Warn("cancel abandoned jobs: %v", err)
}
NotifyWorkflowJobsAndRunsStatusUpdate(ctx, updatedJobs)