fix(actions): ensure all waiting jobs get runners in large workflows (#38200)

## Summary

Fixes two related bugs that cause jobs in large workflows (50+ parallel
jobs) to never get a runner assigned even though runners are free.

### Bug 1 — Concurrent runner race

When N runners all poll `FetchTask` with a stale `tasksVersion`
simultaneously, they all query the same waiting job list sorted by
`(updated, id)` and all pick **job #1**. Only one wins the `UPDATE WHERE
task_id=0` optimistic lock; the rest return empty-handed but still
receive `latestVersion` in the response. They then consider themselves
"up to date" and skip `PickTask` on every subsequent poll, leaving jobs
#2–50 permanently unassigned.

**Fix:** `CreateTaskForRunner` now iterates through all matching waiting
jobs. When the optimistic lock fails on job #1, it immediately tries job
#2, then #3, etc., each in its own independent transaction so a failed
attempt rolls back cleanly before the next candidate is tried.
`PickTask` no longer wraps this call in an outer `db.WithTx` (which
caused `halfCommitter` entanglement that prevented per-attempt
rollbacks).

### Bug 2 — Idle runner doesn't re-check after finishing a task

`tasks_version` only bumps when a job transitions **to** waiting (new
workflow triggered, blocked→unblocked). After a runner finishes its
current task it polls `FetchTask` with `tasksVersion == latestVersion`,
so the server skips `PickTask` entirely — the remaining 45 waiting jobs
are invisible to the now-idle runner.

**Fix:** Also call `IncreaseTaskVersion` in `UpdateRunJob` when a
(non-reusable-caller) job transitions to a **done** state. Idle runners
then see a version mismatch on their next poll and attempt `PickTask`,
picking up the remaining jobs.
This commit is contained in:
bircni
2026-06-27 17:56:12 +00:00
committed by GitHub
parent cbe1b703dc
commit 0f5102427e
5 changed files with 414 additions and 128 deletions
+65
View File
@@ -306,3 +306,68 @@ func TestStopTaskCancellingFallsBackForMissingRunner(t *testing.T) {
assert.Equal(t, StatusCancelled, jobAfterStop.Status)
assert.NotZero(t, jobAfterStop.Stopped)
}
// TestReleaseTaskForRunner verifies that releasing a freshly-claimed task returns
// its job to the waiting queue and deletes the task and its steps, so a failure
// while assembling the runner response cannot strand the job in running state.
func TestReleaseTaskForRunner(t *testing.T) {
require.NoError(t, unittest.PrepareTestDatabase())
run := &ActionRun{
Title: "release-task-test-run",
RepoID: 1,
OwnerID: 2,
WorkflowID: "test.yaml",
Index: 9902,
TriggerUserID: 2,
Ref: "refs/heads/main",
CommitSHA: "c2d72f548424103f01ee1dc02889c1e2bff816b0",
Event: "push",
TriggerEvent: "push",
Status: StatusWaiting,
}
require.NoError(t, db.Insert(t.Context(), run))
job := &ActionRunJob{
RunID: run.ID,
RepoID: run.RepoID,
OwnerID: run.OwnerID,
CommitSHA: run.CommitSHA,
Name: "release-job",
Attempt: 1,
JobID: "release-job",
Status: StatusWaiting,
RunsOn: []string{"ubuntu-latest"},
WorkflowPayload: []byte("on: push\njobs:\n release-job:\n runs-on: ubuntu-latest\n steps:\n - run: echo hi\n"),
}
require.NoError(t, db.Insert(t.Context(), job))
runner := &ActionRunner{
UUID: "release-runner-uuid",
Name: "release-runner",
AgentLabels: []string{"ubuntu-latest"},
}
runner.GenerateAndFillToken()
require.NoError(t, db.Insert(t.Context(), runner))
task, ok, err := CreateTaskForRunner(t.Context(), runner)
require.NoError(t, err)
require.True(t, ok)
require.NotNil(t, task)
claimed := unittest.AssertExistsAndLoadBean(t, &ActionRunJob{ID: job.ID})
require.Equal(t, StatusRunning, claimed.Status)
require.Equal(t, task.ID, claimed.TaskID)
require.NoError(t, ReleaseTaskForRunner(t.Context(), task))
// Job is back in the waiting queue with no task assigned.
released := unittest.AssertExistsAndLoadBean(t, &ActionRunJob{ID: job.ID})
assert.Equal(t, StatusWaiting, released.Status)
assert.Zero(t, released.TaskID)
assert.Zero(t, released.Started)
// The task and its steps are gone.
unittest.AssertNotExistsBean(t, &ActionTask{ID: task.ID})
unittest.AssertNotExistsBean(t, &ActionTaskStep{TaskID: task.ID})
}