Skip to content

Commit 6d3e03b

Browse files
DEVPROD-46935 Ignore stale unprocessed intents in merge queue recovery (#10993)
1 parent b517086 commit 6d3e03b

8 files changed

Lines changed: 224 additions & 20 deletions

File tree

‎model/patch/github.go‎

Lines changed: 20 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -344,19 +344,27 @@ func HasNewerGithubIntent(ctx context.Context, baseRepoName string, prNumber int
344344
return count > 0, nil
345345
}
346346

347-
// FindUnprocessedGithubMergeIntentHeadSHAs returns the head SHAs of GitHub merge
348-
// queue intents that have not yet been processed into a patch. The merge queue
349-
// patch recovery job uses these to avoid creating a duplicate patch for a merge
350-
// group whose webhook intent is still in flight.
351-
func FindUnprocessedGithubMergeIntentHeadSHAs(ctx context.Context) ([]string, error) {
352-
var intents []githubMergeIntent
353-
err := db.FindAllQ(ctx, IntentCollection, db.Query(bson.M{processedKey: false, intentTypeKey: GithubMergeIntentType}), &intents)
347+
// UnprocessedMergeIntent is a GitHub merge queue intent that has not yet been
348+
// processed into a patch.
349+
type UnprocessedMergeIntent struct {
350+
ID string `bson:"_id"`
351+
HeadSHA string `bson:"head_hash"`
352+
CreatedAt time.Time `bson:"created_at"`
353+
}
354+
355+
// FindUnprocessedGithubMergeIntents returns GitHub merge queue intents created
356+
// after createdAfter that have not yet been processed into a patch. The merge
357+
// queue patch recovery job uses these to avoid creating a duplicate patch for a
358+
// merge group whose webhook intent is still in flight.
359+
func FindUnprocessedGithubMergeIntents(ctx context.Context, createdAfter time.Time) ([]UnprocessedMergeIntent, error) {
360+
var intents []UnprocessedMergeIntent
361+
err := db.FindAllQ(ctx, IntentCollection, db.Query(bson.M{
362+
processedKey: false,
363+
intentTypeKey: GithubMergeIntentType,
364+
createdAtKey: bson.M{"$gt": createdAfter},
365+
}).WithFields(documentIDKey, headHashKey, createdAtKey), &intents)
354366
if err != nil {
355367
return nil, err
356368
}
357-
headSHAs := make([]string, 0, len(intents))
358-
for _, intent := range intents {
359-
headSHAs = append(headSHAs, intent.HeadSHA)
360-
}
361-
return headSHAs, nil
369+
return intents, nil
362370
}

‎model/patch/github_merge_test.go‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -214,3 +214,69 @@ func TestExtractBaseBranchFromHeadRef(t *testing.T) {
214214
})
215215
}
216216
}
217+
218+
func TestFindUnprocessedGithubMergeIntents(t *testing.T) {
219+
require.NoError(t, db.Clear(IntentCollection))
220+
t.Cleanup(func() {
221+
assert.NoError(t, db.Clear(IntentCollection))
222+
})
223+
224+
recent := time.Now().UTC().Round(time.Millisecond)
225+
old := recent.Add(-2 * time.Hour)
226+
tooOld := recent.Add(-10 * time.Hour)
227+
require.NoError(t, db.Insert(t.Context(), IntentCollection, githubMergeIntent{
228+
DocumentID: "recent",
229+
IntentType: GithubMergeIntentType,
230+
HeadSHA: "recent_sha",
231+
CreatedAt: recent,
232+
}))
233+
require.NoError(t, db.Insert(t.Context(), IntentCollection, githubMergeIntent{
234+
DocumentID: "old",
235+
IntentType: GithubMergeIntentType,
236+
HeadSHA: "old_sha",
237+
CreatedAt: old,
238+
}))
239+
require.NoError(t, db.Insert(t.Context(), IntentCollection, githubMergeIntent{
240+
DocumentID: "too_old",
241+
IntentType: GithubMergeIntentType,
242+
HeadSHA: "too_old_sha",
243+
CreatedAt: tooOld,
244+
}))
245+
require.NoError(t, db.Insert(t.Context(), IntentCollection, githubMergeIntent{
246+
DocumentID: "processed",
247+
IntentType: GithubMergeIntentType,
248+
HeadSHA: "processed_sha",
249+
CreatedAt: recent,
250+
Processed: true,
251+
}))
252+
require.NoError(t, db.Insert(t.Context(), IntentCollection, githubIntent{
253+
DocumentID: "pr",
254+
IntentType: GithubIntentType,
255+
HeadHash: "pr_sha",
256+
CreatedAt: recent,
257+
}))
258+
259+
intents, err := FindUnprocessedGithubMergeIntents(t.Context(), recent.Add(-6*time.Hour))
260+
require.NoError(t, err)
261+
assert.ElementsMatch(t, []UnprocessedMergeIntent{
262+
{ID: "recent", HeadSHA: "recent_sha", CreatedAt: recent},
263+
{ID: "old", HeadSHA: "old_sha", CreatedAt: old},
264+
}, intents)
265+
}
266+
267+
func TestRemoveIntent(t *testing.T) {
268+
require.NoError(t, db.Clear(IntentCollection))
269+
t.Cleanup(func() {
270+
assert.NoError(t, db.Clear(IntentCollection))
271+
})
272+
273+
require.NoError(t, db.Insert(t.Context(), IntentCollection, githubMergeIntent{DocumentID: "remove", IntentType: GithubMergeIntentType}))
274+
require.NoError(t, db.Insert(t.Context(), IntentCollection, githubMergeIntent{DocumentID: "keep", IntentType: GithubMergeIntentType}))
275+
276+
require.NoError(t, RemoveIntent(t.Context(), "remove"))
277+
278+
var intents []githubMergeIntent
279+
require.NoError(t, db.FindAllQ(t.Context(), IntentCollection, db.Query(bson.M{}), &intents))
280+
require.Len(t, intents, 1)
281+
assert.Equal(t, "keep", intents[0].DocumentID)
282+
}

‎model/patch/intent.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,3 +68,8 @@ func FindIntent(ctx context.Context, id, intentType string) (Intent, error) {
6868

6969
return intent, nil
7070
}
71+
72+
// RemoveIntent deletes the intent with the given ID from the database.
73+
func RemoveIntent(ctx context.Context, id string) error {
74+
return db.Remove(ctx, IntentCollection, bson.M{"_id": id})
75+
}

‎rest/data/patch_intent.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,13 @@ func AddGithubMergeIntent(ctx context.Context, intent patch.Intent, projectID st
7373
"message": "GitHub merge group not queued for processing",
7474
"intent_id": intent.ID(),
7575
}))
76+
// Remove the intent so it doesn't stay unprocessed forever and block
77+
// merge queue patch recovery for this merge group.
78+
grip.Error(ctx, message.WrapError(patch.RemoveIntent(ctx, intent.ID()), message.Fields{
79+
"source": "GitHub hook",
80+
"message": "could not remove unqueued GitHub merge group intent",
81+
"intent_id": intent.ID(),
82+
}))
7683

7784
return gimlet.ErrorResponse{
7885
StatusCode: http.StatusInternalServerError,

‎rest/data/patch_intent_test.go‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,14 @@ import (
44
"context"
55
"testing"
66

7+
"github.com/evergreen-ci/evergreen/db"
8+
"github.com/evergreen-ci/evergreen/model/patch"
9+
"github.com/evergreen-ci/utility"
10+
"github.com/google/go-github/v70/github"
11+
"github.com/mongodb/amboy/queue"
712
"github.com/stretchr/testify/assert"
13+
"github.com/stretchr/testify/require"
14+
"go.mongodb.org/mongo-driver/bson"
815
)
916

1017
func TestContextForQueuedJobPreservesValuesAfterRequestCancellation(t *testing.T) {
@@ -18,3 +25,29 @@ func TestContextForQueuedJobPreservesValuesAfterRequestCancellation(t *testing.T
1825
assert.NoError(t, queueCtx.Err())
1926
assert.Equal(t, "value", queueCtx.Value(key))
2027
}
28+
29+
func TestAddGithubMergeIntentRemovesIntentWhenEnqueueFails(t *testing.T) {
30+
require.NoError(t, db.Clear(patch.IntentCollection))
31+
t.Cleanup(func() {
32+
assert.NoError(t, db.Clear(patch.IntentCollection))
33+
})
34+
35+
intent, err := patch.NewGithubMergeIntent(t.Context(), "delivery_id", "auto", &github.MergeGroupEvent{
36+
MergeGroup: &github.MergeGroup{
37+
HeadSHA: utility.ToStringPtr("head_sha"),
38+
HeadRef: utility.ToStringPtr("refs/heads/gh-readonly-queue/main/pr-1-abc"),
39+
BaseSHA: utility.ToStringPtr("base_sha"),
40+
},
41+
Org: &github.Organization{Login: utility.ToStringPtr("org")},
42+
Repo: &github.Repository{Name: utility.ToStringPtr("repo")},
43+
})
44+
require.NoError(t, err)
45+
46+
// The queue is never started, so enqueueing the processor job fails.
47+
q := queue.NewLocalLimitedSize(1, 10)
48+
assert.Error(t, AddGithubMergeIntent(t.Context(), intent, "project", q))
49+
50+
count, err := db.Count(t.Context(), patch.IntentCollection, bson.M{})
51+
require.NoError(t, err)
52+
assert.Zero(t, count)
53+
}

‎units/merge_queue_patch_recovery.go‎

Lines changed: 32 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,16 @@ import (
2323
const (
2424
mergeQueuePatchRecoveryJobName = "merge-queue-patch-recovery"
2525
minimumStuckTime = 5 * time.Minute
26+
// pendingIntentMaxAge is how long an unprocessed merge intent is treated as
27+
// in flight regardless of its processor job. After that, it's only in flight
28+
// if its processor job is still in the queue and not completed; otherwise
29+
// the job was lost (e.g. never enqueued or crashed) and the intent would
30+
// block recovery indefinitely.
31+
pendingIntentMaxAge = 30 * time.Minute
32+
// unprocessedIntentLookback bounds how far back to look for unprocessed
33+
// merge intents. It matches GitHub's maximum merge queue status check
34+
// timeout, after which the merge group is removed from the queue anyway.
35+
unprocessedIntentLookback = 6 * time.Hour
2636
)
2737

2838
func init() {
@@ -71,17 +81,21 @@ func (j *mergeQueuePatchRecoveryJob) Run(ctx context.Context) {
7181
return
7282
}
7383

74-
// A merge group whose webhook intent has been received but not yet processed
75-
// into a patch already has a pending intent. Treat those head SHAs as covered
76-
// so we never create a second patch for the same merge group.
77-
pendingHeadSHAs, err := patch.FindUnprocessedGithubMergeIntentHeadSHAs(ctx)
84+
// A merge group whose intent has been received but not yet processed into
85+
// a patch already has a pending intent. Treat the head SHAs of intents that
86+
// are still in flight as covered so we never create a second patch for the
87+
// same merge group.
88+
now := time.Now()
89+
unprocessedIntents, err := patch.FindUnprocessedGithubMergeIntents(ctx, now.Add(-unprocessedIntentLookback))
7890
if err != nil {
7991
j.AddRetryableError(errors.Wrap(err, "finding unprocessed merge queue intents"))
8092
return
8193
}
82-
pending := make(map[string]bool, len(pendingHeadSHAs))
83-
for _, sha := range pendingHeadSHAs {
84-
pending[sha] = true
94+
pending := map[string]bool{}
95+
for _, intent := range unprocessedIntents {
96+
if isMergeIntentInFlight(ctx, j.env.RemoteQueue(), intent, now) {
97+
pending[intent.HeadSHA] = true
98+
}
8599
}
86100

87101
for i := range projectRefs {
@@ -96,6 +110,17 @@ func (j *mergeQueuePatchRecoveryJob) Run(ctx context.Context) {
96110
}
97111
}
98112

113+
// isMergeIntentInFlight returns whether an unprocessed merge intent may still be
114+
// processed into a patch, either because it's recent or because its processor
115+
// job is still waiting or running in the queue.
116+
func isMergeIntentInFlight(ctx context.Context, q amboy.Queue, intent patch.UnprocessedMergeIntent, now time.Time) bool {
117+
if now.Sub(intent.CreatedAt) < pendingIntentMaxAge {
118+
return true
119+
}
120+
processor, ok := q.Get(ctx, patchIntentProcessorJobID(patch.GithubMergeIntentType, intent.ID))
121+
return ok && !processor.Status().Completed
122+
}
123+
99124
// recoverProject creates a patch for the merge group at the front of the project's GitHub merge
100125
// queue (queue position 1) if it has neither an active patch nor a pending intent.
101126
func (j *mergeQueuePatchRecoveryJob) recoverProject(ctx context.Context, projectRef *model.ProjectRef, pending map[string]bool) error {

‎units/merge_queue_patch_recovery_test.go‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,15 @@
11
package units
22

33
import (
4+
"context"
45
"testing"
56
"time"
67

8+
"github.com/evergreen-ci/evergreen/model/patch"
79
"github.com/google/go-github/v70/github"
10+
"github.com/mongodb/amboy"
11+
"github.com/mongodb/amboy/pool"
12+
"github.com/mongodb/amboy/queue"
813
"github.com/stretchr/testify/assert"
914
"github.com/stretchr/testify/require"
1015
)
@@ -67,3 +72,54 @@ func TestMergeGroupStagedAt(t *testing.T) {
6772
assert.True(t, mergeGroupStagedAt(&github.RepositoryCommit{}).IsZero())
6873
})
6974
}
75+
76+
func TestIsMergeIntentInFlight(t *testing.T) {
77+
now := time.Now()
78+
oldIntent := patch.UnprocessedMergeIntent{
79+
ID: "intent",
80+
HeadSHA: "sha",
81+
CreatedAt: now.Add(-2 * pendingIntentMaxAge),
82+
}
83+
84+
makeQueue := func(t *testing.T) amboy.Queue {
85+
q := queue.NewLocalLimitedSize(1, 10)
86+
// Use a no-op runner so queued jobs stay pending.
87+
runner := pool.NewNoop()
88+
require.NoError(t, runner.SetQueue(q))
89+
require.NoError(t, q.SetRunner(runner))
90+
ctx, cancel := context.WithCancel(t.Context())
91+
t.Cleanup(cancel)
92+
require.NoError(t, q.Start(ctx))
93+
return q
94+
}
95+
makeProcessor := func(intentID string) *patchIntentProcessor {
96+
j := makePatchIntentProcessor()
97+
j.SetID(patchIntentProcessorJobID(patch.GithubMergeIntentType, intentID))
98+
return j
99+
}
100+
101+
t.Run("RecentIntentWithNoJobIsInFlight", func(t *testing.T) {
102+
intent := patch.UnprocessedMergeIntent{ID: "intent", HeadSHA: "sha", CreatedAt: now.Add(-time.Minute)}
103+
assert.True(t, isMergeIntentInFlight(t.Context(), makeQueue(t), intent, now))
104+
})
105+
t.Run("OldIntentWithQueuedJobIsInFlight", func(t *testing.T) {
106+
q := makeQueue(t)
107+
require.NoError(t, q.Put(t.Context(), makeProcessor(oldIntent.ID)))
108+
assert.True(t, isMergeIntentInFlight(t.Context(), q, oldIntent, now))
109+
})
110+
t.Run("OldIntentWithNoJobIsStale", func(t *testing.T) {
111+
assert.False(t, isMergeIntentInFlight(t.Context(), makeQueue(t), oldIntent, now))
112+
})
113+
t.Run("OldIntentWithCompletedJobIsStale", func(t *testing.T) {
114+
q := makeQueue(t)
115+
j := makeProcessor(oldIntent.ID)
116+
require.NoError(t, q.Put(t.Context(), j))
117+
j.MarkComplete()
118+
assert.False(t, isMergeIntentInFlight(t.Context(), q, oldIntent, now))
119+
})
120+
t.Run("OldIntentWithJobForDifferentIntentIsStale", func(t *testing.T) {
121+
q := makeQueue(t)
122+
require.NoError(t, q.Put(t.Context(), makeProcessor("other_intent")))
123+
assert.False(t, isMergeIntentInFlight(t.Context(), q, oldIntent, now))
124+
})
125+
}

‎units/patch_intent.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ func NewPatchIntentProcessor(env evergreen.Environment, patchID mgobson.ObjectId
8080
j.intent = intent
8181
j.env = env
8282

83-
j.SetID(fmt.Sprintf("%s-%s-%s", patchIntentJobName, j.IntentType, j.IntentID))
83+
j.SetID(patchIntentProcessorJobID(j.IntentType, j.IntentID))
8484
j.UpdateTimeInfo(amboy.JobTimeInfo{
8585
MaxTime: maxPatchIntentJobTime,
8686
})
@@ -94,6 +94,10 @@ func NewGitHubPatchIntentProcessor(env evergreen.Environment, patchID mgobson.Ob
9494
return j
9595
}
9696

97+
func patchIntentProcessorJobID(intentType, intentID string) string {
98+
return fmt.Sprintf("%s-%s-%s", patchIntentJobName, intentType, intentID)
99+
}
100+
97101
func makePatchIntentProcessor() *patchIntentProcessor {
98102
j := &patchIntentProcessor{
99103
Base: job.Base{

0 commit comments

Comments
 (0)
⌂ 换网址