diff --git a/.github/scripts/issue-triage/issue-context.py b/.github/scripts/issue-triage/issue-context.py index cf87d1ead1..056e890fd7 100644 --- a/.github/scripts/issue-triage/issue-context.py +++ b/.github/scripts/issue-triage/issue-context.py @@ -533,6 +533,17 @@ def existing_input_hash(comments): def should_process(event, report_comment): action = event.get("action") + issue = event.get("issue") or {} + issue_labels = { + (label.get("name", "") if isinstance(label, dict) else str(label)).lower() + for label in (issue.get("labels") or []) + } + # Never triage the automated dedupe-digest issue. It is bot-authored and + # aggregates untrusted text from many issues, so re-triaging it wastes AI + # credits and creates a prompt-injection surface. See + # .github/workflows/dedupe-digest.yml (DIGEST_LABEL). + if "dedupe-digest" in issue_labels: + return False, False if "comment" not in event: if ( action == "reopened" diff --git a/.github/scripts/issue-triage/tests/test_issue_context.py b/.github/scripts/issue-triage/tests/test_issue_context.py index a9110abbd6..c8912bd187 100644 --- a/.github/scripts/issue-triage/tests/test_issue_context.py +++ b/.github/scripts/issue-triage/tests/test_issue_context.py @@ -338,6 +338,17 @@ class IssueContextTests(unittest.TestCase): } self.assertEqual(CONTEXT.should_process(event, None), (False, False)) + def test_dedupe_digest_issue_is_never_triaged(self): + event = { + "action": "opened", + "sender": {"login": "github-actions[bot]"}, + "issue": { + "number": 11, + "labels": [{"name": "dedupe-digest"}], + }, + } + self.assertEqual(CONTEXT.should_process(event, None), (False, False)) + def test_unrelated_comment_writes_noop(self): event = { "action": "created", diff --git a/.github/workflows/dedupe-digest.yml b/.github/workflows/dedupe-digest.yml new file mode 100644 index 0000000000..4cd28e4bf0 --- /dev/null +++ b/.github/workflows/dedupe-digest.yml @@ -0,0 +1,390 @@ +# Daily Dedupe Digest +# +# Aggregates the possible-duplicate issues that the "AI Issue Triage" workflow +# has already surfaced, into a single daily review issue assigned to a +# maintainer. It does NOT run its own duplicate detection β€” the triage workflow +# is the single source of truth. Each run: +# 1. Re-checks issues carried over from the previous digest plus issues +# updated in the recent lookback window. +# 2. Reads each issue's triage comment (marker +# ``) and extracts the +# `### πŸ” Possible duplicates` candidates. +# 3. Closes the previous digest and opens a fresh one listing every flagged +# issue (the duplicate to close) with its suggested canonical issue. +# 4. If nothing is flagged, it just closes the previous digest and opens +# nothing. +name: Daily Dedupe Digest + +on: + schedule: + - cron: "0 8 * * *" # 08:00 UTC daily + workflow_dispatch: + inputs: + lookback_hours: + description: "How many hours back to scan for freshly triaged issues" + required: false + default: "26" + +permissions: + issues: write + +concurrency: + group: dedupe-digest + cancel-in-progress: false + +env: + DIGEST_ASSIGNEE: niels9001 + DIGEST_LABEL: dedupe-digest + DIGEST_TITLE_PREFIX: "[Dedupe Digest]" + DIGEST_BODY_MARKER: "" + TRIAGE_MARKER: "" + TRIAGE_BOT_LOGIN: "github-actions[bot]" + LOOKBACK_HOURS: "26" + MAX_ISSUES_SCANNED: "400" + MAX_FLAGGED_IN_DIGEST: "75" + # Labels that mean an issue is already resolved as a duplicate and should + # drop out of the digest. + RESOLVED_DUP_LABELS: "duplicate,Resolution-Duplicate" + +jobs: + build-digest: + runs-on: ubuntu-latest + steps: + - name: Build daily dedupe digest + uses: actions/github-script@v9.0.0 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const { owner, repo } = context.repo; + + const ASSIGNEE = process.env.DIGEST_ASSIGNEE; + const LABEL = process.env.DIGEST_LABEL; + const PREFIX = process.env.DIGEST_TITLE_PREFIX; + const BODY_MARKER = process.env.DIGEST_BODY_MARKER; + const TRIAGE_MARKER = process.env.TRIAGE_MARKER; + const TRIAGE_BOT = process.env.TRIAGE_BOT_LOGIN; + const RESOLVED_LABELS = (process.env.RESOLVED_DUP_LABELS || '') + .split(',').map(s => s.trim().toLowerCase()).filter(Boolean); + const LOOKBACK_HOURS = Number( + context.payload?.inputs?.lookback_hours || process.env.LOOKBACK_HOURS || 26 + ) || 26; + const MAX_SCAN = Number(process.env.MAX_ISSUES_SCANNED) || 400; + const MAX_FLAGGED = Number(process.env.MAX_FLAGGED_IN_DIGEST) || 75; + + const today = new Date().toISOString().slice(0, 10); + + // Neutralize untrusted issue/comment text before the bot republishes + // it: strip HTML comments (marker injection) and control chars, then + // escape Markdown and `@` mentions so a crafted title cannot inject + // links/formatting or notify users/teams on every carried-over digest. + const sanitize = (text, max = 200) => { + const cleaned = String(text == null ? '' : text) + .replace(//g, ' ') + .replace(/[\u0000-\u001F\u007F]/g, ' ') + .replace(/\s+/g, ' ') + .trim() + .slice(0, max); + return cleaned + .replace(/\\/g, '\\\\') + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/@/g, '@\u200B') + .replace(/([`*_{}\[\]()#+.!|~])/g, '\\$1') + .replace(/-/g, '\\-'); + }; + + const labelNames = (issue) => + (issue.labels || []).map(l => (typeof l === 'string' ? l : l.name) || ''); + + const isResolvedDuplicate = (issue) => + issue.state === 'closed' || + labelNames(issue).some(n => RESOLVED_LABELS.includes(n.toLowerCase())); + + // Parse the `### πŸ” Possible duplicates` candidates out of a triage + // canonical comment. Only those
blocks use a `# β€”` + // summary, so scanning the whole body is safe. + const parseDuplicateCandidates = (body) => { + const out = []; + const re = /#(\d+)\s*[β€”-]\s*([^<]*)<\/summary>([\s\S]*?)<\/details>/g; + let m; + while ((m = re.exec(body)) !== null) { + const number = Number(m[1]); + if (!Number.isSafeInteger(number) || number <= 0) continue; + const title = sanitize(m[2], 160); + const reasonMatch = m[3].match(/\*\*Why this may be a duplicate:\*\*\s*([\s\S]*)/); + const reason = sanitize(reasonMatch ? reasonMatch[1] : '', 300); + out.push({ number, title, reason }); + } + return out; + }; + + const findTriageComment = async (issueNumber) => { + const comments = await github.paginate(github.rest.issues.listComments, { + owner, repo, issue_number: issueNumber, per_page: 100 + }); + return comments + .filter(c => + c.user && c.user.login === TRIAGE_BOT && + typeof c.body === 'string' && c.body.includes(TRIAGE_MARKER)) + .sort((a, b) => b.id - a.id)[0] || null; + }; + + // --- 1. Ensure the digest label exists ------------------------------- + try { + await github.rest.issues.getLabel({ owner, repo, name: LABEL }); + } catch (e) { + if (e.status === 404) { + await github.rest.issues.createLabel({ + owner, repo, name: LABEL, color: 'B60205', + description: 'Daily aggregated duplicate-issue review digest' + }); + } else { + throw e; + } + } + + // --- 2. Find prior open digest issues -------------------------------- + const priorDigests = (await github.paginate(github.rest.issues.listForRepo, { + owner, repo, state: 'open', labels: LABEL, per_page: 100 + })).filter(i => !i.pull_request && i.title.startsWith(PREFIX)); + + // --- 3. Build the candidate set -------------------------------------- + // Carry-over: issue numbers remembered from the most recent digest. + const carryNumbers = new Set(); + const mostRecentDigest = priorDigests + .slice() + .sort((a, b) => new Date(b.created_at) - new Date(a.created_at))[0]; + if (mostRecentDigest && typeof mostRecentDigest.body === 'string') { + const re = //g; + let m; + while ((m = re.exec(mostRecentDigest.body)) !== null) { + carryNumbers.add(Number(m[1])); + } + } + + // Fresh: open issues updated within the lookback window. + const sinceIso = new Date(Date.now() - LOOKBACK_HOURS * 3600 * 1000).toISOString(); + const freshNumbers = new Set(); + for await (const { data } of github.paginate.iterator(github.rest.issues.listForRepo, { + owner, repo, state: 'open', since: sinceIso, + sort: 'updated', direction: 'desc', per_page: 100 + })) { + for (const it of data) { + if (it.pull_request) continue; + if (labelNames(it).includes(LABEL)) continue; + freshNumbers.add(it.number); + } + if (freshNumbers.size >= MAX_SCAN) break; + } + + const candidateNumbers = Array.from(new Set([...carryNumbers, ...freshNumbers])) + .slice(0, MAX_SCAN); + + core.info( + `Scanning ${candidateNumbers.length} candidate issue(s) ` + + `(${carryNumbers.size} carried over, ${freshNumbers.size} fresh).` + ); + + // --- 4. Resolve each candidate against its live triage comment ------- + const flagged = []; + for (const number of candidateNumbers) { + let issue; + try { + issue = (await github.rest.issues.get({ owner, repo, issue_number: number })).data; + } catch (e) { + if (e.status === 404) continue; + throw e; + } + if (issue.pull_request) continue; + if (labelNames(issue).includes(LABEL)) continue; // never digest a digest + if (isResolvedDuplicate(issue)) continue; // already closed/resolved + + const triageComment = await findTriageComment(number); + if (!triageComment) continue; + const candidates = parseDuplicateCandidates(triageComment.body); + if (!candidates.length) continue; + + flagged.push({ + number: issue.number, + title: sanitize(issue.title, 200), + url: issue.html_url, + triageUrl: triageComment.html_url, + candidates + }); + } + + // Newest issues first, then keep at most MAX_FLAGGED. Because every + // candidate was inspected above, capping the *sorted* list here means + // a full carry-over set can never starve fresher issues out of the + // scan β€” the newest ones always win the cap. + flagged.sort((a, b) => b.number - a.number); + const totalFlagged = flagged.length; + const tracked = flagged.slice(0, MAX_FLAGGED); + core.info( + `Flagged ${totalFlagged} issue(s) as possible duplicates; ` + + `tracking ${tracked.length}.` + ); + + // --- 5a. Nothing flagged: close priors, open nothing ----------------- + const closePrior = async (noteBody) => { + for (const d of priorDigests) { + try { + await github.rest.issues.createComment({ + owner, repo, issue_number: d.number, body: noteBody + }); + await github.rest.issues.update({ + owner, repo, issue_number: d.number, + state: 'closed', state_reason: 'completed' + }); + } catch (e) { + core.warning(`Failed to close prior digest #${d.number}: ${e.message}`); + } + } + }; + + if (flagged.length === 0) { + await closePrior( + `${BODY_MARKER}\nNo open duplicate candidates remained as of ${today}; closing this digest.` + ); + core.info('No duplicates flagged; closed prior digest(s) and created none.'); + return; + } + + // --- 5b. Group tracked issues under their strongest canonical ------ + // Each tracked issue is a duplicate to close; candidates[0] is the + // strongest suggested canonical (the one to keep). + const groups = new Map(); + for (const f of tracked) { + const primary = f.candidates[0]; + if (!groups.has(primary.number)) { + groups.set(primary.number, { + canonical: { number: primary.number, title: primary.title }, + dupes: [] + }); + } + groups.get(primary.number).dupes.push({ + number: f.number, + title: f.title, + reason: primary.reason, + triageUrl: f.triageUrl, + others: f.candidates.slice(1).map(c => c.number), + allCandidates: f.candidates.map(c => c.number) + }); + } + const groupList = Array.from(groups.values()) + .sort((a, b) => b.canonical.number - a.canonical.number); + for (const g of groupList) { + g.dupes.sort((a, b) => b.number - a.number); + } + + // --- 5c. Build the digest body within GitHub's body-size limit ------ + // GitHub rejects issue bodies over 65,536 chars with a 422. Render + // whole groups only while they fit inside a conservative budget, then + // note how many issues were deferred so nothing looks silently lost. + const MAX_BODY = 60000; + const footer = ''; + + const head = []; + head.push(BODY_MARKER); + head.push(`## 🧭 Daily Duplicate Digest β€” ${today}`); + head.push(''); + head.push( + `AI Issue Triage flagged **${totalFlagged}** open issue(s) as possible ` + + `duplicates, grouped under **${groupList.length}** canonical issue(s) below. ` + + `Triage may have already filed a duplicate-close suggestion on each one β€” ` + + `open the linked triage summary and accept or decline it on the issue itself.` + ); + + const renderGroup = (g) => { + const block = []; + block.push(''); + block.push('---'); + block.push(''); + block.push(`### βœ… Keep #${g.canonical.number} β€” ${g.canonical.title}`); + block.push(''); + block.push( + g.dupes.length === 1 + ? `The following issue looks like a duplicate of #${g.canonical.number} and can be closed:` + : `The following issues look like duplicates of #${g.canonical.number} and can be closed:` + ); + block.push(''); + for (const d of g.dupes) { + const reason = d.reason ? ` β€” ${d.reason}` : ''; + const others = d.others.length + ? ` _(also similar to ${d.others.map(n => `#${n}`).join(', ')})_` + : ''; + block.push( + `- **#${d.number} β€” ${d.title}**${reason}${others} ` + + `([triage summary](${d.triageUrl}))` + ); + } + return block; + }; + + const body = head.slice(); + const markerLines = []; + let renderedGroups = 0; + let renderedDupes = 0; + for (const g of groupList) { + const block = renderGroup(g); + const blockMarkers = g.dupes.flatMap(d => + d.allCandidates.map(cn => ``) + ); + // +240 leaves room for the deferral note and footer. + const projected = body.join('\n').length + block.join('\n').length + + markerLines.join('\n').length + blockMarkers.join('\n').length + + footer.length + 240; + if (projected > MAX_BODY && renderedGroups > 0) break; + body.push(...block); + markerLines.push(...blockMarkers); + renderedGroups++; + renderedDupes += g.dupes.length; + } + + const deferred = totalFlagged - renderedDupes; + if (deferred > 0) { + body.push(''); + body.push('---'); + body.push(''); + body.push( + `_${deferred} additional flagged issue(s) are still being tracked but ` + + `were not shown here (digest size or daily cap). They will surface in a ` + + `later digest as capacity allows._` + ); + } + + // Hidden machine-readable carry-over markers (one per dupβ†’candidate). + body.push(''); + body.push(''); + body.push(...markerLines); + body.push(footer); + + const created = await github.rest.issues.create({ + owner, repo, + title: `${PREFIX} ${today}`, + body: body.join('\n'), + labels: [LABEL] + }); + core.info(`Created digest issue #${created.data.number}.`); + + // Assign the maintainer (best-effort; ignore if not assignable). + if (ASSIGNEE) { + try { + await github.rest.issues.addAssignees({ + owner, repo, issue_number: created.data.number, assignees: [ASSIGNEE] + }); + } catch (e) { + core.warning(`Could not assign @${ASSIGNEE}: ${e.message}`); + } + } + + // Close previous digests, pointing at the new one. + await closePrior( + `${BODY_MARKER}\nSuperseded by #${created.data.number}.` + ); + + core.notice( + `Dedupe digest #${created.data.number} created: ` + + `${renderedDupes} shown, ${deferred} deferred, ${totalFlagged} total.` + );