diff --git a/torchci/clickhouse_queries/autorevert_commits/params.json b/torchci/clickhouse_queries/autorevert_commits/params.json new file mode 100644 index 0000000000..4b82f61639 --- /dev/null +++ b/torchci/clickhouse_queries/autorevert_commits/params.json @@ -0,0 +1,12 @@ +{ + "params": { + "repo": "String", + "shas": "Array(String)" + }, + "tests": [ + { + "repo": "pytorch/pytorch", + "shas": ["abc123", "def456"] + } + ] +} diff --git a/torchci/clickhouse_queries/autorevert_commits/query.sql b/torchci/clickhouse_queries/autorevert_commits/query.sql new file mode 100644 index 0000000000..1b975faede --- /dev/null +++ b/torchci/clickhouse_queries/autorevert_commits/query.sql @@ -0,0 +1,11 @@ +SELECT + commit_sha, + groupArray(workflows) as all_workflows, + groupArray(source_signal_keys) as all_source_signal_keys +FROM misc.autorevert_events_v2 +WHERE repo = {repo: String} + AND action = 'revert' + AND dry_run = 0 + AND failed = 0 + AND commit_sha IN {shas: Array(String)} +GROUP BY commit_sha diff --git a/torchci/clickhouse_queries/autorevert_details/params.json b/torchci/clickhouse_queries/autorevert_details/params.json new file mode 100644 index 0000000000..a6b7ced1af --- /dev/null +++ b/torchci/clickhouse_queries/autorevert_details/params.json @@ -0,0 +1,12 @@ +{ + "params": { + "repo": "String", + "sha": "String" + }, + "tests": [ + { + "repo": "pytorch/pytorch", + "sha": "321e60abc123" + } + ] +} \ No newline at end of file diff --git a/torchci/clickhouse_queries/autorevert_details/query.sql b/torchci/clickhouse_queries/autorevert_details/query.sql new file mode 100644 index 0000000000..91c9e748d5 --- /dev/null +++ b/torchci/clickhouse_queries/autorevert_details/query.sql @@ -0,0 +1,13 @@ +SELECT + commit_sha, + workflows, + source_signal_keys, + ts +FROM misc.autorevert_events_v2 +WHERE + repo = {repo: String} + AND commit_sha = {sha: String} + AND action = 'revert' + AND dry_run = 0 + AND failed = 0 +ORDER BY ts DESC diff --git a/torchci/components/commit/AutorevertBanner.module.css b/torchci/components/commit/AutorevertBanner.module.css new file mode 100644 index 0000000000..9802357ee9 --- /dev/null +++ b/torchci/components/commit/AutorevertBanner.module.css @@ -0,0 +1,52 @@ +.autorevertBanner { + background-color: var(--sev-banner-bg); + border: 2px solid var(--autoreverted-signal-border); + border-radius: 8px; + padding: 16px; + margin: 16px 0; + box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1); +} + +.bannerHeader { + display: flex; + align-items: center; + gap: 8px; + margin-bottom: 12px; + font-size: 1.1em; +} + +.warningIcon { + font-size: 1.2em; +} + +.bannerContent { + color: var(--text-color); +} + +.bannerContent p { + margin: 8px 0; +} + +.workflowList { + margin: 12px 0; + padding-left: 24px; +} + +.workflowList li { + margin: 6px 0; + list-style-type: disc; +} + +.workflowList a { + color: var(--link-color); + text-decoration: none; +} + +.workflowList a:hover { + text-decoration: underline; +} + +.investigateMessage { + font-style: italic; + margin-top: 12px; +} diff --git a/torchci/components/commit/AutorevertBanner.tsx b/torchci/components/commit/AutorevertBanner.tsx new file mode 100644 index 0000000000..378c543e64 --- /dev/null +++ b/torchci/components/commit/AutorevertBanner.tsx @@ -0,0 +1,128 @@ +import useSWR from "swr"; +import styles from "./AutorevertBanner.module.css"; + +interface AutorevertDetails { + commit_sha: string; + workflows: string[]; + source_signal_keys: string[]; + job_ids: number[]; + job_base_names: string[]; + wf_run_ids: number[]; + created_at: string; +} + +interface SignalInfo { + workflow_name: string; + signals: Array<{ + key: string; + job_url?: string; + hud_url?: string; + }>; +} + +export function AutorevertBanner({ + repoOwner, + repoName, + sha, +}: { + repoOwner: string; + repoName: string; + sha: string; +}) { + const { data: autorevertData } = useSWR( + `/api/autorevert/${repoOwner}/${repoName}/${sha}`, + async (url) => { + try { + const response = await fetch(url); + + if (response.status === 404) { + // No autorevert data for this commit + return null; + } + + if (!response.ok) { + const errorText = await response.text(); + throw new Error(`Failed to fetch: ${response.status} - ${errorText}`); + } + + const data = await response.json(); + return data; + } catch (e) { + // Silently fail - no autorevert data + return null; + } + }, + { + refreshInterval: 0, // Don't refresh autorevert data + } + ); + + // Don't show banner if no data or error + if (!autorevertData) { + return null; + } + + // Handle case where arrays might be undefined or have different structure + const workflows = autorevertData.workflows || []; + const sourceSignalKeys = autorevertData.source_signal_keys || []; + + // If no workflows data, don't show the banner + if (!workflows.length) { + return null; + } + + // Group signals by workflow + const signalsByWorkflow = new Map(); + + workflows.forEach((workflow, idx) => { + if (!signalsByWorkflow.has(workflow)) { + signalsByWorkflow.set(workflow, { + workflow_name: workflow, + signals: [], + }); + } + + const signalKey = sourceSignalKeys[idx] || ""; + + const signal = { + key: signalKey, + // Try to create a HUD URL using the signal key as a filter + hud_url: signalKey + ? `/hud/${repoOwner}/${repoName}/main?nameFilter=${encodeURIComponent( + signalKey + )}` + : undefined, + }; + + signalsByWorkflow.get(workflow)!.signals.push(signal); + }); + + return ( +
+
+ ⚠️ + This commit was automatically reverted +
+
+

This PR is attributed to have caused regression in:

+
    + {Array.from(signalsByWorkflow.values()).map((workflowInfo) => ( +
  • + {workflowInfo.workflow_name}:{" "} + {workflowInfo.signals.map((signal, idx) => ( + + {idx > 0 && ", "} + {signal.key} + + ))} +
    • + ))} +
+

+ You can add the label autorevert: disable to disable + autorevert for a specific PR. +

+
+
+ ); +} diff --git a/torchci/components/hud.module.css b/torchci/components/hud.module.css index d189e3918d..568196dcd5 100644 --- a/torchci/components/hud.module.css +++ b/torchci/components/hud.module.css @@ -132,6 +132,15 @@ background-color: var(--forced-merge-failure-bg); } +.autoreverted { + background-color: var(--autoreverted-bg); +} + +.autorevertSignal { + background-color: var(--autoreverted-signal-bg); + border: 2px solid var(--autoreverted-signal-border); +} + .selectedRow { background-color: var(--selected-row-bg); font-weight: bold; diff --git a/torchci/components/job/GroupJobConclusion.tsx b/torchci/components/job/GroupJobConclusion.tsx index ed22bc143d..2e0059567d 100644 --- a/torchci/components/job/GroupJobConclusion.tsx +++ b/torchci/components/job/GroupJobConclusion.tsx @@ -1,4 +1,8 @@ import TooltipTarget from "components/common/tooltipTarget/TooltipTarget"; +import { + isGroupAutorevertSignal, + isJobAutorevertSignal, +} from "lib/autorevertUtils"; import { getGroupConclusionChar } from "lib/JobClassifierUtil"; import { isCancellationSuccessJob, @@ -6,7 +10,7 @@ import { isRerunDisabledTestsJob, isUnstableJob, } from "lib/jobUtils"; -import { IssueData, JobData } from "lib/types"; +import { IssueData, JobData, RowData } from "lib/types"; import { MonsterFailuresContext, PinnedTooltipContext, @@ -175,6 +179,7 @@ export default function HudGroupedCell({ unstableIssues, repoOwner, repoName, + rowData, }: { sha: string; groupName: string; @@ -185,10 +190,25 @@ export default function HudGroupedCell({ unstableIssues: IssueData[]; repoOwner: string; repoName: string; + rowData?: RowData; }) { const [pinnedId, setPinnedId] = useContext(PinnedTooltipContext); const [monsterFailures] = useContext(MonsterFailuresContext); - const style = pinnedId.name == groupName ? hudStyles.highlight : ""; + + // Check if this group contains autorevert signals + const isAutorevertSignal = rowData + ? isGroupAutorevertSignal(jobs, rowData) + : false; + + // Build cell style classes + const cellClasses = []; + if (pinnedId.name == groupName) { + cellClasses.push(hudStyles.highlight); + } + if (isAutorevertSignal) { + cellClasses.push(hudStyles.autorevertSignal); + } + const style = cellClasses.join(" "); const erroredJobs = []; const warningOnlyJobs = []; @@ -255,6 +275,7 @@ export default function HudGroupedCell({ queuedJobs={queuedJobs} failedPreviousRunJobs={failedPreviousRunJobs} sha={sha} + rowData={rowData} /> } > @@ -304,6 +325,7 @@ function GroupTooltip({ queuedJobs, failedPreviousRunJobs, sha, + rowData, }: { conclusion: GroupedJobStatus; groupName: string; @@ -312,6 +334,7 @@ function GroupTooltip({ queuedJobs: JobData[]; failedPreviousRunJobs: JobData[]; sha?: string; + rowData?: RowData; }) { const [monsterFailures] = useContext(MonsterFailuresContext); @@ -355,16 +378,31 @@ function GroupTooltip({ : "1 job with this error type:"} - {group.jobs.map((job: JobData, jobIndex: number) => ( -
- - {job.name} - -
- ))} + {group.jobs.map((job: JobData, jobIndex: number) => { + const isAutorevert = rowData + ? isJobAutorevertSignal(job, rowData) + : false; + return ( +
+ + {job.name} + {isAutorevert && " ⚠️ (triggered autorevert)"} + +
+ ); + })} ))} @@ -377,6 +415,7 @@ function GroupTooltip({ groupName={groupName} jobs={erroredJobs} message={"The following jobs errored out:"} + rowData={rowData} /> ); } else if (conclusion === GroupedJobStatus.Queued) { @@ -386,6 +425,7 @@ function GroupTooltip({ groupName={groupName} jobs={queuedJobs} message={"The following jobs are still in queue:"} + rowData={rowData} /> ); } else if (conclusion === GroupedJobStatus.Pending) { @@ -395,6 +435,7 @@ function GroupTooltip({ groupName={groupName} jobs={pendingJobs} message={"The following jobs are still pending:"} + rowData={rowData} /> ); } else if (conclusion === GroupedJobStatus.Flaky) { @@ -404,6 +445,7 @@ function GroupTooltip({ groupName={groupName} jobs={failedPreviousRunJobs} message={"The following jobs were flaky:"} + rowData={rowData} /> ); } else if (conclusion === GroupedJobStatus.AllNull) { @@ -430,26 +472,34 @@ function ToolTip({ groupName, message, jobs, + rowData, }: { conclusion: string; groupName: string; message: string; jobs: JobData[]; + rowData?: RowData; }) { return (
{`[${conclusion}] ${groupName}`}
{message}
{jobs.map((job, ind) => { + const isAutorevert = rowData + ? isJobAutorevertSignal(job, rowData) + : false; return ( {job.name} + {isAutorevert && " ⚠️ (triggered autorevert)"} ); })} diff --git a/torchci/components/job/JobConclusion.module.css b/torchci/components/job/JobConclusion.module.css index 4c6c99bc00..30b1550d5f 100644 --- a/torchci/components/job/JobConclusion.module.css +++ b/torchci/components/job/JobConclusion.module.css @@ -106,3 +106,8 @@ border-left: 1px solid var(--color-failure); border-right: 1px solid var(--color-failure); } + +.autorevert_tooltip_anchor { + font-weight: bold; + color: #d73a49; +} diff --git a/torchci/components/job/JobTooltip.tsx b/torchci/components/job/JobTooltip.tsx index 03040ee377..0b744e32c0 100644 --- a/torchci/components/job/JobTooltip.tsx +++ b/torchci/components/job/JobTooltip.tsx @@ -6,9 +6,11 @@ import JobLinks from "./JobLinks"; export default function JobTooltip({ job, sha, + isAutorevertSignal, }: { job: JobData; sha?: string; + isAutorevertSignal?: boolean; }) { // For nonexistent jobs, just show something basic: if (!job.hasOwnProperty("id")) { @@ -25,6 +27,11 @@ export default function JobTooltip({ return (
{`[${job.conclusion}] ${job.name}`} + {isAutorevertSignal && ( +
+ Failure in this job has triggered autorevert. +
+ )}
click to pin this tooltip, double-click for job page
diff --git a/torchci/lib/autorevertUtils.ts b/torchci/lib/autorevertUtils.ts new file mode 100644 index 0000000000..639db7a0d2 --- /dev/null +++ b/torchci/lib/autorevertUtils.ts @@ -0,0 +1,66 @@ +import { JobData, RowData } from "./types"; + +/** + * Checks if a job triggered an autorevert by matching against the autorevert workflows and signals + * @param job The job to check + * @param rowData The row data containing autorevert information + * @returns true if the job triggered an autorevert, false otherwise + */ +export function isJobAutorevertSignal( + job: JobData | { name: string; conclusion?: string }, + rowData: RowData +): boolean { + if (!rowData.autorevertWorkflows || !rowData.autorevertSignals) { + return false; + } + + if (job.conclusion?.toLowerCase() !== "failure") { + return false; + } + + const lowAutorevertWorkflows = rowData.autorevertWorkflows.map((w) => + w.toLowerCase() + ); + + const jobFullName = job.name; + if (!jobFullName) { + return false; + } + + const parts = jobFullName + .toLocaleLowerCase() + .split("/") + .map((p) => + p + .trim() + .replace(/ \(.*\)$/, "") + .trim() + ); + const jobWorkflow = parts[0]; + const jobNameOnly = parts.slice(1); + + if (!lowAutorevertWorkflows.includes(jobWorkflow)) { + return false; + } + + return rowData.autorevertSignals.some((signal) => { + const signalLower = signal + .toLowerCase() + .split("/") + .map((p) => p.trim()); + return jobNameOnly.every((p, idx) => p === signalLower[idx]); + }); +} + +/** + * Checks if a group contains any jobs that triggered an autorevert + * @param groupJobs The jobs in the group + * @param rowData The row data containing autorevert information + * @returns true if any job in the group triggered an autorevert, false otherwise + */ +export function isGroupAutorevertSignal( + groupJobs: JobData[], + rowData: RowData +): boolean { + return groupJobs.some((job) => isJobAutorevertSignal(job, rowData)); +} diff --git a/torchci/lib/fetchHud.ts b/torchci/lib/fetchHud.ts index 3250e74d10..96d9a90e65 100644 --- a/torchci/lib/fetchHud.ts +++ b/torchci/lib/fetchHud.ts @@ -77,6 +77,28 @@ export default async function fetchHud( ) ); + // Check if any of these commits were autoreverted + const autorevertedCommits = await queryClickhouseSaved("autorevert_commits", { + repo: `${params.repoOwner}/${params.repoName}`, + shas: shas, + }); + + // Create a map from sha to autorevert data + const autorevertDataBySha = new Map< + string, + { workflows: string[]; signals: string[] } + >(); + autorevertedCommits.forEach((r) => { + // Flatten the nested arrays + const allWorkflows = r.all_workflows.flat(); + const allSignals = r.all_source_signal_keys.flat(); + + autorevertDataBySha.set(r.commit_sha, { + workflows: allWorkflows, + signals: allSignals, + }); + }); + const commitsBySha = _.keyBy(commits, "sha"); if (params.filter_reruns) { @@ -151,11 +173,15 @@ export default async function fetchHud( } } + const autorevertData = autorevertDataBySha.get(commit.sha); const row = { ...commit, jobs: jobs, isForcedMerge: forcedMergeShas.has(commit.sha), isForcedMergeWithFailures: forcedMergeWithFailuresShas.has(commit.sha), + isAutoreverted: autorevertData !== undefined, + autorevertWorkflows: autorevertData?.workflows, + autorevertSignals: autorevertData?.signals, }; shaGrid.push(row); }); diff --git a/torchci/lib/types.ts b/torchci/lib/types.ts index 6d0d3e80c9..ced8e65339 100644 --- a/torchci/lib/types.ts +++ b/torchci/lib/types.ts @@ -90,6 +90,9 @@ export interface Highlight { interface RowDataBase extends CommitData { isForcedMerge: boolean | false; isForcedMergeWithFailures: boolean | false; + isAutoreverted: boolean | false; + autorevertWorkflows?: string[]; + autorevertSignals?: string[]; } export interface RowData extends RowDataBase { diff --git a/torchci/lib/utilization/fetchUtilization.test.ts b/torchci/lib/utilization/fetchUtilization.test.ts index ad39a6a82a..3f5ea4ddb5 100644 --- a/torchci/lib/utilization/fetchUtilization.test.ts +++ b/torchci/lib/utilization/fetchUtilization.test.ts @@ -213,7 +213,9 @@ describe("Test flattenTS to flatten timestamp", () => { // assert log expect(logSpy).toHaveBeenCalledWith( - `Warning: Error parsing JSON:SyntaxError: Expected property name or '}' in JSON at position 1 for data string '{{}dsad}'` + expect.stringMatching( + /Warning: Error parsing JSON:SyntaxError: Expected property name or '\}' in JSON at position 1.*for data string '\{\{\}dsad\}'/ + ) ); }); }); diff --git a/torchci/pages/[repoOwner]/[repoName]/commit/[sha].tsx b/torchci/pages/[repoOwner]/[repoName]/commit/[sha].tsx index 94b6b5b3a9..d3b4cebe6b 100644 --- a/torchci/pages/[repoOwner]/[repoName]/commit/[sha].tsx +++ b/torchci/pages/[repoOwner]/[repoName]/commit/[sha].tsx @@ -1,3 +1,4 @@ +import { AutorevertBanner } from "components/commit/AutorevertBanner"; import { CommitInfo } from "components/commit/CommitInfo"; import { useSetTitle } from "components/layout/DynamicTitle"; import { useRouter } from "next/router"; @@ -24,12 +25,19 @@ export default function Page() { {fancyName} Commit: {sha} {sha !== undefined && ( - + <> + + + )}
); diff --git a/torchci/pages/api/autorevert/[repoOwner]/[repoName]/[sha].ts b/torchci/pages/api/autorevert/[repoOwner]/[repoName]/[sha].ts new file mode 100644 index 0000000000..f84093e317 --- /dev/null +++ b/torchci/pages/api/autorevert/[repoOwner]/[repoName]/[sha].ts @@ -0,0 +1,49 @@ +import { queryClickhouseSaved } from "lib/clickhouse"; +import type { NextApiRequest, NextApiResponse } from "next"; + +export default async function handler( + req: NextApiRequest, + res: NextApiResponse +) { + const { repoOwner, repoName, sha } = req.query; + + if (!repoOwner || !repoName || !sha) { + res.status(400).json({ error: "Missing required parameters" }); + return; + } + + try { + const results = await queryClickhouseSaved("autorevert_details", { + repo: `${repoOwner}/${repoName}`, + sha: sha as string, + }); + + if (results && results.length > 0) { + // Combine data from all rows (in case there are multiple events) + const allWorkflows: string[] = []; + const allSignalKeys: string[] = []; + + results.forEach((row: any) => { + if (row.workflows) allWorkflows.push(...row.workflows); + if (row.source_signal_keys) + allSignalKeys.push(...row.source_signal_keys); + }); + + const response = { + commit_sha: results[0].commit_sha, + workflows: allWorkflows, + source_signal_keys: allSignalKeys, + }; + + res.status(200).json(response); + } else { + res.status(404).json({ error: "No autorevert data found" }); + } + } catch (error: any) { + res.status(500).json({ + error: "Internal server error", + details: + process.env.NODE_ENV === "development" ? error.message : undefined, + }); + } +} diff --git a/torchci/pages/hud/[repoOwner]/[repoName]/[branch]/[[...page]].tsx b/torchci/pages/hud/[repoOwner]/[repoName]/[branch]/[[...page]].tsx index ff0c8f60c4..c0b7320717 100644 --- a/torchci/pages/hud/[repoOwner]/[repoName]/[branch]/[[...page]].tsx +++ b/torchci/pages/hud/[repoOwner]/[repoName]/[branch]/[[...page]].tsx @@ -15,6 +15,7 @@ import JobConclusion from "components/job/JobConclusion"; import JobFilterInput from "components/job/JobFilterInput"; import JobTooltip from "components/job/JobTooltip"; import SettingsPanel from "components/SettingsPanel"; +import { isJobAutorevertSignal } from "lib/autorevertUtils"; import { fetcher } from "lib/GeneralUtils"; import { getGroupingData, @@ -62,23 +63,41 @@ export function JobCell({ sha, job, unstableIssues, + isAutorevertSignal, }: { sha: string; job: JobData; unstableIssues: IssueData[]; + isAutorevertSignal?: boolean; }) { const [pinnedId, setPinnedId] = useContext(PinnedTooltipContext); - const style = pinnedId.name == job.name ? styles.highlight : ""; + + // Build cell style classes + const cellClasses = []; + if (pinnedId.name == job.name) { + cellClasses.push(styles.highlight); + } + if (isAutorevertSignal) { + cellClasses.push(styles.autorevertSignal); + } + const cellStyle = cellClasses.join(" "); + return ( window.open(job.htmlUrl)}> } + tooltipContent={ + + } sha={sha as string} name={job.name as string} > -
+
clickCommit(e)}> + clickCommit(e)}> @@ -242,16 +267,22 @@ function HudJobCells({ numClassified != 0 && numClassified == failedJobs?.length } unstableIssues={unstableIssues} + rowData={rowData} /> ); } else { - const job = rowData.nameToJobs.get(name); + const job = rowData.nameToJobs.get(name) ?? { + name: name, + conclusion: undefined, + }; + return ( ); } diff --git a/torchci/styles/globals.css b/torchci/styles/globals.css index 718cd296bb..dc20593080 100644 --- a/torchci/styles/globals.css +++ b/torchci/styles/globals.css @@ -20,6 +20,9 @@ /* HUD specific colors */ --forced-merge-bg: lightyellow; --forced-merge-failure-bg: #ffe0b3; + --autoreverted-bg: #e8e8e8; + --autoreverted-signal-bg: #ffedcc; + --autoreverted-signal-border: #ffc466; --selected-row-bg: lightblue; --highlight-bg: #ffa; --commit-message-bg: whitesmoke; @@ -84,6 +87,9 @@ /* HUD specific colors - dark mode variants */ --forced-merge-bg: #7e7000; --forced-merge-failure-bg: #a06200; + --autoreverted-bg: #3a3a3a; + --autoreverted-signal-bg: #665a22; + --autoreverted-signal-border: #aa8a44; --selected-row-bg: #164863; --highlight-bg: #555500; --commit-message-bg: #2a2a2a;