element-web/playwright/flaky-reporter.ts
David Langley 491f0cd08a
Change license (#13)
* Copyright headers 1

* Licence headers 2

* Copyright Headers 3

* Copyright Headers 4

* Copyright Headers 5

* Copyright Headers 6

* Copyright headers 7

* Add copyright headers for html and config file

* Replace license files and update package.json

* Update with CLA

* lint
2024-09-09 13:57:16 +00:00

135 lines
4.8 KiB
TypeScript

/*
Copyright 2024 New Vector Ltd.
Copyright 2024 The Matrix.org Foundation C.I.C.
SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only
Please see LICENSE files in the repository root for full details.
*/
/**
* Flaky test reporter, creating & updating GitHub issues
* Only intended to run from within GitHub Actions
*/
import type { Reporter, TestCase } from "@playwright/test/reporter";
const REPO = "element-hq/element-web";
const LABEL = "Z-Flaky-Test";
const ISSUE_TITLE_PREFIX = "Flaky playwright test: ";
type PaginationLinks = {
prev?: string;
next?: string;
last?: string;
first?: string;
};
class FlakyReporter implements Reporter {
private flakes = new Set<string>();
public onTestEnd(test: TestCase): void {
const title = `${test.location.file.split("playwright/e2e/")[1]}: ${test.title}`;
if (test.outcome() === "flaky") {
this.flakes.add(title);
}
}
/**
* Parse link header to retrieve pagination links
* @see https://docs.github.com/en/rest/using-the-rest-api/using-pagination-in-the-rest-api?apiVersion=2022-11-28#using-link-headers
* @param link link header from response or undefined
* @returns an empty object if link is undefined otherwise returns a map from type to link
*/
private parseLinkHeader(link: string): PaginationLinks {
/**
* link looks like:
* <https://api.github.com/repositories/1300192/issues?page=2>; rel="prev", <https://api.github.com/repositories/1300192/issues?page=4>;
*/
const map: PaginationLinks = {};
if (!link) return map;
const matches = link.matchAll(/(<(?<link>.+?)>; rel="(?<type>.+?)")/g);
for (const match of matches) {
const { link, type } = match.groups;
map[type] = link;
}
return map;
}
/**
* Fetch all flaky test issues that were updated since Jan-1-2024
* @returns A promise that resolves to a list of issues
*/
async getAllIssues(): Promise<any[]> {
const issues = [];
const { GITHUB_TOKEN, GITHUB_API_URL } = process.env;
// See https://docs.github.com/en/rest/issues/issues?apiVersion=2022-11-28#list-repository-issues
let url = `${GITHUB_API_URL}/repos/${REPO}/issues?labels=${LABEL}&state=all&per_page=100&sort=updated&since=2024-01-01`;
const headers = {
Authorization: `Bearer ${GITHUB_TOKEN}`,
Accept: "application / vnd.github + json",
};
while (url) {
// Fetch issues and add to list
const issuesResponse = await fetch(url, { headers });
const fetchedIssues = await issuesResponse.json();
issues.push(...fetchedIssues);
// Get the next link for fetching more results
const linkHeader = issuesResponse.headers.get("Link");
const parsed = this.parseLinkHeader(linkHeader);
url = parsed.next;
}
return issues;
}
public async onExit(): Promise<void> {
if (this.flakes.size === 0) {
console.log("No flakes found");
return;
}
console.log("Found flakes: ");
for (const flake of this.flakes) {
console.log(flake);
}
const { GITHUB_TOKEN, GITHUB_API_URL, GITHUB_SERVER_URL, GITHUB_REPOSITORY, GITHUB_RUN_ID } = process.env;
if (!GITHUB_TOKEN) return;
const issues = await this.getAllIssues();
for (const flake of this.flakes) {
const title = ISSUE_TITLE_PREFIX + "`" + flake + "`";
const existingIssue = issues.find((issue) => issue.title === title);
const headers = { Authorization: `Bearer ${GITHUB_TOKEN}` };
const body = `${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}`;
if (existingIssue) {
console.log(`Found issue ${existingIssue.number} for ${flake}, adding comment...`);
// Ensure that the test is open
await fetch(existingIssue.url, {
method: "PATCH",
headers,
body: JSON.stringify({ state: "open" }),
});
await fetch(`${existingIssue.url}/comments`, {
method: "POST",
headers,
body: JSON.stringify({ body }),
});
} else {
console.log(`Creating new issue for ${flake}...`);
await fetch(`${GITHUB_API_URL}/repos/${REPO}/issues`, {
method: "POST",
headers,
body: JSON.stringify({
title,
body,
labels: [LABEL],
}),
});
}
}
}
}
export default FlakyReporter;