Test tiers: smoke suite for the local stack

package.json

@@ -11,6 +11,7 @@
     "start": "next start",
     "lint": "yarn eslint --fix \"src/**/*.{ts,tsx,js,jsx}\"",
     "test": "node tests/setup/pretest.mjs && playwright test",
+    "test:smoke": "playwright test -c playwright.smoke.config.ts",
     "test:unit": "node --experimental-strip-types --test tests/unit/*.test.ts",
     "test:ui": "playwright test --ui",
     "test:headed": "playwright test --headed",

playwright.smoke.config.ts

@@ -0,0 +1,24 @@
+import { defineConfig, devices } from "@playwright/test";
+
+// Smoke: read-only checks against the local stack (LessonsApp has no demo site).
+//   SMOKE_BASE_URL=http://localhost:3501 yarn test:smoke
+const baseURL = process.env.SMOKE_BASE_URL;
+if (!baseURL) throw new Error("SMOKE_BASE_URL is required (e.g. http://localhost:3501)");
+// Smoke never runs against production: only demo and local hosts are allowed.
+if (!/(^|\.)demo|^localhost$|\.localtest\.me$|^127\.0\.0\.1$/.test(new URL(baseURL).hostname)) {
+  throw new Error(`SMOKE_BASE_URL ${baseURL} is not a demo or local host; smoke tests do not run against production`);
+}
+
+export default defineConfig({
+  testDir: "./tests/smoke",
+  testMatch: /\.smoke\.ts$/,
+  fullyParallel: true,
+  forbidOnly: !!process.env.CI,
+  retries: 1,
+  workers: 4,
+  reporter: process.env.CI ? [["list"], ["github"]] : "list",
+  timeout: 60 * 1000,
+  expect: { timeout: 15 * 1000 },
+  use: { baseURL, trace: "retain-on-failure", screenshot: "only-on-failure", navigationTimeout: 30 * 1000 },
+  projects: [{ name: "public", use: { ...devices["Desktop Chrome"], headless: true } }]
+});

tests/README.md

@@ -0,0 +1,11 @@
+# Tests
+
+| Tier | When | Command |
+|---|---|---|
+| Smoke | By hand or in the nightly run, against the local stack | `SMOKE_BASE_URL=http://localhost:3501 yarn test:smoke` |
+| PR | Before pushing a PR branch | `yarn test` (the suite is small enough to run whole) |
+| Full | Nightly | `yarn test` |
+
+**Smoke** (`tests/smoke/`, `playwright.smoke.config.ts`) only reads. It walks home → program → study → lesson using whatever the catalog currently links, runs a search, and checks that the login page reaches the API. Any uncaught page error or 5xx fails it. LessonsApp has no demo site (`demo.lessons.church` does not resolve), so nothing runs it after a deploy. Smoke never runs against production: `playwright.smoke.config.ts` refuses any host that is not a demo or local host.
+
+`harness.spec.ts` (formerly `smoke.spec.ts`) only checks that the test fixtures can sign in; it is part of the regular suite.

tests/smoke/browse.smoke.ts

@@ -0,0 +1,42 @@
+import { test, expect, openPage, childLinks } from "./fixtures";
+
+// Walks the public catalog from the home page down to one lesson. The pages are rendered
+// on the server from LessonsApi, so a page that lists its children proves the data loaded.
+
+test("home page lists programs and studies", async ({ page }) => {
+  await openPage(page, "/");
+  const studies = await page.locator("a[href]").evaluateAll((as) => as.map((a) => a.getAttribute("href") || "").filter((h) => /^\/[\w-]+\/[\w-]+$/.test(h)));
+  expect(studies.length, "study links on the home page").toBeGreaterThan(0);
+});
+
+test("program, study and lesson pages render", async ({ page }) => {
+  await openPage(page, "/");
+  const study = await page.locator("a[href]").evaluateAll((as) => as.map((a) => a.getAttribute("href") || "").find((h) => /^\/[\w-]+\/[\w-]+$/.test(h)));
+  expect(study, "a study link on the home page").toBeTruthy();
+  const program = "/" + study!.split("/")[1];
+
+  await openPage(page, program);
+  const studies = await childLinks(page, program);
+  expect(studies.length, `study links on ${program}`).toBeGreaterThan(0);
+
+  await openPage(page, studies[0]);
+  const lessons = await childLinks(page, studies[0]);
+  expect(lessons.length, `lesson links on ${studies[0]}`).toBeGreaterThan(0);
+
+  await openPage(page, lessons[0]);
+});
+
+test("search returns results", async ({ page }) => {
+  const search = page.waitForResponse((r) => r.url().includes("/api/search"), { timeout: 30000 });
+  await page.goto("/search?q=jesus");
+  const res = await search;
+  expect(res.status()).toBeLessThan(400);
+  expect(((await res.json()).results || []).length, "search results for 'jesus'").toBeGreaterThan(0);
+});
+
+test("login page renders and reaches the API", async ({ page }) => {
+  const providers = page.waitForResponse((r) => /\/membership\/users\/sso\/providers/.test(r.url()), { timeout: 30000 });
+  await page.goto("/login");
+  await expect(page.locator('input[type="email"]')).toBeVisible();
+  expect((await providers).status()).toBeLessThan(400);
+});

tests/smoke/fixtures.ts

@@ -0,0 +1,43 @@
+import { test as base, expect, type Page } from "@playwright/test";
+
+// Smoke specs run against a deployed site. They only navigate and read.
+// Every test fails on an uncaught page error or a 5xx from our own hosts.
+
+const FIRST_PARTY = /(^|\.)(churchapps\.org|lessons\.church)$|^localhost$|^127\.0\.0\.1$/;
+
+type Watch = { pageErrors: string[]; serverErrors: string[] };
+
+export const test = base.extend<{ watch: Watch }>({
+  watch: [
+    async ({ page }, use) => {
+      const watch: Watch = { pageErrors: [], serverErrors: [] };
+      page.on("pageerror", (e) => watch.pageErrors.push(e.message));
+      page.on("response", (r) => {
+        const type = r.request().resourceType();
+        if (type !== "xhr" && type !== "fetch" && type !== "document") return;
+        const url = new URL(r.url());
+        if (FIRST_PARTY.test(url.hostname) && r.status() >= 500) watch.serverErrors.push(`${r.status()} ${r.request().method()} ${url.pathname}`);
+      });
+      await use(watch);
+      expect(watch.pageErrors, "uncaught page errors").toEqual([]);
+      expect(watch.serverErrors, "5xx responses from our servers").toEqual([]);
+    },
+    { auto: true }
+  ]
+});
+
+/** Open a server-rendered page and check it answered with a non-error status and a heading. */
+export async function openPage(page: Page, path: string) {
+  const res = await page.goto(path);
+  expect(res?.status(), `${path} status`).toBeLessThan(400);
+  await expect(page.locator("h1").first()).not.toBeEmpty();
+}
+
+/** Same-site links under the current page, e.g. the studies on a program page. */
+export async function childLinks(page: Page, prefix: string) {
+  const hrefs = await page.locator("a[href]").evaluateAll((as) => as.map((a) => a.getAttribute("href") || ""));
+  const depth = prefix.split("/").length;
+  return [...new Set(hrefs.filter((h) => h.startsWith(prefix + "/") && h.split("/").length === depth + 1))];
+}
+
+export { expect };