also looking at this
test(ios): verify app termination before local notification delivery
#308
5 files
+96
-21
| @@ -512,8 +512,9 @@ | ||
| 512 | 512 | // arrival is not a tap. |
| 513 | 513 | // |
| 514 | 514 | // #258: opened by the local link, the page schedules a notification of |
| 515 | // its own, with data, a few seconds out. The test kills the app before it | |
| 516 | // fires and taps it, and this same report says what the tap carried. | |
| 515 | // its own, with data, far enough out for the UI test to stop the app. | |
| 516 | // The test kills it before it fires and taps it, and this same report | |
| 517 | // says what the tap carried. | |
| 517 | 518 | // |
| 518 | 519 | // Permission is asked for only when the link asks, because nothing |
| 519 | 520 | // answers the prompt on the launches simctl makes. iOS only: Android has |
| @@ -527,12 +528,20 @@ | ||
| 527 | 528 | } |
| 528 | 529 | if (launch && launch.url.indexOf('://e2e/local') !== -1) { |
| 529 | 530 | var scheduled = function (text) { document.getElementById('notification-local').textContent = 'CRAFT-E2E-LOCAL-SCHEDULED ' + text } |
| 531 | var delay = Number(new URL(launch.url).searchParams.get('delayMs')) | |
| 532 | if (!Number.isInteger(delay) || delay < 60000 || delay > 180000) { | |
| 533 | scheduled('rejected invalid scheduling window') | |
| 534 | return | |
| 535 | } | |
| 536 | // Record the earliest delivery time before crossing the bridge. A | |
| 537 | // slow completion must not invent a later deadline for the UI test. | |
| 538 | var fireAt = Date.now() + delay | |
| 530 | 539 | window.craft.notifications.schedule({ |
| 531 | 540 | title: 'Craft E2E', |
| 532 | 541 | body: NONCE + ' local', |
| 533 | delay: 8000, | |
| 542 | delay: delay, | |
| 534 | 543 | data: { craftE2E: NONCE, stage: 'local', nested: { kept: true } } |
| 535 | }).then(function () { scheduled('ok') }, function (error) { scheduled('rejected ' + describeError(error).message) }) | |
| 544 | }).then(function () { scheduled('ok ' + fireAt) }, function (error) { scheduled('rejected ' + describeError(error).message) }) | |
| 536 | 545 | } |
| 537 | 546 | var onTap = window.craft.notifications && window.craft.notifications.onTap |
| 538 | 547 | var seen = { dispatched: 0, taps: [], onTap: typeof onTap, unsubscribe: null } |
| @@ -94,8 +94,9 @@ final class NotificationTapColdStartTests: XCTestCase { | ||
| 94 | 94 | print("CRAFT-E2E-NOTIFICATION-RECEIVED-RESULT \(receipts.label)") |
| 95 | 95 | |
| 96 | 96 | // #258: a notification the page scheduled itself, with data. The |
| 97 | // local link has the page schedule it a few seconds out, and the app | |
| 98 | // is killed before it fires, so its tap is a cold start like the push. | |
| 97 | // local link carries a bounded scheduling window. The page reports | |
| 98 | // its earliest delivery deadline so slow UI calls cannot silently | |
| 99 | // turn this into a foreground-delivery test (#307). | |
| 99 | 100 | app.terminate() |
| 100 | 101 | XCTAssertTrue(app.wait(for: .notRunning, timeout: 15), "the app did not stop before the local link") |
| 101 | 102 | XCUIDevice.shared.system.open(URL(string: localLink)!) |
| @@ -105,10 +106,20 @@ final class NotificationTapColdStartTests: XCTestCase { | ||
| 105 | 106 | .containing(NSPredicate(format: "label BEGINSWITH 'CRAFT-E2E-LOCAL-SCHEDULED '")) |
| 106 | 107 | .firstMatch |
| 107 | 108 | XCTAssertTrue(scheduled.waitForExistence(timeout: 60), "the page never said whether it scheduled the notification") |
| 108 | XCTAssertEqual(scheduled.label, "CRAFT-E2E-LOCAL-SCHEDULED ok") | |
| 109 | let schedulingReport = scheduled.label.split(separator: " ") | |
| 110 | guard schedulingReport.count == 3, schedulingReport[1] == "ok", | |
| 111 | let fireAtMs = Double(schedulingReport[2]), fireAtMs.isFinite, fireAtMs > 0 else { | |
| 112 | XCTFail("invalid local scheduling report: \(scheduled.label)") | |
| 113 | return | |
| 114 | } | |
| 115 | let fireAt = Date(timeIntervalSince1970: fireAtMs / 1000) | |
| 116 | XCTAssertLessThanOrEqual(fireAt.timeIntervalSinceNow, 180, "local scheduling window is unbounded") | |
| 109 | 117 | app.terminate() |
| 110 | 118 | XCTAssertTrue(app.wait(for: .notRunning, timeout: 15), "the app did not stop, so the local tap would not be a cold start") |
| 111 | tapBanner(springboard, app: app, containing: localBody) | |
| 119 | let remaining = fireAt.timeIntervalSinceNow | |
| 120 | print("CRAFT-E2E-LOCAL-TIMING deadlineMs=\(fireAtMs) remainingAfterStopSeconds=\(remaining)") | |
| 121 | XCTAssertGreaterThan(remaining, 0, "local notification deadline elapsed before the app stopped; cold-start precondition was not met") | |
| 122 | tapBanner(springboard, app: app, containing: localBody, timeout: remaining + 30) | |
| 112 | 123 | |
| 113 | 124 | let local = app.staticTexts |
| 114 | 125 | .containing(NSPredicate(format: "label BEGINSWITH 'CRAFT-E2E-NOTIFICATION '")) |
| @@ -130,11 +141,11 @@ final class NotificationTapColdStartTests: XCTestCase { | ||
| 130 | 141 | /// enough for the banner to go, and the test fails saying the tap could |
| 131 | 142 | /// not find it. There is no retrying around that: a tap that misses ends |
| 132 | 143 | /// the test method, even inside XCTExpectFailure. |
| 133 | private func tapBanner(_ springboard: XCUIApplication, app: XCUIApplication, containing text: String) { | |
| 144 | private func tapBanner(_ springboard: XCUIApplication, app: XCUIApplication, containing text: String, timeout: TimeInterval = 120) { | |
| 134 | 145 | let banner = springboard.descendants(matching: .any) |
| 135 | 146 | .matching(NSPredicate(format: "label CONTAINS %@", text)) |
| 136 | 147 | .firstMatch |
| 137 | XCTAssertTrue(banner.waitForExistence(timeout: 120), "no banner for \(text) ever appeared") | |
| 148 | XCTAssertTrue(banner.waitForExistence(timeout: timeout), "no banner for \(text) ever appeared") | |
| 138 | 149 | banner.tap() |
| 139 | 150 | XCTAssertTrue(app.wait(for: .runningForeground, timeout: 60), "tapping the banner did not start the app") |
| 140 | 151 | } |
| @@ -3,7 +3,7 @@ import type { LegOutcome, RunnerOptions } from './types' | ||
| 3 | 3 | import { copyFileSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs' |
| 4 | 4 | import { join } from 'node:path' |
| 5 | 5 | import { bootSimulator, init, pickSimulator } from '../../packages/ios/src/index' |
| 6 | import { deepLinkProblems, deepLinkResults, evaluateRun, hasTerminated, localNotificationProblems, notificationBody, notificationPayload, notificationReceiptProblems, notificationReceiptReport, notificationTapProblems, notificationTapReport, ZIG_REFUSED_ACTIONS, ZIG_SERVED_ACTIONS, ZIG_TESTED_ACTIONS, zigDispatchedActions, zigHandBacks, zigRefusals } from './protocol' | |
| 6 | import { deepLinkProblems, deepLinkResults, evaluateRun, hasTerminated, LOCAL_NOTIFICATION_DELAY_MS, localNotificationProblems, NOTIFICATION_TEST_TIMEOUT_MS, notificationBody, notificationPayload, notificationReceiptProblems, notificationReceiptReport, notificationTapProblems, notificationTapReport, ZIG_REFUSED_ACTIONS, ZIG_SERVED_ACTIONS, ZIG_TESTED_ACTIONS, zigDispatchedActions, zigHandBacks, zigRefusals } from './protocol' | |
| 7 | 7 | import { command, driverPage, waitForFile } from './support' |
| 8 | 8 | |
| 9 | 9 | /** The scheme the probe registers, and the one its cold-start link uses. */ |
| @@ -69,13 +69,6 @@ const UI_TESTS = `${APP_NAME}UITests` | ||
| 69 | 69 | /** The UI test classes, each run on its own with `-only-testing`. */ |
| 70 | 70 | const UI_TEST_CLASSES = ['DeepLinkColdStartTests', 'NotificationTapColdStartTests'] as const |
| 71 | 71 | |
| 72 | /** | |
| 73 | * How long the notification UI test gets in all. It starts a test runner, | |
| 74 | * launches the app four times, answers a permission prompt and waits out a | |
| 75 | * scheduled notification; it usually takes about a minute. | |
| 76 | */ | |
| 77 | const NOTIFICATION_TEST_TIMEOUT_MS = 300_000 | |
| 78 | ||
| 79 | 72 | /** |
| 80 | 73 | * The pushes the notification UI test has asked for, in order, one per |
| 81 | 74 | * `CRAFT-E2E-PUSH-READY <stage>` line. |
| @@ -343,7 +336,7 @@ schemes: | ||
| 343 | 336 | TEST_RUNNER_PROBE_BUNDLE_ID: BUNDLE_ID, |
| 344 | 337 | TEST_RUNNER_PROBE_NOTIFY_LINK: `${DEEP_LINK_SCHEME}://e2e/notify?run=${encodeURIComponent(nonce)}`, |
| 345 | 338 | TEST_RUNNER_PROBE_PUSH_BODY: notificationBody(nonce, 'cold'), |
| 346 | TEST_RUNNER_PROBE_LOCAL_LINK: `${DEEP_LINK_SCHEME}://e2e/local?run=${encodeURIComponent(nonce)}`, | |
| 339 | TEST_RUNNER_PROBE_LOCAL_LINK: `${DEEP_LINK_SCHEME}://e2e/local?run=${encodeURIComponent(nonce)}&delayMs=${LOCAL_NOTIFICATION_DELAY_MS}`, | |
| 347 | 340 | TEST_RUNNER_PROBE_LOCAL_BODY: notificationBody(nonce, 'local'), |
| 348 | 341 | }, |
| 349 | 342 | stdin: 'ignore', |
| @@ -365,6 +358,8 @@ schemes: | ||
| 365 | 358 | } |
| 366 | 359 | if (tapping.exitCode === null) tapping.kill() |
| 367 | 360 | await tapping.exited |
| 361 | if (tapping.exitCode !== 0) | |
| 362 | failures.push(`notification UI test exited ${tapping.exitCode}; see ${notificationEvidence}`) | |
| 368 | 363 | await command(['xcrun', 'simctl', 'io', device.udid, 'screenshot', join(evidence, 'notification-screen.png')], { allowFailure: true }) |
| 369 | 364 | await command(['xcrun', 'simctl', 'terminate', device.udid, BUNDLE_ID], { allowFailure: true }) |
| 370 | 365 | const notificationOutput = readFileSync(notificationLog, 'utf8') |
| @@ -2,12 +2,69 @@ import { describe, expect, it } from 'bun:test' | ||
| 2 | 2 | import { readFileSync } from 'node:fs' |
| 3 | 3 | import { join } from 'node:path' |
| 4 | 4 | import { ANDROID_DECLINE_PHRASES, androidDeclines, awaitedNeeds, deepLinkProblems, deepLinkReports, deepLinkResults, DISMISS_SHARE_MENU, elfSectionNames, evaluateRun, hasTerminated, localNotificationProblems, NOTIFICATION_ROUTE, notificationBody, notificationPayload, notificationReceiptProblems, notificationReceiptReport, notificationTapProblems, notificationTapReport, parseDriverOutput, REQUIRED_CASES, requiredCaseProblems, runtimePermissionGranted, shareMenuInFront, strippedLibraryProblems, ZIG_REFUSED_ACTIONS, ZIG_SERVED_ACTIONS, ZIG_TESTED_ACTIONS, zigDispatchedActions, zigHandBacks, zigRefusals } from './protocol' |
| 5 | import { LOCAL_NOTIFICATION_DELAY_MS, NOTIFICATION_TEST_TIMEOUT_MS } from './protocol' | |
| 5 | 6 | |
| 6 | 7 | const ESC = String.fromCharCode(27) |
| 7 | 8 | |
| 8 | 9 | /** A transcript shaped like the one a real iOS run produces. */ |
| 9 | 10 | const RUN = 'craft-e2e-ios-shim-1234' |
| 10 | 11 | |
| 12 | describe('local notification scheduling window', () => { | |
| 13 | const page = readFileSync(join(import.meta.dir, 'driver.html'), 'utf8') | |
| 14 | const start = page.indexOf(" if (launch && launch.url.indexOf('://e2e/local')") | |
| 15 | const end = page.indexOf(' var onTap', start) | |
| 16 | if (start < 0 || end < 0) throw new Error('Cannot find the actual driver scheduling block') | |
| 17 | const schedule = new Function('window', 'document', 'launch', 'NONCE', 'Date', 'describeError', page.slice(start, end)) | |
| 18 | ||
| 19 | it('keeps the host budget large enough for the bounded delivery window', () => { | |
| 20 | expect(LOCAL_NOTIFICATION_DELAY_MS).toBe(120_000) | |
| 21 | expect(NOTIFICATION_TEST_TIMEOUT_MS - LOCAL_NOTIFICATION_DELAY_MS).toBeGreaterThanOrEqual(300_000) | |
| 22 | const host = readFileSync(join(import.meta.dir, 'ios.ts'), 'utf8') | |
| 23 | expect(host).toContain('&delayMs=${LOCAL_NOTIFICATION_DELAY_MS}') | |
| 24 | expect(host).toContain('Date.now() + NOTIFICATION_TEST_TIMEOUT_MS') | |
| 25 | }) | |
| 26 | ||
| 27 | it('reports the earliest delivery deadline, not a fresh window after a delayed callback', async () => { | |
| 28 | const element = { textContent: '' } | |
| 29 | let options: any | |
| 30 | let now = 1000 | |
| 31 | let resolve!: () => void | |
| 32 | const pending = new Promise<void>((done) => { resolve = done }) | |
| 33 | schedule({ craft: { notifications: { schedule(value: unknown) { options = value; return pending } } } }, | |
| 34 | { getElementById: () => element }, { url: 'crafte2eprobe://e2e/local?delayMs=120000' }, RUN, { now: () => now }, String) | |
| 35 | expect(options.delay).toBe(120_000) | |
| 36 | expect(options.data).toEqual({ craftE2E: RUN, stage: 'local', nested: { kept: true } }) | |
| 37 | expect(element.textContent).toBe('') | |
| 38 | now += 16_000 | |
| 39 | resolve() | |
| 40 | await pending | |
| 41 | expect(element.textContent).toBe('CRAFT-E2E-LOCAL-SCHEDULED ok 121000') | |
| 42 | // Even after the observed 16-second UI lookup and 32-second termination, | |
| 43 | // the deadline is still ahead. The old 8-second window failed this case. | |
| 44 | expect(121_000 - (now + 16_000 + 32_000)).toBeGreaterThan(0) | |
| 45 | }) | |
| 46 | ||
| 47 | it('refuses missing, malformed or unbounded scheduling delays', () => { | |
| 48 | for (const query of ['', '?delayMs=NaN', '?delayMs=0', '?delayMs=8000', '?delayMs=180001']) { | |
| 49 | const element = { textContent: '' } | |
| 50 | let called = false | |
| 51 | schedule({ craft: { notifications: { schedule() { called = true; return Promise.resolve() } } } }, | |
| 52 | { getElementById: () => element }, { url: `crafte2eprobe://e2e/local${query}` }, RUN, Date, String) | |
| 53 | expect(called).toBe(false) | |
| 54 | expect(element.textContent).toContain('CRAFT-E2E-LOCAL-SCHEDULED rejected') | |
| 55 | } | |
| 56 | }) | |
| 57 | ||
| 58 | it('reports native scheduling rejection without claiming a valid deadline', async () => { | |
| 59 | const element = { textContent: '' } | |
| 60 | const rejected = Promise.reject(new Error('permission denied')) | |
| 61 | schedule({ craft: { notifications: { schedule: () => rejected } } }, | |
| 62 | { getElementById: () => element }, { url: 'crafte2eprobe://e2e/local?delayMs=120000' }, RUN, Date, (error: Error) => error) | |
| 63 | await rejected.catch(() => {}) | |
| 64 | expect(element.textContent).toBe('CRAFT-E2E-LOCAL-SCHEDULED rejected permission denied') | |
| 65 | }) | |
| 66 | }) | |
| 67 | ||
| 11 | 68 | function iosTranscript(overrides: { cases?: string[], omit?: string[], fail?: Record<string, string>, done?: boolean, run?: string } = {}): string { |
| 12 | 69 | const planned = overrides.cases ?? REQUIRED_CASES.ios |
| 13 | 70 | const omit = overrides.omit ?? [] |
| @@ -59,6 +59,11 @@ export type DriverEvent = PlanEvent | CaseEvent | DoneEvent | FatalEvent | Obser | ||
| 59 | 59 | |
| 60 | 60 | export type MobilePlatform = 'ios' | 'android' |
| 61 | 61 | |
| 62 | // The local notification must survive UI discovery and synchronous app | |
| 63 | // termination before it fires. CI has spent 32 seconds in terminate() alone. | |
| 64 | export const LOCAL_NOTIFICATION_DELAY_MS = 120_000 | |
| 65 | export const NOTIFICATION_TEST_TIMEOUT_MS = 300_000 + LOCAL_NOTIFICATION_DELAY_MS | |
| 66 | ||
| 62 | 67 | /** |
| 63 | 68 | * The cases each platform must run, named here and nowhere else. |
| 64 | 69 | * |
formatting only · 4 hidden lines
| @@ -867,4 +872,3 @@ export function androidDeclines(text: string): string[] { formatting only | ||
| 867 | 872 | const pattern = new RegExp(`craft: (\\w+) (${phrases}) \\(([^)]*)\\)`, 'g') |
| 868 | 873 | return [...plain.matchAll(pattern)].map(match => `${match[1]}: ${match[2]} (${match[3]})`) |
| 869 | 874 | } |
| 870 | ||