ReviewOS
Reload

also looking at this

craft-native/craft

test(ios): tap a real pushed notification that launches the app, and judge what the page gets

#257
Merged glennmichael123 wants to merge test/ios-e2e-notification-tap into main
5 files +308 -10

Review threads live on the whole diff, not on one commit, so none are shown here - a thread's line means something in the branch's final form, and painting it into an intermediate step would put it on code it is not about.

scripts/mobile-e2e/driver.htmlmodified+44-0
@@ -15,6 +15,8 @@
1515 <h1 id="headline">Craft mobile E2E</h1>
1616 <ul id="results"></ul>
1717 <p id="deeplink"></p>
18 <p id="notification-permission"></p>
19 <p id="notification"></p>
1820 <script>
1921 // The page drives itself and reports through the bridge's own `log` action,
2022 // which is the one channel both runtimes already forward off-device: iOS
@@ -36,6 +38,13 @@
3638 var dispatchedLinks = []
3739 window.addEventListener('craftDeepLink', function (event) { dispatchedLinks.push(event.detail || {}) })
3840
41 // Every notification tap the native side dispatches, counted the same way,
42 // so a missing tap can be put on the right side of the bridge: none here
43 // means the app never handed it over, one here and none through onTap
44 // means the page's replay lost it.
45 var dispatchedTaps = 0
46 window.addEventListener('craftNotificationResponse', function () { dispatchedTaps++ })
47
3948 var NONCE = '__CRAFT_E2E_NONCE__'
4049 // Which platform the harness built this leg for. Substituted in from
4150 // outside, because the page comparing window.craft.platform against a
@@ -487,6 +496,40 @@
487496 setTimeout(report, 250)
488497 }
489498
499 // #255: a tap on a push that launches the app reaches a page that
500 // subscribes late, through notifications.onTap.
501 //
502 // The harness does on iOS what a person does, from an XCUITest: it opens
503 // the permission link and allows the prompt, kills the app, sends a real
504 // push with `simctl push`, and taps the banner. The page subscribes after
505 // the suite, seconds after the bridge appeared, and reports what it was
506 // handed. On any other launch it was handed nothing, and says so.
507 //
508 // Permission is asked for only when the link asks, because nothing
509 // answers the prompt on the launches simctl makes. iOS only: Android has
510 // no tap path to test yet.
511 function probeNotifications() {
512 if (window.craft.platform !== 'ios') return
513 var launch = dispatchedLinks.filter(function (detail) { return detail.initial === true })[0] || null
514 if (launch && launch.url.indexOf('://e2e/notify') !== -1) {
515 var said = function (text) { document.getElementById('notification-permission').textContent = 'CRAFT-E2E-NOTIFY-PERMISSION ' + text }
516 window.craft.permissions.request('notifications').then(said, function (error) { said('rejected ' + describeError(error).message) })
517 }
518 var onTap = window.craft.notifications && window.craft.notifications.onTap
519 var seen = { dispatched: 0, taps: [], onTap: typeof onTap, unsubscribe: null }
520 if (typeof onTap === 'function') {
521 seen.unsubscribe = typeof window.craft.notifications.onTap(function (detail) { seen.taps.push(detail || {}) })
522 }
523 // Long enough for a replay, or a second delivery, to have arrived.
524 setTimeout(function () {
525 seen.dispatched = dispatchedTaps
526 var line = 'CRAFT-E2E-NOTIFICATION ' + JSON.stringify(seen)
527 document.getElementById('notification').textContent = line
528 try { console.log(line) } catch (ignored) {}
529 try { if (window.craft && window.craft.log) window.craft.log(line) } catch (ignored) {}
530 }, 1500)
531 }
532
490533 function waitForBridge(attempt) {
491534 if (window.craft && window.craft.getDeviceInfo) return Promise.resolve(true)
492535 if (attempt >= 150) return Promise.resolve(false)
@@ -517,6 +560,7 @@
517560 emit({ event: 'done', passed: passed, failed: failed })
518561 document.getElementById('headline').textContent = failed ? 'FAILED' : 'PASSED'
519562 probeDeepLinks()
563 probeNotifications()
520564 return
521565 }
522566
scripts/mobile-e2e/ios-uitests/NotificationTapColdStartTests.swiftadded+67-0
@@ -0,0 +1,67 @@
1import XCTest
2
3/// Taps a pushed notification that launches the probe app, and reports what
4/// its page was handed (#255).
5///
6/// A person's sequence, end to end: the app asks for permission and is
7/// allowed, the app is killed, a real push arrives through `simctl push`, and
8/// the banner is tapped. `simctl` runs on the host, so the test prints
9/// `CRAFT-E2E-PUSH-READY` once the app is gone and the harness sends the push
10/// when it reads that line.
11///
12/// As with the deep-link test, no assertions about the answer live here. The
13/// page's report is printed and `scripts/mobile-e2e/protocol.ts` judges it.
14final class NotificationTapColdStartTests: XCTestCase {
15 private let environment = ProcessInfo.processInfo.environment
16 private var bundleId: String { environment["PROBE_BUNDLE_ID"] ?? "" }
17 private var permissionLink: String { environment["PROBE_NOTIFY_LINK"] ?? "" }
18 private var pushBody: String { environment["PROBE_PUSH_BODY"] ?? "" }
19
20 func testTapOnAPushLaunchesTheApp() throws {
21 XCTAssertFalse(bundleId.isEmpty, "PROBE_BUNDLE_ID is not set")
22 XCTAssertFalse(permissionLink.isEmpty, "PROBE_NOTIFY_LINK is not set")
23 XCTAssertFalse(pushBody.isEmpty, "PROBE_PUSH_BODY is not set")
24
25 let app = XCUIApplication(bundleIdentifier: bundleId)
26 let springboard = XCUIApplication(bundleIdentifier: "com.apple.springboard")
27
28 // Permission, answered the way a person answers it. Without it iOS
29 // accepts the push and shows nothing, so there is no banner to tap.
30 app.terminate()
31 XCUIDevice.shared.system.open(URL(string: permissionLink)!)
32 let open = springboard.buttons["Open"]
33 if open.waitForExistence(timeout: 5) { open.tap() }
34 XCTAssertTrue(app.wait(for: .runningForeground, timeout: 60), "the permission link did not start the app")
35
36 let allow = springboard.alerts.buttons["Allow"]
37 XCTAssertTrue(allow.waitForExistence(timeout: 60), "the page never asked for notification permission")
38 allow.tap()
39 let permission = app.staticTexts
40 .containing(NSPredicate(format: "label BEGINSWITH 'CRAFT-E2E-NOTIFY-PERMISSION '"))
41 .firstMatch
42 XCTAssertTrue(permission.waitForExistence(timeout: 30), "the page never said how the permission request ended")
43 XCTAssertEqual(permission.label, "CRAFT-E2E-NOTIFY-PERMISSION granted")
44
45 // Killed, so the tap below is what starts it. A running app would be
46 // handed the tap by a Coordinator that already exists, which is the
47 // case that always worked.
48 app.terminate()
49 XCTAssertTrue(app.wait(for: .notRunning, timeout: 15), "the app did not stop, so this would not be a cold start")
50 print("CRAFT-E2E-PUSH-READY")
51
52 // The banner, found by this run's body so a notification left over
53 // from an earlier run cannot be the one tapped.
54 let banner = springboard.descendants(matching: .any)
55 .matching(NSPredicate(format: "label CONTAINS %@", pushBody))
56 .firstMatch
57 XCTAssertTrue(banner.waitForExistence(timeout: 120), "no banner for the push ever appeared")
58 banner.tap()
59 XCTAssertTrue(app.wait(for: .runningForeground, timeout: 60), "tapping the banner did not start the app")
60
61 let report = app.staticTexts
62 .containing(NSPredicate(format: "label BEGINSWITH 'CRAFT-E2E-NOTIFICATION '"))
63 .firstMatch
64 XCTAssertTrue(report.waitForExistence(timeout: 120), "the page never reported what it was handed")
65 print("CRAFT-E2E-NOTIFICATION-RESULT \(report.label)")
66 }
67}
scripts/mobile-e2e/ios.tsmodified+64-9
@@ -2,7 +2,7 @@ import type { LegOutcome, RunnerOptions } from './types'
22import { copyFileSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
33import { join } from 'node:path'
44import { bootSimulator, init, pickSimulator } from '../../packages/ios/src/index'
5import { deepLinkProblems, deepLinkResults, evaluateRun, hasTerminated, ZIG_REFUSED_ACTIONS, ZIG_SERVED_ACTIONS, ZIG_TESTED_ACTIONS, zigDispatchedActions, zigHandBacks, zigRefusals } from './protocol'
5import { deepLinkProblems, deepLinkResults, evaluateRun, hasTerminated, notificationPayload, notificationTapProblems, notificationTapReport, ZIG_REFUSED_ACTIONS, ZIG_SERVED_ACTIONS, ZIG_TESTED_ACTIONS, zigDispatchedActions, zigHandBacks, zigRefusals } from './protocol'
66import { command, driverPage, waitForFile } from './support'
77
88/** The scheme the probe registers, and the one its cold-start link uses. */
@@ -56,12 +56,25 @@ const APP_NAME = 'CraftE2EProbe'
5656 * The UI test target the harness adds to the generated project.
5757 *
5858 * Added here rather than by the generator, because no app wants it: it exists
59 * to cold-start the probe through a link, which is the one thing neither
60 * `simctl launch` nor `simctl openurl` can do unattended. See
61 * `ios-uitests/DeepLinkColdStartTests.swift` for why.
59 * to cold-start the probe through a link, and through a tap on a pushed
60 * notification, which neither `simctl launch` nor `simctl openurl` can do
61 * unattended. See `ios-uitests/` for why.
6262 */
6363const UI_TESTS = `${APP_NAME}UITests`
6464
65/** The UI test classes, each run on its own with `-only-testing`. */
66const UI_TEST_CLASSES = ['DeepLinkColdStartTests', 'NotificationTapColdStartTests'] as const
67
68/**
69 * How long the notification UI test gets to have the app killed and ready for
70 * a push, and then to tap it. It starts a test runner, launches the app twice
71 * and answers a permission prompt, so it is given more than one launch.
72 */
73const NOTIFICATION_TEST_TIMEOUT_MS = 300_000
74
75/** The line the notification UI test prints once the app is gone. */
76const PUSH_READY = /^CRAFT-E2E-PUSH-READY\s*$/m
77
6578/**
6679 * One leg is one answer to "who served the call".
6780 *
@@ -114,9 +127,10 @@ async function runLeg(leg: Leg, options: RunnerOptions): Promise<LegOutcome> {
114127 if (!leg.requireZigDispatch && linksRuntime)
115128 failures.push(`the ${leg.name} leg was supposed to have no runtime but its project links one`)
116129
117 // The cold-start UI test, as a target in the generated project.
130 // The cold-start UI tests, as a target in the generated project.
118131 mkdirSync(join(project, 'UITests'), { recursive: true })
119 copyFileSync(join(import.meta.dir, 'ios-uitests', 'DeepLinkColdStartTests.swift'), join(project, 'UITests', 'DeepLinkColdStartTests.swift'))
132 for (const testClass of UI_TEST_CLASSES)
133 copyFileSync(join(import.meta.dir, 'ios-uitests', `${testClass}.swift`), join(project, 'UITests', `${testClass}.swift`))
120134 writeFileSync(join(project, 'project.yml'), `${projectYml.trimEnd()}
121135 ${UI_TESTS}:
122136 type: bundle.ui-testing
@@ -279,16 +293,18 @@ schemes:
279293 // #198: cold-start the app through a link, twice, and judge what the page
280294 // received. After the suite and its evidence, because the UI test
281295 // terminates and relaunches the app.
282 const link = `${DEEP_LINK_SCHEME}://e2e/cold?run=${encodeURIComponent(nonce)}`
283 const coldStart = await command([
296 const uiTest = (testClass: typeof UI_TEST_CLASSES[number]) => [
284297 'xcodebuild',
285298 '-project', `${APP_NAME}.xcodeproj`,
286299 '-scheme', APP_NAME,
287300 '-destination', `id=${device.udid}`,
288301 '-derivedDataPath', derived,
302 `-only-testing:${UI_TESTS}/${testClass}`,
289303 'CODE_SIGNING_ALLOWED=NO',
290304 'test-without-building',
291 ], {
305 ]
306 const link = `${DEEP_LINK_SCHEME}://e2e/cold?run=${encodeURIComponent(nonce)}`
307 const coldStart = await command(uiTest('DeepLinkColdStartTests'), {
292308 cwd: project,
293309 // xcodebuild hands TEST_RUNNER_-prefixed variables to the runner without
294310 // the prefix.
@@ -300,6 +316,45 @@ schemes:
300316 await command(['xcrun', 'simctl', 'terminate', device.udid, BUNDLE_ID], { allowFailure: true })
301317 failures.push(...deepLinkProblems(deepLinkResults(`${coldStart.stdout}\n${coldStart.stderr}`), link))
302318
319 // #255: a tap on a push that launches the app, judged by what the page was
320 // handed. The UI test allows notifications, kills the app and prints
321 // CRAFT-E2E-PUSH-READY; `simctl push` runs on the host, so the push is sent
322 // from here when that line appears, and the test taps the banner.
323 //
324 // Spawned rather than run through `command`, which returns only once the
325 // process has exited, and this one waits on the push.
326 const payloadPath = join(evidence, 'notification.apns')
327 writeFileSync(payloadPath, notificationPayload(nonce))
328 const notificationLog = join(evidence, 'xcodebuild-notification.log')
329 const tapping = Bun.spawn(uiTest('NotificationTapColdStartTests'), {
330 cwd: project,
331 env: {
332 ...process.env,
333 TEST_RUNNER_PROBE_BUNDLE_ID: BUNDLE_ID,
334 TEST_RUNNER_PROBE_NOTIFY_LINK: `${DEEP_LINK_SCHEME}://e2e/notify?run=${encodeURIComponent(nonce)}`,
335 TEST_RUNNER_PROBE_PUSH_BODY: nonce,
336 },
337 stdin: 'ignore',
338 stdout: Bun.file(notificationLog),
339 stderr: Bun.file(join(evidence, 'xcodebuild-notification-stderr.log')),
340 })
341 const readyForPush = await waitForFile(
342 notificationLog,
343 NOTIFICATION_TEST_TIMEOUT_MS,
344 text => PUSH_READY.test(text) || tapping.exitCode !== null,
345 ) && PUSH_READY.test(readFileSync(notificationLog, 'utf8'))
346 if (readyForPush)
347 await command(['xcrun', 'simctl', 'push', device.udid, BUNDLE_ID, payloadPath], { logPath: join(evidence, 'simctl.log') })
348 const timer = setTimeout(() => tapping.kill(), NOTIFICATION_TEST_TIMEOUT_MS)
349 await tapping.exited
350 clearTimeout(timer)
351 await command(['xcrun', 'simctl', 'io', device.udid, 'screenshot', join(evidence, 'notification-screen.png')], { allowFailure: true })
352 await command(['xcrun', 'simctl', 'terminate', device.udid, BUNDLE_ID], { allowFailure: true })
353 if (!readyForPush)
354 failures.push(`the notification UI test never had the app killed and ready for a push; see ${label}/xcodebuild-notification.log`)
355 else
356 failures.push(...notificationTapProblems(notificationTapReport(readFileSync(notificationLog, 'utf8')), nonce, `${label}/xcodebuild-notification.log`))
357
303358 const dispatched = zigDispatchedActions(consoleText)
304359 const refused = zigRefusals(consoleText)
305360 const handedBack = zigHandBacks(consoleText)
scripts/mobile-e2e/protocol.test.tsmodified+50-1
@@ -1,7 +1,7 @@
11import { describe, expect, it } from 'bun:test'
22import { readFileSync } from 'node:fs'
33import { join } from 'node:path'
4import { ANDROID_DECLINE_PHRASES, androidDeclines, awaitedNeeds, deepLinkProblems, deepLinkReports, deepLinkResults, DISMISS_SHARE_MENU, elfSectionNames, evaluateRun, hasTerminated, parseDriverOutput, REQUIRED_CASES, requiredCaseProblems, runtimePermissionGranted, shareMenuInFront, strippedLibraryProblems, ZIG_REFUSED_ACTIONS, ZIG_SERVED_ACTIONS, ZIG_TESTED_ACTIONS, zigDispatchedActions, zigHandBacks, zigRefusals } from './protocol'
4import { ANDROID_DECLINE_PHRASES, androidDeclines, awaitedNeeds, deepLinkProblems, deepLinkReports, deepLinkResults, DISMISS_SHARE_MENU, elfSectionNames, evaluateRun, hasTerminated, NOTIFICATION_ROUTE, notificationPayload, notificationTapProblems, notificationTapReport, parseDriverOutput, REQUIRED_CASES, requiredCaseProblems, runtimePermissionGranted, shareMenuInFront, strippedLibraryProblems, ZIG_REFUSED_ACTIONS, ZIG_SERVED_ACTIONS, ZIG_TESTED_ACTIONS, zigDispatchedActions, zigHandBacks, zigRefusals } from './protocol'
55
66const ESC = String.fromCharCode(27)
77
@@ -382,6 +382,55 @@ describe('cold-start deep links', () => {
382382 })
383383})
384384
385describe('notification tap cold start', () => {
386 const RUN = 'craft-e2e-ios-shim-1'
387 const line = (report: object) => `CRAFT-E2E-NOTIFICATION-RESULT CRAFT-E2E-NOTIFICATION ${JSON.stringify(report)}`
388 const tap = { aps: { alert: { title: 'Craft E2E', body: RUN } }, craftE2E: RUN, route: NOTIFICATION_ROUTE }
389 const judge = (report: object) => notificationTapProblems(notificationTapReport(`Test Case started.\n${line(report)}\n** TEST SUCCEEDED **`), RUN)
390
391 it('passes a late subscriber handed the tap that launched the app, once', () => {
392 expect(judge({ dispatched: 1, taps: [tap], onTap: 'function', unsubscribe: 'function' })).toEqual([])
393 })
394
395 // The two halves of #255, told apart.
396 it('blames native code when the app never dispatched the tap', () => {
397 expect(judge({ dispatched: 0, taps: [], onTap: 'function', unsubscribe: 'function' }))
398 .toEqual(['the app never dispatched the tap that launched it; native code dropped it before the page existed'])
399 })
400
401 it('blames the page when the tap was dispatched before anyone subscribed', () => {
402 const problems = judge({ dispatched: 1, taps: [], onTap: 'function', unsubscribe: 'function' })
403 expect(problems).toHaveLength(1)
404 expect(problems[0]).toContain('subscribed to notifications.onTap after launch was handed []')
405 })
406
407 it('fails a tap handed over twice, one from another run, or one that lost its route', () => {
408 expect(judge({ dispatched: 1, taps: [tap, tap], onTap: 'function', unsubscribe: 'function' })[0]).toContain('expected the tap that launched the app once')
409 expect(judge({ dispatched: 1, taps: [{ ...tap, craftE2E: 'an-earlier-run' }], onTap: 'function', unsubscribe: 'function' })[0]).toContain('expected the tap that launched the app once')
410 expect(judge({ dispatched: 1, taps: [{ ...tap, route: undefined }], onTap: 'function', unsubscribe: 'function' })[0]).toContain('without the push\'s route')
411 })
412
413 it('fails a bridge with no onTap, or one that returns no way to unsubscribe', () => {
414 expect(judge({ dispatched: 1, taps: [], onTap: 'undefined', unsubscribe: null }))
415 .toEqual(['notifications.onTap is undefined, so a page has no way to be handed a tap'])
416 expect(judge({ dispatched: 1, taps: [tap], onTap: 'function', unsubscribe: 'undefined' }))
417 .toEqual(['notifications.onTap returned undefined, expected a function that unsubscribes'])
418 })
419
420 it('names a run that never reported', () => {
421 expect(notificationTapProblems(notificationTapReport('** TEST FAILED **'), RUN, 'ios-shim/xcodebuild-notification.log'))
422 .toEqual(['the notification tap cold start never reported; see ios-shim/xcodebuild-notification.log'])
423 })
424
425 it('sends a push simctl accepts, whose body is the run the banner is found by', () => {
426 const payload = notificationPayload(RUN)
427 expect(new TextEncoder().encode(payload).length).toBeLessThanOrEqual(4096)
428 const parsed = JSON.parse(payload)
429 expect(parsed.aps.alert.body).toBe(RUN)
430 expect(parsed).toMatchObject({ craftE2E: RUN, route: NOTIFICATION_ROUTE })
431 })
432})
433
385434describe('cold-start deep links on Android', () => {
386435 const LINK = 'crafte2eprobe://e2e/cold?run=craft-e2e-android-shim-1'
387436 const report = { launch: `${LINK}&receive=subscribe`, receive: 'subscribe', onLink: [{ url: `${LINK}&receive=subscribe`, initial: true }], initialURL: null, unsubscribe: 'function' }
scripts/mobile-e2e/protocol.tsmodified+83-0
@@ -438,6 +438,89 @@ export function deepLinkProblems(
438438 return problems
439439}
440440
441/**
442 * Where the pushed notification says to go, as a custom key beside `aps`.
443 *
444 * The part of a push an app routes on, and the part the tap has to carry
445 * intact: WildLoop opens the plant camera off a key like this one.
446 */
447export const NOTIFICATION_ROUTE = '/e2e/notification-tap'
448
449/**
450 * The push the harness sends with `simctl push` once the app is killed.
451 *
452 * The body is the run's nonce, which the UI test finds the banner by, and
453 * `craftE2E` carries it again as data, for the page to hand back.
454 */
455export function notificationPayload(run: string): string {
456 return `${JSON.stringify({ aps: { alert: { title: 'Craft E2E', body: run } }, craftE2E: run, route: NOTIFICATION_ROUTE })}\n`
457}
458
459/** What a page that subscribed late was handed after a tap launched the app. */
460export interface NotificationTapReport {
461 /** craftNotificationResponse events native code dispatched, counted from the page's first line. */
462 dispatched: number
463 /** What notifications.onTap handed a subscriber that arrived after the suite. */
464 taps: Record<string, unknown>[]
465 /** `typeof` notifications.onTap. */
466 onTap: string
467 /** `typeof` what onTap returned, or null when there was no onTap to call. */
468 unsubscribe: string | null
469}
470
471/**
472 * The page's report the UI test printed, read out of xcodebuild's output.
473 *
474 * `ios-uitests/NotificationTapColdStartTests.swift` writes one
475 * `CRAFT-E2E-NOTIFICATION-RESULT CRAFT-E2E-NOTIFICATION {json}` line. None,
476 * or one whose JSON does not parse, is null: the test never got that far.
477 */
478export function notificationTapReport(output: string): NotificationTapReport | null {
479 for (const line of output.split('\n')) {
480 const match = line.match(/CRAFT-E2E-NOTIFICATION-RESULT CRAFT-E2E-NOTIFICATION (\{.*\})\s*$/)
481 if (!match) continue
482 try {
483 const report = JSON.parse(match[1]!) as NotificationTapReport
484 if (Array.isArray(report.taps)) return report
485 }
486 catch {}
487 }
488 return null
489}
490
491/**
492 * Why the notification tap cold start is not a pass, one line per reason.
493 *
494 * #255 lost a tap in two places, and the report tells them apart: nothing
495 * dispatched is the app dropping it before any page existed, dispatched but
496 * not handed to onTap is the page flushing it before anyone subscribed.
497 */
498export function notificationTapProblems(
499 report: NotificationTapReport | null,
500 run: string,
501 evidence = 'xcodebuild-notification.log',
502): string[] {
503 if (!report)
504 return [`the notification tap cold start never reported; see ${evidence}`]
505 if (report.onTap !== 'function')
506 return [`notifications.onTap is ${report.onTap}, so a page has no way to be handed a tap`]
507
508 const problems: string[] = []
509 if (report.unsubscribe !== 'function')
510 problems.push(`notifications.onTap returned ${report.unsubscribe}, expected a function that unsubscribes`)
511 if (report.dispatched === 0) {
512 problems.push('the app never dispatched the tap that launched it; native code dropped it before the page existed')
513 return problems
514 }
515
516 const ours = report.taps.filter(tap => tap.craftE2E === run)
517 if (ours.length !== 1 || report.taps.length !== 1)
518 problems.push(`a page that subscribed to notifications.onTap after launch was handed ${JSON.stringify(report.taps)}, expected the tap that launched the app once`)
519 else if (ours[0]!.route !== NOTIFICATION_ROUTE)
520 problems.push(`the tap reached the page without the push's route: it carried ${JSON.stringify(ours[0])}`)
521 return problems
522}
523
441524/** The need the Android share case announces before it opens the menu. */
442525export const DISMISS_SHARE_MENU = 'dismiss-share-menu'
443526