Skip to content

fix(tests): Fix test timeouts and add early failure detection #21

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
abueide merged 1 commit into from
Apr 21, 2026
Merged

fix(tests): Fix test timeouts and add early failure detection #21

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
108 changes: 98 additions & 10 deletions examples/android/tests/test-suite.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,14 +87,31 @@ processes:
echo "[ANDROID-EMULATOR] Starting emulator..."
device="${ANDROID_DEFAULT_DEVICE:-max}"

# Capture emulator start result for better error reporting
start_exit=0
if [ "${DEVBOX_PURE_SHELL:-}" = "1" ]; then
echo "Starting fresh Android emulator (pure mode): $device"
android.sh emulator start --pure "$device"
android.sh emulator start --pure "$device" || start_exit=$?
else
echo "Starting Android emulator: $device"
unset ANDROID_EMULATOR_FOREGROUND
android.sh emulator start "$device"
android.sh emulator start "$device" || start_exit=$?
fi

if [ "$start_exit" -ne 0 ]; then
echo ""
echo "ERROR: Emulator start command failed with exit code $start_exit" >&2
echo ""
echo "Common causes:" >&2
echo " - Device '$device' not found (check ANDROID_DEVICES filter)" >&2
echo " - AVD creation failed (check sync-avds logs)" >&2
echo " - System image not available for device" >&2
echo ""
echo "Available AVDs:" >&2
avdmanager list avd 2>/dev/null || echo "(avdmanager not available)"
exit "$start_exit"
fi

# Keep process alive for readiness probe (process-compose services pattern)
tail -f /dev/null
availability:
Expand All @@ -120,9 +137,52 @@ processes:
sleep 5

echo "Verifying emulator is ready..."

# Early failure detection: Check if emulator process exists
echo "Checking if emulator process started..."
initial_wait=30
elapsed=0
emulator_process_found=false

while [ $elapsed -lt $initial_wait ]; do
if pgrep -f "emulator.*-avd" >/dev/null 2>&1; then
emulator_process_found=true
echo "✓ Emulator process detected"
break
fi
sleep 2
elapsed=$((elapsed + 2))
echo " Waiting for emulator process... ${elapsed}s/${initial_wait}s"
done

if [ "$emulator_process_found" = false ]; then
echo ""
echo "ERROR: Emulator process not found after ${initial_wait}s" >&2
echo ""
echo "This usually means:" >&2
echo " 1. Device filtering removed all devices (check sync-avds logs)" >&2
echo " 2. Emulator startup command failed (check android-emulator logs)" >&2
echo " 3. System images not available for selected device" >&2
echo ""
echo "Check process-compose logs above for error details" >&2
printf 'fail\nEmulator process never started - check filtering and device availability\n' > "$_step_dir/$_step.status"
exit 1
fi

# Now wait for emulator to be fully booted and ready
echo ""
echo "Emulator process running, waiting for device to be ready..."
max_wait=300
elapsed=0
while ! android.sh emulator ready 2>/dev/null; do
# Recheck that emulator process is still running
if ! pgrep -f "emulator.*-avd" >/dev/null 2>&1; then
echo ""
echo "ERROR: Emulator process terminated unexpectedly" >&2
printf 'fail\nEmulator process crashed during boot\n' > "$_step_dir/$_step.status"
exit 1
fi

sleep 3
elapsed=$((elapsed + 3))
echo " Waiting for emulator... ${elapsed}s/${max_wait}s"
Expand All @@ -149,6 +209,21 @@ processes:
_logs_dir="reports/logs"
mkdir -p "$_step_dir" "$_logs_dir"

# Check if emulator boot succeeded
if [ -f "$_step_dir/emulator-boot.status" ]; then
boot_status=$(head -n1 "$_step_dir/emulator-boot.status")
if [ "$boot_status" = "fail" ]; then
echo "ERROR: Emulator boot failed, cannot deploy" >&2
cat "$_step_dir/emulator-boot.status"
printf 'fail\nEmulator boot failed - skipping deploy\n' > "$_step_dir/$_step.status"
exit 1
fi
else
echo "ERROR: Emulator boot status not found" >&2
printf 'fail\nEmulator boot status missing\n' > "$_step_dir/$_step.status"
exit 1
fi

serial="${ANDROID_EMULATOR_SERIAL:-emulator-${EMU_PORT:-5554}}"

# Capture emulator state before deploy
Expand Down Expand Up @@ -292,28 +367,41 @@ processes:
build-app:
condition: process_completed_successfully
verify-emulator-ready:
condition: process_completed_successfully
condition: process_completed
availability:
restart: "no"

# Cleanup - capture final logs, then stop everything in pure mode
cleanup-app:
command: |
_step_dir="reports/steps"
_logs_dir="reports/logs"
mkdir -p "$_logs_dir"

# Capture final logcat before teardown
serial="${ANDROID_EMULATOR_SERIAL:-emulator-${EMU_PORT:-5554}}"
adb -s "$serial" logcat -d > "$_logs_dir/logcat-cleanup.txt" 2>&1 || true
echo "Final logcat: $_logs_dir/logcat-cleanup.txt"
# Check if deployment succeeded
deploy_failed=false
if [ -f "$_step_dir/deploy-app.status" ]; then
deploy_status=$(head -n1 "$_step_dir/deploy-app.status")
if [ "$deploy_status" = "fail" ]; then
deploy_failed=true
echo "Deploy failed, skipping adb operations"
fi
fi

# Only capture logs if deployment succeeded
if [ "$deploy_failed" = false ]; then
serial="${ANDROID_EMULATOR_SERIAL:-emulator-${EMU_PORT:-5554}}"
# Use timeout to prevent hanging
timeout 10s adb -s "$serial" logcat -d > "$_logs_dir/logcat-cleanup.txt" 2>&1 || true
echo "Final logcat: $_logs_dir/logcat-cleanup.txt"
fi

# Stop emulator (only in pure mode)
if [ "${DEVBOX_PURE_SHELL:-}" = "1" ]; then
echo "Cleaning up (pure mode)..."
android.sh app stop || true
android.sh emulator stop || true
echo "App stopped, emulator stopped"
else
echo "Test complete - app and emulator still running"
echo "Cleanup complete"
fi
depends_on:
deploy-app:
Expand Down
37 changes: 25 additions & 12 deletions examples/ios/tests/test-suite.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,7 @@ processes:
if ! ios.sh app status; then
echo "FAIL: App is not running"
if [ -n "$UDID" ]; then
xcrun simctl spawn "$UDID" log show --predicate 'eventMessage contains "crash" OR eventMessage contains "fatal" OR eventMessage contains "terminate"' --last 5m --style compact > "$_logs_dir/simulator-crash.log" 2>&1 || true
xcrun simctl spawn "$UDID" log show --predicate 'process == "ios" AND (eventMessage contains "crash" OR eventMessage contains "fatal" OR eventMessage contains "terminate")' --last 5m --style compact > "$_logs_dir/simulator-crash.log" 2>&1 || true
echo "Simulator crash logs: $_logs_dir/simulator-crash.log"
fi
printf 'fail\nApp not found in running processes\n' > "$_step_dir/$_step.status"
Expand All @@ -190,7 +190,7 @@ processes:
if ! ios.sh app status; then
echo "FAIL: App crashed during soak period"
if [ -n "$UDID" ]; then
xcrun simctl spawn "$UDID" log show --predicate 'eventMessage contains "crash" OR eventMessage contains "fatal" OR eventMessage contains "terminate"' --last 5m --style compact > "$_logs_dir/simulator-crash.log" 2>&1 || true
xcrun simctl spawn "$UDID" log show --predicate 'process == "ios" AND (eventMessage contains "crash" OR eventMessage contains "fatal" OR eventMessage contains "terminate")' --last 5m --style compact > "$_logs_dir/simulator-crash.log" 2>&1 || true
echo "Simulator crash logs: $_logs_dir/simulator-crash.log"
fi
printf 'fail\nApp crashed during soak period\n' > "$_step_dir/$_step.status"
Expand All @@ -199,9 +199,10 @@ processes:

# Capture simulator logs for analysis
if [ -n "$UDID" ]; then
xcrun simctl spawn "$UDID" log show --last 2m --style compact > "$_logs_dir/simulator-app.log" 2>&1 || true
# Filter logs to only our test app process to avoid false positives from system services
xcrun simctl spawn "$UDID" log show --last 2m --predicate 'process == "ios"' --style compact > "$_logs_dir/simulator-app.log" 2>&1 || true

# Scan for native crash patterns
# Scan for native crash patterns (only in our app's logs)
error_patterns="Terminating app|SIGABRT|EXC_BAD_ACCESS|EXC_CRASH|Assertion failure"
if grep -qiE "$error_patterns" "$_logs_dir/simulator-app.log" 2>/dev/null; then
echo ""
Expand All @@ -227,23 +228,35 @@ processes:
# Cleanup - capture final logs, then stop everything in pure mode
cleanup:
command: |
_step_dir="reports/steps"
_logs_dir="reports/logs"
mkdir -p "$_logs_dir"

# Capture final simulator logs before teardown
UDID=$(cat "${IOS_RUNTIME_DIR:-/tmp}/ios-e2e/simulator-udid.txt" 2>/dev/null || cat "${IOS_RUNTIME_DIR:-/tmp}/${SUITE_NAME:-ios-e2e}/simulator-udid.txt" 2>/dev/null || true)
if [ -n "$UDID" ]; then
xcrun simctl spawn "$UDID" log show --last 5m --style compact > "$_logs_dir/simulator-final.log" 2>&1 || true
echo "Final simulator logs: $_logs_dir/simulator-final.log"
# Check if app verification succeeded
app_verify_failed=false
if [ -f "$_step_dir/verify-app.status" ]; then
verify_status=$(head -n1 "$_step_dir/verify-app.status")
if [ "$verify_status" = "fail" ]; then
app_verify_failed=true
echo "App verification failed, skipping log capture"
fi
fi

# Only capture logs if verification succeeded
if [ "$app_verify_failed" = false ]; then
UDID=$(cat "${IOS_RUNTIME_DIR:-/tmp}/ios-e2e/simulator-udid.txt" 2>/dev/null || cat "${IOS_RUNTIME_DIR:-/tmp}/${SUITE_NAME:-ios-e2e}/simulator-udid.txt" 2>/dev/null || true)
if [ -n "$UDID" ]; then
timeout 10s xcrun simctl spawn "$UDID" log show --last 5m --predicate 'process == "ios"' --style compact > "$_logs_dir/simulator-final.log" 2>&1 || true
echo "Final simulator logs: $_logs_dir/simulator-final.log"
fi
fi

# Stop services (only in pure mode)
if [ "${DEVBOX_PURE_SHELL:-}" = "1" ]; then
echo "Cleaning up (pure mode)..."
ios.sh app stop || true
ios.sh simulator stop
echo "App stopped, test simulator deleted"
else
echo "Test complete - app and simulator still running"
echo "Cleanup complete"
fi
depends_on:
verify-app-running:
Expand Down
59 changes: 42 additions & 17 deletions examples/react-native/tests/test-suite-all-e2e.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -624,28 +624,55 @@ processes:
# Cleanup - capture final logs, then stop everything in pure mode
cleanup:
command: |
_step_dir="reports/steps"
_logs_dir="reports/logs"
mkdir -p "$_logs_dir"

# Capture final Android logcat before teardown
serial=""
state_dir="${ANDROID_RUNTIME_DIR:-.devbox/virtenv}/${SUITE_NAME:-default}"
if [ -f "$state_dir/emulator-serial.txt" ]; then
serial="$(cat "$state_dir/emulator-serial.txt")"
# Check if Android verification succeeded
android_verify_failed=false
if [ -f "$_step_dir/verify-android.status" ]; then
verify_status=$(head -n1 "$_step_dir/verify-android.status")
if [ "$verify_status" = "fail" ]; then
android_verify_failed=true
echo "Android verification failed, skipping Android adb operations"
fi
fi
if [ -z "$serial" ]; then
serial="emulator-${EMU_PORT:-5554}"

# Check if iOS verification succeeded
ios_verify_failed=false
if [ -f "$_step_dir/verify-ios.status" ]; then
verify_status=$(head -n1 "$_step_dir/verify-ios.status")
if [ "$verify_status" = "fail" ]; then
ios_verify_failed=true
echo "iOS verification failed, skipping iOS log capture"
fi
fi
adb -s "$serial" logcat -d > "$_logs_dir/logcat-android-cleanup.txt" 2>&1 || true
echo "Final Android logcat: $_logs_dir/logcat-android-cleanup.txt"

# Capture final iOS simulator logs before teardown
UDID=$(cat "${IOS_RUNTIME_DIR:-/tmp}/rn-all-e2e/simulator-udid.txt" 2>/dev/null || cat "${IOS_RUNTIME_DIR:-/tmp}/${SUITE_NAME:-rn-all-e2e}/simulator-udid.txt" 2>/dev/null || true)
if [ -n "$UDID" ]; then
xcrun simctl spawn "$UDID" log show --last 5m --style compact > "$_logs_dir/simulator-final.log" 2>&1 || true
echo "Final iOS simulator logs: $_logs_dir/simulator-final.log"
# Only capture Android logs if verification succeeded
if [ "$android_verify_failed" = false ]; then
serial=""
state_dir="${ANDROID_RUNTIME_DIR:-.devbox/virtenv}/${SUITE_NAME:-default}"
if [ -f "$state_dir/emulator-serial.txt" ]; then
serial="$(cat "$state_dir/emulator-serial.txt")"
fi
if [ -z "$serial" ]; then
serial="emulator-${EMU_PORT:-5554}"
fi
# Use timeout to prevent hanging
timeout 10s adb -s "$serial" logcat -d > "$_logs_dir/logcat-android-cleanup.txt" 2>&1 || true
echo "Final Android logcat: $_logs_dir/logcat-android-cleanup.txt"
fi

# Only capture iOS logs if verification succeeded
if [ "$ios_verify_failed" = false ]; then
UDID=$(cat "${IOS_RUNTIME_DIR:-/tmp}/rn-all-e2e/simulator-udid.txt" 2>/dev/null || cat "${IOS_RUNTIME_DIR:-/tmp}/${SUITE_NAME:-rn-all-e2e}/simulator-udid.txt" 2>/dev/null || true)
if [ -n "$UDID" ]; then
timeout 10s xcrun simctl spawn "$UDID" log show --last 5m --style compact > "$_logs_dir/simulator-final.log" 2>&1 || true
echo "Final iOS simulator logs: $_logs_dir/simulator-final.log"
fi
fi

# Stop services (only in pure mode)
if [ "${DEVBOX_PURE_SHELL:-}" = "1" ]; then
echo "Cleaning up (pure mode)..."
metro.sh stop all || true
Expand All @@ -658,9 +685,7 @@ processes:
ios.sh app stop || true
ios.sh simulator stop || true

echo "Metro, apps, and emulator/simulator stopped"
else
echo "Test complete - Metro, apps, and emulator/simulator still running for iteration"
echo "Cleanup complete"
fi
depends_on:
verify-android-running:
Expand Down
37 changes: 25 additions & 12 deletions examples/react-native/tests/test-suite-android-e2e.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -469,29 +469,42 @@ processes:
# Cleanup - capture final logs, then stop everything in pure mode
cleanup:
command: |
_step_dir="reports/steps"
_logs_dir="reports/logs"
mkdir -p "$_logs_dir"

# Capture final logcat before teardown
state_dir="${ANDROID_RUNTIME_DIR:-.devbox/virtenv}/${SUITE_NAME:-default}"
serial=""
if [ -f "$state_dir/emulator-serial.txt" ]; then
serial="$(cat "$state_dir/emulator-serial.txt")"
# Check if app verification succeeded
app_verify_failed=false
if [ -f "$_step_dir/verify-app.status" ]; then
verify_status=$(head -n1 "$_step_dir/verify-app.status")
if [ "$verify_status" = "fail" ]; then
app_verify_failed=true
echo "App verification failed, skipping adb operations"
fi
fi
if [ -z "$serial" ]; then
serial="emulator-${EMU_PORT:-5554}"

# Only capture logs if app verification succeeded
if [ "$app_verify_failed" = false ]; then
state_dir="${ANDROID_RUNTIME_DIR:-.devbox/virtenv}/${SUITE_NAME:-default}"
serial=""
if [ -f "$state_dir/emulator-serial.txt" ]; then
serial="$(cat "$state_dir/emulator-serial.txt")"
fi
if [ -z "$serial" ]; then
serial="emulator-${EMU_PORT:-5554}"
fi
# Use timeout to prevent hanging
timeout 10s adb -s "$serial" logcat -d > "$_logs_dir/logcat-cleanup.txt" 2>&1 || true
echo "Final logcat: $_logs_dir/logcat-cleanup.txt"
fi
adb -s "$serial" logcat -d > "$_logs_dir/logcat-cleanup.txt" 2>&1 || true
echo "Final logcat: $_logs_dir/logcat-cleanup.txt"

# Stop services (only in pure mode)
if [ "${DEVBOX_PURE_SHELL:-}" = "1" ]; then
echo "Cleaning up (pure mode)..."
metro.sh stop android || true
android.sh app stop || true
android.sh emulator stop || true
echo "Metro, app, and emulator stopped"
else
echo "Test complete - Metro, app, and emulator still running for iteration"
echo "Cleanup complete"
fi
depends_on:
verify-app-running:
Expand Down
Loading
Loading