Smoke tests in CI

#!/usr/bin/env bash
# Needs: curl, jq. Env: WEBAGENT_API_KEY, TARGET_URL. Optional: API_URL, EXPECT_TEXT.
set -euo pipefail

API_URL="${API_URL:-https://api.us.webagent.lookover.ai}"
auth=(-H "Authorization: Bearer ${WEBAGENT_API_KEY}")

started=$(curl -sS --fail-with-body -X POST "$API_URL/v1/skills/smoke-uat/run" \
  "${auth[@]}" -H "Content-Type: application/json" \
  -d "$(jq -n --arg url "$TARGET_URL" --arg text "${EXPECT_TEXT:-}" \
        '{inputs: ({url: $url} + (if $text != "" then {expectText: $text} else {} end)), session: {record: true}}')")
run_id=$(jq -r .runId <<<"$started")
echo "run: $run_id"

read_report() { curl -sS --fail-with-body "$API_URL/v1/skill-runs/$run_id/report.json" "${auth[@]}"; }

# Wait up to 10 minutes for a verdict.
for _ in $(seq 1 120); do
  report=$(read_report)
  verdict=$(jq -r .verdict <<<"$report")
  [ "$verdict" != "running" ] && break
  sleep 5
done

# The session log is saved a moment after the run ends: read once more if it is not there yet.
if [ "$verdict" != "running" ] && [ "$(jq -r '.evidence.logsPath // empty' <<<"$report")" = "" ]; then
  sleep 5
  report=$(read_report)
fi

echo "verdict: $verdict"
jq -r '.summary' <<<"$report"
jq -r '.steps[] | select(.status == "fail") | "failed step \(.index + 1) \(.title): \(.error // "no reason given")"' <<<"$report"
jq -r '.consoleErrors[] | "console: \(.text)"' <<<"$report"
jq -r '.networkFailures[] | "network: \(.method // "") \(.url) \(.status // .error // "")"' <<<"$report"

# The report link lives for 5 minutes: save the file now. A failed download must not hide the verdict.
link=$(jq -r '.evidence.reportSignedUrl // empty' <<<"$report")
if [ -n "$link" ]; then
  curl -sS -o smoke-report.html "$link" || echo "could not save the HTML report" >&2
fi

case "$verdict" in
  pass) exit 0 ;;
  fail) exit 1 ;;
  running) echo "still running after 10 minutes" >&2; exit 3 ;;
  *) exit 2 ;;
esac

{
  "inputs": {
    "url": "https://staging.example.com/login",
    "steps": ["type \"[email protected]\" into Email", "type the test password into Password", "click Sign in"],
    "expectText": "Welcome",
    "expectUrl": "/dashboard",
    "expectNoText": "Invalid password"
  }
}

name: Smoke test
on:
  workflow_dispatch:
  push:
    branches: [main]

jobs:
  smoke:
    runs-on: ubuntu-latest
    timeout-minutes: 15
    steps:
      - uses: actions/checkout@v4
      - name: Run the smoke test
        run: bash scripts/smoke.sh
        env:
          WEBAGENT_API_KEY: ${{ secrets.WEBAGENT_API_KEY }}
          TARGET_URL: https://staging.example.com
      - name: Keep the report
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: smoke-report
          path: smoke-report.html
          if-no-files-found: ignore

import { WebagentClient } from "@lookover-ai/webagent-sdk";

const client = new WebagentClient({
  apiKey: process.env.WEBAGENT_API_KEY,
  // baseUrl: "https://...",  // set this when your API host is not the one of your key's region
});

const { runId } = await client.runSkill("smoke-uat", {
  inputs: { url: process.env.TARGET_URL, expectText: "Welcome" },
  session: { record: true },
});
await client.waitForSkillRun(runId, { timeoutMs: 600_000 });
// The session log is saved a moment after the run ends: wait before reading console and network errors.
await new Promise((r) => setTimeout(r, 5_000));
const report = await client.getSkillRunReport(runId);

console.log(report.verdict, report.summary);
for (const s of report.steps.filter((s) => s.status === "fail")) console.log("failed:", s.title, s.error);
for (const e of report.consoleErrors) console.log("console:", e.text);
for (const n of report.networkFailures) console.log("network:", n.method, n.url, n.status ?? n.error);
process.exit(report.verdict === "pass" ? 0 : report.verdict === "fail" ? 1 : 2);