Web Scraping Run Agent API

# POST /v1/web/agent: 25 credits per call. Cache hits are free and failed calls are refunded.
# Check the price first for 0 credits: add dry_run=1 to the query. Nothing is fetched or charged.
# --max-time 60: the API's own time limit for this endpoint.
# This endpoint does not take an Idempotency-Key, so a call that timed out or dropped is not retried: it may have run.
curl -sS --max-time 60 -X POST \
  -H "x-api-key: YOUR_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{"url":"https://example.com","prompt":"Find the pricing page.","model":"spark-1-mini"}' \
  "https://www.socialcrawl.dev/v1/web/agent"

# POST /v1/web/agent: 25 credits per call. Cache hits are free and failed calls are refunded.
# Check the price first for 0 credits: add dry_run=1 to the query. Nothing is fetched or charged.
import os
import random
import time

import requests

API_KEY = os.environ.get("SOCIALCRAWL_API_KEY", "YOUR_API_KEY")
TIMEOUT_S = 60  # the API's own time limit for this endpoint.
MAX_RETRIES = 3
URL = "https://www.socialcrawl.dev/v1/web/agent"


def backoff(attempt):
    # Exponential backoff with jitter: about 1 s, 2 s, then 4 s.
    return 2 ** attempt * (0.5 + random.random() / 2)


def socialcrawl(payload):
    # This endpoint does not take an Idempotency-Key, so a call that timed out or dropped is not retried: it may have run.
    for attempt in range(MAX_RETRIES + 1):
        res = requests.post(
            URL,
            json=payload,
            headers={"x-api-key": API_KEY},
            timeout=TIMEOUT_S,
        )
        body = res.json()
        if body["success"]:
            return body
        error = body["error"]
        # Branch on error["type"], never on the message text.
        if error["type"] == "RESOURCE_NOT_FOUND":
            return None  # refunded: nothing to read
        retry_after = res.headers.get("Retry-After")
        # SERVICE_UNAVAILABLE without Retry-After is a withdrawn endpoint: permanent.
        permanent = error["type"] == "SERVICE_UNAVAILABLE" and retry_after is None
        if not error.get("retryable") or permanent or attempt == MAX_RETRIES:
            raise RuntimeError(f"{error['type']}: {error['message']} (request_id {body['request_id']})")
        # Retry-After (seconds) wins; otherwise back off.
        time.sleep(float(retry_after) if retry_after else backoff(attempt))


payload = {
    "url": "https://example.com",
    "prompt": "Find the pricing page.",
    "model": "spark-1-mini",
}
body = socialcrawl(payload)
if body is not None:
    print(body["data"])

// POST /v1/web/agent: 25 credits per call. Cache hits are free and failed calls are refunded.
// Check the price first for 0 credits: add dry_run=1 to the query. Nothing is fetched or charged.
const API_KEY = process.env.SOCIALCRAWL_API_KEY ?? "YOUR_API_KEY";
const TIMEOUT_MS = 60_000; // the API's own time limit for this endpoint.
const MAX_RETRIES = 3;

type Page = { next_cursor: string | null; has_more: boolean };
type SocialCrawlResponse =
  | { success: true; data: any; pagination?: Page; credits_used: number; request_id: string }
  | { success: false; error: { type: string; message: string; retryable: boolean }; request_id: string };

const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
// Exponential backoff with jitter: about 1 s, 2 s, then 4 s.
const backoffMs = (attempt: number) => 2 ** attempt * 1000 * (0.5 + Math.random() / 2);

async function socialcrawl(url: string, payload: Record<string, unknown>) {
  // This endpoint does not take an Idempotency-Key, so a call that timed out or dropped is not retried: it may have run.
  for (let attempt = 0; ; attempt++) {
    const res = await fetch(url, {
      method: "POST",
      headers: {
        "x-api-key": API_KEY,
        "Content-Type": "application/json",
      },
      body: JSON.stringify(payload),
      signal: AbortSignal.timeout(TIMEOUT_MS),
    });
    const body = (await res.json()) as SocialCrawlResponse;
    if (body.success) return body;
    // Branch on error.type, never on the message text.
    if (body.error.type === "RESOURCE_NOT_FOUND") return null; // refunded: nothing to read
    const retryAfter = res.headers.get("Retry-After");
    // SERVICE_UNAVAILABLE without Retry-After is a withdrawn endpoint: permanent.
    const permanent = body.error.type === "SERVICE_UNAVAILABLE" && retryAfter === null;
    if (!body.error.retryable || permanent || attempt >= MAX_RETRIES) {
      throw new Error(`${body.error.type}: ${body.error.message} (request_id ${body.request_id})`);
    }
    // Retry-After (seconds) wins; otherwise back off.
    await sleep(retryAfter ? Number(retryAfter) * 1000 : backoffMs(attempt));
  }
}

const body = await socialcrawl("https://www.socialcrawl.dev/v1/web/agent", {
  url: "https://example.com",
  prompt: "Find the pricing page.",
  model: "spark-1-mini",
});
if (body) console.log(body.data);