Web Scraping Start Crawl API

# POST /v1/web/crawl: Metered: holds 10 credits per call up front and refunds what the call does not use. Cache hits are free and failed calls are refunded.
# Check the price first for 0 credits: add dry_run=1 to the query. Nothing is fetched or charged.
# --max-time 60: the API's own time limit for this endpoint.
# --retry 3: retries timeouts, 429 and 5xx, honoring Retry-After.
# -o response.json keeps only the final attempt's body.
# One Idempotency-Key per request, reused on every retry, so a retried call is never charged twice.
IDEMPOTENCY_KEY=$(od -An -N16 -tx1 /dev/urandom | tr -d ' \n')
curl -sS --max-time 60 --retry 3 -o response.json -X POST \
  -H "x-api-key: YOUR_API_KEY" \
  -H "Idempotency-Key: $IDEMPOTENCY_KEY" \
  -H "Content-Type: application/json" \
  -d '{"url":"https://example.com","limit":10}' \
  "https://www.socialcrawl.dev/v1/web/crawl" && cat response.json

# POST /v1/web/crawl: Metered: holds 10 credits per call up front and refunds what the call does not use. Cache hits are free and failed calls are refunded.
# Check the price first for 0 credits: add dry_run=1 to the query. Nothing is fetched or charged.
import os
import random
import time
import uuid

import requests

API_KEY = os.environ.get("SOCIALCRAWL_API_KEY", "YOUR_API_KEY")
TIMEOUT_S = 60  # the API's own time limit for this endpoint.
MAX_RETRIES = 3
URL = "https://www.socialcrawl.dev/v1/web/crawl"


def backoff(attempt):
    # Exponential backoff with jitter: about 1 s, 2 s, then 4 s.
    return 2 ** attempt * (0.5 + random.random() / 2)


def socialcrawl(payload):
    # One Idempotency-Key per request, reused on every retry, so a retried call is never charged twice.
    idempotency_key = str(uuid.uuid4())
    for attempt in range(MAX_RETRIES + 1):
        try:
            res = requests.post(
                URL,
                json=payload,
                headers={"x-api-key": API_KEY, "Idempotency-Key": idempotency_key},
                timeout=TIMEOUT_S,
            )
            body = res.json()
        except (requests.RequestException, ValueError):
            # A timeout, a dropped connection or a non-JSON gateway page: retry with the same key.
            if attempt == MAX_RETRIES:
                raise
            time.sleep(backoff(attempt))
            continue
        if body["success"]:
            return body
        error = body["error"]
        # Branch on error["type"], never on the message text.
        if error["type"] == "RESOURCE_NOT_FOUND":
            return None  # refunded: nothing to read
        retry_after = res.headers.get("Retry-After")
        # SERVICE_UNAVAILABLE without Retry-After is a withdrawn endpoint: permanent.
        permanent = error["type"] == "SERVICE_UNAVAILABLE" and retry_after is None
        if not error.get("retryable") or permanent or attempt == MAX_RETRIES:
            raise RuntimeError(f"{error['type']}: {error['message']} (request_id {body['request_id']})")
        # Retry-After (seconds) wins; otherwise back off.
        time.sleep(float(retry_after) if retry_after else backoff(attempt))


payload = {
    "url": "https://example.com",
    "limit": 10,
}
body = socialcrawl(payload)
if body is not None:
    print(body["data"])

// POST /v1/web/crawl: Metered: holds 10 credits per call up front and refunds what the call does not use. Cache hits are free and failed calls are refunded.
// Check the price first for 0 credits: add dry_run=1 to the query. Nothing is fetched or charged.
const API_KEY = process.env.SOCIALCRAWL_API_KEY ?? "YOUR_API_KEY";
const TIMEOUT_MS = 60_000; // the API's own time limit for this endpoint.
const MAX_RETRIES = 3;

type Page = { next_cursor: string | null; has_more: boolean };
type SocialCrawlResponse =
  | { success: true; data: any; pagination?: Page; credits_used: number; request_id: string }
  | { success: false; error: { type: string; message: string; retryable: boolean }; request_id: string };

const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
// Exponential backoff with jitter: about 1 s, 2 s, then 4 s.
const backoffMs = (attempt: number) => 2 ** attempt * 1000 * (0.5 + Math.random() / 2);

async function socialcrawl(url: string, payload: Record<string, unknown>) {
  // One Idempotency-Key per request, reused on every retry, so a retried call is never charged twice.
  const idempotencyKey = crypto.randomUUID();
  for (let attempt = 0; ; attempt++) {
    let res: Response;
    let body: SocialCrawlResponse;
    try {
      res = await fetch(url, {
        method: "POST",
        headers: {
          "x-api-key": API_KEY,
          "Content-Type": "application/json",
          "Idempotency-Key": idempotencyKey,
        },
        body: JSON.stringify(payload),
        signal: AbortSignal.timeout(TIMEOUT_MS),
      });
      body = (await res.json()) as SocialCrawlResponse;
    } catch (error) {
      // A timeout, a dropped connection or a non-JSON gateway page: retry with the same key.
      if (attempt >= MAX_RETRIES) throw error;
      await sleep(backoffMs(attempt));
      continue;
    }
    if (body.success) return body;
    // Branch on error.type, never on the message text.
    if (body.error.type === "RESOURCE_NOT_FOUND") return null; // refunded: nothing to read
    const retryAfter = res.headers.get("Retry-After");
    // SERVICE_UNAVAILABLE without Retry-After is a withdrawn endpoint: permanent.
    const permanent = body.error.type === "SERVICE_UNAVAILABLE" && retryAfter === null;
    if (!body.error.retryable || permanent || attempt >= MAX_RETRIES) {
      throw new Error(`${body.error.type}: ${body.error.message} (request_id ${body.request_id})`);
    }
    // Retry-After (seconds) wins; otherwise back off.
    await sleep(retryAfter ? Number(retryAfter) * 1000 : backoffMs(attempt));
  }
}

const body = await socialcrawl("https://www.socialcrawl.dev/v1/web/crawl", {
  url: "https://example.com",
  limit: 10,
});
if (body) console.log(body.data);