{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://socialcrawl.dev/schemas/web-page.json",
  "title": "SocialCrawl Web page",
  "description": "Canonical unified web page object returned by every SocialCrawl endpoint with the WebPage/WebPageList archetype. Nullability is expressed via JSON Schema 2020-12 type arrays. Generated from the Zod schema in packages/social-api/src/schemas/canonical.ts.",
  "type": "object",
  "properties": {
    "url": {
      "type": [
        "string",
        "null"
      ],
      "description": "Requested URL."
    },
    "final_url": {
      "type": [
        "string",
        "null"
      ],
      "description": "Final resolved URL after redirects."
    },
    "status_code": {
      "type": [
        "number",
        "null"
      ],
      "description": "HTTP status code observed while fetching the page."
    },
    "scrape_id": {
      "type": [
        "string",
        "null"
      ],
      "description": "Opaque scrape identifier for follow-up interactions."
    },
    "fetched_at": {
      "type": [
        "string",
        "null"
      ],
      "description": "Fetch timestamp when reported."
    },
    "published_at": {
      "type": [
        "string",
        "null"
      ],
      "description": "When the page was published: an ISO 8601 instant, or a YYYY-MM-DD date when only the day is printed. From the page's own article:published_time meta tag on a scrape, and from the news date or that tag on a search row. Null when the page states none."
    },
    "content": {
      "type": "object",
      "description": "Extracted page content.",
      "properties": {
        "markdown": {
          "type": [
            "string",
            "null"
          ],
          "description": "Markdown body."
        },
        "html": {
          "type": [
            "string",
            "null"
          ],
          "description": "HTML body."
        },
        "raw_html": {
          "type": [
            "string",
            "null"
          ],
          "description": "Raw HTML body."
        },
        "summary": {
          "type": [
            "string",
            "null"
          ],
          "description": "Generated summary."
        }
      }
    },
    "media": {
      "type": "object",
      "description": "Short-lived media URLs returned for the page.",
      "properties": {
        "screenshot_url": {
          "type": [
            "string",
            "null"
          ],
          "description": "Screenshot URL when requested."
        },
        "audio_url": {
          "type": [
            "string",
            "null"
          ],
          "description": "Audio URL."
        },
        "video_url": {
          "type": [
            "string",
            "null"
          ],
          "description": "Video URL."
        }
      }
    },
    "extraction": {
      "type": [
        "object",
        "null"
      ],
      "description": "Structured extraction result when requested. On `web/extract` every value is checked against the page text: a placeholder such as 'Not specified', or a value the page does not state, comes back as null and is listed in `extraction_check.removed`."
    },
    "extraction_check": {
      "type": "object",
      "description": "`web/extract` only. How each extracted value was checked against the page text. A value is null in `extraction` when it was a placeholder or the page does not state it; `removed` lists those fields with the value the extractor gave, so nothing is hidden. A value kept without a check (the check was unavailable, or the page is too long to check whole) is named in `unverified` and in `_warnings`.",
      "properties": {
        "checked": {
          "type": "integer",
          "description": "Non-null values examined."
        },
        "found_in_text": {
          "type": "integer",
          "description": "Values kept because they appear word for word on the page."
        },
        "confirmed": {
          "type": "integer",
          "description": "Values kept because a check found the page states them, for example in another language."
        },
        "removed": {
          "type": "array",
          "description": "Fields set to null, with the value the extractor gave.",
          "items": {
            "type": "object",
            "description": "One field set to null.",
            "properties": {
              "field": {
                "type": "string",
                "description": "Path of the field, e.g. `faq[2].answer`."
              },
              "value": {
                "description": "The value the extractor gave."
              },
              "reason": {
                "type": "string",
                "enum": [
                  "placeholder",
                  "not_on_page"
                ],
                "description": "`placeholder`: the value holds no answer, either a marker such as 'Not specified' or '/', or a bare link path such as `/sub/info.php` in a field that does not ask for a link. `not_on_page`: the page does not state it."
              }
            }
          }
        },
        "unverified": {
          "type": "array",
          "description": "Fields returned as extracted without a check.",
          "items": {
            "type": "string",
            "description": "Path of the field."
          }
        }
      }
    },
    "fetch": {
      "type": "object",
      "description": "Vendor-neutral fetch provenance.",
      "properties": {
        "cache_state": {
          "type": [
            "string",
            "null"
          ],
          "description": "Upstream cache state when reported."
        },
        "cached_at": {
          "type": [
            "string",
            "null"
          ],
          "description": "Upstream cache timestamp when reported."
        },
        "proxy_tier": {
          "type": [
            "string",
            "null"
          ],
          "description": "Proxy tier used for the fetch."
        }
      }
    }
  },
  "additionalProperties": true
}