The Data Collector — Web Scraping API

{
  "name": "The Data Collector \u2014 Web Scraping API",
  "description": "Web scraping APIs for Bluesky (AT Protocol), Substack newsletters, and Hacker News. Search by keyword, get structured data with engagement metrics. Supports x402 micropayments ($0.05/call) or API key auth. Pre-built scrapers also available on Apify Store.",
  "url": "https://frog03-20494.wykr.es",
  "protocolVersion": "0.3.0",
  "preferredTransport": "JSONRPC",
  "did": "did:web:frog03-20494.wykr.es",
  "a2a_endpoint": "https://frog03-20494.wykr.es/a2a",
  "version": "1.2.0",
  "defaultInputModes": [
    "text/plain",
    "application/json"
  ],
  "defaultOutputModes": [
    "application/json"
  ],
  "capabilities": {
    "streaming": false,
    "pushNotifications": false,
    "stateTransitionHistory": false
  },
  "skills": [
    {
      "id": "bluesky-search",
      "name": "Bluesky Posts Search",
      "description": "Search Bluesky posts by keyword using the AT Protocol. Returns post text, author handle, engagement metrics (likes, reposts, replies), timestamps.",
      "tags": [
        "bluesky",
        "social-media",
        "scraping",
        "at-protocol"
      ],
      "examples": [
        "Search for posts about Bitcoin on Bluesky",
        "Find trending AI discussions on Bluesky"
      ],
      "inputModes": [
        "application/json"
      ],
      "outputModes": [
        "application/json"
      ],
      "endpoint": "/api/bluesky/search",
      "method": "POST",
      "inputSchema": {
        "type": "object",
        "properties": {
          "searchQuery": {
            "type": "string",
            "description": "Search query for Bluesky posts"
          },
          "maxItems": {
            "type": "integer",
            "default": 10,
            "description": "Max posts to return"
          },
          "scrapeType": {
            "type": "string",
            "default": "posts",
            "description": "Type of content to scrape"
          }
        },
        "required": [
          "searchQuery"
        ]
      }
    },
    {
      "id": "substack-search",
      "name": "Substack Newsletter Scraper",
      "description": "Scrape articles from any Substack newsletter. Returns full text, titles, dates, and metadata.",
      "tags": [
        "substack",
        "newsletter",
        "scraping",
        "content"
      ],
      "examples": [
        "Scrape latest articles from a crypto Substack",
        "Get newsletter content from specific publications"
      ],
      "inputModes": [
        "application/json"
      ],
      "outputModes": [
        "application/json"
      ],
      "endpoint": "/api/substack/search",
      "method": "POST",
      "inputSchema": {
        "type": "object",
        "properties": {
          "publications": {
            "type": "array",
            "items": {
              "type": "string"
            },
            "description": "Substack publication slugs"
          },
          "maxArticles": {
            "type": "integer",
            "default": 5,
            "description": "Max articles to return"
          }
        },
        "required": [
          "publications"
        ]
      }
    },
    {
      "id": "hn-search",
      "name": "Hacker News Search",
      "description": "Search Hacker News stories and discussions. Returns titles, URLs, scores, comment counts, and author info.",
      "tags": [
        "hacker-news",
        "tech-news",
        "scraping",
        "search"
      ],
      "examples": [
        "Search Hacker News for AI startup discussions",
        "Find top-scoring posts about Python"
      ],
      "inputModes": [
        "application/json"
      ],
      "outputModes": [
        "application/json"
      ],
      "endpoint": "/api/hn/search",
      "method": "POST",
      "inputSchema": {
        "type": "object",
        "properties": {
          "searchTerms": {
            "type": "array",
            "items": {
              "type": "string"
            },
            "description": "Keywords to search for"
          },
          "maxResults": {
            "type": "integer",
            "default": 10,
            "description": "Max stories to return"
          }
        },
        "required": [
          "searchTerms"
        ]
      }
    }
  ],
  "authentication": {
    "schemes": [
      "apiKey",
      "x402"
    ],
    "apiKey": {
      "headerName": "X-API-Key",
      "description": "API key for authenticated access."
    },
    "x402": {
      "description": "x402 micropayment protocol. Send POST without API key to receive 402 with payment requirements. Pay $0.05 USDC on Base per call.",
      "network": "base",
      "chainId": "eip155:8453",
      "asset": "USDC",
      "assetAddress": "0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913",
      "payTo": "0xa0dad465cdcedc8ecb99ec5ea0cb7f405a4c6eb6",
      "facilitatorUrl": "https://facilitator.xpay.sh",
      "pricePerCall": "$0.05",
      "priceAtomic": "50000"
    }
  },
  "identityUrl": "https://frog03-20494.wykr.es/identity",
  "serviceEndpoints": {
    "bluesky_search": "https://frog03-20494.wykr.es/api/bluesky/search",
    "substack_search": "https://frog03-20494.wykr.es/api/substack/search",
    "hn_search": "https://frog03-20494.wykr.es/api/hn/search",
    "hn_trending_free": "https://frog03-20494.wykr.es/api/hn/trending",
    "mcp": "https://frog03-20494.wykr.es/mcp",
    "health": "https://frog03-20494.wykr.es/health",
    "identity": "https://frog03-20494.wykr.es/identity",
    "agent_card": "https://frog03-20494.wykr.es/.well-known/agent-card.json",
    "mcp_config": "https://frog03-20494.wykr.es/.well-known/mcp.json",
    "x402_discovery": "https://frog03-20494.wykr.es/.well-known/x402"
  },
  "provider": {
    "name": "The Data Collector",
    "organization": "AI Hustler",
    "url": "https://frog03-20494.wykr.es"
  }
}