Card snapshot

This is a frozen copy of the agent's agent-card.json as we observed it at the timestamp above. We capture a new snapshot every time the card's content hash changes. Useful for: forensic drift analysis, verifying downstream callers see the right version, reproducing routing decisions made historically.

{
  "name": "AI Crawler Index",
  "description": "Every AI crawler on the web, what it is for, what blocking it costs you, and the IP ranges its operator publishes \u2014 as JSON, CSV, robots.txt and regex. Ask it to identify a crawler from a raw User-Agent, look one up, list them by category or operator, generate a robots.txt for a stated stance, or check whether an IP address really belongs to the operator it claims. Deterministic and read-only: there is no model behind it \u2014 every answer comes from a public dataset rebuilt every six hours from each operator's own published documentation and IP ranges, and the same skills are also available as files under https://www.pathwren.workers.dev/ and as MCP tools at https://www.pathwren.workers.dev/mcp. No key, no signup, no quota. Independent and unaffiliated with any operator it documents.",
  "supportedInterfaces": [
    {
      "url": "https://www.pathwren.workers.dev/a2a",
      "protocolBinding": "JSONRPC",
      "protocolVersion": "1.0"
    }
  ],
  "url": "https://www.pathwren.workers.dev/a2a",
  "preferredTransport": "JSONRPC",
  "protocolVersion": "1.0",
  "provider": {
    "organization": "Pathwren",
    "url": "https://www.pathwren.workers.dev"
  },
  "version": "1.0.0",
  "documentationUrl": "https://www.pathwren.workers.dev/a2a.html",
  "iconUrl": "https://www.pathwren.workers.dev/icon.png",
  "capabilities": {
    "streaming": false,
    "pushNotifications": false,
    "extendedAgentCard": false,
    "extensions": [
      {
        "uri": "https://www.pathwren.workers.dev/changes.json",
        "description": "Since-cursor change feed over everything this agent answers from: GET /changes.json?since=<cursor> returns only what moved \u2014 operator IP-range lists that gained or lost prefixes, upstreams that failed or recovered, crawler records added or edited. Read `cursor` from the answer and send it back next time; it advances only on a real change, so an unchanged answer is proof and costs a few hundred bytes. The same feed is the changes_since skill on this endpoint.",
        "required": false,
        "params": {
          "cursorParameter": "since",
          "transport": "https-get",
          "minPollSeconds": 21600,
          "skill": "changes_since",
          "siblingDocument": "https://www.pathwren.workers.dev/data/agents.json"
        }
      }
    ]
  },
  "defaultInputModes": [
    "application/json",
    "text/plain"
  ],
  "defaultOutputModes": [
    "application/json",
    "text/plain"
  ],
  "skills": [
    {
      "id": "classify_user_agent",
      "name": "Classify a user-agent string",
      "description": "Identify the crawler behind a raw User-Agent header: operator, category, robots.txt stance, how to verify it, and what blocking it costs. A user-agent is a claim, not proof \u2014 confirm with is_verified_crawler_ip. Example: user_agent='GPTBot/1.2' returns GPTBot, OpenAI, ai-training.",
      "tags": [
        "user-agent",
        "crawlers",
        "bot detection",
        "identification"
      ],
      "examples": [
        "{\"skill\":\"classify_user_agent\",\"user_agent\":\"Mozilla/5.0 (compatible; GPTBot/1.2; +https://openai.com/gptbot)\"}",
        "classify_user_agent ClaudeBot/1.0"
      ],
      "inputModes": [
        "application/json",
        "text/plain"
      ],
      "outputModes": [
        "application/json",
        "text/plain"
      ]
    },
    {
      "id": "lookup_crawler",
      "name": "Look up one crawler",
      "description": "The full index record for one crawler: operator, category, robots.txt token and stance, user-agent substring and example, verification method, mirrored IP-range endpoint, and what blocking it costs. An unknown slug returns near matches. Example: slug='claudebot'.",
      "tags": [
        "crawlers",
        "reference",
        "robots.txt",
        "lookup"
      ],
      "examples": [
        "{\"skill\":\"lookup_crawler\",\"slug\":\"claudebot\"}",
        "lookup_crawler perplexitybot"
      ],
      "inputModes": [
        "application/json",
        "text/plain"
      ],
      "outputModes": [
        "application/json",
        "text/plain"
      ]
    },
    {
      "id": "list_crawlers",
      "name": "List and filter crawlers",
      "description": "The index, filtered or whole: each row says what the crawler is and what blocking it costs, and the reply lists every valid filter value. Example: category='ai-training', limit=20.",
      "tags": [
        "crawlers",
        "catalogue",
        "filter",
        "discovery"
      ],
      "examples": [
        "{\"skill\":\"list_crawlers\",\"category\":\"ai-training\",\"limit\":20}",
        "{\"skill\":\"list_crawlers\",\"operator\":\"openai\"}"
      ],
      "inputModes": [
        "application/json",
        "text/plain"
      ],
      "outputModes": [
        "application/json",
        "text/plain"
      ]
    },
    {
      "id": "generate_robots_txt",
      "name": "Generate a robots.txt for a stance",
      "description": "A ready-to-paste robots.txt block for one stance, with the reasoning and the crawlers it names. Omit the argument to get all eight stances described. Example: stance='block-ai-training'.",
      "tags": [
        "robots.txt",
        "policy",
        "configuration",
        "generator"
      ],
      "examples": [
        "{\"skill\":\"generate_robots_txt\",\"stance\":\"block-ai-training\"}",
        "generate a robots.txt for maximum-ai-visibility"
      ],
      "inputModes": [
        "application/json",
        "text/plain"
      ],
      "outputModes": [
        "application/json",
        "text/plain"
      ]
    },
    {
      "id": "is_verified_crawler_ip",
      "name": "Check an IP against operator-published crawler ranges",
      "description": "Test an IPv4 or IPv6 address against every crawler prefix the operators themselves publish, remirrored every six hours. Returns the matching prefix, its source and the crawlers using it; a miss is reported as a miss, not as a forgery. Example: ip='20.171.206.5'.",
      "tags": [
        "ip ranges",
        "verification",
        "bot detection",
        "security"
      ],
      "examples": [
        "{\"skill\":\"is_verified_crawler_ip\",\"ip\":\"20.171.206.5\"}",
        "is 20.171.206.5 a verified crawler"
      ],
      "inputModes": [
        "application/json",
        "text/plain"
      ],
      "outputModes": [
        "application/json",
        "text/plain"
      ]
    },
    {
      "id": "whats_changed",
      "name": "What changed in the index",
      "description": "Freshness and change state: when the index was built, when each operator source was last fetched, which changed, which are failing, and the dated changelog. The cron check before regenerating a config from this data. Example: since='2026-08-01'.",
      "tags": [
        "freshness",
        "changelog",
        "monitoring",
        "data quality"
      ],
      "examples": [
        "{\"skill\":\"whats_changed\",\"since\":\"2026-08-01\"}",
        "what changed"
      ],
      "inputModes": [
        "application/json",
        "text/plain"
      ],
      "outputModes": [
        "application/json",
        "text/plain"
      ]
    },
    {
      "id": "changes_since",
      "name": "What changed since your cursor",
      "description": "Only what changed since your last call: prefixes added or removed, upstreams failed or recovered, records edited. Send back the `cursor` from the previous result; it advances only on a real change, and the data refreshes every 6h. Example: since='0'.",
      "tags": [
        "changelog",
        "monitoring",
        "polling",
        "incremental sync"
      ],
      "examples": [
        "{\"skill\":\"changes_since\",\"since\":\"0\"}",
        "{\"skill\":\"changes_since\",\"since\":\"2026-09-01T00:00:00Z\",\"limit\":50}"
      ],
      "inputModes": [
        "application/json",
        "text/plain"
      ],
      "outputModes": [
        "application/json",
        "text/plain"
      ]
    }
  ]
}