Data Quality Gate

{
  "name": "Data Quality Gate",
  "description": "Deterministic post-scrape data cleaner and quality gate for AI agents. It repairs how data was ENCODED -- residual HTML tags and entities, mojibake (\"Caf\u00c3\u00a9\" for \"Caf\u00e9\"), zero-width and invisible characters, non-breaking spaces, stray whitespace -- and never touches what the data SAYS: a negative price or an out-of-range rating is reported, never rewritten. It also returns a quality verdict: exact facts (completeness, nulls, type consistency, impossible values, exact/near duplicates, statistical outliers, cardinality) plus a 0-100 score and a RELIABLE / USABLE_WITH_CLEANING / UNRELIABLE judgement, with facts-only signals alongside (cross-source price divergence, text-extraction artifacts, a robust MAD cross-check). 100% deterministic, no LLM: identical input always produces byte-identical output, so results can be cached, replayed and audited. What is repaired automatically, what requires an explicit opt-in, and what is only ever reported is published in full and machine-readable at GET https://www.aidatatools.dev/api/clean -- readable before paying. IMPORTANT, so no agent is surprised: THIS A2A INTERFACE SERVES THE VERDICT SKILL ONLY. Repair is available over plain REST (POST https://www.aidatatools.dev/api/clean, $0.04 via x402; /api/clean/audit adds a replayable, reversible ledger, $0.12) and is discoverable over MCP at https://www.aidatatools.dev/api/mcp_server.",
  "version": "0.2.0",
  "provider": {
    "organization": "aidatatools",
    "url": "https://github.com/aidatatools-dev"
  },
  "documentationUrl": "https://www.aidatatools.dev/llms-full.txt",
  "supportedInterfaces": [
    {
      "url": "https://www.aidatatools.dev/api/a2a",
      "protocolBinding": "JSONRPC",
      "protocolVersion": "1.0"
    }
  ],
  "capabilities": {
    "streaming": false,
    "pushNotifications": false,
    "extendedAgentCard": false
  },
  "defaultInputModes": [
    "application/json"
  ],
  "defaultOutputModes": [
    "application/json"
  ],
  "skills": [
    {
      "id": "check_dataset_quality",
      "name": "Check Dataset Quality",
      "description": "Deterministically verifies the reliability of a tabular JSON dataset before an agent acts on it. Runs 8 checks -- structural homogeneity, completeness, null rate, type consistency, impossible/out-of-range values, exact and near (fuzzy) duplicate detection, statistical outliers (Tukey fence), and field cardinality -- and returns a transparent, recomputable 0-100 score plus a RELIABLE / USABLE_WITH_CLEANING / UNRELIABLE verdict with ranked reasons and a concrete cleanup recommendation. No LLM is involved: the same dataset always produces the exact same facts, score, and verdict. Three further signals report alongside the score without ever moving it. On financial/trading data -- a symbol/ticker/asset field paired with a price/cost/rate field -- it detects cross-source price divergence for the same entity (e.g. the same trading pair quoted very differently by two exchanges), grouped per entity rather than compared globally. On scraped or aggregated text it DETECTS extraction artifacts: leftover HTML and boilerplate, mojibake from wrong-codec decoding, invisible characters, and placeholders such as \"N/A\" or \"null\" that completeness counts as present and types counts as a valid string. And inside the outlier check it reports a robust median/MAD cross-check, surfacing anomalies the Tukey fence structurally cannot see once a cluster of corrupted values widens its bounds. All three are additional facts for review, deliberately not factored into score or verdict. Call it right after scraping, before loading data into a RAG pipeline, before a trading agent acts on aggregated market data, or whenever a dataset comes from an unverified source. Built to be called repeatedly -- once per batch -- as a recurring step in a pipeline, not a one-off check and not a real-time/streaming feed. SCOPE NOTE: this skill DETECTS those text artifacts; it does not repair them, and this A2A interface offers no repair skill. To get the repaired data back, call POST https://www.aidatatools.dev/api/clean over plain HTTP ($0.04 via x402, no account or signup) -- the response body is the cleaned dataset in the shape you posted it.",
      "tags": [
        "data quality",
        "data validation",
        "dataset validation",
        "reliability",
        "verification",
        "deterministic",
        "duplicates",
        "duplicate detection",
        "nulls",
        "missing values",
        "outliers",
        "anomaly detection",
        "data profiling",
        "scraper output validation",
        "post-scrape validation",
        "RAG pipeline guardrail",
        "pre-ingestion check",
        "per-batch validation",
        "pipeline quality gate",
        "recurring data check",
        "financial data validation",
        "trading data quality",
        "on-chain data verification",
        "pre-trade data check",
        "market data validation",
        "price divergence detection",
        "text quality",
        "encoding validation",
        "mojibake detection",
        "scraper artifact detection",
        "invisible character detection"
      ],
      "examples": [
        "Check this scraped product dataset before I load it into my pipeline",
        "Is this dataset reliable enough to use for analysis?",
        "Find duplicates, nulls, and outliers in this JSON dataset",
        "Give me a quality score and verdict for this data before I feed it to my agent",
        "Check this aggregated crypto price feed for cross-exchange divergence before I trade on it",
        "Does this scrape contain mojibake, leftover HTML or invisible characters?"
      ],
      "inputModes": [
        "application/json"
      ],
      "outputModes": [
        "application/json"
      ]
    }
  ],
  "x-additional-services": {
    "note": "Not A2A skills. Declared here for discovery only: these are plain-HTTP resources, not reachable over the A2A interface above. An A2A client must not attempt to invoke them as skills.",
    "clean": {
      "endpoint": "POST https://www.aidatatools.dev/api/clean",
      "price": "$0.04",
      "paidVia": "x402",
      "returns": "the repaired dataset as the response body, in the shape you posted",
      "summary": "Deterministic post-scrape repair: residual HTML stripped, mojibake decoded, invisible characters removed, non-breaking spaces normalised, values trimmed -- across nested objects and arrays. Repairs encoding, never meaning.",
      "tags": [
        "data cleaning",
        "scraping repair",
        "post-scrape sanitization",
        "deterministic data repair",
        "mojibake correction",
        "encoding repair",
        "html stripping",
        "unicode normalization",
        "scraper output cleaning"
      ]
    },
    "cleanAudit": {
      "endpoint": "POST https://www.aidatatools.dev/api/clean/audit",
      "price": "$0.12",
      "paidVia": "x402",
      "returns": "the same repaired dataset plus a complete, replayable, reversible ledger of every transformation (path, rule, before, after) with a replay_id and input/output SHA-256",
      "summary": "For when you must be able to prove later what changed and why. Applying the ledger in reverse reconstructs the input byte for byte.",
      "tags": [
        "auditable data repair",
        "reversible data cleaning",
        "data lineage",
        "compliance data cleaning",
        "replayable transformation log"
      ]
    },
    "repairBoundary": {
      "endpoint": "GET https://www.aidatatools.dev/api/clean",
      "price": "free",
      "returns": "all 20 repair rules and 7 engine invariants, machine-readable, with the reasoning for each",
      "summary": "Auditable before payment. 7 rules applied automatically (information-preserving), 5 requiring an explicit opt-in (they change row count, type or schema), and 8 that are only ever reported with a proposal and that no option can turn on -- near-duplicate merging, ambiguous placeholder nulling, full NFKC, and failed extractions such as captcha or access-denied pages, which are reported rather than deleted because that value tells you the record must be re-scraped."
    },
    "mcp": {
      "endpoint": "https://www.aidatatools.dev/api/mcp_server",
      "transport": "Streamable HTTP (MCP)",
      "tools": {
        "check_dataset_quality": "free -- returns the full verdict",
        "clean_scraped_data": "does NOT return the repaired data over MCP: it reports which rules would change how many values, then returns the paid REST call ($0.04) that hands back the repaired artifact",
        "clean_scraped_data_audited": "same, pointing at the audited route ($0.12)"
      }
    }
  }
}