{ "openapi": "3.1.0", "info": { "title": "RobotsGate API", "version": "0.2.9", "summary": "Generate, validate and check robots.txt rules for AI crawlers.", "description": "Free JSON API behind https://robotsgate.mike-tusa.workers.dev. No key needed: about 20 /api/check requests and 60 /api/generate + /api/validate requests per 60 s per client IP. RobotsGate Pro ($9/mo, https://buy.polar.sh/polar_cl_pJjxZL8dDfTebUMNAEOnWNscJgTjNZPWbiIln3H3k6l) raises them to 300 and 600; a missing or invalid key falls back to the free limits instead of failing. A 429 carries Retry-After. /api/check results are cached for 5 minutes per target. robots.txt is voluntary and is not access control. AI agents can also use the remote MCP server at https://robotsgate.mike-tusa.workers.dev/mcp (see /llms.txt). RobotsGate is not affiliated with Meta or any other crawler or agent operator.", "contact": { "name": "RobotsGate", "email": "digitalpromohub.support@gmail.com", "url": "https://robotsgate.mike-tusa.workers.dev" } }, "servers": [ { "url": "https://robotsgate.mike-tusa.workers.dev" } ], "externalDocs": { "description": "Human-readable API docs", "url": "https://robotsgate.mike-tusa.workers.dev/docs" }, "tags": [ { "name": "robots" }, { "name": "meta" }, { "name": "mcp" } ], "components": { "securitySchemes": { "bearerAuth": { "type": "http", "scheme": "bearer", "description": "Optional RobotsGate Pro license key (prefix `RBTG`). Invalid keys fall back to the free plan." }, "licenseKeyHeader": { "type": "apiKey", "in": "header", "name": "X-License-Key", "description": "Same license key as bearerAuth, alternative header." } }, "schemas": { "Error": { "type": "object", "required": [ "error" ], "properties": { "error": { "type": "object", "required": [ "code", "message" ], "properties": { "code": { "type": "string" }, "message": { "type": "string" }, "details": {} } } } }, "GenerateRequest": { "type": "object", "properties": { "categories": { "type": "object", "properties": { "training": { "type": "string", "enum": [ "allow", "block", "omit" ] }, "dataset": { "type": "string", "enum": [ "allow", "block", "omit" ] }, "search": { "type": "string", "enum": [ "allow", "block", "omit" ] }, "user_fetch": { "type": "string", "enum": [ "allow", "block", "omit" ] }, "other": { "type": "string", "enum": [ "allow", "block", "omit" ] } }, "additionalProperties": false }, "agents": { "type": "object", "maxProperties": 100, "additionalProperties": { "type": "string", "enum": [ "allow", "block", "omit" ] } }, "default": { "type": "string", "enum": [ "allow", "block" ] }, "disallow_paths": { "type": "array", "maxItems": 100, "items": { "type": "string", "maxLength": 512 } }, "allow_paths": { "type": "array", "maxItems": 100, "items": { "type": "string", "maxLength": 512 } }, "sitemaps": { "type": "array", "maxItems": 20, "items": { "type": "string", "maxLength": 2048 } }, "sitemap": { "type": "string", "maxLength": 2048 }, "custom_rules": { "type": "string", "maxLength": 20000 }, "date": { "type": "string", "pattern": "^\\d{4}-\\d{2}-\\d{2}$" } } }, "GenerateResult": { "type": "object", "required": [ "robots_txt", "summary", "notes", "custom_rules_diagnostics", "personal_agents_note" ], "properties": { "robots_txt": { "type": "string" }, "summary": { "type": "array", "items": { "type": "object" } }, "notes": { "type": "array", "items": { "type": "string" } }, "custom_rules_diagnostics": {}, "personal_agents_note": { "type": "string" } } }, "ValidateRequest": { "type": "object", "required": [ "robots_txt" ], "properties": { "robots_txt": { "type": "string", "description": "Up to 512000 bytes (UTF-8)" }, "path": { "type": "string", "maxLength": 2048, "pattern": "^/" } } }, "ValidateResult": { "type": "object", "required": [ "valid", "errors", "warnings", "info", "diagnostics", "stats", "path", "ai_crawlers", "personal_agents" ], "properties": { "valid": { "type": "boolean" }, "errors": { "type": "array", "items": { "type": "object" } }, "warnings": { "type": "array", "items": { "type": "object" } }, "info": { "type": "array", "items": { "type": "object" } }, "diagnostics": { "type": "object" }, "stats": { "type": "object" }, "path": { "type": "string" }, "ai_crawlers": { "type": "array", "items": { "type": "object" } }, "personal_agents": { "type": "object" } } }, "CheckResult": { "type": "object", "required": [ "url", "final_url", "http_status", "redirects", "fetched_at", "cache", "robots_found", "interpretation" ], "properties": { "url": { "type": "string" }, "final_url": { "type": "string" }, "http_status": { "type": "integer" }, "redirects": { "type": "integer" }, "fetched_at": { "type": "string", "format": "date-time" }, "cache": { "type": "object", "properties": { "hit": { "type": "boolean" }, "layer": { "type": [ "string", "null" ] }, "ttl_seconds": { "type": "integer" } } }, "robots_found": { "type": "boolean" }, "interpretation": { "type": "string", "enum": [ "parsed", "unavailable", "unreachable" ] }, "message": { "type": "string" }, "path": { "type": "string" }, "ai_crawlers": { "type": [ "array", "null" ] }, "personal_agents": { "type": [ "object", "null" ] } } }, "JsonRpcMessage": { "type": "object", "required": [ "jsonrpc" ], "properties": { "jsonrpc": { "const": "2.0" }, "id": { "type": [ "string", "integer" ] }, "method": { "type": "string" }, "params": { "type": "object" } } } } }, "paths": { "/api": { "get": { "tags": [ "meta" ], "operationId": "apiIndex", "summary": "API name, version, endpoint list and Pro details", "security": [ {} ], "responses": { "200": { "description": "Index", "content": { "application/json": { "schema": { "type": "object", "required": [ "name", "version", "endpoints" ], "properties": { "name": { "type": "string" }, "version": { "type": "string" }, "endpoints": { "type": "array", "items": { "type": "string" } }, "note": { "type": "string" }, "pro": { "type": "object" } } } } } } } } }, "/api/health": { "get": { "tags": [ "meta" ], "operationId": "health", "summary": "Liveness and deployed version", "security": [ {} ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "type": "object", "required": [ "ok", "version" ], "properties": { "ok": { "const": true }, "version": { "type": "string", "examples": [ "0.2.9" ] } } } } } }, "405": { "description": "`method_not_allowed`: use GET", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } } } }, "/api/crawlers": { "get": { "tags": [ "robots" ], "operationId": "listCrawlers", "summary": "The AI crawler registry", "security": [ {} ], "parameters": [ { "name": "category", "in": "query", "required": false, "schema": { "type": "string", "enum": [ "training", "dataset", "search", "user_fetch", "other" ] } } ], "responses": { "200": { "description": "Registry (cache-control: public, max-age=3600)", "content": { "application/json": { "schema": { "type": "object", "required": [ "categories", "crawlers" ], "properties": { "schema_version": { "type": "integer" }, "generated": { "type": "string" }, "categories": { "type": "object" }, "crawlers": { "type": "array", "items": { "type": "object" } }, "unverified": { "type": "array", "items": { "type": "object" } } } } } } }, "405": { "description": "`method_not_allowed`: use GET", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } } } }, "/api/generate": { "post": { "tags": [ "robots" ], "operationId": "generateRobots", "summary": "Generate a robots.txt for AI crawlers", "security": [ {}, { "bearerAuth": [] }, { "licenseKeyHeader": [] } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/GenerateRequest" } } } }, "responses": { "200": { "description": "Generated file", "headers": { "x-robotsgate-plan": { "description": "`pro` when a valid RobotsGate Pro license was sent (absent on the free plan)", "schema": { "type": "string", "enum": [ "pro" ] } }, "x-ratelimit-remaining": { "description": "Requests left in the current window. Only sent when the in-isolate limiter decided the request (Pro licenses, or if the Cloudflare rate-limit binding is unavailable); the normal free path does not send it.", "schema": { "type": "integer" } } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/GenerateResult" } } } }, "400": { "description": "`invalid_input` (see `details`) or `invalid_json`", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "405": { "description": "`method_not_allowed`: use POST", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "413": { "description": "`payload_too_large`: body over 1,000,000 bytes (or robots_txt over the limit)", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "415": { "description": "`unsupported_media_type`: Content-Type must be application/json", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "`rate_limited`: the shared generate/validate limit", "headers": { "Retry-After": { "description": "Seconds to wait before retrying", "schema": { "type": "integer" } } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "500": { "description": "`internal_error`", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } } } }, "/api/validate": { "post": { "tags": [ "robots" ], "operationId": "validateRobots", "summary": "Validate robots.txt text and test one path for every registry AI crawler", "security": [ {}, { "bearerAuth": [] }, { "licenseKeyHeader": [] } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ValidateRequest" } } } }, "responses": { "200": { "description": "Validation result", "headers": { "x-robotsgate-plan": { "description": "`pro` when a valid RobotsGate Pro license was sent (absent on the free plan)", "schema": { "type": "string", "enum": [ "pro" ] } }, "x-ratelimit-remaining": { "description": "Requests left in the current window. Only sent when the in-isolate limiter decided the request (Pro licenses, or if the Cloudflare rate-limit binding is unavailable); the normal free path does not send it.", "schema": { "type": "integer" } } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ValidateResult" } } } }, "400": { "description": "`invalid_input` (see `details`) or `invalid_json`", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "405": { "description": "`method_not_allowed`: use POST", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "413": { "description": "`payload_too_large`: body over 1,000,000 bytes (or robots_txt over the limit)", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "415": { "description": "`unsupported_media_type`: Content-Type must be application/json", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "422": { "description": "`too_complex`: the file needs more work than the per-request limit allows (`details.stage`)", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "`rate_limited`: the shared generate/validate limit", "headers": { "Retry-After": { "description": "Seconds to wait before retrying", "schema": { "type": "integer" } } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "500": { "description": "`internal_error`", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } } } }, "/api/check": { "get": { "tags": [ "robots" ], "operationId": "checkSite", "summary": "Fetch a site's /robots.txt and report which AI crawlers it allows", "security": [ {}, { "bearerAuth": [] }, { "licenseKeyHeader": [] } ], "parameters": [ { "name": "url", "in": "query", "required": true, "description": "Site URL or bare hostname. http/https on default ports only; private, loopback and internal hosts are refused.", "schema": { "type": "string", "maxLength": 2048 }, "example": "example.com" }, { "name": "path", "in": "query", "required": false, "description": "URL path to test (default \"/\").", "schema": { "type": "string", "maxLength": 2048, "pattern": "^/" } } ], "responses": { "200": { "description": "Check result (also for 4xx/5xx robots.txt responses: see `interpretation`)", "headers": { "x-robotsgate-cache": { "description": "`MISS`, `HIT-memory` or `HIT-cache-api`", "schema": { "type": "string", "enum": [ "MISS", "HIT-memory", "HIT-cache-api" ] } }, "x-robotsgate-plan": { "description": "`pro` when a valid RobotsGate Pro license was sent (absent on the free plan)", "schema": { "type": "string", "enum": [ "pro" ] } }, "x-ratelimit-remaining": { "description": "Requests left in the current window. Only sent when the in-isolate limiter decided the request (Pro licenses, or if the Cloudflare rate-limit binding is unavailable); the normal free path does not send it.", "schema": { "type": "integer" } } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CheckResult" } } } }, "400": { "description": "`url_not_allowed` (missing, invalid, non-http(s), credentials, non-default port, private or internal host) or `invalid_input` (bad path)", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "405": { "description": "`method_not_allowed`: use GET", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "422": { "description": "`too_complex`: the site's robots.txt needs more work than the per-request limit allows", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "`rate_limited`", "headers": { "Retry-After": { "description": "Seconds to wait before retrying", "schema": { "type": "integer" } } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "500": { "description": "`internal_error`", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "502": { "description": "`dns_check_failed`, `upstream_error` or `too_many_redirects`", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "504": { "description": "`upstream_timeout`: robots.txt took longer than 8 s", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } } } }, "/mcp": { "post": { "tags": [ "mcp" ], "operationId": "mcp", "summary": "Remote MCP server (Streamable HTTP, JSON-RPC 2.0)", "description": "Model Context Protocol endpoint. Stateless, JSON responses only (no SSE, no Mcp-Session-Id). Supported protocol versions: 2026-07-28, 2025-11-25, 2025-06-18, 2025-03-26, 2024-11-05. Modern (2026-07-28) requests carry params._meta and the MCP-Protocol-Version, Mcp-Method and Mcp-Name headers; legacy clients use initialize. Tools: list_crawlers, generate_robots, validate_robots, check_site. Same keys and limits as the JSON API (soft-fail: never 401). Body max 65536 bytes; one tools/call per HTTP request; legacy batches max 10 messages.", "security": [ {}, { "bearerAuth": [] }, { "licenseKeyHeader": [] } ], "parameters": [ { "name": "MCP-Protocol-Version", "in": "header", "required": false, "schema": { "type": "string", "enum": [ "2026-07-28", "2025-11-25", "2025-06-18", "2025-03-26", "2024-11-05" ] } }, { "name": "Mcp-Method", "in": "header", "required": false, "description": "Required for 2026-07-28; must equal the body method", "schema": { "type": "string" } }, { "name": "Mcp-Name", "in": "header", "required": false, "description": "Required for 2026-07-28 tools/call; must equal params.name", "schema": { "type": "string" } } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "oneOf": [ { "$ref": "#/components/schemas/JsonRpcMessage" }, { "type": "array", "maxItems": 10, "items": { "$ref": "#/components/schemas/JsonRpcMessage" } } ] } } } }, "responses": { "200": { "description": "JSON-RPC response (or batch of responses). Tool failures, including input errors and rate limits, are results with isError: true (Retry-After set on rate limits).", "headers": { "Retry-After": { "description": "Seconds to wait before retrying", "schema": { "type": "integer" } } }, "content": { "application/json": { "schema": { "type": [ "object", "array" ] } } } }, "202": { "description": "Notification(s) accepted; no body" }, "400": { "description": "Parse error (-32700), invalid request (-32600), header mismatch (-32020), unsupported protocol version (-32022) or missing _meta (-32602)", "content": { "application/json": { "schema": { "type": "object" } } } }, "403": { "description": "Origin header present but not allowed", "content": { "application/json": { "schema": { "type": "object" } } } }, "404": { "description": "Unknown method in a 2026-07-28 request (-32601)", "content": { "application/json": { "schema": { "type": "object" } } } }, "413": { "description": "Body larger than the cap", "content": { "application/json": { "schema": { "type": "object" } } } }, "429": { "description": "Too many non-check MCP messages from this IP", "headers": { "Retry-After": { "description": "Seconds to wait before retrying", "schema": { "type": "integer" } } }, "content": { "application/json": { "schema": { "type": "object" } } } }, "500": { "description": "Unexpected internal error (`internal_error`); per-message failures are JSON-RPC -32603 inside a 200 response instead", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } } }, "get": { "tags": [ "mcp" ], "operationId": "mcp_get_not_allowed", "summary": "Not supported (stateless server: no SSE stream, no sessions)", "responses": { "405": { "description": "Always 405 with `Allow: POST, OPTIONS` and a JSON-RPC error body", "headers": { "Allow": { "schema": { "type": "string", "const": "POST, OPTIONS" } } }, "content": { "application/json": { "schema": { "type": "object" } } } } } }, "delete": { "tags": [ "mcp" ], "operationId": "mcp_delete_not_allowed", "summary": "Not supported (stateless server: no SSE stream, no sessions)", "responses": { "405": { "description": "Always 405 with `Allow: POST, OPTIONS` and a JSON-RPC error body", "headers": { "Allow": { "schema": { "type": "string", "const": "POST, OPTIONS" } } }, "content": { "application/json": { "schema": { "type": "object" } } } } } } } } }