{"record":{"id":"e7da545f2535f922","repo":"unclecode/crawl4ai","slug":"too-many-webhook-headers","errorCode":null,"errorMessage":"too many webhook headers","messagePattern":"too many webhook headers","errorType":"validation","errorClass":"ValueError","httpStatus":null,"severity":"warning","filePath":"deploy/docker/webhook.py","lineNumber":68,"sourceCode":"# Webhook request-header policy: user-controlled outbound headers could inject\n# hop-by-hop / smuggling headers or CRLF. Allow only well-formed names, reject\n# control chars in values, and deny sensitive/hop-by-hop names.\n_WEBHOOK_HEADER_NAME = re.compile(r\"^[A-Za-z0-9-]{1,64}$\")\n_WEBHOOK_DENY_HEADERS = {\n    \"host\", \"content-length\", \"transfer-encoding\", \"connection\",\n    \"content-type\", \"proxy-authorization\", \"authorization\", \"cookie\",\n    \"expect\", \"upgrade\", \"te\", \"trailer\",\n}\n_MAX_WEBHOOK_HEADERS = 20\n_MAX_WEBHOOK_HEADER_VALUE = 2048\n\n\ndef sanitize_webhook_headers(headers: Optional[Dict[str, str]]) -> Dict[str, str]:\n    \"\"\"Validate user-supplied webhook headers; raise ValueError on any bad one.\"\"\"\n    if not headers:\n        return {}\n    if len(headers) > _MAX_WEBHOOK_HEADERS:\n        raise ValueError(\"too many webhook headers\")\n    clean: Dict[str, str] = {}\n    for name, value in headers.items():\n        if not isinstance(name, str) or not _WEBHOOK_HEADER_NAME.match(name):\n            raise ValueError(f\"invalid webhook header name: {name!r}\")\n        if name.lower() in _WEBHOOK_DENY_HEADERS:\n            raise ValueError(f\"webhook header not allowed: {name}\")\n        sval = str(value)\n        if len(sval) > _MAX_WEBHOOK_HEADER_VALUE or any(c in sval for c in \"\\r\\n\\x00\"):\n            raise ValueError(f\"invalid value for webhook header {name}\")\n        clean[name] = sval\n    return clean\n\n\nclass WebhookDeliveryService:\n    \"\"\"Handles webhook delivery with exponential backoff retry logic.\"\"\"\n\n    def __init__(self, config: Dict):\n        \"\"\"","sourceCodeStart":50,"sourceCodeEnd":86,"githubUrl":"https://github.com/unclecode/crawl4ai/blob/7e801521428ee12509994d39151006f64055ebe3/deploy/docker/webhook.py#L50-L86","documentation":"ValueError from sanitize_webhook_headers when the user-supplied webhook headers dict exceeds _MAX_WEBHOOK_HEADERS (20) entries. A hard cap to bound request size and header-forwarding work.","triggerScenarios":"POST /crawl with a webhook.headers object containing more than 20 keys.","commonSituations":"Forwarding an entire outbound request's header set (cookies, tracing, auth, telemetry) verbatim as webhook headers; programmatically generated header dicts (one per locale/feature flag) exceeding the cap.","solutions":["Trim the headers to the <=20 the receiver actually needs (usually just authorization + content-type + an idempotency key)","If more metadata must travel, put it in the webhook payload body, not headers"],"exampleFix":"# before\n\"headers\": dict(all_outbound_headers)  # 30+ entries\n\n# after\n\"headers\": {\n    \"authorization\": all_outbound_headers[\"Authorization\"],\n    \"x-request-id\": all_outbound_headers[\"X-Request-ID\"],\n}","handlingStrategy":"validation","validationCode":"assert isinstance(headers, dict) and 0 < len(headers) <= 20, f\"webhook headers must be 1..20 entries, got {len(headers)}\"","typeGuard":"def is_valid_webhook_headers(h) -> bool:\n    return isinstance(h, dict) and len(h) <= 20","tryCatchPattern":null,"preventionTips":["Whitelist the 2-3 headers your receiver needs instead of forwarding everything","Move bulk metadata into the webhook payload body"],"tags":["crawl4ai","webhook","validation","limits"],"backgroundTag":null,"analyzedSha":"7e801521428ee12509994d39151006f64055ebe3","analyzedAt":"2026-08-14T20:46:20.673Z","schemaVersion":2},"datasetVersion":"2026-08-15T22:17:37.221Z"}