{"record":{"id":"2df7d9d1a4ca7643","repo":"unclecode/crawl4ai","slug":"http-result-status-code-error-for-url-result-u","errorCode":null,"errorMessage":"HTTP {result.status_code} error for URL '{result.url}'","messagePattern":"HTTP (.+?) error for URL '(.+?)'","errorType":"exception","errorClass":"Exception","httpStatus":null,"severity":"error","filePath":"crawl4ai/extraction_strategy.py","lineNumber":1857,"sourceCode":"            urls = [url] if isinstance(url, str) else url\n\n            async with AsyncWebCrawler(config=browser_config) as crawler:\n                if len(urls) == 1:\n                    result = await crawler.arun(url=urls[0], config=crawler_config)\n                    if not result.success:\n                        raise Exception(f\"Failed to fetch URL '{urls[0]}': {result.error_message}\")\n                    if result.status_code >= 400:\n                        raise Exception(f\"HTTP {result.status_code} error for URL '{urls[0]}'\")\n                    html = result.html\n                    original_htmls = [result.html]\n                else:\n                    results = await crawler.arun_many(urls=urls, config=crawler_config)\n                    html_parts = []\n                    for i, result in enumerate(results, 1):\n                        if not result.success:\n                            raise Exception(f\"Failed to fetch URL '{result.url}': {result.error_message}\")\n                        if result.status_code >= 400:\n                            raise Exception(f\"HTTP {result.status_code} error for URL '{result.url}'\")\n                        original_htmls.append(result.html)\n                        cleaned = preprocess_html_for_schema(\n                            html_content=result.html,\n                            text_threshold=2000,\n                            attr_value_threshold=500,\n                            max_size=500_000\n                        )\n                        header = HTML_EXAMPLE_DELIMITER.format(index=i)\n                        html_parts.append(f\"{header}\\n{cleaned}\")\n                    html = \"\\n\\n\".join(html_parts)\n        else:\n            original_htmls = [html]\n\n        # Preprocess HTML for schema generation (skip if already preprocessed from multiple URLs)\n        if url is None or isinstance(url, str):\n            html = preprocess_html_for_schema(\n                html_content=html,\n                text_threshold=2000,","sourceCodeStart":1839,"sourceCodeEnd":1875,"githubUrl":"https://github.com/unclecode/crawl4ai/blob/7e801521428ee12509994d39151006f64055ebe3/crawl4ai/extraction_strategy.py#L1839-L1875","documentation":"Multi-URL variant of the HTTP-status check in generate_schema: during arun_many, a result has success=True but status_code >= 400, and the loop aborts with the failing URL and status. Like its single-URL twin, any one erroring URL stops schema generation for the entire list.","triggerScenarios":"Calling generate_schema(url=[...]) where any URL answers 403/404/429/5xx to the headless browser; expired sitemap entries; endpoints that reject headless clients.","commonSituations":"Bulk schema generation from sitemaps or category-page lists containing dead links; rate-limited targets returning 429 under the concurrency of arun_many; sites returning 403 without browser-like headers.","solutions":["Pre-screen every URL's status code with HEAD requests and drop >= 400 before the call","Reduce concurrency or add delays so the target does not answer 429","Pass locally saved HTML for problematic pages via html= / curated URL lists instead"],"exampleFix":"// before\nschema = await JsonElementExtractionStrategy.generate_schema(url=sitemap_urls)\n# HTTP 403 error for URL '...'\n\n// after\nimport httpx\nasync with httpx.AsyncClient(follow_redirects=True) as hc:\n    ok = [u for u in sitemap_urls\n          if (await hc.head(u)).status_code < 400]\nschema = await JsonElementExtractionStrategy.generate_schema(url=ok)","handlingStrategy":"validation","validationCode":"import httpx\n\nasync def status_ok(u: str) -> bool:\n    try:\n        r = await httpx.AsyncClient(follow_redirects=True).head(u, timeout=10)\n        return r.status_code < 400\n    except httpx.HTTPError:\n        return False\n\nurls = [u for u in urls if await status_ok(u)]","typeGuard":null,"tryCatchPattern":"try:\n    schema = await JsonElementExtractionStrategy.generate_schema(url=urls)\nexcept Exception as e:\n    if \"error for URL\" in str(e):  # HTTP nnn branch\n        bad = extract_url_from_error(str(e))\n        schema = await JsonElementExtractionStrategy.generate_schema(\n            url=[u for u in urls if u != bad])\n    raise","preventionTips":["Filter URLs with status >= 400 before batch generation","Watch for 429 under arun_many concurrency — throttle or add delays","Refresh URL lists periodically; dead links accumulate in scraped sources"],"tags":["extraction","schema-generation","http-status","batch"],"backgroundTag":null,"analyzedSha":"7e801521428ee12509994d39151006f64055ebe3","analyzedAt":"2026-08-14T20:46:20.673Z","schemaVersion":2},"datasetVersion":"2026-08-15T22:17:37.221Z"}