{"record":{"id":"154d7676b19271cf","repo":"vxcontrol/pentagi","slug":"failed-to-parse-html-w","errorCode":null,"errorMessage":"failed to parse HTML: %w","messagePattern":"failed to parse HTML: %w","errorType":"exception","errorClass":null,"httpStatus":null,"severity":"warning","filePath":"backend/pkg/tools/searchers/duckduckgo.go","lineNumber":261,"sourceCode":"\t}\n\n\t// Fallback to regex-based parsing\n\tresults, err = d.parseHTMLRegex(body)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn &searchResponse{\n\t\tResults:   results,\n\t\tNoResults: len(results) == 0,\n\t}, nil\n}\n\n// parseHTMLStructured uses golang.org/x/net/html for structured HTML parsing\nfunc (d *duckduckgo) parseHTMLStructured(body []byte) ([]searchResult, error) {\n\tdoc, err := html.Parse(strings.NewReader(string(body)))\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to parse HTML: %w\", err)\n\t}\n\n\tresults := make([]searchResult, 0)\n\td.findResultNodes(doc, &results)\n\n\treturn results, nil\n}\n\n// findResultNodes recursively finds and extracts search result nodes\nfunc (d *duckduckgo) findResultNodes(n *html.Node, results *[]searchResult) {\n\t// Look for div with class \"result results_links\"\n\tif n.Type == html.ElementNode && n.Data == \"div\" {\n\t\tif d.hasClass(n, \"result\") && d.hasClass(n, \"results_links\") {\n\t\t\tresult := d.extractResultFromNode(n)\n\t\t\tif result.Title != \"\" && result.URL != \"\" {\n\t\t\t\t*results = append(*results, result)\n\t\t\t}\n\t\t}","sourceCodeStart":243,"sourceCodeEnd":279,"githubUrl":"https://github.com/vxcontrol/pentagi/blob/ea665308baaff015b226f308438a68d929d0f29b/backend/pkg/tools/searchers/duckduckgo.go#L243-L279","documentation":"parseHTMLStructured calls golang.org/x/net/html's html.Parse on the response body. html.Parse is extremely lenient — it can error only on unrecoverable internal conditions (e.g. very deeply nested markup overflowing the parser's limits), since it otherwise repairs malformed HTML. This error is usually swallowed by the caller: parseHTMLResponse falls back to regex parsing when the structured pass yields no results.","triggerScenarios":"html.Parse returns a non-nil error for the DuckDuckGo response — practically only with pathologically deep nesting in the body (html.ParseError with huge depth), truncated/corrupt responses, or a non-HTML body large/degenerate enough to break the tokenizer.","commonSituations":"Extremely rare in production; seen with truncated/corrupted response bodies, binary garbage served with 200 by a misbehaving proxy, or x/net/html version regressions on pathological pages.","solutions":["Rely on the existing fallback: parseHTMLResponse already retries with parseHTMLRegex, so confirm the regex path succeeded.","Inspect the body if this recurs — corruption mid-transfer usually indicates network/proxy problems rather than a parser bug.","Update golang.org/x/net if a known html.Parse bug matches the wrapped error.","Log the wrapped error text; it pinpoints whether it's a depth limit or tokenizer failure."],"exampleFix":null,"handlingStrategy":"try-catch","validationCode":"// reject obviously non-HTML bodies before parsing\ntrimmed := bytes.TrimSpace(body)\nif len(trimmed) > 0 && trimmed[0] != '<' {\n    return nil, errors.New(\"response body is not HTML\")\n}","typeGuard":null,"tryCatchPattern":"results, err := d.parseHTMLStructured(body)\nif err != nil {\n    // structured parse failed; regex fallback in parseHTMLResponse handles it\n    log.WithError(err).Debug(\"structured parse failed, falling back to regex\")\n    return d.parseHTMLRegex(body)\n}","preventionTips":["Always pair structured parsing with the regex fallback (already done in parseHTMLResponse)","Keep golang.org/x/net updated for html.Parse fixes","Validate the Content-Type of responses before parsing","Guard against truncated bodies by checking Content-Length vs bytes read"],"tags":["html","parsing","go","x-net"],"backgroundTag":"html-parse-failed","analyzedSha":"ea665308baaff015b226f308438a68d929d0f29b","analyzedAt":"2026-09-01T14:16:31.421Z","contentChangedAt":null,"schemaVersion":2},"datasetVersion":"2026-09-08T10:18:20.063Z"}