{"record":{"id":"00ec3c696c45f1a1","repo":"sgl-project/sglang","slug":"invalid-external-ngram-corpus-record-at-line-line","errorCode":null,"errorMessage":"Invalid external ngram corpus record at line {line_no}: expected a JSON string.","messagePattern":"Invalid external ngram corpus record at line (.+?): expected a JSON string\\.","errorType":"validation","errorClass":"ValueError","httpStatus":null,"severity":"error","filePath":"python/sglang/srt/speculative/cpp_ngram/external_corpus.py","lineNumber":39,"sourceCode":"    if max_tokens <= 0:\n        raise ValueError(\"External ngram corpus max tokens must be positive.\")\n\n    total_tokens = 0\n    has_previous_doc = False\n    with corpus_path.open(\"r\", encoding=\"utf-8\") as f:\n        for line_no, line in enumerate(f, start=1):\n            if not line.strip():\n                continue\n\n            try:\n                record = json.loads(line)\n            except json.JSONDecodeError as e:\n                raise ValueError(\n                    f\"Invalid JSON in external ngram corpus at line {line_no}: {e.msg}\"\n                ) from e\n\n            if not isinstance(record, str):\n                raise ValueError(\n                    \"Invalid external ngram corpus record at line \"\n                    f\"{line_no}: expected a JSON string.\"\n                )\n\n            token_ids = list(tokenizer.encode(record, add_special_tokens=False))\n            if not token_ids:\n                continue\n\n            separator_cost = 1 if has_previous_doc else 0\n            next_total_tokens = total_tokens + separator_cost + len(token_ids)\n            if next_total_tokens > max_tokens:\n                raise ValueError(\n                    \"External ngram corpus exceeds the configured token limit \"\n                    f\"({max_tokens}) at line {line_no} after loading \"\n                    f\"{total_tokens} tokens.\"\n                )\n            total_tokens = next_total_tokens\n","sourceCodeStart":21,"sourceCodeEnd":57,"githubUrl":"https://github.com/sgl-project/sglang/blob/0132848349585cfe6aae51c4941cbae872505f8a/python/sglang/srt/speculative/cpp_ngram/external_corpus.py#L21-L57","documentation":"Each non-blank line of the external ngram corpus must parse to a JSON string (one document per line). If json.loads succeeds but yields a non-str (dict, list, number), this ValueError is raised with the line number.","triggerScenarios":"A corpus file where lines are JSON objects like {\"text\": \"...\"} or JSON arrays instead of plain JSON-encoded strings.","commonSituations":"Corpus exported in a records format (JSONL with objects) rather than one JSON string per line; mixing token-array corpora with text corpora.","solutions":["Rewrite the corpus to one JSON string per line (json.dumps(doc) for doc in docs)","Or unwrap the field first: json.dumps(json.loads(line)[\"text\"])"],"exampleFix":"# before\n{\"text\": \"hello world\"}\n# after\n\"hello world\"","handlingStrategy":"validation","validationCode":"import json\ndef is_string_jsonl(path):\n    with open(path) as f:\n        return all(not line.strip() or isinstance(json.loads(line), str) for line in f)\nassert is_string_jsonl(path)","typeGuard":"def is_corpus_record(line: str) -> bool:\n    try:\n        return isinstance(json.loads(line), str)\n    except json.JSONDecodeError:\n        return False","tryCatchPattern":null,"preventionTips":["Document the one-JSON-string-per-line format next to corpus tools","Add a schema smoke test for generated corpus files"],"tags":["sglang","ngram","json","jsonl","schema-validation"],"backgroundTag":"json-schema-validation-failed","analyzedSha":"0132848349585cfe6aae51c4941cbae872505f8a","analyzedAt":"2026-08-28T05:10:05.995Z","schemaVersion":2},"datasetVersion":"2026-08-28T06:17:29.519Z"}