{"record":{"id":"9e63698e356d35c3","repo":"Hmbown/CodeWhale","slug":"input-exceeds-64-mib","errorCode":null,"errorMessage":"Input exceeds 64 MiB.","messagePattern":"Input exceeds 64 MiB\\.","errorType":"validation","errorClass":"Error","httpStatus":null,"severity":"error","filePath":"pet/scripts/pet.mjs","lineNumber":28,"sourceCode":"import { createPetRecorder } from './lib/pet-recorder.mjs';\n\nconst args = process.argv.slice(2);\nconst option = name => args.find(a => a.startsWith(`--${name}=`))?.slice(name.length + 3);\nif (args.includes('--help')) {\n  console.log('node scripts/pet.mjs --input=trace.jsonl --output=pet.jsonl [--trace=ID] [--format=jsonl|tsv] [--watch]\\nnode scripts/pet.mjs --runtime=http://127.0.0.1:7878 --thread=ID --output=pet.jsonl [--segment-buckets=216000] [--resume]\\nUse --demo instead of --input for synthetic telemetry. Output must not already exist unless --resume is used for live recording. A resumed recorder preserves the previous segment and starts unknown at the same path.\\nLive recording rotates at 216000 buckets or 64 MiB into OUTPUT.segment-NNNNNN.jsonl and continues at the same live path. All archives are retained.\\nRuntime reads only the existing local event journal. Optional authentication comes from CODEWHALE_RUNTIME_TOKEN; never put a token in the URL. No agent or provider is started.');\n  process.exit(0);\n}\nlet output, recorder, monitor, timer, runtime;\ntry {\n  for (const a of args) if (!['--demo', '--watch', '--resume'].includes(a) && !/^--(input|output|trace|format|runtime|thread|segment-buckets)=.+/.test(a)) throw new Error('Unknown or empty option. Use --help.');\n  const input = option('input'), runtimeURL = option('runtime'), path = option('output'), format = option('format') ?? 'jsonl', live = args.includes('--watch') || !!runtimeURL;\n  if (!path || [!!input, args.includes('--demo'), !!runtimeURL].filter(Boolean).length !== 1\n    || !['jsonl', 'tsv'].includes(format) || live && format !== 'jsonl' || args.includes('--watch') && !input\n    || !!runtimeURL !== !!option('thread') || option('trace') && !input || option('segment-buckets') && !live || args.includes('--resume') && !live)\n    throw new Error('Choose one input source, an unused --output path (or --resume), and JSONL for live recording. Runtime requires --thread.');\n  const load = async () => {\n    if (!input) return { events: petDemoEvents(), duration: 80_000 };\n    if ((await stat(input)).size > 64 * 1024 * 1024) throw new Error('Input exceeds 64 MiB.');\n    const traces = importTrace(await readFile(input, 'utf8'), input, { privacy: 'metadata' });\n    const trace = option('trace') ? traces.find(t => t.id === option('trace')) : traces.length === 1 ? traces[0] : undefined;\n    if (!trace) throw new Error('Select an existing --trace ID when input contains multiple traces.');\n    return trace;\n  };\n  let trace = runtimeURL ? undefined : await load(), buckets = compilePetTelemetry(trace?.events ?? [], trace?.duration ?? 0);\n  if (live) recorder = await createPetRecorder(path, { resume: args.includes('--resume'), maxBuckets: option('segment-buckets') === undefined ? 216_000 : Number(option('segment-buckets')), report: text => console.error(text) });\n  else output = await open(path, 'wx', 0o600);\n  if (runtimeURL) runtime = await followRuntime({ baseUrl: runtimeURL, threadId: option('thread'),\n    token: process.env.CODEWHALE_RUNTIME_TOKEN, report: text => console.error(text) });\n  if (!live) {\n    await output.writeFile(format === 'tsv' ? encodePetTSV(buckets) : encodePetJSONL(buckets));\n    await output.close(); output = undefined;\n    console.log(`Wrote ${buckets.length} pet buckets (${args.includes('--demo') ? 'demo' : 'trace replay'}).`);\n  } else {\n    // The driver owns wall time. The core only sees recorded relative timestamps.\n    const started = performance.now(), startedWall = Date.now();\n    const origin = trace && 'originTime' in trace && trace.originTime ? Date.parse(trace.originTime) : NaN;","sourceCodeStart":10,"sourceCodeEnd":46,"githubUrl":"https://github.com/Hmbown/CodeWhale/blob/433685b2024e7bc4c99e1e2e326bcad39b4d9d65/pet/scripts/pet.mjs#L10-L46","documentation":"The load() helper stats the --input file and refuses to read it if it exceeds 64 MiB (64 * 1024 * 1024 bytes). This bound keeps importTrace's full-file readFile into memory bounded, since the converter parses the whole trace into a single string before importing.","triggerScenarios":"Calling load() via scripts/pet.mjs --input=<file> where stat(file).size > 67108864 bytes — e.g. pointing at a very large or live-appending trace JSONL instead of a bounded export.","commonSituations":"Converting a long-running recording journal that grew past 64 MiB; passing the live OUTPUT.segment-NNNNNN.jsonl chain's directory or a merged dump; forgetting that trace exports should be bounded windows.","solutions":["Split the input into traces or time windows under 64 MiB and convert each separately, selecting with --trace=<id>.","Trim or gzip-then-filter the file to only the events you need before converting.","Record from the live source with --runtime/--thread (streaming, segmented) instead of converting a giant captured file.","If the file is genuinely under 64 MiB, check the path — stat on a directory or a growing file may report more than expected."],"exampleFix":"// before\nnode scripts/pet.mjs --input=huge-recording.jsonl --output=pet.jsonl\n// after\nsplit -b 60m huge-recording.jsonl part-\nnode scripts/pet.mjs --input=part-aa --output=pet-aa.jsonl","handlingStrategy":"validation","validationCode":"import { stat } from 'node:fs/promises';\nconst MAX = 64 * 1024 * 1024;\nif ((await stat(inputFile)).size > MAX) {\n  console.error('split or filter the trace before converting');\n} else {\n  // safe to invoke pet.mjs --input=...\n}","typeGuard":null,"tryCatchPattern":"try {\n  await runPetCli(['--input=' + file, '--output=' + out]);\n} catch (err) {\n  if (err.message === 'Input exceeds 64 MiB.') {\n    console.error(`split ${file} (<64 MiB) or convert per-trace windows`);\n  }\n  throw err;\n}","preventionTips":["Stat the input file size before conversion.","Keep trace exports bounded to a session/window instead of unbounded recordings.","Prefer streaming via --runtime/--thread for long journals rather than bulk file conversion."],"tags":["cli","file-size","limit","memory"],"backgroundTag":"file-size-limit-exceeded","analyzedSha":"433685b2024e7bc4c99e1e2e326bcad39b4d9d65","analyzedAt":"2026-09-15T12:24:24.634Z","contentChangedAt":"2026-09-15T12:24:24.634Z","schemaVersion":2},"datasetVersion":"2026-09-22T16:17:23.217Z"}