{"record":{"id":"2885932e6f7fc7a5","repo":"hashicorp/nomad","slug":"failed-to-process-eval-v","errorCode":null,"errorMessage":"failed to process eval: %v","messagePattern":"failed to process eval: (.+?)","errorType":"exception","errorClass":null,"httpStatus":null,"severity":"critical","filePath":"scheduler/generic_sched.go","lineNumber":109,"sourceCode":"// NewBatchScheduler is a factory function to instantiate a new batch scheduler\nfunc NewBatchScheduler(logger log.Logger, eventsCh chan<- any, state sstructs.State, planner sstructs.Planner) sstructs.Scheduler {\n\ts := &GenericScheduler{\n\t\tlogger:   logger.Named(\"batch_sched\"),\n\t\teventsCh: eventsCh,\n\t\tstate:    state,\n\t\tplanner:  planner,\n\t\tbatch:    true,\n\t}\n\treturn s\n}\n\n// Process is used to handle a single evaluation\nfunc (s *GenericScheduler) Process(eval *structs.Evaluation) (err error) {\n\n\tdefer func() {\n\t\tif r := recover(); r != nil {\n\t\t\ts.logger.Error(\"processing eval panicked scheduler - please report this as a bug!\", \"eval_id\", eval.ID, \"error\", r, \"stack_trace\", string(debug.Stack()))\n\t\t\terr = fmt.Errorf(\"failed to process eval: %v\", r)\n\t\t}\n\t}()\n\n\t// Store the evaluation\n\ts.eval = eval\n\n\t// Update our logger with the eval's information\n\ts.logger = s.logger.With(\"eval_id\", eval.ID, \"job_id\", eval.JobID, \"namespace\", eval.Namespace)\n\n\t// Verify the evaluation trigger reason is understood\n\tswitch eval.TriggeredBy {\n\tcase structs.EvalTriggerJobRegister, structs.EvalTriggerJobDeregister,\n\t\tstructs.EvalTriggerNodeDrain, structs.EvalTriggerNodeUpdate,\n\t\tstructs.EvalTriggerAllocStop, structs.EvalTriggerAllocReschedule,\n\t\tstructs.EvalTriggerRollingUpdate, structs.EvalTriggerQueuedAllocs,\n\t\tstructs.EvalTriggerPeriodicJob, structs.EvalTriggerMaxPlans,\n\t\tstructs.EvalTriggerDeploymentWatcher, structs.EvalTriggerRetryFailedAlloc,\n\t\tstructs.EvalTriggerFailedFollowUp, structs.EvalTriggerPreemption,","sourceCodeStart":91,"sourceCodeEnd":127,"githubUrl":"https://github.com/hashicorp/nomad/blob/482b49bf1aec006f089bcfc7e632d8f6ac303e5e/scheduler/generic_sched.go#L91-L127","documentation":"GenericScheduler.Process recovers from any panic raised while processing an evaluation and converts it into this error, asking the user to report it as a bug. It is not an expected operational error — it means the scheduler code panicked (nil map/deref, index out of range, etc.) while handling an eval.","triggerScenarios":"Any panic during eval processing: nil job or task group fields in a malformed job spec, unexpected nil allocation/node from the state store, or a genuine scheduler bug triggered by unusual eval content.","commonSituations":"Malformed job submissions that evade API validation (e.g. via older clients or direct RPC), races with state changes mid-eval, or scheduler bugs on specific Nomad versions.","solutions":["Capture the stack_trace field from the log line and check the Nomad GitHub issues for a matching report","Upgrade Nomad to the latest patch release — many panics are fixed per version","Identify the eval's job and resubmit it after validating its spec with 'nomad job validate'","Report the bug with the eval ID, stack trace, and Nomad version if it persists"],"exampleFix":"// before\njob with nil TaskGroups entry submitted, panicking the scheduler\n// after\nnomad job validate job.nomad.hcl && nomad job run job.nomad.hcl","handlingStrategy":"validation","validationCode":"nomad job validate job.nomad.hcl","typeGuard":"func jobSpecSane(j *api.Job) bool {\n\treturn j != nil && len(j.TaskGroups) > 0\n}","tryCatchPattern":"// Panics are converted to errors by Process(); operators should:\n// 1. grep server logs for 'processing eval panicked' and capture stack_trace\n// 2. check the job with 'nomad job validate'\n// 3. upgrade Nomad if the panic recurs","preventionTips":["Always run 'nomad job validate' before submitting jobs","Use a current Nomad API/CLI version matching the server","Report panics to Nomad GitHub with eval ID and stack trace","Keep Nomad servers and clients on matching versions"],"tags":["nomad","scheduler","panic"],"backgroundTag":"scheduler-eval-panic","analyzedSha":"482b49bf1aec006f089bcfc7e632d8f6ac303e5e","analyzedAt":"2026-09-04T07:54:14.808Z","contentChangedAt":"2026-09-04T07:54:14.808Z","schemaVersion":2},"datasetVersion":"2026-09-08T10:18:20.063Z"}