{"record":{"id":"9ab21433ac33dc64","repo":"vitessio/vitess","slug":"deletetablets-v-failed-to-acquire-toporwpool","errorCode":null,"errorMessage":"DeleteTablets(%+v) failed to acquire topoRWPool: %w","messagePattern":"DeleteTablets\\(%\\+v\\) failed to acquire topoRWPool: %w","errorType":"exception","errorClass":null,"httpStatus":null,"severity":"error","filePath":"go/vt/vtadmin/cluster/cluster.go","lineNumber":555,"sourceCode":"\n\tif err := c.topoRWPool.Acquire(ctx); err != nil {\n\t\treturn nil, fmt.Errorf(\"DeleteShards(%+v) failed to acquire topoRWPool: %w\", req, err)\n\t}\n\tdefer c.topoRWPool.Release()\n\n\treturn c.Vtctld.DeleteShards(ctx, req)\n}\n\n// DeleteTablets deletes one or more tablets in the given cluster.\nfunc (c *Cluster) DeleteTablets(ctx context.Context, req *vtctldatapb.DeleteTabletsRequest) (*vtctldatapb.DeleteTabletsResponse, error) {\n\tspan, ctx := trace.NewSpan(ctx, \"Cluster.DeleteTablets\")\n\tdefer span.Finish()\n\n\tAnnotateSpan(c, span)\n\tspan.Annotate(\"tablet_aliases\", strings.Join(topoproto.TabletAliasList(req.TabletAliases).ToStringSlice(), \",\"))\n\n\tif err := c.topoRWPool.Acquire(ctx); err != nil {\n\t\treturn nil, fmt.Errorf(\"DeleteTablets(%+v) failed to acquire topoRWPool: %w\", req, err)\n\t}\n\tdefer c.topoRWPool.Release()\n\n\treturn c.Vtctld.DeleteTablets(ctx, req)\n}\n\n// EmergencyFailoverShard fails over a shard to a new primary. It assumes the\n// old primary is dead or otherwise not responding.\nfunc (c *Cluster) EmergencyFailoverShard(ctx context.Context, req *vtctldatapb.EmergencyReparentShardRequest) (*vtadminpb.EmergencyFailoverShardResponse, error) {\n\tspan, ctx := trace.NewSpan(ctx, \"Cluster.EmergencyFailoverShard\")\n\tdefer span.Finish()\n\n\tAnnotateSpan(c, span)\n\tspan.Annotate(\"keyspace\", req.Keyspace)\n\tspan.Annotate(\"shard\", req.Shard)\n\tspan.Annotate(\"new_primary\", topoproto.TabletAliasString(req.NewPrimary))\n\tspan.Annotate(\"ignore_replicas\", strings.Join(topoproto.TabletAliasList(req.IgnoreReplicas).ToStringSlice(), \",\"))\n\tspan.Annotate(\"prevent_cross_cell_promotion\", req.PreventCrossCellPromotion)","sourceCodeStart":537,"sourceCodeEnd":573,"githubUrl":"https://github.com/vitessio/vitess/blob/01a25a7d176f94613b8d59d799f438380a8760e4/go/vt/vtadmin/cluster/cluster.go#L537-L573","documentation":"DeleteTablets acquires the read-write topology pool before deleting tablet records via vtctld. If c.topoRWPool.Acquire(ctx) fails, the error is wrapped with the request (including the tablet aliases being deleted). The pool bounds concurrent topology writes across the vtadmin cluster handle.","triggerScenarios":"Calling DeleteTablets while the topoRWPool is fully utilized by other destructive operations, with the context expiring before a slot frees; ctx cancellation (client disconnect) during the wait.","commonSituations":"Large-scale decommissioning scripts removing hundreds of tablets concurrently; overlap with an EmergencyFailoverShard or shard deletion; deadline-driven batch jobs.","solutions":["Retry with an extended context deadline","Throttle/serialize tablet deletions instead of fanning out","Check for concurrent topology-mutating operations in the cluster","Increase topoRWPool size if this recurs under normal load"],"exampleFix":"// before\nfor _, alias := range aliases {\n    go deleteTablet(alias) // Acquire fails under contention\n}\n// after\nfor _, alias := range aliases {\n    if err := deleteTablet(alias); err != nil {\n        return err // sequential: no pool contention\n    }\n}","handlingStrategy":"retry","validationCode":"if req == nil || len(req.TabletAliases) == 0 {\n    return fmt.Errorf(\"DeleteTablets: request and tablet aliases are required\")\n}","typeGuard":"func isValidDeleteTabletsRequest(req *vtctldatapb.DeleteTabletsRequest) bool {\n    return req != nil && len(req.TabletAliases) > 0\n}","tryCatchPattern":"err := cluster.DeleteTablets(ctx, req)\nif err != nil && strings.Contains(err.Error(), \"failed to acquire topoRWPool\") {\n    retryCtx, cancel := context.WithTimeout(context.Background(), 60*time.Second)\n    defer cancel()\n    resp, err = cluster.DeleteTablets(retryCtx, req)\n}","preventionTips":["Delete tablets sequentially or with limited concurrency","Schedule decommissioning jobs away from other topo-mutating workflows","Keep context deadlines comfortably longer than pool wait times"],"tags":["resource-pool","timeout","concurrency","vtadmin"],"backgroundTag":"pool-acquisition-failed","analyzedSha":"01a25a7d176f94613b8d59d799f438380a8760e4","analyzedAt":"2026-09-01T17:28:30.605Z","contentChangedAt":null,"schemaVersion":2},"datasetVersion":"2026-09-08T10:18:20.063Z"}