{"record":{"id":"4dd6329e34f7e5a8","repo":"huggingface/pytorch-image-models","slug":"unsupported-position-embedding-shape-in-checkpoin","errorCode":null,"errorMessage":"Unsupported position embedding shape in {checkpoint_path}: {pos_embed_w.shape}","messagePattern":"Unsupported position embedding shape in (.+?): (.+?)","errorType":"validation","errorClass":"ValueError","httpStatus":null,"severity":"error","filePath":"timm/models/vision_transformer.py","lineNumber":1569,"sourceCode":"                    num_pos_tokens = pos_embed_w.shape[1]\n                    grid_size = int(math.sqrt(num_pos_tokens))\n                    if grid_size * grid_size != num_pos_tokens:\n                        checkpoint_prefix_tokens = (\n                            1 if f'{prefix}cls' in w else getattr(embeds, 'num_prefix_tokens', 0))\n                        num_pos_tokens -= checkpoint_prefix_tokens\n                        grid_size = int(math.sqrt(num_pos_tokens))\n                        if grid_size * grid_size != num_pos_tokens:\n                            raise ValueError(\n                                f'Cannot infer position grid from {pos_embed_w.shape[1]} tokens '\n                                f'in {checkpoint_path}')\n                        prefix_pos_embed = pos_embed_w[:, :checkpoint_prefix_tokens]\n                        pos_embed_w = pos_embed_w[:, checkpoint_prefix_tokens:]\n                    pos_embed_w = pos_embed_w.reshape(1, grid_size, grid_size, pos_embed_w.shape[-1])\n                else:\n                    # Big Vision NaFlex stores the grid directly as HWC.\n                    pos_embed_w = pos_embed_w.unsqueeze(0)\n            if pos_embed_w.ndim != 4:\n                raise ValueError(f'Unsupported position embedding shape in {checkpoint_path}: {pos_embed_w.shape}')\n\n            if prefix_pos_embed is not None:\n                prefix_index = 0\n                if embeds.cls_token is not None and prefix_pos_embed.shape[1] > prefix_index:\n                    embeds.cls_token.add_(prefix_pos_embed[:, prefix_index:prefix_index + 1])\n                    prefix_index += 1\n                if embeds.reg_token is not None and prefix_pos_embed.shape[1] > prefix_index:\n                    num_reg_tokens = min(embeds.reg_token.shape[1], prefix_pos_embed.shape[1] - prefix_index)\n                    embeds.reg_token[:, :num_reg_tokens].add_(\n                        prefix_pos_embed[:, prefix_index:prefix_index + num_reg_tokens])\n\n            if pos_embed_w.shape != embeds.pos_embed.shape:\n                pos_embed_w = resample_abs_pos_embed_nhwc(\n                    pos_embed_w,\n                    new_size=embeds.pos_embed.shape[1:3],\n                    interpolation=interpolation,\n                    antialias=antialias,\n                    verbose=True,","sourceCodeStart":1551,"sourceCodeEnd":1587,"githubUrl":"https://github.com/huggingface/pytorch-image-models/blob/9a5261e31b3b5128526eb2658333b4c0a54464ae/timm/models/vision_transformer.py#L1551-L1587","documentation":"After normalization, the position embedding must be 4D (1, H, W, C) — either reshaped from tokens or already stored HWC (Big Vision NaFlex style). Any other rank (e.g. a 3D or 5D tensor) is unsupported by this loader path.","triggerScenarios":"A checkpoint whose pos_embed tensor has an exotic layout — e.g. factorized (two tensors), extra batch dims, or a per-head layout — reaching the ndim != 4 check.","commonSituations":"Loading experimental or community-converted checkpoints that store position embeddings non-standardly.","solutions":["Pre-convert the pos_embed tensor in the checkpoint to shape (1, H, W, C) or flat tokens for a square grid","Use the standard PyTorch loader (checkpoint_filter / torch format) for non-standard checkpoints"],"exampleFix":"# before\nload_pretrained(model, 'weird_pos.npz')\n# after\nw['pos_embed'] = w['pos_embed'].reshape(1, H, W, C)  # normalize to HWC\nload_pretrained(model, 'weird_pos.npz')","handlingStrategy":"validation","validationCode":"assert pos_embed_w.ndim == 2 or pos_embed_w.ndim == 4, f'bad pos embed rank {pos_embed_w.ndim}'","typeGuard":null,"tryCatchPattern":"try:\n    load_pretrained(model, path)\nexcept ValueError as e:\n    if 'Unsupported position embedding shape' in str(e):\n        w['pos_embed'] = w['pos_embed'].reshape(1, H, W, C)\n        load_pretrained(model, path)\n    else:\n        raise","preventionTips":["Normalize pos_embed layout to HWC before loading","Use torch-format checkpoints when available"],"tags":["timm","vit","checkpoint","pos-embed"],"backgroundTag":"checkpoint-weight-mismatch","analyzedSha":"9a5261e31b3b5128526eb2658333b4c0a54464ae","analyzedAt":"2026-08-27T02:34:25.417Z","schemaVersion":2},"datasetVersion":"2026-08-27T03:17:27.898Z"}