thanos-io/thanos · error

mark old block for deletion from bucket

Error message

mark old block for deletion from bucket

What it means

Wraps cg.deleteBlock while garbage-collecting a source block after successful compaction: specifically the failure to write a deletion mark for the old block in the bucket (the local dir removal precedes it). Retried on the next compaction cycle since the block remains present.

Solutions

1. Check bucket write permissions for deletion-mark.json
2. Inspect wrapped storage errors
3. Rely on retry or re-run compaction

When it happens

Trigger: Thrown at pkg/compact/compact.go:1375 when the library encounters an invalid state.

Common situations: See trigger scenarios.

AI-assisted analysis of thanos-io/thanos@35b8b99117 (2026-09-07). Data as JSON: /api/errors/52a2fef9424e140d. Report an issue: GitHub.

Appendix: source

Thrown at pkg/compact/compact.go:1375

		if err != nil {
			return false, nil, retry(errors.Wrapf(err, "upload of %s failed", compID))
		}
		level.Info(cg.logger).Log("msg", "uploaded block", "result_block", compID, "duration", time.Since(begin), "duration_ms", time.Since(begin).Milliseconds())
		level.Info(cg.logger).Log("msg", "running post compaction callback", "result_block", compID)
		if err := compactionLifecycleCallback.PostCompactionCallback(ctx, cg.logger, cg, compID); err != nil {
			return false, nil, retry(errors.Wrapf(err, "failed to run post compaction callback for result block %s", compID))
		}
		level.Info(cg.logger).Log("msg", "finished running post compaction callback", "result_block", compID)
	}

	// Mark for deletion the blocks we just compacted from the group and bucket so they do not get included
	// into the next planning cycle.
	// Eventually the block we just uploaded should get synced into the group again (including sync-delay).
	for _, meta := range toCompact {
		if err := tracing.DoInSpanWithErr(ctx, "compaction_block_delete", func(ctx context.Context) error {
			return cg.deleteBlock(meta.ULID, filepath.Join(dir, meta.ULID.String()), blockDeletableChecker)
		}, opentracing.Tags{"block.id": meta.ULID}); err != nil {
			return false, nil, retry(errors.Wrapf(err, "mark old block for deletion from bucket"))
		}
		cg.groupGarbageCollectedBlocks.Inc()
	}

	level.Info(cg.logger).Log("msg", "finished compacting blocks", "duration", time.Since(groupCompactionBegin),
		"duration_ms", time.Since(groupCompactionBegin).Milliseconds(), "result_blocks", compIDStrs, "source_blocks", sourceBlockStr)
	return true, compIDs, nil
}

func (cg *Group) deleteBlock(id ulid.ULID, bdir string, blockDeletableChecker BlockDeletableChecker) error {
	if err := os.RemoveAll(bdir); err != nil {
		return errors.Wrapf(err, "remove old block dir %s", id)
	}

	if blockDeletableChecker.CanDelete(cg, id) {
		// Spawn a new context so we always mark a block for deletion in full on shutdown.
		delCtx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
		defer cancel()

View on GitHub (pinned to 35b8b99117)