{"record":{"id":"23dcfac0295c5eef","repo":"apache/beam","slug":"not-a-valid-tfrecord-mismatch-of-data-mask-s","errorCode":null,"errorMessage":"Not a valid TFRecord. Mismatch of data mask: %s","messagePattern":"Not a valid TFRecord\\. Mismatch of data mask: (.+?)","errorType":"validation","errorClass":"ValueError","httpStatus":null,"severity":"error","filePath":"sdks/python/apache_beam/io/tfrecordio.py","lineNumber":174,"sourceCode":"          (buf_length_expected, codecs.encode(buf, 'hex')))\n    length, length_mask_expected = struct.unpack('<QI', buf)\n    length_mask_actual = cls._masked_crc32c(buf[:8])\n    if length_mask_actual != length_mask_expected:\n      raise ValueError(\n          'Not a valid TFRecord. Mismatch of length mask: %s' %\n          codecs.encode(buf, 'hex'))\n\n    # Validate all data related payloads.\n    buf_length_expected = length + 4\n    buf = file_handle.read(buf_length_expected)\n    if len(buf) != buf_length_expected:\n      raise ValueError(\n          'Not a valid TFRecord. Fewer than %d bytes: %s' %\n          (buf_length_expected, codecs.encode(buf, 'hex')))\n    data, data_mask_expected = struct.unpack('<%dsI' % length, buf)\n    data_mask_actual = cls._masked_crc32c(data)\n    if data_mask_actual != data_mask_expected:\n      raise ValueError(\n          'Not a valid TFRecord. Mismatch of data mask: %s' %\n          codecs.encode(buf, 'hex'))\n\n    # All validation checks passed.\n    return data\n\n\nclass _TFRecordSource(FileBasedSource):\n  \"\"\"A File source for reading files of TFRecords.\n\n  For detailed TFRecords format description see:\n    https://www.tensorflow.org/versions/r1.11/api_guides/python/python_io#TFRecords_Format_Details\n  \"\"\"\n  def __init__(self, file_pattern, coder, compression_type, validate):\n    \"\"\"Initialize a TFRecordSource.  See ReadFromTFRecord for details.\"\"\"\n    super().__init__(\n        file_pattern=file_pattern,\n        compression_type=compression_type,","sourceCodeStart":156,"sourceCodeEnd":192,"githubUrl":"https://github.com/apache/beam/blob/12126d8942aaf848030c478b4c6a28c6af861c66/sdks/python/apache_beam/io/tfrecordio.py#L156-L192","documentation":"The reader computes masked CRC32C over the record payload and compares it to the 4-byte mask stored after the data. A mismatch indicates the payload bytes are corrupted, so ValueError is raised with hex of the header buffer (message text uses the header hex in this source, though conceptually it flags the data mask).","triggerScenarios":"Reading a TFRecord whose payload bytes were altered after writing (bit flips, bad compression pipeline, wrong byte patching).","commonSituations":"Storage corruption, manual byte-level edits, or files concatenated incorrectly from different writers.","solutions":["Regenerate the TFRecord from source data","Restore the file from a backup","Check the storage medium / transfer path for corruption"],"exampleFix":"// before\n# reading corrupted payload\n// after\n# rewrite from source:\nwith tf.io.TFRecordWriter(out_path) as w: [w.write(r) for r in records]","handlingStrategy":"validation","validationCode":"# before ingestion, verify data CRC on a sample record with masked_crc32c\nfrom apache_beam.io import tfrecordio  # or compute crc32c directly\ncrc = tfrecordio._TFRecordUtil._masked_crc32c(data)\nassert isinstance(crc, int)","typeGuard":null,"tryCatchPattern":"try:\n    pc | beam.io.ReadFromTFRecord(pattern)\nexcept ValueError as e:\n    if 'Mismatch of data mask' in str(e):\n        raise RuntimeError('Corrupted record payload in %s' % pattern) from e\n    raise","preventionTips":["Avoid editing TFRecord files after writing","Check storage integrity (e2e checksums on copy)","Regenerate from source instead of patching payload bytes"],"tags":["python","beam","tfrecord","corruption"],"backgroundTag":"checksum-mismatch","analyzedSha":"12126d8942aaf848030c478b4c6a28c6af861c66","analyzedAt":"2026-09-13T01:50:10.254Z","contentChangedAt":"2026-09-13T01:50:10.254Z","schemaVersion":2},"datasetVersion":"2026-09-14T21:17:11.552Z"}