{"record":{"id":"addf0921ede157d6","repo":"apache/beam","slug":"pipeline-options-is-not-set","errorCode":null,"errorMessage":"pipeline_options is not set","messagePattern":"pipeline_options is not set","errorType":"validation","errorClass":"ValueError","httpStatus":null,"severity":"error","filePath":"sdks/python/apache_beam/io/hadoopfilesystem.py","lineNumber":120,"sourceCode":"class HadoopFileSystem(FileSystem):\n  \"\"\"``FileSystem`` implementation that supports HDFS.\n\n  URL arguments to methods expect strings starting with ``hdfs://``.\n  \"\"\"\n  def __init__(self, pipeline_options):\n    \"\"\"Initializes a connection to HDFS.\n\n    Connection configuration is done by passing pipeline options.\n    See :class:`~apache_beam.options.pipeline_options.HadoopFileSystemOptions`.\n    \"\"\"\n    super().__init__(pipeline_options)\n    if hdfs is None:\n      raise ImportError(\n          'Failed to import hdfs. You can ensure it is '\n          'installed by installing the hadoop beam extra')\n    logging.getLogger('hdfs.client').setLevel(logging.WARN)\n    if pipeline_options is None:\n      raise ValueError('pipeline_options is not set')\n    if isinstance(pipeline_options, PipelineOptions):\n      hdfs_options = pipeline_options.view_as(HadoopFileSystemOptions)\n      hdfs_host = hdfs_options.hdfs_host\n      hdfs_port = hdfs_options.hdfs_port\n      hdfs_user = hdfs_options.hdfs_user\n      self._full_urls = hdfs_options.hdfs_full_urls\n    else:\n      hdfs_host = pipeline_options.get('hdfs_host')\n      hdfs_port = pipeline_options.get('hdfs_port')\n      hdfs_user = pipeline_options.get('hdfs_user')\n      self._full_urls = pipeline_options.get('hdfs_full_urls', False)\n\n    if hdfs_host is None:\n      raise ValueError('hdfs_host is not set')\n    if hdfs_port is None:\n      raise ValueError('hdfs_port is not set')\n    if hdfs_user is None:\n      raise ValueError('hdfs_user is not set')","sourceCodeStart":102,"sourceCodeEnd":138,"githubUrl":"https://github.com/apache/beam/blob/12126d8942aaf848030c478b4c6a28c6af861c66/sdks/python/apache_beam/io/hadoopfilesystem.py#L102-L138","documentation":"HadoopFileSystem.__init__ needs connection configuration supplied via pipeline options (hdfs_host, hdfs_port, hdfs_user). If pipeline_options is None it cannot read any configuration and raises ValueError 'pipeline_options is not set'.","triggerScenarios":"Constructing HadoopFileSystem(pipeline_options=None) directly, or a filesystem registration path that passes no options object.","commonSituations":"Programmatic use of the HDFS filesystem without setting HadoopFileSystemOptions; tests instantiating the class with defaults; pipeline launched without --hdfs_host etc. so options resolution yields None.","solutions":["Pass a PipelineOptions (or dict) containing hdfs_host, hdfs_port, hdfs_user.","Provide CLI flags --hdfs_host, --hdfs_port, --hdfs_user when launching the pipeline.","Wrap raw dict values in PipelineOptions as the class accepts both forms.","Add an assertion/default pipeline_options before constructing the filesystem."],"exampleFix":"// before\nfs = HadoopFileSystem(pipeline_options=None)\n// after\nopts = PipelineOptions(['--hdfs_host=namenode', '--hdfs_port=8020', '--hdfs_user=hduser'])\nfs = HadoopFileSystem(pipeline_options=opts)","handlingStrategy":"validation","validationCode":"def has_hdfs_options(opts) -> bool:\n    if opts is None:\n        return False\n    if isinstance(opts, dict):\n        return 'hdfs_host' in opts\n    from apache_beam.options.pipeline_options import PipelineOptions, HadoopFileSystemOptions\n    return opts.view_as(HadoopFileSystemOptions).hdfs_host is not None","typeGuard":"None","tryCatchPattern":"try:\n    fs = HadoopFileSystem(pipeline_options=opts)\nexcept ValueError as e:\n    if str(e) == 'pipeline_options is not set':\n        fs = HadoopFileSystem(pipeline_options=build_default_hdfs_options())\n    else:\n        raise","preventionTips":["Always pass PipelineOptions or a dict when constructing HadoopFileSystem","Supply --hdfs_host/--hdfs_port/--hdfs_user flags on launch","Assert options are present in your launcher script","Centralize HDFS options construction in one helper"],"tags":["hdfs","configuration","python","apache-beam"],"backgroundTag":"missing-required-argument","analyzedSha":"12126d8942aaf848030c478b4c6a28c6af861c66","analyzedAt":"2026-09-13T01:50:10.254Z","contentChangedAt":"2026-09-13T01:50:10.254Z","schemaVersion":2},"datasetVersion":"2026-09-14T16:17:12.679Z"}