fsmanager.py 1.7 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253
  1. #!/usr/bin/env python
  2. # Licensed to Cloudera, Inc. under one
  3. # or more contributor license agreements. See the NOTICE file
  4. # distributed with this work for additional information
  5. # regarding copyright ownership. Cloudera, Inc. licenses this file
  6. # to you under the Apache License, Version 2.0 (the
  7. # "License"); you may not use this file except in compliance
  8. # with the License. You may obtain a copy of the License at
  9. #
  10. # http://www.apache.org/licenses/LICENSE-2.0
  11. #
  12. # Unless required by applicable law or agreed to in writing, software
  13. # distributed under the License is distributed on an "AS IS" BASIS,
  14. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. # See the License for the specific language governing permissions and
  16. # limitations under the License.
  17. from hadoop.cluster import get_all_hdfs
  18. _filesystems = None
  19. def _init_filesystems():
  20. """Initialize the module-scoped filesystem dictionary."""
  21. global _filesystems
  22. if _filesystems is not None:
  23. return
  24. _filesystems = get_all_hdfs()
  25. def get_filesystem(name):
  26. """Return the filesystem with the given name. If the filesystem is not defined,
  27. raises KeyError"""
  28. _init_filesystems()
  29. return _filesystems[name]
  30. def get_default_hdfs():
  31. """
  32. Return the (name, fs) for the default hdfs.
  33. Return (None, None) if no hdfs cluster configured
  34. """
  35. _init_filesystems()
  36. for name, fs in _filesystems.iteritems():
  37. # Return the first HDFS encountered
  38. if fs.uri.startswith('hdfs') or fs.uri.startswith('http'):
  39. return name, fs
  40. return None, None
  41. def reset():
  42. """
  43. reset() -- Forget all cached filesystems and go to a pristine state.
  44. """
  45. global _filesystems
  46. _filesystems = None