configloader.py 8.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243
  1. # Copyright (c) 2012-2013 Mitch Garnaat http://garnaat.org/
  2. # Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
  3. #
  4. # Modifications made by Cloudera are:
  5. # Copyright (c) 2016 Cloudera, Inc. All rights reserved.
  6. #
  7. # Licensed under the Apache License, Version 2.0 (the "License"). You
  8. # may not use this file except in compliance with the License. A copy of
  9. # the License is located at
  10. #
  11. # http://aws.amazon.com/apache2.0/
  12. #
  13. # or in the "license" file accompanying this file. This file is
  14. # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
  15. # ANY KIND, either express or implied. See the License for the specific
  16. # language governing permissions and limitations under the License.
  17. import copy
  18. import os
  19. import shlex
  20. from altuscli.exceptions import ConfigNotFound, ConfigParseError
  21. from six.moves import configparser
  22. def multi_file_load_config(*filenames):
  23. """Load and combine multiple INI configs with profiles.
  24. This function will take a list of filesnames and return
  25. a single dictionary that represents the merging of the loaded
  26. config files.
  27. If any of the provided filenames does not exist, then that file
  28. is ignored. It is therefore ok to provide a list of filenames,
  29. some of which may not exist.
  30. Configuration files are **not** deep merged, only the top level
  31. keys are merged. The filenames should be passed in order of
  32. precedence. The first config file has precedence over the
  33. second config file, which has precedence over the third config file,
  34. etc. The only exception to this is that the "profiles" key is
  35. merged to combine profiles from multiple config files into a
  36. single profiles mapping. However, if a profile is defined in
  37. multiple config files, then the config file with the highest
  38. precedence is used. Profile values themselves are not merged.
  39. For example::
  40. FileA FileB FileC
  41. [foo] [foo] [bar]
  42. a=1 a=2 a=3
  43. b=2
  44. [bar] [baz] [profile a]
  45. a=2 a=3 region=e
  46. [profile a] [profile b] [profile c]
  47. region=c region=d region=f
  48. The final result of ``multi_file_load_config(FileA, FileB, FileC)``
  49. would be::
  50. {"foo": {"a": 1}, "bar": {"a": 2}, "baz": {"a": 3},
  51. "profiles": {"a": {"region": "c"}}, {"b": {"region": d"}},
  52. {"c": {"region": "f"}}}
  53. Note that the "foo" key comes from A, even though it's defined in both
  54. FileA and FileB. Because "foo" was defined in FileA first, then the values
  55. for "foo" from FileA are used and the values for "foo" from FileB are
  56. ignored. Also note where the profiles originate from. Profile "a"
  57. comes FileA, profile "b" comes from FileB, and profile "c" comes
  58. from FileC.
  59. """
  60. configs = []
  61. profiles = []
  62. for filename in filenames:
  63. try:
  64. loaded = load_config(filename)
  65. except ConfigNotFound:
  66. continue
  67. profiles.append(loaded.pop('profiles'))
  68. configs.append(loaded)
  69. merged_config = _merge_list_of_dicts(configs)
  70. merged_profiles = _merge_list_of_dicts(profiles)
  71. merged_config['profiles'] = merged_profiles
  72. return merged_config
  73. def _merge_list_of_dicts(list_of_dicts):
  74. merged_dicts = {}
  75. for single_dict in list_of_dicts:
  76. for key, value in single_dict.items():
  77. if key not in merged_dicts:
  78. merged_dicts[key] = value
  79. return merged_dicts
  80. def load_config(config_filename):
  81. """Parse a INI config with profiles.
  82. This will parse an INI config file and map top level profiles
  83. into a top level "profile" key.
  84. If you want to parse an INI file and map all section names to
  85. top level keys, use ``raw_config_parse`` instead.
  86. """
  87. parsed = raw_config_parse(config_filename)
  88. return build_profile_map(parsed)
  89. def raw_config_parse(config_filename):
  90. """Returns the parsed INI config contents.
  91. Each section name is a top level key.
  92. :returns: A dict with keys for each profile found in the config
  93. file and the value of each key being a dict containing name
  94. value pairs found in that profile.
  95. :raises: ConfigNotFound, ConfigParseError
  96. """
  97. config = {}
  98. path = config_filename
  99. if path is not None:
  100. path = os.path.expandvars(path)
  101. path = os.path.expanduser(path)
  102. if not os.path.isfile(path):
  103. raise ConfigNotFound(path=path)
  104. cp = configparser.RawConfigParser()
  105. try:
  106. cp.read(path)
  107. except configparser.Error:
  108. raise ConfigParseError(path=path)
  109. else:
  110. for section in cp.sections():
  111. config[section] = {}
  112. for option in cp.options(section):
  113. config_value = cp.get(section, option)
  114. if config_value.startswith('\n'):
  115. # Then we need to parse the inner contents as
  116. # hierarchical. We support a single level
  117. # of nesting for now.
  118. try:
  119. config_value = _parse_nested(config_value)
  120. except ValueError:
  121. raise ConfigParseError(path=path)
  122. config[section][option] = config_value
  123. return config
  124. def _parse_nested(config_value):
  125. # Given a value like this:
  126. # \n
  127. # foo = bar
  128. # bar = baz
  129. # We need to parse this into
  130. # {'foo': 'bar', 'bar': 'baz}
  131. parsed = {}
  132. for line in config_value.splitlines():
  133. line = line.strip()
  134. if not line:
  135. continue
  136. # The caller will catch ValueError
  137. # and raise an appropriate error
  138. # if this fails.
  139. key, value = line.split('=', 1)
  140. parsed[key.strip()] = value.strip()
  141. return parsed
  142. def build_profile_map(parsed_ini_config):
  143. """Convert the parsed INI config into a profile map.
  144. The config file format requires that every profile except the
  145. default to be prepended with "profile", e.g.::
  146. [profile test]
  147. altus_... = foo
  148. altus_... = bar
  149. [profile bar]
  150. altus_... = foo
  151. altus_... = bar
  152. # This is *not* a profile
  153. [preview]
  154. otherstuff = 1
  155. # Neither is this
  156. [foobar]
  157. morestuff = 2
  158. The build_profile_map will take a parsed INI config file where each top
  159. level key represents a section name, and convert into a format where all
  160. the profiles are under a single top level "profiles" key, and each key in
  161. the sub dictionary is a profile name. For example, the above config file
  162. would be converted from::
  163. {"profile test": {"altus_...": "foo", "altus_...": "bar"},
  164. "profile bar": {"altus_...": "foo", "altus_...": "bar"},
  165. "preview": {"otherstuff": ...},
  166. "foobar": {"morestuff": ...},
  167. }
  168. into::
  169. {"profiles": {"test": {"altus_...": "foo", "altus_...": "bar"},
  170. "bar": {"altus_...": "foo", "altus_...": "bar"},
  171. "preview": {"otherstuff": ...},
  172. "foobar": {"morestuff": ...},
  173. }
  174. If there are no profiles in the provided parsed INI contents, then
  175. an empty dict will be the value associated with the ``profiles`` key.
  176. .. note::
  177. This will not mutate the passed in parsed_ini_config. Instead it will
  178. make a deepcopy and return that value.
  179. """
  180. parsed_config = copy.deepcopy(parsed_ini_config)
  181. profiles = {}
  182. final_config = {}
  183. for key, values in parsed_config.items():
  184. if key.startswith("profile"):
  185. try:
  186. parts = shlex.split(key)
  187. except ValueError:
  188. continue
  189. if len(parts) == 2:
  190. profiles[parts[1]] = values
  191. elif key == 'default':
  192. # default section is special and is considered a profile
  193. # name but we don't require you use 'profile "default"'
  194. # as a section.
  195. profiles[key] = values
  196. else:
  197. final_config[key] = values
  198. final_config['profiles'] = profiles
  199. return final_config