Эх сурвалжийг харах

HUE-5712 [metadata] Update API that now contains stat upload

Romain Rigaux 8 жил өмнө
parent
commit
6e61c7b
29 өөрчлөгдсөн 5107 нэмэгдсэн , 53 устгасан
  1. 4 0
      desktop/core/ext-py/navoptapi-0.1.0/MANIFEST.in
  2. 1 1
      desktop/core/ext-py/navoptapi-0.1.0/PKG-INFO
  3. 77 0
      desktop/core/ext-py/navoptapi-0.1.0/ccscli/__init__.py
  4. 127 0
      desktop/core/ext-py/navoptapi-0.1.0/ccscli/auth.py
  5. 165 0
      desktop/core/ext-py/navoptapi-0.1.0/ccscli/compat.py
  6. 243 0
      desktop/core/ext-py/navoptapi-0.1.0/ccscli/configloader.py
  7. 293 0
      desktop/core/ext-py/navoptapi-0.1.0/ccscli/credentials.py
  8. 179 0
      desktop/core/ext-py/navoptapi-0.1.0/ccscli/exceptions.py
  9. 81 0
      desktop/core/ext-py/navoptapi-0.1.0/ccscli/serialize.py
  10. 59 0
      desktop/core/ext-py/navoptapi-0.1.0/ccscli/signers.py
  11. 0 0
      desktop/core/ext-py/navoptapi-0.1.0/ccscli/thirdparty/__init__.py
  12. 762 0
      desktop/core/ext-py/navoptapi-0.1.0/ccscli/thirdparty/six.py
  13. 255 0
      desktop/core/ext-py/navoptapi-0.1.0/ccscli/utils.py
  14. 243 0
      desktop/core/ext-py/navoptapi-0.1.0/ccscli/validate.py
  15. 6 1
      desktop/core/ext-py/navoptapi-0.1.0/navoptapi/__init__.py
  16. 21 0
      desktop/core/ext-py/navoptapi-0.1.0/navoptapi/_version.py
  17. 8 5
      desktop/core/ext-py/navoptapi-0.1.0/navoptapi/api_lib.py
  18. 5 8
      desktop/core/ext-py/navoptapi-0.1.0/navoptapi/auth.py
  19. 165 0
      desktop/core/ext-py/navoptapi-0.1.0/navoptapi/compat.py
  20. 254 9
      desktop/core/ext-py/navoptapi-0.1.0/navoptapi/credentials.py
  21. 130 0
      desktop/core/ext-py/navoptapi-0.1.0/navoptapi/old_auth.py
  22. 48 0
      desktop/core/ext-py/navoptapi-0.1.0/navoptapi/old_credentials.py
  23. 80 0
      desktop/core/ext-py/navoptapi-0.1.0/navoptapi/old_serialize.py
  24. 58 0
      desktop/core/ext-py/navoptapi-0.1.0/navoptapi/old_signers.py
  25. 2 8
      desktop/core/ext-py/navoptapi-0.1.0/navoptapi/serialize.py
  26. 6 5
      desktop/core/ext-py/navoptapi-0.1.0/navoptapi/signers.py
  27. 9 0
      desktop/core/ext-py/navoptapi-0.1.0/setup.cfg
  28. 4 16
      desktop/core/ext-py/navoptapi-0.1.0/setup.py
  29. 1822 0
      desktop/core/ext-py/navoptapi-0.1.0/versioneer.py

+ 4 - 0
desktop/core/ext-py/navoptapi-0.1.0/MANIFEST.in

@@ -0,0 +1,4 @@
+include README.rst
+include versioneer.py
+include ccscli/compat.py
+include navoptapi/_version.py

+ 1 - 1
desktop/core/ext-py/navoptapi-0.1.0/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: navoptapi
-Version: 0.1.0
+Version: 0-untagged.2312.g2ef4bee.dirty
 Summary: Cloudera Navigator Optimizer Api
 Home-page: http://www.cloudera.com/
 Author: UNKNOWN

+ 77 - 0
desktop/core/ext-py/navoptapi-0.1.0/ccscli/__init__.py

@@ -0,0 +1,77 @@
+# Copyright 2012-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Modifications made by Cloudera are:
+#     Copyright (c) 2016 Cloudera, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+import os
+import re
+
+VERSION = "0.1.0"
+
+CCSCLI_ROOT = os.path.dirname(os.path.abspath(__file__))
+
+
+# Used to specify anonymous (unsigned) request signature
+UNSIGNED = object()
+
+
+SCALAR_TYPES = set(['string',
+                    'float',
+                    'integer',
+                    'long',
+                    'boolean',
+                    'double',
+                    'blob',
+                    'timestamp'])
+
+LIST_TYPE = 'array'
+OBJECT_TYPE = 'object'
+REF_KEY = '$ref'
+REF_NAME_PREFIX = '#/definitions/'
+
+COMPLEX_TYPES = set([OBJECT_TYPE,
+                     LIST_TYPE])
+
+DEFAULT_PROFILE_NAME = 'default'
+CCS_ACCESS_KEY_ID_KEY_NAME = 'ccs_access_key_id'
+CCS_PRIVATE_KEY_KEY_NAME = 'ccs_private_key'
+
+# Prepopulate the cache with special cases that don't match our regular
+# transformation.
+_xform_cache = {}
+_first_cap_regex = re.compile('(.)([A-Z][a-z]+)')
+_number_cap_regex = re.compile('([a-z])([0-9]+)')
+_end_cap_regex = re.compile('([a-z0-9])([A-Z])')
+# The regex below handles the special case where some acryonym
+# name is pluralized, e.g GatewayARNs, ListWebACLs, SomeCNAMEs.
+_special_case_transform = re.compile('[A-Z]{3,}s$')
+
+
+def xform_name(name, sep='_', _xform_cache=_xform_cache):
+    if sep in name:
+        # If the sep is in the name, assume that it's already
+        # transformed and return the string unchanged.
+        return name
+    key = (name, sep)
+    if key not in _xform_cache:
+        if _special_case_transform.search(name) is not None:
+            is_special = _special_case_transform.search(name)
+            matched = is_special.group()
+            # Replace something like CRNs, ACLs with _arns, _acls.
+            name = name[:-len(matched)] + sep + matched.lower()
+        s1 = _first_cap_regex.sub(r'\1' + sep + r'\2', name)
+        s2 = _number_cap_regex.sub(r'\1' + sep + r'\2', s1)
+        transformed = _end_cap_regex.sub(r'\1' + sep + r'\2', s2).lower()
+        _xform_cache[key] = transformed
+    return _xform_cache[key]

+ 127 - 0
desktop/core/ext-py/navoptapi-0.1.0/ccscli/auth.py

@@ -0,0 +1,127 @@
+# Copyright 2012-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Modifications made by Cloudera are:
+#     Copyright (c) 2016 Cloudera, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+from base64 import urlsafe_b64encode
+from email.utils import formatdate
+import logging
+
+from ccscli.compat import json
+from ccscli.compat import OrderedDict
+from ccscli.compat import urlsplit
+from ccscli.exceptions import NoCredentialsError
+from Crypto.Hash import SHA256
+from Crypto.PublicKey import RSA
+from Crypto.Signature import PKCS1_v1_5
+
+
+LOG = logging.getLogger('ccscli.auth')
+
+
+class BaseSigner(object):
+    def add_auth(self, request):
+        raise NotImplementedError("add_auth")
+
+
+class RSAv1Auth(BaseSigner):
+    """
+    RSA signing with a SHA-256 hash returning a base64 encoded signature.
+    """
+    AUTH_METHOD_NAME = 'rsav1'
+
+    def __init__(self, credentials):
+        self.credentials = credentials
+
+    def sign_string(self, string_to_sign):
+        try:
+            # We expect the private key to be the an PKCS8 pem formatted string.
+            key = RSA.importKey(self.credentials.private_key)
+        except:
+            message = \
+                "Failed to import private key from: '%s'. The private key is " \
+                "corrupted or it is not in PKCS8 PEM format. The private key " \
+                "was extracted either from 'env' (environment variables), " \
+                "'shared-credentials-file' (a profile in the shared " \
+                "credential file, by default under ~/.ccs/credentials), or " \
+                "'auth-config-file' (a file containing the credentials whose " \
+                "location was supplied on the command line.)" % \
+                self.credentials.method
+            LOG.debug(message, exc_info=True)
+            raise Exception(message)
+        # We sign the hash.
+        h = SHA256.new(string_to_sign.encode('utf-8'))
+        signer = PKCS1_v1_5.new(key)
+        return urlsafe_b64encode(signer.sign(h)).strip().decode('utf-8')
+
+    def canonical_standard_headers(self, headers):
+        interesting_headers = ['content-type', 'x-ccs-date']
+        hoi = []
+        if 'x-ccs-date' in headers:
+            raise Exception("x-ccs-date found in headers!")
+        headers['x-ccs-date'] = self._get_date()
+        for ih in interesting_headers:
+            found = False
+            for key in headers:
+                lk = key.lower()
+                if headers[key] is not None and lk == ih:
+                    hoi.append(headers[key].strip())
+                    found = True
+            if not found:
+                hoi.append('')
+        return '\n'.join(hoi)
+
+    def canonical_string(self, method, split, headers):
+        cs = method.upper() + '\n'
+        cs += self.canonical_standard_headers(headers) + '\n'
+        cs += split.path + '\n'
+        cs += RSAv1Auth.AUTH_METHOD_NAME
+        return cs
+
+    def get_signature(self, method, split, headers):
+        string_to_sign = self.canonical_string(method, split, headers)
+        LOG.debug('StringToSign:\n%s', string_to_sign)
+        return self.sign_string(string_to_sign)
+
+    def add_auth(self, request):
+        if self.credentials is None:
+            raise NoCredentialsError
+        LOG.debug("Calculating signature using RSAv1Auth.")
+        LOG.debug('HTTP request method: %s', request.method)
+        split = urlsplit(request.url)
+        signature = self.get_signature(request.method,
+                                       split,
+                                       request.headers)
+        self._inject_signature(request, signature)
+
+    def _get_date(self):
+        return formatdate(usegmt=True)
+
+    def _inject_signature(self, request, signature):
+        if 'x-ccs-auth' in request.headers:
+            raise Exception("x-ccs-auth found in headers!")
+        request.headers['x-ccs-auth'] = self._get_signature_header(signature)
+
+    def _get_signature_header(self, signature):
+        auth_params = OrderedDict()
+        auth_params['access_key_id'] = self.credentials.access_key_id
+        auth_params['auth_method'] = RSAv1Auth.AUTH_METHOD_NAME
+        encoded_auth_params = json.dumps(auth_params).encode('utf-8')
+        return "%s.%s" % (
+            urlsafe_b64encode(encoded_auth_params).strip().decode('utf-8'),
+            signature)
+
+
+AUTH_TYPE_MAPS = {
+    RSAv1Auth.AUTH_METHOD_NAME: RSAv1Auth,
+}

+ 165 - 0
desktop/core/ext-py/navoptapi-0.1.0/ccscli/compat.py

@@ -0,0 +1,165 @@
+# Copyright 2012-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Modifications made by Cloudera are:
+#     Copyright (c) 2016 Cloudera, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+import copy
+import sys
+
+from ccscli.thirdparty import six  # noqa
+
+if six.PY3:
+    from base64 import encodebytes  # noqa
+    from email.utils import formatdate  # noqa
+    from http.client import HTTPResponse  # noqa
+    import locale
+    from six.moves import http_client
+    from urllib.parse import urlsplit  # noqa
+    from urllib.parse import urlunsplit  # noqa
+
+    raw_input = input
+
+    class HTTPHeaders(http_client.HTTPMessage):
+        pass
+
+    def get_stdout_text_writer():
+        return sys.stdout
+
+    def ensure_unicode(s, encoding=None, errors=None):
+        # NOOP in Python 3, because every string is already unicode
+        return s
+
+    def compat_open(filename, mode='r', encoding=None):
+        """Back-port open() that accepts an encoding argument.
+
+        In python3 this uses the built in open() and in python2 this
+        uses the io.open() function.
+
+        If the file is not being opened in binary mode, then we'll
+        use locale.getpreferredencoding() to find the preferred
+        encoding.
+
+        """
+        if 'b' not in mode:
+            encoding = locale.getpreferredencoding()
+        return open(filename, mode, encoding=encoding)
+
+else:
+    from base64 import encodestring as encodebytes  # noqa
+    import codecs
+    from email.message import Message
+    from email.Utils import formatdate  # noqa
+    from httplib import HTTPResponse  # noqa
+    import io
+    import locale
+    from urlparse import urlsplit  # noqa
+    from urlparse import urlunsplit  # noqa
+
+    raw_input = raw_input
+
+    class HTTPHeaders(Message):
+
+        # The __iter__ method is not available in python2.x, so we have
+        # to port the py3 version.
+        def __iter__(self):
+            for field, value in self._headers:
+                yield field
+
+    def get_stdout_text_writer():
+        # In python3, all the sys.stdout/sys.stderr streams are in text
+        # mode.  This means they expect unicode, and will encode the
+        # unicode automatically before actually writing to stdout/stderr.
+        # In python2, that's not the case.  In order to provide a consistent
+        # interface, we can create a wrapper around sys.stdout that will take
+        # unicode, and automatically encode it to the preferred encoding.
+        # That way consumers can just call get_stdout_text_writer() and write
+        # unicode to the returned stream.  Note that get_stdout_text_writer
+        # just returns sys.stdout in the PY3 section above because python3
+        # handles this.
+        return codecs.getwriter(locale.getpreferredencoding())(sys.stdout)
+
+    def ensure_unicode(s, encoding='utf-8', errors='strict'):
+        if isinstance(s, six.text_type):
+            return s
+        return unicode(s, encoding, errors)
+
+    def compat_open(filename, mode='r', encoding=None):
+        # See docstring for compat_open in the PY3 section above.
+        if 'b' not in mode:
+            encoding = locale.getpreferredencoding()
+        return io.open(filename, mode, encoding=encoding)
+
+try:
+    from collections import OrderedDict
+except ImportError:
+    from ordereddict import OrderedDict  # noqa
+
+if sys.version_info[:2] == (2, 6):
+    import simplejson as json
+else:
+    import json  # noqa
+
+
+@classmethod
+def from_dict(cls, d):
+    new_instance = cls()
+    for key, value in d.items():
+        new_instance[key] = value
+    return new_instance
+
+
+@classmethod
+def from_pairs(cls, pairs):
+    new_instance = cls()
+    for key, value in pairs:
+        new_instance[key] = value
+    return new_instance
+
+
+HTTPHeaders.from_dict = from_dict
+HTTPHeaders.from_pairs = from_pairs
+
+
+def copy_kwargs(kwargs):
+    """
+    There is a bug in Python versions < 2.6.5 that prevents you from passing
+    unicode keyword args (#4978).  This function takes a dictionary of kwargs and
+    returns a copy.  If you are using Python < 2.6.5, it also encodes the keys to
+    avoid this bug. Oh, and version_info wasn't a namedtuple back then, either!
+    """
+    vi = sys.version_info
+    if vi[0] == 2 and vi[1] <= 6 and vi[3] < 5:
+        copy_kwargs = {}
+        for key in kwargs:
+            copy_kwargs[key.encode('utf-8')] = kwargs[key]
+    else:
+        copy_kwargs = copy.copy(kwargs)
+    return copy_kwargs
+
+
+def compat_input(prompt):
+    """
+    Cygwin's pty's are based on pipes. Therefore, when it interacts with a Win32
+    program (such as Win32 python), what that program sees is a pipe instead of
+    a console. This is important because python buffers pipes, and so on a
+    pty-based terminal, text will not necessarily appear immediately. In most
+    cases, this isn't a big deal. But when we're doing an interactive prompt,
+    the result is that the prompts won't display until we fill the buffer. Since
+    raw_input does not flush the prompt, we need to manually write and flush it.
+
+    See https://github.com/mintty/mintty/issues/56 for more details.
+    """
+    sys.stdout.write(prompt)
+    sys.stdout.flush()
+    return raw_input()

+ 243 - 0
desktop/core/ext-py/navoptapi-0.1.0/ccscli/configloader.py

@@ -0,0 +1,243 @@
+# Copyright (c) 2012-2013 Mitch Garnaat http://garnaat.org/
+# Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Modifications made by Cloudera are:
+#     Copyright (c) 2016 Cloudera, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+import copy
+import os
+import shlex
+
+from ccscli.exceptions import ConfigNotFound, ConfigParseError
+
+from six.moves import configparser
+
+
+def multi_file_load_config(*filenames):
+    """Load and combine multiple INI configs with profiles.
+
+    This function will take a list of filesnames and return
+    a single dictionary that represents the merging of the loaded
+    config files.
+
+    If any of the provided filenames does not exist, then that file
+    is ignored.  It is therefore ok to provide a list of filenames,
+    some of which may not exist.
+
+    Configuration files are **not** deep merged, only the top level
+    keys are merged.  The filenames should be passed in order of
+    precedence.  The first config file has precedence over the
+    second config file, which has precedence over the third config file,
+    etc.  The only exception to this is that the "profiles" key is
+    merged to combine profiles from multiple config files into a
+    single profiles mapping.  However, if a profile is defined in
+    multiple config files, then the config file with the highest
+    precedence is used.  Profile values themselves are not merged.
+    For example::
+
+        FileA              FileB                FileC
+        [foo]             [foo]                 [bar]
+        a=1               a=2                   a=3
+                          b=2
+
+        [bar]             [baz]                [profile a]
+        a=2               a=3                  region=e
+
+        [profile a]       [profile b]          [profile c]
+        region=c          region=d             region=f
+
+    The final result of ``multi_file_load_config(FileA, FileB, FileC)``
+    would be::
+
+        {"foo": {"a": 1}, "bar": {"a": 2}, "baz": {"a": 3},
+        "profiles": {"a": {"region": "c"}}, {"b": {"region": d"}},
+                    {"c": {"region": "f"}}}
+
+    Note that the "foo" key comes from A, even though it's defined in both
+    FileA and FileB.  Because "foo" was defined in FileA first, then the values
+    for "foo" from FileA are used and the values for "foo" from FileB are
+    ignored.  Also note where the profiles originate from.  Profile "a"
+    comes FileA, profile "b" comes from FileB, and profile "c" comes
+    from FileC.
+
+    """
+    configs = []
+    profiles = []
+    for filename in filenames:
+        try:
+            loaded = load_config(filename)
+        except ConfigNotFound:
+            continue
+        profiles.append(loaded.pop('profiles'))
+        configs.append(loaded)
+    merged_config = _merge_list_of_dicts(configs)
+    merged_profiles = _merge_list_of_dicts(profiles)
+    merged_config['profiles'] = merged_profiles
+    return merged_config
+
+
+def _merge_list_of_dicts(list_of_dicts):
+    merged_dicts = {}
+    for single_dict in list_of_dicts:
+        for key, value in single_dict.items():
+            if key not in merged_dicts:
+                merged_dicts[key] = value
+    return merged_dicts
+
+
+def load_config(config_filename):
+    """Parse a INI config with profiles.
+
+    This will parse an INI config file and map top level profiles
+    into a top level "profile" key.
+
+    If you want to parse an INI file and map all section names to
+    top level keys, use ``raw_config_parse`` instead.
+
+    """
+    parsed = raw_config_parse(config_filename)
+    return build_profile_map(parsed)
+
+
+def raw_config_parse(config_filename):
+    """Returns the parsed INI config contents.
+
+    Each section name is a top level key.
+
+    :returns: A dict with keys for each profile found in the config
+        file and the value of each key being a dict containing name
+        value pairs found in that profile.
+
+    :raises: ConfigNotFound, ConfigParseError
+    """
+    config = {}
+    path = config_filename
+    if path is not None:
+        path = os.path.expandvars(path)
+        path = os.path.expanduser(path)
+        if not os.path.isfile(path):
+            raise ConfigNotFound(path=path)
+        cp = configparser.RawConfigParser()
+        try:
+            cp.read(path)
+        except configparser.Error:
+            raise ConfigParseError(path=path)
+        else:
+            for section in cp.sections():
+                config[section] = {}
+                for option in cp.options(section):
+                    config_value = cp.get(section, option)
+                    if config_value.startswith('\n'):
+                        # Then we need to parse the inner contents as
+                        # hierarchical.  We support a single level
+                        # of nesting for now.
+                        try:
+                            config_value = _parse_nested(config_value)
+                        except ValueError:
+                            raise ConfigParseError(path=path)
+                    config[section][option] = config_value
+    return config
+
+
+def _parse_nested(config_value):
+    # Given a value like this:
+    # \n
+    # foo = bar
+    # bar = baz
+    # We need to parse this into
+    # {'foo': 'bar', 'bar': 'baz}
+    parsed = {}
+    for line in config_value.splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        # The caller will catch ValueError
+        # and raise an appropriate error
+        # if this fails.
+        key, value = line.split('=', 1)
+        parsed[key.strip()] = value.strip()
+    return parsed
+
+
+def build_profile_map(parsed_ini_config):
+    """Convert the parsed INI config into a profile map.
+
+    The config file format requires that every profile except the
+    default to be prepended with "profile", e.g.::
+
+        [profile test]
+        ccs_... = foo
+        ccs_... = bar
+
+        [profile bar]
+        ccs_... = foo
+        ccs_... = bar
+
+        # This is *not* a profile
+        [preview]
+        otherstuff = 1
+
+        # Neither is this
+        [foobar]
+        morestuff = 2
+
+    The build_profile_map will take a parsed INI config file where each top
+    level key represents a section name, and convert into a format where all
+    the profiles are under a single top level "profiles" key, and each key in
+    the sub dictionary is a profile name.  For example, the above config file
+    would be converted from::
+
+        {"profile test": {"ccs_...": "foo", "ccs...": "bar"},
+         "profile bar": {"ccs...": "foo", "ccs...": "bar"},
+         "preview": {"otherstuff": ...},
+         "foobar": {"morestuff": ...},
+         }
+
+    into::
+
+        {"profiles": {"test": {"ccs_...": "foo", "ccs...": "bar"},
+                      "bar": {"ccs...": "foo", "ccs...": "bar"},
+         "preview": {"otherstuff": ...},
+         "foobar": {"morestuff": ...},
+        }
+
+    If there are no profiles in the provided parsed INI contents, then
+    an empty dict will be the value associated with the ``profiles`` key.
+
+    .. note::
+
+        This will not mutate the passed in parsed_ini_config.  Instead it will
+        make a deepcopy and return that value.
+
+    """
+    parsed_config = copy.deepcopy(parsed_ini_config)
+    profiles = {}
+    final_config = {}
+    for key, values in parsed_config.items():
+        if key.startswith("profile"):
+            try:
+                parts = shlex.split(key)
+            except ValueError:
+                continue
+            if len(parts) == 2:
+                profiles[parts[1]] = values
+        elif key == 'default':
+            # default section is special and is considered a profile
+            # name but we don't require you use 'profile "default"'
+            # as a section.
+            profiles[key] = values
+        else:
+            final_config[key] = values
+    final_config['profiles'] = profiles
+    return final_config

+ 293 - 0
desktop/core/ext-py/navoptapi-0.1.0/ccscli/credentials.py

@@ -0,0 +1,293 @@
+# Copyright 2012-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Modifications made by Cloudera are:
+#     Copyright (c) 2016 Cloudera, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+from collections import namedtuple
+import logging
+import os
+
+from ccscli import CCS_ACCESS_KEY_ID_KEY_NAME, CCS_PRIVATE_KEY_KEY_NAME
+import ccscli.compat
+from ccscli.compat import json
+from ccscli.configloader import raw_config_parse
+from ccscli.exceptions import ConfigNotFound
+from ccscli.exceptions import NoCredentialsError
+from ccscli.exceptions import PartialCredentialsError
+from ccscli.exceptions import UnknownCredentialError
+
+LOG = logging.getLogger('ccscli.credentials')
+ReadOnlyCredentials = namedtuple('ReadOnlyCredentials',
+                                 ['access_key_id', 'private_key', 'method'])
+ACCESS_KEY_ID = 'access_key_id'
+PRIVATE_KEY = 'private_key'
+
+
+def create_credential_resolver(context):
+    """Create a default credential resolver.
+
+    This creates a pre-configured credential resolver
+    that includes the default lookup chain for
+    credentials.
+    """
+    profile_name = context.effective_profile
+    auth_file = context.get_config_variable('auth_config')
+    shared_credential_file = context.get_config_variable('credentials_file')
+
+    env_provider = EnvProvider()
+    providers = [
+        env_provider,
+        AuthConfigFile(auth_file),
+        SharedCredentialProvider(
+            creds_filename=shared_credential_file,
+            profile_name=profile_name
+        ),
+    ]
+
+    explicit_profile = context.get_config_variable('profile',
+                                                   methods=('instance',))
+    if explicit_profile is not None:
+        # An explicitly provided profile will negate an EnvProvider.
+        # We will defer to providers that understand the "profile"
+        # concept to retrieve credentials.
+        # The one edge case is if all three values are provided via
+        # env vars:
+        # export CCS_ACCESS_KEY_ID=foo
+        # export CCS_PRIVATE_KEY=bar
+        # export CCS_PROFILE=baz
+        # Then, just like our client() calls, the explicit credentials
+        # will take precedence.
+        #
+        # This precedence is enforced by leaving the EnvProvider in the chain.
+        # This means that the only way a "profile" would win is if the
+        # EnvProvider does not return credentials, which is what we want
+        # in this scenario.
+        providers.remove(env_provider)
+        LOG.debug('Skipping environment variable credential check because '
+                  'profile name was explicitly set.')
+
+    resolver = CredentialResolver(providers=providers)
+    return resolver
+
+
+def get_credentials(context):
+    resolver = create_credential_resolver(context)
+    return resolver.load_credentials()
+
+
+class Credentials(object):
+    """
+    Holds the credentials needed to authenticate requests.
+    """
+
+    def __init__(self, access_key_id, private_key, method):
+        self.access_key_id = access_key_id
+        self.private_key = private_key
+        self.method = method
+        self._normalize()
+
+    def _normalize(self):
+        self.access_key_id = ccscli.compat.ensure_unicode(self.access_key_id)
+        self.private_key = ccscli.compat.ensure_unicode(self.private_key)
+
+    def get_frozen_credentials(self):
+        return ReadOnlyCredentials(self.access_key_id,
+                                   self.private_key,
+                                   self.method)
+
+
+class CredentialProvider(object):
+
+    # Implementations must provide a method.
+    METHOD = None
+
+    def load(self):
+        return True
+
+    def _extract_creds_from_mapping(self, mapping, *key_names):
+        found = []
+        for key_name in key_names:
+            try:
+                found.append(mapping[key_name])
+            except KeyError:
+                raise PartialCredentialsError(provider=self.METHOD,
+                                              cred_var=key_name)
+        return found
+
+
+class EnvProvider(CredentialProvider):
+    METHOD = 'env'
+    ACCESS_KEY_ID_ENV_VAR = 'CCS_ACCESS_KEY_ID'
+    PRIVATE_KEY_ENV_VAR = 'CCS_PRIVATE_KEY'
+
+    def __init__(self, environ=None, mapping=None):
+        super(EnvProvider, self).__init__()
+        if environ is None:
+            environ = os.environ
+        self.environ = environ
+        self._mapping = self._build_mapping(mapping)
+
+    def _build_mapping(self, mapping):
+        # Mapping of variable name to env var name.
+        var_mapping = {}
+        if mapping is None:
+            # Use the class var default.
+            var_mapping[ACCESS_KEY_ID] = self.ACCESS_KEY_ID_ENV_VAR
+            var_mapping[PRIVATE_KEY] = self.PRIVATE_KEY_ENV_VAR
+        else:
+            var_mapping[ACCESS_KEY_ID] = mapping.get(
+                ACCESS_KEY_ID, self.ACCESS_KEY_ID_ENV_VAR)
+            var_mapping[PRIVATE_KEY] = mapping.get(
+                PRIVATE_KEY, self.PRIVATE_KEY_ENV_VAR)
+        return var_mapping
+
+    def load(self):
+        """
+        Search for credentials in explicit environment variables.
+        """
+        if self._mapping[ACCESS_KEY_ID] in self.environ:
+            access_key_id, private_key = self._extract_creds_from_mapping(
+                self.environ, self._mapping[ACCESS_KEY_ID],
+                self._mapping[PRIVATE_KEY])
+            LOG.info('Found credentials in environment variables.')
+            if not os.path.isfile(private_key):
+                LOG.debug("Private key at %s does not exist!" % private_key)
+                raise NoCredentialsError()
+            pem = open(private_key).read()
+            return Credentials(access_key_id, pem, method=self.METHOD)
+        else:
+            return None
+
+
+class CredentialResolver(object):
+
+    def __init__(self, providers):
+        self.providers = providers
+
+    def insert_before(self, name, credential_provider):
+        """
+        Inserts a new instance of ``CredentialProvider`` into the chain that will
+        be tried before an existing one.
+        """
+        try:
+            offset = [p.METHOD for p in self.providers].index(name)
+        except ValueError:
+            raise UnknownCredentialError(name=name)
+        self.providers.insert(offset, credential_provider)
+
+    def insert_after(self, name, credential_provider):
+        """
+        Inserts a new type of ``Credentials`` instance into the chain that will
+        be tried after an existing one.
+        """
+        offset = self._get_provider_offset(name)
+        self.providers.insert(offset + 1, credential_provider)
+
+    def remove(self, name):
+        """
+        Removes a given ``Credentials`` instance from the chain.
+        """
+        available_methods = [p.METHOD for p in self.providers]
+        if name not in available_methods:
+            # It's not present. Fail silently.
+            return
+
+        offset = available_methods.index(name)
+        self.providers.pop(offset)
+
+    def get_provider(self, name):
+        """
+        Return a credential provider by name.
+        """
+        return self.providers[self._get_provider_offset(name)]
+
+    def _get_provider_offset(self, name):
+        try:
+            return [p.METHOD for p in self.providers].index(name)
+        except ValueError:
+            raise UnknownCredentialError(name=name)
+
+    def load_credentials(self):
+        """
+        Goes through the credentials chain, returning the first ``Credentials``
+        that could be loaded.
+        """
+        # First provider to return a non-None response wins.
+        for provider in self.providers:
+            LOG.debug("Looking for credentials via: %s", provider.METHOD)
+            creds = provider.load()
+            if creds is not None:
+                return creds
+
+        raise NoCredentialsError()
+
+
+class AuthConfigFile(CredentialProvider):
+    METHOD = 'auth_config_file'
+
+    def __init__(self, conf):
+        super(AuthConfigFile, self).__init__()
+        self._conf = conf
+
+    def load(self):
+        """
+        load the credential from the json configuration file.
+        """
+        if self._conf is None:
+            return None
+
+        if not os.path.isfile(self._conf):
+            LOG.debug("Conf file at %s does not exist!" % self._conf)
+            raise NoCredentialsError()
+        try:
+            conf = json.loads(open(self._conf).read())
+        except Exception:
+            LOG.debug("Could not read conf: %s", exc_info=True)
+            return None
+
+        if ACCESS_KEY_ID in conf:
+            LOG.debug('Found credentials for key: %s in configuration file.',
+                      conf[ACCESS_KEY_ID])
+            access_key_id, private_key = self._extract_creds_from_mapping(
+                conf,
+                ACCESS_KEY_ID,
+                PRIVATE_KEY)
+            return Credentials(access_key_id, private_key, self.METHOD)
+        raise NoCredentialsError()
+
+
+class SharedCredentialProvider(CredentialProvider):
+    METHOD = 'shared-credentials-file'
+
+    def __init__(self, creds_filename, profile_name):
+        self._creds_filename = creds_filename
+        self._profile_name = profile_name
+
+    def load(self):
+        try:
+            available_creds = raw_config_parse(self._creds_filename)
+        except ConfigNotFound:
+            return None
+        if self._profile_name in available_creds:
+            config = available_creds[self._profile_name]
+            access_key_id, private_key = self._extract_creds_from_mapping(
+                config, CCS_ACCESS_KEY_ID_KEY_NAME, CCS_PRIVATE_KEY_KEY_NAME)
+            # We store the private key in the credentials file as a one-line
+            # value in which the newlines in the PEM file are replaced with
+            # '\n'. We need to replace them back as the RawConfigParser we use
+            # does not do it for us. Note that if the value in the configuration
+            # IS a PEM formatted value this is a no-op.
+            private_key = private_key.replace('\\n', '\n')
+            LOG.info("Found credentials in shared credentials file: %s",
+                     self._creds_filename)
+            return Credentials(access_key_id, private_key, method=self.METHOD)

+ 179 - 0
desktop/core/ext-py/navoptapi-0.1.0/ccscli/exceptions.py

@@ -0,0 +1,179 @@
+# Copyright 2012-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Modifications made by Cloudera are:
+#     Copyright (c) 2016 Cloudera, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+
+class CCSCLIError(Exception):
+    """
+    The base exception class for CCS CLI exceptions.
+    """
+    fmt = 'An unspecified error occured'
+
+    def __init__(self, **kwargs):
+        msg = self.fmt.format(**kwargs)
+        Exception.__init__(self, msg)
+        self.kwargs = kwargs
+
+
+class ValidationError(CCSCLIError):
+    """
+    An exception occurred validating parameters.
+    """
+    fmt = "Invalid value ('{value}') for param {param} of type {type_name}"
+
+
+class ParamValidationError(CCSCLIError):
+    fmt = 'Parameter validation failed:\n{report}'
+
+
+class DataNotFoundError(CCSCLIError):
+    """
+    The data associated with a particular path could not be loaded.
+    """
+    fmt = 'Unable to load data for: {data_path}'
+
+
+class ExecutableNotFoundError(CCSCLIError):
+    """
+    The executable was not found.
+    """
+    fmt = 'Could not find executable named: {executable_name}'
+
+
+class OperationNotPageableError(CCSCLIError):
+    fmt = 'Operation cannot be paginated: {operation_name}'
+
+
+class ClientError(Exception):
+    MSG_TEMPLATE = (
+        'An error occurred: {error_message} ('
+        'Status Code: {http_status_code}; '
+        'Error Code: {error_code}; '
+        'Service: {service_name}; '
+        'Operation: {operation_name}; '
+        'Request ID: {request_id};)')
+
+    def __init__(self, error_response, operation_name, service_name,
+                 http_status_code, request_id):
+        msg = self.MSG_TEMPLATE.format(
+            error_code=error_response['error'].get('code', 'Unknown'),
+            error_message=error_response['error'].get('message', 'Unknown'),
+            operation_name=operation_name,
+            service_name=service_name,
+            http_status_code=http_status_code,
+            request_id=request_id)
+        super(ClientError, self).__init__(msg)
+        self.response = error_response
+
+
+class UnseekableStreamError(CCSCLIError):
+    """
+    Need to seek a stream, but stream does not support seeking.
+    """
+    fmt = ('Need to rewind the stream {stream_object}, but stream '
+           'is not seekable.')
+
+
+class EndpointConnectionError(CCSCLIError):
+    fmt = (
+        'Could not connect to the endpoint URL: "{endpoint_url}"')
+
+
+class IncompleteReadError(CCSCLIError):
+    """
+    HTTP response did not return expected number of bytes.
+    """
+    fmt = ('{actual_bytes} read, but total bytes '
+           'expected is {expected_bytes}.')
+
+
+class PaginationError(CCSCLIError):
+    fmt = 'Error during pagination: {message}'
+
+
+class UnknownSignatureVersionError(CCSCLIError):
+    """
+    Requested Signature Version is not known.
+    """
+    fmt = 'Unknown Signature Version: {signature_version}.'
+
+
+class UnsupportedSignatureVersionError(CCSCLIError):
+    """
+    Error when trying to access a method on a client that does not exist.
+    """
+    fmt = 'Signature version is not supported: {signature_version}'
+
+
+class NoCredentialsError(CCSCLIError):
+    """
+    No credentials could be found
+    """
+    fmt = 'Unable to locate credentials'
+
+
+class UnknownCredentialError(CCSCLIError):
+    """
+    Tried to insert before/after an unregistered credential type.
+    """
+    fmt = 'Credential named {name} not found.'
+
+
+class PartialCredentialsError(CCSCLIError):
+    """
+    Only partial credentials were found.
+    """
+    fmt = 'Partial credentials found in {provider}, missing: {cred_var}'
+
+
+class BaseEndpointResolverError(CCSCLIError):
+    """
+    Base error for endpoint resolving errors.
+
+    Should never be raised directly, but clients can catch
+    this exception if they want to generically handle any errors
+    during the endpoint resolution process.
+
+    """
+
+
+class NoRegionError(BaseEndpointResolverError):
+    """
+    No region was specified.
+    """
+    fmt = 'You must specify a region.'
+
+
+class ProfileNotFound(CCSCLIError):
+    """
+    The specified configuration profile was not found in the
+    configuration file.
+
+    """
+    fmt = 'The config profile ({profile}) could not be found'
+
+
+class ConfigNotFound(CCSCLIError):
+    """
+    The specified configuration file could not be found.
+    """
+    fmt = 'The specified config file ({path}) could not be found.'
+
+
+class ConfigParseError(CCSCLIError):
+    """
+    The configuration file could not be parsed.
+    """
+    fmt = 'Unable to parse config file: {path}'

+ 81 - 0
desktop/core/ext-py/navoptapi-0.1.0/ccscli/serialize.py

@@ -0,0 +1,81 @@
+# Copyright 2012-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Modifications made by Cloudera are:
+#     Copyright (c) 2016 Cloudera, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+from ccscli import validate
+from ccscli.compat import json
+from ccscli.compat import OrderedDict
+
+
+def create_serializer():
+    serializer = Serializer()
+    validator = validate.ParamValidator()
+    return validate.ParamValidationDecorator(validator, serializer)
+
+
+class Serializer(object):
+    DEFAULT_ENCODING = 'utf-8'
+
+    def serialize_to_request(self, parameters, operation_model):
+        # Don't serialize any parameter with a None value.
+        filtered_parameters = OrderedDict(
+            (k, v) for k, v in parameters.items() if v is not None)
+
+        serialized = {}
+        serialized['method'] = operation_model.http['method']
+        serialized['headers'] = {'Content-Type': 'application/json'}
+        serialized['url_path'] = operation_model.http['requestUri']
+
+        serialized_body = OrderedDict()
+        if len(filtered_parameters) != 0:
+            self._serialize(serialized_body,
+                            filtered_parameters,
+                            operation_model.input_shape)
+        serialized['body'] = json.dumps(serialized_body).encode(self.DEFAULT_ENCODING)
+
+        return serialized
+
+    def _serialize(self, serialized, value, shape, key=None):
+        serialize_method_name = '_serialize_type_%s' % shape.type_name
+        method = getattr(self, serialize_method_name, self._default_serialize)
+        method(serialized, value, shape, key)
+
+    def _serialize_type_object(self, serialized, value, shape, key):
+        if key is not None:
+            # If a key is provided, this is a result of a recursive call, so we
+            # need to add a new child dict as the value of the passed in dict.
+            # Below we will add all the structure members to the new serialized
+            # dictionary we just created.
+            serialized[key] = OrderedDict()
+            serialized = serialized[key]
+
+        for member_key, member_value in value.items():
+            member_shape = shape.members[member_key]
+            self._serialize(serialized, member_value, member_shape, member_key)
+
+    def _serialize_type_array(self, serialized, value, shape, key):
+        array_obj = []
+        serialized[key] = array_obj
+        for array_item in value:
+            wrapper = {}
+            # JSON list serialization is the only case where we aren't setting
+            # a key on a dict.  We handle this by using a __current__ key on a
+            # wrapper dict to serialize each list item before appending it to
+            # the serialized list.
+            self._serialize(wrapper, array_item, shape.member, "__current__")
+            array_obj.append(wrapper["__current__"])
+
+    def _default_serialize(self, serialized, value, shape, key):
+        serialized[key] = value

+ 59 - 0
desktop/core/ext-py/navoptapi-0.1.0/ccscli/signers.py

@@ -0,0 +1,59 @@
+# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Modifications made by Cloudera are:
+#     Copyright (c) 2016 Cloudera, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+from ccscli import UNSIGNED
+import ccscli.auth
+from ccscli.exceptions import UnknownSignatureVersionError
+
+
+class RequestSigner(object):
+    """
+    An object to sign requests before they go out over the wire using
+    one of the authentication mechanisms defined in ``auth.py``.
+    """
+    def __init__(self, signature_version, credentials):
+        self._signature_version = signature_version
+        self._credentials = credentials
+
+    @property
+    def signature_version(self):
+        return self._signature_version
+
+    def sign(self, request):
+        """
+        Sign a request before it goes out over the wire.
+        """
+        if self._signature_version != UNSIGNED:
+            signer = self.get_auth_instance(self._signature_version)
+            signer.add_auth(request)
+
+    def get_auth_instance(self, signature_version, **kwargs):
+        """
+        Get an auth instance which can be used to sign a request
+        using the given signature version.
+        """
+        cls = ccscli.auth.AUTH_TYPE_MAPS.get(signature_version)
+        if cls is None:
+            raise UnknownSignatureVersionError(
+                signature_version=signature_version)
+        # If there's no credentials provided (i.e credentials is None),
+        # then we'll pass a value of "None" over to the auth classes,
+        # which already handle the cases where no credentials have
+        # been provided.
+        frozen_credentials = self._credentials.get_frozen_credentials()
+        kwargs['credentials'] = frozen_credentials
+        auth = cls(**kwargs)
+        return auth

+ 0 - 0
desktop/core/ext-py/navoptapi-0.1.0/ccscli/thirdparty/__init__.py


+ 762 - 0
desktop/core/ext-py/navoptapi-0.1.0/ccscli/thirdparty/six.py

@@ -0,0 +1,762 @@
+"""Utilities for writing code that runs on Python 2 and 3"""
+
+# Copyright (c) 2010-2014 Benjamin Peterson
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+from __future__ import absolute_import
+
+import functools
+import operator
+import sys
+import types
+
+__author__ = "Benjamin Peterson <benjamin@python.org>"
+__version__ = "1.8.0"
+
+
+# Useful for very coarse version differentiation.
+PY2 = sys.version_info[0] == 2
+PY3 = sys.version_info[0] == 3
+
+if PY3:
+    string_types = str,
+    integer_types = int,
+    class_types = type,
+    text_type = str
+    binary_type = bytes
+
+    MAXSIZE = sys.maxsize
+else:
+    string_types = basestring,
+    integer_types = (int, long)
+    class_types = (type, types.ClassType)
+    text_type = unicode
+    binary_type = str
+
+    if sys.platform.startswith("java"):
+        # Jython always uses 32 bits.
+        MAXSIZE = int((1 << 31) - 1)
+    else:
+        # It's possible to have sizeof(long) != sizeof(Py_ssize_t).
+        class X(object):
+            def __len__(self):
+                return 1 << 31
+        try:
+            len(X())
+        except OverflowError:
+            # 32-bit
+            MAXSIZE = int((1 << 31) - 1)
+        else:
+            # 64-bit
+            MAXSIZE = int((1 << 63) - 1)
+        del X
+
+
+def _add_doc(func, doc):
+    """Add documentation to a function."""
+    func.__doc__ = doc
+
+
+def _import_module(name):
+    """Import module, returning the module after the last dot."""
+    __import__(name)
+    return sys.modules[name]
+
+
+class _LazyDescr(object):
+
+    def __init__(self, name):
+        self.name = name
+
+    def __get__(self, obj, tp):
+        result = self._resolve()
+        setattr(obj, self.name, result) # Invokes __set__.
+        # This is a bit ugly, but it avoids running this again.
+        delattr(obj.__class__, self.name)
+        return result
+
+
+class MovedModule(_LazyDescr):
+
+    def __init__(self, name, old, new=None):
+        super(MovedModule, self).__init__(name)
+        if PY3:
+            if new is None:
+                new = name
+            self.mod = new
+        else:
+            self.mod = old
+
+    def _resolve(self):
+        return _import_module(self.mod)
+
+    def __getattr__(self, attr):
+        _module = self._resolve()
+        value = getattr(_module, attr)
+        setattr(self, attr, value)
+        return value
+
+
+class _LazyModule(types.ModuleType):
+
+    def __init__(self, name):
+        super(_LazyModule, self).__init__(name)
+        self.__doc__ = self.__class__.__doc__
+
+    def __dir__(self):
+        attrs = ["__doc__", "__name__"]
+        attrs += [attr.name for attr in self._moved_attributes]
+        return attrs
+
+    # Subclasses should override this
+    _moved_attributes = []
+
+
+class MovedAttribute(_LazyDescr):
+
+    def __init__(self, name, old_mod, new_mod, old_attr=None, new_attr=None):
+        super(MovedAttribute, self).__init__(name)
+        if PY3:
+            if new_mod is None:
+                new_mod = name
+            self.mod = new_mod
+            if new_attr is None:
+                if old_attr is None:
+                    new_attr = name
+                else:
+                    new_attr = old_attr
+            self.attr = new_attr
+        else:
+            self.mod = old_mod
+            if old_attr is None:
+                old_attr = name
+            self.attr = old_attr
+
+    def _resolve(self):
+        module = _import_module(self.mod)
+        return getattr(module, self.attr)
+
+
+class _SixMetaPathImporter(object):
+    """
+    A meta path importer to import six.moves and its submodules.
+
+    This class implements a PEP302 finder and loader. It should be compatible
+    with Python 2.5 and all existing versions of Python3
+    """
+    def __init__(self, six_module_name):
+        self.name = six_module_name
+        self.known_modules = {}
+
+    def _add_module(self, mod, *fullnames):
+        for fullname in fullnames:
+            self.known_modules[self.name + "." + fullname] = mod
+
+    def _get_module(self, fullname):
+        return self.known_modules[self.name + "." + fullname]
+
+    def find_module(self, fullname, path=None):
+        if fullname in self.known_modules:
+            return self
+        return None
+
+    def __get_module(self, fullname):
+        try:
+            return self.known_modules[fullname]
+        except KeyError:
+            raise ImportError("This loader does not know module " + fullname)
+
+    def load_module(self, fullname):
+        try:
+            # in case of a reload
+            return sys.modules[fullname]
+        except KeyError:
+            pass
+        mod = self.__get_module(fullname)
+        if isinstance(mod, MovedModule):
+            mod = mod._resolve()
+        else:
+            mod.__loader__ = self
+        sys.modules[fullname] = mod
+        return mod
+
+    def is_package(self, fullname):
+        """
+        Return true, if the named module is a package.
+
+        We need this method to get correct spec objects with
+        Python 3.4 (see PEP451)
+        """
+        return hasattr(self.__get_module(fullname), "__path__")
+
+    def get_code(self, fullname):
+        """Return None
+
+        Required, if is_package is implemented"""
+        self.__get_module(fullname)  # eventually raises ImportError
+        return None
+    get_source = get_code  # same as get_code
+
+_importer = _SixMetaPathImporter(__name__)
+
+
+class _MovedItems(_LazyModule):
+    """Lazy loading of moved objects"""
+    __path__ = []  # mark as package
+
+
+_moved_attributes = [
+    MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"),
+    MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"),
+    MovedAttribute("filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse"),
+    MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"),
+    MovedAttribute("intern", "__builtin__", "sys"),
+    MovedAttribute("map", "itertools", "builtins", "imap", "map"),
+    MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"),
+    MovedAttribute("reload_module", "__builtin__", "imp", "reload"),
+    MovedAttribute("reduce", "__builtin__", "functools"),
+    MovedAttribute("shlex_quote", "pipes", "shlex", "quote"),
+    MovedAttribute("StringIO", "StringIO", "io"),
+    MovedAttribute("UserDict", "UserDict", "collections"),
+    MovedAttribute("UserList", "UserList", "collections"),
+    MovedAttribute("UserString", "UserString", "collections"),
+    MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"),
+    MovedAttribute("zip", "itertools", "builtins", "izip", "zip"),
+    MovedAttribute("zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"),
+
+    MovedModule("builtins", "__builtin__"),
+    MovedModule("configparser", "ConfigParser"),
+    MovedModule("copyreg", "copy_reg"),
+    MovedModule("dbm_gnu", "gdbm", "dbm.gnu"),
+    MovedModule("_dummy_thread", "dummy_thread", "_dummy_thread"),
+    MovedModule("http_cookiejar", "cookielib", "http.cookiejar"),
+    MovedModule("http_cookies", "Cookie", "http.cookies"),
+    MovedModule("html_entities", "htmlentitydefs", "html.entities"),
+    MovedModule("html_parser", "HTMLParser", "html.parser"),
+    MovedModule("http_client", "httplib", "http.client"),
+    MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"),
+    MovedModule("email_mime_nonmultipart", "email.MIMENonMultipart", "email.mime.nonmultipart"),
+    MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"),
+    MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"),
+    MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"),
+    MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"),
+    MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"),
+    MovedModule("cPickle", "cPickle", "pickle"),
+    MovedModule("queue", "Queue"),
+    MovedModule("reprlib", "repr"),
+    MovedModule("socketserver", "SocketServer"),
+    MovedModule("_thread", "thread", "_thread"),
+    MovedModule("tkinter", "Tkinter"),
+    MovedModule("tkinter_dialog", "Dialog", "tkinter.dialog"),
+    MovedModule("tkinter_filedialog", "FileDialog", "tkinter.filedialog"),
+    MovedModule("tkinter_scrolledtext", "ScrolledText", "tkinter.scrolledtext"),
+    MovedModule("tkinter_simpledialog", "SimpleDialog", "tkinter.simpledialog"),
+    MovedModule("tkinter_tix", "Tix", "tkinter.tix"),
+    MovedModule("tkinter_ttk", "ttk", "tkinter.ttk"),
+    MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"),
+    MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"),
+    MovedModule("tkinter_colorchooser", "tkColorChooser",
+                "tkinter.colorchooser"),
+    MovedModule("tkinter_commondialog", "tkCommonDialog",
+                "tkinter.commondialog"),
+    MovedModule("tkinter_tkfiledialog", "tkFileDialog", "tkinter.filedialog"),
+    MovedModule("tkinter_font", "tkFont", "tkinter.font"),
+    MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"),
+    MovedModule("tkinter_tksimpledialog", "tkSimpleDialog",
+                "tkinter.simpledialog"),
+    MovedModule("urllib_parse", __name__ + ".moves.urllib_parse", "urllib.parse"),
+    MovedModule("urllib_error", __name__ + ".moves.urllib_error", "urllib.error"),
+    MovedModule("urllib", __name__ + ".moves.urllib", __name__ + ".moves.urllib"),
+    MovedModule("urllib_robotparser", "robotparser", "urllib.robotparser"),
+    MovedModule("xmlrpc_client", "xmlrpclib", "xmlrpc.client"),
+    MovedModule("xmlrpc_server", "SimpleXMLRPCServer", "xmlrpc.server"),
+    MovedModule("winreg", "_winreg"),
+]
+for attr in _moved_attributes:
+    setattr(_MovedItems, attr.name, attr)
+    if isinstance(attr, MovedModule):
+        _importer._add_module(attr, "moves." + attr.name)
+del attr
+
+_MovedItems._moved_attributes = _moved_attributes
+
+moves = _MovedItems(__name__ + ".moves")
+_importer._add_module(moves, "moves")
+
+
+class Module_six_moves_urllib_parse(_LazyModule):
+    """Lazy loading of moved objects in six.moves.urllib_parse"""
+
+
+_urllib_parse_moved_attributes = [
+    MovedAttribute("ParseResult", "urlparse", "urllib.parse"),
+    MovedAttribute("SplitResult", "urlparse", "urllib.parse"),
+    MovedAttribute("parse_qs", "urlparse", "urllib.parse"),
+    MovedAttribute("parse_qsl", "urlparse", "urllib.parse"),
+    MovedAttribute("urldefrag", "urlparse", "urllib.parse"),
+    MovedAttribute("urljoin", "urlparse", "urllib.parse"),
+    MovedAttribute("urlparse", "urlparse", "urllib.parse"),
+    MovedAttribute("urlsplit", "urlparse", "urllib.parse"),
+    MovedAttribute("urlunparse", "urlparse", "urllib.parse"),
+    MovedAttribute("urlunsplit", "urlparse", "urllib.parse"),
+    MovedAttribute("quote", "urllib", "urllib.parse"),
+    MovedAttribute("quote_plus", "urllib", "urllib.parse"),
+    MovedAttribute("unquote", "urllib", "urllib.parse"),
+    MovedAttribute("unquote_plus", "urllib", "urllib.parse"),
+    MovedAttribute("urlencode", "urllib", "urllib.parse"),
+    MovedAttribute("splitquery", "urllib", "urllib.parse"),
+    MovedAttribute("splittag", "urllib", "urllib.parse"),
+    MovedAttribute("splituser", "urllib", "urllib.parse"),
+    MovedAttribute("uses_fragment", "urlparse", "urllib.parse"),
+    MovedAttribute("uses_netloc", "urlparse", "urllib.parse"),
+    MovedAttribute("uses_params", "urlparse", "urllib.parse"),
+    MovedAttribute("uses_query", "urlparse", "urllib.parse"),
+    MovedAttribute("uses_relative", "urlparse", "urllib.parse"),
+]
+for attr in _urllib_parse_moved_attributes:
+    setattr(Module_six_moves_urllib_parse, attr.name, attr)
+del attr
+
+Module_six_moves_urllib_parse._moved_attributes = _urllib_parse_moved_attributes
+
+_importer._add_module(Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse"),
+                      "moves.urllib_parse", "moves.urllib.parse")
+
+
+class Module_six_moves_urllib_error(_LazyModule):
+    """Lazy loading of moved objects in six.moves.urllib_error"""
+
+
+_urllib_error_moved_attributes = [
+    MovedAttribute("URLError", "urllib2", "urllib.error"),
+    MovedAttribute("HTTPError", "urllib2", "urllib.error"),
+    MovedAttribute("ContentTooShortError", "urllib", "urllib.error"),
+]
+for attr in _urllib_error_moved_attributes:
+    setattr(Module_six_moves_urllib_error, attr.name, attr)
+del attr
+
+Module_six_moves_urllib_error._moved_attributes = _urllib_error_moved_attributes
+
+_importer._add_module(Module_six_moves_urllib_error(__name__ + ".moves.urllib.error"),
+                      "moves.urllib_error", "moves.urllib.error")
+
+
+class Module_six_moves_urllib_request(_LazyModule):
+    """Lazy loading of moved objects in six.moves.urllib_request"""
+
+
+_urllib_request_moved_attributes = [
+    MovedAttribute("urlopen", "urllib2", "urllib.request"),
+    MovedAttribute("install_opener", "urllib2", "urllib.request"),
+    MovedAttribute("build_opener", "urllib2", "urllib.request"),
+    MovedAttribute("pathname2url", "urllib", "urllib.request"),
+    MovedAttribute("url2pathname", "urllib", "urllib.request"),
+    MovedAttribute("getproxies", "urllib", "urllib.request"),
+    MovedAttribute("Request", "urllib2", "urllib.request"),
+    MovedAttribute("OpenerDirector", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPDefaultErrorHandler", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPRedirectHandler", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPCookieProcessor", "urllib2", "urllib.request"),
+    MovedAttribute("ProxyHandler", "urllib2", "urllib.request"),
+    MovedAttribute("BaseHandler", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPPasswordMgr", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPPasswordMgrWithDefaultRealm", "urllib2", "urllib.request"),
+    MovedAttribute("AbstractBasicAuthHandler", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPBasicAuthHandler", "urllib2", "urllib.request"),
+    MovedAttribute("ProxyBasicAuthHandler", "urllib2", "urllib.request"),
+    MovedAttribute("AbstractDigestAuthHandler", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPDigestAuthHandler", "urllib2", "urllib.request"),
+    MovedAttribute("ProxyDigestAuthHandler", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPHandler", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPSHandler", "urllib2", "urllib.request"),
+    MovedAttribute("FileHandler", "urllib2", "urllib.request"),
+    MovedAttribute("FTPHandler", "urllib2", "urllib.request"),
+    MovedAttribute("CacheFTPHandler", "urllib2", "urllib.request"),
+    MovedAttribute("UnknownHandler", "urllib2", "urllib.request"),
+    MovedAttribute("HTTPErrorProcessor", "urllib2", "urllib.request"),
+    MovedAttribute("urlretrieve", "urllib", "urllib.request"),
+    MovedAttribute("urlcleanup", "urllib", "urllib.request"),
+    MovedAttribute("URLopener", "urllib", "urllib.request"),
+    MovedAttribute("FancyURLopener", "urllib", "urllib.request"),
+    MovedAttribute("proxy_bypass", "urllib", "urllib.request"),
+]
+for attr in _urllib_request_moved_attributes:
+    setattr(Module_six_moves_urllib_request, attr.name, attr)
+del attr
+
+Module_six_moves_urllib_request._moved_attributes = _urllib_request_moved_attributes
+
+_importer._add_module(Module_six_moves_urllib_request(__name__ + ".moves.urllib.request"),
+                      "moves.urllib_request", "moves.urllib.request")
+
+
+class Module_six_moves_urllib_response(_LazyModule):
+    """Lazy loading of moved objects in six.moves.urllib_response"""
+
+
+_urllib_response_moved_attributes = [
+    MovedAttribute("addbase", "urllib", "urllib.response"),
+    MovedAttribute("addclosehook", "urllib", "urllib.response"),
+    MovedAttribute("addinfo", "urllib", "urllib.response"),
+    MovedAttribute("addinfourl", "urllib", "urllib.response"),
+]
+for attr in _urllib_response_moved_attributes:
+    setattr(Module_six_moves_urllib_response, attr.name, attr)
+del attr
+
+Module_six_moves_urllib_response._moved_attributes = _urllib_response_moved_attributes
+
+_importer._add_module(Module_six_moves_urllib_response(__name__ + ".moves.urllib.response"),
+                      "moves.urllib_response", "moves.urllib.response")
+
+
+class Module_six_moves_urllib_robotparser(_LazyModule):
+    """Lazy loading of moved objects in six.moves.urllib_robotparser"""
+
+
+_urllib_robotparser_moved_attributes = [
+    MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser"),
+]
+for attr in _urllib_robotparser_moved_attributes:
+    setattr(Module_six_moves_urllib_robotparser, attr.name, attr)
+del attr
+
+Module_six_moves_urllib_robotparser._moved_attributes = _urllib_robotparser_moved_attributes
+
+_importer._add_module(Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser"),
+                      "moves.urllib_robotparser", "moves.urllib.robotparser")
+
+
+class Module_six_moves_urllib(types.ModuleType):
+    """Create a six.moves.urllib namespace that resembles the Python 3 namespace"""
+    __path__ = []  # mark as package
+    parse = _importer._get_module("moves.urllib_parse")
+    error = _importer._get_module("moves.urllib_error")
+    request = _importer._get_module("moves.urllib_request")
+    response = _importer._get_module("moves.urllib_response")
+    robotparser = _importer._get_module("moves.urllib_robotparser")
+
+    def __dir__(self):
+        return ['parse', 'error', 'request', 'response', 'robotparser']
+
+_importer._add_module(Module_six_moves_urllib(__name__ + ".moves.urllib"),
+                      "moves.urllib")
+
+
+def add_move(move):
+    """Add an item to six.moves."""
+    setattr(_MovedItems, move.name, move)
+
+
+def remove_move(name):
+    """Remove item from six.moves."""
+    try:
+        delattr(_MovedItems, name)
+    except AttributeError:
+        try:
+            del moves.__dict__[name]
+        except KeyError:
+            raise AttributeError("no such move, %r" % (name,))
+
+
+if PY3:
+    _meth_func = "__func__"
+    _meth_self = "__self__"
+
+    _func_closure = "__closure__"
+    _func_code = "__code__"
+    _func_defaults = "__defaults__"
+    _func_globals = "__globals__"
+else:
+    _meth_func = "im_func"
+    _meth_self = "im_self"
+
+    _func_closure = "func_closure"
+    _func_code = "func_code"
+    _func_defaults = "func_defaults"
+    _func_globals = "func_globals"
+
+
+try:
+    advance_iterator = next
+except NameError:
+    def advance_iterator(it):
+        return it.next()
+next = advance_iterator
+
+
+try:
+    callable = callable
+except NameError:
+    def callable(obj):
+        return any("__call__" in klass.__dict__ for klass in type(obj).__mro__)
+
+
+if PY3:
+    def get_unbound_function(unbound):
+        return unbound
+
+    create_bound_method = types.MethodType
+
+    Iterator = object
+else:
+    def get_unbound_function(unbound):
+        return unbound.im_func
+
+    def create_bound_method(func, obj):
+        return types.MethodType(func, obj, obj.__class__)
+
+    class Iterator(object):
+
+        def next(self):
+            return type(self).__next__(self)
+
+    callable = callable
+_add_doc(get_unbound_function,
+         """Get the function out of a possibly unbound function""")
+
+
+get_method_function = operator.attrgetter(_meth_func)
+get_method_self = operator.attrgetter(_meth_self)
+get_function_closure = operator.attrgetter(_func_closure)
+get_function_code = operator.attrgetter(_func_code)
+get_function_defaults = operator.attrgetter(_func_defaults)
+get_function_globals = operator.attrgetter(_func_globals)
+
+
+if PY3:
+    def iterkeys(d, **kw):
+        return iter(d.keys(**kw))
+
+    def itervalues(d, **kw):
+        return iter(d.values(**kw))
+
+    def iteritems(d, **kw):
+        return iter(d.items(**kw))
+
+    def iterlists(d, **kw):
+        return iter(d.lists(**kw))
+else:
+    def iterkeys(d, **kw):
+        return iter(d.iterkeys(**kw))
+
+    def itervalues(d, **kw):
+        return iter(d.itervalues(**kw))
+
+    def iteritems(d, **kw):
+        return iter(d.iteritems(**kw))
+
+    def iterlists(d, **kw):
+        return iter(d.iterlists(**kw))
+
+_add_doc(iterkeys, "Return an iterator over the keys of a dictionary.")
+_add_doc(itervalues, "Return an iterator over the values of a dictionary.")
+_add_doc(iteritems,
+         "Return an iterator over the (key, value) pairs of a dictionary.")
+_add_doc(iterlists,
+         "Return an iterator over the (key, [values]) pairs of a dictionary.")
+
+
+if PY3:
+    def b(s):
+        return s.encode("latin-1")
+    def u(s):
+        return s
+    unichr = chr
+    if sys.version_info[1] <= 1:
+        def int2byte(i):
+            return bytes((i,))
+    else:
+        # This is about 2x faster than the implementation above on 3.2+
+        int2byte = operator.methodcaller("to_bytes", 1, "big")
+    byte2int = operator.itemgetter(0)
+    indexbytes = operator.getitem
+    iterbytes = iter
+    import io
+    StringIO = io.StringIO
+    BytesIO = io.BytesIO
+else:
+    def b(s):
+        return s
+    # Workaround for standalone backslash
+    def u(s):
+        return unicode(s.replace(r'\\', r'\\\\'), "unicode_escape")
+    unichr = unichr
+    int2byte = chr
+    def byte2int(bs):
+        return ord(bs[0])
+    def indexbytes(buf, i):
+        return ord(buf[i])
+    def iterbytes(buf):
+        return (ord(byte) for byte in buf)
+    import StringIO
+    StringIO = BytesIO = StringIO.StringIO
+_add_doc(b, """Byte literal""")
+_add_doc(u, """Text literal""")
+
+
+if PY3:
+    exec_ = getattr(moves.builtins, "exec")
+
+
+    def reraise(tp, value, tb=None):
+        if value is None:
+            value = tp()
+        if value.__traceback__ is not tb:
+            raise value.with_traceback(tb)
+        raise value
+
+else:
+    def exec_(_code_, _globs_=None, _locs_=None):
+        """Execute code in a namespace."""
+        if _globs_ is None:
+            frame = sys._getframe(1)
+            _globs_ = frame.f_globals
+            if _locs_ is None:
+                _locs_ = frame.f_locals
+            del frame
+        elif _locs_ is None:
+            _locs_ = _globs_
+        exec("""exec _code_ in _globs_, _locs_""")
+
+
+    exec_("""def reraise(tp, value, tb=None):
+    raise tp, value, tb
+""")
+
+
+print_ = getattr(moves.builtins, "print", None)
+if print_ is None:
+    def print_(*args, **kwargs):
+        """The new-style print function for Python 2.4 and 2.5."""
+        fp = kwargs.pop("file", sys.stdout)
+        if fp is None:
+            return
+        def write(data):
+            if not isinstance(data, basestring):
+                data = str(data)
+            # If the file has an encoding, encode unicode with it.
+            if (isinstance(fp, file) and
+                isinstance(data, unicode) and
+                fp.encoding is not None):
+                errors = getattr(fp, "errors", None)
+                if errors is None:
+                    errors = "strict"
+                data = data.encode(fp.encoding, errors)
+            fp.write(data)
+        want_unicode = False
+        sep = kwargs.pop("sep", None)
+        if sep is not None:
+            if isinstance(sep, unicode):
+                want_unicode = True
+            elif not isinstance(sep, str):
+                raise TypeError("sep must be None or a string")
+        end = kwargs.pop("end", None)
+        if end is not None:
+            if isinstance(end, unicode):
+                want_unicode = True
+            elif not isinstance(end, str):
+                raise TypeError("end must be None or a string")
+        if kwargs:
+            raise TypeError("invalid keyword arguments to print()")
+        if not want_unicode:
+            for arg in args:
+                if isinstance(arg, unicode):
+                    want_unicode = True
+                    break
+        if want_unicode:
+            newline = unicode("\n")
+            space = unicode(" ")
+        else:
+            newline = "\n"
+            space = " "
+        if sep is None:
+            sep = space
+        if end is None:
+            end = newline
+        for i, arg in enumerate(args):
+            if i:
+                write(sep)
+            write(arg)
+        write(end)
+
+_add_doc(reraise, """Reraise an exception.""")
+
+if sys.version_info[0:2] < (3, 4):
+    def wraps(wrapped, assigned=functools.WRAPPER_ASSIGNMENTS,
+              updated=functools.WRAPPER_UPDATES):
+        def wrapper(f):
+            f = functools.wraps(wrapped)(f)
+            f.__wrapped__ = wrapped
+            return f
+        return wrapper
+else:
+    wraps = functools.wraps
+
+def with_metaclass(meta, *bases):
+    """Create a base class with a metaclass."""
+    # This requires a bit of explanation: the basic idea is to make a dummy
+    # metaclass for one level of class instantiation that replaces itself with
+    # the actual metaclass.
+    class metaclass(meta):
+        def __new__(cls, name, this_bases, d):
+            return meta(name, bases, d)
+    return type.__new__(metaclass, 'temporary_class', (), {})
+
+
+def add_metaclass(metaclass):
+    """Class decorator for creating a class with a metaclass."""
+    def wrapper(cls):
+        orig_vars = cls.__dict__.copy()
+        slots = orig_vars.get('__slots__')
+        if slots is not None:
+            if isinstance(slots, str):
+                slots = [slots]
+            for slots_var in slots:
+                orig_vars.pop(slots_var)
+        orig_vars.pop('__dict__', None)
+        orig_vars.pop('__weakref__', None)
+        return metaclass(cls.__name__, cls.__bases__, orig_vars)
+    return wrapper
+
+# Complete the moves implementation.
+# This code is at the end of this module to speed up module loading.
+# Turn this module into a package.
+__path__ = []  # required for PEP 302 and PEP 451
+__package__ = __name__  # see PEP 366 @ReservedAssignment
+if globals().get("__spec__") is not None:
+    __spec__.submodule_search_locations = []  # PEP 451 @UndefinedVariable
+# Remove other six meta path importers, since they cause problems. This can
+# happen if six is removed from sys.modules and then reloaded. (Setuptools does
+# this for some reason.)
+if sys.meta_path:
+    for i, importer in enumerate(sys.meta_path):
+        # Here's some real nastiness: Another "instance" of the six module might
+        # be floating around. Therefore, we can't use isinstance() to check for
+        # the six meta path importer, since the other six instance will have
+        # inserted an importer with different class.
+        if (type(importer).__name__ == "_SixMetaPathImporter" and
+            importer.name == __name__):
+            del sys.meta_path[i]
+            break
+    del i, importer
+# Finally, add the importer to the meta path import hook.
+sys.meta_path.append(_importer)

+ 255 - 0
desktop/core/ext-py/navoptapi-0.1.0/ccscli/utils.py

@@ -0,0 +1,255 @@
+# Copyright 2012-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Modifications made by Cloudera are:
+#     Copyright (c) 2016 Cloudera, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+import contextlib
+import datetime
+import functools
+import re
+import signal
+
+from ccscli import LIST_TYPE
+from ccscli import OBJECT_TYPE
+from ccscli.compat import OrderedDict
+import dateutil.parser
+from dateutil.tz import tzlocal
+from dateutil.tz import tzutc
+
+# These are chars that do not need to be urlencoded based on rfc2986, section 2.3.
+SAFE_CHARS = '-._~'
+
+
+def get_service_module_name(service_model):
+    name = service_model.service_name
+    name = name.replace('Cloudera', '')
+    name = name.replace('CCS', '')
+    name = re.sub('\W+', '', name)
+    return name
+
+
+def json_encoder(obj):
+    """JSON encoder that formats datetimes as ISO8601 format."""
+    if isinstance(obj, datetime.datetime):
+        return obj.isoformat()
+    else:
+        return obj
+
+
+class CachedProperty(object):
+    def __init__(self, fget):
+        self._fget = fget
+
+    def __get__(self, obj, cls):
+        if obj is None:
+            return self
+        else:
+            computed_value = self._fget(obj)
+            obj.__dict__[self._fget.__name__] = computed_value
+            return computed_value
+
+
+def instance_cache(func):
+    """Method decorator for caching method calls to a single instance.
+
+    **This is not a general purpose caching decorator.**
+
+    In order to use this, you *must* provide an ``_instance_cache``
+    attribute on the instance.
+
+    This decorator is used to cache method calls.  The cache is only
+    scoped to a single instance though such that multiple instances
+    will maintain their own cache.  In order to keep things simple,
+    this decorator requires that you provide an ``_instance_cache``
+    attribute on your instance.
+
+    """
+    func_name = func.__name__
+
+    @functools.wraps(func)
+    def _cache_guard(self, *args, **kwargs):
+        cache_key = (func_name, args)
+        if kwargs:
+            kwarg_items = tuple(sorted(kwargs.items()))
+            cache_key = (func_name, args, kwarg_items)
+        result = self._instance_cache.get(cache_key)
+        if result is not None:
+            return result
+        result = func(self, *args, **kwargs)
+        self._instance_cache[cache_key] = result
+        return result
+    return _cache_guard
+
+
+def parse_timestamp(value):
+    """Parse a timestamp into a datetime object.
+
+    Supported formats:
+
+        * iso8601
+        * rfc822
+        * epoch (value is an integer)
+
+    This will return a ``datetime.datetime`` object.
+
+    """
+    if isinstance(value, (int, float)):
+        # Possibly an epoch time.
+        return datetime.datetime.fromtimestamp(value, tzlocal())
+    else:
+        try:
+            return datetime.datetime.fromtimestamp(float(value), tzlocal())
+        except (TypeError, ValueError):
+            pass
+    try:
+        return dateutil.parser.parse(value)
+    except (TypeError, ValueError) as e:
+        raise ValueError('Invalid timestamp "%s": %s' % (value, e))
+
+
+def parse_to_aware_datetime(value):
+    """Converted the passed in value to a datetime object with tzinfo.
+
+    This function can be used to normalize all timestamp inputs.  This
+    function accepts a number of different types of inputs, but
+    will always return a datetime.datetime object with time zone
+    information.
+
+    The input param ``value`` can be one of several types:
+
+        * A datetime object (both naive and aware)
+        * An integer representing the epoch time (can also be a string
+          of the integer, i.e '0', instead of 0).  The epoch time is
+          considered to be UTC.
+        * An iso8601 formatted timestamp.  This does not need to be
+          a complete timestamp, it can contain just the date portion
+          without the time component.
+
+    The returned value will be a datetime object that will have tzinfo.
+    If no timezone info was provided in the input value, then UTC is
+    assumed, not local time.
+
+    """
+    # This is a general purpose method that handles several cases of
+    # converting the provided value to a string timestamp suitable to be
+    # serialized to an http request. It can handle:
+    # 1) A datetime.datetime object.
+    if isinstance(value, datetime.datetime):
+        datetime_obj = value
+    else:
+        # 2) A string object that's formatted as a timestamp.
+        #    We document this as being an iso8601 timestamp, although
+        #    parse_timestamp is a bit more flexible.
+        datetime_obj = parse_timestamp(value)
+    if datetime_obj.tzinfo is None:
+        # I think a case would be made that if no time zone is provided,
+        # we should use the local time.  However, to restore backwards
+        # compat, the previous behavior was to assume UTC, which is
+        # what we're going to do here.
+        datetime_obj = datetime_obj.replace(tzinfo=tzutc())
+    else:
+        datetime_obj = datetime_obj.astimezone(tzutc())
+    return datetime_obj
+
+
+@contextlib.contextmanager
+def ignore_ctrl_c():
+    original = signal.signal(signal.SIGINT, signal.SIG_IGN)
+    try:
+        yield
+    finally:
+        signal.signal(signal.SIGINT, original)
+
+
+def datetime2timestamp(dt, default_timezone=None):
+    """Calculate the timestamp based on the given datetime instance.
+
+    :type dt: datetime
+    :param dt: A datetime object to be converted into timestamp
+    :type default_timezone: tzinfo
+    :param default_timezone: If it is provided as None, we treat it as tzutc().
+                             But it is only used when dt is a naive datetime.
+    :returns: The timestamp
+    """
+    epoch = datetime.datetime(1970, 1, 1)
+    if dt.tzinfo is None:
+        if default_timezone is None:
+            default_timezone = tzutc()
+        dt = dt.replace(tzinfo=default_timezone)
+    d = dt.replace(tzinfo=None) - dt.utcoffset() - epoch
+    if hasattr(d, "total_seconds"):
+        return d.total_seconds()  # Works in Python 2.7+
+    return (d.microseconds + (d.seconds + d.days * 24 * 3600) * 10**6) / 10**6
+
+
+class ArgumentGenerator(object):
+    """Generate sample input based on a shape model.
+
+    This class contains a ``generate_skeleton`` method that will take
+    an input shape (created from ``ccscli.model``) and generate
+    a sample dictionary corresponding to the input shape.
+
+    The specific values used are place holder values. For strings an
+    empty string is used, for numbers 0 or 0.0 is used. For datetime a date in
+    RFC822 is used. The intended usage of this class is to generate the *shape*
+    of the input object. In the future we might take the defaults from the
+    model.
+
+    This can be useful for operations that have complex input shapes.
+    This allows a user to just fill in the necessary data instead of
+    worrying about the specific object of the input arguments.
+
+    Example usage::
+
+        clidriver = CLIDriver
+        ddb = clidriver.get_service_model('mastodon')
+        arg_gen = ArgumentGenerator()
+        sample_input = arg_gen.generate_skeleton(
+            ddb.operation_model('createCluster').input_shape)
+        print("Sample input for mastodon.createCluster: %s" % sample_input)
+
+    """
+    def __init__(self):
+        pass
+
+    def generate_skeleton(self, shape):
+        if shape.type_name == OBJECT_TYPE:
+            return self._generate_type_object(shape)
+        elif shape.type_name == LIST_TYPE:
+            return self._generate_type_array(shape)
+        elif shape.type_name == 'string':
+            return ''
+        elif shape.type_name in ['integer']:
+            return 0
+        elif shape.type_name == 'number':
+            return 0.0
+        elif shape.type_name == 'boolean':
+            return True
+        elif shape.type_name == 'datetime':
+            return 'Wed, 02 Oct 2002 13:00:00 GMT'
+        else:
+            raise Exception("Unknown shape type: %s" % shape.type_name)
+
+    def _generate_type_object(self, shape):
+        skeleton = OrderedDict()
+        for member_name, member_shape in shape.members.items():
+            skeleton[member_name] = self.generate_skeleton(member_shape)
+        return skeleton
+
+    def _generate_type_array(self, shape):
+        # For list elements we've arbitrarily decided to return the first
+        # element for the skeleton list.
+        return [
+            self.generate_skeleton(shape.member),
+        ]

+ 243 - 0
desktop/core/ext-py/navoptapi-0.1.0/ccscli/validate.py

@@ -0,0 +1,243 @@
+# Copyright 2012-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Modifications made by Cloudera are:
+#     Copyright (c) 2016 Cloudera, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+import datetime
+import decimal
+
+from ccscli.compat import six
+from ccscli.exceptions import ParamValidationError
+from ccscli.utils import parse_to_aware_datetime
+
+
+def validate_parameters(params, shape):
+    """Validates input parameters against a schema.
+
+    This is a convenience function that validates parameters against a schema.
+    You can also instantiate and use the ParamValidator class directly if you
+    want more control.
+
+    If there are any validation errors then a ParamValidationError
+    will be raised.  If there are no validation errors than no exception
+    is raised and a value of None is returned.
+    """
+    validator = ParamValidator()
+    report = validator.validate(params, shape)
+    if report.has_errors():
+        raise ParamValidationError(report=report.generate_report())
+
+
+def type_check(valid_types):
+    def _create_type_check_guard(func):
+        def _on_passes_type_check(self, param, shape, errors, name):
+            if _type_check(param, errors, name):
+                return func(self, param, shape, errors, name)
+
+        def _type_check(param, errors, name):
+            if not isinstance(param, valid_types):
+                valid_type_names = [six.text_type(t) for t in valid_types]
+                errors.report(name, 'invalid type', param=param,
+                              valid_types=valid_type_names)
+                return False
+            return True
+
+        return _on_passes_type_check
+    return _create_type_check_guard
+
+
+def range_check(name, value, shape, error_type, errors):
+    failed = False
+    min_allowed = float('-inf')
+    max_allowed = float('inf')
+    if shape.minimum is not None:
+        min_allowed = shape.minimum
+        if value < min_allowed:
+            failed = True
+    if shape.maximum is not None:
+        max_allowed = shape.maximum
+        if value > max_allowed:
+            failed = True
+    if failed:
+        errors.report(name, error_type, param=value,
+                      valid_range=[min_allowed, max_allowed])
+
+
+def length_check(name, value, shape, error_type, errors):
+    failed = False
+    min_allowed = float('-inf')
+    max_allowed = float('inf')
+    if shape.min_length is not None:
+        min_allowed = shape.min_length
+        if value < min_allowed:
+            failed = True
+    if shape.max_length is not None:
+        max_allowed = shape.max_length
+        if value > max_allowed:
+            failed = True
+    if failed:
+        errors.report(name, error_type, param=value,
+                      valid_range=[min_allowed, max_allowed])
+
+
+class ValidationErrors(object):
+    def __init__(self):
+        self._errors = []
+
+    def has_errors(self):
+        if self._errors:
+            return True
+        return False
+
+    def generate_report(self):
+        error_messages = []
+        for error in self._errors:
+            error_messages.append(self._format_error(error))
+        return '\n'.join(error_messages)
+
+    def _format_error(self, error):
+        error_type, name, additional = error
+        name = self._get_name(name)
+        if error_type == 'missing required field':
+            return 'Missing required parameter in %s: "%s"' % (
+                name, additional['required_name'])
+        elif error_type == 'unknown field':
+            return 'Unknown parameter in %s: "%s", must be one of: %s' % (
+                name, additional['unknown_param'], ', '.join(additional['valid_names']))
+        elif error_type == 'invalid type':
+            return 'Invalid type for parameter %s, value: %s, type: %s, valid types: %s' \
+                % (name, additional['param'],
+                   str(type(additional['param'])),
+                   ', '.join(additional['valid_types']))
+        elif error_type == 'invalid enum':
+            return ('Invalid value for parameter %s, value: %s, type: %s, valid '
+                    'values: %s') \
+                % (name, additional['param'],
+                   str(type(additional['param'])),
+                   ', '.join(additional['valid_values']))
+        elif error_type == 'invalid range':
+            min_allowed = additional['valid_range'][0]
+            max_allowed = additional['valid_range'][1]
+            return ('Invalid range for parameter %s, value: %s, valid range: '
+                    '%s-%s' % (name, additional['param'],
+                               min_allowed, max_allowed))
+        elif error_type == 'invalid length':
+            min_allowed = additional['valid_range'][0]
+            max_allowed = additional['valid_range'][1]
+            return ('Invalid length for parameter %s, value: %s, valid range: '
+                    '%s-%s' % (name, additional['param'],
+                               min_allowed, max_allowed))
+
+    def _get_name(self, name):
+        if not name:
+            return 'input'
+        elif name.startswith('.'):
+            return name[1:]
+        else:
+            return name
+
+    def report(self, name, reason, **kwargs):
+        self._errors.append((reason, name, kwargs))
+
+
+class ParamValidator(object):
+
+    def validate(self, params, shape):
+        errors = ValidationErrors()
+        self._validate(params, shape, errors, name='')
+        return errors
+
+    def _validate(self, params, shape, errors, name):
+        getattr(self, '_validate_%s' % shape.type_name)(params, shape, errors, name)
+
+    @type_check(valid_types=(dict,))
+    def _validate_object(self, params, shape, errors, name):
+        # Validate required fields.
+        for required_member in shape.required_members:
+            if required_member not in params:
+                errors.report(name, 'missing required field',
+                              required_name=required_member, user_params=params)
+        members = shape.members
+        known_params = []
+        # Validate known params.
+        for param in params:
+            if param not in members:
+                errors.report(name, 'unknown field', unknown_param=param,
+                              valid_names=list(members))
+            else:
+                known_params.append(param)
+        # Validate structure members.
+        for param in known_params:
+            self._validate(params[param], shape.members[param],
+                           errors, '%s.%s' % (name, param))
+
+    @type_check(valid_types=(bool,))
+    def _validate_boolean(self, param, shape, errors, name):
+        pass
+
+    @type_check(valid_types=six.integer_types)
+    def _validate_integer(self, param, shape, errors, name):
+        range_check(name, param, shape, 'invalid range', errors)
+
+    @type_check(valid_types=(float, decimal.Decimal) + six.integer_types)
+    def _validate_number(self, param, shape, errors, name):
+        range_check(name, param, shape, 'invalid range', errors)
+
+    @type_check(valid_types=six.string_types)
+    def _validate_string(self, param, shape, errors, name):
+        if len(shape.enum) > 0 and param not in shape.enum:
+            errors.report(name, 'invalid enum', param=param,
+                          valid_values=shape.enum)
+        length_check(name, len(param), shape, 'invalid length', errors)
+
+    @type_check(valid_types=(list, tuple))
+    def _validate_array(self, param, shape, errors, name):
+        member_shape = shape.member
+        length_check(name, len(param), shape, 'invalid length', errors)
+        for i, item in enumerate(param):
+            self._validate(item, member_shape, errors, '%s[%s]' % (name, i))
+
+    def _validate_datetime(self, param, shape, errors, name):
+        # We don't use @type_check because datetimes are a bit more flexible.
+        # You can either provide a datetime object, or a string that parses
+        # to a datetime.
+        is_valid_type = self._type_check_datetime(param)
+        if not is_valid_type:
+            valid_type_names = [six.text_type(datetime), 'timestamp-string']
+            errors.report(name, 'invalid type', param=param,
+                          valid_types=valid_type_names)
+
+    def _type_check_datetime(self, value):
+        try:
+            parse_to_aware_datetime(value)
+            return True
+        except (TypeError, ValueError, AttributeError):
+            # Yes, dateutil can sometimes raise an AttributeError when parsing
+            # timestamps.
+            return False
+
+
+class ParamValidationDecorator(object):
+    def __init__(self, param_validator, serializer):
+        self._param_validator = param_validator
+        self._serializer = serializer
+
+    def serialize_to_request(self, parameters, operation_model):
+        input_shape = operation_model.input_shape
+        if input_shape is not None:
+            report = self._param_validator.validate(parameters,
+                                                    operation_model.input_shape)
+            if report.has_errors():
+                raise ParamValidationError(report=report.generate_report())
+        return self._serializer.serialize_to_request(parameters, operation_model)

+ 6 - 1
desktop/core/ext-py/navoptapi-0.1.0/navoptapi/__init__.py

@@ -17,7 +17,12 @@
 import os
 import re
 
-VERSION = "0.1.0"
+from ._version import get_versions
+
+__version__ = get_versions()['version']
+del get_versions
+
+VERSION = __version__
 
 CCSCLI_ROOT = os.path.dirname(os.path.abspath(__file__))
 

+ 21 - 0
desktop/core/ext-py/navoptapi-0.1.0/navoptapi/_version.py

@@ -0,0 +1,21 @@
+
+# This file was generated by 'versioneer.py' (0.17) from
+# revision-control system data, or from the parent directory name of an
+# unpacked source archive. Distribution tarballs contain a pre-generated copy
+# of this file.
+
+import json
+
+version_json = '''
+{
+ "date": "2017-01-20T09:07:40-0800",
+ "dirty": true,
+ "error": null,
+ "full-revisionid": "2ef4beebbe2cfac713b2422ce1e2b71612a87b60",
+ "version": "0+untagged.2312.g2ef4bee.dirty"
+}
+'''  # END VERSION_JSON
+
+
+def get_versions():
+    return json.loads(version_json)

+ 8 - 5
desktop/core/ext-py/navoptapi-0.1.0/navoptapi/api_lib.py

@@ -19,10 +19,11 @@ import logging
 import os
 import platform
 
-from navoptapi.auth import RSAv1Auth
-from navoptapi.credentials import Credentials
+from ccscli.auth import RSAv1Auth
+from ccscli.credentials import Credentials
+from ccscli.signers import RequestSigner
+
 from navoptapi.serialize import Serializer
-from navoptapi.signers import RequestSigner
 
 from requests import put, Request, Session
 
@@ -42,7 +43,7 @@ class ApiLib(object):
         # get Credentials
         self._access_key = access_key
         self._private_key = private_key
-        self._endpoint_url = "http://"+host_name+":8982/"+service_name+"/"
+        self._endpoint_url = "http://" + host_name + ":8982/" + service_name + "/"
         self._service_name = service_name
         self._cred = Credentials(access_key, private_key,
                                  method='shared-credentials-file')
@@ -100,7 +101,7 @@ class ApiLib(object):
             put(resp['url'], data=open(params['fileLocation']).read())
             # build upload parameters
             upload_params = {'rowDelim': '', 'colDelim': '', 'headerFields': [],
-                             'tenant': ''}
+                             'tenant': '', 'fileType': 0}
             # now do actual upload
             if 'tenant' in params and params['tenant']:
                 upload_params['tenant'] = params['tenant']
@@ -118,6 +119,8 @@ class ApiLib(object):
                 upload_params['rowDelim'] = params['rowDelim']
             if 'headerFields' in params and params['headerFields']:
                 upload_params['headerFields'] = params['headerFields']
+            if 'fileType' in params and params['fileType']:
+                upload_params['fileType'] = params['fileType']
             # prepare the body
             serializer = Serializer()
             serial_obj = serializer.serialize_to_request(upload_params, None)

+ 5 - 8
desktop/core/ext-py/navoptapi-0.1.0/navoptapi/auth.py

@@ -14,16 +14,13 @@
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
 from base64 import urlsafe_b64encode
-try:
-  from collections import OrderedDict
-except ImportError:
-  from ordereddict import OrderedDict # Python 2.6
 from email.utils import formatdate
-import json
 import logging
 
-from urlparse import urlsplit
-
+from ccscli.compat import json
+from ccscli.compat import OrderedDict
+from ccscli.compat import urlsplit
+from ccscli.exceptions import NoCredentialsError
 from Crypto.Hash import SHA256
 from Crypto.PublicKey import RSA
 from Crypto.Signature import PKCS1_v1_5
@@ -98,7 +95,7 @@ class RSAv1Auth(BaseSigner):
 
     def add_auth(self, request):
         if self.credentials is None:
-            return
+            raise NoCredentialsError
         LOG.debug("Calculating signature using RSAv1Auth.")
         LOG.debug('HTTP request method: %s', request.method)
         split = urlsplit(request.url)

+ 165 - 0
desktop/core/ext-py/navoptapi-0.1.0/navoptapi/compat.py

@@ -0,0 +1,165 @@
+# Copyright 2012-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Modifications made by Cloudera are:
+#     Copyright (c) 2016 Cloudera, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+import copy
+import sys
+
+from ccscli.thirdparty import six  # noqa
+
+if six.PY3:
+    from base64 import encodebytes  # noqa
+    from email.utils import formatdate  # noqa
+    from http.client import HTTPResponse  # noqa
+    import locale
+    from six.moves import http_client
+    from urllib.parse import urlsplit  # noqa
+    from urllib.parse import urlunsplit  # noqa
+
+    raw_input = input
+
+    class HTTPHeaders(http_client.HTTPMessage):
+        pass
+
+    def get_stdout_text_writer():
+        return sys.stdout
+
+    def ensure_unicode(s, encoding=None, errors=None):
+        # NOOP in Python 3, because every string is already unicode
+        return s
+
+    def compat_open(filename, mode='r', encoding=None):
+        """Back-port open() that accepts an encoding argument.
+
+        In python3 this uses the built in open() and in python2 this
+        uses the io.open() function.
+
+        If the file is not being opened in binary mode, then we'll
+        use locale.getpreferredencoding() to find the preferred
+        encoding.
+
+        """
+        if 'b' not in mode:
+            encoding = locale.getpreferredencoding()
+        return open(filename, mode, encoding=encoding)
+
+else:
+    from base64 import encodestring as encodebytes  # noqa
+    import codecs
+    from email.message import Message
+    from email.Utils import formatdate  # noqa
+    from httplib import HTTPResponse  # noqa
+    import io
+    import locale
+    from urlparse import urlsplit  # noqa
+    from urlparse import urlunsplit  # noqa
+
+    raw_input = raw_input
+
+    class HTTPHeaders(Message):
+
+        # The __iter__ method is not available in python2.x, so we have
+        # to port the py3 version.
+        def __iter__(self):
+            for field, value in self._headers:
+                yield field
+
+    def get_stdout_text_writer():
+        # In python3, all the sys.stdout/sys.stderr streams are in text
+        # mode.  This means they expect unicode, and will encode the
+        # unicode automatically before actually writing to stdout/stderr.
+        # In python2, that's not the case.  In order to provide a consistent
+        # interface, we can create a wrapper around sys.stdout that will take
+        # unicode, and automatically encode it to the preferred encoding.
+        # That way consumers can just call get_stdout_text_writer() and write
+        # unicode to the returned stream.  Note that get_stdout_text_writer
+        # just returns sys.stdout in the PY3 section above because python3
+        # handles this.
+        return codecs.getwriter(locale.getpreferredencoding())(sys.stdout)
+
+    def ensure_unicode(s, encoding='utf-8', errors='strict'):
+        if isinstance(s, six.text_type):
+            return s
+        return unicode(s, encoding, errors)
+
+    def compat_open(filename, mode='r', encoding=None):
+        # See docstring for compat_open in the PY3 section above.
+        if 'b' not in mode:
+            encoding = locale.getpreferredencoding()
+        return io.open(filename, mode, encoding=encoding)
+
+try:
+    from collections import OrderedDict
+except ImportError:
+    from ordereddict import OrderedDict  # noqa
+
+if sys.version_info[:2] == (2, 6):
+    import simplejson as json
+else:
+    import json  # noqa
+
+
+@classmethod
+def from_dict(cls, d):
+    new_instance = cls()
+    for key, value in d.items():
+        new_instance[key] = value
+    return new_instance
+
+
+@classmethod
+def from_pairs(cls, pairs):
+    new_instance = cls()
+    for key, value in pairs:
+        new_instance[key] = value
+    return new_instance
+
+
+HTTPHeaders.from_dict = from_dict
+HTTPHeaders.from_pairs = from_pairs
+
+
+def copy_kwargs(kwargs):
+    """
+    There is a bug in Python versions < 2.6.5 that prevents you from passing
+    unicode keyword args (#4978).  This function takes a dictionary of kwargs and
+    returns a copy.  If you are using Python < 2.6.5, it also encodes the keys to
+    avoid this bug. Oh, and version_info wasn't a namedtuple back then, either!
+    """
+    vi = sys.version_info
+    if vi[0] == 2 and vi[1] <= 6 and vi[3] < 5:
+        copy_kwargs = {}
+        for key in kwargs:
+            copy_kwargs[key.encode('utf-8')] = kwargs[key]
+    else:
+        copy_kwargs = copy.copy(kwargs)
+    return copy_kwargs
+
+
+def compat_input(prompt):
+    """
+    Cygwin's pty's are based on pipes. Therefore, when it interacts with a Win32
+    program (such as Win32 python), what that program sees is a pipe instead of
+    a console. This is important because python buffers pipes, and so on a
+    pty-based terminal, text will not necessarily appear immediately. In most
+    cases, this isn't a big deal. But when we're doing an interactive prompt,
+    the result is that the prompts won't display until we fill the buffer. Since
+    raw_input does not flush the prompt, we need to manually write and flush it.
+
+    See https://github.com/mintty/mintty/issues/56 for more details.
+    """
+    sys.stdout.write(prompt)
+    sys.stdout.flush()
+    return raw_input()

+ 254 - 9
desktop/core/ext-py/navoptapi-0.1.0/navoptapi/credentials.py

@@ -13,13 +13,76 @@
 # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
-
 from collections import namedtuple
+import logging
+import os
 
-import six
+from ccscli import CCS_ACCESS_KEY_ID_KEY_NAME, CCS_PRIVATE_KEY_KEY_NAME
+import ccscli.compat
+from ccscli.compat import json
+from ccscli.configloader import raw_config_parse
+from ccscli.exceptions import ConfigNotFound
+from ccscli.exceptions import NoCredentialsError
+from ccscli.exceptions import PartialCredentialsError
+from ccscli.exceptions import UnknownCredentialError
 
+LOG = logging.getLogger('ccscli.credentials')
 ReadOnlyCredentials = namedtuple('ReadOnlyCredentials',
                                  ['access_key_id', 'private_key', 'method'])
+ACCESS_KEY_ID = 'access_key_id'
+PRIVATE_KEY = 'private_key'
+
+
+def create_credential_resolver(context):
+    """Create a default credential resolver.
+
+    This creates a pre-configured credential resolver
+    that includes the default lookup chain for
+    credentials.
+    """
+    profile_name = context.effective_profile
+    auth_file = context.get_config_variable('auth_config')
+    shared_credential_file = context.get_config_variable('credentials_file')
+
+    env_provider = EnvProvider()
+    providers = [
+        env_provider,
+        AuthConfigFile(auth_file),
+        SharedCredentialProvider(
+            creds_filename=shared_credential_file,
+            profile_name=profile_name
+        ),
+    ]
+
+    explicit_profile = context.get_config_variable('profile',
+                                                   methods=('instance',))
+    if explicit_profile is not None:
+        # An explicitly provided profile will negate an EnvProvider.
+        # We will defer to providers that understand the "profile"
+        # concept to retrieve credentials.
+        # The one edge case is if all three values are provided via
+        # env vars:
+        # export CCS_ACCESS_KEY_ID=foo
+        # export CCS_PRIVATE_KEY=bar
+        # export CCS_PROFILE=baz
+        # Then, just like our client() calls, the explicit credentials
+        # will take precedence.
+        #
+        # This precedence is enforced by leaving the EnvProvider in the chain.
+        # This means that the only way a "profile" would win is if the
+        # EnvProvider does not return credentials, which is what we want
+        # in this scenario.
+        providers.remove(env_provider)
+        LOG.debug('Skipping environment variable credential check because '
+                  'profile name was explicitly set.')
+
+    resolver = CredentialResolver(providers=providers)
+    return resolver
+
+
+def get_credentials(context):
+    resolver = create_credential_resolver(context)
+    return resolver.load_credentials()
 
 
 class Credentials(object):
@@ -33,16 +96,198 @@ class Credentials(object):
         self.method = method
         self._normalize()
 
-    def ensure_unicode(self, s, encoding='utf-8', errors='strict'):
-        if isinstance(s, six.text_type):
-            return s
-        return unicode(s, encoding, errors)
-
     def _normalize(self):
-        self.access_key_id = self.ensure_unicode(self.access_key_id)
-        self.private_key = self.ensure_unicode(self.private_key)
+        self.access_key_id = ccscli.compat.ensure_unicode(self.access_key_id)
+        self.private_key = ccscli.compat.ensure_unicode(self.private_key)
 
     def get_frozen_credentials(self):
         return ReadOnlyCredentials(self.access_key_id,
                                    self.private_key,
                                    self.method)
+
+
+class CredentialProvider(object):
+
+    # Implementations must provide a method.
+    METHOD = None
+
+    def load(self):
+        return True
+
+    def _extract_creds_from_mapping(self, mapping, *key_names):
+        found = []
+        for key_name in key_names:
+            try:
+                found.append(mapping[key_name])
+            except KeyError:
+                raise PartialCredentialsError(provider=self.METHOD,
+                                              cred_var=key_name)
+        return found
+
+
+class EnvProvider(CredentialProvider):
+    METHOD = 'env'
+    ACCESS_KEY_ID_ENV_VAR = 'CCS_ACCESS_KEY_ID'
+    PRIVATE_KEY_ENV_VAR = 'CCS_PRIVATE_KEY'
+
+    def __init__(self, environ=None, mapping=None):
+        super(EnvProvider, self).__init__()
+        if environ is None:
+            environ = os.environ
+        self.environ = environ
+        self._mapping = self._build_mapping(mapping)
+
+    def _build_mapping(self, mapping):
+        # Mapping of variable name to env var name.
+        var_mapping = {}
+        if mapping is None:
+            # Use the class var default.
+            var_mapping[ACCESS_KEY_ID] = self.ACCESS_KEY_ID_ENV_VAR
+            var_mapping[PRIVATE_KEY] = self.PRIVATE_KEY_ENV_VAR
+        else:
+            var_mapping[ACCESS_KEY_ID] = mapping.get(
+                ACCESS_KEY_ID, self.ACCESS_KEY_ID_ENV_VAR)
+            var_mapping[PRIVATE_KEY] = mapping.get(
+                PRIVATE_KEY, self.PRIVATE_KEY_ENV_VAR)
+        return var_mapping
+
+    def load(self):
+        """
+        Search for credentials in explicit environment variables.
+        """
+        if self._mapping[ACCESS_KEY_ID] in self.environ:
+            access_key_id, private_key = self._extract_creds_from_mapping(
+                self.environ, self._mapping[ACCESS_KEY_ID],
+                self._mapping[PRIVATE_KEY])
+            LOG.info('Found credentials in environment variables.')
+            if not os.path.isfile(private_key):
+                LOG.debug("Private key at %s does not exist!" % private_key)
+                raise NoCredentialsError()
+            pem = open(private_key).read()
+            return Credentials(access_key_id, pem, method=self.METHOD)
+        else:
+            return None
+
+
+class CredentialResolver(object):
+
+    def __init__(self, providers):
+        self.providers = providers
+
+    def insert_before(self, name, credential_provider):
+        """
+        Inserts a new instance of ``CredentialProvider`` into the chain that will
+        be tried before an existing one.
+        """
+        try:
+            offset = [p.METHOD for p in self.providers].index(name)
+        except ValueError:
+            raise UnknownCredentialError(name=name)
+        self.providers.insert(offset, credential_provider)
+
+    def insert_after(self, name, credential_provider):
+        """
+        Inserts a new type of ``Credentials`` instance into the chain that will
+        be tried after an existing one.
+        """
+        offset = self._get_provider_offset(name)
+        self.providers.insert(offset + 1, credential_provider)
+
+    def remove(self, name):
+        """
+        Removes a given ``Credentials`` instance from the chain.
+        """
+        available_methods = [p.METHOD for p in self.providers]
+        if name not in available_methods:
+            # It's not present. Fail silently.
+            return
+
+        offset = available_methods.index(name)
+        self.providers.pop(offset)
+
+    def get_provider(self, name):
+        """
+        Return a credential provider by name.
+        """
+        return self.providers[self._get_provider_offset(name)]
+
+    def _get_provider_offset(self, name):
+        try:
+            return [p.METHOD for p in self.providers].index(name)
+        except ValueError:
+            raise UnknownCredentialError(name=name)
+
+    def load_credentials(self):
+        """
+        Goes through the credentials chain, returning the first ``Credentials``
+        that could be loaded.
+        """
+        # First provider to return a non-None response wins.
+        for provider in self.providers:
+            LOG.debug("Looking for credentials via: %s", provider.METHOD)
+            creds = provider.load()
+            if creds is not None:
+                return creds
+
+        raise NoCredentialsError()
+
+
+class AuthConfigFile(CredentialProvider):
+    METHOD = 'auth_config_file'
+
+    def __init__(self, conf):
+        super(AuthConfigFile, self).__init__()
+        self._conf = conf
+
+    def load(self):
+        """
+        load the credential from the json configuration file.
+        """
+        if self._conf is None:
+            return None
+
+        if not os.path.isfile(self._conf):
+            LOG.debug("Conf file at %s does not exist!" % self._conf)
+            raise NoCredentialsError()
+        try:
+            conf = json.loads(open(self._conf).read())
+        except Exception:
+            LOG.debug("Could not read conf: %s", exc_info=True)
+            return None
+
+        if ACCESS_KEY_ID in conf:
+            LOG.debug('Found credentials for key: %s in configuration file.',
+                      conf[ACCESS_KEY_ID])
+            access_key_id, private_key = self._extract_creds_from_mapping(
+                conf,
+                ACCESS_KEY_ID,
+                PRIVATE_KEY)
+            return Credentials(access_key_id, private_key, self.METHOD)
+        raise NoCredentialsError()
+
+
+class SharedCredentialProvider(CredentialProvider):
+    METHOD = 'shared-credentials-file'
+
+    def __init__(self, creds_filename, profile_name):
+        self._creds_filename = creds_filename
+        self._profile_name = profile_name
+
+    def load(self):
+        try:
+            available_creds = raw_config_parse(self._creds_filename)
+        except ConfigNotFound:
+            return None
+        if self._profile_name in available_creds:
+            config = available_creds[self._profile_name]
+            access_key_id, private_key = self._extract_creds_from_mapping(
+                config, CCS_ACCESS_KEY_ID_KEY_NAME, CCS_PRIVATE_KEY_KEY_NAME)
+            # We store the private key in the credentials file as a one-line
+            # value in which the newlines in the PEM file are replaced with
+            # '\n'. We need to replace them back as the RawConfigParser we use
+            # does not do it for us. Note that if the value in the configuration
+            # IS a PEM formatted value this is a no-op.
+            private_key = private_key.replace('\\n', '\n')
+            LOG.info("Found credentials in shared credentials file: %s",
+                     self._creds_filename)
+            return Credentials(access_key_id, private_key, method=self.METHOD)

+ 130 - 0
desktop/core/ext-py/navoptapi-0.1.0/navoptapi/old_auth.py

@@ -0,0 +1,130 @@
+# Copyright 2012-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Modifications made by Cloudera are:
+#     Copyright (c) 2016 Cloudera, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+from base64 import urlsafe_b64encode
+try:
+    from collections import OrderedDict
+except ImportError:
+    from ordereddict import OrderedDict   # Python 2.6
+from email.utils import formatdate
+import json
+import logging
+
+from urlparse import urlsplit
+
+from Crypto.Hash import SHA256
+from Crypto.PublicKey import RSA
+from Crypto.Signature import PKCS1_v1_5
+
+
+LOG = logging.getLogger('ccscli.auth')
+
+
+class BaseSigner(object):
+    def add_auth(self, request):
+        raise NotImplementedError("add_auth")
+
+
+class RSAv1Auth(BaseSigner):
+    """
+    RSA signing with a SHA-256 hash returning a base64 encoded signature.
+    """
+    AUTH_METHOD_NAME = 'rsav1'
+
+    def __init__(self, credentials):
+        self.credentials = credentials
+
+    def sign_string(self, string_to_sign):
+        try:
+            # We expect the private key to be the an PKCS8 pem formatted string.
+            key = RSA.importKey(self.credentials.private_key)
+        except:
+            message = \
+                "Failed to import private key from: '%s'. The private key is " \
+                "corrupted or it is not in PKCS8 PEM format. The private key " \
+                "was extracted either from 'env' (environment variables), " \
+                "'shared-credentials-file' (a profile in the shared " \
+                "credential file, by default under ~/.ccs/credentials), or " \
+                "'auth-config-file' (a file containing the credentials whose " \
+                "location was supplied on the command line.)" % \
+                self.credentials.method
+            LOG.debug(message, exc_info=True)
+            raise Exception(message)
+        # We sign the hash.
+        h = SHA256.new(string_to_sign.encode('utf-8'))
+        signer = PKCS1_v1_5.new(key)
+        return urlsafe_b64encode(signer.sign(h)).strip().decode('utf-8')
+
+    def canonical_standard_headers(self, headers):
+        interesting_headers = ['content-type', 'x-ccs-date']
+        hoi = []
+        if 'x-ccs-date' in headers:
+            raise Exception("x-ccs-date found in headers!")
+        headers['x-ccs-date'] = self._get_date()
+        for ih in interesting_headers:
+            found = False
+            for key in headers:
+                lk = key.lower()
+                if headers[key] is not None and lk == ih:
+                    hoi.append(headers[key].strip())
+                    found = True
+            if not found:
+                hoi.append('')
+        return '\n'.join(hoi)
+
+    def canonical_string(self, method, split, headers):
+        cs = method.upper() + '\n'
+        cs += self.canonical_standard_headers(headers) + '\n'
+        cs += split.path + '\n'
+        cs += RSAv1Auth.AUTH_METHOD_NAME
+        return cs
+
+    def get_signature(self, method, split, headers):
+        string_to_sign = self.canonical_string(method, split, headers)
+        LOG.debug('StringToSign:\n%s', string_to_sign)
+        return self.sign_string(string_to_sign)
+
+    def add_auth(self, request):
+        if self.credentials is None:
+            return
+        LOG.debug("Calculating signature using RSAv1Auth.")
+        LOG.debug('HTTP request method: %s', request.method)
+        split = urlsplit(request.url)
+        signature = self.get_signature(request.method,
+                                       split,
+                                       request.headers)
+        self._inject_signature(request, signature)
+
+    def _get_date(self):
+        return formatdate(usegmt=True)
+
+    def _inject_signature(self, request, signature):
+        if 'x-ccs-auth' in request.headers:
+            raise Exception("x-ccs-auth found in headers!")
+        request.headers['x-ccs-auth'] = self._get_signature_header(signature)
+
+    def _get_signature_header(self, signature):
+        auth_params = OrderedDict()
+        auth_params['access_key_id'] = self.credentials.access_key_id
+        auth_params['auth_method'] = RSAv1Auth.AUTH_METHOD_NAME
+        encoded_auth_params = json.dumps(auth_params).encode('utf-8')
+        return "%s.%s" % (
+            urlsafe_b64encode(encoded_auth_params).strip().decode('utf-8'),
+            signature)
+
+
+AUTH_TYPE_MAPS = {
+    RSAv1Auth.AUTH_METHOD_NAME: RSAv1Auth,
+}

+ 48 - 0
desktop/core/ext-py/navoptapi-0.1.0/navoptapi/old_credentials.py

@@ -0,0 +1,48 @@
+# Copyright 2012-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Modifications made by Cloudera are:
+#     Copyright (c) 2016 Cloudera, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+from collections import namedtuple
+
+import six
+
+ReadOnlyCredentials = namedtuple('ReadOnlyCredentials',
+                                 ['access_key_id', 'private_key', 'method'])
+
+
+class Credentials(object):
+    """
+    Holds the credentials needed to authenticate requests.
+    """
+
+    def __init__(self, access_key_id, private_key, method):
+        self.access_key_id = access_key_id
+        self.private_key = private_key
+        self.method = method
+        self._normalize()
+
+    def ensure_unicode(self, s, encoding='utf-8', errors='strict'):
+        if isinstance(s, six.text_type):
+            return s
+        return unicode(s, encoding, errors)
+
+    def _normalize(self):
+        self.access_key_id = self.ensure_unicode(self.access_key_id)
+        self.private_key = self.ensure_unicode(self.private_key)
+
+    def get_frozen_credentials(self):
+        return ReadOnlyCredentials(self.access_key_id,
+                                   self.private_key,
+                                   self.method)

+ 80 - 0
desktop/core/ext-py/navoptapi-0.1.0/navoptapi/old_serialize.py

@@ -0,0 +1,80 @@
+# Copyright 2012-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Modifications made by Cloudera are:
+#     Copyright (c) 2016 Cloudera, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+try:
+    from collections import OrderedDict
+except ImportError:
+    from ordereddict import OrderedDict  # Python 2.6
+import json
+
+
+class Serializer(object):
+    DEFAULT_ENCODING = 'utf-8'
+
+    def serialize_to_request(self, parameters, operation_model):
+        # Don't serialize any parameter with a None value.
+        filtered_parameters = OrderedDict(
+            (k, v) for k, v in parameters.items() if v is not None)
+
+        serialized = {}
+        # serialized['method'] = operation_model.http['method']
+        # serialized['headers'] = {'Content-Type': 'application/json'}
+        # serialized['url_path'] = operation_model.http['requestUri']
+
+        serialized_body = OrderedDict()
+        if len(filtered_parameters) != 0:
+            self._serialize(serialized_body, filtered_parameters, None)
+
+        serialized['body'] = json.dumps(serialized_body).encode(self.DEFAULT_ENCODING)
+
+        return serialized
+
+    def _serialize(self, serialized, value, shape, key=None):
+        # serialize_method_name = '_serialize_type_%s' % shape.type_name
+        # method = getattr(self, serialize_method_name, self._default_serialize)
+        self._default_serialize(serialized, value, shape, key)
+
+    def _serialize_type_object(self, serialized, value, shape, key):
+        if key is not None:
+            # If a key is provided, this is a result of a recursive call, so we
+            # need to add a new child dict as the value of the passed in dict.
+            # Below we will add all the structure members to the new serialized
+            # dictionary we just created.
+            serialized[key] = OrderedDict()
+            serialized = serialized[key]
+
+        for member_key, member_value in value.items():
+            member_shape = shape.members[member_key]
+            self._serialize(serialized, member_value, member_shape, member_key)
+
+    def _serialize_type_array(self, serialized, value, shape, key):
+        array_obj = []
+        serialized[key] = array_obj
+        for array_item in value:
+            wrapper = {}
+            # JSON list serialization is the only case where we aren't setting
+            # a key on a dict.  We handle this by using a __current__ key on a
+            # wrapper dict to serialize each list item before appending it to
+            # the serialized list.
+            self._serialize(wrapper, array_item, shape.member, "__current__")
+            array_obj.append(wrapper["__current__"])
+
+    def _default_serialize(self, serialized, value, shape, key):
+        if key:
+            serialized[key] = value
+        else:
+            for member_key, member_value in value.items():
+                serialized[member_key] = member_value

+ 58 - 0
desktop/core/ext-py/navoptapi-0.1.0/navoptapi/old_signers.py

@@ -0,0 +1,58 @@
+# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Modifications made by Cloudera are:
+#     Copyright (c) 2016 Cloudera, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+import navoptapi.auth
+
+UNSIGNED = object()
+
+
+class RequestSigner(object):
+    """
+    An object to sign requests before they go out over the wire using
+    one of the authentication mechanisms defined in ``auth.py``.
+    """
+    def __init__(self, signature_version, credentials):
+        self._signature_version = signature_version
+        self._credentials = credentials
+
+    @property
+    def signature_version(self):
+        return self._signature_version
+
+    def sign(self, request):
+        """
+        Sign a request before it goes out over the wire.
+        """
+        if self._signature_version != UNSIGNED:
+            signer = self.get_auth_instance(self._signature_version)
+            signer.add_auth(request)
+
+    def get_auth_instance(self, signature_version, **kwargs):
+        """
+        Get an auth instance which can be used to sign a request
+        using the given signature version.
+        """
+        cls = navoptapi.auth.AUTH_TYPE_MAPS.get(signature_version)
+        if cls is None:
+            return
+        # If there's no credentials provided (i.e credentials is None),
+        # then we'll pass a value of "None" over to the auth classes,
+        # which already handle the cases where no credentials have
+        # been provided.
+        frozen_credentials = self._credentials.get_frozen_credentials()
+        kwargs['credentials'] = frozen_credentials
+        auth = cls(**kwargs)
+        return auth

+ 2 - 8
desktop/core/ext-py/navoptapi-0.1.0/navoptapi/serialize.py

@@ -15,9 +15,9 @@
 # language governing permissions and limitations under the License.
 
 try:
-  from collections import OrderedDict
+    from collections import OrderedDict
 except ImportError:
-  from ordereddict import OrderedDict # Python 2.6
+    from ordereddict import OrderedDict  # Python 2.6
 import json
 
 
@@ -30,10 +30,6 @@ class Serializer(object):
             (k, v) for k, v in parameters.items() if v is not None)
 
         serialized = {}
-        # serialized['method'] = operation_model.http['method']
-        # serialized['headers'] = {'Content-Type': 'application/json'}
-        # serialized['url_path'] = operation_model.http['requestUri']
-
         serialized_body = OrderedDict()
         if len(filtered_parameters) != 0:
             self._serialize(serialized_body, filtered_parameters, None)
@@ -43,8 +39,6 @@ class Serializer(object):
         return serialized
 
     def _serialize(self, serialized, value, shape, key=None):
-        # serialize_method_name = '_serialize_type_%s' % shape.type_name
-        # method = getattr(self, serialize_method_name, self._default_serialize)
         self._default_serialize(serialized, value, shape, key)
 
     def _serialize_type_object(self, serialized, value, shape, key):

+ 6 - 5
desktop/core/ext-py/navoptapi-0.1.0/navoptapi/signers.py

@@ -14,9 +14,9 @@
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
 
-import navoptapi.auth
-
-UNSIGNED = object()
+from ccscli import UNSIGNED
+import ccscli.auth
+from ccscli.exceptions import UnknownSignatureVersionError
 
 
 class RequestSigner(object):
@@ -45,9 +45,10 @@ class RequestSigner(object):
         Get an auth instance which can be used to sign a request
         using the given signature version.
         """
-        cls = navoptapi.auth.AUTH_TYPE_MAPS.get(signature_version)
+        cls = ccscli.auth.AUTH_TYPE_MAPS.get(signature_version)
         if cls is None:
-            return
+            raise UnknownSignatureVersionError(
+                signature_version=signature_version)
         # If there's no credentials provided (i.e credentials is None),
         # then we'll pass a value of "None" over to the auth classes,
         # which already handle the cases where no credentials have

+ 9 - 0
desktop/core/ext-py/navoptapi-0.1.0/setup.cfg

@@ -1,6 +1,15 @@
 [flake8]
 max-line-length = 90
 import-order-style = google
+exclude = ccscli/thirdparty/*
+
+[versioneer]
+vcs = git
+style = pep440
+versionfile_source = navoptapi/_version.py
+versionfile_build = navoptapi/_version.py
+parentdir_prefix = navoptapi-
+tag_prefix = navoptapi-
 
 [egg_info]
 tag_build = 

+ 4 - 16
desktop/core/ext-py/navoptapi-0.1.0/setup.py

@@ -1,18 +1,4 @@
-# Copyright 2012-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-# Modifications made by Cloudera are:
-#     Copyright (c) 2016 Cloudera, Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"). You
-# may not use this file except in compliance with the License. A copy of
-# the License is located at
-#
-#     http://aws.amazon.com/apache2.0/
-#
-# or in the "license" file accompanying this file. This file is
-# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
-# ANY KIND, either express or implied. See the License for the specific
-# language governing permissions and limitations under the License.
+# Copyright (c) 2016 Cloudera, Inc. All rights reserved.
 
 from codecs import open
 from os import path
@@ -20,6 +6,7 @@ import sys
 
 from setuptools import find_packages
 from setuptools import setup
+import versioneer
 
 here = path.abspath(path.dirname(__file__))
 
@@ -40,7 +27,7 @@ if sys.version_info[:2] == (2, 6):
 
 setup(
     name='navoptapi',
-    version='0.1.0',
+    version=versioneer.get_version(),
     description='Cloudera Navigator Optimizer Api',
     long_description=long_description,
     url='http://www.cloudera.com/',
@@ -61,4 +48,5 @@ setup(
     packages=find_packages(exclude=['tests']),
     include_package_data=True,
     install_requires=requirements,
+    cmdclass=versioneer.get_cmdclass(),
 )

+ 1822 - 0
desktop/core/ext-py/navoptapi-0.1.0/versioneer.py

@@ -0,0 +1,1822 @@
+
+# Version: 0.17
+
+"""The Versioneer - like a rocketeer, but for versions.
+
+The Versioneer
+==============
+
+* like a rocketeer, but for versions!
+* https://github.com/warner/python-versioneer
+* Brian Warner
+* License: Public Domain
+* Compatible With: python2.6, 2.7, 3.2, 3.3, 3.4, 3.5, and pypy
+* [![Latest Version]
+(https://pypip.in/version/versioneer/badge.svg?style=flat)
+](https://pypi.python.org/pypi/versioneer/)
+* [![Build Status]
+(https://travis-ci.org/warner/python-versioneer.png?branch=master)
+](https://travis-ci.org/warner/python-versioneer)
+
+This is a tool for managing a recorded version number in distutils-based
+python projects. The goal is to remove the tedious and error-prone "update
+the embedded version string" step from your release process. Making a new
+release should be as easy as recording a new tag in your version-control
+system, and maybe making new tarballs.
+
+
+## Quick Install
+
+* `pip install versioneer` to somewhere to your $PATH
+* add a `[versioneer]` section to your setup.cfg (see below)
+* run `versioneer install` in your source tree, commit the results
+
+## Version Identifiers
+
+Source trees come from a variety of places:
+
+* a version-control system checkout (mostly used by developers)
+* a nightly tarball, produced by build automation
+* a snapshot tarball, produced by a web-based VCS browser, like github's
+  "tarball from tag" feature
+* a release tarball, produced by "setup.py sdist", distributed through PyPI
+
+Within each source tree, the version identifier (either a string or a number,
+this tool is format-agnostic) can come from a variety of places:
+
+* ask the VCS tool itself, e.g. "git describe" (for checkouts), which knows
+  about recent "tags" and an absolute revision-id
+* the name of the directory into which the tarball was unpacked
+* an expanded VCS keyword ($Id$, etc)
+* a `_version.py` created by some earlier build step
+
+For released software, the version identifier is closely related to a VCS
+tag. Some projects use tag names that include more than just the version
+string (e.g. "myproject-1.2" instead of just "1.2"), in which case the tool
+needs to strip the tag prefix to extract the version identifier. For
+unreleased software (between tags), the version identifier should provide
+enough information to help developers recreate the same tree, while also
+giving them an idea of roughly how old the tree is (after version 1.2, before
+version 1.3). Many VCS systems can report a description that captures this,
+for example `git describe --tags --dirty --always` reports things like
+"0.7-1-g574ab98-dirty" to indicate that the checkout is one revision past the
+0.7 tag, has a unique revision id of "574ab98", and is "dirty" (it has
+uncommitted changes.
+
+The version identifier is used for multiple purposes:
+
+* to allow the module to self-identify its version: `myproject.__version__`
+* to choose a name and prefix for a 'setup.py sdist' tarball
+
+## Theory of Operation
+
+Versioneer works by adding a special `_version.py` file into your source
+tree, where your `__init__.py` can import it. This `_version.py` knows how to
+dynamically ask the VCS tool for version information at import time.
+
+`_version.py` also contains `$Revision$` markers, and the installation
+process marks `_version.py` to have this marker rewritten with a tag name
+during the `git archive` command. As a result, generated tarballs will
+contain enough information to get the proper version.
+
+To allow `setup.py` to compute a version too, a `versioneer.py` is added to
+the top level of your source tree, next to `setup.py` and the `setup.cfg`
+that configures it. This overrides several distutils/setuptools commands to
+compute the version when invoked, and changes `setup.py build` and `setup.py
+sdist` to replace `_version.py` with a small static file that contains just
+the generated version data.
+
+## Installation
+
+See [INSTALL.md](./INSTALL.md) for detailed installation instructions.
+
+## Version-String Flavors
+
+Code which uses Versioneer can learn about its version string at runtime by
+importing `_version` from your main `__init__.py` file and running the
+`get_versions()` function. From the "outside" (e.g. in `setup.py`), you can
+import the top-level `versioneer.py` and run `get_versions()`.
+
+Both functions return a dictionary with different flavors of version
+information:
+
+* `['version']`: A condensed version string, rendered using the selected
+  style. This is the most commonly used value for the project's version
+  string. The default "pep440" style yields strings like `0.11`,
+  `0.11+2.g1076c97`, or `0.11+2.g1076c97.dirty`. See the "Styles" section
+  below for alternative styles.
+
+* `['full-revisionid']`: detailed revision identifier. For Git, this is the
+  full SHA1 commit id, e.g. "1076c978a8d3cfc70f408fe5974aa6c092c949ac".
+
+* `['date']`: Date and time of the latest `HEAD` commit. For Git, it is the
+  commit date in ISO 8601 format. This will be None if the date is not
+  available.
+
+* `['dirty']`: a boolean, True if the tree has uncommitted changes. Note that
+  this is only accurate if run in a VCS checkout, otherwise it is likely to
+  be False or None
+
+* `['error']`: if the version string could not be computed, this will be set
+  to a string describing the problem, otherwise it will be None. It may be
+  useful to throw an exception in setup.py if this is set, to avoid e.g.
+  creating tarballs with a version string of "unknown".
+
+Some variants are more useful than others. Including `full-revisionid` in a
+bug report should allow developers to reconstruct the exact code being tested
+(or indicate the presence of local changes that should be shared with the
+developers). `version` is suitable for display in an "about" box or a CLI
+`--version` output: it can be easily compared against release notes and lists
+of bugs fixed in various releases.
+
+The installer adds the following text to your `__init__.py` to place a basic
+version in `YOURPROJECT.__version__`:
+
+    from ._version import get_versions
+    __version__ = get_versions()['version']
+    del get_versions
+
+## Styles
+
+The setup.cfg `style=` configuration controls how the VCS information is
+rendered into a version string.
+
+The default style, "pep440", produces a PEP440-compliant string, equal to the
+un-prefixed tag name for actual releases, and containing an additional "local
+version" section with more detail for in-between builds. For Git, this is
+TAG[+DISTANCE.gHEX[.dirty]] , using information from `git describe --tags
+--dirty --always`. For example "0.11+2.g1076c97.dirty" indicates that the
+tree is like the "1076c97" commit but has uncommitted changes (".dirty"), and
+that this commit is two revisions ("+2") beyond the "0.11" tag. For released
+software (exactly equal to a known tag), the identifier will only contain the
+stripped tag, e.g. "0.11".
+
+Other styles are available. See details.md in the Versioneer source tree for
+descriptions.
+
+## Debugging
+
+Versioneer tries to avoid fatal errors: if something goes wrong, it will tend
+to return a version of "0+unknown". To investigate the problem, run `setup.py
+version`, which will run the version-lookup code in a verbose mode, and will
+display the full contents of `get_versions()` (including the `error` string,
+which may help identify what went wrong).
+
+## Known Limitations
+
+Some situations are known to cause problems for Versioneer. This details the
+most significant ones. More can be found on Github
+[issues page](https://github.com/warner/python-versioneer/issues).
+
+### Subprojects
+
+Versioneer has limited support for source trees in which `setup.py` is not in
+the root directory (e.g. `setup.py` and `.git/` are *not* siblings). The are
+two common reasons why `setup.py` might not be in the root:
+
+* Source trees which contain multiple subprojects, such as
+  [Buildbot](https://github.com/buildbot/buildbot), which contains both
+  "master" and "slave" subprojects, each with their own `setup.py`,
+  `setup.cfg`, and `tox.ini`. Projects like these produce multiple PyPI
+  distributions (and upload multiple independently-installable tarballs).
+* Source trees whose main purpose is to contain a C library, but which also
+  provide bindings to Python (and perhaps other langauges) in subdirectories.
+
+Versioneer will look for `.git` in parent directories, and most operations
+should get the right version string. However `pip` and `setuptools` have bugs
+and implementation details which frequently cause `pip install .` from a
+subproject directory to fail to find a correct version string (so it usually
+defaults to `0+unknown`).
+
+`pip install --editable .` should work correctly. `setup.py install` might
+work too.
+
+Pip-8.1.1 is known to have this problem, but hopefully it will get fixed in
+some later version.
+
+[Bug #38](https://github.com/warner/python-versioneer/issues/38) is tracking
+this issue. The discussion in
+[PR #61](https://github.com/warner/python-versioneer/pull/61) describes the
+issue from the Versioneer side in more detail.
+[pip PR#3176](https://github.com/pypa/pip/pull/3176) and
+[pip PR#3615](https://github.com/pypa/pip/pull/3615) contain work to improve
+pip to let Versioneer work correctly.
+
+Versioneer-0.16 and earlier only looked for a `.git` directory next to the
+`setup.cfg`, so subprojects were completely unsupported with those releases.
+
+### Editable installs with setuptools <= 18.5
+
+`setup.py develop` and `pip install --editable .` allow you to install a
+project into a virtualenv once, then continue editing the source code (and
+test) without re-installing after every change.
+
+"Entry-point scripts" (`setup(entry_points={"console_scripts": ..})`) are a
+convenient way to specify executable scripts that should be installed along
+with the python package.
+
+These both work as expected when using modern setuptools. When using
+setuptools-18.5 or earlier, however, certain operations will cause
+`pkg_resources.DistributionNotFound` errors when running the entrypoint
+script, which must be resolved by re-installing the package. This happens
+when the install happens with one version, then the egg_info data is
+regenerated while a different version is checked out. Many setup.py commands
+cause egg_info to be rebuilt (including `sdist`, `wheel`, and installing into
+a different virtualenv), so this can be surprising.
+
+[Bug #83](https://github.com/warner/python-versioneer/issues/83) describes
+this one, but upgrading to a newer version of setuptools should probably
+resolve it.
+
+### Unicode version strings
+
+While Versioneer works (and is continually tested) with both Python 2 and
+Python 3, it is not entirely consistent with bytes-vs-unicode distinctions.
+Newer releases probably generate unicode version strings on py2. It's not
+clear that this is wrong, but it may be surprising for applications when then
+write these strings to a network connection or include them in bytes-oriented
+APIs like cryptographic checksums.
+
+[Bug #71](https://github.com/warner/python-versioneer/issues/71) investigates
+this question.
+
+
+## Updating Versioneer
+
+To upgrade your project to a new release of Versioneer, do the following:
+
+* install the new Versioneer (`pip install -U versioneer` or equivalent)
+* edit `setup.cfg`, if necessary, to include any new configuration settings
+  indicated by the release notes. See [UPGRADING](./UPGRADING.md) for details.
+* re-run `versioneer install` in your source tree, to replace
+  `SRC/_version.py`
+* commit any changed files
+
+## Future Directions
+
+This tool is designed to make it easily extended to other version-control
+systems: all VCS-specific components are in separate directories like
+src/git/ . The top-level `versioneer.py` script is assembled from these
+components by running make-versioneer.py . In the future, make-versioneer.py
+will take a VCS name as an argument, and will construct a version of
+`versioneer.py` that is specific to the given VCS. It might also take the
+configuration arguments that are currently provided manually during
+installation by editing setup.py . Alternatively, it might go the other
+direction and include code from all supported VCS systems, reducing the
+number of intermediate scripts.
+
+
+## License
+
+To make Versioneer easier to embed, all its code is dedicated to the public
+domain. The `_version.py` that it creates is also in the public domain.
+Specifically, both are released under the Creative Commons "Public Domain
+Dedication" license (CC0-1.0), as described in
+https://creativecommons.org/publicdomain/zero/1.0/ .
+
+"""
+
+from __future__ import print_function
+try:
+    import configparser
+except ImportError:
+    import ConfigParser as configparser
+import errno
+import json
+import os
+import re
+import subprocess
+import sys
+
+
+class VersioneerConfig:
+    """Container for Versioneer configuration parameters."""
+
+
+def get_root():
+    """Get the project root directory.
+
+    We require that all commands are run from the project root, i.e. the
+    directory that contains setup.py, setup.cfg, and versioneer.py .
+    """
+    root = os.path.realpath(os.path.abspath(os.getcwd()))
+    setup_py = os.path.join(root, "setup.py")
+    versioneer_py = os.path.join(root, "versioneer.py")
+    if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)):
+        # allow 'python path/to/setup.py COMMAND'
+        root = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0])))
+        setup_py = os.path.join(root, "setup.py")
+        versioneer_py = os.path.join(root, "versioneer.py")
+    if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)):
+        err = ("Versioneer was unable to run the project root directory. "
+               "Versioneer requires setup.py to be executed from "
+               "its immediate directory (like 'python setup.py COMMAND'), "
+               "or in a way that lets it use sys.argv[0] to find the root "
+               "(like 'python path/to/setup.py COMMAND').")
+        raise VersioneerBadRootError(err)
+    try:
+        # Certain runtime workflows (setup.py install/develop in a setuptools
+        # tree) execute all dependencies in a single python process, so
+        # "versioneer" may be imported multiple times, and python's shared
+        # module-import table will cache the first one. So we can't use
+        # os.path.dirname(__file__), as that will find whichever
+        # versioneer.py was first imported, even in later projects.
+        me = os.path.realpath(os.path.abspath(__file__))
+        me_dir = os.path.normcase(os.path.splitext(me)[0])
+        vsr_dir = os.path.normcase(os.path.splitext(versioneer_py)[0])
+        if me_dir != vsr_dir:
+            print("Warning: build in %s is using versioneer.py from %s"
+                  % (os.path.dirname(me), versioneer_py))
+    except NameError:
+        pass
+    return root
+
+
+def get_config_from_root(root):
+    """Read the project setup.cfg file to determine Versioneer config."""
+    # This might raise EnvironmentError (if setup.cfg is missing), or
+    # configparser.NoSectionError (if it lacks a [versioneer] section), or
+    # configparser.NoOptionError (if it lacks "VCS="). See the docstring at
+    # the top of versioneer.py for instructions on writing your setup.cfg .
+    setup_cfg = os.path.join(root, "setup.cfg")
+    parser = configparser.SafeConfigParser()
+    with open(setup_cfg, "r") as f:
+        parser.readfp(f)
+    VCS = parser.get("versioneer", "VCS")  # mandatory
+
+    def get(parser, name):
+        if parser.has_option("versioneer", name):
+            return parser.get("versioneer", name)
+        return None
+    cfg = VersioneerConfig()
+    cfg.VCS = VCS
+    cfg.style = get(parser, "style") or ""
+    cfg.versionfile_source = get(parser, "versionfile_source")
+    cfg.versionfile_build = get(parser, "versionfile_build")
+    cfg.tag_prefix = get(parser, "tag_prefix")
+    if cfg.tag_prefix in ("''", '""'):
+        cfg.tag_prefix = ""
+    cfg.parentdir_prefix = get(parser, "parentdir_prefix")
+    cfg.verbose = get(parser, "verbose")
+    return cfg
+
+
+class NotThisMethod(Exception):
+    """Exception raised if a method is not valid for the current scenario."""
+
+
+# these dictionaries contain VCS-specific tools
+LONG_VERSION_PY = {}
+HANDLERS = {}
+
+
+def register_vcs_handler(vcs, method):  # decorator
+    """Decorator to mark a method as the handler for a particular VCS."""
+    def decorate(f):
+        """Store f in HANDLERS[vcs][method]."""
+        if vcs not in HANDLERS:
+            HANDLERS[vcs] = {}
+        HANDLERS[vcs][method] = f
+        return f
+    return decorate
+
+
+def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False,
+                env=None):
+    """Call the given command(s)."""
+    assert isinstance(commands, list)
+    p = None
+    for c in commands:
+        try:
+            dispcmd = str([c] + args)
+            # remember shell=False, so use git.cmd on windows, not just git
+            p = subprocess.Popen([c] + args, cwd=cwd, env=env,
+                                 stdout=subprocess.PIPE,
+                                 stderr=(subprocess.PIPE if hide_stderr
+                                         else None))
+            break
+        except EnvironmentError:
+            e = sys.exc_info()[1]
+            if e.errno == errno.ENOENT:
+                continue
+            if verbose:
+                print("unable to run %s" % dispcmd)
+                print(e)
+            return None, None
+    else:
+        if verbose:
+            print("unable to find command, tried %s" % (commands,))
+        return None, None
+    stdout = p.communicate()[0].strip()
+    if sys.version_info[0] >= 3:
+        stdout = stdout.decode()
+    if p.returncode != 0:
+        if verbose:
+            print("unable to run %s (error)" % dispcmd)
+            print("stdout was %s" % stdout)
+        return None, p.returncode
+    return stdout, p.returncode
+
+
+LONG_VERSION_PY['git'] = '''
+# This file helps to compute a version number in source trees obtained from
+# git-archive tarball (such as those provided by githubs download-from-tag
+# feature). Distribution tarballs (built by setup.py sdist) and build
+# directories (produced by setup.py build) will contain a much shorter file
+# that just contains the computed version number.
+
+# This file is released into the public domain. Generated by
+# versioneer-0.17 (https://github.com/warner/python-versioneer)
+
+"""Git implementation of _version.py."""
+
+import errno
+import os
+import re
+import subprocess
+import sys
+
+
+def get_keywords():
+    """Get the keywords needed to look up the version information."""
+    # these strings will be replaced by git during git-archive.
+    # setup.py/versioneer.py will grep for the variable names, so they must
+    # each be defined on a line of their own. _version.py will just call
+    # get_keywords().
+    git_refnames = "%(DOLLAR)sFormat:%%d%(DOLLAR)s"
+    git_full = "%(DOLLAR)sFormat:%%H%(DOLLAR)s"
+    git_date = "%(DOLLAR)sFormat:%%ci%(DOLLAR)s"
+    keywords = {"refnames": git_refnames, "full": git_full, "date": git_date}
+    return keywords
+
+
+class VersioneerConfig:
+    """Container for Versioneer configuration parameters."""
+
+
+def get_config():
+    """Create, populate and return the VersioneerConfig() object."""
+    # these strings are filled in when 'setup.py versioneer' creates
+    # _version.py
+    cfg = VersioneerConfig()
+    cfg.VCS = "git"
+    cfg.style = "%(STYLE)s"
+    cfg.tag_prefix = "%(TAG_PREFIX)s"
+    cfg.parentdir_prefix = "%(PARENTDIR_PREFIX)s"
+    cfg.versionfile_source = "%(VERSIONFILE_SOURCE)s"
+    cfg.verbose = False
+    return cfg
+
+
+class NotThisMethod(Exception):
+    """Exception raised if a method is not valid for the current scenario."""
+
+
+LONG_VERSION_PY = {}
+HANDLERS = {}
+
+
+def register_vcs_handler(vcs, method):  # decorator
+    """Decorator to mark a method as the handler for a particular VCS."""
+    def decorate(f):
+        """Store f in HANDLERS[vcs][method]."""
+        if vcs not in HANDLERS:
+            HANDLERS[vcs] = {}
+        HANDLERS[vcs][method] = f
+        return f
+    return decorate
+
+
+def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False,
+                env=None):
+    """Call the given command(s)."""
+    assert isinstance(commands, list)
+    p = None
+    for c in commands:
+        try:
+            dispcmd = str([c] + args)
+            # remember shell=False, so use git.cmd on windows, not just git
+            p = subprocess.Popen([c] + args, cwd=cwd, env=env,
+                                 stdout=subprocess.PIPE,
+                                 stderr=(subprocess.PIPE if hide_stderr
+                                         else None))
+            break
+        except EnvironmentError:
+            e = sys.exc_info()[1]
+            if e.errno == errno.ENOENT:
+                continue
+            if verbose:
+                print("unable to run %%s" %% dispcmd)
+                print(e)
+            return None, None
+    else:
+        if verbose:
+            print("unable to find command, tried %%s" %% (commands,))
+        return None, None
+    stdout = p.communicate()[0].strip()
+    if sys.version_info[0] >= 3:
+        stdout = stdout.decode()
+    if p.returncode != 0:
+        if verbose:
+            print("unable to run %%s (error)" %% dispcmd)
+            print("stdout was %%s" %% stdout)
+        return None, p.returncode
+    return stdout, p.returncode
+
+
+def versions_from_parentdir(parentdir_prefix, root, verbose):
+    """Try to determine the version from the parent directory name.
+
+    Source tarballs conventionally unpack into a directory that includes both
+    the project name and a version string. We will also support searching up
+    two directory levels for an appropriately named parent directory
+    """
+    rootdirs = []
+
+    for i in range(3):
+        dirname = os.path.basename(root)
+        if dirname.startswith(parentdir_prefix):
+            return {"version": dirname[len(parentdir_prefix):],
+                    "full-revisionid": None,
+                    "dirty": False, "error": None, "date": None}
+        else:
+            rootdirs.append(root)
+            root = os.path.dirname(root)  # up a level
+
+    if verbose:
+        print("Tried directories %%s but none started with prefix %%s" %%
+              (str(rootdirs), parentdir_prefix))
+    raise NotThisMethod("rootdir doesn't start with parentdir_prefix")
+
+
+@register_vcs_handler("git", "get_keywords")
+def git_get_keywords(versionfile_abs):
+    """Extract version information from the given file."""
+    # the code embedded in _version.py can just fetch the value of these
+    # keywords. When used from setup.py, we don't want to import _version.py,
+    # so we do it with a regexp instead. This function is not used from
+    # _version.py.
+    keywords = {}
+    try:
+        f = open(versionfile_abs, "r")
+        for line in f.readlines():
+            if line.strip().startswith("git_refnames ="):
+                mo = re.search(r'=\s*"(.*)"', line)
+                if mo:
+                    keywords["refnames"] = mo.group(1)
+            if line.strip().startswith("git_full ="):
+                mo = re.search(r'=\s*"(.*)"', line)
+                if mo:
+                    keywords["full"] = mo.group(1)
+            if line.strip().startswith("git_date ="):
+                mo = re.search(r'=\s*"(.*)"', line)
+                if mo:
+                    keywords["date"] = mo.group(1)
+        f.close()
+    except EnvironmentError:
+        pass
+    return keywords
+
+
+@register_vcs_handler("git", "keywords")
+def git_versions_from_keywords(keywords, tag_prefix, verbose):
+    """Get version information from git keywords."""
+    if not keywords:
+        raise NotThisMethod("no keywords at all, weird")
+    date = keywords.get("date")
+    if date is not None:
+        # git-2.2.0 added "%%cI", which expands to an ISO-8601 -compliant
+        # datestamp. However we prefer "%%ci" (which expands to an "ISO-8601
+        # -like" string, which we must then edit to make compliant), because
+        # it's been around since git-1.5.3, and it's too difficult to
+        # discover which version we're using, or to work around using an
+        # older one.
+        date = date.strip().replace(" ", "T", 1).replace(" ", "", 1)
+    refnames = keywords["refnames"].strip()
+    if refnames.startswith("$Format"):
+        if verbose:
+            print("keywords are unexpanded, not using")
+        raise NotThisMethod("unexpanded keywords, not a git-archive tarball")
+    refs = set([r.strip() for r in refnames.strip("()").split(",")])
+    # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of
+    # just "foo-1.0". If we see a "tag: " prefix, prefer those.
+    TAG = "tag: "
+    tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)])
+    if not tags:
+        # Either we're using git < 1.8.3, or there really are no tags. We use
+        # a heuristic: assume all version tags have a digit. The old git %%d
+        # expansion behaves like git log --decorate=short and strips out the
+        # refs/heads/ and refs/tags/ prefixes that would let us distinguish
+        # between branches and tags. By ignoring refnames without digits, we
+        # filter out many common branch names like "release" and
+        # "stabilization", as well as "HEAD" and "master".
+        tags = set([r for r in refs if re.search(r'\d', r)])
+        if verbose:
+            print("discarding '%%s', no digits" %% ",".join(refs - tags))
+    if verbose:
+        print("likely tags: %%s" %% ",".join(sorted(tags)))
+    for ref in sorted(tags):
+        # sorting will prefer e.g. "2.0" over "2.0rc1"
+        if ref.startswith(tag_prefix):
+            r = ref[len(tag_prefix):]
+            if verbose:
+                print("picking %%s" %% r)
+            return {"version": r,
+                    "full-revisionid": keywords["full"].strip(),
+                    "dirty": False, "error": None,
+                    "date": date}
+    # no suitable tags, so version is "0+unknown", but full hex is still there
+    if verbose:
+        print("no suitable tags, using unknown + full revision id")
+    return {"version": "0+unknown",
+            "full-revisionid": keywords["full"].strip(),
+            "dirty": False, "error": "no suitable tags", "date": None}
+
+
+@register_vcs_handler("git", "pieces_from_vcs")
+def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
+    """Get version from 'git describe' in the root of the source tree.
+
+    This only gets called if the git-archive 'subst' keywords were *not*
+    expanded, and _version.py hasn't already been rewritten with a short
+    version string, meaning we're inside a checked out source tree.
+    """
+    GITS = ["git"]
+    if sys.platform == "win32":
+        GITS = ["git.cmd", "git.exe"]
+
+    out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root,
+                          hide_stderr=True)
+    if rc != 0:
+        if verbose:
+            print("Directory %%s not under git control" %% root)
+        raise NotThisMethod("'git rev-parse --git-dir' returned error")
+
+    # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty]
+    # if there isn't one, this yields HEX[-dirty] (no NUM)
+    describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty",
+                                          "--always", "--long",
+                                          "--match", "%%s*" %% tag_prefix],
+                                   cwd=root)
+    # --long was added in git-1.5.5
+    if describe_out is None:
+        raise NotThisMethod("'git describe' failed")
+    describe_out = describe_out.strip()
+    full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root)
+    if full_out is None:
+        raise NotThisMethod("'git rev-parse' failed")
+    full_out = full_out.strip()
+
+    pieces = {}
+    pieces["long"] = full_out
+    pieces["short"] = full_out[:7]  # maybe improved later
+    pieces["error"] = None
+
+    # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty]
+    # TAG might have hyphens.
+    git_describe = describe_out
+
+    # look for -dirty suffix
+    dirty = git_describe.endswith("-dirty")
+    pieces["dirty"] = dirty
+    if dirty:
+        git_describe = git_describe[:git_describe.rindex("-dirty")]
+
+    # now we have TAG-NUM-gHEX or HEX
+
+    if "-" in git_describe:
+        # TAG-NUM-gHEX
+        mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe)
+        if not mo:
+            # unparseable. Maybe git-describe is misbehaving?
+            pieces["error"] = ("unable to parse git-describe output: '%%s'"
+                               %% describe_out)
+            return pieces
+
+        # tag
+        full_tag = mo.group(1)
+        if not full_tag.startswith(tag_prefix):
+            if verbose:
+                fmt = "tag '%%s' doesn't start with prefix '%%s'"
+                print(fmt %% (full_tag, tag_prefix))
+            pieces["error"] = ("tag '%%s' doesn't start with prefix '%%s'"
+                               %% (full_tag, tag_prefix))
+            return pieces
+        pieces["closest-tag"] = full_tag[len(tag_prefix):]
+
+        # distance: number of commits since tag
+        pieces["distance"] = int(mo.group(2))
+
+        # commit: short hex revision ID
+        pieces["short"] = mo.group(3)
+
+    else:
+        # HEX: no tags
+        pieces["closest-tag"] = None
+        count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"],
+                                    cwd=root)
+        pieces["distance"] = int(count_out)  # total number of commits
+
+    # commit date: see ISO-8601 comment in git_versions_from_keywords()
+    date = run_command(GITS, ["show", "-s", "--format=%%ci", "HEAD"],
+                       cwd=root)[0].strip()
+    pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1)
+
+    return pieces
+
+
+def plus_or_dot(pieces):
+    """Return a + if we don't already have one, else return a ."""
+    if "+" in pieces.get("closest-tag", ""):
+        return "."
+    return "+"
+
+
+def render_pep440(pieces):
+    """Build up version string, with post-release "local version identifier".
+
+    Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you
+    get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty
+
+    Exceptions:
+    1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += plus_or_dot(pieces)
+            rendered += "%%d.g%%s" %% (pieces["distance"], pieces["short"])
+            if pieces["dirty"]:
+                rendered += ".dirty"
+    else:
+        # exception #1
+        rendered = "0+untagged.%%d.g%%s" %% (pieces["distance"],
+                                          pieces["short"])
+        if pieces["dirty"]:
+            rendered += ".dirty"
+    return rendered
+
+
+def render_pep440_pre(pieces):
+    """TAG[.post.devDISTANCE] -- No -dirty.
+
+    Exceptions:
+    1: no tags. 0.post.devDISTANCE
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"]:
+            rendered += ".post.dev%%d" %% pieces["distance"]
+    else:
+        # exception #1
+        rendered = "0.post.dev%%d" %% pieces["distance"]
+    return rendered
+
+
+def render_pep440_post(pieces):
+    """TAG[.postDISTANCE[.dev0]+gHEX] .
+
+    The ".dev0" means dirty. Note that .dev0 sorts backwards
+    (a dirty tree will appear "older" than the corresponding clean one),
+    but you shouldn't be releasing software with -dirty anyways.
+
+    Exceptions:
+    1: no tags. 0.postDISTANCE[.dev0]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += ".post%%d" %% pieces["distance"]
+            if pieces["dirty"]:
+                rendered += ".dev0"
+            rendered += plus_or_dot(pieces)
+            rendered += "g%%s" %% pieces["short"]
+    else:
+        # exception #1
+        rendered = "0.post%%d" %% pieces["distance"]
+        if pieces["dirty"]:
+            rendered += ".dev0"
+        rendered += "+g%%s" %% pieces["short"]
+    return rendered
+
+
+def render_pep440_old(pieces):
+    """TAG[.postDISTANCE[.dev0]] .
+
+    The ".dev0" means dirty.
+
+    Eexceptions:
+    1: no tags. 0.postDISTANCE[.dev0]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += ".post%%d" %% pieces["distance"]
+            if pieces["dirty"]:
+                rendered += ".dev0"
+    else:
+        # exception #1
+        rendered = "0.post%%d" %% pieces["distance"]
+        if pieces["dirty"]:
+            rendered += ".dev0"
+    return rendered
+
+
+def render_git_describe(pieces):
+    """TAG[-DISTANCE-gHEX][-dirty].
+
+    Like 'git describe --tags --dirty --always'.
+
+    Exceptions:
+    1: no tags. HEX[-dirty]  (note: no 'g' prefix)
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"]:
+            rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"])
+    else:
+        # exception #1
+        rendered = pieces["short"]
+    if pieces["dirty"]:
+        rendered += "-dirty"
+    return rendered
+
+
+def render_git_describe_long(pieces):
+    """TAG-DISTANCE-gHEX[-dirty].
+
+    Like 'git describe --tags --dirty --always -long'.
+    The distance/hash is unconditional.
+
+    Exceptions:
+    1: no tags. HEX[-dirty]  (note: no 'g' prefix)
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"])
+    else:
+        # exception #1
+        rendered = pieces["short"]
+    if pieces["dirty"]:
+        rendered += "-dirty"
+    return rendered
+
+
+def render(pieces, style):
+    """Render the given version pieces into the requested style."""
+    if pieces["error"]:
+        return {"version": "unknown",
+                "full-revisionid": pieces.get("long"),
+                "dirty": None,
+                "error": pieces["error"],
+                "date": None}
+
+    if not style or style == "default":
+        style = "pep440"  # the default
+
+    if style == "pep440":
+        rendered = render_pep440(pieces)
+    elif style == "pep440-pre":
+        rendered = render_pep440_pre(pieces)
+    elif style == "pep440-post":
+        rendered = render_pep440_post(pieces)
+    elif style == "pep440-old":
+        rendered = render_pep440_old(pieces)
+    elif style == "git-describe":
+        rendered = render_git_describe(pieces)
+    elif style == "git-describe-long":
+        rendered = render_git_describe_long(pieces)
+    else:
+        raise ValueError("unknown style '%%s'" %% style)
+
+    return {"version": rendered, "full-revisionid": pieces["long"],
+            "dirty": pieces["dirty"], "error": None,
+            "date": pieces.get("date")}
+
+
+def get_versions():
+    """Get version information or return default if unable to do so."""
+    # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have
+    # __file__, we can work backwards from there to the root. Some
+    # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which
+    # case we can only use expanded keywords.
+
+    cfg = get_config()
+    verbose = cfg.verbose
+
+    try:
+        return git_versions_from_keywords(get_keywords(), cfg.tag_prefix,
+                                          verbose)
+    except NotThisMethod:
+        pass
+
+    try:
+        root = os.path.realpath(__file__)
+        # versionfile_source is the relative path from the top of the source
+        # tree (where the .git directory might live) to this file. Invert
+        # this to find the root from __file__.
+        for i in cfg.versionfile_source.split('/'):
+            root = os.path.dirname(root)
+    except NameError:
+        return {"version": "0+unknown", "full-revisionid": None,
+                "dirty": None,
+                "error": "unable to find root of source tree",
+                "date": None}
+
+    try:
+        pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose)
+        return render(pieces, cfg.style)
+    except NotThisMethod:
+        pass
+
+    try:
+        if cfg.parentdir_prefix:
+            return versions_from_parentdir(cfg.parentdir_prefix, root, verbose)
+    except NotThisMethod:
+        pass
+
+    return {"version": "0+unknown", "full-revisionid": None,
+            "dirty": None,
+            "error": "unable to compute version", "date": None}
+'''
+
+
+@register_vcs_handler("git", "get_keywords")
+def git_get_keywords(versionfile_abs):
+    """Extract version information from the given file."""
+    # the code embedded in _version.py can just fetch the value of these
+    # keywords. When used from setup.py, we don't want to import _version.py,
+    # so we do it with a regexp instead. This function is not used from
+    # _version.py.
+    keywords = {}
+    try:
+        f = open(versionfile_abs, "r")
+        for line in f.readlines():
+            if line.strip().startswith("git_refnames ="):
+                mo = re.search(r'=\s*"(.*)"', line)
+                if mo:
+                    keywords["refnames"] = mo.group(1)
+            if line.strip().startswith("git_full ="):
+                mo = re.search(r'=\s*"(.*)"', line)
+                if mo:
+                    keywords["full"] = mo.group(1)
+            if line.strip().startswith("git_date ="):
+                mo = re.search(r'=\s*"(.*)"', line)
+                if mo:
+                    keywords["date"] = mo.group(1)
+        f.close()
+    except EnvironmentError:
+        pass
+    return keywords
+
+
+@register_vcs_handler("git", "keywords")
+def git_versions_from_keywords(keywords, tag_prefix, verbose):
+    """Get version information from git keywords."""
+    if not keywords:
+        raise NotThisMethod("no keywords at all, weird")
+    date = keywords.get("date")
+    if date is not None:
+        # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant
+        # datestamp. However we prefer "%ci" (which expands to an "ISO-8601
+        # -like" string, which we must then edit to make compliant), because
+        # it's been around since git-1.5.3, and it's too difficult to
+        # discover which version we're using, or to work around using an
+        # older one.
+        date = date.strip().replace(" ", "T", 1).replace(" ", "", 1)
+    refnames = keywords["refnames"].strip()
+    if refnames.startswith("$Format"):
+        if verbose:
+            print("keywords are unexpanded, not using")
+        raise NotThisMethod("unexpanded keywords, not a git-archive tarball")
+    refs = set([r.strip() for r in refnames.strip("()").split(",")])
+    # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of
+    # just "foo-1.0". If we see a "tag: " prefix, prefer those.
+    TAG = "tag: "
+    tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)])
+    if not tags:
+        # Either we're using git < 1.8.3, or there really are no tags. We use
+        # a heuristic: assume all version tags have a digit. The old git %d
+        # expansion behaves like git log --decorate=short and strips out the
+        # refs/heads/ and refs/tags/ prefixes that would let us distinguish
+        # between branches and tags. By ignoring refnames without digits, we
+        # filter out many common branch names like "release" and
+        # "stabilization", as well as "HEAD" and "master".
+        tags = set([r for r in refs if re.search(r'\d', r)])
+        if verbose:
+            print("discarding '%s', no digits" % ",".join(refs - tags))
+    if verbose:
+        print("likely tags: %s" % ",".join(sorted(tags)))
+    for ref in sorted(tags):
+        # sorting will prefer e.g. "2.0" over "2.0rc1"
+        if ref.startswith(tag_prefix):
+            r = ref[len(tag_prefix):]
+            if verbose:
+                print("picking %s" % r)
+            return {"version": r,
+                    "full-revisionid": keywords["full"].strip(),
+                    "dirty": False, "error": None,
+                    "date": date}
+    # no suitable tags, so version is "0+unknown", but full hex is still there
+    if verbose:
+        print("no suitable tags, using unknown + full revision id")
+    return {"version": "0+unknown",
+            "full-revisionid": keywords["full"].strip(),
+            "dirty": False, "error": "no suitable tags", "date": None}
+
+
+@register_vcs_handler("git", "pieces_from_vcs")
+def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
+    """Get version from 'git describe' in the root of the source tree.
+
+    This only gets called if the git-archive 'subst' keywords were *not*
+    expanded, and _version.py hasn't already been rewritten with a short
+    version string, meaning we're inside a checked out source tree.
+    """
+    GITS = ["git"]
+    if sys.platform == "win32":
+        GITS = ["git.cmd", "git.exe"]
+
+    out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root,
+                          hide_stderr=True)
+    if rc != 0:
+        if verbose:
+            print("Directory %s not under git control" % root)
+        raise NotThisMethod("'git rev-parse --git-dir' returned error")
+
+    # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty]
+    # if there isn't one, this yields HEX[-dirty] (no NUM)
+    describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty",
+                                          "--always", "--long",
+                                          "--match", "%s*" % tag_prefix],
+                                   cwd=root)
+    # --long was added in git-1.5.5
+    if describe_out is None:
+        raise NotThisMethod("'git describe' failed")
+    describe_out = describe_out.strip()
+    full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root)
+    if full_out is None:
+        raise NotThisMethod("'git rev-parse' failed")
+    full_out = full_out.strip()
+
+    pieces = {}
+    pieces["long"] = full_out
+    pieces["short"] = full_out[:7]  # maybe improved later
+    pieces["error"] = None
+
+    # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty]
+    # TAG might have hyphens.
+    git_describe = describe_out
+
+    # look for -dirty suffix
+    dirty = git_describe.endswith("-dirty")
+    pieces["dirty"] = dirty
+    if dirty:
+        git_describe = git_describe[:git_describe.rindex("-dirty")]
+
+    # now we have TAG-NUM-gHEX or HEX
+
+    if "-" in git_describe:
+        # TAG-NUM-gHEX
+        mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe)
+        if not mo:
+            # unparseable. Maybe git-describe is misbehaving?
+            pieces["error"] = ("unable to parse git-describe output: '%s'"
+                               % describe_out)
+            return pieces
+
+        # tag
+        full_tag = mo.group(1)
+        if not full_tag.startswith(tag_prefix):
+            if verbose:
+                fmt = "tag '%s' doesn't start with prefix '%s'"
+                print(fmt % (full_tag, tag_prefix))
+            pieces["error"] = ("tag '%s' doesn't start with prefix '%s'"
+                               % (full_tag, tag_prefix))
+            return pieces
+        pieces["closest-tag"] = full_tag[len(tag_prefix):]
+
+        # distance: number of commits since tag
+        pieces["distance"] = int(mo.group(2))
+
+        # commit: short hex revision ID
+        pieces["short"] = mo.group(3)
+
+    else:
+        # HEX: no tags
+        pieces["closest-tag"] = None
+        count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"],
+                                    cwd=root)
+        pieces["distance"] = int(count_out)  # total number of commits
+
+    # commit date: see ISO-8601 comment in git_versions_from_keywords()
+    date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"],
+                       cwd=root)[0].strip()
+    pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1)
+
+    return pieces
+
+
+def do_vcs_install(manifest_in, versionfile_source, ipy):
+    """Git-specific installation logic for Versioneer.
+
+    For Git, this means creating/changing .gitattributes to mark _version.py
+    for export-subst keyword substitution.
+    """
+    GITS = ["git"]
+    if sys.platform == "win32":
+        GITS = ["git.cmd", "git.exe"]
+    files = [manifest_in, versionfile_source]
+    if ipy:
+        files.append(ipy)
+    try:
+        me = __file__
+        if me.endswith(".pyc") or me.endswith(".pyo"):
+            me = os.path.splitext(me)[0] + ".py"
+        versioneer_file = os.path.relpath(me)
+    except NameError:
+        versioneer_file = "versioneer.py"
+    files.append(versioneer_file)
+    present = False
+    try:
+        f = open(".gitattributes", "r")
+        for line in f.readlines():
+            if line.strip().startswith(versionfile_source):
+                if "export-subst" in line.strip().split()[1:]:
+                    present = True
+        f.close()
+    except EnvironmentError:
+        pass
+    if not present:
+        f = open(".gitattributes", "a+")
+        f.write("%s export-subst\n" % versionfile_source)
+        f.close()
+        files.append(".gitattributes")
+    run_command(GITS, ["add", "--"] + files)
+
+
+def versions_from_parentdir(parentdir_prefix, root, verbose):
+    """Try to determine the version from the parent directory name.
+
+    Source tarballs conventionally unpack into a directory that includes both
+    the project name and a version string. We will also support searching up
+    two directory levels for an appropriately named parent directory
+    """
+    rootdirs = []
+
+    for i in range(3):
+        dirname = os.path.basename(root)
+        if dirname.startswith(parentdir_prefix):
+            return {"version": dirname[len(parentdir_prefix):],
+                    "full-revisionid": None,
+                    "dirty": False, "error": None, "date": None}
+        else:
+            rootdirs.append(root)
+            root = os.path.dirname(root)  # up a level
+
+    if verbose:
+        print("Tried directories %s but none started with prefix %s" %
+              (str(rootdirs), parentdir_prefix))
+    raise NotThisMethod("rootdir doesn't start with parentdir_prefix")
+
+
+SHORT_VERSION_PY = """
+# This file was generated by 'versioneer.py' (0.17) from
+# revision-control system data, or from the parent directory name of an
+# unpacked source archive. Distribution tarballs contain a pre-generated copy
+# of this file.
+
+import json
+
+version_json = '''
+%s
+'''  # END VERSION_JSON
+
+
+def get_versions():
+    return json.loads(version_json)
+"""
+
+
+def versions_from_file(filename):
+    """Try to determine the version from _version.py if present."""
+    try:
+        with open(filename) as f:
+            contents = f.read()
+    except EnvironmentError:
+        raise NotThisMethod("unable to read _version.py")
+    mo = re.search(r"version_json = '''\n(.*)'''  # END VERSION_JSON",
+                   contents, re.M | re.S)
+    if not mo:
+        mo = re.search(r"version_json = '''\r\n(.*)'''  # END VERSION_JSON",
+                       contents, re.M | re.S)
+    if not mo:
+        raise NotThisMethod("no version_json in _version.py")
+    return json.loads(mo.group(1))
+
+
+def write_to_version_file(filename, versions):
+    """Write the given version number to the given _version.py file."""
+    os.unlink(filename)
+    contents = json.dumps(versions, sort_keys=True,
+                          indent=1, separators=(",", ": "))
+    with open(filename, "w") as f:
+        f.write(SHORT_VERSION_PY % contents)
+
+    print("set %s to '%s'" % (filename, versions["version"]))
+
+
+def plus_or_dot(pieces):
+    """Return a + if we don't already have one, else return a ."""
+    if "+" in pieces.get("closest-tag", ""):
+        return "."
+    return "+"
+
+
+def render_pep440(pieces):
+    """Build up version string, with post-release "local version identifier".
+
+    Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you
+    get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty
+
+    Exceptions:
+    1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += plus_or_dot(pieces)
+            rendered += "%d.g%s" % (pieces["distance"], pieces["short"])
+            if pieces["dirty"]:
+                rendered += ".dirty"
+    else:
+        # exception #1
+        rendered = "0+untagged.%d.g%s" % (pieces["distance"],
+                                          pieces["short"])
+        if pieces["dirty"]:
+            rendered += ".dirty"
+    return rendered
+
+
+def render_pep440_pre(pieces):
+    """TAG[.post.devDISTANCE] -- No -dirty.
+
+    Exceptions:
+    1: no tags. 0.post.devDISTANCE
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"]:
+            rendered += ".post.dev%d" % pieces["distance"]
+    else:
+        # exception #1
+        rendered = "0.post.dev%d" % pieces["distance"]
+    return rendered
+
+
+def render_pep440_post(pieces):
+    """TAG[.postDISTANCE[.dev0]+gHEX] .
+
+    The ".dev0" means dirty. Note that .dev0 sorts backwards
+    (a dirty tree will appear "older" than the corresponding clean one),
+    but you shouldn't be releasing software with -dirty anyways.
+
+    Exceptions:
+    1: no tags. 0.postDISTANCE[.dev0]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += ".post%d" % pieces["distance"]
+            if pieces["dirty"]:
+                rendered += ".dev0"
+            rendered += plus_or_dot(pieces)
+            rendered += "g%s" % pieces["short"]
+    else:
+        # exception #1
+        rendered = "0.post%d" % pieces["distance"]
+        if pieces["dirty"]:
+            rendered += ".dev0"
+        rendered += "+g%s" % pieces["short"]
+    return rendered
+
+
+def render_pep440_old(pieces):
+    """TAG[.postDISTANCE[.dev0]] .
+
+    The ".dev0" means dirty.
+
+    Eexceptions:
+    1: no tags. 0.postDISTANCE[.dev0]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += ".post%d" % pieces["distance"]
+            if pieces["dirty"]:
+                rendered += ".dev0"
+    else:
+        # exception #1
+        rendered = "0.post%d" % pieces["distance"]
+        if pieces["dirty"]:
+            rendered += ".dev0"
+    return rendered
+
+
+def render_git_describe(pieces):
+    """TAG[-DISTANCE-gHEX][-dirty].
+
+    Like 'git describe --tags --dirty --always'.
+
+    Exceptions:
+    1: no tags. HEX[-dirty]  (note: no 'g' prefix)
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"]:
+            rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
+    else:
+        # exception #1
+        rendered = pieces["short"]
+    if pieces["dirty"]:
+        rendered += "-dirty"
+    return rendered
+
+
+def render_git_describe_long(pieces):
+    """TAG-DISTANCE-gHEX[-dirty].
+
+    Like 'git describe --tags --dirty --always -long'.
+    The distance/hash is unconditional.
+
+    Exceptions:
+    1: no tags. HEX[-dirty]  (note: no 'g' prefix)
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
+    else:
+        # exception #1
+        rendered = pieces["short"]
+    if pieces["dirty"]:
+        rendered += "-dirty"
+    return rendered
+
+
+def render(pieces, style):
+    """Render the given version pieces into the requested style."""
+    if pieces["error"]:
+        return {"version": "unknown",
+                "full-revisionid": pieces.get("long"),
+                "dirty": None,
+                "error": pieces["error"],
+                "date": None}
+
+    if not style or style == "default":
+        style = "pep440"  # the default
+
+    if style == "pep440":
+        rendered = render_pep440(pieces)
+    elif style == "pep440-pre":
+        rendered = render_pep440_pre(pieces)
+    elif style == "pep440-post":
+        rendered = render_pep440_post(pieces)
+    elif style == "pep440-old":
+        rendered = render_pep440_old(pieces)
+    elif style == "git-describe":
+        rendered = render_git_describe(pieces)
+    elif style == "git-describe-long":
+        rendered = render_git_describe_long(pieces)
+    else:
+        raise ValueError("unknown style '%s'" % style)
+
+    return {"version": rendered, "full-revisionid": pieces["long"],
+            "dirty": pieces["dirty"], "error": None,
+            "date": pieces.get("date")}
+
+
+class VersioneerBadRootError(Exception):
+    """The project root directory is unknown or missing key files."""
+
+
+def get_versions(verbose=False):
+    """Get the project version from whatever source is available.
+
+    Returns dict with two keys: 'version' and 'full'.
+    """
+    if "versioneer" in sys.modules:
+        # see the discussion in cmdclass.py:get_cmdclass()
+        del sys.modules["versioneer"]
+
+    root = get_root()
+    cfg = get_config_from_root(root)
+
+    assert cfg.VCS is not None, "please set [versioneer]VCS= in setup.cfg"
+    handlers = HANDLERS.get(cfg.VCS)
+    assert handlers, "unrecognized VCS '%s'" % cfg.VCS
+    verbose = verbose or cfg.verbose
+    assert cfg.versionfile_source is not None, \
+        "please set versioneer.versionfile_source"
+    assert cfg.tag_prefix is not None, "please set versioneer.tag_prefix"
+
+    versionfile_abs = os.path.join(root, cfg.versionfile_source)
+
+    # extract version from first of: _version.py, VCS command (e.g. 'git
+    # describe'), parentdir. This is meant to work for developers using a
+    # source checkout, for users of a tarball created by 'setup.py sdist',
+    # and for users of a tarball/zipball created by 'git archive' or github's
+    # download-from-tag feature or the equivalent in other VCSes.
+
+    get_keywords_f = handlers.get("get_keywords")
+    from_keywords_f = handlers.get("keywords")
+    if get_keywords_f and from_keywords_f:
+        try:
+            keywords = get_keywords_f(versionfile_abs)
+            ver = from_keywords_f(keywords, cfg.tag_prefix, verbose)
+            if verbose:
+                print("got version from expanded keyword %s" % ver)
+            return ver
+        except NotThisMethod:
+            pass
+
+    try:
+        ver = versions_from_file(versionfile_abs)
+        if verbose:
+            print("got version from file %s %s" % (versionfile_abs, ver))
+        return ver
+    except NotThisMethod:
+        pass
+
+    from_vcs_f = handlers.get("pieces_from_vcs")
+    if from_vcs_f:
+        try:
+            pieces = from_vcs_f(cfg.tag_prefix, root, verbose)
+            ver = render(pieces, cfg.style)
+            if verbose:
+                print("got version from VCS %s" % ver)
+            return ver
+        except NotThisMethod:
+            pass
+
+    try:
+        if cfg.parentdir_prefix:
+            ver = versions_from_parentdir(cfg.parentdir_prefix, root, verbose)
+            if verbose:
+                print("got version from parentdir %s" % ver)
+            return ver
+    except NotThisMethod:
+        pass
+
+    if verbose:
+        print("unable to compute version")
+
+    return {"version": "0+unknown", "full-revisionid": None,
+            "dirty": None, "error": "unable to compute version",
+            "date": None}
+
+
+def get_version():
+    """Get the short version string for this project."""
+    return get_versions()["version"]
+
+
+def get_cmdclass():
+    """Get the custom setuptools/distutils subclasses used by Versioneer."""
+    if "versioneer" in sys.modules:
+        del sys.modules["versioneer"]
+        # this fixes the "python setup.py develop" case (also 'install' and
+        # 'easy_install .'), in which subdependencies of the main project are
+        # built (using setup.py bdist_egg) in the same python process. Assume
+        # a main project A and a dependency B, which use different versions
+        # of Versioneer. A's setup.py imports A's Versioneer, leaving it in
+        # sys.modules by the time B's setup.py is executed, causing B to run
+        # with the wrong versioneer. Setuptools wraps the sub-dep builds in a
+        # sandbox that restores sys.modules to it's pre-build state, so the
+        # parent is protected against the child's "import versioneer". By
+        # removing ourselves from sys.modules here, before the child build
+        # happens, we protect the child from the parent's versioneer too.
+        # Also see https://github.com/warner/python-versioneer/issues/52
+
+    cmds = {}
+
+    # we add "version" to both distutils and setuptools
+    from distutils.core import Command
+
+    class cmd_version(Command):
+        description = "report generated version string"
+        user_options = []
+        boolean_options = []
+
+        def initialize_options(self):
+            pass
+
+        def finalize_options(self):
+            pass
+
+        def run(self):
+            vers = get_versions(verbose=True)
+            print("Version: %s" % vers["version"])
+            print(" full-revisionid: %s" % vers.get("full-revisionid"))
+            print(" dirty: %s" % vers.get("dirty"))
+            print(" date: %s" % vers.get("date"))
+            if vers["error"]:
+                print(" error: %s" % vers["error"])
+    cmds["version"] = cmd_version
+
+    # we override "build_py" in both distutils and setuptools
+    #
+    # most invocation pathways end up running build_py:
+    #  distutils/build -> build_py
+    #  distutils/install -> distutils/build ->..
+    #  setuptools/bdist_wheel -> distutils/install ->..
+    #  setuptools/bdist_egg -> distutils/install_lib -> build_py
+    #  setuptools/install -> bdist_egg ->..
+    #  setuptools/develop -> ?
+    #  pip install:
+    #   copies source tree to a tempdir before running egg_info/etc
+    #   if .git isn't copied too, 'git describe' will fail
+    #   then does setup.py bdist_wheel, or sometimes setup.py install
+    #  setup.py egg_info -> ?
+
+    # we override different "build_py" commands for both environments
+    if "setuptools" in sys.modules:
+        from setuptools.command.build_py import build_py as _build_py
+    else:
+        from distutils.command.build_py import build_py as _build_py
+
+    class cmd_build_py(_build_py):
+        def run(self):
+            root = get_root()
+            cfg = get_config_from_root(root)
+            versions = get_versions()
+            _build_py.run(self)
+            # now locate _version.py in the new build/ directory and replace
+            # it with an updated value
+            if cfg.versionfile_build:
+                target_versionfile = os.path.join(self.build_lib,
+                                                  cfg.versionfile_build)
+                print("UPDATING %s" % target_versionfile)
+                write_to_version_file(target_versionfile, versions)
+    cmds["build_py"] = cmd_build_py
+
+    if "cx_Freeze" in sys.modules:  # cx_freeze enabled?
+        from cx_Freeze.dist import build_exe as _build_exe
+        # nczeczulin reports that py2exe won't like the pep440-style string
+        # as FILEVERSION, but it can be used for PRODUCTVERSION, e.g.
+        # setup(console=[{
+        #   "version": versioneer.get_version().split("+", 1)[0], # FILEVERSION
+        #   "product_version": versioneer.get_version(),
+        #   ...
+
+        class cmd_build_exe(_build_exe):
+            def run(self):
+                root = get_root()
+                cfg = get_config_from_root(root)
+                versions = get_versions()
+                target_versionfile = cfg.versionfile_source
+                print("UPDATING %s" % target_versionfile)
+                write_to_version_file(target_versionfile, versions)
+
+                _build_exe.run(self)
+                os.unlink(target_versionfile)
+                with open(cfg.versionfile_source, "w") as f:
+                    LONG = LONG_VERSION_PY[cfg.VCS]
+                    f.write(LONG %
+                            {"DOLLAR": "$",
+                             "STYLE": cfg.style,
+                             "TAG_PREFIX": cfg.tag_prefix,
+                             "PARENTDIR_PREFIX": cfg.parentdir_prefix,
+                             "VERSIONFILE_SOURCE": cfg.versionfile_source,
+                             })
+        cmds["build_exe"] = cmd_build_exe
+        del cmds["build_py"]
+
+    if 'py2exe' in sys.modules:  # py2exe enabled?
+        try:
+            from py2exe.distutils_buildexe import py2exe as _py2exe  # py3
+        except ImportError:
+            from py2exe.build_exe import py2exe as _py2exe  # py2
+
+        class cmd_py2exe(_py2exe):
+            def run(self):
+                root = get_root()
+                cfg = get_config_from_root(root)
+                versions = get_versions()
+                target_versionfile = cfg.versionfile_source
+                print("UPDATING %s" % target_versionfile)
+                write_to_version_file(target_versionfile, versions)
+
+                _py2exe.run(self)
+                os.unlink(target_versionfile)
+                with open(cfg.versionfile_source, "w") as f:
+                    LONG = LONG_VERSION_PY[cfg.VCS]
+                    f.write(LONG %
+                            {"DOLLAR": "$",
+                             "STYLE": cfg.style,
+                             "TAG_PREFIX": cfg.tag_prefix,
+                             "PARENTDIR_PREFIX": cfg.parentdir_prefix,
+                             "VERSIONFILE_SOURCE": cfg.versionfile_source,
+                             })
+        cmds["py2exe"] = cmd_py2exe
+
+    # we override different "sdist" commands for both environments
+    if "setuptools" in sys.modules:
+        from setuptools.command.sdist import sdist as _sdist
+    else:
+        from distutils.command.sdist import sdist as _sdist
+
+    class cmd_sdist(_sdist):
+        def run(self):
+            versions = get_versions()
+            self._versioneer_generated_versions = versions
+            # unless we update this, the command will keep using the old
+            # version
+            self.distribution.metadata.version = versions["version"]
+            return _sdist.run(self)
+
+        def make_release_tree(self, base_dir, files):
+            root = get_root()
+            cfg = get_config_from_root(root)
+            _sdist.make_release_tree(self, base_dir, files)
+            # now locate _version.py in the new base_dir directory
+            # (remembering that it may be a hardlink) and replace it with an
+            # updated value
+            target_versionfile = os.path.join(base_dir, cfg.versionfile_source)
+            print("UPDATING %s" % target_versionfile)
+            write_to_version_file(target_versionfile,
+                                  self._versioneer_generated_versions)
+    cmds["sdist"] = cmd_sdist
+
+    return cmds
+
+
+CONFIG_ERROR = """
+setup.cfg is missing the necessary Versioneer configuration. You need
+a section like:
+
+ [versioneer]
+ VCS = git
+ style = pep440
+ versionfile_source = src/myproject/_version.py
+ versionfile_build = myproject/_version.py
+ tag_prefix =
+ parentdir_prefix = myproject-
+
+You will also need to edit your setup.py to use the results:
+
+ import versioneer
+ setup(version=versioneer.get_version(),
+       cmdclass=versioneer.get_cmdclass(), ...)
+
+Please read the docstring in ./versioneer.py for configuration instructions,
+edit setup.cfg, and re-run the installer or 'python versioneer.py setup'.
+"""
+
+SAMPLE_CONFIG = """
+# See the docstring in versioneer.py for instructions. Note that you must
+# re-run 'versioneer.py setup' after changing this section, and commit the
+# resulting files.
+
+[versioneer]
+#VCS = git
+#style = pep440
+#versionfile_source =
+#versionfile_build =
+#tag_prefix =
+#parentdir_prefix =
+
+"""
+
+INIT_PY_SNIPPET = """
+from ._version import get_versions
+__version__ = get_versions()['version']
+del get_versions
+"""
+
+
+def do_setup():
+    """Main VCS-independent setup function for installing Versioneer."""
+    root = get_root()
+    try:
+        cfg = get_config_from_root(root)
+    except (EnvironmentError, configparser.NoSectionError,
+            configparser.NoOptionError) as e:
+        if isinstance(e, (EnvironmentError, configparser.NoSectionError)):
+            print("Adding sample versioneer config to setup.cfg",
+                  file=sys.stderr)
+            with open(os.path.join(root, "setup.cfg"), "a") as f:
+                f.write(SAMPLE_CONFIG)
+        print(CONFIG_ERROR, file=sys.stderr)
+        return 1
+
+    print(" creating %s" % cfg.versionfile_source)
+    with open(cfg.versionfile_source, "w") as f:
+        LONG = LONG_VERSION_PY[cfg.VCS]
+        f.write(LONG % {"DOLLAR": "$",
+                        "STYLE": cfg.style,
+                        "TAG_PREFIX": cfg.tag_prefix,
+                        "PARENTDIR_PREFIX": cfg.parentdir_prefix,
+                        "VERSIONFILE_SOURCE": cfg.versionfile_source,
+                        })
+
+    ipy = os.path.join(os.path.dirname(cfg.versionfile_source),
+                       "__init__.py")
+    if os.path.exists(ipy):
+        try:
+            with open(ipy, "r") as f:
+                old = f.read()
+        except EnvironmentError:
+            old = ""
+        if INIT_PY_SNIPPET not in old:
+            print(" appending to %s" % ipy)
+            with open(ipy, "a") as f:
+                f.write(INIT_PY_SNIPPET)
+        else:
+            print(" %s unmodified" % ipy)
+    else:
+        print(" %s doesn't exist, ok" % ipy)
+        ipy = None
+
+    # Make sure both the top-level "versioneer.py" and versionfile_source
+    # (PKG/_version.py, used by runtime code) are in MANIFEST.in, so
+    # they'll be copied into source distributions. Pip won't be able to
+    # install the package without this.
+    manifest_in = os.path.join(root, "MANIFEST.in")
+    simple_includes = set()
+    try:
+        with open(manifest_in, "r") as f:
+            for line in f:
+                if line.startswith("include "):
+                    for include in line.split()[1:]:
+                        simple_includes.add(include)
+    except EnvironmentError:
+        pass
+    # That doesn't cover everything MANIFEST.in can do
+    # (http://docs.python.org/2/distutils/sourcedist.html#commands), so
+    # it might give some false negatives. Appending redundant 'include'
+    # lines is safe, though.
+    if "versioneer.py" not in simple_includes:
+        print(" appending 'versioneer.py' to MANIFEST.in")
+        with open(manifest_in, "a") as f:
+            f.write("include versioneer.py\n")
+    else:
+        print(" 'versioneer.py' already in MANIFEST.in")
+    if cfg.versionfile_source not in simple_includes:
+        print(" appending versionfile_source ('%s') to MANIFEST.in" %
+              cfg.versionfile_source)
+        with open(manifest_in, "a") as f:
+            f.write("include %s\n" % cfg.versionfile_source)
+    else:
+        print(" versionfile_source already in MANIFEST.in")
+
+    # Make VCS-specific changes. For git, this means creating/changing
+    # .gitattributes to mark _version.py for export-subst keyword
+    # substitution.
+    do_vcs_install(manifest_in, cfg.versionfile_source, ipy)
+    return 0
+
+
+def scan_setup_py():
+    """Validate the contents of setup.py against Versioneer's expectations."""
+    found = set()
+    setters = False
+    errors = 0
+    with open("setup.py", "r") as f:
+        for line in f.readlines():
+            if "import versioneer" in line:
+                found.add("import")
+            if "versioneer.get_cmdclass()" in line:
+                found.add("cmdclass")
+            if "versioneer.get_version()" in line:
+                found.add("get_version")
+            if "versioneer.VCS" in line:
+                setters = True
+            if "versioneer.versionfile_source" in line:
+                setters = True
+    if len(found) != 3:
+        print("")
+        print("Your setup.py appears to be missing some important items")
+        print("(but I might be wrong). Please make sure it has something")
+        print("roughly like the following:")
+        print("")
+        print(" import versioneer")
+        print(" setup( version=versioneer.get_version(),")
+        print("        cmdclass=versioneer.get_cmdclass(),  ...)")
+        print("")
+        errors += 1
+    if setters:
+        print("You should remove lines like 'versioneer.VCS = ' and")
+        print("'versioneer.versionfile_source = ' . This configuration")
+        print("now lives in setup.cfg, and should be removed from setup.py")
+        print("")
+        errors += 1
+    return errors
+
+
+if __name__ == "__main__":
+    cmd = sys.argv[1]
+    if cmd == "setup":
+        errors = do_setup()
+        errors += scan_setup_py()
+        if errors:
+            sys.exit(1)