Browse Source

[metadata] Initial creation of metadata lib with navigator API module

Jenny Kim 10 years ago
parent
commit
4a85fc5

+ 1 - 0
desktop/Makefile

@@ -51,6 +51,7 @@ APPS := core \
 	libs/libsentry \
 	libs/libsolr \
 	libs/libzookeeper \
+	libs/metadata \
 	libs/notebook
 
 .PHONY: default

+ 18 - 0
desktop/conf.dist/hue.ini

@@ -1321,3 +1321,21 @@
 
   # Whitelisted domains (only applies to Google OAuth). CSV format.
   ## whitelisted_domains_google=
+
+
+###########################################################################
+# Settings to configure Metadata
+###########################################################################
+
+[metadata]
+  # For metadata tagging and enrichment features
+
+  [[navigator]]
+  # Navigator API URL with version
+  ## api_url=http://jennykim-1.vpc.cloudera.com:7187/api/v2
+
+  # Navigator API HTTP authentication username and password
+  # Override the desktop default username and password of the hue user used for authentications with other services.
+  # e.g. Used for LDAP/PAM pass-through authentication.
+  ## auth_username=hue
+  ## auth_password=

+ 18 - 0
desktop/conf/pseudo-distributed.ini.tmpl

@@ -1323,3 +1323,21 @@
 
   # Whitelisted domains (only applies to Google OAuth). CSV format.
   ## whitelisted_domains_google=
+
+
+###########################################################################
+# Settings to configure Metadata
+###########################################################################
+
+[metadata]
+  # For metadata tagging and enrichment features
+
+  [[navigator]]
+  # Navigator API URL with version
+  ## api_url=http://jennykim-1.vpc.cloudera.com:7187/api/v2
+
+  # Navigator API HTTP authentication username and password
+  # Override the desktop default username and password of the hue user used for authentications with other services.
+  # e.g. Used for LDAP/PAM pass-through authentication.
+  ## auth_username=hue
+  ## auth_password=

+ 36 - 0
desktop/libs/metadata/Makefile

@@ -0,0 +1,36 @@
+#
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+ifeq ($(ROOT),)
+  $(error "Error: Expect the environment variable $$ROOT to point to the Desktop installation")
+endif
+
+include $(ROOT)/Makefile.sdk
+
+default::
+	@echo '  env-install    : Install into virtual-env'
+
+#
+# env-install
+#   Install app into the virtual environment.
+#
+.PHONY: env-install
+env-install: compile ext-env-install
+	@echo '--- Installing $(APP_NAME) into virtual-env'
+	@$(ENV_PYTHON) setup.py develop -N -q

+ 0 - 0
desktop/libs/metadata/__init__.py


+ 1 - 0
desktop/libs/metadata/babel.cfg

@@ -0,0 +1 @@
+[python: src/metadata/**.py]

+ 1 - 0
desktop/libs/metadata/hueversion.py

@@ -0,0 +1 @@
+../../../VERSION

+ 31 - 0
desktop/libs/metadata/setup.py

@@ -0,0 +1,31 @@
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from setuptools import setup, find_packages
+from hueversion import VERSION
+
+setup(
+      name = "metadata",
+      version = VERSION,
+      url = 'http://github.com/cloudera/hue',
+      description = "Search Libraries",
+      packages = find_packages('src'),
+      package_dir = {'': 'src' },
+      install_requires = ['setuptools', 'desktop'],
+      # Even libraries need to be registered as desktop_apps,
+      # if they have configuration, like this one.
+      entry_points = { 'desktop.sdk.lib': 'metadata=metadata' },
+)

+ 15 - 0
desktop/libs/metadata/src/metadata/__init__.py

@@ -0,0 +1,15 @@
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

+ 64 - 0
desktop/libs/metadata/src/metadata/conf.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from django.utils.translation import ugettext_lazy as _t
+
+from desktop.conf import AUTH_USERNAME as DEFAULT_AUTH_USERNAME, AUTH_PASSWORD as DEFAULT_AUTH_PASSWORD, \
+  AUTH_PASSWORD_SCRIPT, coerce_password_from_script
+from desktop.lib.conf import Config, ConfigSection, coerce_string
+
+
+def get_auth_username():
+  """Get from top level default from desktop"""
+  return DEFAULT_AUTH_USERNAME.get()
+
+
+def get_auth_password():
+  """Get from script or backward compatibility"""
+  password = AUTH_PASSWORD_SCRIPT.get()
+  if password:
+    return password
+  return DEFAULT_AUTH_PASSWORD.get()
+
+
+NAVIGATOR = ConfigSection(
+  key='navigator',
+  help=_t("""Configuration options for Navigator API"""),
+  members=dict(
+    API_URL=Config(
+      key='api_url',
+      help=_t('Base URL to Navigator API'),
+      default='http://localhost:7187/api/v2',
+      type=coerce_string),
+    AUTH_USERNAME=Config(
+      key="auth_username",
+      help=_t("Auth username of the hue user used for authentications."),
+      private=True,
+      dynamic_default=get_auth_username),
+    AUTH_PASSWORD=Config(
+      key="auth_password",
+      help=_t("LDAP/PAM/.. password of the hue user used for authentications."),
+      private=True,
+      dynamic_default=get_auth_password),
+    AUTH_PASSWORD_SCRIPT=Config(
+      key="auth_password_script",
+      help=_t("Execute this script to produce the auth password. This will be used when `auth_password` is not set."),
+      private=True,
+      type=coerce_password_from_script,
+      default=None),
+  )
+)

+ 74 - 0
desktop/libs/metadata/src/metadata/navigator.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python
+# -- coding: utf-8 --
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+from desktop.lib.rest.http_client import HttpClient, RestException
+from desktop.lib.rest import resource
+
+from metadata.conf import NAVIGATOR
+
+
+LOG = logging.getLogger(__name__)
+
+
+class NavigatorApiException(Exception):
+  pass
+
+
+class NavigatorApi(object):
+  """
+  http://cloudera.github.io/navigator/apidocs/v2/index.html
+  """
+
+  def __init__(self, api_url=None, user=None, password=None):
+    self._api_url = (api_url or NAVIGATOR.API_URL.get()).strip('/')
+    self._username = user or NAVIGATOR.AUTH_USERNAME.get()
+    self._password = password or NAVIGATOR.AUTH_PASSWORD.get()
+
+    self._client = HttpClient(self._api_url, logger=LOG)
+    self._client.set_basic_auth(self._username, self._password)
+    self._root = resource.Resource(self._client)
+
+    self.__headers = {}
+    self.__params = ()
+
+
+  def find_entity(self, type, name):
+    """
+    GET /api/v2/interactive/entities?query=((originalName:<name>)AND(type:<type>))
+    http://cloudera.github.io/navigator/apidocs/v2/path__v2_interactive_entities.html
+    """
+    try:
+      params = self.__params
+      filter_query = '((originalName:%(name)s)AND(type:%(type)s))' % {'name': name, 'type': type}
+      params += (
+        ('query', filter_query),
+        ('offset', 0),
+      )
+
+      response = self._root.get('interactive/entities', headers=self.__headers, params=params)
+      if response['totalMatched'] == 0:
+        raise NavigatorApiException('Could not find entity with type %s and name %s') % (type, name)
+      elif response['totalMatched'] > 1:
+        raise NavigatorApiException('Found more than 1 entity with type %s and name %s') % (type, name)
+      else:
+        return response['results'][0]
+
+    except RestException, e:
+      raise NavigatorApiException('Failed to find entity with type %s and name %s: %s' % (type, name, str(e)))

+ 18 - 0
desktop/libs/metadata/src/metadata/tests.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from nose.tools import assert_equal, assert_true, assert_false