Browse Source

[metadata] Basic skeleton on API for an SQL Optimizer

API is not implemented completely, working on authenticate now.
Romain Rigaux 10 years ago
parent
commit
36553b7

+ 30 - 8
desktop/conf.dist/hue.ini

@@ -1354,12 +1354,34 @@
 [metadata]
   # For metadata tagging and enrichment features
 
-  [[navigator]]
-  # Navigator API URL with version
-  ## api_url=http://localhost:7187/api/v2
+  [[optimizer]]
+    # For SQL query and table analysis
+    # Base URL to Optimizer API.
+    ## api_url=https://alpha.optimizer.cloudera.com
+    # The name of the product or group which will have API access to the emails associated with it.
+    ## product_name=hue
+    # A secret passphrase associated with the productName
+    ## product_secret=hue
+    # Execute this script to produce the product secret. This will be used when `product_secret` is not set.
+    ## product_secret_script=
+
+    # The email of the Optimizer account you want to associate with the Product.
+    ## email=hue@gethue.com
+    # The password associated with the Optimizer account you to associate with the Product.
+    ## email_password=hue
+    # Execute this script to produce the email password. This will be used when `email_password` is not set.
+    ## password_script=
+
+    # In secure mode (HTTPS), if Optimizer SSL certificates have to be verified against certificate authority.
+    ## ssl_cert_ca_verify=True
 
-  # Navigator API HTTP authentication username and password
-  # Override the desktop default username and password of the hue user used for authentications with other services.
-  # e.g. Used for LDAP/PAM pass-through authentication.
-  ## auth_username=hue
-  ## auth_password=
+  [[navigator]]
+    # For tagging tables, files and getting lineage of data.
+    # Navigator API URL with version
+    ## api_url=http://localhost:7187/api/v2
+
+    # Navigator API HTTP authentication username and password
+    # Override the desktop default username and password of the hue user used for authentications with other services.
+    # e.g. Used for LDAP/PAM pass-through authentication.
+    ## auth_username=hue
+    ## auth_password=

+ 30 - 8
desktop/conf/pseudo-distributed.ini.tmpl

@@ -1356,12 +1356,34 @@
 [metadata]
   # For metadata tagging and enrichment features
 
-  [[navigator]]
-  # Navigator API URL with version
-  ## api_url=http://localhost:7187/api/v2
+  [[optimizer]]
+    # For SQL query and table analysis
+    # Base URL to Optimizer API.
+    ## api_url=https://alpha.optimizer.cloudera.com
+    # The name of the product or group which will have API access to the emails associated with it.
+    ## product_name=hue
+    # A secret passphrase associated with the productName
+    ## product_secret=hue
+    # Execute this script to produce the product secret. This will be used when `product_secret` is not set.
+    ## product_secret_script=
+
+    # The email of the Optimizer account you want to associate with the Product.
+    ## email=hue@gethue.com
+    # The password associated with the Optimizer account you to associate with the Product.
+    ## email_password=hue
+    # Execute this script to produce the email password. This will be used when `email_password` is not set.
+    ## password_script=
+
+    # In secure mode (HTTPS), if Optimizer SSL certificates have to be verified against certificate authority.
+    ## ssl_cert_ca_verify=True
 
-  # Navigator API HTTP authentication username and password
-  # Override the desktop default username and password of the hue user used for authentications with other services.
-  # e.g. Used for LDAP/PAM pass-through authentication.
-  ## auth_username=hue
-  ## auth_password=
+  [[navigator]]
+    # For tagging tables, files and getting lineage of data.
+    # Navigator API URL with version
+    ## api_url=http://localhost:7187/api/v2
+
+    # Navigator API HTTP authentication username and password
+    # Override the desktop default username and password of the hue user used for authentications with other services.
+    # e.g. Used for LDAP/PAM pass-through authentication.
+    ## auth_username=hue
+    ## auth_password=

+ 55 - 2
desktop/libs/metadata/src/metadata/conf.py

@@ -18,8 +18,8 @@
 from django.utils.translation import ugettext_lazy as _t
 
 from desktop.conf import AUTH_USERNAME as DEFAULT_AUTH_USERNAME, AUTH_PASSWORD as DEFAULT_AUTH_PASSWORD, \
-                         AUTH_PASSWORD_SCRIPT, coerce_password_from_script
-from desktop.lib.conf import Config, ConfigSection
+                         AUTH_PASSWORD_SCRIPT, coerce_password_from_script, default_ssl_validate
+from desktop.lib.conf import Config, ConfigSection, coerce_bool
 
 
 def get_auth_username():
@@ -35,6 +35,59 @@ def get_auth_password():
   return DEFAULT_AUTH_PASSWORD.get()
 
 
+OPTIMIZER = ConfigSection(
+  key='optimizer',
+  help=_t("""Configuration options for Optimizer API"""),
+  members=dict(
+    API_URL=Config(
+      key='api_url',
+      help=_t('Base URL to Optimizer API (e.g. - https://alpha.optimizer.cloudera.com/)'),
+      default=None),
+
+    PRODUCT_NAME=Config(
+      key="product_name",
+      help=_t("The name of the product or group which will have API access to the emails associated with it."),
+      private=True,
+      dynamic_default=get_auth_username),
+    PRODUCT_SECRET=Config(
+      key="product_secret",
+      help=_t("A secret passphrase associated with the productName."),
+      private=True,
+      dynamic_default=get_auth_password),
+    PRODUCT_SECRET_SCRIPT=Config(
+      key="product_secret_script",
+      help=_t("Execute this script to produce the product secret. This will be used when `product_secret` is not set."),
+      private=True,
+      type=coerce_password_from_script,
+      default=None),
+
+    EMAIL=Config(
+      key="email",
+      help=_t("The email of the Optimizer account you want to associate with the Product."),
+      private=True,
+      dynamic_default=get_auth_username),
+    EMAIL_PASSWORD=Config(
+      key="email_password",
+      help=_t("The password associated with the Optimizer account you to associate with the Product."),
+      private=True,
+      dynamic_default=get_auth_password),
+    EMAIL_PASSWORD_SCRIPT=Config(
+      key="password_script",
+      help=_t("Execute this script to produce the email password. This will be used when `email_password` is not set."),
+      private=True,
+      type=coerce_password_from_script,
+      default=None),
+
+    SSL_CERT_CA_VERIFY = Config(
+      key="ssl_cert_ca_verify",
+      help=_t("In secure mode (HTTPS), if Optimizer SSL certificates have to be verified against certificate authority"),
+      dynamic_default=default_ssl_validate,
+      type=coerce_bool
+    )
+  )
+)
+
+
 NAVIGATOR = ConfigSection(
   key='navigator',
   help=_t("""Configuration options for Navigator API"""),

+ 0 - 0
desktop/libs/metadata/src/metadata/navigator.py → desktop/libs/metadata/src/metadata/navigator_client.py


+ 1 - 1
desktop/libs/metadata/src/metadata/tests.py → desktop/libs/metadata/src/metadata/navigator_tests.py

@@ -28,7 +28,7 @@ from hadoop.pseudo_hdfs4 import is_live_cluster
 from desktop.lib.django_test_util import make_logged_in_client
 from desktop.lib.test_utils import add_to_group, grant_access
 
-from metadata.navigator import NavigatorApi, is_navigator_enabled
+from metadata.navigator_client import NavigatorApi, is_navigator_enabled
 
 
 LOG = logging.getLogger(__name__)

+ 122 - 0
desktop/libs/metadata/src/metadata/optimizer_client.py

@@ -0,0 +1,122 @@
+#!/usr/bin/env python
+# -- coding: utf-8 --
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import logging
+
+from django.utils.translation import ugettext as _
+
+from desktop.lib.exceptions_renderable import PopupException
+from desktop.lib.rest.http_client import HttpClient, RestException
+from desktop.lib.rest import resource
+
+from metadata.conf import OPTIMIZER
+
+
+LOG = logging.getLogger(__name__)
+
+
+def is_optimizer_enabled():
+  return OPTIMIZER.API_URL.get()
+
+
+class OptimizerApiException(Exception):
+  pass
+
+
+class OptimizerApi(object):
+
+  def __init__(self, api_url=None, product_name=None, product_secret=None, ssl_cert_ca_verify=OPTIMIZER.SSL_CERT_CA_VERIFY.get()):
+    self._api_url = (api_url or OPTIMIZER.API_URL.get()).strip('/')
+    self._product_name = product_name if product_name else OPTIMIZER.PRODUCT_NAME.get()
+    self._product_secret = product_secret if product_secret else OPTIMIZER.PRODUCT_SECRET.get()
+
+    self._client = HttpClient(self._api_url, logger=LOG)
+    self._client.set_verify(ssl_cert_ca_verify)
+
+    self._root = resource.Resource(self._client)
+
+  def create_product(self, product_name, product_secret):
+    try:
+      data = {
+          'productName': product_name,
+          'productSecret': product_secret,
+          'authCode': ''
+      }
+      return self._root.post('/api/createProduct', data)
+    except RestException, e:
+      raise PopupException(e, title=_('Error while accessing Optimizer'))
+
+
+  def add_email_to_product(self, email):
+    try:
+      data = {
+          'productName': self._product_name,
+          'productSecret': self._product_secret,
+          'email': '',
+          'password': ''
+      }
+      return self._root.post('/api/addEmailToProduct', data)
+    except RestException, e:
+      raise PopupException(e, title=_('Error while accessing Optimizer'))
+
+
+  def authenticate(self):
+    try:
+      data = {
+          'productName': self._product_name,
+          'productSecret': self._product_secret,
+      }
+      return self._root.post('/api/createProduct', data)
+    except RestException, e:
+      raise PopupException(e, title=_('Error while accessing Optimizer'))
+
+
+  def delete_workload(self):
+    try:
+      data = {
+          'email': email,
+          'token': token,
+      }
+      return self._root.post('/api/deleteWorkload', data)
+    except RestException, e:
+      raise PopupException(e, title=_('Error while accessing Optimizer'))
+
+
+  def get_status(self):
+    try:
+      data = {
+          'email': email,
+          'token': token,
+      }
+      return self._root.post('/api/getStatus', data)
+    except RestException, e:
+      raise PopupException(e, title=_('Error while accessing Optimizer'))
+
+
+  def upload(self):
+    try:
+      data = {
+          'email': email,
+          'token': token,
+          'sourcePlatform': 'generic',
+          'file': 'file'
+      }
+      return self._root.post('/api/upload', data)
+    except RestException, e:
+      raise PopupException(e, title=_('Error while accessing Optimizer'))

+ 63 - 0
desktop/libs/metadata/src/metadata/optimizer_tests.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import json
+
+from nose.plugins.skip import SkipTest
+from nose.tools import assert_equal, assert_true
+
+from django.contrib.auth.models import User
+from django.core.urlresolvers import reverse
+
+from hadoop.pseudo_hdfs4 import is_live_cluster
+from desktop.lib.django_test_util import make_logged_in_client
+from desktop.lib.test_utils import add_to_group, grant_access
+
+from metadata.optimizer_client import OptimizerApi, is_optimizer_enabled
+
+
+LOG = logging.getLogger(__name__)
+
+
+class TestOptimizerApi(object):
+
+  @classmethod
+  def setup_class(cls):
+
+    if not is_optimizer_enabled():
+      raise SkipTest
+
+    cls.client = make_logged_in_client(username='test', is_superuser=False)
+    cls.user = User.objects.get(username='test')
+    add_to_group('test')
+    grant_access("test", "test", "metadata")
+    grant_access("test", "test", "optimizer")
+
+    cls.api = OptimizerApi()
+
+
+  @classmethod
+  def teardown_class(cls):
+    cls.user.is_superuser = False
+    cls.user.save()
+
+
+  def test_api_authenticate(self):
+    resp = self.api.authenticate()
+
+    assert_equal('success', resp['status'], resp)