Selaa lähdekoodia

HUE-8758 [connectors] Adding first part of Hive metastore connector

Romain 6 vuotta sitten
vanhempi
commit
72c0cd6175

+ 3 - 1
apps/beeswax/src/beeswax/server/dbms.py

@@ -49,16 +49,18 @@ from beeswax.design import hql_query
 from beeswax.hive_site import hiveserver2_use_ssl
 from beeswax.models import QueryHistory, QUERY_TYPES
 
-LOG = logging.getLogger(__name__)
 
+LOG = logging.getLogger(__name__)
 
 DBMS_CACHE = {}
 DBMS_CACHE_LOCK = threading.Lock()
 cache = caches[CACHES_HIVE_DISCOVERY_KEY]
+
 # Using file cache to make sure eventlet threads are uniform, this cache is persistent on startup
 # So we clear it to make sure the server resets hiveserver2 host.
 cache.clear()
 
+
 def get(user, query_server=None, cluster=None):
   global DBMS_CACHE
   global DBMS_CACHE_LOCK

+ 6 - 1
apps/impala/src/impala/dbms.py

@@ -181,7 +181,12 @@ class ImpalaDbms(HiveServer2Dbms):
 
 
   def _get_beeswax_tables(self, database):
-    beeswax_query_server = dbms.get(user=self.client.user, query_server=beeswax_query_server_config(name=Cluster(self.client.user).get_app_config().get_hive_metastore_interpreters()[0]))
+    beeswax_query_server = dbms.get(
+      user=self.client.user,
+      query_server=beeswax_query_server_config(
+        name=Cluster(self.client.user).get_app_config().get_hive_metastore_interpreters()[0]
+      )
+    )
     return beeswax_query_server.get_tables(database=database)
 
 

+ 6 - 2
apps/metastore/src/metastore/templates/metastore.mako

@@ -78,7 +78,9 @@ ${ components.menubar(is_embeddable) }
 <script type="text/html" id="metastore-breadcrumbs">
   <div style="font-size: 14px; margin: 0 12px; line-height: 27px;">
     <!-- ko if: sources().length >= 2 -->
-    <div data-bind="component: { name: 'hue-drop-down', params: { value: source, entries: sources, onSelect: sourceChanged, labelAttribute: 'name', searchable: true, linkTitle: '${ _ko('Source') }' } }" style="display: inline-block"></div>
+    <div data-bind="component: { name: 'hue-drop-down', params: { value: source, entries: sources, onSelect: sourceChanged, labelAttribute: 'name', searchable: true, linkTitle: '${ _ko('Source') }' } }"
+      style="display: inline-block">
+    </div>
     <!-- /ko -->
     <!-- ko with: source -->
     <!-- ko if: window.HAS_MULTI_CLUSTER -->
@@ -86,7 +88,9 @@ ${ components.menubar(is_embeddable) }
     <i class="margin-left-10 fa fa-warning"></i> ${ _('No namespaces found') }
     <!-- /ko -->
     <!-- ko if: namespaces().length > 0 -->
-    <div class="margin-left-10" data-bind="component: { name: 'hue-drop-down', params: { value: namespace, entries: namespaces, onSelect: namespaceChanged, labelAttribute: 'name', searchable: true, linkTitle: '${ _ko('Namespace') }' } }" style="display: inline-block"></div>
+    <div class="margin-left-10" data-bind="component: { name: 'hue-drop-down', params: { value: namespace, entries: namespaces, onSelect: namespaceChanged, labelAttribute: 'name', searchable: true, linkTitle: '${ _ko('Namespace') }' } }"
+      style="display: inline-block">
+    </div>
     <!-- /ko -->
     <!-- /ko -->
     <!-- /ko -->

+ 17 - 7
desktop/core/src/desktop/lib/connectors/api.py

@@ -74,9 +74,17 @@ CONNECTOR_CLASSES += [
   {'nice_name': "S3", 'dialect': 's3', 'settings': [], 'category': 'browsers', 'description': '', 'properties': {}},
   {'nice_name': "ADLS", 'dialect': 'adls-v1', 'settings': [], 'category': 'browsers', 'description': '', 'properties': {}},
 
-  {'nice_name': "Hive Metastore", 'dialect': 'hms', 'settings': [], 'category': 'catalogs', 'description': '', 'properties': {}},
-  {'nice_name': "Atlas", 'dialect': 'atlas', 'settings': [], 'category': 'catalogs', 'description': '', 'properties': {}},
-  {'nice_name': "Navigator", 'dialect': 'navigator', 'settings': [], 'category': 'catalogs', 'description': '', 'properties': {}},
+  {
+    'nice_name': "Hive Metastore",
+    'dialect': 'hms',
+    'interface': 'hiveserver2',
+    'settings': [{'name': 'server_host', 'value': ''}, {'name': 'server_port', 'value': ''},],
+    'category': 'catalogs',
+    'description': '',
+    'properties': {}
+  },
+  {'nice_name': "Atlas", 'dialect': 'atlas', 'interface': 'rest', 'settings': [], 'category': 'catalogs', 'description': '', 'properties': {}},
+  {'nice_name': "Navigator", 'dialect': 'navigator', 'interface': 'rest', 'settings': [], 'category': 'catalogs', 'description': '', 'properties': {}},
 
   {'nice_name': "Optimizer", 'dialect': 'optimizer', 'settings': [], 'category': 'optimizers', 'description': '', 'properties': {}},
 
@@ -195,7 +203,7 @@ def delete_connector(request):
     raise PopupException(_('No connector with the name %(name)s found.') % connector)
 
 
-def _get_installed_connectors(category=None, dialect=None, interface=None):
+def _get_installed_connectors(category=None, categories=None, dialect=None, interface=None):
   global CONNECTOR_INSTANCES
   global CONNECTOR_IDS
   config_connectors = CONNECTORS.get()
@@ -224,12 +232,14 @@ def _get_installed_connectors(category=None, dialect=None, interface=None):
 
   connectors = CONNECTOR_INSTANCES
 
+  if categories is not None:
+    connectors = [connector for connector in connectors if connector['category'] in categories]
   if category is not None:
-    connectors = [connector for connector in connectors if category == connector['category']]
+    connectors = [connector for connector in connectors if connector['category'] == category]
   if dialect is not None:
-    connectors = [connector for connector in connectors if dialect == connector['dialect']]
+    connectors = [connector for connector in connectors if connector['dialect'] == dialect]
   if interface is not None:
-    connectors = [connector for connector in connectors if interface == connector['interface']]
+    connectors = [connector for connector in connectors if connector['interface'] == interface]
 
   return connectors
 

+ 6 - 6
desktop/core/src/desktop/models.py

@@ -1581,10 +1581,6 @@ class ClusterConfig(object):
     self.apps = appmanager.get_apps_dict(self.user) if apps is None else apps
     self.cluster_type = cluster_type
 
-  def refreshConfig(self):
-    # TODO: reload "some ini sections"
-    pass
-
 
   def get_config(self):
     app_config = self.get_apps()
@@ -1711,12 +1707,16 @@ class ClusterConfig(object):
       return None
 
   def _get_catalogs(self):
+    '''
+    Any connector implementing Notebook Assist API.
+    Currently SQL connectors, Hive Metastore. Possibly Metadata Catalogs later.
+    '''
     interpreters = []
 
     _interpreters = get_ordered_interpreters(self.user)
 
     for interpreter in _interpreters:
-      if interpreter['interface'] == 'hms':
+      if interpreter['is_sql'] or interpreter['interface'] == 'hms':
         interpreters.append({
           'name': interpreter['name'],
           'type': interpreter['type'],
@@ -1725,7 +1725,7 @@ class ClusterConfig(object):
           'tooltip': _('%s Query') % interpreter['type'].title(),
           'page': '/editor/?type=%(type)s' % interpreter,
           'is_sql': interpreter['is_sql'],
-          'is_catalog': interpreter['is_catalog'],
+          'is_catalog': True,
         })
 
     return interpreters if interpreters else None

+ 1 - 1
desktop/core/src/desktop/views.py

@@ -133,7 +133,7 @@ def home(request):
 
 def home2(request, is_embeddable=False):
   apps = appmanager.get_apps_dict(request.user)
-
+  
   return render('home2.mako', request, {
     'apps': apps,
     'is_embeddable': request.GET.get('is_embeddable', False)

+ 4 - 2
desktop/libs/metadata/src/metadata/catalog_api.py

@@ -513,7 +513,8 @@ def create_namespace_property(request):
   "enumValues" : null,
   "type" : "TEXT",
   "createdDate" : "2018-04-02T22:36:19.001Z"
-}"""
+  }
+  """
   interface = request.POST.get('interface', CATALOG.INTERFACE.get())
   namespace = request.POST.get('namespace')
   properties = json.loads(request.POST.get('properties', '{}'))
@@ -531,7 +532,8 @@ def map_namespace_property(request):
   {
   namespace: "huecatalog",
   name: "relatedEntities"
-  }"""
+  }
+  """
   interface = request.POST.get('interface', CATALOG.INTERFACE.get())
   clazz = request.POST.get('class')
   properties = json.loads(request.POST.get('properties', '[]'))

+ 2 - 2
desktop/libs/notebook/src/notebook/conf.py

@@ -71,10 +71,10 @@ def get_ordered_interpreters(user=None):
         'type': connector['name'],
         'dialect': connector['dialect'],
         'category': connector['category'],
-        'is_sql': connector['is_sql'],
+        'is_sql': connector.get('is_sql', False),
         'interface': connector['interface'],
         'options': {setting['name']: setting['value'] for setting in connector['settings']}
-      } for connector in _get_installed_connectors(category='editor')
+      } for connector in _get_installed_connectors(categories=['editor', 'catalogs'])
     ]
   else:
     reordered_interpreters = interpreters_shown_on_wheel + [i for i in user_interpreters if i not in interpreters_shown_on_wheel]

+ 1 - 1
desktop/libs/notebook/src/notebook/connectors/base.py

@@ -336,7 +336,7 @@ def get_api(request, snippet):
     interpreter.get('compute') and interpreter['compute']['name'])
   )
 
-  if interface == 'hiveserver2':
+  if interface == 'hiveserver2' or interface == 'hms':
     from notebook.connectors.hiveserver2 import HS2Api
     return HS2Api(user=request.user, request=request, interpreter=interpreter)
   elif interface == 'oozie':