瀏覽代碼

HUE-3228 [search] Skeleton of common API

Romain Rigaux 8 年之前
父節點
當前提交
9d4b9f995d

+ 167 - 0
apps/impala/src/impala/dashboard_api.py

@@ -0,0 +1,167 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import json
+
+from django.utils.html import escape
+
+from beeswax.server.dbms import get_query_server_config
+from beeswax.design import hql_query
+from beeswax.server import dbms
+from search.models import Collection2
+
+
+LOG = logging.getLogger(__name__)
+
+
+# To Split in Impala, DBMS..
+# To inherit from DashboardApi
+class SQLApi():
+
+  def __init__(self, user):
+    self.user = user
+
+  def query(self, dashboard, query):
+    database, table = self._get_database_table_names(dashboard['name'])
+    filters = []
+
+    fields =  '*'
+    hql = "SELECT %(fields)s FROM `%(database)s`.`%(table)s`" % {
+        'database': database,
+        'table': table,
+        'fields': fields
+    }
+    if filters:
+      hql += ' WHERE ' + filters
+    hql += ' LIMIT 100'
+
+#     sample = get_api(request, {'type': 'hive'}).get_sample_data({'type': 'hive'}, database=file_format['databaseName'], table=file_format['tableName'])
+#     db = dbms.get(request.user)
+#     table_metadata = db.get_table(database=file_format['databaseName'], table_name=file_format['tableName'])
+#
+#     format_ = {
+#         "sample": sample['rows'][:4],
+#         "columns": [
+#             Field(col.name, HiveFormat.FIELD_TYPE_TRANSLATE.get(col.type, 'string')).to_dict()
+#             for col in table_metadata.cols
+#         ]
+#     }
+
+    query_server = get_query_server_config(name='impala') # To move to notebook API
+    db = dbms.get(self.user, query_server=query_server)
+
+    query = hql_query(hql)
+    handle = db.execute_and_wait(query, timeout_sec=35.0)
+
+    if handle:
+      result = db.fetch(handle, rows=100)
+      db.close(handle)
+
+    return self._convert_impala_results(result, dashboard, query)
+
+  def datasets(self):
+    return ['sample_07', 'web_logs']
+
+  def fields(self, dashboard):
+    database, table = self._get_database_table_names(dashboard)
+
+    db = dbms.get(self.user)
+    table_metadata = db.get_table(database=database, table_name=table)
+    return [{
+        'name': str(escape(col.name)),
+        'type': str(col.type),
+        'isId': False, # TODO Kudu
+        'isDynamic': False,
+        'indexed': False,
+        'stored': True
+        # isNested
+      } for col in table_metadata.cols
+    ]
+
+  def schema_fields(self, collection):
+    return {'fields': self.fields(collection)}
+
+  def luke(self, collection):
+    fields = self.schema_fields(collection)
+    return {'fields': Collection2._make_luke_from_schema_fields(fields)}
+
+  def _get_database_table_names(self, name):
+    if '.' in name:
+      database, table_name = name.split('.', 1)
+    else:
+      database = 'default'
+      table_name = name
+
+    return database, table_name
+
+  def _convert_impala_results(self, result, dashboard, query):
+    cols = list(result.cols())
+
+    docs = []
+    for row in result.rows():
+      docs.append(dict((header, cell) for header, cell in zip(cols, row)))
+
+    response = json.loads('''{
+   "highlighting":{
+      "F8V7067-APL-KIT":{
+
+      },
+      "USD":{
+
+      },
+      "NOK":{
+
+      },
+      "GBP":{
+
+      },
+      "EUR":{
+
+      }
+   },
+   "normalized_facets":[
+
+   ],
+   "responseHeader":{
+      "status":0,
+      "QTime":0,
+      "params":{
+         "rows":"5",
+         "hl.fragsize":"1000",
+         "hl.snippets":"5",
+         "doAs":"romain",
+         "q":"*:*",
+         "start":"0",
+         "wt":"json",
+         "user.name":"hue",
+         "hl":"true",
+         "hl.fl":"*",
+         "fl":"*"
+      }
+   },
+   "response":{
+      "start":0,
+      "numFound":32,
+      "docs":[]
+   }
+}''')
+
+    response['response']['docs'] = docs
+    response['response']['numFound'] = len(docs)
+
+    return response

+ 6 - 4
apps/search/src/search/api.py

@@ -48,7 +48,7 @@ def search(request):
 
   if collection:
     try:
-      response = get_engine(request.user).query(collection, query)
+      response = get_engine(request.user, collection).query(collection, query)
     except RestException, e:
       try:
         message = json.loads(e.message)
@@ -98,8 +98,9 @@ def index_fields_dynamic(request):
 
   try:
     name = request.POST['name']
+    engine = request.POST['engine']
 
-    dynamic_fields = SolrApi(SOLR_URL.get(), request.user).luke(name)
+    dynamic_fields = get_engine(request.user, engine).luke(name)
 
     result['message'] = ''
     result['fields'] = [
@@ -126,7 +127,7 @@ def nested_documents(request):
   query = {'qs': [{'q': '_root_:*'}], 'fqs': [], 'start': 0, 'limit': 0}
 
   try:
-    response = SolrApi(SOLR_URL.get(), request.user).query(collection, query)
+    response = get_engine(request.user, collection).query(collection, query)
     result['has_nested_documents'] = response['response']['numFound'] > 0
     result['status'] = 0
   except Exception, e:
@@ -466,8 +467,9 @@ def get_collection(request):
 
   try:
     name = request.POST['name']
+    engine = request.POST['engine']
 
-    collection = Collection2(request.user, name=name)
+    collection = Collection2(request.user, name=name, engine=engine)
     collection_json = collection.get_json(request.user)
 
     result['collection'] = json.loads(collection_json)

+ 27 - 6
apps/search/src/search/api_engines.py

@@ -21,14 +21,19 @@ from libsolr.api import SolrApi
 
 from search.conf import SOLR_URL
 from search.models import augment_solr_response
+from search.search_controller import SearchController
 
 
 LOG = logging.getLogger(__name__)
 
 
-def get_engine(user, name='solr'):
-  if name == 'db':
-    return DBApi(user)
+def get_engine(user, engine='solr'):
+  if isinstance(engine, dict):
+    engine = engine.get('engine', 'solr')
+
+  if engine == 'db':
+    from impala.dashboard_api import SQLApi
+    return SQLApi(user)
   else:
     return SearchApi(user)
 
@@ -38,6 +43,8 @@ class DashboardApi(object):
   def __init__(self, user):
     self.user = user
 
+  def datasets(self): pass
+
   def query(self, collection, query): pass
 
   def suggest(self, collection, query): pass
@@ -59,10 +66,24 @@ class DashboardApi(object):
 
 class SearchApi(DashboardApi):
 
+  def __init__(self, user):
+    DashboardApi.__init__(self, user)
+    self.api = SolrApi(SOLR_URL.get(), self.user)
+
   def query(self, collection, query):
-    response = SolrApi(SOLR_URL.get(), self.user).query(collection, query)
+    response = self.api.query(collection, query)
     return augment_solr_response(response, collection, query)
+  
+  def datasets(self):
+    return SearchController(self.user).get_all_indexes()
+
+  def fields(self, collection):
+    return self.api.fields(collection)
 
+  def schema_fields(self, collection):
+    return self.api.fields(collection)
+  
+  def luke(self, collection):
+    return self.api.luke(collection)
 
-class DBApi(DashboardApi):
-  pass
+  

+ 12 - 11
apps/search/src/search/models.py

@@ -447,7 +447,7 @@ class Collection(models.Model):
 
 class Collection2(object):
 
-  def __init__(self, user, name='Default', data=None, document=None):
+  def __init__(self, user, name='Default', data=None, document=None, engine='solr'):
     self.document = document
 
     if document is not None:
@@ -456,7 +456,7 @@ class Collection2(object):
       self.data = json.loads(data)
     else:
       self.data = {
-          'collection': self.get_default(user, name),
+          'collection': self.get_default(user, name, engine),
           'layout': []
       }
 
@@ -490,6 +490,8 @@ class Collection2(object):
       }
     if 'enabled' not in props['collection']:
       props['collection']['enabled'] = True
+    if 'engine' not in props['collection']:
+      props['collection']['engine'] = 'solr'
     if 'leafletmap' not in props['collection']['template']:
       props['collection']['template']['leafletmap'] = {'latitudeField': None, 'longitudeField': None, 'labelField': None}
     if 'timeFilter' not in props['collection']:
@@ -541,8 +543,8 @@ class Collection2(object):
 
     return json.dumps(props)
 
-  def get_default(self, user, name):
-    fields = self.fields_data(user, name)
+  def get_default(self, user, name, engine='solr'):
+    fields = self.fields_data(user, name, engine)
     id_field = [field['name'] for field in fields if field.get('isId')]
 
     if id_field:
@@ -588,6 +590,7 @@ class Collection2(object):
     return {
       'id': None,
       'name': name,
+      'engine': engine,
       'label': name,
       'enabled': False,
       'template': TEMPLATE,
@@ -628,8 +631,9 @@ class Collection2(object):
   def fields(self, user):
     return sorted([str(field.get('name', '')) for field in self.fields_data(user)])
 
-  def fields_data(self, user, name):
-    api = SolrApi(SOLR_URL.get(), user)
+  def fields_data(self, user, name, engine='solr'):
+    from search.api_engines import get_engine
+    api = get_engine(user, engine)
     try:
       schema_fields = api.fields(name)
       schema_fields = schema_fields['schema']['fields']
@@ -888,11 +892,8 @@ def augment_solr_response(response, collection, query):
           if len(agg_keys) == 1 and agg_keys[0].lower().startswith('dim_'):
             agg_keys.insert(0, 'count')
           counts = _augment_stats_2d(name, facet, counts, selected_values, agg_keys, rows)
-#             _augment_stats_2d(name, facet, counts, selected_values, agg_keys, rows)
-          
-          print counts
-#           count = response['facets'][name]
-#           _convert_nested_to_augmented_pivot_nd(facet_fields, facet['id'], count, selected_values, dimension=2)
+
+          #_convert_nested_to_augmented_pivot_nd(facet_fields, facet['id'], count, selected_values, dimension=2)
           dimension = len(facet_fields)
         elif not collection_facet['properties']['facets'] or (collection_facet['properties']['facets'][0]['aggregate']['function'] != 'count' and len(collection_facet['properties']['facets']) == 1):
           # Dimension 1 with 1 count or agg

+ 8 - 3
apps/search/src/search/static/search/js/search.ko.js

@@ -451,6 +451,7 @@ var Collection = function (vm, collection) {
   self.label = ko.mapping.fromJS(collection.label);
   self.description = ko.observable(typeof collection.description != "undefined" && collection.description != null ? collection.description : "");
   self.suggest = ko.mapping.fromJS(collection.suggest);
+  self.engine = ko.observable(typeof collection.engine != "undefined" && collection.engine != null ? collection.engine : "solr");
   self.nested = ko.mapping.fromJS(collection.nested);
   self.nestedNames = ko.computed(function() {
     function flatten(values) {
@@ -1099,7 +1100,8 @@ var Collection = function (vm, collection) {
 
   self.switchCollection = function() {
     $.post("/search/get_collection", {
-        name: self.name()
+        name: self.name(),
+        engine: self.engine()
     }, function (data) {
       if (data.status == 0) {
         self.idField(data.collection.collection.idField);
@@ -1159,7 +1161,8 @@ var Collection = function (vm, collection) {
 
   self.syncFields = function() {
     $.post("/search/get_collection", {
-        name: self.name()
+        name: self.name(),
+        engine: self.engine()
       }, function (data) {
         if (data.status == 0) {
           self.idField(data.collection.collection.idField);
@@ -1173,7 +1176,8 @@ var Collection = function (vm, collection) {
 
   self.syncDynamicFields = function () {
     $.post("/search/index/fields/dynamic", {
-        name: self.name()
+        name: self.name(),
+        engine: self.engine()
       }, function (data) {
         if (data.status == 0) {
           syncArray(self.template.fieldsAttributes, data.gridlayout_header_fields, true);
@@ -1189,6 +1193,7 @@ var Collection = function (vm, collection) {
   self.getNestedDocuments = function () {
     $.post("/search/index/fields/nested_documents", {
         collection: ko.mapping.toJSON(self),
+        engine: self.engine()
       }, function (data) {
         if (data.status == 0) {
           self.nested.enabled(data.has_nested_documents);

+ 4 - 2
apps/search/src/search/views.py

@@ -28,6 +28,7 @@ from desktop.models import Document2, Document
 
 from indexer.management.commands import indexer_setup
 
+from search.api_engines import get_engine
 from search.conf import LATEST
 from search.data_export import download as export_download
 from search.decorators import allow_owner_only, allow_viewer_only
@@ -88,11 +89,12 @@ def index_embeddable(request):
   return index(request, False, True)
 
 def new_search(request, is_embeddable=False):
-  collections = SearchController(request.user).get_all_indexes()
+  engine = request.GET.get('engine', 'solr')
+  collections = get_engine(request.user, engine).datasets()
   if not collections:
     return no_collections(request)
 
-  collection = Collection2(user=request.user, name=collections[0])
+  collection = Collection2(user=request.user, name=collections[0], engine=engine)
   query = {'qs': [{'q': ''}], 'fqs': [], 'start': 0}
 
   template = 'search.mako'

+ 0 - 2
desktop/libs/indexer/src/indexer/smart_indexer.py

@@ -22,10 +22,8 @@ from collections import deque
 from django.contrib.auth.models import User
 from django.utils.translation import ugettext as _
 from mako.lookup import TemplateLookup
-from mako.template import Template
 
 from desktop.models import Document2
-from notebook.api import _execute_notebook
 from notebook.connectors.base import get_api
 from notebook.models import Notebook