Ver código fonte

HUE-2465 [beeswax] Nice to have Natural Sort for database drop-down-list

Jenny Kim 10 anos atrás
pai
commit
a3cbddcb1a

+ 1 - 1
apps/beeswax/src/beeswax/api.py

@@ -34,10 +34,10 @@ from metastore import parser
 
 import beeswax.models
 
-from beeswax.forms import QueryForm
 from beeswax.data_export import upload
 from beeswax.design import HQLdesign
 from beeswax.conf import USE_GET_LOG_API
+from beeswax.forms import QueryForm
 from beeswax.server import dbms
 from beeswax.server.dbms import expand_exception, get_query_server_config, QueryServerException, QueryServerTimeoutException
 from beeswax.views import authorized_get_design, authorized_get_query_history, make_parameterization_form,\

+ 26 - 0
apps/beeswax/src/beeswax/common.py

@@ -20,6 +20,7 @@ Common utils for beeswax.
 """
 
 import re
+import time
 
 from django import forms
 
@@ -50,6 +51,15 @@ TERMINATORS = [
   (' ', "Space", 32),
 ]
 
+def timing(fn):
+  def decorator(*args, **kwargs):
+    time1 = time.time()
+    ret = fn(*args, **kwargs)
+    time2 = time.time()
+    print '%s elapsed time: %0.3f ms' % (fn.func_name, (time2-time1)*1000.0)
+    return ret
+  return decorator
+
 
 def to_choices(x):
   """
@@ -59,6 +69,22 @@ def to_choices(x):
   return [ (y, y) for y in x ]
 
 
+def apply_natural_sort(collection, key=None):
+  """
+  Applies a natural sort (http://rosettacode.org/wiki/Natural_sorting) to a list or dictionary
+  Dictionary types require a sort key to be specified
+  Adapted from http://blog.codinghorror.com/sorting-for-humans-natural-sort-order/
+  """
+  convert = lambda text: int(text) if text.isdigit() else text
+
+  def tokenize_and_convert(item, key=None):
+    if key:
+      item = item[key]
+    return [ convert(c) for c in re.split('([0-9]+)', item) ]
+
+  return sorted(collection, key=lambda i: tokenize_and_convert(i, key=key))
+
+
 class HiveIdentifierField(forms.RegexField):
   """
   Corresponds to 'Identifier' in Hive.g (Hive's grammar)

+ 7 - 0
apps/beeswax/src/beeswax/conf.py

@@ -84,6 +84,13 @@ DOWNLOAD_ROW_LIMIT = Config(
   type=int,
   help=_t('A limit to the number of rows that can be downloaded from a query. A value of -1 means there will be no limit. A maximum of 65,000 is applied to XLS downloads.'))
 
+APPLY_NATURAL_SORT_MAX = Config(
+  key="apply_natural_sort_max",
+  help=_t("The max number of records in the result set permitted to apply a natural sort to the database or tables list."),
+  type=int,
+  default=2000
+)
+
 CLOSE_QUERIES = Config(
   key="close_queries",
   help=_t("Hue will try to close the Hive query when the user leaves the editor page. "

+ 15 - 4
apps/beeswax/src/beeswax/server/dbms.py

@@ -29,7 +29,9 @@ from desktop.lib.parameterization import substitute_variables
 from filebrowser.views import location_to_url
 
 from beeswax import hive_site
-from beeswax.conf import HIVE_SERVER_HOST, HIVE_SERVER_PORT, BROWSE_PARTITIONED_TABLE_LIMIT, SERVER_CONN_TIMEOUT
+from beeswax.conf import HIVE_SERVER_HOST, HIVE_SERVER_PORT, BROWSE_PARTITIONED_TABLE_LIMIT, SERVER_CONN_TIMEOUT, \
+                         APPLY_NATURAL_SORT_MAX
+from beeswax.common import apply_natural_sort
 from beeswax.design import hql_query
 from beeswax.hive_site import hiveserver2_use_ssl
 from beeswax.models import QueryHistory, QUERY_TYPES
@@ -145,7 +147,10 @@ class HiveServer2Dbms(object):
     if handle:
       result = self.fetch(handle, rows=5000)
       self.close(handle)
-      return [name for database in result.rows() for name in database]
+      databases = [name for database in result.rows() for name in database]
+      if len(databases) <= APPLY_NATURAL_SORT_MAX.get():
+        databases = apply_natural_sort(databases)
+      return databases
     else:
       return []
 
@@ -156,7 +161,10 @@ class HiveServer2Dbms(object):
 
   def get_tables_meta(self, database='default', table_names='*'):
     identifier = self.to_matching_wildcard(table_names)
-    return self.client.get_tables_meta(database, identifier)
+    tables = self.client.get_tables_meta(database, identifier)
+    if len(tables) <= APPLY_NATURAL_SORT_MAX.get():
+      tables = apply_natural_sort(tables, key='name')
+    return tables
 
 
   def get_tables(self, database='default', table_names='*'):
@@ -171,7 +179,10 @@ class HiveServer2Dbms(object):
     if handle:
       result = self.fetch(handle, rows=5000)
       self.close(handle)
-      return [name for table in result.rows() for name in table]
+      tables = [name for table in result.rows() for name in table]
+      if len(tables) <= APPLY_NATURAL_SORT_MAX.get():
+        tables = apply_natural_sort(tables)
+      return tables
     else:
       return []
 

+ 15 - 0
apps/beeswax/src/beeswax/tests.py

@@ -59,6 +59,7 @@ import beeswax.models
 import beeswax.views
 
 from beeswax import conf, hive_site
+from beeswax.common import apply_natural_sort
 from beeswax.conf import HIVE_SERVER_HOST
 from beeswax.views import collapse_whitespace, _save_design
 from beeswax.test_base import make_query, wait_for_query_to_finish, verify_history, get_query_server_config,\
@@ -2852,3 +2853,17 @@ def test_to_matching_wildcard():
     assert_equal(match_fn('*'), '*')
     assert_equal(match_fn('test'), '*test*')
     assert_equal(match_fn('test*'), '*test*')
+
+
+def test_apply_natural_sort():
+  test_strings = ['test_1', 'test_100', 'test_2', 'test_200']
+  assert_equal(apply_natural_sort(test_strings), ['test_1', 'test_2', 'test_100', 'test_200'])
+
+  test_dicts = [{'name': 'test_1', 'comment': 'Test'},
+                {'name': 'test_100', 'comment': 'Test'},
+                {'name': 'test_2', 'comment': 'Test'},
+                {'name': 'test_200', 'comment': 'Test'}]
+  assert_equal(apply_natural_sort(test_dicts, key='name'), [{'name': 'test_1', 'comment': 'Test'},
+                                                            {'name': 'test_2', 'comment': 'Test'},
+                                                            {'name': 'test_100', 'comment': 'Test'},
+                                                            {'name': 'test_200', 'comment': 'Test'}])

+ 1 - 1
apps/metastore/src/metastore/conf.py

@@ -24,5 +24,5 @@ HS2_GET_TABLES_MAX = Config(
   key="hs2_get_tables_max",
   help=_("The max number of records in the result set permitted to do a HS2 GetTables call."),
   type=int,
-  default=1000
+  default=250
 )

+ 1 - 1
apps/metastore/src/metastore/views.py

@@ -116,7 +116,6 @@ def get_database_metadata(request, database):
 """
 Table Views
 """
-
 def show_tables(request, database=None):
   if database is None:
     database = request.COOKIES.get('hueBeeswaxLastDatabase', 'default') # Assume always 'default'
@@ -168,6 +167,7 @@ def show_tables(request, database=None):
     'has_write_access': has_write_access(request.user),
   })
   resp.set_cookie("hueBeeswaxLastDatabase", database, expires=90)
+
   return resp