소스 검색

HUE-1431 [hbase] Truncate the columns in the backend

Kevin Wang 12 년 전
부모
커밋
9ade5ab43e
5개의 변경된 파일14개의 추가작업 그리고 33개의 파일을 삭제
  1. 8 10
      apps/hbase/src/hbase/api.py
  2. 2 2
      apps/hbase/src/hbase/templates/app.mako
  3. 2 15
      apps/hbase/src/hbase/views.py
  4. 2 3
      apps/hbase/static/js/api.js
  5. 0 3
      apps/hbase/static/js/controls.js

+ 8 - 10
apps/hbase/src/hbase/api.py

@@ -143,19 +143,17 @@ class HbaseApi(object):
   def getRowQuerySet(self, cluster, tableName, columns, queries):
     client = self.connectCluster(cluster)
     aggregate_data = []
+    limit = conf.TRUNCATE_LIMIT.get()
     queries = sorted(queries, key=lambda query: query['scan_length']) #sort by scan length
     for query in queries:
       scan_length = int(query['scan_length'])
       if query['row_key'] == "null":
         query['row_key'] = ""
-      filterstring = query.get('filter', None)
-      if filterstring:
-        scan = get_thrift_type('TScan')(startRow=query['row_key'], stopRow=None, timestamp=None, columns=query['columns'] or columns, caching=None, filterString=filterstring, batchSize=None)
-        scanner = client.scannerOpenWithScan(tableName, scan, None)
-        aggregate_data += client.scannerGetList(scanner, query['scan_length'])
-      else:
-        if scan_length == 1:
-          aggregate_data += self.getRows(cluster, tableName, query['columns'] or columns, query['row_key'], query['scan_length'], query['prefix']) #to be replaced in future
-        elif scan_length > 1:
-          aggregate_data += self.getRows(cluster, tableName, query['columns'] or columns, query['row_key'], query['scan_length'], query['prefix'])
+      fs = query.get('filter', None)
+      if fs:
+        fs = " AND (" + fs + ")"
+      filterstring = "(ColumnPaginationFilter(" + str(limit) + ",0) AND PageFilter(" + str(limit) + "))" + (fs or "")
+      scan = get_thrift_type('TScan')(startRow=query['row_key'], stopRow=None, timestamp=None, columns=query['columns'] or columns, caching=None, filterString=filterstring, batchSize=None)
+      scanner = client.scannerOpenWithScan(tableName, scan, None)
+      aggregate_data += client.scannerGetList(scanner, query['scan_length'])
     return aggregate_data

+ 2 - 2
apps/hbase/src/hbase/templates/app.mako

@@ -95,8 +95,8 @@ ${ commonheader(None, "hbase", user) | n,unicode }
     </div>
   </div>
   <br/>
-  <div class="alert alert-error" data-bind="visible: ${datasource}.truncateCount() > 0 && !${datasource}.isLoading()">
-    <b>${_('Notice:')} <i data-bind="text: ${datasource}.truncateCount"></i> ${_('entries truncated')}</b> ${_('at')} <i data-bind="text: ${datasource}.truncateLimit"></i> ${_('items per set.')}
+  <div class="alert alert-error" data-bind="visible: !${datasource}.isLoading()">
+    <b>${_('Notice:')}</b> ${_(' entries may be truncated with limit set at')} <i data-bind="text: ${datasource}.truncateLimit"></i> ${_('items per set.')}
   </div>
   <div class="alert alert-warning" data-bind="visible: !${datasource}.isLoading()">
       ${_('Query executed in')} <i data-bind="text:${datasource}.lastReloadTime() + 's'"></i> ${_('fetching')} <i data-bind="text: ${datasource}.items().length"></i> ${_('rows')}.

+ 2 - 15
apps/hbase/src/hbase/views.py

@@ -61,9 +61,7 @@ def api_router(request, url): #on split, deserialize anything
 
 def api_dump(response):
   ignored_fields = ('thrift_spec', "__.+__")
-  HARD_LIMIT = conf.TRUNCATE_LIMIT.get()
-  truncateCount = [0]
-  truncated = [False]
+  trunc_limit = conf.TRUNCATE_LIMIT.get()
   def clean(data):
     try:
       json.dumps(data)
@@ -71,34 +69,23 @@ def api_dump(response):
     except:
       cleaned = {}
       lim = [0]
-      def step(data):
-        lim[0] += 1
-        if lim[0] > HARD_LIMIT:
-          truncateCount[0] += len(data) - HARD_LIMIT
-          truncated[0] = True
-          return True
-        return False
       if isinstance(data, str): #not JSON dumpable, meaning some sort of bytestring or byte data
         return base64.b64encode(data)
       if hasattr(data, "__iter__"):
         if type(data) is dict:
           for i in data:
             cleaned[i] = clean(data[i])
-            if step(data): break
         elif type(data) is list:
           cleaned = []
           for i, item in enumerate(data):
             cleaned += [clean(item)]
-            if step(data): break
         else:
           for i, item in enumerate(data):
             cleaned[i] = clean(item)
-            if step(data): break
       else:
         for key in dir(data):
           value = getattr(data, key)
           if value is not None and not hasattr(value, '__call__') and sum([int(bool(re.search(ignore, key))) for ignore in ignored_fields]) == 0:
             cleaned[key] = clean(value)
-            if step(dir(data)): break
       return cleaned
-  return HttpResponse(json.dumps({ 'data': clean(response), 'truncated': truncated[0], 'limit': HARD_LIMIT, 'truncate_count': truncateCount[0] }), content_type="application/json")
+  return HttpResponse(json.dumps({ 'data': clean(response), 'truncated': True, 'limit': trunc_limit }), content_type="application/json")

+ 2 - 3
apps/hbase/static/js/api.js

@@ -40,10 +40,9 @@ var API = {
     var doneHandle = handler.done;
     handler.done = function() {
       var cb = arguments[0];
-      return doneHandle.apply(handler, [function(data) {
-        app.views.tabledata.truncated(data.truncated); //change later
+      return doneHandle.apply(handler, [function(data)
+      {
         app.views.tabledata.truncateLimit(data.limit);
-        app.views.tabledata.truncateCount(data.truncate_count);
         data = data.data;
         return cb(data);
       }].concat(Array.prototype.slice.call(arguments).slice(1)));

+ 0 - 3
apps/hbase/static/js/controls.js

@@ -239,7 +239,6 @@ var SmartViewModel = function(options) {
   var _reload = self.reload;
   self.reload = function(callback) {
   	logGA('view_table');
-    self.truncated = ko.observable(false);
     var queryStart = new Date();
     _reload(function() {
       self.lastReloadTime((new Date() - queryStart)/1000);
@@ -249,9 +248,7 @@ var SmartViewModel = function(options) {
     });
   };
 
-  self.truncated = ko.observable(false);
   self.truncateLimit = ko.observable(1500);
-  self.truncateCount = ko.observable(0);
 };
 
 var SmartViewDataRow = function(options) {