Browse Source

HUE-4530 [indexer] Dynamically pickup the column separator for test tables

Romain Rigaux 9 years ago
parent
commit
05992220f2

+ 9 - 3
apps/beeswax/src/beeswax/server/hive_server2_lib.py

@@ -146,6 +146,12 @@ class HiveServerTable(Table):
     end_cols_index = map(itemgetter('data_type'), rows[col_row_index:]).index(None)
     return rows[col_row_index:][:end_cols_index]
 
+  @property
+  def storage_details(self):
+    rows = self.properties
+    col_row_index = map(itemgetter('col_name'), rows).index('Storage Desc Params:') + 1
+    return rows[col_row_index:][:col_row_index + 2]
+
   @property
   def has_complex(self):
     has_complex = False
@@ -168,15 +174,15 @@ class HiveServerTable(Table):
     if self._details is None:
       props = dict([(stat['col_name'], stat['data_type']) for stat in self.properties if stat['col_name'] != 'Table Parameters:'])
       serde = props.get('SerDe Library:', '')
-      
+
       self._details = {
           'stats': dict([(stat['data_type'], stat['comment']) for stat in self.stats]),
           'properties': {
             'owner': props.get('Owner:'),
             'create_time': props.get('CreateTime:'),
             'compressed': props.get('Compressed:', 'No') != 'No',
-            'format': 'parquet' if 'ParquetHiveSerDe' in serde else ('text' if 'LazySimpleSerDe' in serde else serde.rsplit('.', 1)[-1])
-        } 
+            'format': 'parquet' if 'ParquetHiveSerDe' in serde else ('text' if 'LazySimpleSerDe' in serde else serde.rsplit('.', 1)[-1]),
+        }
       }
 
     return self._details

+ 15 - 9
desktop/libs/indexer/src/indexer/api3.py

@@ -22,6 +22,7 @@ from django.utils.translation import ugettext as _
 
 from beeswax.server import dbms
 from desktop.lib.django_util import JsonResponse
+from desktop.lib.exceptions_renderable import PopupException
 from notebook.connectors.base import get_api
 
 from indexer.smart_indexer import Indexer
@@ -34,10 +35,10 @@ LOG = logging.getLogger(__name__)
 
 def _escape_white_space_characters(s, inverse = False):
   MAPPINGS = {
-    "\n":"\\n",
-    "\t":"\\t",
-    "\r":"\\r",
-    " ":"\\s"
+    "\n": "\\n",
+    "\t": "\\t",
+    "\r": "\\r",
+    " ": "\\s"
   }
 
   to = 1 if inverse else 0
@@ -67,9 +68,14 @@ def guess_format(request):
       })
     _convert_format(format_)
   elif file_format['inputFormat'] == 'table':
-    #TODO get table file format
-    # if not CSV?
-    format_ = {"quoteChar": "\"", "recordSeparator": "\\n", "type": "csv", "hasHeader": False, "fieldSeparator": "\t"}
+    db = dbms.get(request.user)
+    table_metadata = db.get_table(database=file_format['databaseName'], table_name=file_format['tableName'])
+
+    storage = dict([(delim['data_type'], delim['comment']) for delim in table_metadata.storage_details])
+    if table_metadata.details['properties']['format'] == 'text':
+      format_ = {"quoteChar": "\"", "recordSeparator": '\\n', "type": "csv", "hasHeader": False, "fieldSeparator": storage['serialization.format']}
+    else:
+      raise PopupException('Hive table format %s is not supported.' % table_metadata.details['properties']['format'])
   elif file_format['inputFormat'] == 'query':
     format_ = {"quoteChar": "\"", "recordSeparator": "\\n", "type": "csv", "hasHeader": False, "fieldSeparator": "\t"} # \t --> CTRL+A
 
@@ -128,8 +134,8 @@ def index_file(request):
 
   if file_format['inputFormat'] == 'table':
     db = dbms.get(request.user)
-    table_metadata = db.get_table(database=file_format['databaseName'], table=file_format['tableName'])
-    input_path = table_metadata.hdfs_link
+    table_metadata = db.get_table(database=file_format['databaseName'], table_name=file_format['tableName'])
+    input_path = table_metadata.path_location
   else:
     input_path = file_format["path"]
 

+ 1 - 1
desktop/libs/indexer/src/indexer/smart_indexer.py

@@ -84,7 +84,7 @@ class Indexer(object):
           zkensemble(),
           u'--collection',
           collection_name,
-          u'${nameNode}%s' % input_path,
+          input_path,
       ],
       u'archives': [],
     }

+ 1 - 1
desktop/libs/indexer/src/indexer/templates/indexer.mako

@@ -168,7 +168,7 @@ ${ assist.assistPanel() }
             3
           <!-- /ko -->
         </div>
-        <div class="caption">${ _('Index it') }</div>
+        <div class="caption">${ _('Index it!') }</div>
       </li>
     </ol>