Преглед на файлове

[beeswax] Execute the create table SQL automatically instead of showing it

At the end of the create table wizard, the table is created automatically.
Romain Rigaux преди 13 години
родител
ревизия
c5a540f
променени са 3 файла, в които са добавени 26 реда и са изтрити 23 реда
  1. 6 7
      apps/beeswax/src/beeswax/create_table.py
  2. 15 15
      apps/beeswax/src/beeswax/tests.py
  3. 5 1
      desktop/libs/liboozie/src/liboozie/oozie_api_test.py

+ 6 - 7
apps/beeswax/src/beeswax/create_table.py

@@ -32,7 +32,7 @@ from hadoop.fs import hadoopfs
 
 
 import beeswax.common
 import beeswax.common
 import beeswax.forms
 import beeswax.forms
-from beeswax.views import describe_table, confirm_query, execute_directly
+from beeswax.views import describe_table, execute_directly
 from beeswax.views import make_beeswax_query
 from beeswax.views import make_beeswax_query
 from beeswax import db_utils
 from beeswax import db_utils
 
 
@@ -65,9 +65,8 @@ def create_table(request):
       )
       )
       # Mako outputs bytestring in utf8
       # Mako outputs bytestring in utf8
       proposed_query = proposed_query.decode('utf-8')
       proposed_query = proposed_query.decode('utf-8')
-      tablename = form.table.cleaned_data['name']
-      on_success_url = urlresolvers.reverse(describe_table, kwargs={'table': tablename})
-      return confirm_query(request, proposed_query, on_success_url)
+      table_name = form.table.cleaned_data['name']
+      return _submit_create_and_load(request, proposed_query, table_name, None, False)
   else:
   else:
     form.bind()
     form.bind()
   return render("create_table_manually.mako", request, dict(
   return render("create_table_manually.mako", request, dict(
@@ -82,9 +81,9 @@ def create_table(request):
 IMPORT_PEEK_SIZE = 8192
 IMPORT_PEEK_SIZE = 8192
 IMPORT_PEEK_NLINES = 10
 IMPORT_PEEK_NLINES = 10
 DELIMITERS = [ hive_val for hive_val, desc, ascii in beeswax.common.TERMINATORS ]
 DELIMITERS = [ hive_val for hive_val, desc, ascii in beeswax.common.TERMINATORS ]
-DELIMITER_READABLE = {'\\001' : 'ctrl-As',
-                      '\\002' : 'ctrl-Bs',
-                      '\\003' : 'ctrl-Cs',
+DELIMITER_READABLE = {'\\001' : _('ctrl-As'),
+                      '\\002' : _('ctrl-Bs'),
+                      '\\003' : _('ctrl-Cs'),
                       '\\t'   : _('tabs'),
                       '\\t'   : _('tabs'),
                       ','     : _('commas'),
                       ','     : _('commas'),
                       ' '     : _('spaces')}
                       ' '     : _('spaces')}

+ 15 - 15
apps/beeswax/src/beeswax/tests.py

@@ -848,22 +848,22 @@ for x in sys.stdin:
       'columns-0-_exists': 'True',
       'columns-0-_exists': 'True',
       'columns-next_form_id': '1',
       'columns-next_form_id': '1',
       'partitions-next_form_id': '0',
       'partitions-next_form_id': '0',
-    })
+    }, follow=True)
 
 
-    assert_equal_mod_whitespace(r"""
+    assert_equal_mod_whitespace("""
         CREATE EXTERNAL TABLE `my_table`
         CREATE EXTERNAL TABLE `my_table`
         (
         (
          `my_col` string
          `my_col` string
         )
         )
         COMMENT "Yo>>>>dude"
         COMMENT "Yo>>>>dude"
         ROW FORMAT DELIMITED
         ROW FORMAT DELIMITED
-          FIELDS TERMINATED BY '\001'
-          COLLECTION ITEMS TERMINATED BY '\002'
-          MAP KEYS TERMINATED BY '\003'
+          FIELDS TERMINATED BY '\\001'
+          COLLECTION ITEMS TERMINATED BY '\\002'
+          MAP KEYS TERMINATED BY '\\003'
           STORED AS TextFile LOCATION "/tmp/foo"
           STORED AS TextFile LOCATION "/tmp/foo"
-    """, resp.context["form"].query.initial["query"])
+    """, resp.context['query'].query)
 
 
-    assert_true('/beeswax/table/my_table' in resp.context['on_success_url'])
+    assert_true('on_success_url=%2Fbeeswax%2Ftable%2Fmy_table' in resp.context['fwd_params'])
 
 
 
 
   def test_partitioned_create_table(self):
   def test_partitioned_create_table(self):
@@ -875,7 +875,7 @@ for x in sys.stdin:
     assert_true("Field terminator" in resp.content)
     assert_true("Field terminator" in resp.content)
     # Make a submission
     # Make a submission
     resp = self.client.post("/beeswax/create/create_table", {
     resp = self.client.post("/beeswax/create/create_table", {
-      'table-name': 'my_table',
+      'table-name': 'my_table2',
       'table-row_format': 'Delimited',
       'table-row_format': 'Delimited',
       'table-field_terminator_0': r'\001',
       'table-field_terminator_0': r'\001',
       'table-collection_terminator_0': r'\002',
       'table-collection_terminator_0': r'\002',
@@ -890,10 +890,10 @@ for x in sys.stdin:
       'partitions-0-column_type': 'string',
       'partitions-0-column_type': 'string',
       'partitions-0-_exists': 'True',
       'partitions-0-_exists': 'True',
       'partitions-next_form_id': '1',
       'partitions-next_form_id': '1',
-    })
+    }, follow=True)
 
 
-    assert_equal_mod_whitespace(r"""
-        CREATE TABLE `my_table`
+    assert_equal_mod_whitespace("""
+        CREATE TABLE `my_table2`
         (
         (
          `my_col` string
          `my_col` string
         )
         )
@@ -902,11 +902,11 @@ for x in sys.stdin:
           `my_partition` string
           `my_partition` string
         )
         )
         ROW FORMAT DELIMITED
         ROW FORMAT DELIMITED
-          FIELDS TERMINATED BY '\001'
-          COLLECTION ITEMS TERMINATED BY '\002'
-          MAP KEYS TERMINATED BY '\003'
+          FIELDS TERMINATED BY '\\001'
+          COLLECTION ITEMS TERMINATED BY '\\002'
+          MAP KEYS TERMINATED BY '\\003'
           STORED AS TextFile
           STORED AS TextFile
-    """, resp.context["form"].query.initial["query"])
+    """, resp.context['query'].query)
 
 
 
 
   def test_create_table_dependencies(self):
   def test_create_table_dependencies(self):

+ 5 - 1
desktop/libs/liboozie/src/liboozie/oozie_api_test.py

@@ -53,10 +53,13 @@ class OozieServerProvider(object):
     cls.shutdown = [callback]
     cls.shutdown = [callback]
 
 
   @classmethod
   @classmethod
-  def wait_until_completion(cls, oozie_jobid, timeout=900.0, step=5):
+
+  def wait_until_completion(cls, oozie_jobid, timeout=1800.0, step=5):
     job = cls.oozie.get_job(oozie_jobid)
     job = cls.oozie.get_job(oozie_jobid)
     start = time.time()
     start = time.time()
 
 
+    LOG.info('[%d] cluster status: %s' % (time.time(), cls.cluster.jt.cluster_status()))
+
     while job.is_running() and time.time() - start < timeout:
     while job.is_running() and time.time() - start < timeout:
       time.sleep(step)
       time.sleep(step)
       LOG.info('Checking status of %s...' % oozie_jobid)
       LOG.info('Checking status of %s...' % oozie_jobid)
@@ -71,6 +74,7 @@ class OozieServerProvider(object):
     else:
     else:
       LOG.info('[%d] Job duration %s: %d' % (time.time(), job.id, time.time() - start))
       LOG.info('[%d] Job duration %s: %d' % (time.time(), job.id, time.time() - start))
 
 
+    LOG.info('[%d] cluster status: %s' % (time.time(), cls.cluster.jt.cluster_status()))
     return job
     return job
 
 
   @classmethod
   @classmethod