|
@@ -68,6 +68,7 @@ class Command(NoArgsCommand):
|
|
|
import_workflow_root(workflow=workflow, workflow_definition_root=workflow_root, metadata=metadata, fs=self.fs)
|
|
import_workflow_root(workflow=workflow, workflow_definition_root=workflow_root, metadata=metadata, fs=self.fs)
|
|
|
workflow.doc.all().delete() # Delete doc as it messes up the example sharing
|
|
workflow.doc.all().delete() # Delete doc as it messes up the example sharing
|
|
|
|
|
|
|
|
|
|
+
|
|
|
def _import_coordinators(self, directory):
|
|
def _import_coordinators(self, directory):
|
|
|
|
|
|
|
|
for example_directory_name in os.listdir(directory):
|
|
for example_directory_name in os.listdir(directory):
|
|
@@ -87,6 +88,7 @@ class Command(NoArgsCommand):
|
|
|
coordinator.save()
|
|
coordinator.save()
|
|
|
import_coordinator_root(coordinator=coordinator, coordinator_definition_root=coordinator_root, metadata=metadata)
|
|
import_coordinator_root(coordinator=coordinator, coordinator_definition_root=coordinator_root, metadata=metadata)
|
|
|
|
|
|
|
|
|
|
+
|
|
|
def _import_bundles(self, directory):
|
|
def _import_bundles(self, directory):
|
|
|
|
|
|
|
|
for example_directory_name in os.listdir(directory):
|
|
for example_directory_name in os.listdir(directory):
|
|
@@ -107,9 +109,10 @@ class Command(NoArgsCommand):
|
|
|
bundle.save()
|
|
bundle.save()
|
|
|
import_bundle_root(bundle=bundle, bundle_definition_root=bundle_root, metadata=metadata)
|
|
import_bundle_root(bundle=bundle, bundle_definition_root=bundle_root, metadata=metadata)
|
|
|
|
|
|
|
|
|
|
+
|
|
|
def _install_mapreduce_example(self):
|
|
def _install_mapreduce_example(self):
|
|
|
doc2 = None
|
|
doc2 = None
|
|
|
- name = 'MapReduce Sleep Job (example)'
|
|
|
|
|
|
|
+ name = 'MapReduce Sleep Job'
|
|
|
|
|
|
|
|
if Document2.objects.filter(owner=self.user, name=name, type='query-mapreduce').exists():
|
|
if Document2.objects.filter(owner=self.user, name=name, type='query-mapreduce').exists():
|
|
|
LOG.info("Sample mapreduce editor job already installed.")
|
|
LOG.info("Sample mapreduce editor job already installed.")
|
|
@@ -162,12 +165,65 @@ class Command(NoArgsCommand):
|
|
|
|
|
|
|
|
return doc2
|
|
return doc2
|
|
|
|
|
|
|
|
|
|
+ def _install_java_example(self):
|
|
|
|
|
+ doc2 = None
|
|
|
|
|
+ name = 'Java Terasort Job'
|
|
|
|
|
+
|
|
|
|
|
+ if Document2.objects.filter(owner=self.user, name=name, type='query-mapreduce').exists():
|
|
|
|
|
+ LOG.info("Sample mapreduce editor job already installed.")
|
|
|
|
|
+ doc2 = Document2.objects.get(owner=self.user, name=name, type='query-mapreduce')
|
|
|
|
|
+ else:
|
|
|
|
|
+ snippet_properties = {
|
|
|
|
|
+ 'app_jar': '/user/hue/oozie/workspaces/lib/hadoop-examples.jar',
|
|
|
|
|
+ 'class': 'org.apache.hadoop.examples.terasort.TeraSort',
|
|
|
|
|
+ 'args': '${output_dir}/teragen ${output_dir}/terasort',
|
|
|
|
|
+ 'java_opts': '',
|
|
|
|
|
+ 'hadoopProperties': [],
|
|
|
|
|
+ 'archives': [],
|
|
|
|
|
+ 'files': []
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ notebook = make_notebook(
|
|
|
|
|
+ name=name,
|
|
|
|
|
+ description='Terasort: Example Java job',
|
|
|
|
|
+ editor_type='java',
|
|
|
|
|
+ statement='',
|
|
|
|
|
+ status='ready',
|
|
|
|
|
+ snippet_properties=snippet_properties,
|
|
|
|
|
+ is_saved=True
|
|
|
|
|
+ )
|
|
|
|
|
+
|
|
|
|
|
+ # Remove files, functions, settings from snippet properties
|
|
|
|
|
+ data = notebook.get_data()
|
|
|
|
|
+ data['snippets'][0]['properties'].pop('functions')
|
|
|
|
|
+ data['snippets'][0]['properties'].pop('settings')
|
|
|
|
|
+
|
|
|
|
|
+ try:
|
|
|
|
|
+ with transaction.atomic():
|
|
|
|
|
+ doc2 = Document2.objects.create(
|
|
|
|
|
+ owner=self.user,
|
|
|
|
|
+ name=data['name'],
|
|
|
|
|
+ type='query-java',
|
|
|
|
|
+ description=data['description'],
|
|
|
|
|
+ data=json.dumps(data)
|
|
|
|
|
+ )
|
|
|
|
|
+ except Exception, e:
|
|
|
|
|
+ LOG.exception("Failed to create sample Java job document: %s" % e)
|
|
|
|
|
+ # Just to be sure we delete Doc2 object incase of exception.
|
|
|
|
|
+ # Possible when there are mixed InnoDB and MyISAM tables
|
|
|
|
|
+ if doc2 and Document2.objects.filter(id=doc2.id).exists():
|
|
|
|
|
+ doc2.delete()
|
|
|
|
|
+
|
|
|
|
|
+ return doc2
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
def install_examples(self):
|
|
def install_examples(self):
|
|
|
data_dir = LOCAL_SAMPLE_DIR.get()
|
|
data_dir = LOCAL_SAMPLE_DIR.get()
|
|
|
|
|
|
|
|
unmanaged_dir = os.path.join(data_dir, 'unmanaged')
|
|
unmanaged_dir = os.path.join(data_dir, 'unmanaged')
|
|
|
self._import_workflows(unmanaged_dir, managed=False)
|
|
self._import_workflows(unmanaged_dir, managed=False)
|
|
|
|
|
|
|
|
|
|
+
|
|
|
def handle_noargs(self, **options):
|
|
def handle_noargs(self, **options):
|
|
|
self.user = install_sample_user()
|
|
self.user = install_sample_user()
|
|
|
self.fs = cluster.get_hdfs()
|
|
self.fs = cluster.get_hdfs()
|
|
@@ -210,13 +266,12 @@ class Command(NoArgsCommand):
|
|
|
LOG.info("Using Hue 4, will install oozie editor samples.")
|
|
LOG.info("Using Hue 4, will install oozie editor samples.")
|
|
|
|
|
|
|
|
example_jobs = []
|
|
example_jobs = []
|
|
|
- mr_job = self._install_mapreduce_example()
|
|
|
|
|
- if mr_job:
|
|
|
|
|
- example_jobs.append(mr_job)
|
|
|
|
|
|
|
+ example_jobs.append(self._install_mapreduce_example())
|
|
|
|
|
+ example_jobs.append(self._install_java_example())
|
|
|
|
|
|
|
|
# If documents exist but have been trashed, recover from Trash
|
|
# If documents exist but have been trashed, recover from Trash
|
|
|
for doc in example_jobs:
|
|
for doc in example_jobs:
|
|
|
- if doc.parent_directory != examples_dir:
|
|
|
|
|
|
|
+ if doc is not None and doc.parent_directory != examples_dir:
|
|
|
doc.parent_directory = examples_dir
|
|
doc.parent_directory = examples_dir
|
|
|
doc.save()
|
|
doc.save()
|
|
|
|
|
|