瀏覽代碼

Usefull scripts (#2761)

These are useful scripts which we use day to day
1. to share workflows,
2. setting up default editors,
3. changing of ownership of docs
4. test the backend with curl command.
5. If oozie is not installed then cleaning up of documents should not fail.

Internal Jira: CDPD-35949
Mahesh Balakrishnan 3 年之前
父節點
當前提交
7613d5caa7

+ 85 - 0
apps/beeswax/src/beeswax/management/commands/set_default_editor.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import sys
+from desktop.models import set_user_preferences
+from django.contrib.auth.models import User
+from django.core.management.base import BaseCommand
+
+if sys.version_info[0] > 2:
+  from django.utils.translation import gettext_lazy as _t, gettext as _
+else:
+  from django.utils.translation import ugettext_lazy as _t, ugettext as _
+
+logging.basicConfig()
+LOG = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+  """
+  Handler for renaming duplicate User objects
+  """
+
+  try:
+    from optparse import make_option
+    option_list = BaseCommand.option_list + (
+      make_option("--hive", help=_t("Set Hive as default."),
+                  action="store_true", default=False, dest='sethive'),
+      make_option("--impala", help=_t("Set Impala as default."),
+                  action="store_true", default=False, dest='setimpala'),
+      make_option("--username", help=_t("User to set."),
+                  action="store", default="all", dest='username'),
+    )
+
+  except AttributeError, e:
+    baseoption_test = 'BaseCommand' in str(e) and 'option_list' in str(e)
+    if baseoption_test:
+      def add_arguments(self, parser):
+        parser.add_argument("--hive", help=_t("Set Hive as default."),
+                            action="store_true", default=False, dest='sethive'),
+        parser.add_argument("--impala", help=_t("Set Impala as default."),
+                            action="store_true", default=False, dest='setimpala'),
+        parser.add_argument("--username", help=_t("User to set."),
+                            action="store", default="all", dest='username'),
+
+    else:
+      LOG.exception(str(e))
+      sys.exit(1)
+
+  def handle(self, *args, **options):
+    key = "default_app"
+    set_props = None
+    if options['sethive']:
+      set_props = '{"app":"editor","interpreter":"hive"}'
+      editor = "hive"
+    if options['setimpala']:
+      set_props = '{"app":"editor","interpreter":"impala"}'
+      editor = "impala"
+    if set_props is None:
+      set_props = '{"app":"editor","interpreter":"impala"}'
+      editor = "impala"
+
+    if options['username'] != "all":
+      LOG.info("Setting default interpreter to %s for user %s" % (editor, options['username']))
+      user = User.objects.get(username=options['username'])
+      set_user_preferences(user, key, set_props)
+
+    else:
+      for user in User.objects.filter():
+        LOG.info("Setting default interpreter to %s for user %s" % (editor, options['username']))
+        set_user_preferences(user, key, set_props)

+ 150 - 0
apps/oozie/src/oozie/management/commands/share_all_workflows.py

@@ -0,0 +1,150 @@
+#!/usr/bin/env python
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging.handlers
+import sys
+from desktop.models import Document2
+from django.contrib.auth.models import User, Group
+from django.core.management.base import BaseCommand
+
+if sys.version_info[0] > 2:
+  from django.utils.translation import gettext_lazy as _t, gettext as _
+else:
+  from django.utils.translation import ugettext_lazy as _t, ugettext as _
+
+LOG = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+  """
+  Handler for sharing workflows
+  """
+
+  try:
+    from optparse import make_option
+    option_list = BaseCommand.option_list + (
+      make_option("--shareusers", help=_t("Comma separated list of users to share all workflows with."),
+                  action="store"),
+      make_option("--sharegroups", help=_t("Comma separated list of groups to share all workflows with."),
+                  action="store"),
+      make_option("--owner", help=_t("Give permissions to only workflows owned by this user."),
+                  action="store"),
+      make_option("--permissions", help=_t("Comma separated list of permissions for the users and groups."
+                                           "read, write or read,write"), action="store"),
+    )
+
+  except AttributeError, e:
+    baseoption_test = 'BaseCommand' in str(e) and 'option_list' in str(e)
+    if baseoption_test:
+      def add_arguments(self, parser):
+        parser.add_argument("--shareusers", help=_t("Comma separated list of users to share all workflows with."),
+                            action="store"),
+        parser.add_argument("--sharegroups", help=_t("Comma separated list of groups to share all workflows with."),
+                            action="store"),
+        parser.add_argument("--owner", help=_t("Give permissions to only workflows owned by this user."),
+                            action="store"),
+        parser.add_argument("--permissions", help=_t("Comma separated list of permissions for the users and groups."
+                                                     "read, write or read,write"), action="store")
+    else:
+      LOG.exception(str(e))
+      sys.exit(1)
+
+  def handle(self, *args, **options):
+
+    if not options['shareusers'] and not options['sharegroups']:
+      LOG.warn("You must set either shareusers or sharegroups or both")
+      sys.exit(1)
+
+    if not options['permissions']:
+      LOG.warn("permissions option required either read, write or read,write")
+      sys.exit(1)
+
+    if options['shareusers']:
+      users = options['shareusers'].split(",")
+    else:
+      users = []
+
+    if options['sharegroups']:
+      groups = options['sharegroups'].split(",")
+    else:
+      groups = []
+
+    perms = options['permissions'].split(",")
+
+    LOG.info("Setting permissions %s on all workflows for users: %s" % (perms, users))
+    LOG.info("Setting permissions %s on all workflows for groups: %s" % (perms, groups))
+
+    shareusers = User.objects.filter(username__in=users)
+    sharegroups = Group.objects.filter(name__in=groups)
+
+    doc_types = ['oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2']
+    workflow_owner = User.objects.get(username=options['owner'])
+
+    if options['owner']:
+      LOG.info("Only setting permissions for workflows owned by %s" % options['owner'])
+      oozie_docs = Document2.objects.filter(type__in=doc_types, owner=workflow_owner)
+    else:
+      oozie_docs = Document2.objects.filter(type__in=doc_types)
+
+    for perm in perms:
+      if perm in ['read', 'write']:
+        for oozie_doc in oozie_docs:
+          owner = User.objects.get(id=oozie_doc.owner_id)
+          read_perms = oozie_doc.to_dict()['perms']['read']
+          write_perms = oozie_doc.to_dict()['perms']['write']
+
+          read_users = []
+          write_users = []
+          read_groups = []
+          write_groups = []
+
+          for user in read_perms['users']:
+            read_users.append(user['id'])
+
+          for group in read_perms['groups']:
+            read_groups.append(group['id'])
+
+          for user in write_perms['users']:
+            write_users.append(user['id'])
+
+          for group in write_perms['groups']:
+            write_groups.append(group['id'])
+
+          for user in shareusers:
+            if perm == 'read':
+              read_users.append(user.id)
+
+            if perm == 'write':
+              write_users.append(user.id)
+
+          for group in sharegroups:
+            if perm == 'read':
+              read_groups.append(group.id)
+
+            if perm == 'write':
+              write_groups.append(group.id)
+
+          if perm == 'read':
+            users = User.objects.in_bulk(read_users)
+            groups = Group.objects.in_bulk(read_groups)
+
+          if perm == 'write':
+            users = User.objects.in_bulk(write_users)
+            groups = Group.objects.in_bulk(write_groups)
+
+          LOG.warn("Setting %s on %s for users: %s : groups: %s" % (perm, oozie_doc.name, users, groups))
+          oozie_doc.share(owner, name=perm, users=users, groups=groups)

+ 225 - 0
desktop/core/src/desktop/cm_environment.py

@@ -0,0 +1,225 @@
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+import os.path
+import re
+import subprocess
+import sys
+
+from hue_shared import which
+
+LOG = logging.getLogger(__name__)
+
+
+"""
+Class to configure Hue environment from CM
+"""
+
+
+def set_cm_environment():
+  """
+  Collect environment from CM supervisor
+  """
+  hue_config = {}
+  hue_bin_dir = "/usr/lib/hue"
+  hue_path = "/usr/lib/hue"
+  parcel_name = "CDH"
+  parcel_dir = "/opt/cloudera/parcels"
+  dbengine = None
+  cm_agent_process = subprocess.Popen('ps -ef | grep "[c]m agent\|[c]mf-agent" | awk \'{print $2}\'', shell=True,
+                                      stdout=subprocess.PIPE)
+  cm_agent_pid = cm_agent_process.communicate()[0].split('\n')[0]
+  if cm_agent_pid != '':
+    try:
+      supervisor_process = subprocess.Popen('ps -ef | grep [s]upervisord | awk \'{print $2}\'', shell=True,
+                                            stdout=subprocess.PIPE)
+      supervisor_pid = supervisor_process.communicate()[0].split('\n')[0]
+      cm_supervisor_dir = os.path.realpath('/proc/%s/cwd' % supervisor_pid)
+      if supervisor_pid == '':
+        LOG.exception("This appears to be a CM enabled cluster and supervisord is not running")
+        LOG.exception("Make sure you are running as root and CM supervisord is running")
+        sys.exit(1)
+    except Exception, e:
+      LOG.exception("This appears to be a CM enabled cluster and supervisord is not running")
+      LOG.exception("Make sure you are running as root and CM supervisord is running")
+      sys.exit(1)
+
+    # Parse CM supervisor include file for Hue and set env vars
+    cm_supervisor_dir = cm_supervisor_dir + '/include'
+    cm_agent_run_dir = os.path.dirname(cm_supervisor_dir)
+    hue_env_conf = None
+    envline = None
+    cm_hue_string = "HUE_SERVER"
+
+    for file in os.listdir(cm_supervisor_dir):
+      if cm_hue_string in file:
+        hue_env_conf = file
+
+    if not hue_env_conf == None:
+      if os.path.isfile(cm_supervisor_dir + "/" + hue_env_conf):
+        hue_env_conf_file = open(cm_supervisor_dir + "/" + hue_env_conf, "r")
+        LOG.info("Setting CM managed environment using supervisor include: %s" % hue_env_conf_file)
+        for line in hue_env_conf_file:
+          if "environment" in line:
+            envline = line
+          if "directory" in line:
+            empty, hue_conf_dir = line.split("directory=")
+            os.environ["HUE_CONF_DIR"] = hue_conf_dir.rstrip()
+            sys.path.append(os.environ["HUE_CONF_DIR"])
+
+    if not envline == None:
+      empty, environment = envline.split("environment=")
+      for envvar in environment.split(","):
+        if "HADOOP_C" in envvar or "PARCEL" in envvar or "DESKTOP" in envvar or "ORACLE" in envvar or "LIBRARY" in \
+            envvar or "CMF" in envvar:
+          envkey, envval = envvar.split("=")
+          envval = envval.replace("'", "").rstrip()
+          if "LIBRARY" not in envkey:
+            os.environ[envkey] = envval
+          elif "LD_LIBRARY_PATH" not in os.environ.keys():
+            os.environ[envkey] = envval
+
+    if "PARCELS_ROOT" in os.environ:
+      parcel_dir = os.environ["PARCELS_ROOT"]
+
+    if "PARCEL_DIRNAMES" in os.environ:
+      parcel_names = os.environ["PARCEL_DIRNAMES"].split(':')
+      for parcel_name_temp in parcel_names:
+        if parcel_name_temp.startswith("CDH"):
+          parcel_name = parcel_name_temp
+
+    if os.path.isdir("%s/%s/lib/hue" % (parcel_dir, parcel_name)):
+      hue_path = "%s/%s/lib/hue" % (parcel_dir, parcel_name)
+    hue_bin_dir = "%s/build/env/bin" % hue_path
+
+    cloudera_config_script = None
+    if os.path.isfile('/usr/lib64/cmf/service/common/cloudera-config.sh'):
+      cloudera_config_script = '/usr/lib64/cmf/service/common/cloudera-config.sh'
+    elif os.path.isfile('/opt/cloudera/cm-agent/service/common/cloudera-config.sh'):
+      cloudera_config_script = '/opt/cloudera/cm-agent/service/common/cloudera-config.sh'
+
+    JAVA_HOME = None
+    if cloudera_config_script is not None:
+      locate_java = subprocess.Popen(['bash', '-c', '. %s; locate_java_home' % cloudera_config_script],
+                                     stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+      for line in iter(locate_java.stdout.readline, ''):
+        if 'JAVA_HOME' in line:
+          JAVA_HOME = line.rstrip().split('=')[1]
+
+    if JAVA_HOME is not None:
+      os.environ["JAVA_HOME"] = JAVA_HOME
+
+    if "JAVA_HOME" not in os.environ:
+      print "JAVA_HOME must be set and can't be found, please set JAVA_HOME environment variable"
+      sys.exit(1)
+
+    hue_config["LD_LIBRARY_PATH"] = None
+    for line in open(os.environ["HUE_CONF_DIR"] + "/hue_safety_valve_server.ini"):
+      if re.search("engine=", line):
+        dbengine = line
+    if dbengine is None:
+      for line in open(os.environ["HUE_CONF_DIR"] + "/hue_safety_valve.ini"):
+        if re.search("engine=", line):
+          dbengine = line
+    if dbengine is None:
+      for line in open(os.environ["HUE_CONF_DIR"] + "/hue.ini"):
+        if re.search("engine=", line):
+          dbengine = line
+
+    if dbengine is not None and "oracle" in dbengine.lower():
+      # Make sure we set Oracle Client if configured
+      if "LD_LIBRARY_PATH" not in os.environ.keys():
+        if "SCM_DEFINES_SCRIPTS" in os.environ.keys():
+          for scm_script in os.environ["SCM_DEFINES_SCRIPTS"].split(":"):
+            if "ORACLE_INSTANT_CLIENT" in scm_script:
+              if os.path.isfile(scm_script):
+                oracle_source = subprocess.Popen(". %s; env" % scm_script, stdout=subprocess.PIPE, shell=True,
+                                                 executable="/bin/bash")
+                for line in oracle_source.communicate()[0].splitlines():
+                  if "LD_LIBRARY_PATH" in line:
+                    var, oracle_ld_path = line.split("=")
+                    os.environ["LD_LIBRARY_PATH"] = oracle_ld_path
+
+      if "LD_LIBRARY_PATH" not in os.environ.keys() or not os.path.isfile(
+              "%s/libclntsh.so.11.1" % os.environ["LD_LIBRARY_PATH"]):
+        print "You are using Oracle for backend DB"
+        if "LD_LIBRARY_PATH" in os.environ.keys():
+          print "LD_LIBRARY_PATH set to %s" % os.environ["LD_LIBRARY_PATH"]
+          print "LD_LIBRARY_PATH does not contain libclntsh.so.11.1"
+          print "Please set LD_LIBRARY_PATH correctly and rerun"
+
+        else:
+          print "LD_LIBRARY_PATH can't be found, if you are using ORACLE for your Hue database"
+          print "then it must be set, if not, you can ignore"
+
+        print "Here is an exmple, ONLY INCLUDE ONE PATH and NO VARIABLES"
+        print "  export LD_LIBRARY_PATH=/path/to/instantclient"
+        sys.exit(1)
+
+  else:
+    print "CM does not appear to be running on this server"
+    print "If this is a CM managed cluster make sure the agent and supervisor are running"
+    print "Running with /etc/hue/conf as the HUE_CONF_DIR"
+    os.environ["HUE_CONF_DIR"] = "/etc/hue/conf"
+
+  hue_config['hue_path'] = hue_path
+  hue_config['hue_bin_dir'] = hue_bin_dir
+  hue_config['HUE_CONF_DIR'] = os.environ["HUE_CONF_DIR"]
+  hue_config['parcel_name'] = parcel_name
+  hue_config['parcel_dir'] = parcel_dir
+  if dbengine is not None and "oracle" in dbengine.lower():
+    hue_config['LD_LIBRARY_PATH'] = os.environ["LD_LIBRARY_PATH"]
+
+  return hue_config
+
+
+def reload_with_cm_env():
+  try:
+    from django.db.backends.oracle.base import Oracle_datetime
+  except:
+    os.environ["SKIP_RELOAD"] = "True"
+    if 'LD_LIBRARY_PATH' in os.environ:
+      LOG.info("We need to reload the process to include any LD_LIBRARY_PATH changes")
+      try:
+        os.execv(sys.argv[0], sys.argv)
+      except Exception, exc:
+        LOG.warn('Failed re-exec:', exc)
+        sys.exit(1)
+
+
+def check_security():
+  from hadoop import conf
+  hdfs_config = conf.HDFS_CLUSTERS['default']
+  security_enabled = False
+  if hdfs_config.SECURITY_ENABLED.get():
+    KLIST = which('klist')
+    if KLIST is None:
+      LOG.exception("klist is required, please install and rerun")
+      sys.exit(1)
+    klist_cmd = '%s | grep "Default principal"' % KLIST
+    klist_check = subprocess.Popen(klist_cmd, shell=True, stdout=subprocess.PIPE)
+    klist_princ = klist_check.communicate()[0].split(': ')[1]
+    if not 'hue/' in klist_princ:
+      LOG.exception("klist failed, please contact support: %s" % klist_princ)
+      sys.exit(1)
+    LOG.info("Security enabled using principal: %s" % klist_princ)
+    LOG.info("You can imitate by running following export:")
+    LOG.info("OSRUN: export KRB5CCNAME=%s" % os.environ['KRB5CCNAME'])
+    security_enabled = True
+
+  return security_enabled

+ 75 - 0
desktop/core/src/desktop/hue_curl.py

@@ -0,0 +1,75 @@
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+import logging
+import datetime
+import time
+import subprocess
+
+from cm_environment import check_security
+from hue_shared import which
+
+
+LOG = logging.getLogger(__name__)
+
+class Curl(object):
+
+  def __init__(self, verbose=False):
+    self.curl = which('curl')
+    if self.curl is None:
+      LOG.exception("curl is required, please install and rerun")
+      sys.exit(1)
+
+    # We will change to handle certs later
+    self.basecmd = self.curl + ' -k'
+    LOG.info("Checking security status")
+    self.security_enabled = check_security()
+    self.verbose = verbose
+
+    if self.security_enabled:
+      self.basecmd = self.basecmd + ' --negotiate -u :'
+
+    if self.verbose:
+      self.basecmd = self.basecmd + ' -v'
+    else:
+      self.basecmd = self.basecmd + ' -s'
+
+  def do_curl(self, url, method='GET', follow=False, args=None):
+
+    cmd = self.basecmd + ' -X ' + method
+    if follow:
+      cmd = cmd + ' -L'
+
+    if args is not None:
+      cmd = cmd + ' ' + args
+
+    cmd = cmd + ' \'' + url + '\''
+    LOG.info("OSRUN: %s" % cmd)
+    curl_process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
+    curl_response = curl_process.communicate()[0]
+    curl_ret = curl_process.returncode
+    if curl_ret > 0:
+      LOG.exception("Curl failed to run succesfully: %s" % curl_response)
+    return curl_response
+
+
+  def do_curl_available_services(self, service_test):
+    url = service_test['url']
+    method = service_test['method']
+    response = self.do_curl(url, method=method)
+    return response

+ 24 - 0
desktop/core/src/desktop/hue_shared.py

@@ -0,0 +1,24 @@
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+def which(file_name):
+  for path in os.environ["PATH"].split(os.pathsep):
+    full_path = os.path.join(path, file_name)
+    if os.path.exists(full_path) and os.access(full_path, os.X_OK):
+      return full_path
+  return None

+ 79 - 0
desktop/core/src/desktop/management/commands/change_owner_of_docs.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python
+
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import sys
+from desktop.models import Document2
+from django.contrib.auth.models import User
+from django.core.management.base import BaseCommand
+
+if sys.version_info[0] > 2:
+  from django.utils.translation import gettext_lazy as _t, gettext as _
+else:
+  from django.utils.translation import ugettext_lazy as _t, ugettext as _
+
+LOG = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+  """
+  Handler for changing ownership of docs
+  """
+
+  try:
+    from optparse import make_option
+    option_list = BaseCommand.option_list + (
+      make_option("--olduser", help=_t("User who's docs need to change ownership. "),
+                  action="store"),
+      make_option("--newuser", help=_t("User who will own the docs. "),
+                  action="store"),
+    )
+
+  except AttributeError, e:
+    baseoption_test = 'BaseCommand' in str(e) and 'option_list' in str(e)
+    if baseoption_test:
+      def add_arguments(self, parser):
+        parser.add_argument("--olduser", help=_t("User who's docs need to change ownership. "),
+                            action="store"),
+        parser.add_argument("--newuser", help=_t("User who will own the docs. "),
+                            action="store")
+
+    else:
+      LOG.exception(str(e))
+      sys.exit(1)
+
+  def handle(self, *args, **options):
+    LOG.warn("Changing ownership of all docs owned by %s to %s" % (options['olduser'], options['newuser']))
+
+    if not options['olduser']:
+      LOG.exception("--olduser option required")
+      sys.exit(1)
+
+    if not options['newuser']:
+      LOG.exception("--newuser option required")
+      sys.exit(1)
+
+    try:
+      newuser = User.objects.get(username=options['newuser'])
+      olduser = User.objects.get(username=options['olduser'])
+      docs = Document2.objects.filter(owner=olduser)
+      Document2.objects.filter(owner=olduser).update(owner=newuser)
+
+    except Exception as e:
+      LOG.warn(
+        "EXCEPTION: Changing ownership of %s's docs to %s failed: %s" % (options['olduser'], options['newuser'], e))

+ 120 - 119
desktop/core/src/desktop/management/commands/desktop_document_cleanup.py

@@ -1,4 +1,3 @@
-
 # adapted from django-extensions (http://code.google.com/p/django-command-extensions/)
 # Licensed to Cloudera, Inc. under one
 # or more contributor license agreements.  See the NOTICE file
@@ -16,25 +15,21 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import desktop.conf
+import desktop.conf
+import logging.handlers
 import os
 import sys
 import time
-
-from importlib import import_module
-
-from django.conf import settings
-from django.core.management.base import BaseCommand, CommandError
 from beeswax.models import SavedQuery
 from beeswax.models import Session
 from datetime import date, timedelta
-from oozie.models import Workflow
-from django.db.utils import DatabaseError
-import desktop.conf
 from desktop.models import Document2
-import logging
-import logging.handlers
-
-import desktop.conf
+from django.conf import settings
+from django.core.management.base import BaseCommand
+from django.db.utils import DatabaseError
+from importlib import import_module
+from oozie.models import Workflow
 
 if sys.version_info[0] > 2:
   from django.utils.translation import gettext_lazy as _t, gettext as _
@@ -45,114 +40,120 @@ LOG = logging.getLogger(__name__)
 
 
 class Command(BaseCommand):
-    """
-    Handler for purging old Query History, Workflow documents and Session data
-    """
+  """
+  Handler for purging old Query History, Workflow documents and Session data
+  """
+
+  try:
+    from optparse import make_option
+    option_list = BaseCommand.option_list + (
+      make_option("--keep-days", help=_t("Number of days of history data to keep."),
+                  action="store",
+                  type=int,
+                  default=30),
+    )
+
+  except AttributeError as e:
+    baseoption_test = 'BaseCommand' in str(e) and 'option_list' in str(e)
+    if baseoption_test:
+      def add_arguments(self, parser):
+        parser.add_argument("--keep-days", help=_t("Number of days of history data to keep."),
+                            action="store",
+                            type=int,
+                            default=30)
+    else:
+      LOG.exception(str(e))
+      sys.exit(1)
+
+  def objectCleanup(self, objClass, filterType, filterValue, dateField):
+    errorCount = 0
+    checkCount = 0
+    resets = 0
+    deleteRecords = self.deleteRecordsBase
+
+    totalObjects = objClass.objects.filter(
+      **{'%s' % filterType: filterValue, '%s__lte' % dateField: self.timeDeltaObj,}) \
+      .values_list("id", flat=True)
+    LOG.info("Looping through %s objects. %s objects to be deleted." % (objClass.__name__, totalObjects.count()))
+    while totalObjects.count():
+      if deleteRecords < 30 and resets < self.resetMax:
+        checkCount += 1
+      if checkCount == self.resetCount:
+        deleteRecords = self.deleteRecordsBase
+        resets += 1
+        checkCount = 0
+      LOG.info("%s objects left: %s" % (objClass.__name__, totalObjects.count()))
+      deleteObjects = objClass.objects.filter(
+        **{'%s' % filterType: filterValue, '%s__lte' % dateField: self.timeDeltaObj,}) \
+                        .values_list("id", flat=True)[:deleteRecords]
+      try:
+        objClass.objects.filter(pk__in=list(deleteObjects)).delete()
+        errorCount = 0
+      except DatabaseError as e:
+        LOG.info("Non Fatal Exception: %s: %s" % (e.__class__.__name__, e))
+        errorCount += 1
+        if errorCount > 9 and deleteRecords == 1:
+          raise
+        if deleteRecords > 100:
+          deleteRecords = max(deleteRecords - 100, 1)
+        else:
+          deleteRecords = max(deleteRecords - 10, 1)
+        LOG.info("Decreasing max delete records to: %s" % deleteRecords)
+      totalObjects = objClass.objects.filter(
+        **{'%s' % filterType: filterValue, '%s__lte' % dateField: self.timeDeltaObj,}) \
+        .values_list("id", flat=True)
+
+  def handle(self, *args, **options):
+
+    self.keepDays = options['keep_days']
+    self.timeDeltaObj = date.today() - timedelta(days=self.keepDays)
+    self.resetCount = 15
+    self.resetMax = 5
+    self.deleteRecordsBase = 999  # number of documents to delete in a batch
+    # to avoid Non Fatal Exception: DatabaseError: too many SQL variables
+
+    LOG.warning("HUE_CONF_DIR: %s" % os.environ['HUE_CONF_DIR'])
+    LOG.info("DB Engine: %s" % desktop.conf.DATABASE.ENGINE.get())
+    LOG.info("DB Name: %s" % desktop.conf.DATABASE.NAME.get())
+    LOG.info("DB User: %s" % desktop.conf.DATABASE.USER.get())
+    LOG.info("DB Host: %s" % desktop.conf.DATABASE.HOST.get())
+    LOG.info("DB Port: %s" % str(desktop.conf.DATABASE.PORT.get()))
+    LOG.info(
+      "Cleaning up anything in the Hue tables django_session, oozie*, desktop* and beeswax* older than %s old" % self.keepDays)
+
+    start = time.time()
+
+    # Clean out Hive / Impala Query History
+    self.objectCleanup(SavedQuery, 'is_auto', True, 'mtime')
+
+    # Clear out old Hive/Impala sessions
+    self.objectCleanup(Session, 'status_code__gte', -10000, 'last_used')
+
+    # Clean out Trashed Workflows
+    try:
+      self.objectCleanup(Workflow, 'is_trashed', True, 'last_modified')
+    except NameError as NE:
+      LOG.info('Oozie app is not configured to clean out trashed workflows')
 
+    # Clean out Workflows without a name
     try:
-        from optparse import make_option
-        option_list = BaseCommand.option_list + (
-            make_option("--keep-days", help=_t("Number of days of history data to keep."),
-                    action="store",
-                    type=int,
-                    default=30),
-        )
-
-    except AttributeError as e:
-        baseoption_test = 'BaseCommand' in str(e) and 'option_list' in str(e)
-        if baseoption_test:
-            def add_arguments(self, parser):
-                parser.add_argument("--keep-days", help=_t("Number of days of history data to keep."),
-                    action="store",
-                    type=int,
-                    default=30)
-        else:
-            LOG.exception(str(e))
-            sys.exit(1)
+      self.objectCleanup(Workflow, 'name', '', 'last_modified')
+    except NameError as NE:
+      LOG.info('Oozie app is not configured to clean out workflows without a name')
 
+    # Clean out history Doc2 objects
+    self.objectCleanup(Document2, 'is_history', True, 'last_modified')
 
-    def objectCleanup(self, objClass, filterType, filterValue, dateField):
-        errorCount = 0
-        checkCount = 0
-        resets = 0
-        deleteRecords = self.deleteRecordsBase
+    # Clean out expired sessions
+    LOG.debug("Cleaning out expired sessions from django_session table")
 
-        totalObjects = objClass.objects.filter(**{ '%s' % filterType: filterValue, '%s__lte' % dateField: self.timeDeltaObj, })\
-                                                .values_list("id", flat=True)
-        LOG.info("Looping through %s objects. %s objects to be deleted." % (objClass.__name__, totalObjects.count()))
-        while totalObjects.count():
-            if deleteRecords < 30 and resets < self.resetMax:
-                checkCount += 1
-            if checkCount == self.resetCount:
-                deleteRecords = self.deleteRecordsBase
-                resets += 1
-                checkCount = 0
-            LOG.info("%s objects left: %s" % (objClass.__name__, totalObjects.count()))
-            deleteObjects = objClass.objects.filter(**{ '%s' % filterType: filterValue, '%s__lte' % dateField: self.timeDeltaObj, })\
-                                                    .values_list("id", flat=True)[:deleteRecords]
-            try:
-                objClass.objects.filter(pk__in=list(deleteObjects)).delete()
-                errorCount = 0
-            except DatabaseError as e:
-                LOG.info("Non Fatal Exception: %s: %s" % (e.__class__.__name__, e))
-                errorCount += 1
-                if errorCount > 9 and deleteRecords == 1:
-                    raise
-                if deleteRecords > 100:
-                    deleteRecords = max(deleteRecords - 100, 1)
-                else:
-                    deleteRecords = max(deleteRecords - 10, 1)
-                LOG.info("Decreasing max delete records to: %s" % deleteRecords)
-            totalObjects = objClass.objects.filter(**{'%s' % filterType: filterValue, '%s__lte' % dateField: self.timeDeltaObj, })\
-                                                    .values_list("id", flat=True)
-
-
-    def handle(self, *args, **options):
-
-
-        self.keepDays = options['keep_days']
-        self.timeDeltaObj = date.today() - timedelta(days=self.keepDays)
-        self.resetCount = 15
-        self.resetMax = 5
-        self.deleteRecordsBase = 999  #number of documents to delete in a batch
-                                      #to avoid Non Fatal Exception: DatabaseError: too many SQL variables
-
-        LOG.warning("HUE_CONF_DIR: %s" % os.environ['HUE_CONF_DIR'])
-        LOG.info("DB Engine: %s" % desktop.conf.DATABASE.ENGINE.get())
-        LOG.info("DB Name: %s" % desktop.conf.DATABASE.NAME.get())
-        LOG.info("DB User: %s" % desktop.conf.DATABASE.USER.get())
-        LOG.info("DB Host: %s" % desktop.conf.DATABASE.HOST.get())
-        LOG.info("DB Port: %s" % str(desktop.conf.DATABASE.PORT.get()))
-        LOG.info("Cleaning up anything in the Hue tables django_session, oozie*, desktop* and beeswax* older than %s old" % self.keepDays)
-
-        start = time.time()
-
-        #Clean out Hive / Impala Query History
-        self.objectCleanup(SavedQuery, 'is_auto', True, 'mtime')
-
-        #Clear out old Hive/Impala sessions
-        self.objectCleanup(Session, 'status_code__gte', -10000, 'last_used')
-
-        #Clean out Trashed Workflows
-        self.objectCleanup(Workflow, 'is_trashed', True, 'last_modified')
-
-        #Clean out Workflows without a name
-        self.objectCleanup(Workflow, 'name', '', 'last_modified')
-
-        #Clean out history Doc2 objects
-        self.objectCleanup(Document2, 'is_history', True, 'last_modified')
-
-        #Clean out expired sessions
-        LOG.debug("Cleaning out expired sessions from django_session table")
-
-        engine = import_module(settings.SESSION_ENGINE)
-        try:
-            engine.SessionStore.clear_expired()
-        except NotImplementedError:
-            LOG.error("Session engine '%s' doesn't support clearing "
-                            "expired sessions.\n" % settings.SESSION_ENGINE)
-
-
-        end = time.time()
-        elapsed = (end - start)
-        LOG.debug("Total time elapsed (seconds): %.2f" % elapsed)
+    engine = import_module(settings.SESSION_ENGINE)
+    try:
+      engine.SessionStore.clear_expired()
+    except NotImplementedError:
+      LOG.error("Session engine '%s' doesn't support clearing "
+                "expired sessions.\n" % settings.SESSION_ENGINE)
+
+    end = time.time()
+    elapsed = (end - start)
+    LOG.debug("Total time elapsed (seconds): %.2f" % elapsed)

+ 321 - 0
desktop/core/src/desktop/management/commands/get_backend_curl.py

@@ -0,0 +1,321 @@
+#!/usr/bin/env python
+
+# Licensed to Cloudera, Inc. under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  Cloudera, Inc. licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+import subprocess
+import sys
+import time
+from desktop.conf import TIME_ZONE
+from django.core.management.base import BaseCommand
+from hadoop import cluster
+from hadoop import conf as hdfs_conf
+from desktop.hue_curl import Curl
+from liboozie.conf import OOZIE_URL, SECURITY_ENABLED as OOZIE_SECURITY_ENABLED
+from search.conf import SOLR_URL, SECURITY_ENABLED as SOLR_SECURITY_ENABLED
+
+if sys.version_info[0] > 2:
+  from django.utils.translation import gettext_lazy as _t, gettext as _
+else:
+  from django.utils.translation import ugettext_lazy as _t, ugettext as _
+
+LOG = logging.getLogger(__name__)
+
+DEFAULT_LOG_DIR = 'logs'
+log_dir = os.getenv("DESKTOP_LOG_DIR", DEFAULT_LOG_DIR)
+
+current_milli_time = lambda: int(round(time.time() * 1000))
+
+
+def get_service_info(service):
+  service_info = {}
+  if service.lower() == 'solr':
+    service_info['url'] = SOLR_URL.get()
+    service_info['security_enabled'] = SOLR_SECURITY_ENABLED.get()
+  if service.lower() == 'oozie':
+    service_info['url'] = OOZIE_URL.get()
+    service_info['security_enabled'] = OOZIE_SECURITY_ENABLED.get()
+  if service.lower() == 'httpfs':
+    hdfs_config = hdfs_conf.HDFS_CLUSTERS['default']
+    service_info['url'] = hdfs_config.WEBHDFS_URL.get()
+    service_info['security_enabled'] = hdfs_config.SECURITY_ENABLED.get()
+  if service.lower() == 'rm':
+    yarn_cluster = cluster.get_cluster_conf_for_job_submission()
+    service_info['url'] = yarn_cluster.RESOURCE_MANAGER_API_URL.get()
+    service_info['security_enabled'] = yarn_cluster.SECURITY_ENABLED.get()
+  if service.lower() == 'jhs':
+    yarn_cluster = cluster.get_cluster_conf_for_job_submission()
+    service_info['url'] = yarn_cluster.HISTORY_SERVER_API_URL.get()
+    service_info['security_enabled'] = yarn_cluster.SECURITY_ENABLED.get()
+  if service.lower() == 'sparkhs':
+    yarn_cluster = cluster.get_cluster_conf_for_job_submission()
+    service_info['url'] = yarn_cluster.SPARK_HISTORY_SERVER_URL.get()
+    service_info['security_enabled'] = yarn_cluster.SPARK_HISTORY_SERVER_SECURITY_ENABLED.get()
+
+  if 'url' not in service_info:
+    LOG.info("Hue does not have %s configured, cannot test %s" % (service, service))
+  elif service_info['url'] is None:
+    LOG.info("Hue does not have %s configured, cannot test %s" % (service, service))
+
+  if service_info['url'].endswith('/'):
+    service_info['url'] = service_info['url'][:-1]
+
+  return service_info
+
+
+def add_service_test(available_services, options='all', service_name=None, testname=None, suburl=None, method='GET',
+                     teststring=None, test_options=None):
+  if options['service'] == "all" or options['service'] == service_name.lower():
+    if not service_name in available_services:
+      service_info = get_service_info(service_name)
+      url = service_info['url']
+      security_enabled = service_info['security_enabled']
+      available_services[service_name] = {}
+      available_services[service_name]['url'] = url
+      available_services[service_name]['security_enabled'] = security_enabled
+    # Tests
+    if not 'tests' in available_services[service_name]:
+      available_services[service_name]['tests'] = {}
+    if not testname in available_services[service_name]['tests']:
+      for test_option in test_options.keys():
+        suburl = suburl.replace(test_option, str(test_options[test_option]))
+      available_services[service_name]['tests'][testname] = {}
+      available_services[service_name]['tests'][testname]['url'] = '%s/%s' % (
+        available_services[service_name]['url'], suburl)
+      available_services[service_name]['tests'][testname]['method'] = method
+      available_services[service_name]['tests'][testname]['test'] = teststring
+
+
+class Command(BaseCommand):
+  """
+  Handler for renaming duplicate User objects
+  """
+
+  try:
+    from optparse import make_option
+    option_list = BaseCommand.option_list + (
+      make_option("--service", help=_t("Comma separated services to test, all, httpfs, solr, oozie, rm, jhs, sparkhs."),
+                  action="store", default='all', dest='service'),
+      make_option("--testname", help=_t("Test for a given service, must only include one service name."),
+                  action="store", default=None, dest='testname'),
+      make_option("--testoptions", help=_t(
+        "Comma separated list of options for test. IE: oozie_job=0000778-190820133637006-oozie-oozi-C,getlogs=true"),
+                  action="store", default=None, dest='testoptions'),
+      make_option("--showcurl", help=_t("Show curl commands."),
+                  action="store_true", default=False, dest='showcurl'),
+      make_option("--response", help=_t("Show entire REST response."),
+                  action="store_true", default=False, dest='entireresponse'),
+      make_option("--username", help=_t("User to doAs."),
+                  action="store", default="admin", dest='username'),
+      make_option("--verbose", help=_t("Verbose."),
+                  action="store_true", default=False, dest='verbose'),
+    )
+
+  except AttributeError, e:
+    baseoption_test = 'BaseCommand' in str(e) and 'option_list' in str(e)
+    if baseoption_test:
+      def add_arguments(self, parser):
+        parser.add_argument("--service",
+                            help=_t("Comma separated services to test, all, httpfs, solr, oozie, rm, jhs, sparkhs."),
+                            action="store", default='all', dest='service'),
+        parser.add_argument("--testname", help=_t("Test for a given service, must only include one service name."),
+                            action="store", default=None, dest='testname'),
+        parser.add_argument("--testoptions", help=_t(
+          "Comma separated list of options for test. IE: oozie_job=0000778-190820133637006-oozie-oozi-C,getlogs=true"),
+                            action="store", default=None, dest='testoptions'),
+        parser.add_argument("--showcurl", help=_t("Show curl commands."),
+                            action="store_true", default=False, dest='showcurl'),
+        parser.add_argument("--response", help=_t("Show entire REST response."),
+                            action="store_true", default=False, dest='entireresponse'),
+        parser.add_argument("--username", help=_t("User to doAs."),
+                            action="store", default="admin", dest='username'),
+        parser.add_argument("--verbose", help=_t("Verbose."),
+                            action="store_true", default=False, dest='verbose')
+    else:
+      LOG.warn(str(e))
+      sys.exit(1)
+
+  def handle(self, *args, **options):
+    test_options = {}
+    test_options['TIME_ZONE'] = TIME_ZONE.get()
+    test_options['DOAS'] = options['username']
+    test_options['NOW'] = current_milli_time()
+    test_options['NOWLESSMIN'] = test_options['NOW'] - 60000
+    if options['testoptions'] is not None:
+      for test_option in options['testoptions'].split(','):
+        option, option_value = test_option.split('=')
+        test_options[option.upper()] = option_value
+
+    test_services = options['service'].split(',')
+    supported_services = ['all', 'httpfs', 'solr', 'oozie', 'rm', 'jhs', 'sparkhs']
+    allowed_tests = {}
+    allowed_tests['httpfs'] = {}
+    allowed_tests['httpfs']['USERHOME'] = None
+
+    allowed_tests['jhs'] = {}
+    allowed_tests['jhs']['FINISHED'] = None
+
+    allowed_tests['oozie'] = {}
+    allowed_tests['oozie']['STATUS'] = None
+    allowed_tests['oozie']['CONFIGURATION'] = None
+    allowed_tests['oozie']['WORKFLOWS'] = None
+    allowed_tests['oozie']['COORDS'] = None
+    allowed_tests['oozie']['WORKFLOW'] = "oozie_id=0000001-190820133637006-oozie-oozi-W"
+    allowed_tests['oozie']['WORKFLOWLOG'] = "oozie_id=0000001-190820133637006-oozie-oozi-W"
+    allowed_tests['oozie']['WORKFLOWDEF'] = "oozie_id=0000001-190820133637006-oozie-oozi-W"
+    allowed_tests['oozie']['COORD'] = "oozie_id=0000001-190820133637006-oozie-oozi-C"
+
+    allowed_tests['rm'] = {}
+    allowed_tests['rm']['CLUSTERINFO'] = None
+
+    allowed_tests['solr'] = {}
+    allowed_tests['solr']['JMX'] = None
+
+    if options['testname'] is not None:
+      if len(test_services) > 1 or "all" in test_services:
+        LOG.warn("When using --testname you must only submit one service name and you must not use all")
+        sys.exit(1)
+
+      if options['testname'] not in allowed_tests[options['service'].lower()].keys():
+        LOG.warn(
+          "--testname %s not found in allowed_tests for service %s" % (options['testname'], options['service']))
+        LOG.warn("Allowed tests for service:")
+        for test in allowed_tests[options['service'].lower()].keys():
+          if allowed_tests[options['service'].lower()][test] is None:
+            testoptions = "NONE"
+          else:
+            testoptions = allowed_tests[options['service'].lower()][test]
+          LOG.warn("testname: %s : testoptions: %s" % (test, testoptions))
+        sys.exit(1)
+
+    if not any(elem in test_services for elem in supported_services):
+      LOG.warn("Your service list does not contain a supported service: %s" % options['service'])
+      LOG.warn("Supported services: all, httpfs, solr, oozie, rm, jhs, sparkhs")
+      LOG.warn("Format: httpfs,solr,oozie")
+      sys.exit(1)
+
+    if not all(elem in supported_services for elem in test_services):
+      LOG.warn("Your service list contains an unsupported service: %s" % options['service'])
+      LOG.warn("Supported services: all, httpfs, solr, oozie, rm, jhs, sparkhs")
+      LOG.warn("Format: httpfs,solr,oozie")
+      sys.exit(1)
+
+    if options['service'] == 'sparkhs':
+      LOG.warn("Spark History Server not supported yet")
+      sys.exit(1)
+
+    LOG.info("TEST: %s" % str(test_options['NOW']))
+    LOG.info("Running REST API Tests on Services: %s" % options['service'])
+    curl = Curl(verbose=options['verbose'])
+
+    available_services = {}
+
+    # Add Solr
+    add_service_test(available_services, options=options, service_name="Solr", testname="JMX",
+                     suburl='jmx', method='GET', teststring='solr.solrxml.location', test_options=test_options)
+
+    # Add Oozie
+    if options['testname'] is None or options['testname'].upper() == "STATUS":
+      add_service_test(available_services, options=options, service_name="Oozie", testname="STATUS",
+                       suburl='v1/admin/status?timezone=TIME_ZONE&user.name=hue&doAs=DOAS', method='GET',
+                       teststring='{"systemMode":"NORMAL"}', test_options=test_options)
+
+    elif options['testname'].upper() == 'CONFIGURATION':
+      add_service_test(available_services, options=options, service_name="Oozie", testname="CONFIGURATION",
+                       suburl='v2/admin/configuration?timezone=TIME_ZONE&user.name=hue&doAs=DOAS', method='GET',
+                       teststring='{"oozie.email.smtp.auth', test_options=test_options)
+
+    elif options['testname'].upper() == 'WORKFLOWS':
+      add_service_test(available_services, options=options, service_name="Oozie", testname="WORKFLOWS",
+                       suburl='v1/jobs?len=100&doAs=DOAS&filter=user=admin;startcreatedtime=-7d&user.name=hue&offset'
+                              '=1&timezone=TIME_ZONE&jobtype=wf',
+                       method='GET',
+                       teststring='"workflows":[', test_options=test_options)
+
+    elif options['testname'].upper() == 'WORKFLOW':
+      add_service_test(available_services, options=options, service_name="Oozie", testname="WORKFLOW",
+                       suburl='v1/job/OOZIE_ID?timezone=TIME_ZONE&suser.name=hue&logfilter=&doAs=DOAS', method='GET',
+                       teststring='{"appName":', test_options=test_options)
+
+    elif options['testname'].upper() == 'WORKFLOWLOG':
+      add_service_test(available_services, options=options, service_name="Oozie", testname="WORKFLOWLOG",
+                       suburl='v2/job/OOZIE_ID?timezone=TIME_ZONE&show=log&user.name=hue&logfilter=&doAs=DOAS',
+                       method='GET',
+                       teststring='org.apache.oozie.service.JPAService: SERVER', test_options=test_options)
+
+    elif options['testname'].upper() == 'WORKFLOWDEF':
+      add_service_test(available_services, options=options, service_name="Oozie", testname="WORKFLOWDEF",
+                       suburl='v2/job/OOZIE_ID?timezone=TIME_ZONE&show=definition&user.name=hue&logfilter=&doAs=DOAS',
+                       method='GET',
+                       teststring='xmlns="uri', test_options=test_options)
+
+    elif options['testname'].upper() == 'COORDS':
+      add_service_test(available_services, options=options, service_name="Oozie", testname="COORDS",
+                       suburl='v1/jobs?len=100&doAs=DOAS&filter=user=admin;startcreatedtime=-7d&user.name=hue&offset'
+                              '=1&timezone=TIME_ZONE&jobtype=coord',
+                       method='GET',
+                       teststring='"coordinatorjobs":[', test_options=test_options)
+
+    elif options['testname'].upper() == 'COORD':
+      add_service_test(available_services, options=options, service_name="Oozie", testname="COORD",
+                       suburl='v1/job/OOZIE_ID?timezone=TIME_ZONE&suser.name=hue&logfilter=&doAs=DOAS', method='GET',
+                       teststring='{"appName":', test_options=test_options)
+
+    # Add HTTPFS
+    add_service_test(available_services, options=options, service_name="Httpfs", testname="USERHOME",
+                     suburl='user/DOAS?op=GETFILESTATUS&user.name=hue&DOAS=%s', method='GET',
+                     teststring='"type":"DIRECTORY"', test_options=test_options)
+
+    # Add RM
+    add_service_test(available_services, options=options, service_name="RM", testname="CLUSTERINFO",
+                     suburl='ws/v1/cluster/info', method='GET', teststring='"clusterInfo"', test_options=test_options)
+
+    # Add JHS
+    add_service_test(available_services, options=options, service_name="JHS", testname="FINISHED",
+                     suburl='ws/v1/history/mapreduce/jobs?finishedTimeBegin=NOWLESSMIN&finishedTimeEnd=NOW',
+                     method='GET',
+                     teststring='{"jobs"', test_options=test_options)
+
+    for service in available_services:
+      for service_test in available_services[service]['tests']:
+        LOG.info("Running %s %s Test:" % (service, service_test))
+        start_time = time.time()
+        response = curl.do_curl_available_services(available_services[service]['tests'][service_test])
+        returned_in = (time.time() - start_time) * 1000
+        if available_services[service]['tests'][service_test]['test'] in response:
+          LOG.info("TEST: %s %s: Passed in %dms: %s found in response" % (
+            service, service_test, returned_in, available_services[service]['tests'][service_test]['test']))
+          if options['entireresponse']:
+            LOG.info("TEST: %s %s: Response: %s" % (service, service_test, response))
+        else:
+          LOG.info("TEST: %s %s: Failed in %dms: Response: %s" % (service, service_test, returned_in, response))
+
+    log_file = log_dir + '/backend_test_curl.log'
+    print ""
+    print "Tests completed, view logs here: %s" % log_file
+    print "Report:"
+    cmd = 'grep -A1000 "%s" %s | grep "TEST:" | sed "s/.*INFO.*TEST:/  TEST:/g"' % (str(test_options['NOW']), log_file)
+    grep_process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
+    grep_response = grep_process.communicate()[0]
+    print "%s" % grep_response
+    print ""
+    print "OS Repro Commands are:"
+    cmd = 'grep -A1000 "%s" %s | grep "OSRUN:" | sed "s/.*INFO.*OSRUN:/  /g"' % (str(test_options['NOW']), log_file)
+    grep_process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
+    grep_response = grep_process.communicate()[0]
+    print "%s" % grep_response