Jelajahi Sumber

HUE-8835 [core] Minor fixes to cm-managed

cconner 6 tahun lalu
induk
melakukan
ac44c7b869
1 mengubah file dengan 76 tambahan dan 90 penghapusan
  1. 76 90
      desktop/core/src/desktop/manage_entry.py

+ 76 - 90
desktop/core/src/desktop/manage_entry.py

@@ -37,13 +37,10 @@ def reload_with_cm_env(cm_managed):
   try:
   try:
     from django.db.backends.oracle.base import Oracle_datetime
     from django.db.backends.oracle.base import Oracle_datetime
   except:
   except:
+    os.environ["SKIP_RELOAD"] = "True"
     if 'LD_LIBRARY_PATH' in os.environ:
     if 'LD_LIBRARY_PATH' in os.environ:
-      print "We need to reload the process to include LD_LIBRARY_PATH for Oracle backend"
+      print "We need to reload the process to include any LD_LIBRARY_PATH changes"
       try:
       try:
-        if cm_managed:
-          sys.argv.append("--cm-managed")
- 
-        sys.argv.append("--skip-reload")
         os.execv(sys.argv[0], sys.argv)
         os.execv(sys.argv[0], sys.argv)
       except Exception, exc:
       except Exception, exc:
         print 'Failed re-exec: %s' % exc
         print 'Failed re-exec: %s' % exc
@@ -58,24 +55,19 @@ def entry():
   from django.core.management.base import BaseCommand
   from django.core.management.base import BaseCommand
 
 
   os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'desktop.settings')
   os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'desktop.settings')
-  cm_config_file = '/etc/cloudera-scm-agent/config.ini'
-  ld_path_orig = None
-  if "LD_LIBRARY_PATH" in os.environ.keys():
-    ld_path_orig = os.environ["LD_LIBRARY_PATH"]
 
 
   # What's the subcommand being run?
   # What's the subcommand being run?
   # This code uses the same logic from django.core.management to handle command args
   # This code uses the same logic from django.core.management to handle command args
   subcommand = None
   subcommand = None
-  if "--skip-reload" in sys.argv:
-    skip_reload = True
-    sys.argv.remove("--skip-reload")
-  else:
-    skip_reload = False
 
 
-  # Check if --cm-managed flag is set and strip it out
-  # to prevent from sending to subcommands
+  # Check if legacy --cm-managed flag is set and strip it out
   if "--cm-managed" in sys.argv:
   if "--cm-managed" in sys.argv:
     sys.argv.remove("--cm-managed")
     sys.argv.remove("--cm-managed")
+
+  # Legit check for cm managed so we don't need flags
+  cm_agent_process = subprocess.Popen('ps -ef | grep "[c]m agent\|[c]mf-agent" | awk \'{print $2}\'', shell=True, stdout=subprocess.PIPE)
+  cm_agent_pid = cm_agent_process.communicate()[0].split('\n')[0]
+  if cm_agent_pid != '':
     cm_managed = True
     cm_managed = True
   else:
   else:
     cm_managed = False
     cm_managed = False
@@ -87,28 +79,22 @@ def entry():
 
 
   if len(sys.argv) > 1:
   if len(sys.argv) > 1:
     prof_id = subcommand = sys.argv[1]
     prof_id = subcommand = sys.argv[1]
-    #Check if this is a CM managed cluster
-    if os.path.isfile(cm_config_file) and not cm_managed and not skip_reload:
-        print "ALERT: This appears to be a CM Managed environment"
-        print "ALERT: HUE_CONF_DIR must be set when running hue commands in CM Managed environment"
-        print "ALERT: Please run 'hue <command> --cm-managed'"
   else:
   else:
     prof_id = str(os.getpid())
     prof_id = str(os.getpid())
 
 
   # CM managed configure env vars
   # CM managed configure env vars
-  if cm_managed:
-    import ConfigParser
-    from ConfigParser import NoOptionError
-    config = ConfigParser.RawConfigParser()
-    config.read(cm_config_file)
+  # Only run if CM isn't running the command
+  if cm_managed and not "HADOOP_CREDSTORE_PASSWORD" in os.environ.keys():
     try:
     try:
-      cm_agent_run_dir = config.get('General', 'agent_wide_credential_cache_location')
-    except NoOptionError:
-      cm_agent_run_dir = '/var/run/cloudera-scm-agent'
-      pass
+      supervisor_process = subprocess.Popen('ps -ef | grep [s]upervisord | awk \'{print $2}\'', shell=True, stdout=subprocess.PIPE)
+      supervisor_pid = supervisor_process.communicate()[0].split('\n')[0]
+      cm_supervisor_dir = os.path.realpath('/proc/%s/cwd' % supervisor_pid)
+    except Exception, e:
+      LOG.exception("Unable to get valid supervisord, make sure you are running as root and make sure the CM supervisor is running")
 
 
     #Parse CM supervisor include file for Hue and set env vars
     #Parse CM supervisor include file for Hue and set env vars
-    cm_supervisor_dir = cm_agent_run_dir + '/supervisor/include'
+    print "cm_supervisor_dir: %s" % cm_supervisor_dir
+    cm_agent_run_dir = os.path.dirname(cm_supervisor_dir)
     cm_process_dir = cm_agent_run_dir + '/process'
     cm_process_dir = cm_agent_run_dir + '/process'
     hue_env_conf = None
     hue_env_conf = None
     envline = None
     envline = None
@@ -119,13 +105,6 @@ def entry():
         hue_env_conf = file
         hue_env_conf = file
         hue_env_conf = cm_supervisor_dir + "/" + hue_env_conf
         hue_env_conf = cm_supervisor_dir + "/" + hue_env_conf
 
 
-    if hue_env_conf == None:
-      process_dirs = fnmatch.filter(os.listdir(cm_process_dir), '*%s*' % cm_hue_string)
-      process_dirs.sort()
-      hue_process_dir = cm_process_dir + "/" + process_dirs[-1]
-      hue_env_conf = fnmatch.filter(os.listdir(hue_process_dir), 'supervisor.conf')[0]
-      hue_env_conf = hue_process_dir + "/" + hue_env_conf
-
     if not hue_env_conf == None:
     if not hue_env_conf == None:
       if os.path.isfile(hue_env_conf):
       if os.path.isfile(hue_env_conf):
         hue_env_conf_file = open(hue_env_conf, "r")
         hue_env_conf_file = open(hue_env_conf, "r")
@@ -136,13 +115,14 @@ def entry():
             empty, hue_conf_dir = line.split("directory=")
             empty, hue_conf_dir = line.split("directory=")
             os.environ["HUE_CONF_DIR"] = hue_conf_dir.rstrip()
             os.environ["HUE_CONF_DIR"] = hue_conf_dir.rstrip()
     else:
     else:
+      os.environ["HUE_CONF_DIR"] = "/etc/hue/conf"
       print "This appears to be a CM managed cluster, but the"
       print "This appears to be a CM managed cluster, but the"
       print "supervisor/include file for Hue could not be found"
       print "supervisor/include file for Hue could not be found"
       print "in order to successfully run commands that access"
       print "in order to successfully run commands that access"
       print "the database you need to set the following env vars:"
       print "the database you need to set the following env vars:"
       print ""
       print ""
       print "  export JAVA_HOME=<java_home>"
       print "  export JAVA_HOME=<java_home>"
-      print "  export HUE_CONF_DIR=\"%s/`ls -1 %s | grep %s | sort -n | tail -1 `\"" % (cm_processs_dir, cm_process_dir, cm_hue_string)
+      print "  export HUE_CONF_DIR=/path/to/cloudera-scm-agent/process/<id>-hue-HUE_SERVER"
       print "  export HUE_IGNORE_PASSWORD_SCRIPT_ERRORS=1"
       print "  export HUE_IGNORE_PASSWORD_SCRIPT_ERRORS=1"
       print "  export HUE_DATABASE_PASSWORD=<hueDBpassword>"
       print "  export HUE_DATABASE_PASSWORD=<hueDBpassword>"
       print "If using Oracle as your database:"
       print "If using Oracle as your database:"
@@ -159,58 +139,65 @@ def entry():
           envval = envval.replace("'", "").rstrip()
           envval = envval.replace("'", "").rstrip()
           os.environ[envkey] = envval
           os.environ[envkey] = envval
 
 
-    #Set JAVA_HOME
-    if "JAVA_HOME" not in os.environ.keys():
-      if os.path.isfile('/usr/lib64/cmf/service/common/cloudera-config.sh'):
-        locate_java = subprocess.Popen(
-          ['bash', '-c', '. /usr/lib64/cmf/service/common/cloudera-config.sh; locate_java_home'], stdout=subprocess.PIPE,
-          stderr=subprocess.PIPE)
-      elif os.path.isfile('/opt/cloudera/cm-agent/service/common/cloudera-config.sh'):
-        locate_java = subprocess.Popen(
-          ['bash', '-c', '. /opt/cloudera/cm-agent/service/common/cloudera-config.sh; locate_java_home'],
-          stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-      else:
-        locate_java = None
-
-      JAVA_HOME = "UNKNOWN"
-
-      if locate_java is not None:
-        for line in iter(locate_java.stdout.readline, ''):
-          if 'JAVA_HOME' in line:
-            JAVA_HOME = line.rstrip().split('=')[1]
-
-      if JAVA_HOME != "UNKNOWN":
-        os.environ["JAVA_HOME"] = JAVA_HOME
-
-      if "JAVA_HOME" not in os.environ.keys():
-        print "JAVA_HOME must be set and can't be found, please set JAVA_HOME environment variable"
-        print "  export JAVA_HOME=<java_home>"
-        sys.exit(1)
-
-    #Make sure we set Oracle Client if configured
-    if "LD_LIBRARY_PATH" not in os.environ.keys():
-      if "SCM_DEFINES_SCRIPTS" in os.environ.keys():
-        for scm_script in os.environ["SCM_DEFINES_SCRIPTS"].split(":"):
-          if "ORACLE" in scm_script:
-            if os.path.isfile(scm_script):
-              oracle_source = subprocess.Popen(". %s; env" % scm_script, stdout=subprocess.PIPE, shell=True, executable="/bin/bash")
-              for line in oracle_source.communicate()[0].splitlines():
-                if "LD_LIBRARY_PATH" in line:
-                  var, oracle_ld_path = line.split("=")
-                  os.environ["LD_LIBRARY_PATH"] = oracle_ld_path
-
-    if "LD_LIBRARY_PATH" not in os.environ.keys():
-      print "LD_LIBRARY_PATH can't be found, if you are using ORACLE for your Hue database"
-      print "then it must be set, if not, you can ignore"
-      print "  export LD_LIBRARY_PATH=/path/to/instantclient"
-
-  if "LD_LIBRARY_PATH" in os.environ.keys():
-    if ld_path_orig is not None and ld_path_orig == os.environ["LD_LIBRARY_PATH"]:
-      skip_reload = True
-
-  if not skip_reload:
+    cloudera_config_script = None
+    if os.path.isfile('/usr/lib64/cmf/service/common/cloudera-config.sh'):
+      cloudera_config_script = '/usr/lib64/cmf/service/common/cloudera-config.sh'
+    elif os.path.isfile('/opt/cloudera/cm-agent/service/common/cloudera-config.sh'):
+      cloudera_config_script = '/opt/cloudera/cm-agent/service/common/cloudera-config.sh'
+
+    JAVA_HOME = None
+    if cloudera_config_script is not None:
+      locate_java = subprocess.Popen(['bash', '-c', '. %s; locate_java_home' % cloudera_config_script], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+      for line in iter(locate_java.stdout.readline,''):
+        if 'JAVA_HOME' in line:
+          JAVA_HOME = line.rstrip().split('=')[1]
+
+    if JAVA_HOME is not None:
+      os.environ["JAVA_HOME"] = JAVA_HOME
+    else:
+      print "JAVA_HOME must be set and can't be found, please set JAVA_HOME environment variable"
+      sys.exit(1)
+
+    oracle_check_process = subprocess.Popen('grep -i oracle %s/hue*ini' % os.environ["HUE_CONF_DIR"], shell=True, stdout=subprocess.PIPE)
+    oracle_check = oracle_check_process.communicate()[0]
+    if not oracle_check == '':
+      #Make sure we set Oracle Client if configured
+      if "LD_LIBRARY_PATH" not in os.environ.keys():
+        if "SCM_DEFINES_SCRIPTS" in os.environ.keys():
+          for scm_script in os.environ["SCM_DEFINES_SCRIPTS"].split(":"):
+            if "ORACLE" in scm_script:
+              if os.path.isfile(scm_script):
+                oracle_source = subprocess.Popen(". %s; env" % scm_script, stdout=subprocess.PIPE, shell=True, executable="/bin/bash")
+                for line in oracle_source.communicate()[0].splitlines():
+                  if "LD_LIBRARY_PATH" in line:
+                    var, oracle_ld_path = line.split("=")
+                    os.environ["LD_LIBRARY_PATH"] = oracle_ld_path
+
+      if "LD_LIBRARY_PATH" not in os.environ.keys():
+        print "LD_LIBRARY_PATH can't be found, if you are using ORACLE for your Hue database"
+        print "then it must be set, if not, you can ignore"
+        print "  export LD_LIBRARY_PATH=/path/to/instantclient"
+
+  if not "SKIP_RELOAD" in os.environ.keys():
     reload_with_cm_env(cm_managed)
     reload_with_cm_env(cm_managed)
 
 
+  #Get desktop settings so we can give guidance
+  from django.conf import settings
+  from django.core.exceptions import ImproperlyConfigured
+  try:
+    settings.INSTALLED_APPS
+  except ImproperlyConfigured as exc:
+    self.settings_exception = exc
+  from desktop.conf import DATABASE as desktop_database
+
+  print "Using the following config make sure it looks correct"
+  print "HUE_CONF_DIR: %s" % os.environ['HUE_CONF_DIR']
+  print "DB Engine: %s" % desktop_database.ENGINE.get()
+  print "DB Name: %s" % desktop_database.NAME.get()
+  print "DB User: %s" % desktop_database.USER.get()
+  print "DB Host: %s" % desktop_database.HOST.get()
+  print "DB Port: %s" % str(desktop_database.PORT.get())
+
   try:
   try:
     # Let django handle the normal execution
     # Let django handle the normal execution
     if os.getenv("DESKTOP_PROFILE"):
     if os.getenv("DESKTOP_PROFILE"):
@@ -226,7 +213,6 @@ def entry():
   except subprocess.CalledProcessError, e:
   except subprocess.CalledProcessError, e:
     if "altscript.sh" in str(e).lower():
     if "altscript.sh" in str(e).lower():
       print "%s" % e
       print "%s" % e
-      print "HUE_CONF_DIR seems to be set to CM location and '--cm-managed' flag not used"
 
 
 def _profile(prof_id, func):
 def _profile(prof_id, func):
   """
   """