浏览代码

Revert "HUE-8835 [core] Minor fixes to cm-managed"

This reverts commit ac44c7b86952c0cc1bbe2eab7f6ace69605b6d93.
Romain 6 年之前
父节点
当前提交
53fdf1e1f0
共有 1 个文件被更改,包括 90 次插入76 次删除
  1. 90 76
      desktop/core/src/desktop/manage_entry.py

+ 90 - 76
desktop/core/src/desktop/manage_entry.py

@@ -37,10 +37,13 @@ def reload_with_cm_env(cm_managed):
   try:
   try:
     from django.db.backends.oracle.base import Oracle_datetime
     from django.db.backends.oracle.base import Oracle_datetime
   except:
   except:
-    os.environ["SKIP_RELOAD"] = "True"
     if 'LD_LIBRARY_PATH' in os.environ:
     if 'LD_LIBRARY_PATH' in os.environ:
-      print "We need to reload the process to include any LD_LIBRARY_PATH changes"
+      print "We need to reload the process to include LD_LIBRARY_PATH for Oracle backend"
       try:
       try:
+        if cm_managed:
+          sys.argv.append("--cm-managed")
+ 
+        sys.argv.append("--skip-reload")
         os.execv(sys.argv[0], sys.argv)
         os.execv(sys.argv[0], sys.argv)
       except Exception, exc:
       except Exception, exc:
         print 'Failed re-exec: %s' % exc
         print 'Failed re-exec: %s' % exc
@@ -55,19 +58,24 @@ def entry():
   from django.core.management.base import BaseCommand
   from django.core.management.base import BaseCommand
 
 
   os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'desktop.settings')
   os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'desktop.settings')
+  cm_config_file = '/etc/cloudera-scm-agent/config.ini'
+  ld_path_orig = None
+  if "LD_LIBRARY_PATH" in os.environ.keys():
+    ld_path_orig = os.environ["LD_LIBRARY_PATH"]
 
 
   # What's the subcommand being run?
   # What's the subcommand being run?
   # This code uses the same logic from django.core.management to handle command args
   # This code uses the same logic from django.core.management to handle command args
   subcommand = None
   subcommand = None
+  if "--skip-reload" in sys.argv:
+    skip_reload = True
+    sys.argv.remove("--skip-reload")
+  else:
+    skip_reload = False
 
 
-  # Check if legacy --cm-managed flag is set and strip it out
+  # Check if --cm-managed flag is set and strip it out
+  # to prevent from sending to subcommands
   if "--cm-managed" in sys.argv:
   if "--cm-managed" in sys.argv:
     sys.argv.remove("--cm-managed")
     sys.argv.remove("--cm-managed")
-
-  # Legit check for cm managed so we don't need flags
-  cm_agent_process = subprocess.Popen('ps -ef | grep "[c]m agent\|[c]mf-agent" | awk \'{print $2}\'', shell=True, stdout=subprocess.PIPE)
-  cm_agent_pid = cm_agent_process.communicate()[0].split('\n')[0]
-  if cm_agent_pid != '':
     cm_managed = True
     cm_managed = True
   else:
   else:
     cm_managed = False
     cm_managed = False
@@ -79,22 +87,28 @@ def entry():
 
 
   if len(sys.argv) > 1:
   if len(sys.argv) > 1:
     prof_id = subcommand = sys.argv[1]
     prof_id = subcommand = sys.argv[1]
+    #Check if this is a CM managed cluster
+    if os.path.isfile(cm_config_file) and not cm_managed and not skip_reload:
+        print "ALERT: This appears to be a CM Managed environment"
+        print "ALERT: HUE_CONF_DIR must be set when running hue commands in CM Managed environment"
+        print "ALERT: Please run 'hue <command> --cm-managed'"
   else:
   else:
     prof_id = str(os.getpid())
     prof_id = str(os.getpid())
 
 
   # CM managed configure env vars
   # CM managed configure env vars
-  # Only run if CM isn't running the command
-  if cm_managed and not "HADOOP_CREDSTORE_PASSWORD" in os.environ.keys():
+  if cm_managed:
+    import ConfigParser
+    from ConfigParser import NoOptionError
+    config = ConfigParser.RawConfigParser()
+    config.read(cm_config_file)
     try:
     try:
-      supervisor_process = subprocess.Popen('ps -ef | grep [s]upervisord | awk \'{print $2}\'', shell=True, stdout=subprocess.PIPE)
-      supervisor_pid = supervisor_process.communicate()[0].split('\n')[0]
-      cm_supervisor_dir = os.path.realpath('/proc/%s/cwd' % supervisor_pid)
-    except Exception, e:
-      LOG.exception("Unable to get valid supervisord, make sure you are running as root and make sure the CM supervisor is running")
+      cm_agent_run_dir = config.get('General', 'agent_wide_credential_cache_location')
+    except NoOptionError:
+      cm_agent_run_dir = '/var/run/cloudera-scm-agent'
+      pass
 
 
     #Parse CM supervisor include file for Hue and set env vars
     #Parse CM supervisor include file for Hue and set env vars
-    print "cm_supervisor_dir: %s" % cm_supervisor_dir
-    cm_agent_run_dir = os.path.dirname(cm_supervisor_dir)
+    cm_supervisor_dir = cm_agent_run_dir + '/supervisor/include'
     cm_process_dir = cm_agent_run_dir + '/process'
     cm_process_dir = cm_agent_run_dir + '/process'
     hue_env_conf = None
     hue_env_conf = None
     envline = None
     envline = None
@@ -105,6 +119,13 @@ def entry():
         hue_env_conf = file
         hue_env_conf = file
         hue_env_conf = cm_supervisor_dir + "/" + hue_env_conf
         hue_env_conf = cm_supervisor_dir + "/" + hue_env_conf
 
 
+    if hue_env_conf == None:
+      process_dirs = fnmatch.filter(os.listdir(cm_process_dir), '*%s*' % cm_hue_string)
+      process_dirs.sort()
+      hue_process_dir = cm_process_dir + "/" + process_dirs[-1]
+      hue_env_conf = fnmatch.filter(os.listdir(hue_process_dir), 'supervisor.conf')[0]
+      hue_env_conf = hue_process_dir + "/" + hue_env_conf
+
     if not hue_env_conf == None:
     if not hue_env_conf == None:
       if os.path.isfile(hue_env_conf):
       if os.path.isfile(hue_env_conf):
         hue_env_conf_file = open(hue_env_conf, "r")
         hue_env_conf_file = open(hue_env_conf, "r")
@@ -115,14 +136,13 @@ def entry():
             empty, hue_conf_dir = line.split("directory=")
             empty, hue_conf_dir = line.split("directory=")
             os.environ["HUE_CONF_DIR"] = hue_conf_dir.rstrip()
             os.environ["HUE_CONF_DIR"] = hue_conf_dir.rstrip()
     else:
     else:
-      os.environ["HUE_CONF_DIR"] = "/etc/hue/conf"
       print "This appears to be a CM managed cluster, but the"
       print "This appears to be a CM managed cluster, but the"
       print "supervisor/include file for Hue could not be found"
       print "supervisor/include file for Hue could not be found"
       print "in order to successfully run commands that access"
       print "in order to successfully run commands that access"
       print "the database you need to set the following env vars:"
       print "the database you need to set the following env vars:"
       print ""
       print ""
       print "  export JAVA_HOME=<java_home>"
       print "  export JAVA_HOME=<java_home>"
-      print "  export HUE_CONF_DIR=/path/to/cloudera-scm-agent/process/<id>-hue-HUE_SERVER"
+      print "  export HUE_CONF_DIR=\"%s/`ls -1 %s | grep %s | sort -n | tail -1 `\"" % (cm_processs_dir, cm_process_dir, cm_hue_string)
       print "  export HUE_IGNORE_PASSWORD_SCRIPT_ERRORS=1"
       print "  export HUE_IGNORE_PASSWORD_SCRIPT_ERRORS=1"
       print "  export HUE_DATABASE_PASSWORD=<hueDBpassword>"
       print "  export HUE_DATABASE_PASSWORD=<hueDBpassword>"
       print "If using Oracle as your database:"
       print "If using Oracle as your database:"
@@ -139,64 +159,57 @@ def entry():
           envval = envval.replace("'", "").rstrip()
           envval = envval.replace("'", "").rstrip()
           os.environ[envkey] = envval
           os.environ[envkey] = envval
 
 
-    cloudera_config_script = None
-    if os.path.isfile('/usr/lib64/cmf/service/common/cloudera-config.sh'):
-      cloudera_config_script = '/usr/lib64/cmf/service/common/cloudera-config.sh'
-    elif os.path.isfile('/opt/cloudera/cm-agent/service/common/cloudera-config.sh'):
-      cloudera_config_script = '/opt/cloudera/cm-agent/service/common/cloudera-config.sh'
-
-    JAVA_HOME = None
-    if cloudera_config_script is not None:
-      locate_java = subprocess.Popen(['bash', '-c', '. %s; locate_java_home' % cloudera_config_script], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-      for line in iter(locate_java.stdout.readline,''):
-        if 'JAVA_HOME' in line:
-          JAVA_HOME = line.rstrip().split('=')[1]
-
-    if JAVA_HOME is not None:
-      os.environ["JAVA_HOME"] = JAVA_HOME
-    else:
-      print "JAVA_HOME must be set and can't be found, please set JAVA_HOME environment variable"
-      sys.exit(1)
-
-    oracle_check_process = subprocess.Popen('grep -i oracle %s/hue*ini' % os.environ["HUE_CONF_DIR"], shell=True, stdout=subprocess.PIPE)
-    oracle_check = oracle_check_process.communicate()[0]
-    if not oracle_check == '':
-      #Make sure we set Oracle Client if configured
-      if "LD_LIBRARY_PATH" not in os.environ.keys():
-        if "SCM_DEFINES_SCRIPTS" in os.environ.keys():
-          for scm_script in os.environ["SCM_DEFINES_SCRIPTS"].split(":"):
-            if "ORACLE" in scm_script:
-              if os.path.isfile(scm_script):
-                oracle_source = subprocess.Popen(". %s; env" % scm_script, stdout=subprocess.PIPE, shell=True, executable="/bin/bash")
-                for line in oracle_source.communicate()[0].splitlines():
-                  if "LD_LIBRARY_PATH" in line:
-                    var, oracle_ld_path = line.split("=")
-                    os.environ["LD_LIBRARY_PATH"] = oracle_ld_path
-
-      if "LD_LIBRARY_PATH" not in os.environ.keys():
-        print "LD_LIBRARY_PATH can't be found, if you are using ORACLE for your Hue database"
-        print "then it must be set, if not, you can ignore"
-        print "  export LD_LIBRARY_PATH=/path/to/instantclient"
-
-  if not "SKIP_RELOAD" in os.environ.keys():
-    reload_with_cm_env(cm_managed)
+    #Set JAVA_HOME
+    if "JAVA_HOME" not in os.environ.keys():
+      if os.path.isfile('/usr/lib64/cmf/service/common/cloudera-config.sh'):
+        locate_java = subprocess.Popen(
+          ['bash', '-c', '. /usr/lib64/cmf/service/common/cloudera-config.sh; locate_java_home'], stdout=subprocess.PIPE,
+          stderr=subprocess.PIPE)
+      elif os.path.isfile('/opt/cloudera/cm-agent/service/common/cloudera-config.sh'):
+        locate_java = subprocess.Popen(
+          ['bash', '-c', '. /opt/cloudera/cm-agent/service/common/cloudera-config.sh; locate_java_home'],
+          stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+      else:
+        locate_java = None
+
+      JAVA_HOME = "UNKNOWN"
+
+      if locate_java is not None:
+        for line in iter(locate_java.stdout.readline, ''):
+          if 'JAVA_HOME' in line:
+            JAVA_HOME = line.rstrip().split('=')[1]
+
+      if JAVA_HOME != "UNKNOWN":
+        os.environ["JAVA_HOME"] = JAVA_HOME
+
+      if "JAVA_HOME" not in os.environ.keys():
+        print "JAVA_HOME must be set and can't be found, please set JAVA_HOME environment variable"
+        print "  export JAVA_HOME=<java_home>"
+        sys.exit(1)
 
 
-  #Get desktop settings so we can give guidance
-  from django.conf import settings
-  from django.core.exceptions import ImproperlyConfigured
-  try:
-    settings.INSTALLED_APPS
-  except ImproperlyConfigured as exc:
-    self.settings_exception = exc
-  from desktop.conf import DATABASE as desktop_database
-
-  print "Using the following config make sure it looks correct"
-  print "HUE_CONF_DIR: %s" % os.environ['HUE_CONF_DIR']
-  print "DB Engine: %s" % desktop_database.ENGINE.get()
-  print "DB Name: %s" % desktop_database.NAME.get()
-  print "DB User: %s" % desktop_database.USER.get()
-  print "DB Host: %s" % desktop_database.HOST.get()
-  print "DB Port: %s" % str(desktop_database.PORT.get())
+    #Make sure we set Oracle Client if configured
+    if "LD_LIBRARY_PATH" not in os.environ.keys():
+      if "SCM_DEFINES_SCRIPTS" in os.environ.keys():
+        for scm_script in os.environ["SCM_DEFINES_SCRIPTS"].split(":"):
+          if "ORACLE" in scm_script:
+            if os.path.isfile(scm_script):
+              oracle_source = subprocess.Popen(". %s; env" % scm_script, stdout=subprocess.PIPE, shell=True, executable="/bin/bash")
+              for line in oracle_source.communicate()[0].splitlines():
+                if "LD_LIBRARY_PATH" in line:
+                  var, oracle_ld_path = line.split("=")
+                  os.environ["LD_LIBRARY_PATH"] = oracle_ld_path
+
+    if "LD_LIBRARY_PATH" not in os.environ.keys():
+      print "LD_LIBRARY_PATH can't be found, if you are using ORACLE for your Hue database"
+      print "then it must be set, if not, you can ignore"
+      print "  export LD_LIBRARY_PATH=/path/to/instantclient"
+
+  if "LD_LIBRARY_PATH" in os.environ.keys():
+    if ld_path_orig is not None and ld_path_orig == os.environ["LD_LIBRARY_PATH"]:
+      skip_reload = True
+
+  if not skip_reload:
+    reload_with_cm_env(cm_managed)
 
 
   try:
   try:
     # Let django handle the normal execution
     # Let django handle the normal execution
@@ -213,6 +226,7 @@ def entry():
   except subprocess.CalledProcessError, e:
   except subprocess.CalledProcessError, e:
     if "altscript.sh" in str(e).lower():
     if "altscript.sh" in str(e).lower():
       print "%s" % e
       print "%s" % e
+      print "HUE_CONF_DIR seems to be set to CM location and '--cm-managed' flag not used"
 
 
 def _profile(prof_id, func):
 def _profile(prof_id, func):
   """
   """