Browse Source

[ozone] Add initial config under desktop

Harshg999 2 years ago
parent
commit
6b03c71486

+ 27 - 0
desktop/conf.dist/hue.ini

@@ -921,6 +921,30 @@ tls=no
 # The JSON credentials to authenticate to Google Cloud e.g. '{ "type": "service_account", "project_id": .... }'
 # The JSON credentials to authenticate to Google Cloud e.g. '{ "type": "service_account", "project_id": .... }'
 # json_credentials=None
 # json_credentials=None
 
 
+## Configuration for Ozone File System
+# ------------------------------------------------------------------------
+[[ozone]]
+# Default Ozone cluster
+[[[default]]]
+# Enter the filesystem URI
+## fs_defaultfs=ofs://localhost:8020
+
+# NameNode logical name.
+## logical_name=
+
+# Use WebHdfs/HttpFS as the communication mechanism.
+# Domain should be the NameNode or HttpFS host.
+# Default port is 14000 for HttpFS.
+## webhdfs_url=http://localhost:50070/webhdfs/v1
+
+# Whether Ozone requires client to perform Kerberos authentication.
+## security_enabled=false
+
+# Choose whether Hue should validate certificates received from the server.
+## ssl_cert_ca_verify=true
+
+# Directory of the Ozone configuration
+## ozone_conf_dir=$OZONE_CONF_DIR when set or '/etc/ozone/conf'
 
 
 ## Configuration for RAZ service integration
 ## Configuration for RAZ service integration
 # ------------------------------------------------------------------------
 # ------------------------------------------------------------------------
@@ -1223,6 +1247,9 @@ fs_defaultfs=hdfs://localhost:8020
 # Whether Hue should list this HDFS cluster. For historical reason there is no way to disable HDFS.
 # Whether Hue should list this HDFS cluster. For historical reason there is no way to disable HDFS.
 ## is_enabled=true
 ## is_enabled=true
 
 
+# Size, in bytes, of the chunks Django should store into memory and feed into the handler. Default is 64MB.
+## upload_chunk_size=64*1024*1024
+
 # Configuration for YARN (MR2)
 # Configuration for YARN (MR2)
 # ------------------------------------------------------------------------
 # ------------------------------------------------------------------------
 [[yarn_clusters]]
 [[yarn_clusters]]

+ 27 - 0
desktop/conf/pseudo-distributed.ini.tmpl

@@ -904,6 +904,30 @@
         # The JSON credentials to authenticate to Google Cloud e.g. '{ "type": "service_account", "project_id": .... }'
         # The JSON credentials to authenticate to Google Cloud e.g. '{ "type": "service_account", "project_id": .... }'
         # json_credentials=None
         # json_credentials=None
 
 
+  ## Configuration for Ozone File System
+  # ------------------------------------------------------------------------
+  [[ozone]]
+    # Default Ozone cluster
+    [[[default]]]
+      # Enter the filesystem URI
+      fs_defaultfs=ofs://localhost:8020
+
+      # NameNode logical name.
+      ## logical_name=
+
+      # Use WebHdfs/HttpFS as the communication mechanism.
+      # Domain should be the NameNode or HttpFS host.
+      # Default port is 14000 for HttpFS.
+      ## webhdfs_url=http://localhost:50070/webhdfs/v1
+
+      # Whether Ozone requires client to perform Kerberos authentication.
+      ## security_enabled=false
+
+      # Choose whether Hue should validate certificates received from the server.
+      ## ssl_cert_ca_verify=true
+
+      # Directory of the Ozone configuration
+      ## ozone_conf_dir=$OZONE_CONF_DIR when set or '/etc/ozone/conf'
 
 
   ## Configuration for RAZ service integration
   ## Configuration for RAZ service integration
   # ------------------------------------------------------------------------
   # ------------------------------------------------------------------------
@@ -1206,6 +1230,9 @@
       # Whether Hue should list this HDFS cluster. For historical reason there is no way to disable HDFS.
       # Whether Hue should list this HDFS cluster. For historical reason there is no way to disable HDFS.
       ## is_enabled=true
       ## is_enabled=true
 
 
+  # Size, in bytes, of the chunks Django should store into memory and feed into the handler. Default is 64MB.
+  ## upload_chunk_size=64*1024*1024
+  
   # Configuration for YARN (MR2)
   # Configuration for YARN (MR2)
   # ------------------------------------------------------------------------
   # ------------------------------------------------------------------------
   [[yarn_clusters]]
   [[yarn_clusters]]

+ 70 - 0
desktop/core/src/desktop/conf.py

@@ -2545,3 +2545,73 @@ def is_gs_enabled():
 def has_gs_access(user):
 def has_gs_access(user):
   from desktop.auth.backend import is_admin
   from desktop.auth.backend import is_admin
   return user.is_authenticated and user.is_active and (is_admin(user) or user.has_hue_permission(action="gs_access", app="filebrowser"))
   return user.is_authenticated and user.is_active and (is_admin(user) or user.has_hue_permission(action="gs_access", app="filebrowser"))
+
+
+def get_ozone_conf_dir_default():
+  """
+  Get from environment variable OZONE_CONF_DIR or '/etc/ozone/conf'
+  """
+  return os.environ.get("OZONE_CONF_DIR", "/etc/ozone/conf")
+
+
+PERMISSION_ACTION_OFS = "ofs_access"
+
+OZONE = UnspecifiedConfigSection(
+  "ozone",
+  help="One entry for each Ozone cluster",
+  each=ConfigSection(
+    help="Information about a single Ozone cluster",
+    members=dict(
+      FS_DEFAULTFS=Config(
+          "fs_defaultfs",
+          help="The equivalent of fs.defaultFS (aka fs.default.name)",
+          type=str,
+          default=None
+      ),
+      LOGICAL_NAME=Config(
+          "logical_name",
+          default="",
+          type=str,
+          help=_('NameNode logical name.')
+      ),
+      WEBHDFS_URL=Config(
+          "webhdfs_url",
+          help="The URL to WebHDFS/HttpFS service. Defaults to the WebHDFS URL on the NameNode.",
+          type=str,
+          default=None
+      ),
+      SECURITY_ENABLED=Config(
+          "security_enabled",
+          help="Whether Ozone requires client to perform Kerberos authentication",
+          type=coerce_bool,
+          default=False
+      ),
+      SSL_CERT_CA_VERIFY=Config(
+          "ssl_cert_ca_verify",
+          help="Choose whether Hue should validate certificates received from the server.",
+          dynamic_default=default_ssl_validate,
+          type=coerce_bool
+      ),
+      TEMP_DIR=Config(
+          "temp_dir",
+          help="Ozone directory for temporary files",
+          default='/tmp',
+          type=str
+      ),
+      OZONE_CONF_DIR=Config(
+          key="ozone_conf_dir",
+          dynamic_default=get_ozone_conf_dir_default,
+          help="Directory of the Ozone configuration. Defaults to the env variable OZONE_CONF_DIR when set, or '/etc/ozone/conf'",
+          type=str
+      ),
+    )
+  )
+)
+
+def is_ofs_enabled():
+  return ('default' in list(OZONE.keys()) and OZONE['default'].get_raw() and OZONE['default'].WEBHDFS_URL.get())
+
+
+def has_ofs_access(user):
+  from desktop.auth.backend import is_admin
+  return user.is_authenticated and user.is_active and (is_admin(user) or user.has_hue_permission(action="ofs_access", app="filebrowser"))

+ 0 - 15
desktop/core/src/desktop/lib/ozone/__init__.py

@@ -1,15 +0,0 @@
-# Licensed to Cloudera, Inc. under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  Cloudera, Inc. licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.

+ 0 - 15
desktop/core/src/desktop/lib/ozone/conf.py

@@ -1,15 +0,0 @@
-# Licensed to Cloudera, Inc. under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  Cloudera, Inc. licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.