Browse Source

HUE-2962 [desktop] Add API doc for configuration and fix tests

Jenny Kim 9 years ago
parent
commit
45b42bcb32

+ 320 - 0
desktop/core/src/desktop/configuration/README.md

@@ -0,0 +1,320 @@
+# DefaultConfiguration API
+## API Endpoints
+
+* [/desktop/api/configurations/apps/](#get_configurable_apps)
+* [/desktop/api/configurations/user/](#get_default_configuration_for_user)
+* [/desktop/api/configurations/save/](#save_default_configuration)
+* [desktop/api/configurations/delete/](#delete_default_configuration)
+
+### <a name="get_configurable_apps"></a> GET /desktop/api/configurations/apps
+Returns a JSON response with `status` and `apps` where apps contains a dictionary of all configurable apps and their defined configuration, as well as any default and group saved configurations.
+
+Each record in `apps` will map to a dictionary that contains a **required** `properties` record which maps to a list of defined properties for the app. Optionally, the app may also contain a `default` list of properties, and/or `group` properties where each configured group ID is returned with corresponding properties.
+
+#### Example Request
+GET /desktop/api/configurations/apps
+
+#### Example Response
+```
+{
+    "status": 0,
+    "apps": {
+        "impala": {
+            "properties": [
+                {
+                    "multiple": true,
+                    "value": [],
+                    "nice_name": "Settings",
+                    "key": "settings",
+                    "help_text": "Impala configuration properties.",
+                    "type": "settings",
+                    "options": [
+                        "debug_action",
+                        "explain_level",
+                        "mem_limit",
+                        "optimize_partition_key_scans",
+                        "query_timeout_s"
+                    ]
+                }
+            ]
+        },
+        "hive": {
+            "properties": [
+                {
+                    "multiple": true,
+                    "value": [],
+                    "nice_name": "Files",
+                    "key": "files",
+                    "help_text": "Add one or more files, jars, or archives to the list of resources.",
+                    "type": "hdfs-files"
+                },
+                {
+                    "multiple": true,
+                    "value": [],
+                    "nice_name": "Functions",
+                    "key": "functions",
+                    "help_text": "Add one or more registered UDFs (requires function name and fully-qualified class name).",
+                    "type": "functions"
+                },
+                {
+                    "multiple": true,
+                    "value": [],
+                    "nice_name": "Settings",
+                    "key": "settings",
+                    "help_text": "Hive and Hadoop configuration properties.",
+                    "type": "settings",
+                    "options": [
+                        "hive.map.aggr",
+                        "hive.exec.compress.output",
+                        "hive.exec.parallel",
+                        "hive.execution.engine",
+                        "mapreduce.job.queuename"
+                    ]
+                }
+            ],
+            "default": [
+                {
+                    "multiple": true,
+                    "value": [],
+                    "nice_name": "Files",
+                    "key": "files",
+                    "help_text": "Add one or more files, jars, or archives to the list of resources.",
+                    "type": "hdfs-files"
+                },
+                {
+                    "multiple": true,
+                    "value": [],
+                    "nice_name": "Functions",
+                    "key": "functions",
+                    "help_text": "Add one or more registered UDFs (requires function name and fully-qualified class name).",
+                    "type": "functions"
+                },
+                {
+                    "multiple": true,
+                    "value": [
+                        {
+                            "mapreduce.job.queuename": "mr"
+                        }
+                    ],
+                    "nice_name": "Settings",
+                    "key": "settings",
+                    "help_text": "Hive and Hadoop configuration properties.",
+                    "type": "settings",
+                    "options": [
+                        "hive.map.aggr",
+                        "hive.exec.compress.output",
+                        "hive.exec.parallel",
+                        "hive.execution.engine",
+                        "mapreduce.job.queuename"
+                    ]
+                }
+            ],
+            "groups": {
+                "1": [
+                    {
+                        "multiple": true,
+                        "value": [
+                            {
+                                "path": "/user/test/myudfs.jar",
+                                "type": "jar"
+                            }
+                        ],
+                        "nice_name": "Files",
+                        "key": "files",
+                        "help_text": "Add one or more files, jars, or archives to the list of resources.",
+                        "type": "hdfs-files"
+                    },
+                    {
+                        "multiple": true,
+                        "value": [
+                            {
+                                "class_name": "org.hue.udf.MyUpper",
+                                "name": "myUpper"
+                            }
+                        ],
+                        "nice_name": "Functions",
+                        "key": "functions",
+                        "help_text": "Add one or more registered UDFs (requires function name and fully-qualified class name).",
+                        "type": "functions"
+                    },
+                    {
+                        "multiple": true,
+                        "value": [
+                            {
+                                "key": "mapreduce.job.queuename",
+                                "value": "mr"
+                            }
+                        ],
+                        "nice_name": "Settings",
+                        "key": "settings",
+                        "help_text": "Hive and Hadoop configuration properties.",
+                        "type": "settings",
+                        "options": [
+                            "hive.map.aggr",
+                            "hive.exec.compress.output",
+                            "hive.exec.parallel",
+                            "hive.execution.engine",
+                            "mapreduce.job.queuename"
+                        ]
+                    }
+                ]
+            }
+        }
+    }
+}
+```
+
+
+### <a name="get_default_configuration_for_user"></a> GET /desktop/api/configurations/user
+Returns a JSON response with configuration for a given `app` type and user designated by `user_id`.
+
+If no saved configuration is found in the DB (i.e. - no default, group or user specific saved configuration), null is returned.
+
+#### Example Request
+GET /desktop/api/configurations/user
+
+#### Example Response
+TODO: write me
+
+
+### <a name="save_default_configuration"></a> POST /desktop/api/configurations/save/
+Saves a configuration for either `default` (all users), a specific `group` designated by `group_id` or a specific `user` designated by `user_id`.
+
+#### Parameters
+
+* (**Required**) app: app type (e.g. - hive, impala, jdbc, etc.)
+* (**Required**) properties: JSON of saved properties
+* (**Optional**) is_default: boolean indicating if this is a default configuration or not
+* (**Optional**) group_id: group ID if this is a saved configuration for a specific group
+* (**Optional**) user_id: user ID if this is a saved configuration for a specific user
+
+Either `is_default`, `group_id` or `user_id` must be passed
+
+
+#### Example Request
+POST /desktop/api/configurations/apps
+
+```
+{
+    "app": "hive",
+    "group_id": 1,
+    "properties": [
+        {
+            "multiple": true,
+            "value": [
+                {
+                    "path": "/user/test/myudfs.jar",
+                    "type": "jar"
+                }
+            ],
+            "nice_name": "Files",
+            "key": "files",
+            "help_text": "Add one or more files, jars, or archives to the list of resources.",
+            "type": "hdfs-files"
+        },
+        {
+            "multiple": true,
+            "value": [
+                {
+                    "class_name": "org.hue.udf.MyUpper",
+                    "name": "myUpper"
+                }
+            ],
+            "nice_name": "Functions",
+            "key": "functions",
+            "help_text": "Add one or more registered UDFs (requires function name and fully-qualified class name).",
+            "type": "functions"
+        },
+        {
+            "multiple": true,
+            "value": [
+                {
+                    "key": "mapreduce.job.queuename",
+                    "value": "mr"
+                }
+            ],
+            "nice_name": "Settings",
+            "key": "settings",
+            "help_text": "Hive and Hadoop configuration properties.",
+            "type": "settings",
+            "options": [
+                "hive.map.aggr",
+                "hive.exec.compress.output",
+                "hive.exec.parallel",
+                "hive.execution.engine",
+                "mapreduce.job.queuename"
+            ]
+        }
+    ]
+}
+```
+
+#### Example Response
+```
+{
+    "status": 0,
+    "configuration": {
+        "is_default": false,
+        "app": "hive",
+        "group": "default",
+        "properties": [
+            {
+                "multiple": true,
+                "value": [
+                    {
+                        "path": "/user/test/myudfs.jar",
+                        "type": "jar"
+                    }
+                ],
+                "nice_name": "Files",
+                "key": "files",
+                "help_text": "Add one or more files, jars, or archives to the list of resources.",
+                "type": "hdfs-files"
+            },
+            {
+                "multiple": true,
+                "value": [
+                    {
+                        "class_name": "org.hue.udf.MyUpper",
+                        "name": "myUpper"
+                    }
+                ],
+                "nice_name": "Functions",
+                "key": "functions",
+                "help_text": "Add one or more registered UDFs (requires function name and fully-qualified class name).",
+                "type": "functions"
+            },
+            {
+                "multiple": true,
+                "value": [
+                    {
+                        "key": "mapreduce.job.queuename",
+                        "value": "mr"
+                    }
+                ],
+                "nice_name": "Settings",
+                "key": "settings",
+                "help_text": "Hive and Hadoop configuration properties.",
+                "type": "settings",
+                "options": [
+                    "hive.map.aggr",
+                    "hive.exec.compress.output",
+                    "hive.exec.parallel",
+                    "hive.execution.engine",
+                    "mapreduce.job.queuename"
+                ]
+            }
+        ],
+        "user": null
+    }
+}
+```
+
+### <a name="delete_default_configuration"></a> GET /desktop/api/configurations/delete
+TODO: Write me
+
+#### Example Request
+GET /desktop/api/configurations/delete
+
+#### Example Response
+TODO: write me

+ 29 - 1
desktop/core/src/desktop/configuration/api.py

@@ -48,6 +48,34 @@ def api_error_handler(func):
 
   return decorator
 
+def get_configurable():
+  # TODO: Use metaclasses to self-register configurable apps
+  app_configs = {}
+  config_classes = [HiveConfiguration, ImpalaConfiguration]
+
+  for config_cls in config_classes:
+    if not hasattr(config_cls, 'APP_NAME') or not hasattr(config_cls, 'PROPERTIES'):
+      LOG.exception('Configurable classes must define APP_NAME and PROPERTIES.')
+    app_name = config_cls.APP_NAME
+    app_configs[app_name] = {
+      'properties': config_cls.PROPERTIES
+    }
+
+    # Get default config
+    if DefaultConfiguration.objects.filter(app=app_name, is_default=True).exists():
+      default_config = DefaultConfiguration.objects.get(app=app_name, is_default=True)
+      app_configs[app_name].update({'default': default_config.properties_list})
+
+    # Get group configs
+    if DefaultConfiguration.objects.filter(app=app_name, group__isnull=False).exists():
+      app_configs[app_name].update({'groups': {}})
+      for grp_config in DefaultConfiguration.objects.filter(app=app_name, group__isnull=False).all():
+        app_configs[app_name]['groups'].update({grp_config.group.id: grp_config.properties_list})
+
+  return {
+    'status': 0,
+    'apps': app_configs
+  }
 
 @api_error_handler
 def get_configurable_apps(request):
@@ -71,7 +99,7 @@ def get_configurable_apps(request):
     # Get group configs
     if DefaultConfiguration.objects.filter(app=app_name, group__isnull=False).exists():
       app_configs[app_name].update({'groups': {}})
-      for grp_config in DefaultConfiguration.objects.filter(app=app_name, group_isnull=False).all():
+      for grp_config in DefaultConfiguration.objects.filter(app=app_name, group__isnull=False).all():
         app_configs[app_name]['groups'].update({grp_config.group.id: grp_config.properties_list})
 
   return JsonResponse({

+ 42 - 3
desktop/core/src/desktop/configuration/tests.py

@@ -96,9 +96,48 @@ class TestDefaultConfiguration(object):
 
   def test_get_default_configurations(self):
     app = 'hive'
-    properties = {
-        'settings': [{'key': 'hive.execution.engine', 'value': 'spark'}]
-    }
+    properties = [
+        {
+            "multiple": True,
+            "value": [{
+                "path": "/user/test/myudfs.jar",
+                "type": "jar"
+            }],
+            "nice_name": "Files",
+            "key": "files",
+            "help_text": "Add one or more files, jars, or archives to the list of resources.",
+            "type": "hdfs-files"
+        },
+        {
+            "multiple": True,
+            "value": [{
+                "class_name": "org.hue.udf.MyUpper",
+                "name": "myUpper"
+            }],
+            "nice_name": "Functions",
+            "key": "functions",
+            "help_text": "Add one or more registered UDFs (requires function name and fully-qualified class name).",
+            "type": "functions"
+        },
+        {
+            "multiple": True,
+            "value": [{
+                "key": "mapreduce.job.queuename",
+                "value": "mr"
+            }],
+            "nice_name": "Settings",
+            "key": "settings",
+            "help_text": "Hive and Hadoop configuration properties.",
+            "type": "settings",
+            "options": [
+                "hive.map.aggr",
+                "hive.exec.compress.output",
+                "hive.exec.parallel",
+                "hive.execution.engine",
+                "mapreduce.job.queuename"
+            ]
+        }
+    ]
 
     # No configurations returns null
     response = self.client.get("/desktop/api/configurations/user", {