|
|
@@ -25,22 +25,25 @@ import time
|
|
|
|
|
|
from django import forms
|
|
|
|
|
|
+from beeswax.models import Namespace, Compute
|
|
|
|
|
|
HIVE_IDENTIFER_REGEX = re.compile("(^[a-zA-Z0-9]\w*\.)?[a-zA-Z0-9]\w*$")
|
|
|
|
|
|
-DL_FORMATS = [ 'csv', 'xls' ]
|
|
|
+DL_FORMATS = ['csv', 'xls']
|
|
|
|
|
|
-SELECTION_SOURCE = [ '', 'table', 'constant', ]
|
|
|
+SELECTION_SOURCE = ['', 'table', 'constant',]
|
|
|
|
|
|
-AGGREGATIONS = [ '', 'COUNT', 'SUM', 'AVG', 'MIN', 'MAX' ]
|
|
|
+AGGREGATIONS = ['', 'COUNT', 'SUM', 'AVG', 'MIN', 'MAX']
|
|
|
|
|
|
-JOIN_TYPES = [ '', 'LEFT OUTER JOIN', 'RIGHT OUTER JOIN', 'FULL OUTER JOIN', 'JOIN' ]
|
|
|
+JOIN_TYPES = ['', 'LEFT OUTER JOIN', 'RIGHT OUTER JOIN', 'FULL OUTER JOIN', 'JOIN']
|
|
|
|
|
|
-SORT_OPTIONS = [ '', 'ascending', 'descending' ]
|
|
|
+SORT_OPTIONS = ['', 'ascending', 'descending']
|
|
|
|
|
|
-RELATION_OPS_UNARY = [ 'IS NULL', 'IS NOT NULL', 'NOT' ]
|
|
|
+RELATION_OPS_UNARY = ['IS NULL', 'IS NOT NULL', 'NOT']
|
|
|
|
|
|
-RELATION_OPS = [ '=', '<>', '<', '<=', '>', '>=' ] + RELATION_OPS_UNARY
|
|
|
+RELATION_OPS = ['=', '<>', '<', '<=', '>', '>='] + RELATION_OPS_UNARY
|
|
|
+
|
|
|
+COMPUTE_TYPES = ['hive-compute', 'impala-compute']
|
|
|
|
|
|
TERMINATORS = [
|
|
|
# (hive representation, description, ascii value)
|
|
|
@@ -67,7 +70,7 @@ def to_choices(x):
|
|
|
Maps [a, b, c] to [(a,a), (b,b), (c,c)].
|
|
|
Useful for making ChoiceField's.
|
|
|
"""
|
|
|
- return [ (y, y) for y in x ]
|
|
|
+ return [(y, y) for y in x]
|
|
|
|
|
|
|
|
|
def apply_natural_sort(collection, key=None):
|
|
|
@@ -85,6 +88,68 @@ def apply_natural_sort(collection, key=None):
|
|
|
return sorted(collection, key=lambda i: tokenize_and_convert(i, key=key))
|
|
|
|
|
|
|
|
|
+def is_compute(cluster):
|
|
|
+ if not cluster:
|
|
|
+ return False
|
|
|
+ connector = cluster.get('connector')
|
|
|
+ compute = cluster.get('compute')
|
|
|
+ compute_check = lambda x: x and x.get('type') in COMPUTE_TYPES
|
|
|
+ return compute_check(cluster) or compute_check(connector) or compute_check(compute)
|
|
|
+
|
|
|
+
|
|
|
+'''
|
|
|
+find_compute attempts to find a compute based on the provided criteria.
|
|
|
+Following is the priority order
|
|
|
+1. A full/partial compute object available in cluster
|
|
|
+2. Lookup namespace based on namespace_id and return the first compute
|
|
|
+ filtered by user-access. Needs valid user and namespace_id
|
|
|
+3. Lookup namespace based on dialect from cluster or prpvided dialect
|
|
|
+ and return the first compute filtered by user-access. Needs valid user
|
|
|
+'''
|
|
|
+def find_compute(cluster=None, user=None, dialect=None, namespace_id=None):
|
|
|
+ if cluster:
|
|
|
+ # If we find a full/partial cluster object, we will attempt to load a compute
|
|
|
+ connector = cluster.get('connector')
|
|
|
+ compute = cluster.get('compute')
|
|
|
+ compute_check = lambda x: x and x.get('type') in COMPUTE_TYPES
|
|
|
+
|
|
|
+ # Pick the most probable compute object
|
|
|
+ selected_compute = (cluster if compute_check(cluster)
|
|
|
+ else compute if compute_check(compute)
|
|
|
+ else connector if compute_check(connector) else None)
|
|
|
+
|
|
|
+ # If found, we will attempt to reload it, first by id then by name
|
|
|
+ if selected_compute:
|
|
|
+ if selected_compute.get('id'):
|
|
|
+ c = Compute.objects.filter(id=selected_compute['id']).first().to_dict()
|
|
|
+ if c:
|
|
|
+ return c
|
|
|
+
|
|
|
+ if selected_compute.get('name'):
|
|
|
+ c = Compute.objects.filter(name=selected_compute['name']).first().to_dict()
|
|
|
+ if c:
|
|
|
+ return c
|
|
|
+
|
|
|
+ # If we could not load by id or name, then we want to pick a default compute based on dialect
|
|
|
+ dialect = selected_compute['dialect'] if selected_compute.get('dialect') else dialect
|
|
|
+ if not dialect and cluster.get('type'):
|
|
|
+ t = cluster['type']
|
|
|
+ dialect = 'hive' if t.startswith('hive') else 'impala' if t.startswith('impala') else None
|
|
|
+
|
|
|
+ # We will attempt to find a default compute based on other criteria
|
|
|
+ ns = None
|
|
|
+ if namespace_id:
|
|
|
+ ns = Namespace.objects.filter(id=namespace_id).first()
|
|
|
+
|
|
|
+ if not ns and dialect:
|
|
|
+ ns = Namespace.objects.filter(dialect=dialect).first()
|
|
|
+
|
|
|
+ if ns and user:
|
|
|
+ computes = ns.get_computes(user) if ns else None
|
|
|
+ if computes:
|
|
|
+ return computes[0]
|
|
|
+
|
|
|
+
|
|
|
class HiveIdentifierField(forms.RegexField):
|
|
|
"""
|
|
|
Corresponds to 'Identifier' in Hive.g (Hive's grammar)
|