Przeglądaj źródła

HUE-8737 [core] Replace past.utils.old_div with simple division

Ying Chen 6 lat temu
rodzic
commit
3df46ae403

+ 2 - 2
apps/beeswax/src/beeswax/conf.py

@@ -17,8 +17,8 @@
 
 from __future__ import division
 from builtins import str
-from past.utils import old_div
 import logging
+import math
 import os.path
 
 from django.utils.translation import ugettext_lazy as _t, ugettext as _
@@ -194,7 +194,7 @@ DOWNLOAD_CELL_LIMIT = Config(
 
 def get_deprecated_download_cell_limit():
   """Get the old default"""
-  return old_div(DOWNLOAD_CELL_LIMIT.get(), 100) if DOWNLOAD_CELL_LIMIT.get() > 0 else DOWNLOAD_CELL_LIMIT.get()
+  return math.floor(DOWNLOAD_CELL_LIMIT.get() / 100) if DOWNLOAD_CELL_LIMIT.get() > 0 else DOWNLOAD_CELL_LIMIT.get()
 
 DOWNLOAD_ROW_LIMIT = Config(
   key='download_row_limit',

+ 3 - 3
apps/beeswax/src/beeswax/create_table.py

@@ -20,11 +20,11 @@ from __future__ import division
 from builtins import str
 from builtins import range
 from builtins import object
-from past.utils import old_div
 import csv
 import gzip
 import json
 import logging
+import math
 import re
 
 from django.urls import reverse
@@ -401,11 +401,11 @@ def _readfields(lines, delimiters):
     if min(len_list) == 1:
       return 0
 
-    avg_n_fields = old_div(sum(len_list), n_lines)
+    avg_n_fields = math.floor(sum(len_list) / n_lines)
     sq_of_exp = avg_n_fields * avg_n_fields
 
     len_list_sq = [l * l for l in len_list]
-    exp_of_sq = old_div(sum(len_list_sq), n_lines)
+    exp_of_sq = math.floor(sum(len_list_sq) / n_lines)
     var = exp_of_sq - sq_of_exp
     # Favour more fields
     return (1000.0 / (var + 1)) + avg_n_fields

+ 2 - 2
apps/filebrowser/src/filebrowser/lib/xxd.py

@@ -20,7 +20,7 @@ Implements xxd-like functionality.
 from __future__ import division
 from builtins import map
 from builtins import range
-from past.utils import old_div
+import math
 import string
 import sys
 
@@ -94,7 +94,7 @@ def main(input, output):
         return "%02x" % ord
       hex = " ".join([ "".join(map(ashex, sentence)) for sentence in ordinals])
       # 2 characters per byte, 1 extra for spacing, and 1 extra at the end.
-      hex = hex.ljust(bytes_per_line*2 + (old_div(bytes_per_line,bytes_per_sentence)) - 1)
+      hex = hex.ljust(bytes_per_line*2 + (math.floor(bytes_per_line / bytes_per_sentence)) - 1)
       output.write("%07x: %s  %s\n" % (off, hex, printable))
 
     offset += len(data)

+ 2 - 2
apps/jobbrowser/src/jobbrowser/models.py

@@ -17,10 +17,10 @@
 
 from __future__ import division
 from builtins import str
-from past.utils import old_div
 from builtins import object
 import datetime
 import logging
+import math
 import functools
 import re
 
@@ -115,6 +115,6 @@ def format_unixtime_ms(unixtime):
   Format a unix timestamp in ms to a human readable string
   """
   if unixtime:
-    return str(datetime.datetime.fromtimestamp(old_div(unixtime,1000)).strftime("%x %X %Z"))
+    return str(datetime.datetime.fromtimestamp(math.floor(unixtime / 1000)).strftime("%x %X %Z"))
   else:
     return ""

+ 3 - 2
apps/jobbrowser/src/jobbrowser/templatetags/unix_ms_to_datetime.py

@@ -16,9 +16,10 @@
 # limitations under the License.
 
 from __future__ import division
-from past.utils import old_div
 import datetime
 import django
+import math
+
 from django.utils.translation import ugettext as _
 
 register = django.template.Library()
@@ -27,7 +28,7 @@ register = django.template.Library()
 def unix_ms_to_datetime(unixtime):
   """unixtime is seconds since the epoch"""
   if unixtime:
-    return datetime.datetime.fromtimestamp(old_div(unixtime,1000))
+    return datetime.datetime.fromtimestamp(math.floor(unixtime / 1000))
   return _("No time")
 unix_ms_to_datetime.is_safe = True
 

+ 2 - 2
apps/jobbrowser/src/jobbrowser/yarn_models.py

@@ -19,9 +19,9 @@ from __future__ import division
 from future import standard_library
 standard_library.install_aliases()
 from builtins import str
-from past.utils import old_div
 from builtins import object
 import logging
+import math
 import os
 import re
 import time
@@ -252,7 +252,7 @@ class Job(object):
 
       if self.desiredReduces > 0:
         if self.progress is not None:
-          self.progress = int(old_div((self.progress + self.reduces_percent_complete), 2))
+          self.progress = int(math.floor((self.progress + self.reduces_percent_complete) / 2))
         else:
           self.progress = self.reduces_percent_complete
 

+ 5 - 5
apps/oozie/src/oozie/models2.py

@@ -18,10 +18,10 @@
 from __future__ import division
 from builtins import str
 from past.builtins import basestring
-from past.utils import old_div
 from builtins import object
 import json
 import logging
+import math
 import os
 import re
 import sys
@@ -696,14 +696,14 @@ def _create_workflow_layout(nodes, adj_list, nodes_uuid_set, size=12):
           "columns":[
              {
                 "id": str(uuid.uuid4()),
-                "size": (old_div(size, len(node[1]))),
+                "size": (math.floor(size / len(node[1]))),
                 "rows":
                    [{
                       "id": str(uuid.uuid4()),
                       "widgets": c['widgets'],
                       "columns":c.get('columns') or []
                     } for c in col],
-                "klass":"card card-home card-column span%s" % (old_div(size, len(node[1])))
+                "klass":"card card-home card-column span%s" % (math.floor(size / len(node[1])))
              }
              for col in [_create_workflow_layout(item, adj_list, nodes_uuid_set, size) for item in node[1]]
           ]
@@ -3127,7 +3127,7 @@ def import_workflow_from_hue_3_7(old_wf):
             "columns":[
                {
                   "id": str(uuid.uuid4()),
-                  "size": (old_div(size, len(node[1]))),
+                  "size": (math.floor(size / len(node[1]))),
                   "rows":
                      [{
                         "id": str(uuid.uuid4()),
@@ -3141,7 +3141,7 @@ def import_workflow_from_hue_3_7(old_wf):
                       }
                    ]
                   ,
-                  "klass":"card card-home card-column span%s" % (old_div(size, len(node[1])))
+                  "klass":"card card-home card-column span%s" % (math.floor(size / len(node[1])))
                }
                for col in _create_layout(node[1], size)
             ]

+ 2 - 2
apps/useradmin/src/useradmin/middleware.py

@@ -18,9 +18,9 @@
 from __future__ import division
 from __future__ import absolute_import
 from builtins import next
-from past.utils import old_div
 from builtins import object
 import logging
+import math
 from datetime import datetime
 
 from django.contrib import messages
@@ -114,7 +114,7 @@ class LastActivityMiddleware(object):
     if hasattr(dt, 'total_seconds'):
       return dt.total_seconds()
     else:
-      return old_div((dt.microseconds + (dt.seconds + dt.days * 24 * 3600) * 10**6), 10**6)
+      return math.floor((dt.microseconds + (dt.seconds + dt.days * 24 * 3600) * 10**6) / 10**6)
 
 class ConcurrentUserSessionMiddleware(object):
   """

+ 3 - 3
desktop/core/src/desktop/lib/rest/resource.py

@@ -15,9 +15,9 @@ from __future__ import division
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from past.utils import old_div
 from builtins import object
 import logging
+import math
 import posixpath
 import urllib
 import time
@@ -214,9 +214,9 @@ class Resource(object):
 
 # Same in thrift_util.py for not losing the trace class
 def log_if_slow_call(duration, message, logger):
-  if duration >= old_div(WARN_LEVEL_CALL_DURATION_MS, 1000):
+  if duration >= math.floor(WARN_LEVEL_CALL_DURATION_MS / 1000):
     logger.warn('SLOW: %.2f - %s' % (duration, message))
-  elif duration >= old_div(INFO_LEVEL_CALL_DURATION_MS, 1000):
+  elif duration >= math.floor(INFO_LEVEL_CALL_DURATION_MS / 1000):
     logger.info('SLOW: %.2f - %s' % (duration, message))
   else:
     logger.debug(message)

+ 3 - 3
desktop/core/src/desktop/lib/thrift_util.py

@@ -21,11 +21,11 @@ standard_library.install_aliases()
 from builtins import map
 from builtins import range
 from past.builtins import basestring
-from past.utils import old_div
 from builtins import object
 import base64
 import queue
 import logging
+import math
 import socket
 import threading
 import time
@@ -820,9 +820,9 @@ def is_thrift_struct(o):
 
 # Same in resource.py for not losing the trace class
 def log_if_slow_call(duration, message):
-  if duration >= old_div(WARN_LEVEL_CALL_DURATION_MS, 1000):
+  if duration >= math.floor(WARN_LEVEL_CALL_DURATION_MS / 1000):
     LOG.warn('SLOW: %.2f - %s' % (duration, message))
-  elif duration >= old_div(INFO_LEVEL_CALL_DURATION_MS, 1000):
+  elif duration >= math.floor(INFO_LEVEL_CALL_DURATION_MS / 1000):
     LOG.info('SLOW: %.2f - %s' % (duration, message))
   else:
     LOG.debug(message)

+ 1 - 2
desktop/core/src/desktop/lib/view_util.py

@@ -17,7 +17,6 @@
 """Utilities for views (text and number formatting, etc)"""
 from __future__ import division
 
-from past.utils import old_div
 import datetime
 import logging
 import math
@@ -44,7 +43,7 @@ def big_filesizeformat(bytes):
   index = int(math.floor(math.log(bytes, 1024)))
   index = min(len(units) - 1, index)
 
-  return( "%.1f %s" % (old_div(bytes, math.pow(1024, index)), units[index]) )
+  return( "%.1f %s" % ((bytes / math.pow(1024, index)), units[index]) )
 
 def format_time_diff(start=None, end=None):
   """

+ 2 - 2
desktop/core/src/desktop/lib/view_util_test.py

@@ -16,12 +16,12 @@
 # limitations under the License.
 
 from __future__ import division
-from past.utils import old_div
 from nose.tools import *
 
 from desktop.lib.view_util import big_filesizeformat, format_time_diff, format_duration_in_millis
 
 import datetime
+import math
 
 def test_big_filesizeformat():
   assert_equal("N/A", big_filesizeformat(None))
@@ -32,7 +32,7 @@ def test_big_filesizeformat():
   assert_equal("1.0 MB", big_filesizeformat(1024*1024))
   assert_equal("1.1 GB", big_filesizeformat(int(1.1*1024*1024*1024)))
   assert_equal("2.0 TB", big_filesizeformat(2*1024*1024*1024*1024))
-  assert_equal("1.5 PB", big_filesizeformat(old_div(3*1024*1024*1024*1024*1024,2)))
+  assert_equal("1.5 PB", big_filesizeformat(math.floor(3*1024*1024*1024*1024*1024 / 2)))
 
 def test_format_time_diff():
   assert_equal("1h:0m:0s", format_time_diff(datetime.datetime.fromtimestamp(0), datetime.datetime.fromtimestamp(60*60*1)))

+ 2 - 2
desktop/core/src/desktop/log/access.py

@@ -21,8 +21,8 @@ This assumes a single-threaded server.
 """
 from __future__ import division
 
-from past.utils import old_div
 import logging
+import math
 import re
 import resource
 import sys
@@ -96,7 +96,7 @@ class AccessInfo(dict):
     if sys.platform == 'darwin':
       rusage_denom = rusage_denom * 1024
     # get peak memory usage, bytes on OSX, Kilobytes on Linux
-    return old_div(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss, rusage_denom)
+    return math.floor(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / rusage_denom)
 
   def log(self, level, msg=None, start_time=None, response=None):
     is_instrumentation = desktop.conf.INSTRUMENTATION.get()

+ 7 - 7
desktop/libs/dashboard/src/dashboard/facet_builder.py

@@ -22,8 +22,8 @@ from future import standard_library
 standard_library.install_aliases()
 from builtins import str
 from builtins import range
-from past.utils import old_div
 import logging
+import math
 import numbers
 import urllib.request, urllib.parse, urllib.error
 import re
@@ -114,13 +114,13 @@ def _guess_range_facet(widget_type, solr_api, collection, facet_field, propertie
 
 def _get_interval(domain_ms, SLOTS):
   biggest_interval = TIME_INTERVALS[len(TIME_INTERVALS) - 1]
-  biggest_interval_is_too_small = old_div(domain_ms, biggest_interval['ms']) > SLOTS
+  biggest_interval_is_too_small = math.floor(domain_ms / biggest_interval['ms']) > SLOTS
   if biggest_interval_is_too_small:
-    coeff = min(ceil(old_div(domain_ms, SLOTS)), 100) # If we go over 100 years, something has gone wrong.
+    coeff = min(ceil(math.floor(domain_ms / SLOTS)), 100) # If we go over 100 years, something has gone wrong.
     return {'ms': YEAR_MS * coeff, 'coeff': coeff, 'unit': 'YEARS'}
 
   for i in range(len(TIME_INTERVALS) - 2, 0, -1):
-    slots = old_div(domain_ms, TIME_INTERVALS[i]['ms'])
+    slots = math.floor(domain_ms / TIME_INTERVALS[i]['ms'])
     if slots > SLOTS:
       return TIME_INTERVALS[i + 1]
 
@@ -175,7 +175,7 @@ def _compute_range_facet(widget_type, stat_facet, properties, start=None, end=No
         SLOTS = 5
       elif widget_type == 'facet-widget' or widget_type == 'text-facet-widget' or widget_type == 'histogram-widget' or widget_type == 'bar-widget' or widget_type == 'bucket-widget' or widget_type == 'timeline-widget':
         if window_size:
-          SLOTS = old_div(int(window_size), 75) # Value is determined as the thinnest space required to display a timestamp on x axis
+          SLOTS = math.floor(int(window_size) / 75) # Value is determined as the thinnest space required to display a timestamp on x axis
         else:
           SLOTS = 10
       else:
@@ -202,7 +202,7 @@ def _compute_range_facet(widget_type, stat_facet, properties, start=None, end=No
         end = int(end)
 
       if gap is None:
-        gap = int(old_div((end - start), SLOTS))
+        gap = int(math.floor((end - start) / SLOTS))
       if gap < 1:
         gap = 1
 
@@ -248,7 +248,7 @@ def _compute_range_facet(widget_type, stat_facet, properties, start=None, end=No
       is_date = True
       domain_ms = _get_interval_duration(stat_facet['min'])
       interval = _get_interval(domain_ms, SLOTS)
-      nb_slot = old_div(domain_ms, interval['ms'])
+      nb_slot = math.floor(domain_ms / interval['ms'])
       gap = _format_interval(interval)
       end_ts = datetime.utcnow()
       end_ts_clamped = _clamp_date(interval, end_ts)

+ 3 - 3
desktop/libs/hadoop/src/hadoop/fs/__init__.py

@@ -35,11 +35,11 @@ from functools import reduce
 standard_library.install_aliases()
 from builtins import map
 from builtins import range
-from past.utils import old_div
 from builtins import object
 import errno
 import grp
 import logging
+import math
 import os
 import posixpath
 import pwd
@@ -251,9 +251,9 @@ class FakeStatus(object):
     o = dict()
     GB = 1024*1024*1024
     o["bytesTotal"] = 5*GB
-    o["bytesUsed"] = old_div(5*GB,2)
+    o["bytesUsed"] = math.floor(5*GB / 2)
     o["bytesRemaining"] = 2*GB
-    o["bytesNonDfs"] = old_div(GB,2)
+    o["bytesNonDfs"] = math.floor(GB / 2)
     o["liveDataNodes"] = 13
     o["deadDataNodes"] = 2
     o["upgradeStatus"] = dict(version=13, percentComplete=100, finalized=True)

+ 2 - 2
desktop/libs/hadoop/src/hadoop/fs/hadoopfs.py

@@ -26,10 +26,10 @@ from __future__ import division
 from past.builtins import cmp
 from future import standard_library
 standard_library.install_aliases()
-from past.utils import old_div
 from builtins import object
 import errno
 import logging
+import math
 import os
 import posixpath
 import random
@@ -521,7 +521,7 @@ class BlockCache(object):
     if _max_idx < _min_idx:
       return None
 
-    pivot_idx = old_div((_max_idx + _min_idx), 2)
+    pivot_idx = math.floor((_max_idx + _min_idx) / 2)
     pivot_block = self.blocks[pivot_idx]
     if pos < pivot_block.startOffset:
       return self.find_block(pos, _min_idx, pivot_idx - 1)

+ 3 - 3
desktop/libs/hadoop/src/hadoop/fs/webhdfs_types.py

@@ -22,7 +22,7 @@ from __future__ import division
 
 from builtins import oct
 from builtins import object
-from past.utils import old_div
+import math
 import stat
 
 from django.utils.encoding import smart_str
@@ -40,8 +40,8 @@ class WebHdfsStat(object):
     self.path = Hdfs.join(parent_path, self.name)
     self.isDir = file_status['type'] == 'DIRECTORY'
     self.type = file_status['type']
-    self.atime = old_div(file_status['accessTime'], 1000)
-    self.mtime = old_div(file_status['modificationTime'], 1000)
+    self.atime = math.floor(file_status['accessTime'] / 1000)
+    self.mtime = math.floor(file_status['modificationTime'] / 1000)
     self.user = file_status['owner']
     self.group = file_status['group']
     self.size = file_status['length']

+ 6 - 6
desktop/libs/libanalyze/src/libanalyze/rules.py

@@ -19,11 +19,11 @@ from builtins import zip
 from builtins import range
 from builtins import object
 from functools import reduce
-from past.utils import old_div
 import copy
 import glob
 import json
 import logging
+import math
 import os
 import re
 import types
@@ -63,7 +63,7 @@ class ProfileContext(object):
                 dtparse(node.info_strings["Start Time"])).total_seconds()
 
     def percentage_of_total(self, compare):
-        return old_div(compare, self.query_duration())
+        return math.floor(compare / self.query_duration())
 
 
 class SQLOperatorReason(object):
@@ -291,7 +291,7 @@ class JoinOrderStrategyCheck(SQLOperatorReason):
             rhsRows = buildRows * hosts
             lhsRows = probeRows * hosts
 
-        impact = old_div((rhsRows - lhsRows * 1.5), hosts / 0.01)
+        impact = math.floor((rhsRows - lhsRows * 1.5) / hosts / 0.01)
         if (impact > 0):
             return {
                 "impact": impact,
@@ -301,7 +301,7 @@ class JoinOrderStrategyCheck(SQLOperatorReason):
 
         bcost = rhsRows * hosts
         scost = lhsRows + rhsRows
-        impact = old_div((networkcost - min(bcost, scost) - 1), hosts / 0.01)
+        impact = math.floor((networkcost - min(bcost, scost) - 1) / hosts / 0.01)
         return {
             "impact": impact,
             "message": "RHS %d; LHS %d" % (rhsRows, lhsRows),
@@ -331,7 +331,7 @@ class ExplodingJoinCheck(SQLOperatorReason):
 
         impact = 0
         if (rowsReturned > 0):
-            impact = old_div(probeTime * (rowsReturned - probeRows), rowsReturned)
+            impact = math.floor(probeTime * (rowsReturned - probeRows) / rowsReturned)
         return {
             "impact": impact,
             "message": "%d input rows are exploded to %d output rows" % (probeRows, rowsReturned),
@@ -356,7 +356,7 @@ class NNRpcCheck(SQLOperatorReason):
         hdfsRawReadTime = models.query_node_by_id_value(profile, plan_node_id, "TotalRawHdfsReadTime(*)", True)
         avgReadThreads = models.query_node_by_id_value(profile, plan_node_id, "AverageHdfsReadThreadConcurrency", True)
         avgReadThreads = max(1, to_double(avgReadThreads))
-        impact = max(0, old_div((totalStorageTime - hdfsRawReadTime), avgReadThreads))
+        impact = max(0, math.floor((totalStorageTime - hdfsRawReadTime) / avgReadThreads))
         return {
             "impact": impact,
             "message": "This is the time waiting for HDFS NN RPC.",

+ 4 - 4
desktop/libs/liboozie/src/liboozie/types.py

@@ -25,9 +25,9 @@ from __future__ import division
 
 from future import standard_library
 standard_library.install_aliases()
-from past.utils import old_div
 from builtins import object
 import logging
+import math
 import re
 import sys
 import time
@@ -307,7 +307,7 @@ class BundleAction(Action):
     end = mktime(parse_timestamp(self.endTime))
 
     if end != start:
-      progress = min(int((1 - old_div((end - next), (end - start))) * 100), 100)
+      progress = min(int((1 - math.floor((end - next) / (end - start))) * 100), 100)
     else:
       progress = 100
 
@@ -578,14 +578,14 @@ class Coordinator(Job):
     end = mktime(self.endTime)
 
     if end != start:
-      progress = min(int((1 - old_div((end - next), (end - start))) * 100), 100)
+      progress = min(int((1 - math.floor((end - next) / (end - start))) * 100), 100)
     else:
       progress = 100
 
     # Manage case of a rerun
     action_count = float(len(self.actions))
     if action_count != 0 and progress == 100:
-      progress = int(old_div(sum([action.is_finished() for action in self.actions]), action_count * 100))
+      progress = int(math.floor(sum([action.is_finished() for action in self.actions]) / action_count * 100))
 
     return progress
 

+ 1 - 2
desktop/libs/notebook/src/notebook/connectors/hiveserver2.py

@@ -19,7 +19,6 @@ from __future__ import division
 from future import standard_library
 standard_library.install_aliases()
 from builtins import next
-from past.utils import old_div
 from builtins import object
 import binascii
 import copy
@@ -428,7 +427,7 @@ class HS2Api(Api):
       started = logs.count('Starting Job')
       ended = logs.count('Ended Job')
 
-      progress = int(old_div((started + ended) * 100, (total * 2)))
+      progress = int((started + ended) * 100 / (total * 2))
       return max(progress, 5)  # Return 5% progress as a minimum
     elif snippet['type'] == 'impala':
       match = re.findall('(\d+)% Complete', logs, re.MULTILINE)

+ 3 - 3
desktop/libs/notebook/src/notebook/connectors/jdbc_vertica.py

@@ -16,13 +16,13 @@
 # limitations under the License.
 
 from __future__ import division
-from past.utils import old_div
 from librdbms.jdbc import query_and_fetch
 
 from notebook.connectors.jdbc import JdbcApi
 from notebook.connectors.jdbc import Assist
 import time
 import logging
+import math
 
 
 LOG = logging.getLogger(__name__)
@@ -65,8 +65,8 @@ class VerticaAssist(Assist):
                     + ", cache is used in "
                     + "%.2f"
                     % (
-                        old_div(100
-                        * float(self.cache_use_stat["cache"]), (self.cache_use_stat["query"] + self.cache_use_stat["cache"]))
+                        math.floor(100
+                        * float(self.cache_use_stat["cache"]) / (self.cache_use_stat["query"] + self.cache_use_stat["cache"]))
                     )
                     + "% cases"
                 )

+ 2 - 2
desktop/libs/notebook/src/notebook/dashboard_api.py

@@ -19,9 +19,9 @@ from __future__ import division
 from __future__ import print_function
 from builtins import next
 from builtins import zip
-from past.utils import old_div
 from builtins import object
 import logging
+import math
 import json
 import numbers
 import re
@@ -547,7 +547,7 @@ class SQLDashboardApi(DashboardApi):
     elif value <= 1:
       return value
     else:
-      return old_div(value, 100)
+      return math.floor(value / 100)
 
   @classmethod
   def _supports_cume_dist(self):