Prechádzať zdrojové kódy

[core] make snappy a first class citizen

Abraham Elmahrek 11 rokov pred
rodič
commit
8d15fcf

+ 3 - 0
README.rst

@@ -85,6 +85,7 @@ your system:
       * python-dev
       * python-dev
       * python-simplejson
       * python-simplejson
       * python-setuptools
       * python-setuptools
+      * libsnappy-dev
 
 
     CentOS:
     CentOS:
       * ant
       * ant
@@ -104,6 +105,7 @@ your system:
       * python-devel
       * python-devel
       * python-simplejson
       * python-simplejson
       * sqlite-devel
       * sqlite-devel
+      * snappy-devel
 
 
     MacOS (mac port):
     MacOS (mac port):
       * liblxml
       * liblxml
@@ -112,6 +114,7 @@ your system:
       * mysql5-devel
       * mysql5-devel
       * simplejson (easy_install)
       * simplejson (easy_install)
       * sqlite3
       * sqlite3
+      * snappy
 
 
 
 
 File Layout
 File Layout

+ 4 - 20
apps/filebrowser/src/filebrowser/views.py

@@ -23,6 +23,7 @@ import operator
 import posixpath
 import posixpath
 import re
 import re
 import shutil
 import shutil
+import snappy
 import stat as stat_module
 import stat as stat_module
 import os
 import os
 
 
@@ -621,9 +622,9 @@ def read_contents(codec_type, path, fs, offset, length):
                 offset = 0
                 offset = 0
             elif path.endswith('.avro') and detect_avro(contents):
             elif path.endswith('.avro') and detect_avro(contents):
                 codec_type = 'avro'
                 codec_type = 'avro'
-            elif snappy_installed() and path.endswith('.snappy'):
+            elif path.endswith('.snappy'):
                 codec_type = 'snappy'
                 codec_type = 'snappy'
-            elif snappy_installed() and stats.size <= MAX_SNAPPY_DECOMPRESSION_SIZE.get() and detect_snappy(contents + fhandle.read()):
+            elif stats.size <= MAX_SNAPPY_DECOMPRESSION_SIZE.get() and detect_snappy(contents + fhandle.read()):
                 codec_type = 'snappy'
                 codec_type = 'snappy'
 
 
         fhandle.seek(0)
         fhandle.seek(0)
@@ -645,17 +646,10 @@ def read_contents(codec_type, path, fs, offset, length):
 
 
 
 
 def _decompress_snappy(compressed_content):
 def _decompress_snappy(compressed_content):
-    try:
-        import snappy
-        return snappy.decompress(compressed_content)
-    except Exception, e:
-        raise PopupException(_('Failed to decompress snappy compressed file.'), detail=e)
+    return snappy.decompress(compressed_content)
 
 
 
 
 def _read_snappy(fhandle, path, offset, length, stats):
 def _read_snappy(fhandle, path, offset, length, stats):
-    if not snappy_installed():
-        raise PopupException(_('Failed to decompress snappy compressed file. Snappy is not installed.'))
-
     if stats.size > MAX_SNAPPY_DECOMPRESSION_SIZE.get():
     if stats.size > MAX_SNAPPY_DECOMPRESSION_SIZE.get():
         raise PopupException(_('Failed to decompress snappy compressed file. File size is greater than allowed max snappy decompression size of %d.') % MAX_SNAPPY_DECOMPRESSION_SIZE.get())
         raise PopupException(_('Failed to decompress snappy compressed file. File size is greater than allowed max snappy decompression size of %d.') % MAX_SNAPPY_DECOMPRESSION_SIZE.get())
 
 
@@ -726,21 +720,11 @@ def detect_snappy(contents):
     This will also return false if snappy decompression if we do not have the library available.
     This will also return false if snappy decompression if we do not have the library available.
     '''
     '''
     try:
     try:
-        import snappy
         return snappy.isValidCompressed(contents)
         return snappy.isValidCompressed(contents)
     except:
     except:
         return False
         return False
 
 
 
 
-def snappy_installed():
-    '''Snappy is library that isn't supported by python2.4'''
-    try:
-        import snappy
-        return True
-    except:
-        return False
-
-
 def _calculate_navigation(offset, length, size):
 def _calculate_navigation(offset, length, size):
     """
     """
     List of (offset, length, string) tuples for suggested navigation through the file.
     List of (offset, length, string) tuples for suggested navigation through the file.

+ 1 - 9
apps/filebrowser/src/filebrowser/views_test.py

@@ -20,6 +20,7 @@ import json
 import logging
 import logging
 import os
 import os
 import re
 import re
+import snappy
 import urlparse
 import urlparse
 from avro import schema, datafile, io
 from avro import schema, datafile, io
 
 
@@ -35,7 +36,6 @@ from filebrowser.views import location_to_url
 
 
 from conf import MAX_SNAPPY_DECOMPRESSION_SIZE
 from conf import MAX_SNAPPY_DECOMPRESSION_SIZE
 from lib.rwx import expand_mode
 from lib.rwx import expand_mode
-from views import snappy_installed
 
 
 
 
 LOG = logging.getLogger(__name__)
 LOG = logging.getLogger(__name__)
@@ -583,10 +583,6 @@ def test_chooser():
 
 
 @attr('requires_hadoop')
 @attr('requires_hadoop')
 def test_view_snappy_compressed():
 def test_view_snappy_compressed():
-  if not snappy_installed():
-    raise SkipTest
-  import snappy
-
   cluster = pseudo_hdfs4.shared_cluster()
   cluster = pseudo_hdfs4.shared_cluster()
   finish = []
   finish = []
   try:
   try:
@@ -639,10 +635,6 @@ def test_view_snappy_compressed():
 
 
 @attr('requires_hadoop')
 @attr('requires_hadoop')
 def test_view_snappy_compressed_avro():
 def test_view_snappy_compressed_avro():
-  if not snappy_installed():
-    raise SkipTest
-  import snappy
-
   cluster = pseudo_hdfs4.shared_cluster()
   cluster = pseudo_hdfs4.shared_cluster()
   finish = []
   finish = []
   try:
   try: