浏览代码

[core] make snappy a first class citizen

Abraham Elmahrek 11 年之前
父节点
当前提交
8d15fcf37f
共有 3 个文件被更改,包括 8 次插入29 次删除
  1. 3 0
      README.rst
  2. 4 20
      apps/filebrowser/src/filebrowser/views.py
  3. 1 9
      apps/filebrowser/src/filebrowser/views_test.py

+ 3 - 0
README.rst

@@ -85,6 +85,7 @@ your system:
       * python-dev
       * python-simplejson
       * python-setuptools
+      * libsnappy-dev
 
     CentOS:
       * ant
@@ -104,6 +105,7 @@ your system:
       * python-devel
       * python-simplejson
       * sqlite-devel
+      * snappy-devel
 
     MacOS (mac port):
       * liblxml
@@ -112,6 +114,7 @@ your system:
       * mysql5-devel
       * simplejson (easy_install)
       * sqlite3
+      * snappy
 
 
 File Layout

+ 4 - 20
apps/filebrowser/src/filebrowser/views.py

@@ -23,6 +23,7 @@ import operator
 import posixpath
 import re
 import shutil
+import snappy
 import stat as stat_module
 import os
 
@@ -621,9 +622,9 @@ def read_contents(codec_type, path, fs, offset, length):
                 offset = 0
             elif path.endswith('.avro') and detect_avro(contents):
                 codec_type = 'avro'
-            elif snappy_installed() and path.endswith('.snappy'):
+            elif path.endswith('.snappy'):
                 codec_type = 'snappy'
-            elif snappy_installed() and stats.size <= MAX_SNAPPY_DECOMPRESSION_SIZE.get() and detect_snappy(contents + fhandle.read()):
+            elif stats.size <= MAX_SNAPPY_DECOMPRESSION_SIZE.get() and detect_snappy(contents + fhandle.read()):
                 codec_type = 'snappy'
 
         fhandle.seek(0)
@@ -645,17 +646,10 @@ def read_contents(codec_type, path, fs, offset, length):
 
 
 def _decompress_snappy(compressed_content):
-    try:
-        import snappy
-        return snappy.decompress(compressed_content)
-    except Exception, e:
-        raise PopupException(_('Failed to decompress snappy compressed file.'), detail=e)
+    return snappy.decompress(compressed_content)
 
 
 def _read_snappy(fhandle, path, offset, length, stats):
-    if not snappy_installed():
-        raise PopupException(_('Failed to decompress snappy compressed file. Snappy is not installed.'))
-
     if stats.size > MAX_SNAPPY_DECOMPRESSION_SIZE.get():
         raise PopupException(_('Failed to decompress snappy compressed file. File size is greater than allowed max snappy decompression size of %d.') % MAX_SNAPPY_DECOMPRESSION_SIZE.get())
 
@@ -726,21 +720,11 @@ def detect_snappy(contents):
     This will also return false if snappy decompression if we do not have the library available.
     '''
     try:
-        import snappy
         return snappy.isValidCompressed(contents)
     except:
         return False
 
 
-def snappy_installed():
-    '''Snappy is library that isn't supported by python2.4'''
-    try:
-        import snappy
-        return True
-    except:
-        return False
-
-
 def _calculate_navigation(offset, length, size):
     """
     List of (offset, length, string) tuples for suggested navigation through the file.

+ 1 - 9
apps/filebrowser/src/filebrowser/views_test.py

@@ -20,6 +20,7 @@ import json
 import logging
 import os
 import re
+import snappy
 import urlparse
 from avro import schema, datafile, io
 
@@ -35,7 +36,6 @@ from filebrowser.views import location_to_url
 
 from conf import MAX_SNAPPY_DECOMPRESSION_SIZE
 from lib.rwx import expand_mode
-from views import snappy_installed
 
 
 LOG = logging.getLogger(__name__)
@@ -583,10 +583,6 @@ def test_chooser():
 
 @attr('requires_hadoop')
 def test_view_snappy_compressed():
-  if not snappy_installed():
-    raise SkipTest
-  import snappy
-
   cluster = pseudo_hdfs4.shared_cluster()
   finish = []
   try:
@@ -639,10 +635,6 @@ def test_view_snappy_compressed():
 
 @attr('requires_hadoop')
 def test_view_snappy_compressed_avro():
-  if not snappy_installed():
-    raise SkipTest
-  import snappy
-
   cluster = pseudo_hdfs4.shared_cluster()
   finish = []
   try: