浏览代码

Revert "[core] make snappy a first class citizen"

This reverts commit 8d15fcf37fe9deedbd0579f1744db7728642db4b.

Conflicts:
	apps/filebrowser/src/filebrowser/views.py
Abraham Elmahrek 11 年之前
父节点
当前提交
d09dce2
共有 3 个文件被更改,包括 25 次插入7 次删除
  1. 0 3
      README.rst
  2. 16 3
      apps/filebrowser/src/filebrowser/views.py
  3. 9 1
      apps/filebrowser/src/filebrowser/views_test.py

+ 0 - 3
README.rst

@@ -85,7 +85,6 @@ your system:
       * python-dev
       * python-dev
       * python-simplejson
       * python-simplejson
       * python-setuptools
       * python-setuptools
-      * libsnappy-dev
 
 
     CentOS:
     CentOS:
       * ant
       * ant
@@ -105,7 +104,6 @@ your system:
       * python-devel
       * python-devel
       * python-simplejson
       * python-simplejson
       * sqlite-devel
       * sqlite-devel
-      * snappy-devel
 
 
     MacOS (mac port):
     MacOS (mac port):
       * liblxml
       * liblxml
@@ -114,7 +112,6 @@ your system:
       * mysql5-devel
       * mysql5-devel
       * simplejson (easy_install)
       * simplejson (easy_install)
       * sqlite3
       * sqlite3
-      * snappy
 
 
 
 
 File Layout
 File Layout

+ 16 - 3
apps/filebrowser/src/filebrowser/views.py

@@ -24,7 +24,6 @@ import parquet
 import posixpath
 import posixpath
 import re
 import re
 import shutil
 import shutil
-import snappy
 import stat as stat_module
 import stat as stat_module
 import os
 import os
 
 
@@ -634,9 +633,9 @@ def read_contents(codec_type, path, fs, offset, length):
                 codec_type = 'avro'
                 codec_type = 'avro'
             elif path.endswith('.parquet') and detect_parquet(fhandle):
             elif path.endswith('.parquet') and detect_parquet(fhandle):
                 codec_type = 'parquet'
                 codec_type = 'parquet'
-            elif path.endswith('.snappy'):
+            elif snappy_installed() and path.endswith('.snappy'):
                 codec_type = 'snappy'
                 codec_type = 'snappy'
-            elif stats.size <= MAX_SNAPPY_DECOMPRESSION_SIZE.get() and detect_snappy(fhandle.read()):
+            elif snappy_installed() and stats.size <= MAX_SNAPPY_DECOMPRESSION_SIZE.get() and detect_snappy(fhandle.read()):
                 codec_type = 'snappy'
                 codec_type = 'snappy'
 
 
         fhandle.seek(0)
         fhandle.seek(0)
@@ -661,12 +660,16 @@ def read_contents(codec_type, path, fs, offset, length):
 
 
 def _decompress_snappy(compressed_content):
 def _decompress_snappy(compressed_content):
     try:
     try:
+        import snappy
         return snappy.decompress(compressed_content)
         return snappy.decompress(compressed_content)
     except Exception, e:
     except Exception, e:
         raise PopupException(_('Failed to decompress snappy compressed file.'), detail=e)
         raise PopupException(_('Failed to decompress snappy compressed file.'), detail=e)
 
 
 
 
 def _read_snappy(fhandle, path, offset, length, stats):
 def _read_snappy(fhandle, path, offset, length, stats):
+    if not snappy_installed():
+        raise PopupException(_('Failed to decompress snappy compressed file. Snappy is not installed.'))
+
     if stats.size > MAX_SNAPPY_DECOMPRESSION_SIZE.get():
     if stats.size > MAX_SNAPPY_DECOMPRESSION_SIZE.get():
         raise PopupException(_('Failed to decompress snappy compressed file. File size is greater than allowed max snappy decompression size of %d.') % MAX_SNAPPY_DECOMPRESSION_SIZE.get())
         raise PopupException(_('Failed to decompress snappy compressed file. File size is greater than allowed max snappy decompression size of %d.') % MAX_SNAPPY_DECOMPRESSION_SIZE.get())
 
 
@@ -748,6 +751,7 @@ def detect_snappy(contents):
     This will also return false if snappy decompression if we do not have the library available.
     This will also return false if snappy decompression if we do not have the library available.
     '''
     '''
     try:
     try:
+        import snappy
         return snappy.isValidCompressed(contents)
         return snappy.isValidCompressed(contents)
     except:
     except:
         return False
         return False
@@ -760,6 +764,15 @@ def detect_parquet(fhandle):
     return parquet._check_header_magic_bytes(fhandle)
     return parquet._check_header_magic_bytes(fhandle)
 
 
 
 
+def snappy_installed():
+    '''Snappy is library that isn't supported by python2.4'''
+    try:
+        import snappy
+        return True
+    except:
+        return False
+
+
 def _calculate_navigation(offset, length, size):
 def _calculate_navigation(offset, length, size):
     """
     """
     List of (offset, length, string) tuples for suggested navigation through the file.
     List of (offset, length, string) tuples for suggested navigation through the file.

+ 9 - 1
apps/filebrowser/src/filebrowser/views_test.py

@@ -20,7 +20,6 @@ import json
 import logging
 import logging
 import os
 import os
 import re
 import re
-import snappy
 import urlparse
 import urlparse
 from avro import schema, datafile, io
 from avro import schema, datafile, io
 
 
@@ -36,6 +35,7 @@ from filebrowser.views import location_to_url
 
 
 from conf import MAX_SNAPPY_DECOMPRESSION_SIZE
 from conf import MAX_SNAPPY_DECOMPRESSION_SIZE
 from lib.rwx import expand_mode
 from lib.rwx import expand_mode
+from views import snappy_installed
 
 
 
 
 LOG = logging.getLogger(__name__)
 LOG = logging.getLogger(__name__)
@@ -583,6 +583,10 @@ def test_chooser():
 
 
 @attr('requires_hadoop')
 @attr('requires_hadoop')
 def test_view_snappy_compressed():
 def test_view_snappy_compressed():
+  if not snappy_installed():
+    raise SkipTest
+  import snappy
+
   cluster = pseudo_hdfs4.shared_cluster()
   cluster = pseudo_hdfs4.shared_cluster()
   finish = []
   finish = []
   try:
   try:
@@ -635,6 +639,10 @@ def test_view_snappy_compressed():
 
 
 @attr('requires_hadoop')
 @attr('requires_hadoop')
 def test_view_snappy_compressed_avro():
 def test_view_snappy_compressed_avro():
+  if not snappy_installed():
+    raise SkipTest
+  import snappy
+
   cluster = pseudo_hdfs4.shared_cluster()
   cluster = pseudo_hdfs4.shared_cluster()
   finish = []
   finish = []
   try:
   try: