|
|
@@ -0,0 +1,305 @@
|
|
|
+#!/usr/bin/env python
|
|
|
+#
|
|
|
+# Copyright (c) 2011, Andres Moreira <andres@andresmoreira.com>
|
|
|
+# 2011, Felipe Cruz <felipecruz@loogica.net>
|
|
|
+# 2012, JT Olds <jt@spacemonkey.com>
|
|
|
+# All rights reserved.
|
|
|
+#
|
|
|
+# Redistribution and use in source and binary forms, with or without
|
|
|
+# modification, are permitted provided that the following conditions are met:
|
|
|
+# * Redistributions of source code must retain the above copyright
|
|
|
+# notice, this list of conditions and the following disclaimer.
|
|
|
+# * Redistributions in binary form must reproduce the above copyright
|
|
|
+# notice, this list of conditions and the following disclaimer in the
|
|
|
+# documentation and/or other materials provided with the distribution.
|
|
|
+# * Neither the name of the authors nor the
|
|
|
+# names of its contributors may be used to endorse or promote products
|
|
|
+# derived from this software without specific prior written permission.
|
|
|
+#
|
|
|
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
|
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
|
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
|
+# ARE DISCLAIMED. IN NO EVENT SHALL ANDRES MOREIRA BE LIABLE FOR ANY DIRECT,
|
|
|
+# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
|
|
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
|
|
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
|
|
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
|
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
|
|
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
+#
|
|
|
+
|
|
|
+"""python-snappy
|
|
|
+
|
|
|
+Python library for the snappy compression library from Google.
|
|
|
+Expected usage like:
|
|
|
+
|
|
|
+ import snappy
|
|
|
+
|
|
|
+ compressed = snappy.compress("some data")
|
|
|
+ assert "some data" == snappy.uncompress(compressed)
|
|
|
+
|
|
|
+"""
|
|
|
+
|
|
|
+import struct
|
|
|
+
|
|
|
+from _snappy import CompressError, CompressedLengthError, \
|
|
|
+ InvalidCompressedInputError, UncompressError, \
|
|
|
+ compress, decompress, isValidCompressed, uncompress, \
|
|
|
+ _crc32c
|
|
|
+
|
|
|
+_CHUNK_MAX = 65536
|
|
|
+_STREAM_TO_STREAM_BLOCK_SIZE = _CHUNK_MAX
|
|
|
+_STREAM_IDENTIFIER = b"sNaPpY"
|
|
|
+_COMPRESSED_CHUNK = 0x00
|
|
|
+_UNCOMPRESSED_CHUNK = 0x01
|
|
|
+_IDENTIFIER_CHUNK = 0xff
|
|
|
+_RESERVED_UNSKIPPABLE = (0x02, 0x80) # chunk ranges are [inclusive, exclusive)
|
|
|
+_RESERVED_SKIPPABLE = (0x80, 0xff)
|
|
|
+
|
|
|
+# the minimum percent of bytes compression must save to be enabled in automatic
|
|
|
+# mode
|
|
|
+_COMPRESSION_THRESHOLD = .125
|
|
|
+
|
|
|
+def _masked_crc32c(data):
|
|
|
+ # see the framing format specification
|
|
|
+ crc = _crc32c(data)
|
|
|
+ return (((crc >> 15) | (crc << 17)) + 0xa282ead8) & 0xffffffff
|
|
|
+
|
|
|
+_compress = compress
|
|
|
+_uncompress = uncompress
|
|
|
+
|
|
|
+
|
|
|
+class StreamCompressor(object):
|
|
|
+
|
|
|
+ """This class implements the compressor-side of the proposed Snappy framing
|
|
|
+ format, found at
|
|
|
+
|
|
|
+ http://code.google.com/p/snappy/source/browse/trunk/framing_format.txt
|
|
|
+ ?spec=svn68&r=71
|
|
|
+
|
|
|
+ This class matches the interface found for the zlib module's compression
|
|
|
+ objects (see zlib.compressobj), but also provides some additions, such as
|
|
|
+ the snappy framing format's ability to intersperse uncompressed data.
|
|
|
+
|
|
|
+ Keep in mind that this compressor object does no buffering for you to
|
|
|
+ appropriately size chunks. Every call to StreamCompressor.compress results
|
|
|
+ in a unique call to the underlying snappy compression method.
|
|
|
+ """
|
|
|
+
|
|
|
+ __slots__ = ["_header_chunk_written"]
|
|
|
+
|
|
|
+ def __init__(self):
|
|
|
+ self._header_chunk_written = False
|
|
|
+
|
|
|
+ def add_chunk(self, data, compress=None):
|
|
|
+ """Add a chunk containing 'data', returning a string that is framed and
|
|
|
+ (optionally, default) compressed. This data should be concatenated to
|
|
|
+ the tail end of an existing Snappy stream. In the absence of any
|
|
|
+ internal buffering, no data is left in any internal buffers, and so
|
|
|
+ unlike zlib.compress, this method returns everything.
|
|
|
+
|
|
|
+ If compress is None, compression is determined automatically based on
|
|
|
+ snappy's performance. If compress == True, compression always happens,
|
|
|
+ and if compress == False, compression never happens.
|
|
|
+ """
|
|
|
+ if not self._header_chunk_written:
|
|
|
+ self._header_chunk_written = True
|
|
|
+ out = [struct.pack("<L", _IDENTIFIER_CHUNK +
|
|
|
+ (len(_STREAM_IDENTIFIER) << 8)),
|
|
|
+ _STREAM_IDENTIFIER]
|
|
|
+ else:
|
|
|
+ out = []
|
|
|
+ for i in range(0, len(data), _CHUNK_MAX):
|
|
|
+ chunk = data[i:i + _CHUNK_MAX]
|
|
|
+ crc = _masked_crc32c(chunk)
|
|
|
+ if compress is None:
|
|
|
+ compressed_chunk = _compress(chunk)
|
|
|
+ if (len(compressed_chunk) <=
|
|
|
+ (1 - _COMPRESSION_THRESHOLD) * len(chunk)):
|
|
|
+ chunk = compressed_chunk
|
|
|
+ chunk_type = _COMPRESSED_CHUNK
|
|
|
+ else:
|
|
|
+ chunk_type = _UNCOMPRESSED_CHUNK
|
|
|
+ compressed_chunk = None
|
|
|
+ elif compress:
|
|
|
+ chunk = _compress(chunk)
|
|
|
+ chunk_type = _COMPRESSED_CHUNK
|
|
|
+ else:
|
|
|
+ chunk_type = _UNCOMPRESSED_CHUNK
|
|
|
+ out.append(struct.pack("<LL", chunk_type + ((len(chunk) + 4) << 8),
|
|
|
+ crc))
|
|
|
+ out.append(chunk)
|
|
|
+ return b"".join(out)
|
|
|
+
|
|
|
+ def compress(self, data):
|
|
|
+ """This method is simply an alias for compatibility with zlib
|
|
|
+ compressobj's compress method.
|
|
|
+ """
|
|
|
+ return self.add_chunk(data)
|
|
|
+
|
|
|
+ def flush(self, mode=None):
|
|
|
+ """This method does nothing and only exists for compatibility with
|
|
|
+ the zlib compressobj
|
|
|
+ """
|
|
|
+ pass
|
|
|
+
|
|
|
+ def copy(self):
|
|
|
+ """This method exists for compatibility with the zlib compressobj.
|
|
|
+ """
|
|
|
+ copy = StreamCompressor()
|
|
|
+ copy._header_chunk_written = self._header_chunk_written
|
|
|
+ return copy
|
|
|
+
|
|
|
+
|
|
|
+class StreamDecompressor(object):
|
|
|
+
|
|
|
+ """This class implements the decompressor-side of the proposed Snappy
|
|
|
+ framing format, found at
|
|
|
+
|
|
|
+ http://code.google.com/p/snappy/source/browse/trunk/framing_format.txt
|
|
|
+ ?spec=svn68&r=71
|
|
|
+
|
|
|
+ This class matches a subset of the interface found for the zlib module's
|
|
|
+ decompression objects (see zlib.decompressobj). Specifically, it currently
|
|
|
+ implements the decompress method without the max_length option, the flush
|
|
|
+ method without the length option, and the copy method.
|
|
|
+ """
|
|
|
+
|
|
|
+ __slots__ = ["_buf", "_header_found"]
|
|
|
+
|
|
|
+ def __init__(self):
|
|
|
+ self._buf = b""
|
|
|
+ self._header_found = False
|
|
|
+
|
|
|
+ def decompress(self, data):
|
|
|
+ """Decompress 'data', returning a string containing the uncompressed
|
|
|
+ data corresponding to at least part of the data in string. This data
|
|
|
+ should be concatenated to the output produced by any preceding calls to
|
|
|
+ the decompress() method. Some of the input data may be preserved in
|
|
|
+ internal buffers for later processing.
|
|
|
+ """
|
|
|
+ self._buf += data
|
|
|
+ uncompressed = []
|
|
|
+ while True:
|
|
|
+ if len(self._buf) < 4:
|
|
|
+ return b"".join(uncompressed)
|
|
|
+ chunk_type = struct.unpack("<L", self._buf[:4])[0]
|
|
|
+ size = (chunk_type >> 8)
|
|
|
+ chunk_type &= 0xff
|
|
|
+ if not self._header_found:
|
|
|
+ if (chunk_type != _IDENTIFIER_CHUNK or
|
|
|
+ size != len(_STREAM_IDENTIFIER)):
|
|
|
+ raise UncompressError("stream missing snappy identifier")
|
|
|
+ self._header_found = True
|
|
|
+ if (_RESERVED_UNSKIPPABLE[0] <= chunk_type and
|
|
|
+ chunk_type < _RESERVED_UNSKIPPABLE[1]):
|
|
|
+ raise UncompressError(
|
|
|
+ "stream received unskippable but unknown chunk")
|
|
|
+ if len(self._buf) < 4 + size:
|
|
|
+ return b"".join(uncompressed)
|
|
|
+ chunk, self._buf = self._buf[4:4 + size], self._buf[4 + size:]
|
|
|
+ if chunk_type == _IDENTIFIER_CHUNK:
|
|
|
+ if chunk != _STREAM_IDENTIFIER:
|
|
|
+ raise UncompressError(
|
|
|
+ "stream has invalid snappy identifier")
|
|
|
+ continue
|
|
|
+ if (_RESERVED_SKIPPABLE[0] <= chunk_type and
|
|
|
+ chunk_type < _RESERVED_SKIPPABLE[1]):
|
|
|
+ continue
|
|
|
+ assert chunk_type in (_COMPRESSED_CHUNK, _UNCOMPRESSED_CHUNK)
|
|
|
+ crc, chunk = chunk[:4], chunk[4:]
|
|
|
+ if chunk_type == _COMPRESSED_CHUNK:
|
|
|
+ chunk = _uncompress(chunk)
|
|
|
+ if struct.pack("<L", _masked_crc32c(chunk)) != crc:
|
|
|
+ raise UncompressError("crc mismatch")
|
|
|
+ uncompressed.append(chunk)
|
|
|
+
|
|
|
+ def flush(self):
|
|
|
+ """All pending input is processed, and a string containing the
|
|
|
+ remaining uncompressed output is returned. After calling flush(), the
|
|
|
+ decompress() method cannot be called again; the only realistic action
|
|
|
+ is to delete the object.
|
|
|
+ """
|
|
|
+ if self._buf != b"":
|
|
|
+ raise UncompressError("chunk truncated")
|
|
|
+ return b""
|
|
|
+
|
|
|
+ def copy(self):
|
|
|
+ """Returns a copy of the decompression object. This can be used to save
|
|
|
+ the state of the decompressor midway through the data stream in order
|
|
|
+ to speed up random seeks into the stream at a future point.
|
|
|
+ """
|
|
|
+ copy = StreamDecompressor()
|
|
|
+ copy._buf, copy._header_found = self._buf, self._header_found
|
|
|
+ return copy
|
|
|
+
|
|
|
+
|
|
|
+def stream_compress(src, dst, blocksize=_STREAM_TO_STREAM_BLOCK_SIZE):
|
|
|
+ """Takes an incoming file-like object and an outgoing file-like object,
|
|
|
+ reads data from src, compresses it, and writes it to dst. 'src' should
|
|
|
+ support the read method, and 'dst' should support the write method.
|
|
|
+
|
|
|
+ The default blocksize is good for almost every scenario.
|
|
|
+ """
|
|
|
+ compressor = StreamCompressor()
|
|
|
+ while True:
|
|
|
+ buf = src.read(blocksize)
|
|
|
+ if not buf: break
|
|
|
+ buf = compressor.add_chunk(buf)
|
|
|
+ if buf: dst.write(buf)
|
|
|
+
|
|
|
+
|
|
|
+def stream_decompress(src, dst, blocksize=_STREAM_TO_STREAM_BLOCK_SIZE):
|
|
|
+ """Takes an incoming file-like object and an outgoing file-like object,
|
|
|
+ reads data from src, decompresses it, and writes it to dst. 'src' should
|
|
|
+ support the read method, and 'dst' should support the write method.
|
|
|
+
|
|
|
+ The default blocksize is good for almost every scenario.
|
|
|
+ """
|
|
|
+ decompressor = StreamDecompressor()
|
|
|
+ while True:
|
|
|
+ buf = src.read(blocksize)
|
|
|
+ if not buf: break
|
|
|
+ buf = decompressor.decompress(buf)
|
|
|
+ if buf: dst.write(buf)
|
|
|
+ decompressor.flush() # makes sure the stream ended well
|
|
|
+
|
|
|
+
|
|
|
+def cmdline_main():
|
|
|
+ """This method is what is run when invoking snappy via the commandline.
|
|
|
+ Try python -m snappy --help
|
|
|
+ """
|
|
|
+ import sys
|
|
|
+ if (len(sys.argv) < 2 or len(sys.argv) > 4 or "--help" in sys.argv or
|
|
|
+ "-h" in sys.argv or sys.argv[1] not in ("-c", "-d")):
|
|
|
+ print("Usage: python -m snappy <-c/-d> [src [dst]]")
|
|
|
+ print(" -c compress")
|
|
|
+ print(" -d decompress")
|
|
|
+ print("output is stdout if dst is omitted or '-'")
|
|
|
+ print("input is stdin if src and dst are omitted or src is '-'.")
|
|
|
+ sys.exit(1)
|
|
|
+
|
|
|
+ if len(sys.argv) >= 4 and sys.argv[3] != "-":
|
|
|
+ dst = open(sys.argv[3], "wb")
|
|
|
+ elif hasattr(sys.stdout, 'buffer'):
|
|
|
+ dst = sys.stdout.buffer
|
|
|
+ else:
|
|
|
+ dst = sys.stdout
|
|
|
+
|
|
|
+ if len(sys.argv) >= 3 and sys.argv[2] != "-":
|
|
|
+ src = open(sys.argv[2], "rb")
|
|
|
+ elif hasattr(sys.stdin, "buffer"):
|
|
|
+ src = sys.stdin.buffer
|
|
|
+ else:
|
|
|
+ src = sys.stdin
|
|
|
+
|
|
|
+ if sys.argv[1] == "-c":
|
|
|
+ method = stream_compress
|
|
|
+ else:
|
|
|
+ method = stream_decompress
|
|
|
+
|
|
|
+ method(src, dst)
|
|
|
+
|
|
|
+
|
|
|
+if __name__ == "__main__":
|
|
|
+ cmdline_main()
|