#!/usr/bin/env python # # Copyright (c) 2011, Andres Moreira # 2011, Felipe Cruz # 2012, JT Olds # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of the authors nor the # names of its contributors may be used to endorse or promote products # derived from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL ANDRES MOREIRA BE LIABLE FOR ANY DIRECT, # INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # """python-snappy Python library for the snappy compression library from Google. Expected usage like: import snappy compressed = snappy.compress("some data") assert "some data" == snappy.uncompress(compressed) """ import struct from _snappy import CompressError, CompressedLengthError, \ InvalidCompressedInputError, UncompressError, \ compress, decompress, isValidCompressed, uncompress, \ _crc32c _CHUNK_MAX = 65536 _STREAM_TO_STREAM_BLOCK_SIZE = _CHUNK_MAX _STREAM_IDENTIFIER = b"sNaPpY" _COMPRESSED_CHUNK = 0x00 _UNCOMPRESSED_CHUNK = 0x01 _IDENTIFIER_CHUNK = 0xff _RESERVED_UNSKIPPABLE = (0x02, 0x80) # chunk ranges are [inclusive, exclusive) _RESERVED_SKIPPABLE = (0x80, 0xff) # the minimum percent of bytes compression must save to be enabled in automatic # mode _COMPRESSION_THRESHOLD = .125 def _masked_crc32c(data): # see the framing format specification crc = _crc32c(data) return (((crc >> 15) | (crc << 17)) + 0xa282ead8) & 0xffffffff _compress = compress _uncompress = uncompress class StreamCompressor(object): """This class implements the compressor-side of the proposed Snappy framing format, found at http://code.google.com/p/snappy/source/browse/trunk/framing_format.txt ?spec=svn68&r=71 This class matches the interface found for the zlib module's compression objects (see zlib.compressobj), but also provides some additions, such as the snappy framing format's ability to intersperse uncompressed data. Keep in mind that this compressor object does no buffering for you to appropriately size chunks. Every call to StreamCompressor.compress results in a unique call to the underlying snappy compression method. """ __slots__ = ["_header_chunk_written"] def __init__(self): self._header_chunk_written = False def add_chunk(self, data, compress=None): """Add a chunk containing 'data', returning a string that is framed and (optionally, default) compressed. This data should be concatenated to the tail end of an existing Snappy stream. In the absence of any internal buffering, no data is left in any internal buffers, and so unlike zlib.compress, this method returns everything. If compress is None, compression is determined automatically based on snappy's performance. If compress == True, compression always happens, and if compress == False, compression never happens. """ if not self._header_chunk_written: self._header_chunk_written = True out = [struct.pack("> 8) chunk_type &= 0xff if not self._header_found: if (chunk_type != _IDENTIFIER_CHUNK or size != len(_STREAM_IDENTIFIER)): raise UncompressError("stream missing snappy identifier") self._header_found = True if (_RESERVED_UNSKIPPABLE[0] <= chunk_type and chunk_type < _RESERVED_UNSKIPPABLE[1]): raise UncompressError( "stream received unskippable but unknown chunk") if len(self._buf) < 4 + size: return b"".join(uncompressed) chunk, self._buf = self._buf[4:4 + size], self._buf[4 + size:] if chunk_type == _IDENTIFIER_CHUNK: if chunk != _STREAM_IDENTIFIER: raise UncompressError( "stream has invalid snappy identifier") continue if (_RESERVED_SKIPPABLE[0] <= chunk_type and chunk_type < _RESERVED_SKIPPABLE[1]): continue assert chunk_type in (_COMPRESSED_CHUNK, _UNCOMPRESSED_CHUNK) crc, chunk = chunk[:4], chunk[4:] if chunk_type == _COMPRESSED_CHUNK: chunk = _uncompress(chunk) if struct.pack(" 4 or "--help" in sys.argv or "-h" in sys.argv or sys.argv[1] not in ("-c", "-d")): print("Usage: python -m snappy <-c/-d> [src [dst]]") print(" -c compress") print(" -d decompress") print("output is stdout if dst is omitted or '-'") print("input is stdin if src and dst are omitted or src is '-'.") sys.exit(1) if len(sys.argv) >= 4 and sys.argv[3] != "-": dst = open(sys.argv[3], "wb") elif hasattr(sys.stdout, 'buffer'): dst = sys.stdout.buffer else: dst = sys.stdout if len(sys.argv) >= 3 and sys.argv[2] != "-": src = open(sys.argv[2], "rb") elif hasattr(sys.stdin, "buffer"): src = sys.stdin.buffer else: src = sys.stdin if sys.argv[1] == "-c": method = stream_compress else: method = stream_decompress method(src, dst) if __name__ == "__main__": cmdline_main()