You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
2363 lines
90 KiB
2363 lines
90 KiB
#!python3
|
|
"""Program to dump contents of Brotli compressed files showing the compression format.
|
|
Jurjen N.E. Bos, 2016.
|
|
I found the following issues with the Brotli format:
|
|
- The distance alphabet has size 16+(48<<POSTFIX),
|
|
but the last symbols are useless.
|
|
It could be lowered to 16+(44-POSTFIX<<POSTFIX), and this could matter.
|
|
- The block type code is useless if NBLTYPES==2, you would only need 1 symbol
|
|
anyway, so why don't you just switch to "the other" type?
|
|
"""
|
|
import struct
|
|
from operator import itemgetter, methodcaller
|
|
from itertools import accumulate, repeat
|
|
from collections import defaultdict, deque
|
|
from functools import partial
|
|
|
|
class InvalidStream(Exception): pass
|
|
#lookup table
|
|
L, I, D = "literal", "insert©", "distance"
|
|
pL, pI, pD = 'P'+L, 'P'+I, 'P'+D
|
|
|
|
def outputCharFormatter(c):
|
|
"""Show character in readable format
|
|
"""
|
|
#TODO 2: allow hex only output
|
|
if 32<c<127: return chr(c)
|
|
elif c==10: return '\\n'
|
|
elif c==13: return '\\r'
|
|
elif c==32: return '" "'
|
|
else: return '\\x{:02x}'.format(c)
|
|
|
|
def outputFormatter(s):
|
|
"""Show string or char.
|
|
"""
|
|
result = ''
|
|
def formatSubString(s):
|
|
for c in s:
|
|
if c==32: yield ' '
|
|
else: yield outputCharFormatter(c)
|
|
if len(result)<200: return ''.join(formatSubString(s))
|
|
else:
|
|
return ''.join(formatSubString(s[:100]))+'...'+ \
|
|
''.join(formatSubString(s[-100:]))
|
|
|
|
|
|
class BitStream:
|
|
"""Represent a bytes object. Can read bits and prefix codes the way
|
|
Brotli does.
|
|
"""
|
|
def __init__(self, byteString):
|
|
self.data = byteString
|
|
#position in bits: byte pos is pos>>3, bit pos is pos&7
|
|
self.pos = 0
|
|
|
|
def __repr__(self):
|
|
"""Representation
|
|
>>> olleke
|
|
BitStream(pos=0:0)
|
|
"""
|
|
return "BitStream(pos={:x}:{})".format(self.pos>>3, self.pos&7)
|
|
|
|
def read(self, n):
|
|
"""Read n bits from the stream and return as an integer.
|
|
Produces zero bits beyond the stream.
|
|
>>> olleke.data[0]==27
|
|
True
|
|
>>> olleke.read(5)
|
|
27
|
|
|
|
>>> olleke
|
|
BitStream(pos=0:5)
|
|
"""
|
|
value = self.peek(n)
|
|
self.pos += n
|
|
if self.pos>len(self.data)*8:
|
|
raise ValueError('Read past end of stream')
|
|
return value
|
|
|
|
def peek(self, n):
|
|
"""Peek an n bit integer from the stream without updating the pointer.
|
|
It is not an error to read beyond the end of the stream.
|
|
>>> olleke.data[:2]==b'\x1b\x2e' and 0x2e1b==11803
|
|
True
|
|
>>> olleke.peek(15)
|
|
11803
|
|
>>> hex(olleke.peek(32))
|
|
'0x2e1b'
|
|
"""
|
|
#read bytes that contain the data: self.data[self.pos>>3:self.pos+n+7>>3]
|
|
#convert to int: int.from_bytes(..., 'little')
|
|
#shift out the bits from the first byte: >>(self.pos&7)
|
|
#mask unwanted bits: & (1<<n)-1
|
|
return int.from_bytes(
|
|
self.data[self.pos>>3:self.pos+n+7>>3],
|
|
'little')>>(self.pos&7) & (1<<n)-1
|
|
|
|
def readBytes(self, n):
|
|
"""Read n bytes from the stream on a byte boundary.
|
|
"""
|
|
if self.pos&7: raise ValueError('readBytes: need byte boundary')
|
|
result = self.data[self.pos>>3:(self.pos>>3)+n]
|
|
self.pos += 8*n
|
|
return result
|
|
|
|
#-----------------------Symbol-------------------------------------------
|
|
class Symbol:
|
|
"""A symbol in a code.
|
|
Refers back to the code that contains it.
|
|
Index is the place in the alphabet of the symbol.
|
|
"""
|
|
__slots__ = 'code', 'index'
|
|
def __init__(self, code, value):
|
|
self.code = code
|
|
self.index = value
|
|
|
|
def __repr__(self):
|
|
return 'Symbol({}, {})'.format(self.code.name, self.index)
|
|
|
|
def __len__(self):
|
|
"""Number of bits in the prefix notation of this symbol
|
|
"""
|
|
return self.code.length(self.index)
|
|
|
|
def __int__(self):
|
|
return self.index
|
|
|
|
#these routines call equivalent routine in Code class
|
|
def bitPattern(self):
|
|
"""Value of the symbol in the stream
|
|
"""
|
|
return self.code.bitPattern(self.index)
|
|
|
|
def extraBits(self):
|
|
"""Number of extra bits to read for this symbol
|
|
"""
|
|
return self.code.extraBits(self.index)
|
|
|
|
def __str__(self):
|
|
"""Short descriptor of the symbol without extra bits.
|
|
"""
|
|
return self.code.mnemonic(self.index)
|
|
|
|
#requiring optional extra bits, if self.code supports them
|
|
def value(self, extra=None):
|
|
"""The value used for processing. Can be a tuple.
|
|
with optional extra bits
|
|
"""
|
|
if isinstance(self.code, WithExtra):
|
|
if not 0<=extra<1<<self.extraBits():
|
|
raise ValueError("value: extra value doesn't fit in extraBits")
|
|
return self.code.value(self.index, extra)
|
|
if extra is not None:
|
|
raise ValueError('value: no extra bits for this code')
|
|
return self.code.value(self.index)
|
|
|
|
def explanation(self, extra=None):
|
|
"""Long explanation of the value from the numeric value
|
|
with optional extra bits
|
|
Used by Layout.verboseRead when printing the value
|
|
"""
|
|
if isinstance(self.code, WithExtra):
|
|
return self.code.callback(self, extra)
|
|
return self.code.callback(self)
|
|
|
|
#========================Code definitions==================================
|
|
class RangeDecoder:
|
|
"""A decoder for the Code class that assumes the symbols
|
|
are encoded consecutively in binary.
|
|
It all depends on the "alphabetSize" property.
|
|
The range runs from 0 to alphabetSize-1.
|
|
This is the default decoder.
|
|
"""
|
|
def __init__(self, *, alphabetSize=None, bitLength=None, **args):
|
|
if bitLength is not None: alphabetSize = 1<<bitLength
|
|
if alphabetSize is not None:
|
|
self.alphabetSize = alphabetSize
|
|
self.maxLength = (alphabetSize-1).bit_length()
|
|
|
|
def __len__(self):
|
|
return self.alphabetSize
|
|
|
|
def __iter__(self):
|
|
"""Produce all symbols.
|
|
"""
|
|
return map(partial(Symbol, self), range(len(self)))
|
|
|
|
def __getitem__(self, index):
|
|
if index>=self.alphabetSize: raise ValueError('index out of range')
|
|
return Symbol(self, index)
|
|
|
|
def bitPattern(self, index):
|
|
return '{:0{}b}'.format(index, self.maxLength)
|
|
|
|
def length(self, index):
|
|
"""Encoding length of given symbol.
|
|
Does not depend on index in this case.
|
|
"""
|
|
return self.maxLength
|
|
|
|
def decodePeek(self, data):
|
|
"""Find which symbol index matches the given data (from peek, as a number)
|
|
and return the number of bits decoded.
|
|
Can also be used to figure out length of a symbol.
|
|
"""
|
|
return self.maxLength, Symbol(self, data&(1<<self.maxLength)-1)
|
|
|
|
class PrefixDecoder:
|
|
"""A decoder for the Code class that uses a prefix code.
|
|
The code is determined by encoding:
|
|
encode[p] gives the index corresponding to bit pattern p.
|
|
Used setDecode(decodeTable) to switch the decoder from the default
|
|
to a prefix decoder, or pass decodeTable at init.
|
|
You can also use setLength(lengthTable)
|
|
to define the encoding from the lengths.
|
|
The set of symbol values does not need to be consecutive.
|
|
"""
|
|
def __init__(self, *, decodeTable=None, **args):
|
|
if decodeTable is not None: self.setDecode(decodeTable)
|
|
|
|
def __len__(self):
|
|
return len(self.decodeTable)
|
|
|
|
def __iter__(self):
|
|
def revBits(index):
|
|
return self.bitPattern(index)[::-1]
|
|
return (
|
|
Symbol(self, index)
|
|
for index in sorted(self.decodeTable.values(), key=revBits)
|
|
)
|
|
|
|
def __getitem__(self, index):
|
|
if index not in self.lengthTable:
|
|
raise ValueError('No symbol {}[{}]'.format(
|
|
self.__class__.__name__, index))
|
|
return Symbol(self, index)
|
|
|
|
def bitPattern(self, index):
|
|
bits = next(b for (b,s) in self.decodeTable.items() if s==index)
|
|
return '{:0{}b}'.format(bits, self.length(index))
|
|
|
|
def length(self, index):
|
|
"""Encoding length of given symbol.
|
|
"""
|
|
return self.lengthTable[index]
|
|
|
|
def decodePeek(self, data):
|
|
"""Find which symbol index matches the given data (from peek, as a number)
|
|
and return the number of bits decoded.
|
|
Can also be used to figure out length of a symbol.
|
|
"""
|
|
#do binary search for word length
|
|
#invariant: lo<=length<=hi
|
|
lo, hi = self.minLength, self.maxLength
|
|
while lo<=hi:
|
|
mid = lo+hi>>1
|
|
#note lo<=mid<hi at this point
|
|
mask = (1<<mid)-1
|
|
#lets see what happens if we guess length is mid
|
|
try: index = self.decodeTable[data&mask]
|
|
except KeyError:
|
|
#too many bits specified, reduce estimated length
|
|
hi = mid-1
|
|
continue
|
|
#we found a symbol, but there could be a longer match
|
|
symbolLength = self.lengthTable[index]
|
|
if symbolLength<=mid:
|
|
#all bits match, symbol must be right
|
|
return symbolLength, Symbol(self, index)
|
|
#there must be more bits to match
|
|
lo = mid+1
|
|
return lo, Symbol(self, index)
|
|
|
|
#routine to set up the tables
|
|
def setDecode(self, decodeTable):
|
|
"""Store decodeTable,
|
|
and compute lengthTable, minLength, maxLength from encodings.
|
|
"""
|
|
self.decodeTable = decodeTable
|
|
#set of symbols with unknown length
|
|
todo = set(decodeTable)
|
|
#bit size under investigation
|
|
maskLength = 0
|
|
lengthTable = {}
|
|
while todo:
|
|
mask = (1<<maskLength)-1
|
|
#split the encodings that we didn't find yet using b bits
|
|
splitSymbols = defaultdict(list)
|
|
for s in todo: splitSymbols[s&mask].append(s)
|
|
#unique encodings have a length of maskLength bits
|
|
#set length, and remove from todo list
|
|
for s,subset in splitSymbols.items():
|
|
if len(subset)==1:
|
|
lengthTable[self.decodeTable[s]] = maskLength
|
|
todo.remove(s)
|
|
#now investigate with longer mask
|
|
maskLength +=1
|
|
#save result
|
|
self.lengthTable = lengthTable
|
|
self.minLength = min(lengthTable.values())
|
|
self.maxLength = max(lengthTable.values())
|
|
self.switchToPrefix()
|
|
|
|
def setLength(self, lengthTable):
|
|
"""Given the bit pattern lengths for symbols given in lengthTable,
|
|
set decodeTable, minLength, maxLength
|
|
"""
|
|
self.lengthTable = lengthTable
|
|
self.minLength = min(lengthTable.values())
|
|
self.maxLength = max(lengthTable.values())
|
|
#compute the backwards codes first; then reverse them
|
|
#compute (backwards) first code for every separate lengths
|
|
nextCodes = []
|
|
#build codes for each length, from right to left
|
|
code = 0
|
|
for bits in range(self.maxLength+1):
|
|
code <<= 1
|
|
nextCodes.append(code)
|
|
code += sum(x==bits for x in lengthTable.values())
|
|
self.decodeTable = {}
|
|
#count codes for each length, and store reversed in the table
|
|
for symbol in sorted(lengthTable):
|
|
bits = lengthTable[symbol]
|
|
bitpattern = '{:0{}b}'.format(nextCodes[bits], bits)
|
|
self.decodeTable[int(bitpattern[::-1], 2)] = symbol
|
|
nextCodes[bits] += 1
|
|
self.switchToPrefix()
|
|
|
|
def switchToPrefix(self):
|
|
"""This routine makes sure the prefix decoder is activated.
|
|
"""
|
|
self.mode = PrefixDecoder
|
|
|
|
class Code(RangeDecoder, PrefixDecoder):
|
|
"""An alphabet of symbols, that can be read from a stream.
|
|
If you use setDecode or setLength, you have a prefix code,
|
|
otherwise you have a range code.
|
|
Features:
|
|
code[index] produces symbol with given index
|
|
value(index): value of symbol
|
|
mnemonic(index): short description of symbol
|
|
explanation(index): show meaning of symbol, shown in Layout.verboseRead
|
|
iter(code): produce all symbols in some order
|
|
name: show as context in Layout.verboseRead
|
|
"""
|
|
name = '?'
|
|
#callback is a function that gets the symbol and the extra bits
|
|
#default callback calls explanation
|
|
def __init__(self, name=None, *, callback=None, description='', **args):
|
|
"""Don't forget to set either alphabetSize or decodeTable
|
|
"""
|
|
#set name when provided, otherwise take class variable
|
|
if name is not None: self.name = name
|
|
if callback is not None: self.callback = callback
|
|
self.description = description
|
|
#mode switch
|
|
if 'bitLength' in args or 'alphabetSize' in args:
|
|
self.mode = RangeDecoder
|
|
RangeDecoder.__init__(self, **args)
|
|
elif 'decodeTable' in args:
|
|
self.mode = PrefixDecoder
|
|
PrefixDecoder.__init__(self, **args)
|
|
else:
|
|
super().__init__(**args)
|
|
|
|
def __repr__(self):
|
|
return self.__class__.__name__+' '+self.name
|
|
|
|
#the routines that get switched between RangeDecoder and PrefixDecoder
|
|
def __len__(self): return self.mode.__len__(self)
|
|
def __iter__(self): return self.mode.__iter__(self)
|
|
def __getitem__(self, index): return self.mode.__getitem__(self, index)
|
|
def bitPattern(self, index): return self.mode.bitPattern(self, index)
|
|
def length(self, index): return self.mode.length(self, index)
|
|
def decodePeek(self, data): return self.mode.decodePeek(self, data)
|
|
#general routines
|
|
def value(self, index, extra=None):
|
|
"""Get value of symbol for computations.
|
|
Override where needed.
|
|
"""
|
|
if extra is not None:
|
|
raise ValueError('value: no extra for this symbol')
|
|
return index
|
|
|
|
def mnemonic(self, index):
|
|
"""Give mnemonic of symbol.
|
|
Override where needed.
|
|
"""
|
|
return str(self.value(index))
|
|
|
|
def callback(self, symbol):
|
|
return self.explanation(symbol.index)
|
|
|
|
def explanation(self, index):
|
|
"""Long explanation of the value from the numeric value
|
|
This is a default routine.
|
|
You can customize in three ways:
|
|
- set description to add some text
|
|
- override to get more control
|
|
- set callback to make it dependent on you local variables
|
|
"""
|
|
value = self.value(index)
|
|
return '{0}{1}: {2}'.format(
|
|
self.description and self.description+': ',
|
|
self.bitPattern(index),
|
|
value,
|
|
)
|
|
|
|
def extraBits(self, index):
|
|
return 0
|
|
|
|
#Routines that use the decode interface
|
|
def showCode(self, width=80):
|
|
"""Show all words of the code in a nice format.
|
|
"""
|
|
#make table of all symbols with binary strings
|
|
symbolStrings = [
|
|
(self.bitPattern(s.index), self.mnemonic(s.index))
|
|
for s in self
|
|
]
|
|
#determine column widths the way Lisp programmers do it
|
|
leftColWidth, rightColWidth = map(max, map(
|
|
map,
|
|
repeat(len),
|
|
zip(*symbolStrings)
|
|
))
|
|
colwidth = leftColWidth+rightColWidth
|
|
columns = 81//(colwidth+2)
|
|
rows = -(-len(symbolStrings)//columns)
|
|
def justify(bs):
|
|
b,s = bs
|
|
return b.rjust(leftColWidth)+':'+s.ljust(rightColWidth)
|
|
for i in range(rows):
|
|
print(' '.join(map(justify, symbolStrings[i::rows])).rstrip())
|
|
|
|
def readTuple(self, stream):
|
|
"""Read symbol from stream. Returns symbol, length.
|
|
"""
|
|
length, symbol = self.decodePeek(stream.peek(self.maxLength))
|
|
stream.pos += length
|
|
return length, symbol
|
|
|
|
def readTupleAndExtra(self, stream):
|
|
return self.readTuple(stream)+(0, None)
|
|
|
|
class WithExtra(Code):
|
|
"""Extension for Code so that symbol may have extra bits associated.
|
|
If you supply an extraTable, you can use extraBits
|
|
You can define an extraTable,
|
|
which allows to call extraBits to get the number of extraBits.
|
|
Otherwise, you can supply extraBits yourself.
|
|
Routine readTupleAndExtra now reads the extra bits too.
|
|
Value probably needs to be overridden; see Enumerator.
|
|
Note: this does not give you an decodeTable.
|
|
"""
|
|
#redefine these if you don't want to use an extraTable
|
|
def extraBits(self, index):
|
|
"""Get the number of extra bits for this symbol.
|
|
"""
|
|
return self.extraTable[index]
|
|
|
|
def mnemonic(self, index):
|
|
"""This value must be independent of extra.
|
|
"""
|
|
return str(index)
|
|
|
|
def readTupleAndExtra(self, stream):
|
|
"""Read symbol and extrabits from stream.
|
|
Returns symbol length, symbol, extraBits, extra
|
|
>>> olleke.pos = 6
|
|
>>> MetablockLengthAlphabet().readTupleAndExtra(olleke)
|
|
(2, Symbol(MLEN, 4), 16, 46)
|
|
"""
|
|
length, symbol = self.decodePeek(stream.peek(self.maxLength))
|
|
stream.pos += length
|
|
extraBits = self.extraBits(symbol.index)
|
|
return length, symbol, extraBits, stream.read(extraBits)
|
|
|
|
def explanation(self, index, extra=None):
|
|
"""Expanded version of Code.explanation supporting extra bits.
|
|
If you don't supply extra, it is not mentioned.
|
|
"""
|
|
extraBits = 0 if extra is None else self.extraBits(index)
|
|
if not hasattr(self, 'extraTable'):
|
|
formatString = '{0}{3}'
|
|
lo = hi = value = self.value(index, extra)
|
|
elif extraBits==0:
|
|
formatString = '{0}{2}: {3}'
|
|
lo, hi = self.span(index)
|
|
value = lo
|
|
else:
|
|
formatString = '{0}{1} {2}: {3}-{4}; {3}+{5}={6}'
|
|
lo, hi = self.span(index)
|
|
value = lo+extra
|
|
return formatString.format(
|
|
self.description and self.description+': ',
|
|
'x'*extraBits,
|
|
self.bitPattern(index),
|
|
lo, hi,
|
|
extra,
|
|
value,
|
|
)
|
|
|
|
def callback(self, symbol, extra):
|
|
return self.explanation(symbol.index, extra)
|
|
|
|
class BoolCode(Code):
|
|
"""Same as Code(bitLength=1), but shows a boolean.
|
|
"""
|
|
def __init__(self, name=None, **args):
|
|
super().__init__(name, bitLength=1, **args)
|
|
|
|
def value(self, index, extra=None):
|
|
return bool(super().value(index, extra))
|
|
|
|
class Enumerator(WithExtra):
|
|
"""Code that is defined by the ExtraTable.
|
|
extraTable is a class variable that contains
|
|
the extraBits of the symbols from 0
|
|
value0 contains the value of symbol 0
|
|
encodings is not neccessary, but allowed.
|
|
Note: place for FixedCode to make sure extraBits works
|
|
"""
|
|
def __init__(self, name=None, **args):
|
|
#if there is no decodeTable to determine length, compute it ourselves
|
|
if 'decodeTable' not in args:
|
|
args['alphabetSize'] = len(self.extraTable)
|
|
super().__init__(name, **args)
|
|
|
|
def __len__(self):
|
|
return len(self.extraTable)
|
|
|
|
def __getitem__(self, index):
|
|
"""Faster than PrefixDecoder
|
|
"""
|
|
if index>=len(self.extraTable):
|
|
raise ValueError("No symbol {}[{}]".format(
|
|
self.__class__.__name__, index))
|
|
return Symbol(self, index)
|
|
|
|
def value(self, index, extra):
|
|
"""Override if you don't define value0 and extraTable
|
|
"""
|
|
lower, upper = self.span(index)
|
|
value = lower+(extra or 0)
|
|
if value>upper:
|
|
raise ValueError('value: extra out of range')
|
|
return value
|
|
|
|
def span(self, index):
|
|
"""Give the range of possible values in a tuple
|
|
Useful for mnemonic and explanation
|
|
"""
|
|
lower = self.value0+sum(1<<x for x in self.extraTable[:index])
|
|
upper = lower+(1<<self.extraTable[index])
|
|
return lower, upper-1
|
|
|
|
#======================Code subclasses======================================
|
|
#Alphabets used in the metablock header----------------------------------
|
|
#For prefix codes
|
|
class PrefixCodeHeader(WithExtra):
|
|
"""Header of prefix codes.
|
|
"""
|
|
def __init__(self, codename):
|
|
super().__init__('PFX', bitLength=2)
|
|
#this is the name of the code that it describes
|
|
self.codename = codename
|
|
|
|
def extraBits(self, index):
|
|
return 2 if index==1 else 0
|
|
|
|
def value(self, index, extra):
|
|
"""Returns ('Simple', #codewords) or ('Complex', HSKIP)
|
|
"""
|
|
if index==1:
|
|
if extra>3:
|
|
raise ValueError('value: extra out of range')
|
|
return 'Simple', extra+1
|
|
if extra:
|
|
raise ValueError('value: extra out of range')
|
|
return 'Complex', index
|
|
|
|
def explanation(self, index, extra):
|
|
if index==1:
|
|
return '{} is simple with {} code word{}'.format(
|
|
self.codename, extra+1, 's' if extra else '')
|
|
lengths = [1, 2, 3, 4, 0, 5, 17, 6]
|
|
return '{} is complex with lengths {}...'.format(
|
|
self.codename,
|
|
','.join(
|
|
map(str, lengths[index:index+5]))
|
|
)
|
|
|
|
class TreeShapeAlhabet(BoolCode):
|
|
"""The bit used to indicate if four word code is "deep" or "wide"
|
|
"""
|
|
name = 'SHAPE'
|
|
def value(self, index):
|
|
return [(2,2,2,2), (1,2,3,3)][index]
|
|
|
|
def explanation(self, index):
|
|
return str(bool(index))+': lengths {},{},{},{}'.format(*self.value(index))
|
|
|
|
class LengthOfLengthAlphabet(Code):
|
|
"""For use in decoding complex code descriptors.
|
|
>>> lengthOfLengthAlphabet = LengthOfLengthAlphabet('')
|
|
>>> print(lengthOfLengthAlphabet[2])
|
|
coded with 2 bits
|
|
>>> len(lengthOfLengthAlphabet[0])
|
|
2
|
|
>>> [len(lengthOfLengthAlphabet[x]) for x in range(6)]
|
|
[2, 4, 3, 2, 2, 4]
|
|
>>> lengthOfLengthAlphabet.showCode()
|
|
00:skipped 01:coded with 4 bits 0111:coded with 1 bits
|
|
10:coded with 3 bits 011:coded with 2 bits 1111:coded with 5 bits
|
|
"""
|
|
decodeTable = {
|
|
0b00:0, 0b10:3,
|
|
0b0111:1, 0b01:4,
|
|
0b011:2, 0b1111:5,
|
|
}
|
|
|
|
def __init__(self, name=None, **args):
|
|
super().__init__(name, decodeTable=self.decodeTable, **args)
|
|
|
|
def mnemonic(self, index):
|
|
if index==0: return 'skipped'
|
|
return 'coded with {} bits'.format(index)
|
|
|
|
def explanation(self, index, extra=None):
|
|
return self.description+': '+self.mnemonic(index)
|
|
|
|
class LengthAlphabet(WithExtra):
|
|
"""Length of symbols
|
|
Used during construction of a code.
|
|
"""
|
|
def __init__(self, name):
|
|
super().__init__(name, alphabetSize=18)
|
|
|
|
def extraBits(self, index):
|
|
return {16:2, 17:3}.get(index, 0)
|
|
|
|
def mnemonic(self, index):
|
|
if index==0: return 'unused'
|
|
elif index==16: return 'rep xx'
|
|
elif index==17: return 'zero xxx'
|
|
else: return 'len {}'.format(index)
|
|
|
|
def explanation(self, index, extra):
|
|
return self.description.format(self[index], extra)
|
|
|
|
def value(self, index, extra):
|
|
#the caller got the length already, so extra is enough
|
|
return extra
|
|
|
|
#Stream header
|
|
class WindowSizeAlphabet(Code):
|
|
"""The alphabet used for window size in the stream header.
|
|
>>> WindowSizeAlphabet()[10].explanation()
|
|
'windowsize=(1<<10)-16=1008'
|
|
"""
|
|
decodeTable = {
|
|
0b0100001: 10, 0b1100001: 14, 0b0011: 18, 0b1011: 22,
|
|
0b0110001: 11, 0b1110001: 15, 0b0101: 19, 0b1101: 23,
|
|
0b1000001: 12, 0b0: 16, 0b0111: 20, 0b1111: 24,
|
|
0b1010001: 13, 0b0000001: 17, 0b1001: 21,
|
|
0b0010001: None,
|
|
}
|
|
|
|
name = 'WSIZE'
|
|
|
|
def __init__(self, name=None):
|
|
super().__init__(name, decodeTable=self.decodeTable)
|
|
|
|
def value(self, index):
|
|
#missing value gives index None
|
|
if index is None: return None
|
|
return (1<<index)-16
|
|
|
|
def explanation(self, index):
|
|
return 'windowsize=(1<<{})-16={}'.format(
|
|
index, (1<<index)-16)
|
|
|
|
#Metablock
|
|
class MetablockLengthAlphabet(WithExtra):
|
|
"""Used for the meta block length;
|
|
also indicates a block with no data
|
|
>>> metablockLengthAlphabet = MetablockLengthAlphabet()
|
|
>>> metablockLengthAlphabet[0]; str(metablockLengthAlphabet[0])
|
|
Symbol(MLEN, 0)
|
|
'empty'
|
|
>>> metablockLengthAlphabet[3]
|
|
Traceback (most recent call last):
|
|
...
|
|
ValueError: No symbol MetablockLengthAlphabet[3]
|
|
>>> print(metablockLengthAlphabet[4])
|
|
hhhh00
|
|
>>> metablockLengthAlphabet[4].value(0x1000)
|
|
4097
|
|
>>> metablockLengthAlphabet[5].value(0x1000)
|
|
Traceback (most recent call last):
|
|
...
|
|
InvalidStream: Zeros in high nibble of MLEN
|
|
>>> metablockLengthAlphabet[5].explanation(0x12345)
|
|
'data length: 12345h+1=74566'
|
|
>>> metablockLengthAlphabet.showCode()
|
|
00:hhhh00 10:hhhhhh10 01:hhhhh01 11:empty
|
|
"""
|
|
decodeTable = {0b11:0, 0b00:4, 0b01:5, 0b10:6}
|
|
|
|
name = 'MLEN'
|
|
def __init__(self, name=None):
|
|
super().__init__(name, decodeTable=self.decodeTable)
|
|
|
|
def extraBits(self, index):
|
|
return index*4
|
|
|
|
def mnemonic(self, index):
|
|
if index==0: return 'empty'
|
|
return 'h'*(self.extraBits(index)//4)+self.bitPattern(index)
|
|
|
|
def value(self, index, extra):
|
|
extraBits = self.extraBits(index)
|
|
if not 0<=extra<1<<extraBits:
|
|
raise ValueError('value: extra out of range')
|
|
if index==0: return 0
|
|
if index>4 and extra>>extraBits-4==0: raise InvalidStream(
|
|
'Zeros in high nibble of MLEN')
|
|
return extra+1
|
|
|
|
def explanation(self, index, extra):
|
|
if index==0: return '11: empty block'
|
|
extraBits = self.extraBits(index)
|
|
return 'data length: {:0{}x}h+1={}'.format(extra, extraBits//4, extra+1)
|
|
|
|
|
|
class ReservedAlphabet(BoolCode):
|
|
"""The reserved bit that must be zero.
|
|
"""
|
|
name = 'RSVD'
|
|
def value(self, index):
|
|
if index: raise ValueError('Reserved bit is not zero')
|
|
|
|
def explanation(self, index):
|
|
return 'Reserved (must be zero)'
|
|
|
|
class FillerAlphabet(Code):
|
|
def __init__(self, *, streamPos):
|
|
super().__init__('SKIP', bitLength=(-streamPos)&7)
|
|
|
|
def explanation(self, index):
|
|
return '{} bit{} ignored'.format(
|
|
self.length(index),
|
|
'' if self.length(index)==1 else 's',
|
|
)
|
|
|
|
class SkipLengthAlphabet(WithExtra):
|
|
"""Used for the skip length in an empty metablock
|
|
>>> skipLengthAlphabet = SkipLengthAlphabet()
|
|
>>> skipLengthAlphabet[0]; str(skipLengthAlphabet[0])
|
|
Symbol(SKIP, 0)
|
|
'empty'
|
|
>>> skipLengthAlphabet[4]
|
|
Traceback (most recent call last):
|
|
...
|
|
ValueError: index out of range
|
|
>>> print(skipLengthAlphabet[3])
|
|
hhhhhh11
|
|
>>> skipLengthAlphabet[2].value(0x1000)
|
|
4097
|
|
>>> skipLengthAlphabet[3].value(0x1000)
|
|
Traceback (most recent call last):
|
|
...
|
|
InvalidStream: Zeros in high byte of SKIPBYTES
|
|
>>> skipLengthAlphabet[3].explanation(0x12345)
|
|
'skip length: 12345h+1=74566'
|
|
>>> skipLengthAlphabet.showCode()
|
|
00:empty 01:hh01 10:hhhh10 11:hhhhhh11
|
|
"""
|
|
def __init__(self):
|
|
super().__init__('SKIP', bitLength=2)
|
|
|
|
def extraBits(self, index):
|
|
return index*8
|
|
|
|
def mnemonic(self, index):
|
|
if index==0: return 'empty'
|
|
return 'h'*(self.extraBits(index)//4)+self.bitPattern(index)
|
|
|
|
def value(self, index, extra):
|
|
extraBits = self.extraBits(index)
|
|
if not 0<=extra<1<<extraBits:
|
|
raise ValueError('value: extra out of range')
|
|
if index==0: return 0
|
|
if index>1 and extra>>extraBits-8==0:
|
|
raise InvalidStream('Zeros in high byte of SKIPBYTES')
|
|
return extra+1
|
|
|
|
def explanation(self, index, extra):
|
|
if index==0: return '00: no skip'
|
|
extraBits = self.extraBits(index)
|
|
return 'skip length: {:{}x}h+1={}'.format(extra, extraBits//8, extra+1)
|
|
|
|
|
|
class TypeCountAlphabet(Enumerator):
|
|
"""Used for giving block type counts and tree counts.
|
|
>>> TypeCountAlphabet(description='').showCode()
|
|
0:0 0101:xx,0101 1011:xxxxx,1011
|
|
0001:0001 1101:xxxxxx,1101 0111:xxx,0111
|
|
1001:xxxx,1001 0011:x,0011 1111:xxxxxxx,1111
|
|
"""
|
|
decodeTable = {
|
|
0b0: 0, 0b1001: 5,
|
|
0b0001: 1, 0b1011: 6,
|
|
0b0011: 2, 0b1101: 7,
|
|
0b0101: 3, 0b1111: 8,
|
|
0b0111: 4,
|
|
}
|
|
|
|
value0 = 1
|
|
extraTable = [0, 0, 1, 2, 3, 4, 5, 6, 7]
|
|
name = 'BT#'
|
|
|
|
def __init__(self, name=None, *, description):
|
|
super().__init__(
|
|
name,
|
|
decodeTable=self.decodeTable,
|
|
description=description)
|
|
|
|
def mnemonic(self, index):
|
|
if index==0: return '0'
|
|
if index==1: return '0001'
|
|
return 'x'*(self.extraBits(index))+','+self.bitPattern(index)
|
|
|
|
def explanation(self, index, extra):
|
|
value = self.value(index, extra)
|
|
description = self.description
|
|
if value==1: description = description[:-1]
|
|
return '{}: {} {}'.format(
|
|
self.mnemonic(index),
|
|
value,
|
|
description)
|
|
|
|
class BlockTypeAlphabet(Code):
|
|
"""The block types; this code works for all three kinds.
|
|
>>> b = BlockTypeAlphabet('T', NBLTYPES=5)
|
|
>>> print(*(x for x in b))
|
|
prev +1 #0 #1 #2 #3 #4
|
|
"""
|
|
def __init__(self, name, NBLTYPES, **args):
|
|
super().__init__(name, alphabetSize=NBLTYPES+2, **args)
|
|
self.NBLTYPES = NBLTYPES
|
|
|
|
def mnemonic(self, index):
|
|
if index==0: return 'prev'
|
|
elif index==1: return '+1'
|
|
else: return '#'+str(index-2)
|
|
|
|
def value(self, index):
|
|
return index-2
|
|
|
|
def explanation(self, index):
|
|
if index==0: return '0: previous'
|
|
elif index==1: return '1: increment'
|
|
else: return 'Set block type to: '+str(index-2)
|
|
|
|
class BlockCountAlphabet(Enumerator):
|
|
"""Block counts
|
|
>>> b = BlockCountAlphabet('L')
|
|
>>> print(b[25])
|
|
[24*x]: BC16625-16793840
|
|
"""
|
|
|
|
value0 = 1
|
|
extraTable = [2,2,2,2,3, 3,3,3,4,4, 4,4,5,5,5, 5,6,6,7,8, 9,10,11,12,13, 24]
|
|
def __init__(self, name, **args):
|
|
super().__init__(name, alphabetSize=26, **args)
|
|
|
|
def mnemonic(self, index):
|
|
extraBits = self.extraBits(index)
|
|
return '{}: BC{}-{}'.format(
|
|
'x'*extraBits if index<5 else '[{}*x]'.format(extraBits),
|
|
*self.span(index))
|
|
|
|
def explanation(self, index, extra):
|
|
return 'Block count: '+super().explanation(index, extra)
|
|
|
|
class DistanceParamAlphabet(WithExtra):
|
|
"""The distance parameters NPOSTFIX and NDIRECT.
|
|
Although these are treated as two in the description, this is easier.
|
|
"""
|
|
def __init__(self):
|
|
super().__init__('DIST', bitLength=2)
|
|
|
|
def extraBits(self, index):
|
|
return 4
|
|
|
|
def value(self, index, extra):
|
|
"""Returns NPOSTFIX and NDIRECT<<NPOSTFIX
|
|
"""
|
|
if extra>15:
|
|
raise ValueError('value: extra out of range')
|
|
return index, extra<<index
|
|
|
|
def explanation(self, index, extra):
|
|
return '{} postfix bits and {:04b}<<{}={} direct codes'.format(
|
|
index, extra, index, extra<<index)
|
|
|
|
def mnemonic(self, index):
|
|
return 'PF'+str(index)
|
|
|
|
class LiteralContextMode(Code):
|
|
"""For the literal context modes.
|
|
>>> LiteralContextMode().showCode()
|
|
00:LSB6 01:MSB6 10:UTF8 11:Signed
|
|
>>> LiteralContextMode().explanation(2)
|
|
'Context mode for type 9: 2(UTF8)'
|
|
"""
|
|
|
|
def __init__(self, *, number=9):
|
|
super().__init__('LC'+str(number), bitLength=2)
|
|
self.number = number
|
|
|
|
def mnemonic(self, index):
|
|
return ['LSB6', 'MSB6', 'UTF8', 'Signed'][index]
|
|
|
|
def explanation(self, index):
|
|
return 'Context mode for type {}: {}({})'.format(
|
|
self.number,
|
|
index,
|
|
self.mnemonic(index))
|
|
|
|
class RLEmaxAlphabet(Enumerator):
|
|
"""Used for describing the run length encoding used for describing context maps.
|
|
>>> RLEmaxAlphabet().showCode()
|
|
0:1 1:more
|
|
"""
|
|
value0 = 0
|
|
extraTable = [0, 4]
|
|
name = 'RLE#'
|
|
|
|
def mnemonic(self, index):
|
|
return ['1', 'more'][index]
|
|
|
|
def explanation(self, index, extra):
|
|
description = self.description and self.description+': '
|
|
if index==0: return description+'No RLE coding'
|
|
return '{}xxxx 1: RLEMAX={}'.format(description, extra+1)
|
|
|
|
class TreeAlphabet(WithExtra):
|
|
"""The alphabet to enumerate entries (called trees) in the context map.
|
|
parameters are RLEMAX and NTREES
|
|
>>> t = TreeAlphabet('', RLEMAX=3, NTREES=5)
|
|
>>> len(t)
|
|
8
|
|
>>> print(t[2])
|
|
xx+4 zeroes
|
|
>>> t[3].explanation(2)
|
|
'8+010=10 zeroes'
|
|
>>> t[0].value(0)
|
|
(1, 0)
|
|
"""
|
|
name = 'CMI'
|
|
def __init__(self, name=None, *, RLEMAX, NTREES, **args):
|
|
super().__init__(name, alphabetSize=RLEMAX+NTREES, **args)
|
|
self.RLEMAX = RLEMAX
|
|
self.NTREES = NTREES
|
|
|
|
def extraBits(self, index):
|
|
if 0<index<=self.RLEMAX: return index
|
|
return 0
|
|
|
|
def mnemonic(self, index):
|
|
if index==0: return 'map #0'
|
|
if index<=self.RLEMAX:
|
|
return '{}+{} zeroes'.format('x'*index, 1<<index)
|
|
return 'map #{}'.format(index-self.RLEMAX)
|
|
|
|
def value(self, index, extra):
|
|
"""Give count and value."""
|
|
index = index
|
|
if index==0: return 1, 0
|
|
if index<=self.RLEMAX: return (1<<index)+extra, 0
|
|
return 1, index-self.RLEMAX
|
|
|
|
def explanation(self, index, extra):
|
|
description = self.description and self.description+': '
|
|
if index==0: return description+'map #0'
|
|
if index<=self.RLEMAX:
|
|
return '{}+{:0{}b}={} zeroes'.format(
|
|
(1<<index),
|
|
extra, self.extraBits(index),
|
|
(1<<index)+extra)
|
|
return '{}map #{}-{}={}'.format(
|
|
description,
|
|
index, self.RLEMAX, index-self.RLEMAX)
|
|
|
|
#Prefix alphabets for the data stream----------------------------------
|
|
class LiteralAlphabet(Code):
|
|
"""Alphabet of symbols.
|
|
"""
|
|
minLength = maxLength = 8
|
|
def __init__(self, number):
|
|
super().__init__('L'+str(number), alphabetSize=1<<8)
|
|
|
|
def mnemonic(self, index):
|
|
return outputCharFormatter(index)
|
|
|
|
def value(self, index, extra=None):
|
|
return index
|
|
|
|
def explanation(self, index, extra=None):
|
|
return self.mnemonic(index)
|
|
|
|
class InsertLengthAlphabet(Enumerator):
|
|
"""Intern code for insert counts
|
|
"""
|
|
value0 = 0
|
|
extraTable = [0,0,0,0,0, 0,1,1,2,2, 3,3,4,4,5, 5,6,7,8,9, 10,12,14,24]
|
|
|
|
class CopyLengthAlphabet(Enumerator):
|
|
value0 = 2
|
|
extraTable = [0,0,0,0,0, 0,0,0,1,1, 2,2,3,3,4, 4,5,5,6,7, 8,9,10,24]
|
|
|
|
class InsertAndCopyAlphabet(WithExtra):
|
|
"""The insert and copy code
|
|
>>> for x in range(0,704,704//13):
|
|
... print('{:10b}'.format(x), InsertAndCopyAlphabet()[x])
|
|
0 I0C2&D=0
|
|
110110 I6+xC8&D=0
|
|
1101100 I5C22+xxx&D=0
|
|
10100010 I4C4
|
|
11011000 I3C10+x
|
|
100001110 I14+xxC8
|
|
101000100 I10+xxC22+xxx
|
|
101111010 I98+xxxxxC14+xx
|
|
110110000 I6+xC70+xxxxx
|
|
111100110 I1090+[10*x]C8
|
|
1000011100 I26+xxxC326+[8*x]
|
|
1001010010 I322+[8*x]C14+xx
|
|
1010001000 I194+[7*x]C70+xxxxx
|
|
1010111110 I22594+[24*x]C1094+[10*x]
|
|
"""
|
|
insertLengthAlphabet = InsertLengthAlphabet(None)
|
|
copyLengthAlphabet = CopyLengthAlphabet(None)
|
|
|
|
def __init__(self, number=''):
|
|
super().__init__('IC'+str(number), bitLength=10)
|
|
|
|
def __len__(self):
|
|
return 704
|
|
|
|
def extraBits(self, index):
|
|
insertSymbol, copySymbol, dist0 = self.splitSymbol(index)
|
|
return InsertLengthAlphabet.extraTable[insertSymbol.index] + \
|
|
CopyLengthAlphabet.extraTable[copySymbol.index]
|
|
|
|
def splitSymbol(self, index):
|
|
"""Give relevant values for computations:
|
|
(insertSymbol, copySymbol, dist0flag)
|
|
"""
|
|
#determine insert and copy upper bits from table
|
|
row = [0,0,1,1,2,2,1,3,2,3,3][index>>6]
|
|
col = [0,1,0,1,0,1,2,0,2,1,2][index>>6]
|
|
#determine inserts and copy sub codes
|
|
insertLengthCode = row<<3 | index>>3&7
|
|
if row: insertLengthCode -= 8
|
|
copyLengthCode = col<<3 | index&7
|
|
return (
|
|
Symbol(self.insertLengthAlphabet, insertLengthCode),
|
|
Symbol(self.copyLengthAlphabet, copyLengthCode),
|
|
row==0
|
|
)
|
|
|
|
def mnemonic(self, index):
|
|
"""Make a nice mnemonic
|
|
"""
|
|
i,c,d0 = self.splitSymbol(index)
|
|
iLower, _ = i.code.span(i.index)
|
|
iExtra = i.extraBits()
|
|
cLower, _ = c.code.span(c.index)
|
|
cExtra = c.extraBits()
|
|
return 'I{}{}{}C{}{}{}{}'.format(
|
|
iLower,
|
|
'+' if iExtra else '',
|
|
'x'*iExtra if iExtra<6 else '[{}*x]'.format(iExtra),
|
|
cLower,
|
|
'+' if cExtra else '',
|
|
'x'*cExtra if cExtra<6 else '[{}*x]'.format(cExtra),
|
|
'&D=0' if d0 else '')
|
|
|
|
def value(self, index, extra):
|
|
i,c,d0 = self.splitSymbol(index)
|
|
iExtra = i.extraBits()
|
|
ce, ie = extra>>iExtra, extra&(1<<iExtra)-1
|
|
insert = i.value(ie)
|
|
copy = c.value(ce)
|
|
return insert, copy, d0
|
|
|
|
def explanation(self, index, extra):
|
|
insert, copy, d0 = self.value(index, extra)
|
|
if d0: return 'Literal: {}, copy: {}, same distance'.format(insert, copy)
|
|
else: return 'Literal: {}, copy: {}'.format(insert, copy)
|
|
|
|
class DistanceAlphabet(WithExtra):
|
|
"""Represent the distance encoding.
|
|
Dynamically generated alphabet.
|
|
This is what the documentation should have said:
|
|
Ignoring offsets for the moment, the "long" encoding works as follows:
|
|
Write the distance in binary as follows:
|
|
1xy..yz..z, then the distance symbol consists of n..nxz..z
|
|
Where:
|
|
n is one less than number of bits in y
|
|
x is a single bit
|
|
y..y are n+1 extra bits (encoded in the bit stream)
|
|
z..z is NPOSTFIX bits that are part of the symbol
|
|
The offsets are so as to start at the lowest useable value:
|
|
if 1xyyyyz = distance +(4<<POSTFIX)-NDIRECT-1
|
|
then n..nxz..z is symbol -NDIRECT-16
|
|
>>> d = DistanceAlphabet('D', NPOSTFIX=2, NDIRECT=10)
|
|
>>> print(d[4], d[17], d[34])
|
|
last-1 1 10xx00-5
|
|
>>> [str(d[x]) for x in range(26, 32)]
|
|
['10x00-5', '10x01-5', '10x10-5', '10x11-5', '11x00-5', '11x01-5']
|
|
"""
|
|
def __init__(self, number, *, NPOSTFIX, NDIRECT):
|
|
self.NPOSTFIX = NPOSTFIX
|
|
self.NDIRECT = NDIRECT
|
|
#set length
|
|
#Actually, not all symbols are used,
|
|
#only NDIRECT+16+(44-2*POSTFIX<<NPOSTFIX)
|
|
super().__init__('D'+str(number),
|
|
alphabetSize=self.NDIRECT+16+(48<<self.NPOSTFIX))
|
|
|
|
def extraBits(self, index):
|
|
"""Indicate how many extra bits are needed to interpret symbol
|
|
>>> d = DistanceAlphabet('D', NPOSTFIX=2, NDIRECT=10)
|
|
>>> [d[i].extraBits() for i in range(26)]
|
|
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
|
|
>>> [d[i].extraBits() for i in range(26,36)]
|
|
[1, 1, 1, 1, 1, 1, 1, 1, 2, 2]
|
|
"""
|
|
if index<16+self.NDIRECT: return 0
|
|
return 1 + ((index - self.NDIRECT - 16) >> (self.NPOSTFIX + 1))
|
|
|
|
def value(self, dcode, dextra):
|
|
"""Decode value of symbol together with the extra bits.
|
|
>>> d = DistanceAlphabet('D', NPOSTFIX=2, NDIRECT=10)
|
|
>>> d[34].value(2)
|
|
(0, 35)
|
|
"""
|
|
if dcode<16:
|
|
return [(1,0),(2,0),(3,0),(4,0),
|
|
(1,-1),(1,+1),(1,-2),(1,+2),(1,-3),(1,+3),
|
|
(2,-1),(2,+1),(2,-2),(2,+2),(2,-3),(2,+3)
|
|
][dcode]
|
|
if dcode<16+self.NDIRECT:
|
|
return (0,dcode-16)
|
|
#we use the original formulas, instead of my clear explanation
|
|
POSTFIX_MASK = (1 << self.NPOSTFIX) - 1
|
|
ndistbits = 1 + ((dcode - self.NDIRECT - 16) >> (self.NPOSTFIX + 1))
|
|
hcode = (dcode - self.NDIRECT - 16) >> self.NPOSTFIX
|
|
lcode = (dcode - self.NDIRECT - 16) & POSTFIX_MASK
|
|
offset = ((2 + (hcode & 1)) << ndistbits) - 4
|
|
distance = ((offset + dextra) << self.NPOSTFIX) + lcode + self.NDIRECT + 1
|
|
return (0,distance)
|
|
|
|
def mnemonic(self, index, verbose=False):
|
|
"""Give mnemonic representation of meaning.
|
|
verbose compresses strings of x's
|
|
"""
|
|
if index<16:
|
|
return ['last', '2last', '3last', '4last',
|
|
'last-1', 'last+1', 'last-2', 'last+2', 'last-3', 'last+3',
|
|
'2last-1', '2last+1', '2last-2', '2last+2', '2last-3', '2last+3'
|
|
][index]
|
|
if index<16+self.NDIRECT:
|
|
return str(index-16)
|
|
#construct strings like "1xx01-15"
|
|
index -= self.NDIRECT+16
|
|
hcode = index >> self.NPOSTFIX
|
|
lcode = index & (1<<self.NPOSTFIX)-1
|
|
if self.NPOSTFIX: formatString = '1{0}{1}{2:0{3}b}{4:+d}'
|
|
else: formatString = '1{0}{1}{4:+d}'
|
|
return formatString.format(
|
|
hcode&1,
|
|
'x'*(2+hcode>>1) if hcode<13 or verbose else '[{}*x]'.format(2+hcode>>1),
|
|
lcode, self.NPOSTFIX,
|
|
self.NDIRECT+1-(4<<self.NPOSTFIX))
|
|
|
|
def explanation(self, index, extra):
|
|
"""
|
|
>>> d = DistanceAlphabet('D', NPOSTFIX=2, NDIRECT=10)
|
|
>>> d[55].explanation(13)
|
|
'11[1101]01-5: [0]+240'
|
|
"""
|
|
extraBits = self.extraBits(index)
|
|
extraString = '[{:0{}b}]'.format(extra, extraBits)
|
|
return '{0}: [{1[0]}]{1[1]:+d}'.format(
|
|
self.mnemonic(index, True).replace('x'*(extraBits or 1), extraString),
|
|
self.value(index, extra))
|
|
|
|
#Classes for doing actual work------------------------------------------
|
|
class ContextModeKeeper:
|
|
"""For computing the literal context mode.
|
|
You feed it characters, and it computes indices in the context map.
|
|
"""
|
|
def __init__(self, mode):
|
|
self.chars = deque([0,0], maxlen=2)
|
|
self.mode = mode
|
|
|
|
def setContextMode(self, mode):
|
|
"""Switch to given context mode (0..3)"""
|
|
self.mode = mode
|
|
def getIndex(self):
|
|
if self.mode==0: #LSB6
|
|
return self.chars[1]&0x3f
|
|
elif self.mode==1: #MSB6
|
|
return self.chars[1]>>2
|
|
elif self.mode==2: #UTF8: character class of previous and a bit of the second
|
|
p2,p1 = self.chars
|
|
return self.lut0[p1]|self.lut1[p2]
|
|
elif self.mode==3: #Signed: initial bits of last two bytes
|
|
p2,p1 = self.chars
|
|
return self.lut2[p1]<<3|self.lut2[p2]
|
|
|
|
def add(self, index):
|
|
"""Adjust the context for output char (as int)."""
|
|
self.chars.append(index)
|
|
|
|
#0: control #16: quote #32: ,:; #48: AEIOU
|
|
#4: tab/lf/cr #20: % #36: . #52: BC..Z
|
|
#8: space #24: (<[{ #40: = #56: aeiou
|
|
#12:!#$&*+-/?@| #28: )>]} #44: 0-9 #60: bc..z
|
|
lut0 = [0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 4, 0, 0,
|
|
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
8, 12, 16, 12, 12, 20, 12, 16, 24, 28, 12, 12, 32, 12, 36, 12,
|
|
44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 32, 32, 24, 40, 28, 12,
|
|
12, 48, 52, 52, 52, 48, 52, 52, 52, 48, 52, 52, 52, 52, 52, 48,
|
|
52, 52, 52, 52, 52, 48, 52, 52, 52, 52, 52, 24, 12, 28, 12, 12,
|
|
12, 56, 60, 60, 60, 56, 60, 60, 60, 56, 60, 60, 60, 60, 60, 56,
|
|
60, 60, 60, 60, 60, 56, 60, 60, 60, 60, 60, 24, 12, 28, 12, 0
|
|
]+[0,1]*32+[2,3]*32
|
|
#0: space 1:punctuation 2:digit/upper 3:lower
|
|
lut1 = [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
|
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1,
|
|
1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
|
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1,
|
|
1, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
|
|
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 1, 1, 1, 1, 0
|
|
]+[0]*96+[2]*32
|
|
#initial bits: 8*0, 4*0, 2*0, 1*0, 1*1, 2*1, 4*1, 8*1
|
|
lut2 = [0]+[1]*15+[2]*48+[3]*64+[4]*64+[5]*48+[6]*15+[7]
|
|
assert len(lut0)==len(lut1)==len(lut2)==256
|
|
|
|
class WordList:
|
|
"""Word list.
|
|
>>> WordList().word(7, 35555)
|
|
b'Program to '
|
|
"""
|
|
NDBITS = [0, 0, 0, 0, 10, 10, 11, 11, 10, 10,
|
|
10, 10, 10, 9, 9, 8, 7, 7, 8, 7,
|
|
7, 6, 6, 5, 5]
|
|
def __init__(self):
|
|
self.file = open('dict', 'rb')
|
|
self.compileActions()
|
|
|
|
def word(self, size, dist):
|
|
"""Get word
|
|
"""
|
|
#split dist in index and action
|
|
ndbits = self.NDBITS[size]
|
|
index = dist&(1<<ndbits)-1
|
|
action = dist>>ndbits
|
|
#compute position in file
|
|
position = sum(n<<self.NDBITS[n] for n in range(4,size))+size*index
|
|
self.file.seek(position)
|
|
return self.doAction(self.file.read(size), action)
|
|
|
|
def upperCase1(self, word):
|
|
word = word.decode('utf8')
|
|
word = word[0].upper()+word[1:]
|
|
return word.encode('utf8')
|
|
|
|
|
|
#Super compact form of action table.
|
|
#_ means space, .U means UpperCaseAll, U(w) means UpperCaseFirst
|
|
actionTable = r"""
|
|
0:w 25:w+_for_ 50:w+\n\t 75:w+. This_100:w+ize_
|
|
1:w+_ 26:w[3:] 51:w+: 76:w+, 101:w.U+.
|
|
2:_+w+_ 27:w[:-2] 52:_+w+._ 77:.+w+_ 102:\xc2\xa0+w
|
|
3:w[1:] 28:w+_a_ 53:w+ed_ 78:U(w)+( 103:_+w+,
|
|
4:U(w)+_ 29:w+_that_ 54:w[9:] 79:U(w)+. 104:U(w)+="
|
|
5:w+_the_ 30:_+U(w) 55:w[7:] 80:w+_not_ 105:w.U+="
|
|
6:_+w 31:w+._ 56:w[:-6] 81:_+w+=" 106:w+ous_
|
|
7:s_+w+_ 32:.+w 57:w+( 82:w+er_ 107:w.U+,_
|
|
8:w+_of_ 33:_+w+,_ 58:U(w)+,_ 83:_+w.U+_ 108:U(w)+=\'
|
|
9:U(w) 34:w[4:] 59:w[:-8] 84:w+al_ 109:_+U(w)+,
|
|
10:w+_and_ 35:w+_with_ 60:w+_at_ 85:_+w.U 110:_+w.U+="
|
|
11:w[2:] 36:w+\' 61:w+ly_ 86:w+=\' 111:_+w.U+,_
|
|
12:w[:-1] 37:w+_from_ 62:_the_+w+_of_ 87:w.U+" 112:_+w.U+,
|
|
13:,_+w+_ 38:w+_by_ 63:w[:-5] 88:U(w)+._ 113:w.U+(
|
|
14:w+,_ 39:w[5:] 64:w[:-9] 89:_+w+( 114:w.U+._
|
|
15:_+U(w)+_ 40:w[6:] 65:_+U(w)+,_ 90:w+ful_ 115:_+w.U+.
|
|
16:w+_in_ 41:_the_+w 66:U(w)+" 91:_+U(w)+._116:w.U+=\'
|
|
17:w+_to_ 42:w[:-4] 67:.+w+( 92:w+ive_ 117:_+w.U+._
|
|
18:e_+w+_ 43:w+. The_ 68:w.U+_ 93:w+less_ 118:_+U(w)+="
|
|
19:w+" 44:w.U 69:U(w)+"> 94:w.U+\' 119:_+w.U+=\'
|
|
20:w+. 45:w+_on_ 70:w+=" 95:w+est_ 120:_+U(w)+=\'
|
|
21:w+"> 46:w+_as_ 71:_+w+. 96:_+U(w)+.
|
|
22:w+\n 47:w+_is_ 72:.com/+w 97:w.U+">
|
|
23:w[:-3] 48:w[:-7] 98:_+w+=\'
|
|
24:w+] 49:w[:-1]+ing_ 74:U(w)+\' 99:U(w)+,
|
|
"""
|
|
|
|
def compileActions(self):
|
|
"""Build the action table from the text above
|
|
"""
|
|
import re
|
|
self.actionList = actions = [None]*121
|
|
#Action 73, which is too long, looks like this when expanded:
|
|
actions[73] = "b' the '+w+b' of the '"
|
|
#find out what the columns are
|
|
actionLines = self.actionTable.splitlines()
|
|
colonPositions = [m.start()
|
|
for m in re.finditer(':',actionLines[1])
|
|
]+[100]
|
|
columns = [(colonPositions[i]-3,colonPositions[i+1]-3)
|
|
for i in range(len(colonPositions)-1)]
|
|
for line in self.actionTable.splitlines(keepends=False):
|
|
for start,end in columns:
|
|
action = line[start:end]
|
|
#skip empty actions
|
|
if not action or action.isspace(): continue
|
|
#chop it up, and check if the colon is properly placed
|
|
index, colon, action = action[:3], action[3], action[4:]
|
|
assert colon==':'
|
|
#remove filler spaces at right
|
|
action = action.rstrip()
|
|
#replace space symbols
|
|
action = action.replace('_', ' ')
|
|
wPos = action.index('w')
|
|
#add quotes around left string when present
|
|
#translation: any pattern from beginning, up to
|
|
#(but not including) a + following by a w later on
|
|
action = re.sub(r"^(.*)(?=\+[U(]*w)", r"b'\1'", action)
|
|
#add quotes around right string when present
|
|
#translation: anything with a w in it, followed by a +
|
|
#and a pattern up to the end
|
|
#(there is no variable lookbehind assertion,
|
|
#so we have to copy the pattern)
|
|
action = re.sub(r"(w[[:\-1\]).U]*)\+(.*)$", r"\1+b'\2'", action)
|
|
#expand shortcut for uppercaseAll
|
|
action = action.replace(".U", ".upper()")
|
|
#store action
|
|
actions[int(index)] = action
|
|
|
|
def doAction(self, w, action):
|
|
"""Perform the proper action
|
|
"""
|
|
#set environment for the UpperCaseFirst
|
|
U = self.upperCase1
|
|
return eval(self.actionList[action], locals())
|
|
|
|
class Layout:
|
|
"""Class to layout the output.
|
|
"""
|
|
#display width of hexdata+bitdata
|
|
width = 25
|
|
#general
|
|
def __init__(self, stream):
|
|
self.stream = stream
|
|
self.bitPtr = self.width
|
|
|
|
def makeHexData(self, pos):
|
|
"""Produce hex dump of all data containing the bits
|
|
from pos to stream.pos
|
|
"""
|
|
firstAddress = pos+7>>3
|
|
lastAddress = self.stream.pos+7>>3
|
|
return ''.join(map('{:02x} '.format,
|
|
self.stream.data[firstAddress:lastAddress]))
|
|
|
|
def formatBitData(self, pos, width1, width2=0):
|
|
"""Show formatted bit data:
|
|
Bytes are separated by commas
|
|
whole bytes are displayed in hex
|
|
>>> Layout(olleke).formatBitData(6, 2, 16)
|
|
'|00h|2Eh,|00'
|
|
>>> Layout(olleke).formatBitData(4, 1, 0)
|
|
'1'
|
|
"""
|
|
result = []
|
|
#make empty prefix code explicit
|
|
if width1==0: result = ['()', ',']
|
|
for width in width1, width2:
|
|
#skip empty width2
|
|
if width==0: continue
|
|
#build result backwards in a list
|
|
while width>0:
|
|
availableBits = 8-(pos&7)
|
|
if width<availableBits:
|
|
#read partial byte, beginning nor ending at boundary
|
|
data = self.stream.data[pos>>3] >> (pos&7) & (1<<width)-1
|
|
result.append('{:0{}b}'.format(data, width))
|
|
elif availableBits<8:
|
|
#read rest of byte, ending at boundary
|
|
data = self.stream.data[pos>>3] >> (pos&7)
|
|
result.append('|{:0{}b}'.format(data, availableBits))
|
|
else:
|
|
#read whole byte (in hex), beginning and ending at boundary
|
|
data = self.stream.data[pos>>3]
|
|
result.append('|{:02X}h'.format(data))
|
|
width -= availableBits
|
|
pos += availableBits
|
|
#if width overshot from the availableBits subtraction, fix it
|
|
pos += width
|
|
#add comma to separate fields
|
|
result.append(',')
|
|
#concatenate pieces, reversed, skipping the last space
|
|
return ''.join(result[-2::-1])
|
|
|
|
def readPrefixCode(self, alphabet):
|
|
"""give alphabet the prefix code that is read from the stream
|
|
Called for the following alphabets, in this order:
|
|
The alphabet in question must have a "logical" order,
|
|
otherwise the assignment of symbols doesn't work.
|
|
"""
|
|
mode, numberOfSymbols = self.verboseRead(PrefixCodeHeader(alphabet.name))
|
|
if mode=='Complex':
|
|
#for a complex code, numberOfSymbols means hskip
|
|
self.readComplexCode(numberOfSymbols, alphabet)
|
|
return alphabet
|
|
else:
|
|
table = []
|
|
#Set table of lengths for mnemonic function
|
|
lengths = [[0], [1,1], [1,2,2], '????'][numberOfSymbols-1]
|
|
#adjust mnemonic function of alphabet class
|
|
def myMnemonic(index):
|
|
return '{} bit{}: {}'.format(
|
|
lengths[i],
|
|
'' if lengths[i]==1 else 's',
|
|
alphabet.__class__.mnemonic(alphabet, index)
|
|
)
|
|
alphabet.mnemonic = myMnemonic
|
|
for i in range(numberOfSymbols):
|
|
table.append(self.verboseRead(alphabet, skipExtra=True).index)
|
|
#restore mnemonic
|
|
del alphabet.mnemonic
|
|
if numberOfSymbols==4:
|
|
#read tree shape to redefine lengths
|
|
lengths = self.verboseRead(TreeShapeAlhabet())
|
|
#construct the alphabet prefix code
|
|
alphabet.setLength(dict(zip(table, lengths)))
|
|
return alphabet
|
|
|
|
def readComplexCode(self, hskip, alphabet):
|
|
"""Read complex code"""
|
|
stream = self.stream
|
|
#read the lengths for the length code
|
|
lengths = [1,2,3,4,0,5,17,6,16,7,8,9,10,11,12,13,14,15][hskip:]
|
|
codeLengths = {}
|
|
total = 0
|
|
lol = LengthOfLengthAlphabet('##'+alphabet.name)
|
|
#lengthCode will be used for coding the lengths of the new code
|
|
#we use it for display until now; definition comes below
|
|
lengthCode = LengthAlphabet('#'+alphabet.name)
|
|
lengthIter = iter(lengths)
|
|
lengthsLeft = len(lengths)
|
|
while total<32 and lengthsLeft>0:
|
|
lengthsLeft -= 1
|
|
newSymbol = next(lengthIter)
|
|
lol.description = str(lengthCode[newSymbol])
|
|
length = self.verboseRead(lol)
|
|
if length:
|
|
codeLengths[newSymbol] = length
|
|
total += 32>>length
|
|
if total>32: raise ValueError("Stream format")
|
|
if len(codeLengths)==1: codeLengths[list(codeLengths.keys())[0]] = 0
|
|
#Now set the encoding of the lengthCode
|
|
lengthCode.setLength(codeLengths)
|
|
print("***** Lengths for {} will be coded as:".format(alphabet.name))
|
|
lengthCode.showCode()
|
|
#Now determine the symbol lengths with the lengthCode
|
|
symbolLengths = {}
|
|
total = 0
|
|
lastLength = 8
|
|
alphabetIter = iter(alphabet)
|
|
while total<32768:
|
|
#look ahead to see what is going to happen
|
|
length = lengthCode.decodePeek(
|
|
self.stream.peek(lengthCode.maxLength))[1].index
|
|
#in every branch, set lengthCode.description to explanatory text
|
|
#lengthCode calls format(symbol, extra) with this string
|
|
if length==0:
|
|
symbol = next(alphabetIter)
|
|
lengthCode.description = 'symbol {} unused'.format(symbol)
|
|
self.verboseRead(lengthCode)
|
|
#unused symbol
|
|
continue
|
|
if length==16:
|
|
lengthCode.description = \
|
|
'{1}+3 symbols of length '+str(lastLength)
|
|
extra = self.verboseRead(lengthCode)
|
|
#scan series of 16s (repeat counts)
|
|
#start with repeat count 2
|
|
repeat = 2
|
|
startSymbol = next(alphabetIter)
|
|
endSymbol = next(alphabetIter)
|
|
symbolLengths[startSymbol.index] = \
|
|
symbolLengths[endSymbol.index] = lastLength
|
|
#count the two just defined symbols
|
|
total += 2*32768>>lastLength
|
|
#note: loop may end because we're there
|
|
#even if a 16 _appears_ to follow
|
|
while True:
|
|
#determine last symbol
|
|
oldRepeat = repeat
|
|
repeat = (repeat-2<<2)+extra+3
|
|
#read as many symbols as repeat increased
|
|
for i in range(oldRepeat, repeat):
|
|
endSymbol = next(alphabetIter)
|
|
symbolLengths[endSymbol.index] = lastLength
|
|
#compute new total; it may be end of loop
|
|
total += (repeat-oldRepeat)*32768>>lastLength
|
|
if total>=32768: break
|
|
#see if there is more to do
|
|
length = lengthCode.decodePeek(
|
|
self.stream.peek(lengthCode.maxLength))[1].index
|
|
if length!=16: break
|
|
lengthCode.description = 'total {}+{{1}} symbols'.format(
|
|
(repeat-2<<2)+3)
|
|
extra = self.verboseRead(lengthCode)
|
|
elif length==17:
|
|
#read, and show explanation
|
|
lengthCode.description = '{1}+3 unused'
|
|
extra = self.verboseRead(lengthCode)
|
|
#scan series of 17s (groups of zero counts)
|
|
#start with repeat count 2
|
|
repeat = 2
|
|
startSymbol = next(alphabetIter)
|
|
endSymbol = next(alphabetIter)
|
|
#note: loop will not end with total==32768,
|
|
#since total doesn't change here
|
|
while True:
|
|
#determine last symbol
|
|
oldRepeat = repeat
|
|
repeat = (repeat-2<<3)+extra+3
|
|
#read as many symbols as repeat increases
|
|
for i in range(repeat-oldRepeat):
|
|
endSymbol = next(alphabetIter)
|
|
#see if there is more to do
|
|
length = lengthCode.decodePeek(
|
|
self.stream.peek(lengthCode.maxLength))[1].index
|
|
if length!=17: break
|
|
lengthCode.description = 'total {}+{{1}} unused'.format(
|
|
(repeat-2<<3)+3)
|
|
extra = self.verboseRead(lengthCode)
|
|
else:
|
|
symbol = next(alphabetIter)
|
|
#double braces for format
|
|
char = str(symbol)
|
|
if char in '{}': char *= 2
|
|
lengthCode.description = \
|
|
'Length for {} is {{0.index}} bits'.format(char)
|
|
#output is not needed (will be 0)
|
|
self.verboseRead(lengthCode)
|
|
symbolLengths[symbol.index] = length
|
|
total += 32768>>length
|
|
lastLength = length
|
|
assert total==32768
|
|
alphabet.setLength(symbolLengths)
|
|
print('End of table. Prefix code '+alphabet.name+':')
|
|
alphabet.showCode()
|
|
|
|
#stream
|
|
def processStream(self):
|
|
"""Process a brotli stream.
|
|
"""
|
|
print('addr hex{:{}s}binary context explanation'.format(
|
|
'', self.width-10))
|
|
print('Stream header'.center(60, '-'))
|
|
self.windowSize = self.verboseRead(WindowSizeAlphabet())
|
|
print('Metablock header'.center(60, '='))
|
|
self.ISLAST = False
|
|
self.output = bytearray()
|
|
while not self.ISLAST:
|
|
self.ISLAST = self.verboseRead(
|
|
BoolCode('LAST', description="Last block"))
|
|
if self.ISLAST:
|
|
if self.verboseRead(
|
|
BoolCode('EMPTY', description="Empty block")): break
|
|
if self.metablockLength(): continue
|
|
if not self.ISLAST and self.uncompressed(): continue
|
|
print('Block type descriptors'.center(60, '-'))
|
|
self.numberOfBlockTypes = {}
|
|
self.currentBlockCounts = {}
|
|
self.blockTypeCodes = {}
|
|
self.blockCountCodes = {}
|
|
for blockType in (L,I,D): self.blockType(blockType)
|
|
print('Distance code parameters'.center(60, '-'))
|
|
self.NPOSTFIX, self.NDIRECT = self.verboseRead(DistanceParamAlphabet())
|
|
self.readLiteralContextModes()
|
|
print('Context maps'.center(60, '-'))
|
|
self.cmaps = {}
|
|
#keep the number of each kind of prefix tree for the last loop
|
|
numberOfTrees = {I: self.numberOfBlockTypes[I]}
|
|
for blockType in (L,D):
|
|
numberOfTrees[blockType] = self.contextMap(blockType)
|
|
print('Prefix code lists'.center(60, '-'))
|
|
self.prefixCodes = {}
|
|
for blockType in (L,I,D):
|
|
self.readPrefixArray(blockType, numberOfTrees[blockType])
|
|
self.metablock()
|
|
|
|
#metablock header
|
|
def verboseRead(self, alphabet, context='', skipExtra=False):
|
|
"""Read symbol and extra from stream and explain what happens.
|
|
Returns the value of the symbol
|
|
>>> olleke.pos = 0
|
|
>>> l = Layout(olleke)
|
|
>>> l.verboseRead(WindowSizeAlphabet())
|
|
0000 1b 1011 WSIZE windowsize=(1<<22)-16=4194288
|
|
4194288
|
|
"""
|
|
#TODO 2: verbosity level, e.g. show only codes and maps in header
|
|
stream = self.stream
|
|
pos = stream.pos
|
|
if skipExtra:
|
|
length, symbol = alphabet.readTuple(stream)
|
|
extraBits, extra = 0, None
|
|
else:
|
|
length, symbol, extraBits, extra = alphabet.readTupleAndExtra(
|
|
stream)
|
|
#fields: address, hex data, binary data, name of alphabet, explanation
|
|
hexdata = self.makeHexData(pos)
|
|
addressField = '{:04x}'.format(pos+7>>3) if hexdata else ''
|
|
bitdata = self.formatBitData(pos, length, extraBits)
|
|
#bitPtr moves bitdata so that the bytes are easier to read
|
|
#jump back to right if a new byte starts
|
|
if '|' in bitdata[1:]:
|
|
#start over on the right side
|
|
self.bitPtr = self.width
|
|
fillWidth = self.bitPtr-(len(hexdata)+len(bitdata))
|
|
if fillWidth<0: fillWidth = 0
|
|
print('{:<5s} {:<{}s} {:7s} {}'.format(
|
|
addressField,
|
|
hexdata+' '*fillWidth+bitdata, self.width,
|
|
context+alphabet.name,
|
|
symbol if skipExtra else symbol.explanation(extra),
|
|
))
|
|
#jump to the right if we started with a '|'
|
|
#because we didn't jump before printing
|
|
if bitdata.startswith('|'): self.bitPtr = self.width
|
|
else: self.bitPtr -= len(bitdata)
|
|
return symbol if skipExtra else symbol.value(extra)
|
|
|
|
def metablockLength(self):
|
|
"""Read MNIBBLES and meta block length;
|
|
if empty block, skip block and return true.
|
|
"""
|
|
self.MLEN = self.verboseRead(MetablockLengthAlphabet())
|
|
if self.MLEN:
|
|
return False
|
|
#empty block; skip and return False
|
|
self.verboseRead(ReservedAlphabet())
|
|
MSKIP = self.verboseRead(SkipLengthAlphabet())
|
|
self.verboseRead(FillerAlphabet(streamPos=self.stream.pos))
|
|
self.stream.pos += 8*MSKIP
|
|
print("Skipping to {:x}".format(self.stream.pos>>3))
|
|
return True
|
|
|
|
def uncompressed(self):
|
|
"""If true, handle uncompressed data
|
|
"""
|
|
ISUNCOMPRESSED = self.verboseRead(
|
|
BoolCode('UNCMPR', description='Is uncompressed?'))
|
|
if ISUNCOMPRESSED:
|
|
self.verboseRead(FillerAlphabet(streamPos=self.stream.pos))
|
|
print('Uncompressed data:')
|
|
self.output += self.stream.readBytes(self.MLEN)
|
|
print(outputFormatter(self.output[-self.MLEN:]))
|
|
return ISUNCOMPRESSED
|
|
|
|
def blockType(self, kind):
|
|
"""Read block type switch descriptor for given kind of blockType."""
|
|
NBLTYPES = self.verboseRead(TypeCountAlphabet(
|
|
'BT#'+kind[0].upper(),
|
|
description='{} block types'.format(kind),
|
|
))
|
|
self.numberOfBlockTypes[kind] = NBLTYPES
|
|
if NBLTYPES>=2:
|
|
self.blockTypeCodes[kind] = self.readPrefixCode(
|
|
BlockTypeAlphabet('BT'+kind[0].upper(), NBLTYPES))
|
|
self.blockCountCodes[kind] = self.readPrefixCode(
|
|
BlockCountAlphabet('BC'+kind[0].upper()))
|
|
blockCount = self.verboseRead(self.blockCountCodes[kind])
|
|
else:
|
|
blockCount = 1<<24
|
|
self.currentBlockCounts[kind] = blockCount
|
|
|
|
def readLiteralContextModes(self):
|
|
"""Read literal context modes.
|
|
LSB6: lower 6 bits of last char
|
|
MSB6: upper 6 bits of last char
|
|
UTF8: rougly dependent on categories:
|
|
upper 4 bits depend on category of last char:
|
|
control/whitespace/space/ punctuation/quote/%/open/close/
|
|
comma/period/=/digits/ VOWEL/CONSONANT/vowel/consonant
|
|
lower 2 bits depend on category of 2nd last char:
|
|
space/punctuation/digit or upper/lowercase
|
|
signed: hamming weight of last 2 chars
|
|
"""
|
|
print('Context modes'.center(60, '-'))
|
|
self.literalContextModes = []
|
|
for i in range(self.numberOfBlockTypes[L]):
|
|
self.literalContextModes.append(
|
|
self.verboseRead(LiteralContextMode(number=i)))
|
|
|
|
def contextMap(self, kind):
|
|
"""Read context maps
|
|
Returns the number of differnt values on the context map
|
|
(In other words, the number of prefix trees)
|
|
"""
|
|
NTREES = self.verboseRead(TypeCountAlphabet(
|
|
kind[0].upper()+'T#',
|
|
description='{} prefix trees'.format(kind)))
|
|
mapSize = {L:64, D:4}[kind]
|
|
if NTREES<2:
|
|
self.cmaps[kind] = [0]*mapSize
|
|
else:
|
|
#read CMAPkind
|
|
RLEMAX = self.verboseRead(RLEmaxAlphabet(
|
|
'RLE#'+kind[0].upper(),
|
|
description=kind+' context map'))
|
|
alphabet = TreeAlphabet('CM'+kind[0].upper(), NTREES=NTREES, RLEMAX=RLEMAX)
|
|
cmapCode = self.readPrefixCode(alphabet)
|
|
tableSize = mapSize*self.numberOfBlockTypes[kind]
|
|
cmap = []
|
|
while len(cmap)<tableSize:
|
|
cmapCode.description = 'map {}, entry {}'.format(
|
|
*divmod(len(cmap), mapSize))
|
|
count, value = self.verboseRead(cmapCode)
|
|
cmap.extend([value]*count)
|
|
assert len(cmap)==tableSize
|
|
IMTF = self.verboseRead(BoolCode('IMTF', description='Apply inverse MTF'))
|
|
if IMTF:
|
|
self.IMTF(cmap)
|
|
if kind==L:
|
|
print('Context maps for literal data:')
|
|
for i in range(0, len(cmap), 64):
|
|
print(*(
|
|
''.join(map(str, cmap[j:j+8]))
|
|
for j in range(i, i+64, 8)
|
|
))
|
|
else:
|
|
print('Context map for distances:')
|
|
print(*(
|
|
''.join(map('{:x}'.format, cmap[i:i+4]))
|
|
for i in range(0, len(cmap), 4)
|
|
))
|
|
self.cmaps[kind] = cmap
|
|
return NTREES
|
|
|
|
@staticmethod
|
|
def IMTF(v):
|
|
"""In place inverse move to front transform.
|
|
"""
|
|
#mtf is initialized virtually with range(infinity)
|
|
mtf = []
|
|
for i, vi in enumerate(v):
|
|
#get old value from mtf. If never seen, take virtual value
|
|
try: value = mtf.pop(vi)
|
|
except IndexError: value = vi
|
|
#put value at front
|
|
mtf.insert(0, value)
|
|
#replace transformed value
|
|
v[i] = value
|
|
|
|
def readPrefixArray(self, kind, numberOfTrees):
|
|
"""Read prefix code array"""
|
|
prefixes = []
|
|
for i in range(numberOfTrees):
|
|
if kind==L: alphabet = LiteralAlphabet(i)
|
|
elif kind==I: alphabet = InsertAndCopyAlphabet(i)
|
|
elif kind==D: alphabet = DistanceAlphabet(
|
|
i, NPOSTFIX=self.NPOSTFIX, NDIRECT=self.NDIRECT)
|
|
self.readPrefixCode(alphabet)
|
|
prefixes.append(alphabet)
|
|
self.prefixCodes[kind] = prefixes
|
|
|
|
#metablock data
|
|
def metablock(self):
|
|
"""Process the data.
|
|
Relevant variables of self:
|
|
numberOfBlockTypes[kind]: number of block types
|
|
currentBlockTypes[kind]: current block types (=0)
|
|
literalContextModes: the context modes for the literal block types
|
|
currentBlockCounts[kind]: counters for block types
|
|
blockTypeCodes[kind]: code for block type
|
|
blockCountCodes[kind]: code for block count
|
|
cmaps[kind]: the context maps (not for I)
|
|
prefixCodes[kind][#]: the prefix codes
|
|
lastDistances: the last four distances
|
|
lastChars: the last two chars
|
|
output: the result
|
|
"""
|
|
print('Meta block contents'.center(60, '='))
|
|
self.currentBlockTypes = {L:0, I:0, D:0, pL:1, pI:1, pD:1}
|
|
self.lastDistances = deque([17,16,11,4], maxlen=4)
|
|
#the current context mode is for block type 0
|
|
self.contextMode = ContextModeKeeper(self.literalContextModes[0])
|
|
wordList = WordList()
|
|
|
|
#setup distance callback function
|
|
def distanceCallback(symbol, extra):
|
|
"callback function for displaying decoded distance"
|
|
index, offset = symbol.value(extra)
|
|
if index:
|
|
#recent distance
|
|
distance = self.lastDistances[-index]+offset
|
|
return 'Distance: {}last{:+d}={}'.format(index, offset, distance)
|
|
#absolute value
|
|
if offset<=maxDistance:
|
|
return 'Absolute value: {} (pos {})'.format(offset, maxDistance-offset)
|
|
#word list value
|
|
action, word = divmod(offset-maxDistance, 1<<wordList.NDBITS[copyLen])
|
|
return '{}-{} gives word {},{} action {}'.format(
|
|
offset, maxDistance, copyLen, word, action)
|
|
for dpc in self.prefixCodes[D]: dpc.callback = distanceCallback
|
|
|
|
blockLen = 0
|
|
#there we go
|
|
while blockLen<self.MLEN:
|
|
#get insert© command
|
|
litLen, copyLen, dist0Flag = self.verboseRead(
|
|
self.prefixCodes[I][
|
|
self.figureBlockType(I)])
|
|
#literal data
|
|
for i in range(litLen):
|
|
bt = self.figureBlockType(L)
|
|
cm = self.contextMode.getIndex()
|
|
ct = self.cmaps[L][bt<<6|cm]
|
|
char = self.verboseRead(
|
|
self.prefixCodes[L][ct],
|
|
context='{},{}='.format(bt,cm))
|
|
self.contextMode.add(char)
|
|
self.output.append(char)
|
|
blockLen += litLen
|
|
#check if we're done
|
|
if blockLen>=self.MLEN: return
|
|
#distance
|
|
#distances are computed relative to output length, at most window size
|
|
maxDistance = min(len(self.output), self.windowSize)
|
|
if dist0Flag:
|
|
distance = self.lastDistances[-1]
|
|
else:
|
|
bt = self.figureBlockType(D)
|
|
cm = {2:0, 3:1, 4:2}.get(copyLen, 3)
|
|
ct = self.cmaps[D][bt<<2|cm]
|
|
index, offset = self.verboseRead(
|
|
self.prefixCodes[D][ct],
|
|
context='{},{}='.format(bt,cm))
|
|
distance = self.lastDistances[-index]+offset if index else offset
|
|
if index==1 and offset==0:
|
|
#to make sure distance is not put in last distance list
|
|
dist0Flag = True
|
|
if distance<=maxDistance:
|
|
#copy from output
|
|
for i in range(
|
|
maxDistance-distance,
|
|
maxDistance-distance+copyLen):
|
|
self.output.append(self.output[i])
|
|
if not dist0Flag: self.lastDistances.append(distance)
|
|
comment = 'Seen before'
|
|
else:
|
|
#fetch from wordlist
|
|
newWord = wordList.word(copyLen, distance-maxDistance-1)
|
|
self.output.extend(newWord)
|
|
#adjust copyLen to reflect actual new data
|
|
copyLen = len(newWord)
|
|
comment = 'From wordlist'
|
|
blockLen += copyLen
|
|
print(' '*40,
|
|
comment,
|
|
': "',
|
|
outputFormatter(self.output[-copyLen:]),
|
|
'"',
|
|
sep='')
|
|
self.contextMode.add(self.output[-2])
|
|
self.contextMode.add(self.output[-1])
|
|
|
|
def figureBlockType(self, kind):
|
|
counts, types = self.currentBlockCounts, self.currentBlockTypes
|
|
if counts[kind]==0:
|
|
newType = self.verboseRead(self.blockTypeCodes[kind])
|
|
if newType==-2: newType = types['P'+kind]
|
|
elif newType==-1:
|
|
newType = (types[kind]+1)%self.numberOfBlockTypes[kind]
|
|
types['P'+kind] = types[kind]
|
|
types[kind] = newType
|
|
counts[kind] = self.verboseRead(self.blockCountCodes[kind])
|
|
counts[kind] -=1
|
|
return types[kind]
|
|
|
|
__test__ = {
|
|
'BitStream': """
|
|
>>> bs = BitStream(b'Jurjen')
|
|
>>> bs.readBytes(2)
|
|
b'Ju'
|
|
>>> bs.read(6) #r=01110010
|
|
50
|
|
>>> bs
|
|
BitStream(pos=2:6)
|
|
>>> bs.peek(5) #j=01101010
|
|
9
|
|
>>> bs.readBytes(2)
|
|
Traceback (most recent call last):
|
|
...
|
|
ValueError: readBytes: need byte boundary
|
|
""",
|
|
|
|
'Symbol': """
|
|
>>> a=Symbol(MetablockLengthAlphabet(),5)
|
|
>>> len(a)
|
|
2
|
|
>>> int(a)
|
|
5
|
|
>>> a.bitPattern()
|
|
'01'
|
|
>>> a.value(200000)
|
|
200001
|
|
>>> a.explanation(300000)
|
|
'data length: 493e0h+1=300001'
|
|
""",
|
|
|
|
'RangeDecoder': """
|
|
>>> a=RangeDecoder(bitLength=3)
|
|
>>> len(a)
|
|
8
|
|
>>> a.name='t'
|
|
>>> list(a)
|
|
[Symbol(t, 0), Symbol(t, 1), Symbol(t, 2), Symbol(t, 3), Symbol(t, 4), Symbol(t, 5), Symbol(t, 6), Symbol(t, 7)]
|
|
>>> a[2]
|
|
Symbol(t, 2)
|
|
>>> a.bitPattern(4)
|
|
'100'
|
|
>>> a.length(2)
|
|
3
|
|
>>> a.decodePeek(15)
|
|
(3, Symbol(t, 7))
|
|
>>>
|
|
|
|
""",
|
|
|
|
'PrefixDecoder': """
|
|
>>> a=PrefixDecoder(decodeTable={0:1,1:2,3:3,7:4})
|
|
>>> len(a)
|
|
4
|
|
>>> a.name='t'
|
|
>>> list(a)
|
|
[Symbol(t, 1), Symbol(t, 2), Symbol(t, 3), Symbol(t, 4)]
|
|
>>> a.decodePeek(22)
|
|
(1, Symbol(t, 1))
|
|
>>> a.decodePeek(27)
|
|
(3, Symbol(t, 3))
|
|
>>> a.length(1)
|
|
1
|
|
>>> a.length(4)
|
|
3
|
|
""",
|
|
|
|
'Code': """
|
|
>>> a=Code('t',alphabetSize=10)
|
|
>>> len(a)
|
|
10
|
|
>>> a.showCode()
|
|
0000:0 0001:1 0010:2 0011:3 0100:4 0101:5 0110:6 0111:7 1000:8 1001:9
|
|
>>> a.setLength({2:1,3:2,5:3,6:3})
|
|
>>> a.showCode()
|
|
0:2 01:3 011:5 111:6
|
|
>>> len(a)
|
|
4
|
|
>>> def callback(i): return 'call{}back'.format(i)
|
|
>>> a=Code('t',callback=callback,bitLength=3)
|
|
>>> a[6].explanation()
|
|
'call6back'
|
|
""",
|
|
|
|
'WithExtra': """
|
|
>>> class A(WithExtra):
|
|
... extraTable = [0,1,1,2,2]
|
|
>>> a=A('t',alphabetSize=5)
|
|
>>> a[1]
|
|
Symbol(t, 1)
|
|
>>> a.extraBits(2)
|
|
1
|
|
>>> a.mnemonic(4)
|
|
'4'
|
|
>>> a.readTupleAndExtra(BitStream(b'\x5b'))
|
|
(3, Symbol(t, 3), 2, 3)
|
|
""",
|
|
|
|
'BoolCode': """
|
|
>>> BoolCode('test')[0].explanation()
|
|
'0: False'
|
|
""",
|
|
|
|
'Enumerator': """
|
|
>>> class A(Enumerator):
|
|
... extraTable = [0,1,1,2,2]
|
|
... value0=3
|
|
>>> a=A(alphabetLength=5)
|
|
>>> a.value(3)
|
|
Traceback (most recent call last):
|
|
...
|
|
TypeError: value() missing 1 required positional argument: 'extra'
|
|
>>> a.explanation(3,4)
|
|
'xx 011: 8-11; 8+4=12'
|
|
""",
|
|
|
|
'WindowSizeAlphabet': """
|
|
>>> windowSizeAlphabet = WindowSizeAlphabet()
|
|
>>> windowSizeAlphabet[0]
|
|
Traceback (most recent call last):
|
|
...
|
|
ValueError: No symbol WindowSizeAlphabet[0]
|
|
>>> len(windowSizeAlphabet)
|
|
16
|
|
>>> windowSizeAlphabet[21]
|
|
Symbol(WSIZE, 21)
|
|
>>> windowSizeAlphabet[21].bitPattern()
|
|
'1001'
|
|
>>> windowSizeAlphabet[21].extraBits()
|
|
0
|
|
>>> windowSizeAlphabet[21].index
|
|
21
|
|
>>> windowSizeAlphabet[10].value()
|
|
1008
|
|
>>> windowSizeAlphabet[10].explanation()
|
|
'windowsize=(1<<10)-16=1008'
|
|
>>> windowSizeAlphabet.showCode()
|
|
0:65520 1100001:16368 1110001:32752 0011:262128
|
|
0000001:131056 0010001:None 1001:2097136 1011:4194288
|
|
1000001:4080 1010001:8176 0101:524272 0111:1048560
|
|
0100001:1008 0110001:2032 1101:8388592 1111:16777200
|
|
""",
|
|
|
|
'TypeCountAlphabet': """
|
|
>>> typeCountAlphabet = TypeCountAlphabet(description='bananas')
|
|
>>> len(typeCountAlphabet)
|
|
9
|
|
>>> typeCountAlphabet[3]
|
|
Symbol(BT#, 3)
|
|
>>> typeCountAlphabet[9]
|
|
Traceback (most recent call last):
|
|
...
|
|
ValueError: No symbol TypeCountAlphabet[9]
|
|
>>> print(typeCountAlphabet[3])
|
|
xx,0101
|
|
>>> typeCountAlphabet[8].value(127)
|
|
256
|
|
>>> typeCountAlphabet[4].explanation(2)
|
|
'xxx,0111: 11 bananas'
|
|
>>> typeCountAlphabet[0].explanation()
|
|
'0: 1 banana'
|
|
""",
|
|
|
|
'DistanceParamAlphabet': """
|
|
>>> dpa = DistanceParamAlphabet()
|
|
>>> dpa.showCode()
|
|
00:PF0 01:PF1 10:PF2 11:PF3
|
|
>>> dpa.readTupleAndExtra(BitStream(b'\\x29'))
|
|
(2, Symbol(DIST, 1), 4, 10)
|
|
>>> dpa.explanation(2, 5)
|
|
'2 postfix bits and 0101<<2=20 direct codes'
|
|
""",
|
|
|
|
'LiteralAlphabet': """
|
|
>>> LiteralAlphabet(-1).showCode() #doctest: +ELLIPSIS
|
|
00000000:\\x00 00110100:4 01101000:h 10011100:\\x9c 11010000:\\xd0
|
|
00000001:\\x01 00110101:5 01101001:i 10011101:\\x9d 11010001:\\xd1
|
|
00000010:\\x02 00110110:6 01101010:j 10011110:\\x9e 11010010:\\xd2
|
|
...
|
|
00101111:/ 01100011:c 10010111:\\x97 11001011:\\xcb 11111111:\\xff
|
|
00110000:0 01100100:d 10011000:\\x98 11001100:\\xcc
|
|
00110001:1 01100101:e 10011001:\\x99 11001101:\\xcd
|
|
00110010:2 01100110:f 10011010:\\x9a 11001110:\\xce
|
|
00110011:3 01100111:g 10011011:\\x9b 11001111:\\xcf
|
|
""",
|
|
|
|
'BlockCountAlphabet': """
|
|
>>> bc=BlockCountAlphabet('BCL')
|
|
>>> len(bc)
|
|
26
|
|
>>> bs=BitStream(b'\\x40\\x83\\xc8\\x59\\12\\x02')
|
|
>>> x = bc.readTupleAndExtra(bs); x[1].explanation(x[3])
|
|
'Block count: xx 00000: 1-4; 1+2=3'
|
|
>>> x = bc.readTupleAndExtra(bs); x[1].explanation(x[3])
|
|
'Block count: xxx 00110: 33-40; 33+0=33'
|
|
>>> x = bc.readTupleAndExtra(bs); x[1].explanation(x[3])
|
|
'Block count: xxxxxx 10001: 305-368; 305+28=333'
|
|
>>> x = bc.readTupleAndExtra(bs); x[1].explanation(x[3])
|
|
'Block count: xxxxxxxxxxx 10110: 2289-4336; 2289+1044=3333'
|
|
""",
|
|
|
|
'Layout': """
|
|
>>> olleke.pos = 0
|
|
>>> l = Layout(olleke)
|
|
>>> l.verboseRead(WindowSizeAlphabet())
|
|
0000 1b 1011 WSIZE windowsize=(1<<22)-16=4194288
|
|
4194288
|
|
>>> l.verboseRead(BoolCode('LAST', description="Last block"))
|
|
1 LAST Last block: 1: True
|
|
True
|
|
>>> l.verboseRead(BoolCode('EMPTY', description="Empty block"))
|
|
0 EMPTY Empty block: 0: False
|
|
False
|
|
>>> l.verboseRead(MetablockLengthAlphabet())
|
|
0001 2e 00 |00h|2Eh,|00 MLEN data length: 002eh+1=47
|
|
47
|
|
>>> olleke.pos = 76
|
|
>>> l = Layout(olleke)
|
|
>>> x = l.verboseRead(DistanceAlphabet(0,NPOSTFIX=0,NDIRECT=0), skipExtra=True)
|
|
000a 82 10|1100 D0 10[15*x]-3
|
|
>>> x.explanation(0x86a3)
|
|
'10[1000011010100011]-3: [0]+100000'
|
|
""",
|
|
|
|
'olleke': """
|
|
>>> olleke.pos = 0
|
|
>>> try: Layout(olleke).processStream()
|
|
... except NotImplementedError: pass
|
|
... #doctest: +REPORT_NDIFF
|
|
addr hex binary context explanation
|
|
-----------------------Stream header------------------------
|
|
0000 1b 1011 WSIZE windowsize=(1<<22)-16=4194288
|
|
======================Metablock header======================
|
|
1 LAST Last block: 1: True
|
|
0 EMPTY Empty block: 0: False
|
|
0001 2e 00 |00h|2Eh,|00 MLEN data length: 002eh+1=47
|
|
-------------------Block type descriptors-------------------
|
|
0003 00 0 BT#L 0: 1 literal block type
|
|
0 BT#I 0: 1 insert© block type
|
|
0 BT#D 0: 1 distance block type
|
|
------------------Distance code parameters------------------
|
|
0004 44 0|000,00 DIST 0 postfix bits and 0000<<0=0 direct codes
|
|
-----------------------Context modes------------------------
|
|
10 LC0 Context mode for type 0: 2(UTF8)
|
|
------------------------Context maps------------------------
|
|
0 LT# 0: 1 literal prefix tree
|
|
0 DT# 0: 1 distance prefix tree
|
|
---------------------Prefix code lists----------------------
|
|
10 PFX L0 is complex with lengths 3,4,0,5,17...
|
|
0005 4f 1|0 ##L0 len 3: coded with 3 bits
|
|
0111 ##L0 len 4: coded with 1 bits
|
|
10 ##L0 unused: coded with 3 bits
|
|
0006 d6 0|0 ##L0 len 5: skipped
|
|
011 ##L0 zero xxx: coded with 2 bits
|
|
***** Lengths for L0 will be coded as:
|
|
0:len 4 01:zero xxx 011:unused 111:len 3
|
|
0007 95 1|11,01 #L0 7+3 unused
|
|
0 #L0 Length for \\n is 4 bits
|
|
001,01 #L0 1+3 unused
|
|
0008 44 010,0|1 #L0 total 19+2 unused
|
|
0 #L0 Length for " " is 4 bits
|
|
0 #L0 Length for ! is 4 bits
|
|
0009 cb 011,|01 #L0 3+3 unused
|
|
|110,01 #L0 total 35+6 unused
|
|
000a 82 0 #L0 Length for K is 4 bits
|
|
000,01 #L0 0+3 unused
|
|
0 #L0 Length for O is 4 bits
|
|
000b 4d 01|1 #L0 symbol P unused
|
|
011 #L0 symbol Q unused
|
|
0 #L0 Length for R is 4 bits
|
|
000c 88 000,|01 #L0 0+3 unused
|
|
|100,01 #L0 total 11+4 unused
|
|
000d b6 0 #L0 Length for b is 4 bits
|
|
011 #L0 symbol c unused
|
|
011 #L0 symbol d unused
|
|
000e 27 11|1 #L0 Length for e is 3 bits
|
|
010,01 #L0 2+3 unused
|
|
|0 #L0 Length for k is 4 bits
|
|
000f 1f 111 #L0 Length for l is 3 bits
|
|
011 #L0 symbol m unused
|
|
0 #L0 Length for n is 4 bits
|
|
|0 #L0 Length for o is 4 bits
|
|
0010 c1 000,01 #L0 0+3 unused
|
|
0 #L0 Length for s is 4 bits
|
|
0011 b4 0|11 #L0 symbol t unused
|
|
0 #L0 Length for u is 4 bits
|
|
End of table. Prefix code L0:
|
|
000:e 0010:\\n 0110:! 0001:O 0101:b 0011:n 0111:s
|
|
100:l 1010:" " 1110:K 1001:R 1101:k 1011:o 1111:u
|
|
11,01 PFX IC0 is simple with 4 code words
|
|
0012 2a |2Ah|10 IC0 ? bits: I5C4
|
|
0013 b5 ec 00|B5h IC0 ? bits: I6+xC7
|
|
0015 22 0010|111011 IC0 ? bits: I8+xC5
|
|
0016 8c 001100|0010 IC0 ? bits: I0C14+xx
|
|
0 SHAPE False: lengths 2,2,2,2
|
|
0017 74 10,0|1 PFX D0 is simple with 3 code words
|
|
0018 a6 0|01110 D0 1 bit: 2last-3
|
|
010011 D0 2 bits: 11xx-3
|
|
0019 aa 01010|1 D0 2 bits: 11xxx-3
|
|
====================Meta block contents=====================
|
|
|1,01 IC0 Literal: 9, copy: 5
|
|
001a 41 0001 0,0=L0 O
|
|
100 0,48=L0 l
|
|
001b a2 10|0 0,62=L0 l
|
|
000 0,63=L0 e
|
|
001c a1 1|101 0,59=L0 k
|
|
000 0,63=L0 e
|
|
|1010 0,59=L0 " "
|
|
001d b5 0101 0,11=L0 b
|
|
|1011 0,60=L0 o
|
|
001e 24 0 0,3=D0 Distance: 2last-3=8
|
|
Seen before: "lleke"
|
|
0,10 IC0 Literal: 6, copy: 7
|
|
|0010 0,59=L0 \\n
|
|
001f 89 1001 0,7=L0 R
|
|
000 0,52=L0 e
|
|
0020 fa 010|1 0,58=L0 b
|
|
1111 0,63=L0 u
|
|
0021 eb 011|1 0,59=L0 s
|
|
11,01 0,3=D0 Absolute value: 12 (pos 8)
|
|
Seen before: "olleke\\n"
|
|
0022 db 01,1|1 IC0 Literal: 0, copy: 15
|
|
|110,11 0,3=D0 Absolute value: 27 (pos 0)
|
|
Seen before: "Olleke bolleke\\n"
|
|
0023 f8 00 IC0 Literal: 5, copy: 4
|
|
1110 0,7=L0 K
|
|
0024 2c 00|11 0,52=L0 n
|
|
1011 0,62=L0 o
|
|
0025 0d 1|00 0,59=L0 l
|
|
0110 0,63=L0 !
|
|
""",
|
|
|
|
'file': """
|
|
>>> try: Layout(BitStream(
|
|
... open("H:/Downloads/brotli-master/tests/testdata/10x10y.compressed",'rb')
|
|
... .read())).processStream()
|
|
... except NotImplementedError: pass
|
|
addr hex binary context explanation
|
|
-----------------------Stream header------------------------
|
|
0000 1b 1011 WSIZE windowsize=(1<<22)-16=4194288
|
|
======================Metablock header======================
|
|
1 LAST Last block: 1: True
|
|
0 EMPTY Empty block: 0: False
|
|
0001 13 00 |00h|13h,|00 MLEN data length: 0013h+1=20
|
|
-------------------Block type descriptors-------------------
|
|
0003 00 0 BT#L 0: 1 literal block type
|
|
0 BT#I 0: 1 insert© block type
|
|
0 BT#D 0: 1 distance block type
|
|
------------------Distance code parameters------------------
|
|
0004 a4 0|000,00 DIST 0 postfix bits and 0000<<0=0 direct codes
|
|
-----------------------Context modes------------------------
|
|
10 LC0 Context mode for type 0: 2(UTF8)
|
|
------------------------Context maps------------------------
|
|
0 LT# 0: 1 literal prefix tree
|
|
0 DT# 0: 1 distance prefix tree
|
|
---------------------Prefix code lists----------------------
|
|
0005 b0 0|1,01 PFX L0 is simple with 2 code words
|
|
0006 b2 0|1011000 L0 1 bit: X
|
|
0007 ea 0|1011001 L0 1 bit: Y
|
|
01,01 PFX IC0 is simple with 2 code words
|
|
0008 81 0000001|111 IC0 1 bit: I1C9&D=0
|
|
0009 47 02 0|47h|1 IC0 1 bit: I1C9
|
|
00,01 PFX D0 is simple with 1 code word
|
|
000b 8a 010|000 D0 0 bits: 10x-3
|
|
====================Meta block contents=====================
|
|
1 IC0 Literal: 1, copy: 9
|
|
0 0,0=L0 X
|
|
0,() 0,3=D0 Absolute value: 1 (pos 0)
|
|
Seen before: "XXXXXXXXX"
|
|
0 IC0 Literal: 1, copy: 9, same distance
|
|
|1 0,54=L0 Y
|
|
Seen before: "YYYYYYYYY"
|
|
""",
|
|
|
|
'XY': """
|
|
>>> try: Layout(BitStream(brotli.compress('X'*10+'Y'*10))).processStream()
|
|
... except NotImplementedError: pass
|
|
addr hex binary context explanation
|
|
-----------------------Stream header------------------------
|
|
0000 1b 1011 WSIZE windowsize=(1<<22)-16=4194288
|
|
======================Metablock header======================
|
|
1 LAST Last block: 1: True
|
|
0 EMPTY Empty block: 0: False
|
|
0001 13 00 |00h|13h,|00 MLEN data length: 0013h+1=20
|
|
-------------------Block type descriptors-------------------
|
|
0003 00 0 BT#L 0: 1 literal block type
|
|
0 BT#I 0: 1 insert© block type
|
|
0 BT#D 0: 1 distance block type
|
|
------------------Distance code parameters------------------
|
|
0004 a4 0|000,00 DIST 0 postfix bits and 0000<<0=0 direct codes
|
|
-----------------------Context modes------------------------
|
|
10 LC0 Context mode for type 0: 2(UTF8)
|
|
------------------------Context maps------------------------
|
|
0 LT# 0: 1 literal prefix tree
|
|
0 DT# 0: 1 distance prefix tree
|
|
---------------------Prefix code lists----------------------
|
|
0005 b0 0|1,01 PFX L0 is simple with 2 code words
|
|
0006 b2 0|1011000 L0 1 bit: X
|
|
0007 82 0|1011001 L0 1 bit: Y
|
|
00,01 PFX IC0 is simple with 1 code word
|
|
0008 84 0000100|100 IC0 0 bits: I4C6&D=0
|
|
0009 00 00,0|1 PFX D0 is simple with 1 code word
|
|
000a e0 0|00000 D0 0 bits: last
|
|
====================Meta block contents=====================
|
|
() IC0 Literal: 4, copy: 6, same distance
|
|
0 0,0=L0 X
|
|
0 0,52=L0 X
|
|
0 0,54=L0 X
|
|
0 0,54=L0 X
|
|
Seen before: "XXXXXX"
|
|
() IC0 Literal: 4, copy: 6, same distance
|
|
1 0,54=L0 Y
|
|
1 0,54=L0 Y
|
|
|1 0,54=L0 Y
|
|
000b 01 1 0,54=L0 Y
|
|
Seen before: "YYYYYY"
|
|
""",
|
|
|
|
'empty': """
|
|
>>> try: Layout(BitStream(b'\\x81\\x16\\x00\\x58')).processStream()
|
|
... except NotImplementedError: pass
|
|
addr hex binary context explanation
|
|
-----------------------Stream header------------------------
|
|
0000 81 0000001 WSIZE windowsize=(1<<17)-16=131056
|
|
======================Metablock header======================
|
|
|1 LAST Last block: 1: True
|
|
0001 16 0 EMPTY Empty block: 0: False
|
|
11 MLEN 11: empty block
|
|
0 RSVD Reserved (must be zero)
|
|
0002 00 000000|00,01 SKIP skip length: 0h+1=1
|
|
|00 SKIP 2 bits ignored
|
|
Skipping to 4
|
|
""",
|
|
|
|
}
|
|
|
|
if __name__=='__main__':
|
|
import sys
|
|
if len(sys.argv)>1:
|
|
l = Layout(BitStream(open(sys.argv[1],'rb').read()))
|
|
l.processStream()
|
|
else:
|
|
sys.path.append("h:/Persoonlijk/bin")
|
|
try:
|
|
import brotli
|
|
open('brotlidump.br', 'wb').write(
|
|
brotli.compress(
|
|
open('brotlidump.py', 'r').read()
|
|
))
|
|
olleke = BitStream(brotli.compress(
|
|
'Olleke bolleke\nRebusolleke\nOlleke bolleke\nKnol!'))
|
|
except ImportError: pass
|
|
import doctest
|
|
doctest.testmod(optionflags=doctest.REPORT_NDIFF
|
|
#|doctest.FAIL_FAST
|
|
)
|