mirror of https://github.com/go-gitea/gitea
Dump: add output format tar and output to stdout (#10376)
* Dump: Use mholt/archive/v3 to support tar including many compressions Signed-off-by: Philipp Homann <homann.philipp@googlemail.com> * Dump: Allow dump output to stdout Signed-off-by: Philipp Homann <homann.philipp@googlemail.com> * Dump: Fixed bug present since #6677 where SessionConfig.Provider is never "file" Signed-off-by: Philipp Homann <homann.philipp@googlemail.com> * Dump: never pack RepoRootPath, LFS.ContentPath and LogRootPath when they are below AppDataPath Signed-off-by: Philipp Homann <homann.philipp@googlemail.com> * Dump: also dump LFS (fixes #10058) Signed-off-by: Philipp Homann <homann.philipp@googlemail.com> * Dump: never dump CustomPath if CustomPath is a subdir of or equal to AppDataPath (fixes #10365) Signed-off-by: Philipp Homann <homann.philipp@googlemail.com> * Use log.Info instead of fmt.Fprintf Signed-off-by: Philipp Homann <homann.philipp@googlemail.com> * import ordering * make fmt Co-authored-by: zeripath <art27@cantab.net> Co-authored-by: techknowlogick <techknowlogick@gitea.io> Co-authored-by: Matti R <matti@mdranta.net>pull/11769/head^2
PhilippHomann
5 years ago
committed by
GitHub
parent
209b17c4e2
commit
684b7a999f
@ -0,0 +1,19 @@ |
||||
Copyright (c) 2009, 2010, 2013-2016 by the Brotli Authors. |
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy |
||||
of this software and associated documentation files (the "Software"), to deal |
||||
in the Software without restriction, including without limitation the rights |
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
||||
copies of the Software, and to permit persons to whom the Software is |
||||
furnished to do so, subject to the following conditions: |
||||
|
||||
The above copyright notice and this permission notice shall be included in |
||||
all copies or substantial portions of the Software. |
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN |
||||
THE SOFTWARE. |
||||
@ -0,0 +1,5 @@ |
||||
This package is a brotli compressor and decompressor implemented in Go. |
||||
It was translated from the reference implementation (https://github.com/google/brotli) |
||||
with the `c2go` tool at https://github.com/andybalholm/c2go. |
||||
|
||||
I am using it in production with https://github.com/andybalholm/redwood. |
||||
@ -0,0 +1,177 @@ |
||||
package brotli |
||||
|
||||
/* Copyright 2013 Google Inc. All Rights Reserved. |
||||
|
||||
Distributed under MIT license. |
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/ |
||||
|
||||
/* Function to find backward reference copies. */ |
||||
|
||||
func computeDistanceCode(distance uint, max_distance uint, dist_cache []int) uint { |
||||
if distance <= max_distance { |
||||
var distance_plus_3 uint = distance + 3 |
||||
var offset0 uint = distance_plus_3 - uint(dist_cache[0]) |
||||
var offset1 uint = distance_plus_3 - uint(dist_cache[1]) |
||||
if distance == uint(dist_cache[0]) { |
||||
return 0 |
||||
} else if distance == uint(dist_cache[1]) { |
||||
return 1 |
||||
} else if offset0 < 7 { |
||||
return (0x9750468 >> (4 * offset0)) & 0xF |
||||
} else if offset1 < 7 { |
||||
return (0xFDB1ACE >> (4 * offset1)) & 0xF |
||||
} else if distance == uint(dist_cache[2]) { |
||||
return 2 |
||||
} else if distance == uint(dist_cache[3]) { |
||||
return 3 |
||||
} |
||||
} |
||||
|
||||
return distance + numDistanceShortCodes - 1 |
||||
} |
||||
|
||||
/* "commands" points to the next output command to write to, "*num_commands" is |
||||
initially the total amount of commands output by previous |
||||
CreateBackwardReferences calls, and must be incremented by the amount written |
||||
by this call. */ |
||||
func createBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher hasherHandle, dist_cache []int, last_insert_len *uint, commands []command, num_commands *uint, num_literals *uint) { |
||||
var max_backward_limit uint = maxBackwardLimit(params.lgwin) |
||||
var orig_commands []command = commands |
||||
var insert_length uint = *last_insert_len |
||||
var pos_end uint = position + num_bytes |
||||
var store_end uint |
||||
if num_bytes >= hasher.StoreLookahead() { |
||||
store_end = position + num_bytes - hasher.StoreLookahead() + 1 |
||||
} else { |
||||
store_end = position |
||||
} |
||||
var random_heuristics_window_size uint = literalSpreeLengthForSparseSearch(params) |
||||
var apply_random_heuristics uint = position + random_heuristics_window_size |
||||
var gap uint = 0 |
||||
/* Set maximum distance, see section 9.1. of the spec. */ |
||||
|
||||
const kMinScore uint = scoreBase + 100 |
||||
|
||||
/* For speed up heuristics for random data. */ |
||||
|
||||
/* Minimum score to accept a backward reference. */ |
||||
hasher.PrepareDistanceCache(dist_cache) |
||||
var sr2 hasherSearchResult |
||||
var sr hasherSearchResult |
||||
|
||||
for position+hasher.HashTypeLength() < pos_end { |
||||
var max_length uint = pos_end - position |
||||
var max_distance uint = brotli_min_size_t(position, max_backward_limit) |
||||
sr.len = 0 |
||||
sr.len_code_delta = 0 |
||||
sr.distance = 0 |
||||
sr.score = kMinScore |
||||
hasher.FindLongestMatch(¶ms.dictionary, ringbuffer, ringbuffer_mask, dist_cache, position, max_length, max_distance, gap, params.dist.max_distance, &sr) |
||||
if sr.score > kMinScore { |
||||
/* Found a match. Let's look for something even better ahead. */ |
||||
var delayed_backward_references_in_row int = 0 |
||||
max_length-- |
||||
for ; ; max_length-- { |
||||
var cost_diff_lazy uint = 175 |
||||
if params.quality < minQualityForExtensiveReferenceSearch { |
||||
sr2.len = brotli_min_size_t(sr.len-1, max_length) |
||||
} else { |
||||
sr2.len = 0 |
||||
} |
||||
sr2.len_code_delta = 0 |
||||
sr2.distance = 0 |
||||
sr2.score = kMinScore |
||||
max_distance = brotli_min_size_t(position+1, max_backward_limit) |
||||
hasher.FindLongestMatch(¶ms.dictionary, ringbuffer, ringbuffer_mask, dist_cache, position+1, max_length, max_distance, gap, params.dist.max_distance, &sr2) |
||||
if sr2.score >= sr.score+cost_diff_lazy { |
||||
/* Ok, let's just write one byte for now and start a match from the |
||||
next byte. */ |
||||
position++ |
||||
|
||||
insert_length++ |
||||
sr = sr2 |
||||
delayed_backward_references_in_row++ |
||||
if delayed_backward_references_in_row < 4 && position+hasher.HashTypeLength() < pos_end { |
||||
continue |
||||
} |
||||
} |
||||
|
||||
break |
||||
} |
||||
|
||||
apply_random_heuristics = position + 2*sr.len + random_heuristics_window_size |
||||
max_distance = brotli_min_size_t(position, max_backward_limit) |
||||
{ |
||||
/* The first 16 codes are special short-codes, |
||||
and the minimum offset is 1. */ |
||||
var distance_code uint = computeDistanceCode(sr.distance, max_distance+gap, dist_cache) |
||||
if (sr.distance <= (max_distance + gap)) && distance_code > 0 { |
||||
dist_cache[3] = dist_cache[2] |
||||
dist_cache[2] = dist_cache[1] |
||||
dist_cache[1] = dist_cache[0] |
||||
dist_cache[0] = int(sr.distance) |
||||
hasher.PrepareDistanceCache(dist_cache) |
||||
} |
||||
|
||||
initCommand(&commands[0], ¶ms.dist, insert_length, sr.len, sr.len_code_delta, distance_code) |
||||
commands = commands[1:] |
||||
} |
||||
|
||||
*num_literals += insert_length |
||||
insert_length = 0 |
||||
/* Put the hash keys into the table, if there are enough bytes left. |
||||
Depending on the hasher implementation, it can push all positions |
||||
in the given range or only a subset of them. |
||||
Avoid hash poisoning with RLE data. */ |
||||
{ |
||||
var range_start uint = position + 2 |
||||
var range_end uint = brotli_min_size_t(position+sr.len, store_end) |
||||
if sr.distance < sr.len>>2 { |
||||
range_start = brotli_min_size_t(range_end, brotli_max_size_t(range_start, position+sr.len-(sr.distance<<2))) |
||||
} |
||||
|
||||
hasher.StoreRange(ringbuffer, ringbuffer_mask, range_start, range_end) |
||||
} |
||||
|
||||
position += sr.len |
||||
} else { |
||||
insert_length++ |
||||
position++ |
||||
|
||||
/* If we have not seen matches for a long time, we can skip some |
||||
match lookups. Unsuccessful match lookups are very very expensive |
||||
and this kind of a heuristic speeds up compression quite |
||||
a lot. */ |
||||
if position > apply_random_heuristics { |
||||
/* Going through uncompressible data, jump. */ |
||||
if position > apply_random_heuristics+4*random_heuristics_window_size { |
||||
var kMargin uint = brotli_max_size_t(hasher.StoreLookahead()-1, 4) |
||||
/* It is quite a long time since we saw a copy, so we assume |
||||
that this data is not compressible, and store hashes less |
||||
often. Hashes of non compressible data are less likely to |
||||
turn out to be useful in the future, too, so we store less of |
||||
them to not to flood out the hash table of good compressible |
||||
data. */ |
||||
|
||||
var pos_jump uint = brotli_min_size_t(position+16, pos_end-kMargin) |
||||
for ; position < pos_jump; position += 4 { |
||||
hasher.Store(ringbuffer, ringbuffer_mask, position) |
||||
insert_length += 4 |
||||
} |
||||
} else { |
||||
var kMargin uint = brotli_max_size_t(hasher.StoreLookahead()-1, 2) |
||||
var pos_jump uint = brotli_min_size_t(position+8, pos_end-kMargin) |
||||
for ; position < pos_jump; position += 2 { |
||||
hasher.Store(ringbuffer, ringbuffer_mask, position) |
||||
insert_length += 2 |
||||
} |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
insert_length += pos_end - position |
||||
*last_insert_len = insert_length |
||||
*num_commands += uint(-cap(commands) + cap(orig_commands)) |
||||
} |
||||
@ -0,0 +1,795 @@ |
||||
package brotli |
||||
|
||||
import "math" |
||||
|
||||
type zopfliNode struct { |
||||
length uint32 |
||||
distance uint32 |
||||
dcode_insert_length uint32 |
||||
u struct { |
||||
cost float32 |
||||
next uint32 |
||||
shortcut uint32 |
||||
} |
||||
} |
||||
|
||||
const maxEffectiveDistanceAlphabetSize = 544 |
||||
|
||||
const kInfinity float32 = 1.7e38 /* ~= 2 ^ 127 */ |
||||
|
||||
var kDistanceCacheIndex = []uint32{0, 1, 2, 3, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1} |
||||
|
||||
var kDistanceCacheOffset = []int{0, 0, 0, 0, -1, 1, -2, 2, -3, 3, -1, 1, -2, 2, -3, 3} |
||||
|
||||
func initZopfliNodes(array []zopfliNode, length uint) { |
||||
var stub zopfliNode |
||||
var i uint |
||||
stub.length = 1 |
||||
stub.distance = 0 |
||||
stub.dcode_insert_length = 0 |
||||
stub.u.cost = kInfinity |
||||
for i = 0; i < length; i++ { |
||||
array[i] = stub |
||||
} |
||||
} |
||||
|
||||
func zopfliNodeCopyLength(self *zopfliNode) uint32 { |
||||
return self.length & 0x1FFFFFF |
||||
} |
||||
|
||||
func zopfliNodeLengthCode(self *zopfliNode) uint32 { |
||||
var modifier uint32 = self.length >> 25 |
||||
return zopfliNodeCopyLength(self) + 9 - modifier |
||||
} |
||||
|
||||
func zopfliNodeCopyDistance(self *zopfliNode) uint32 { |
||||
return self.distance |
||||
} |
||||
|
||||
func zopfliNodeDistanceCode(self *zopfliNode) uint32 { |
||||
var short_code uint32 = self.dcode_insert_length >> 27 |
||||
if short_code == 0 { |
||||
return zopfliNodeCopyDistance(self) + numDistanceShortCodes - 1 |
||||
} else { |
||||
return short_code - 1 |
||||
} |
||||
} |
||||
|
||||
func zopfliNodeCommandLength(self *zopfliNode) uint32 { |
||||
return zopfliNodeCopyLength(self) + (self.dcode_insert_length & 0x7FFFFFF) |
||||
} |
||||
|
||||
/* Histogram based cost model for zopflification. */ |
||||
type zopfliCostModel struct { |
||||
cost_cmd_ [numCommandSymbols]float32 |
||||
cost_dist_ []float32 |
||||
distance_histogram_size uint32 |
||||
literal_costs_ []float32 |
||||
min_cost_cmd_ float32 |
||||
num_bytes_ uint |
||||
} |
||||
|
||||
func initZopfliCostModel(self *zopfliCostModel, dist *distanceParams, num_bytes uint) { |
||||
var distance_histogram_size uint32 = dist.alphabet_size |
||||
if distance_histogram_size > maxEffectiveDistanceAlphabetSize { |
||||
distance_histogram_size = maxEffectiveDistanceAlphabetSize |
||||
} |
||||
|
||||
self.num_bytes_ = num_bytes |
||||
self.literal_costs_ = make([]float32, (num_bytes + 2)) |
||||
self.cost_dist_ = make([]float32, (dist.alphabet_size)) |
||||
self.distance_histogram_size = distance_histogram_size |
||||
} |
||||
|
||||
func cleanupZopfliCostModel(self *zopfliCostModel) { |
||||
self.literal_costs_ = nil |
||||
self.cost_dist_ = nil |
||||
} |
||||
|
||||
func setCost(histogram []uint32, histogram_size uint, literal_histogram bool, cost []float32) { |
||||
var sum uint = 0 |
||||
var missing_symbol_sum uint |
||||
var log2sum float32 |
||||
var missing_symbol_cost float32 |
||||
var i uint |
||||
for i = 0; i < histogram_size; i++ { |
||||
sum += uint(histogram[i]) |
||||
} |
||||
|
||||
log2sum = float32(fastLog2(sum)) |
||||
missing_symbol_sum = sum |
||||
if !literal_histogram { |
||||
for i = 0; i < histogram_size; i++ { |
||||
if histogram[i] == 0 { |
||||
missing_symbol_sum++ |
||||
} |
||||
} |
||||
} |
||||
|
||||
missing_symbol_cost = float32(fastLog2(missing_symbol_sum)) + 2 |
||||
for i = 0; i < histogram_size; i++ { |
||||
if histogram[i] == 0 { |
||||
cost[i] = missing_symbol_cost |
||||
continue |
||||
} |
||||
|
||||
/* Shannon bits for this symbol. */ |
||||
cost[i] = log2sum - float32(fastLog2(uint(histogram[i]))) |
||||
|
||||
/* Cannot be coded with less than 1 bit */ |
||||
if cost[i] < 1 { |
||||
cost[i] = 1 |
||||
} |
||||
} |
||||
} |
||||
|
||||
func zopfliCostModelSetFromCommands(self *zopfliCostModel, position uint, ringbuffer []byte, ringbuffer_mask uint, commands []command, num_commands uint, last_insert_len uint) { |
||||
var histogram_literal [numLiteralSymbols]uint32 |
||||
var histogram_cmd [numCommandSymbols]uint32 |
||||
var histogram_dist [maxEffectiveDistanceAlphabetSize]uint32 |
||||
var cost_literal [numLiteralSymbols]float32 |
||||
var pos uint = position - last_insert_len |
||||
var min_cost_cmd float32 = kInfinity |
||||
var i uint |
||||
var cost_cmd []float32 = self.cost_cmd_[:] |
||||
var literal_costs []float32 |
||||
|
||||
histogram_literal = [numLiteralSymbols]uint32{} |
||||
histogram_cmd = [numCommandSymbols]uint32{} |
||||
histogram_dist = [maxEffectiveDistanceAlphabetSize]uint32{} |
||||
|
||||
for i = 0; i < num_commands; i++ { |
||||
var inslength uint = uint(commands[i].insert_len_) |
||||
var copylength uint = uint(commandCopyLen(&commands[i])) |
||||
var distcode uint = uint(commands[i].dist_prefix_) & 0x3FF |
||||
var cmdcode uint = uint(commands[i].cmd_prefix_) |
||||
var j uint |
||||
|
||||
histogram_cmd[cmdcode]++ |
||||
if cmdcode >= 128 { |
||||
histogram_dist[distcode]++ |
||||
} |
||||
|
||||
for j = 0; j < inslength; j++ { |
||||
histogram_literal[ringbuffer[(pos+j)&ringbuffer_mask]]++ |
||||
} |
||||
|
||||
pos += inslength + copylength |
||||
} |
||||
|
||||
setCost(histogram_literal[:], numLiteralSymbols, true, cost_literal[:]) |
||||
setCost(histogram_cmd[:], numCommandSymbols, false, cost_cmd) |
||||
setCost(histogram_dist[:], uint(self.distance_histogram_size), false, self.cost_dist_) |
||||
|
||||
for i = 0; i < numCommandSymbols; i++ { |
||||
min_cost_cmd = brotli_min_float(min_cost_cmd, cost_cmd[i]) |
||||
} |
||||
|
||||
self.min_cost_cmd_ = min_cost_cmd |
||||
{ |
||||
literal_costs = self.literal_costs_ |
||||
var literal_carry float32 = 0.0 |
||||
var num_bytes uint = self.num_bytes_ |
||||
literal_costs[0] = 0.0 |
||||
for i = 0; i < num_bytes; i++ { |
||||
literal_carry += cost_literal[ringbuffer[(position+i)&ringbuffer_mask]] |
||||
literal_costs[i+1] = literal_costs[i] + literal_carry |
||||
literal_carry -= literal_costs[i+1] - literal_costs[i] |
||||
} |
||||
} |
||||
} |
||||
|
||||
func zopfliCostModelSetFromLiteralCosts(self *zopfliCostModel, position uint, ringbuffer []byte, ringbuffer_mask uint) { |
||||
var literal_costs []float32 = self.literal_costs_ |
||||
var literal_carry float32 = 0.0 |
||||
var cost_dist []float32 = self.cost_dist_ |
||||
var cost_cmd []float32 = self.cost_cmd_[:] |
||||
var num_bytes uint = self.num_bytes_ |
||||
var i uint |
||||
estimateBitCostsForLiterals(position, num_bytes, ringbuffer_mask, ringbuffer, literal_costs[1:]) |
||||
literal_costs[0] = 0.0 |
||||
for i = 0; i < num_bytes; i++ { |
||||
literal_carry += literal_costs[i+1] |
||||
literal_costs[i+1] = literal_costs[i] + literal_carry |
||||
literal_carry -= literal_costs[i+1] - literal_costs[i] |
||||
} |
||||
|
||||
for i = 0; i < numCommandSymbols; i++ { |
||||
cost_cmd[i] = float32(fastLog2(uint(11 + uint32(i)))) |
||||
} |
||||
|
||||
for i = 0; uint32(i) < self.distance_histogram_size; i++ { |
||||
cost_dist[i] = float32(fastLog2(uint(20 + uint32(i)))) |
||||
} |
||||
|
||||
self.min_cost_cmd_ = float32(fastLog2(11)) |
||||
} |
||||
|
||||
func zopfliCostModelGetCommandCost(self *zopfliCostModel, cmdcode uint16) float32 { |
||||
return self.cost_cmd_[cmdcode] |
||||
} |
||||
|
||||
func zopfliCostModelGetDistanceCost(self *zopfliCostModel, distcode uint) float32 { |
||||
return self.cost_dist_[distcode] |
||||
} |
||||
|
||||
func zopfliCostModelGetLiteralCosts(self *zopfliCostModel, from uint, to uint) float32 { |
||||
return self.literal_costs_[to] - self.literal_costs_[from] |
||||
} |
||||
|
||||
func zopfliCostModelGetMinCostCmd(self *zopfliCostModel) float32 { |
||||
return self.min_cost_cmd_ |
||||
} |
||||
|
||||
/* REQUIRES: len >= 2, start_pos <= pos */ |
||||
/* REQUIRES: cost < kInfinity, nodes[start_pos].cost < kInfinity */ |
||||
/* Maintains the "ZopfliNode array invariant". */ |
||||
func updateZopfliNode(nodes []zopfliNode, pos uint, start_pos uint, len uint, len_code uint, dist uint, short_code uint, cost float32) { |
||||
var next *zopfliNode = &nodes[pos+len] |
||||
next.length = uint32(len | (len+9-len_code)<<25) |
||||
next.distance = uint32(dist) |
||||
next.dcode_insert_length = uint32(short_code<<27 | (pos - start_pos)) |
||||
next.u.cost = cost |
||||
} |
||||
|
||||
type posData struct { |
||||
pos uint |
||||
distance_cache [4]int |
||||
costdiff float32 |
||||
cost float32 |
||||
} |
||||
|
||||
/* Maintains the smallest 8 cost difference together with their positions */ |
||||
type startPosQueue struct { |
||||
q_ [8]posData |
||||
idx_ uint |
||||
} |
||||
|
||||
func initStartPosQueue(self *startPosQueue) { |
||||
self.idx_ = 0 |
||||
} |
||||
|
||||
func startPosQueueSize(self *startPosQueue) uint { |
||||
return brotli_min_size_t(self.idx_, 8) |
||||
} |
||||
|
||||
func startPosQueuePush(self *startPosQueue, posdata *posData) { |
||||
var offset uint = ^(self.idx_) & 7 |
||||
self.idx_++ |
||||
var len uint = startPosQueueSize(self) |
||||
var i uint |
||||
var q []posData = self.q_[:] |
||||
q[offset] = *posdata |
||||
|
||||
/* Restore the sorted order. In the list of |len| items at most |len - 1| |
||||
adjacent element comparisons / swaps are required. */ |
||||
for i = 1; i < len; i++ { |
||||
if q[offset&7].costdiff > q[(offset+1)&7].costdiff { |
||||
var tmp posData = q[offset&7] |
||||
q[offset&7] = q[(offset+1)&7] |
||||
q[(offset+1)&7] = tmp |
||||
} |
||||
|
||||
offset++ |
||||
} |
||||
} |
||||
|
||||
func startPosQueueAt(self *startPosQueue, k uint) *posData { |
||||
return &self.q_[(k-self.idx_)&7] |
||||
} |
||||
|
||||
/* Returns the minimum possible copy length that can improve the cost of any */ |
||||
/* future position. */ |
||||
func computeMinimumCopyLength(start_cost float32, nodes []zopfliNode, num_bytes uint, pos uint) uint { |
||||
var min_cost float32 = start_cost |
||||
var len uint = 2 |
||||
var next_len_bucket uint = 4 |
||||
/* Compute the minimum possible cost of reaching any future position. */ |
||||
|
||||
var next_len_offset uint = 10 |
||||
for pos+len <= num_bytes && nodes[pos+len].u.cost <= min_cost { |
||||
/* We already reached (pos + len) with no more cost than the minimum |
||||
possible cost of reaching anything from this pos, so there is no point in |
||||
looking for lengths <= len. */ |
||||
len++ |
||||
|
||||
if len == next_len_offset { |
||||
/* We reached the next copy length code bucket, so we add one more |
||||
extra bit to the minimum cost. */ |
||||
min_cost += 1.0 |
||||
|
||||
next_len_offset += next_len_bucket |
||||
next_len_bucket *= 2 |
||||
} |
||||
} |
||||
|
||||
return uint(len) |
||||
} |
||||
|
||||
/* REQUIRES: nodes[pos].cost < kInfinity |
||||
REQUIRES: nodes[0..pos] satisfies that "ZopfliNode array invariant". */ |
||||
func computeDistanceShortcut(block_start uint, pos uint, max_backward_limit uint, gap uint, nodes []zopfliNode) uint32 { |
||||
var clen uint = uint(zopfliNodeCopyLength(&nodes[pos])) |
||||
var ilen uint = uint(nodes[pos].dcode_insert_length & 0x7FFFFFF) |
||||
var dist uint = uint(zopfliNodeCopyDistance(&nodes[pos])) |
||||
|
||||
/* Since |block_start + pos| is the end position of the command, the copy part |
||||
starts from |block_start + pos - clen|. Distances that are greater than |
||||
this or greater than |max_backward_limit| + |gap| are static dictionary |
||||
references, and do not update the last distances. |
||||
Also distance code 0 (last distance) does not update the last distances. */ |
||||
if pos == 0 { |
||||
return 0 |
||||
} else if dist+clen <= block_start+pos+gap && dist <= max_backward_limit+gap && zopfliNodeDistanceCode(&nodes[pos]) > 0 { |
||||
return uint32(pos) |
||||
} else { |
||||
return nodes[pos-clen-ilen].u.shortcut |
||||
} |
||||
} |
||||
|
||||
/* Fills in dist_cache[0..3] with the last four distances (as defined by |
||||
Section 4. of the Spec) that would be used at (block_start + pos) if we |
||||
used the shortest path of commands from block_start, computed from |
||||
nodes[0..pos]. The last four distances at block_start are in |
||||
starting_dist_cache[0..3]. |
||||
REQUIRES: nodes[pos].cost < kInfinity |
||||
REQUIRES: nodes[0..pos] satisfies that "ZopfliNode array invariant". */ |
||||
func computeDistanceCache(pos uint, starting_dist_cache []int, nodes []zopfliNode, dist_cache []int) { |
||||
var idx int = 0 |
||||
var p uint = uint(nodes[pos].u.shortcut) |
||||
for idx < 4 && p > 0 { |
||||
var ilen uint = uint(nodes[p].dcode_insert_length & 0x7FFFFFF) |
||||
var clen uint = uint(zopfliNodeCopyLength(&nodes[p])) |
||||
var dist uint = uint(zopfliNodeCopyDistance(&nodes[p])) |
||||
dist_cache[idx] = int(dist) |
||||
idx++ |
||||
|
||||
/* Because of prerequisite, p >= clen + ilen >= 2. */ |
||||
p = uint(nodes[p-clen-ilen].u.shortcut) |
||||
} |
||||
|
||||
for ; idx < 4; idx++ { |
||||
dist_cache[idx] = starting_dist_cache[0] |
||||
starting_dist_cache = starting_dist_cache[1:] |
||||
} |
||||
} |
||||
|
||||
/* Maintains "ZopfliNode array invariant" and pushes node to the queue, if it |
||||
is eligible. */ |
||||
func evaluateNode(block_start uint, pos uint, max_backward_limit uint, gap uint, starting_dist_cache []int, model *zopfliCostModel, queue *startPosQueue, nodes []zopfliNode) { |
||||
/* Save cost, because ComputeDistanceCache invalidates it. */ |
||||
var node_cost float32 = nodes[pos].u.cost |
||||
nodes[pos].u.shortcut = computeDistanceShortcut(block_start, pos, max_backward_limit, gap, nodes) |
||||
if node_cost <= zopfliCostModelGetLiteralCosts(model, 0, pos) { |
||||
var posdata posData |
||||
posdata.pos = pos |
||||
posdata.cost = node_cost |
||||
posdata.costdiff = node_cost - zopfliCostModelGetLiteralCosts(model, 0, pos) |
||||
computeDistanceCache(pos, starting_dist_cache, nodes, posdata.distance_cache[:]) |
||||
startPosQueuePush(queue, &posdata) |
||||
} |
||||
} |
||||
|
||||
/* Returns longest copy length. */ |
||||
func updateNodes(num_bytes uint, block_start uint, pos uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, max_backward_limit uint, starting_dist_cache []int, num_matches uint, matches []backwardMatch, model *zopfliCostModel, queue *startPosQueue, nodes []zopfliNode) uint { |
||||
var cur_ix uint = block_start + pos |
||||
var cur_ix_masked uint = cur_ix & ringbuffer_mask |
||||
var max_distance uint = brotli_min_size_t(cur_ix, max_backward_limit) |
||||
var max_len uint = num_bytes - pos |
||||
var max_zopfli_len uint = maxZopfliLen(params) |
||||
var max_iters uint = maxZopfliCandidates(params) |
||||
var min_len uint |
||||
var result uint = 0 |
||||
var k uint |
||||
var gap uint = 0 |
||||
|
||||
evaluateNode(block_start, pos, max_backward_limit, gap, starting_dist_cache, model, queue, nodes) |
||||
{ |
||||
var posdata *posData = startPosQueueAt(queue, 0) |
||||
var min_cost float32 = (posdata.cost + zopfliCostModelGetMinCostCmd(model) + zopfliCostModelGetLiteralCosts(model, posdata.pos, pos)) |
||||
min_len = computeMinimumCopyLength(min_cost, nodes, num_bytes, pos) |
||||
} |
||||
|
||||
/* Go over the command starting positions in order of increasing cost |
||||
difference. */ |
||||
for k = 0; k < max_iters && k < startPosQueueSize(queue); k++ { |
||||
var posdata *posData = startPosQueueAt(queue, k) |
||||
var start uint = posdata.pos |
||||
var inscode uint16 = getInsertLengthCode(pos - start) |
||||
var start_costdiff float32 = posdata.costdiff |
||||
var base_cost float32 = start_costdiff + float32(getInsertExtra(inscode)) + zopfliCostModelGetLiteralCosts(model, 0, pos) |
||||
var best_len uint = min_len - 1 |
||||
var j uint = 0 |
||||
/* Look for last distance matches using the distance cache from this |
||||
starting position. */ |
||||
for ; j < numDistanceShortCodes && best_len < max_len; j++ { |
||||
var idx uint = uint(kDistanceCacheIndex[j]) |
||||
var backward uint = uint(posdata.distance_cache[idx] + kDistanceCacheOffset[j]) |
||||
var prev_ix uint = cur_ix - backward |
||||
var len uint = 0 |
||||
var continuation byte = ringbuffer[cur_ix_masked+best_len] |
||||
if cur_ix_masked+best_len > ringbuffer_mask { |
||||
break |
||||
} |
||||
|
||||
if backward > max_distance+gap { |
||||
/* Word dictionary -> ignore. */ |
||||
continue |
||||
} |
||||
|
||||
if backward <= max_distance { |
||||
/* Regular backward reference. */ |
||||
if prev_ix >= cur_ix { |
||||
continue |
||||
} |
||||
|
||||
prev_ix &= ringbuffer_mask |
||||
if prev_ix+best_len > ringbuffer_mask || continuation != ringbuffer[prev_ix+best_len] { |
||||
continue |
||||
} |
||||
|
||||
len = findMatchLengthWithLimit(ringbuffer[prev_ix:], ringbuffer[cur_ix_masked:], max_len) |
||||
} else { |
||||
continue |
||||
} |
||||
{ |
||||
var dist_cost float32 = base_cost + zopfliCostModelGetDistanceCost(model, j) |
||||
var l uint |
||||
for l = best_len + 1; l <= len; l++ { |
||||
var copycode uint16 = getCopyLengthCode(l) |
||||
var cmdcode uint16 = combineLengthCodes(inscode, copycode, j == 0) |
||||
var tmp float32 |
||||
if cmdcode < 128 { |
||||
tmp = base_cost |
||||
} else { |
||||
tmp = dist_cost |
||||
} |
||||
var cost float32 = tmp + float32(getCopyExtra(copycode)) + zopfliCostModelGetCommandCost(model, cmdcode) |
||||
if cost < nodes[pos+l].u.cost { |
||||
updateZopfliNode(nodes, pos, start, l, l, backward, j+1, cost) |
||||
result = brotli_max_size_t(result, l) |
||||
} |
||||
|
||||
best_len = l |
||||
} |
||||
} |
||||
} |
||||
|
||||
/* At higher iterations look only for new last distance matches, since |
||||
looking only for new command start positions with the same distances |
||||
does not help much. */ |
||||
if k >= 2 { |
||||
continue |
||||
} |
||||
{ |
||||
/* Loop through all possible copy lengths at this position. */ |
||||
var len uint = min_len |
||||
for j = 0; j < num_matches; j++ { |
||||
var match backwardMatch = matches[j] |
||||
var dist uint = uint(match.distance) |
||||
var is_dictionary_match bool = (dist > max_distance+gap) |
||||
var dist_code uint = dist + numDistanceShortCodes - 1 |
||||
var dist_symbol uint16 |
||||
var distextra uint32 |
||||
var distnumextra uint32 |
||||
var dist_cost float32 |
||||
var max_match_len uint |
||||
/* We already tried all possible last distance matches, so we can use |
||||
normal distance code here. */ |
||||
prefixEncodeCopyDistance(dist_code, uint(params.dist.num_direct_distance_codes), uint(params.dist.distance_postfix_bits), &dist_symbol, &distextra) |
||||
|
||||
distnumextra = uint32(dist_symbol) >> 10 |
||||
dist_cost = base_cost + float32(distnumextra) + zopfliCostModelGetDistanceCost(model, uint(dist_symbol)&0x3FF) |
||||
|
||||
/* Try all copy lengths up until the maximum copy length corresponding |
||||
to this distance. If the distance refers to the static dictionary, or |
||||
the maximum length is long enough, try only one maximum length. */ |
||||
max_match_len = backwardMatchLength(&match) |
||||
|
||||
if len < max_match_len && (is_dictionary_match || max_match_len > max_zopfli_len) { |
||||
len = max_match_len |
||||
} |
||||
|
||||
for ; len <= max_match_len; len++ { |
||||
var len_code uint |
||||
if is_dictionary_match { |
||||
len_code = backwardMatchLengthCode(&match) |
||||
} else { |
||||
len_code = len |
||||
} |
||||
var copycode uint16 = getCopyLengthCode(len_code) |
||||
var cmdcode uint16 = combineLengthCodes(inscode, copycode, false) |
||||
var cost float32 = dist_cost + float32(getCopyExtra(copycode)) + zopfliCostModelGetCommandCost(model, cmdcode) |
||||
if cost < nodes[pos+len].u.cost { |
||||
updateZopfliNode(nodes, pos, start, uint(len), len_code, dist, 0, cost) |
||||
result = brotli_max_size_t(result, uint(len)) |
||||
} |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
return result |
||||
} |
||||
|
||||
func computeShortestPathFromNodes(num_bytes uint, nodes []zopfliNode) uint { |
||||
var index uint = num_bytes |
||||
var num_commands uint = 0 |
||||
for nodes[index].dcode_insert_length&0x7FFFFFF == 0 && nodes[index].length == 1 { |
||||
index-- |
||||
} |
||||
nodes[index].u.next = math.MaxUint32 |
||||
for index != 0 { |
||||
var len uint = uint(zopfliNodeCommandLength(&nodes[index])) |
||||
index -= uint(len) |
||||
nodes[index].u.next = uint32(len) |
||||
num_commands++ |
||||
} |
||||
|
||||
return num_commands |
||||
} |
||||
|
||||
/* REQUIRES: nodes != NULL and len(nodes) >= num_bytes + 1 */ |
||||
func zopfliCreateCommands(num_bytes uint, block_start uint, nodes []zopfliNode, dist_cache []int, last_insert_len *uint, params *encoderParams, commands []command, num_literals *uint) { |
||||
var max_backward_limit uint = maxBackwardLimit(params.lgwin) |
||||
var pos uint = 0 |
||||
var offset uint32 = nodes[0].u.next |
||||
var i uint |
||||
var gap uint = 0 |
||||
for i = 0; offset != math.MaxUint32; i++ { |
||||
var next *zopfliNode = &nodes[uint32(pos)+offset] |
||||
var copy_length uint = uint(zopfliNodeCopyLength(next)) |
||||
var insert_length uint = uint(next.dcode_insert_length & 0x7FFFFFF) |
||||
pos += insert_length |
||||
offset = next.u.next |
||||
if i == 0 { |
||||
insert_length += *last_insert_len |
||||
*last_insert_len = 0 |
||||
} |
||||
{ |
||||
var distance uint = uint(zopfliNodeCopyDistance(next)) |
||||
var len_code uint = uint(zopfliNodeLengthCode(next)) |
||||
var max_distance uint = brotli_min_size_t(block_start+pos, max_backward_limit) |
||||
var is_dictionary bool = (distance > max_distance+gap) |
||||
var dist_code uint = uint(zopfliNodeDistanceCode(next)) |
||||
initCommand(&commands[i], ¶ms.dist, insert_length, copy_length, int(len_code)-int(copy_length), dist_code) |
||||
|
||||
if !is_dictionary && dist_code > 0 { |
||||
dist_cache[3] = dist_cache[2] |
||||
dist_cache[2] = dist_cache[1] |
||||
dist_cache[1] = dist_cache[0] |
||||
dist_cache[0] = int(distance) |
||||
} |
||||
} |
||||
|
||||
*num_literals += insert_length |
||||
pos += copy_length |
||||
} |
||||
|
||||
*last_insert_len += num_bytes - pos |
||||
} |
||||
|
||||
func zopfliIterate(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, gap uint, dist_cache []int, model *zopfliCostModel, num_matches []uint32, matches []backwardMatch, nodes []zopfliNode) uint { |
||||
var max_backward_limit uint = maxBackwardLimit(params.lgwin) |
||||
var max_zopfli_len uint = maxZopfliLen(params) |
||||
var queue startPosQueue |
||||
var cur_match_pos uint = 0 |
||||
var i uint |
||||
nodes[0].length = 0 |
||||
nodes[0].u.cost = 0 |
||||
initStartPosQueue(&queue) |
||||
for i = 0; i+3 < num_bytes; i++ { |
||||
var skip uint = updateNodes(num_bytes, position, i, ringbuffer, ringbuffer_mask, params, max_backward_limit, dist_cache, uint(num_matches[i]), matches[cur_match_pos:], model, &queue, nodes) |
||||
if skip < longCopyQuickStep { |
||||
skip = 0 |
||||
} |
||||
cur_match_pos += uint(num_matches[i]) |
||||
if num_matches[i] == 1 && backwardMatchLength(&matches[cur_match_pos-1]) > max_zopfli_len { |
||||
skip = brotli_max_size_t(backwardMatchLength(&matches[cur_match_pos-1]), skip) |
||||
} |
||||
|
||||
if skip > 1 { |
||||
skip-- |
||||
for skip != 0 { |
||||
i++ |
||||
if i+3 >= num_bytes { |
||||
break |
||||
} |
||||
evaluateNode(position, i, max_backward_limit, gap, dist_cache, model, &queue, nodes) |
||||
cur_match_pos += uint(num_matches[i]) |
||||
skip-- |
||||
} |
||||
} |
||||
} |
||||
|
||||
return computeShortestPathFromNodes(num_bytes, nodes) |
||||
} |
||||
|
||||
/* Computes the shortest path of commands from position to at most |
||||
position + num_bytes. |
||||
|
||||
On return, path->size() is the number of commands found and path[i] is the |
||||
length of the i-th command (copy length plus insert length). |
||||
Note that the sum of the lengths of all commands can be less than num_bytes. |
||||
|
||||
On return, the nodes[0..num_bytes] array will have the following |
||||
"ZopfliNode array invariant": |
||||
For each i in [1..num_bytes], if nodes[i].cost < kInfinity, then |
||||
(1) nodes[i].copy_length() >= 2 |
||||
(2) nodes[i].command_length() <= i and |
||||
(3) nodes[i - nodes[i].command_length()].cost < kInfinity |
||||
|
||||
REQUIRES: nodes != nil and len(nodes) >= num_bytes + 1 */ |
||||
func zopfliComputeShortestPath(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, dist_cache []int, hasher *h10, nodes []zopfliNode) uint { |
||||
var max_backward_limit uint = maxBackwardLimit(params.lgwin) |
||||
var max_zopfli_len uint = maxZopfliLen(params) |
||||
var model zopfliCostModel |
||||
var queue startPosQueue |
||||
var matches [2 * (maxNumMatchesH10 + 64)]backwardMatch |
||||
var store_end uint |
||||
if num_bytes >= hasher.StoreLookahead() { |
||||
store_end = position + num_bytes - hasher.StoreLookahead() + 1 |
||||
} else { |
||||
store_end = position |
||||
} |
||||
var i uint |
||||
var gap uint = 0 |
||||
var lz_matches_offset uint = 0 |
||||
nodes[0].length = 0 |
||||
nodes[0].u.cost = 0 |
||||
initZopfliCostModel(&model, ¶ms.dist, num_bytes) |
||||
zopfliCostModelSetFromLiteralCosts(&model, position, ringbuffer, ringbuffer_mask) |
||||
initStartPosQueue(&queue) |
||||
for i = 0; i+hasher.HashTypeLength()-1 < num_bytes; i++ { |
||||
var pos uint = position + i |
||||
var max_distance uint = brotli_min_size_t(pos, max_backward_limit) |
||||
var skip uint |
||||
var num_matches uint |
||||
num_matches = findAllMatchesH10(hasher, ¶ms.dictionary, ringbuffer, ringbuffer_mask, pos, num_bytes-i, max_distance, gap, params, matches[lz_matches_offset:]) |
||||
if num_matches > 0 && backwardMatchLength(&matches[num_matches-1]) > max_zopfli_len { |
||||
matches[0] = matches[num_matches-1] |
||||
num_matches = 1 |
||||
} |
||||
|
||||
skip = updateNodes(num_bytes, position, i, ringbuffer, ringbuffer_mask, params, max_backward_limit, dist_cache, num_matches, matches[:], &model, &queue, nodes) |
||||
if skip < longCopyQuickStep { |
||||
skip = 0 |
||||
} |
||||
if num_matches == 1 && backwardMatchLength(&matches[0]) > max_zopfli_len { |
||||
skip = brotli_max_size_t(backwardMatchLength(&matches[0]), skip) |
||||
} |
||||
|
||||
if skip > 1 { |
||||
/* Add the tail of the copy to the hasher. */ |
||||
hasher.StoreRange(ringbuffer, ringbuffer_mask, pos+1, brotli_min_size_t(pos+skip, store_end)) |
||||
|
||||
skip-- |
||||
for skip != 0 { |
||||
i++ |
||||
if i+hasher.HashTypeLength()-1 >= num_bytes { |
||||
break |
||||
} |
||||
evaluateNode(position, i, max_backward_limit, gap, dist_cache, &model, &queue, nodes) |
||||
skip-- |
||||
} |
||||
} |
||||
} |
||||
|
||||
cleanupZopfliCostModel(&model) |
||||
return computeShortestPathFromNodes(num_bytes, nodes) |
||||
} |
||||
|
||||
func createZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher *h10, dist_cache []int, last_insert_len *uint, commands []command, num_commands *uint, num_literals *uint) { |
||||
var nodes []zopfliNode |
||||
nodes = make([]zopfliNode, (num_bytes + 1)) |
||||
initZopfliNodes(nodes, num_bytes+1) |
||||
*num_commands += zopfliComputeShortestPath(num_bytes, position, ringbuffer, ringbuffer_mask, params, dist_cache, hasher, nodes) |
||||
zopfliCreateCommands(num_bytes, position, nodes, dist_cache, last_insert_len, params, commands, num_literals) |
||||
nodes = nil |
||||
} |
||||
|
||||
func createHqZopfliBackwardReferences(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint, params *encoderParams, hasher hasherHandle, dist_cache []int, last_insert_len *uint, commands []command, num_commands *uint, num_literals *uint) { |
||||
var max_backward_limit uint = maxBackwardLimit(params.lgwin) |
||||
var num_matches []uint32 = make([]uint32, num_bytes) |
||||
var matches_size uint = 4 * num_bytes |
||||
var store_end uint |
||||
if num_bytes >= hasher.StoreLookahead() { |
||||
store_end = position + num_bytes - hasher.StoreLookahead() + 1 |
||||
} else { |
||||
store_end = position |
||||
} |
||||
var cur_match_pos uint = 0 |
||||
var i uint |
||||
var orig_num_literals uint |
||||
var orig_last_insert_len uint |
||||
var orig_dist_cache [4]int |
||||
var orig_num_commands uint |
||||
var model zopfliCostModel |
||||
var nodes []zopfliNode |
||||
var matches []backwardMatch = make([]backwardMatch, matches_size) |
||||
var gap uint = 0 |
||||
var shadow_matches uint = 0 |
||||
var new_array []backwardMatch |
||||
for i = 0; i+hasher.HashTypeLength()-1 < num_bytes; i++ { |
||||
var pos uint = position + i |
||||
var max_distance uint = brotli_min_size_t(pos, max_backward_limit) |
||||
var max_length uint = num_bytes - i |
||||
var num_found_matches uint |
||||
var cur_match_end uint |
||||
var j uint |
||||
|
||||
/* Ensure that we have enough free slots. */ |
||||
if matches_size < cur_match_pos+maxNumMatchesH10+shadow_matches { |
||||
var new_size uint = matches_size |
||||
if new_size == 0 { |
||||
new_size = cur_match_pos + maxNumMatchesH10 + shadow_matches |
||||
} |
||||
|
||||
for new_size < cur_match_pos+maxNumMatchesH10+shadow_matches { |
||||
new_size *= 2 |
||||
} |
||||
|
||||
new_array = make([]backwardMatch, new_size) |
||||
if matches_size != 0 { |
||||
copy(new_array, matches[:matches_size]) |
||||
} |
||||
|
||||
matches = new_array |
||||
matches_size = new_size |
||||
} |
||||
|
||||
num_found_matches = findAllMatchesH10(hasher.(*h10), ¶ms.dictionary, ringbuffer, ringbuffer_mask, pos, max_length, max_distance, gap, params, matches[cur_match_pos+shadow_matches:]) |
||||
cur_match_end = cur_match_pos + num_found_matches |
||||
for j = cur_match_pos; j+1 < cur_match_end; j++ { |
||||
assert(backwardMatchLength(&matches[j]) <= backwardMatchLength(&matches[j+1])) |
||||
} |
||||
|
||||
num_matches[i] = uint32(num_found_matches) |
||||
if num_found_matches > 0 { |
||||
var match_len uint = backwardMatchLength(&matches[cur_match_end-1]) |
||||
if match_len > maxZopfliLenQuality11 { |
||||
var skip uint = match_len - 1 |
||||
matches[cur_match_pos] = matches[cur_match_end-1] |
||||
cur_match_pos++ |
||||
num_matches[i] = 1 |
||||
|
||||
/* Add the tail of the copy to the hasher. */ |
||||
hasher.StoreRange(ringbuffer, ringbuffer_mask, pos+1, brotli_min_size_t(pos+match_len, store_end)) |
||||
var pos uint = i |
||||
for i := 0; i < int(skip); i++ { |
||||
num_matches[pos+1:][i] = 0 |
||||
} |
||||
i += skip |
||||
} else { |
||||
cur_match_pos = cur_match_end |
||||
} |
||||
} |
||||
} |
||||
|
||||
orig_num_literals = *num_literals |
||||
orig_last_insert_len = *last_insert_len |
||||
copy(orig_dist_cache[:], dist_cache[:4]) |
||||
orig_num_commands = *num_commands |
||||
nodes = make([]zopfliNode, (num_bytes + 1)) |
||||
initZopfliCostModel(&model, ¶ms.dist, num_bytes) |
||||
for i = 0; i < 2; i++ { |
||||
initZopfliNodes(nodes, num_bytes+1) |
||||
if i == 0 { |
||||
zopfliCostModelSetFromLiteralCosts(&model, position, ringbuffer, ringbuffer_mask) |
||||
} else { |
||||
zopfliCostModelSetFromCommands(&model, position, ringbuffer, ringbuffer_mask, commands, *num_commands-orig_num_commands, orig_last_insert_len) |
||||
} |
||||
|
||||
*num_commands = orig_num_commands |
||||
*num_literals = orig_num_literals |
||||
*last_insert_len = orig_last_insert_len |
||||
copy(dist_cache, orig_dist_cache[:4]) |
||||
*num_commands += zopfliIterate(num_bytes, position, ringbuffer, ringbuffer_mask, params, gap, dist_cache, &model, num_matches, matches, nodes) |
||||
zopfliCreateCommands(num_bytes, position, nodes, dist_cache, last_insert_len, params, commands, num_literals) |
||||
} |
||||
|
||||
cleanupZopfliCostModel(&model) |
||||
nodes = nil |
||||
matches = nil |
||||
num_matches = nil |
||||
} |
||||
@ -0,0 +1,436 @@ |
||||
package brotli |
||||
|
||||
/* Copyright 2013 Google Inc. All Rights Reserved. |
||||
|
||||
Distributed under MIT license. |
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/ |
||||
|
||||
/* Functions to estimate the bit cost of Huffman trees. */ |
||||
func shannonEntropy(population []uint32, size uint, total *uint) float64 { |
||||
var sum uint = 0 |
||||
var retval float64 = 0 |
||||
var population_end []uint32 = population[size:] |
||||
var p uint |
||||
for -cap(population) < -cap(population_end) { |
||||
p = uint(population[0]) |
||||
population = population[1:] |
||||
sum += p |
||||
retval -= float64(p) * fastLog2(p) |
||||
} |
||||
|
||||
if sum != 0 { |
||||
retval += float64(sum) * fastLog2(sum) |
||||
} |
||||
*total = sum |
||||
return retval |
||||
} |
||||
|
||||
func bitsEntropy(population []uint32, size uint) float64 { |
||||
var sum uint |
||||
var retval float64 = shannonEntropy(population, size, &sum) |
||||
if retval < float64(sum) { |
||||
/* At least one bit per literal is needed. */ |
||||
retval = float64(sum) |
||||
} |
||||
|
||||
return retval |
||||
} |
||||
|
||||
const kOneSymbolHistogramCost float64 = 12 |
||||
const kTwoSymbolHistogramCost float64 = 20 |
||||
const kThreeSymbolHistogramCost float64 = 28 |
||||
const kFourSymbolHistogramCost float64 = 37 |
||||
|
||||
func populationCostLiteral(histogram *histogramLiteral) float64 { |
||||
var data_size uint = histogramDataSizeLiteral() |
||||
var count int = 0 |
||||
var s [5]uint |
||||
var bits float64 = 0.0 |
||||
var i uint |
||||
if histogram.total_count_ == 0 { |
||||
return kOneSymbolHistogramCost |
||||
} |
||||
|
||||
for i = 0; i < data_size; i++ { |
||||
if histogram.data_[i] > 0 { |
||||
s[count] = i |
||||
count++ |
||||
if count > 4 { |
||||
break |
||||
} |
||||
} |
||||
} |
||||
|
||||
if count == 1 { |
||||
return kOneSymbolHistogramCost |
||||
} |
||||
|
||||
if count == 2 { |
||||
return kTwoSymbolHistogramCost + float64(histogram.total_count_) |
||||
} |
||||
|
||||
if count == 3 { |
||||
var histo0 uint32 = histogram.data_[s[0]] |
||||
var histo1 uint32 = histogram.data_[s[1]] |
||||
var histo2 uint32 = histogram.data_[s[2]] |
||||
var histomax uint32 = brotli_max_uint32_t(histo0, brotli_max_uint32_t(histo1, histo2)) |
||||
return kThreeSymbolHistogramCost + 2*(float64(histo0)+float64(histo1)+float64(histo2)) - float64(histomax) |
||||
} |
||||
|
||||
if count == 4 { |
||||
var histo [4]uint32 |
||||
var h23 uint32 |
||||
var histomax uint32 |
||||
for i = 0; i < 4; i++ { |
||||
histo[i] = histogram.data_[s[i]] |
||||
} |
||||
|
||||
/* Sort */ |
||||
for i = 0; i < 4; i++ { |
||||
var j uint |
||||
for j = i + 1; j < 4; j++ { |
||||
if histo[j] > histo[i] { |
||||
var tmp uint32 = histo[j] |
||||
histo[j] = histo[i] |
||||
histo[i] = tmp |
||||
} |
||||
} |
||||
} |
||||
|
||||
h23 = histo[2] + histo[3] |
||||
histomax = brotli_max_uint32_t(h23, histo[0]) |
||||
return kFourSymbolHistogramCost + 3*float64(h23) + 2*(float64(histo[0])+float64(histo[1])) - float64(histomax) |
||||
} |
||||
{ |
||||
var max_depth uint = 1 |
||||
var depth_histo = [codeLengthCodes]uint32{0} |
||||
/* In this loop we compute the entropy of the histogram and simultaneously |
||||
build a simplified histogram of the code length codes where we use the |
||||
zero repeat code 17, but we don't use the non-zero repeat code 16. */ |
||||
|
||||
var log2total float64 = fastLog2(histogram.total_count_) |
||||
for i = 0; i < data_size; { |
||||
if histogram.data_[i] > 0 { |
||||
var log2p float64 = log2total - fastLog2(uint(histogram.data_[i])) |
||||
/* Compute -log2(P(symbol)) = -log2(count(symbol)/total_count) = |
||||
= log2(total_count) - log2(count(symbol)) */ |
||||
|
||||
var depth uint = uint(log2p + 0.5) |
||||
/* Approximate the bit depth by round(-log2(P(symbol))) */ |
||||
bits += float64(histogram.data_[i]) * log2p |
||||
|
||||
if depth > 15 { |
||||
depth = 15 |
||||
} |
||||
|
||||
if depth > max_depth { |
||||
max_depth = depth |
||||
} |
||||
|
||||
depth_histo[depth]++ |
||||
i++ |
||||
} else { |
||||
var reps uint32 = 1 |
||||
/* Compute the run length of zeros and add the appropriate number of 0 |
||||
and 17 code length codes to the code length code histogram. */ |
||||
|
||||
var k uint |
||||
for k = i + 1; k < data_size && histogram.data_[k] == 0; k++ { |
||||
reps++ |
||||
} |
||||
|
||||
i += uint(reps) |
||||
if i == data_size { |
||||
/* Don't add any cost for the last zero run, since these are encoded |
||||
only implicitly. */ |
||||
break |
||||
} |
||||
|
||||
if reps < 3 { |
||||
depth_histo[0] += reps |
||||
} else { |
||||
reps -= 2 |
||||
for reps > 0 { |
||||
depth_histo[repeatZeroCodeLength]++ |
||||
|
||||
/* Add the 3 extra bits for the 17 code length code. */ |
||||
bits += 3 |
||||
|
||||
reps >>= 3 |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
/* Add the estimated encoding cost of the code length code histogram. */ |
||||
bits += float64(18 + 2*max_depth) |
||||
|
||||
/* Add the entropy of the code length code histogram. */ |
||||
bits += bitsEntropy(depth_histo[:], codeLengthCodes) |
||||
} |
||||
|
||||
return bits |
||||
} |
||||
|
||||
func populationCostCommand(histogram *histogramCommand) float64 { |
||||
var data_size uint = histogramDataSizeCommand() |
||||
var count int = 0 |
||||
var s [5]uint |
||||
var bits float64 = 0.0 |
||||
var i uint |
||||
if histogram.total_count_ == 0 { |
||||
return kOneSymbolHistogramCost |
||||
} |
||||
|
||||
for i = 0; i < data_size; i++ { |
||||
if histogram.data_[i] > 0 { |
||||
s[count] = i |
||||
count++ |
||||
if count > 4 { |
||||
break |
||||
} |
||||
} |
||||
} |
||||
|
||||
if count == 1 { |
||||
return kOneSymbolHistogramCost |
||||
} |
||||
|
||||
if count == 2 { |
||||
return kTwoSymbolHistogramCost + float64(histogram.total_count_) |
||||
} |
||||
|
||||
if count == 3 { |
||||
var histo0 uint32 = histogram.data_[s[0]] |
||||
var histo1 uint32 = histogram.data_[s[1]] |
||||
var histo2 uint32 = histogram.data_[s[2]] |
||||
var histomax uint32 = brotli_max_uint32_t(histo0, brotli_max_uint32_t(histo1, histo2)) |
||||
return kThreeSymbolHistogramCost + 2*(float64(histo0)+float64(histo1)+float64(histo2)) - float64(histomax) |
||||
} |
||||
|
||||
if count == 4 { |
||||
var histo [4]uint32 |
||||
var h23 uint32 |
||||
var histomax uint32 |
||||
for i = 0; i < 4; i++ { |
||||
histo[i] = histogram.data_[s[i]] |
||||
} |
||||
|
||||
/* Sort */ |
||||
for i = 0; i < 4; i++ { |
||||
var j uint |
||||
for j = i + 1; j < 4; j++ { |
||||
if histo[j] > histo[i] { |
||||
var tmp uint32 = histo[j] |
||||
histo[j] = histo[i] |
||||
histo[i] = tmp |
||||
} |
||||
} |
||||
} |
||||
|
||||
h23 = histo[2] + histo[3] |
||||
histomax = brotli_max_uint32_t(h23, histo[0]) |
||||
return kFourSymbolHistogramCost + 3*float64(h23) + 2*(float64(histo[0])+float64(histo[1])) - float64(histomax) |
||||
} |
||||
{ |
||||
var max_depth uint = 1 |
||||
var depth_histo = [codeLengthCodes]uint32{0} |
||||
/* In this loop we compute the entropy of the histogram and simultaneously |
||||
build a simplified histogram of the code length codes where we use the |
||||
zero repeat code 17, but we don't use the non-zero repeat code 16. */ |
||||
|
||||
var log2total float64 = fastLog2(histogram.total_count_) |
||||
for i = 0; i < data_size; { |
||||
if histogram.data_[i] > 0 { |
||||
var log2p float64 = log2total - fastLog2(uint(histogram.data_[i])) |
||||
/* Compute -log2(P(symbol)) = -log2(count(symbol)/total_count) = |
||||
= log2(total_count) - log2(count(symbol)) */ |
||||
|
||||
var depth uint = uint(log2p + 0.5) |
||||
/* Approximate the bit depth by round(-log2(P(symbol))) */ |
||||
bits += float64(histogram.data_[i]) * log2p |
||||
|
||||
if depth > 15 { |
||||
depth = 15 |
||||
} |
||||
|
||||
if depth > max_depth { |
||||
max_depth = depth |
||||
} |
||||
|
||||
depth_histo[depth]++ |
||||
i++ |
||||
} else { |
||||
var reps uint32 = 1 |
||||
/* Compute the run length of zeros and add the appropriate number of 0 |
||||
and 17 code length codes to the code length code histogram. */ |
||||
|
||||
var k uint |
||||
for k = i + 1; k < data_size && histogram.data_[k] == 0; k++ { |
||||
reps++ |
||||
} |
||||
|
||||
i += uint(reps) |
||||
if i == data_size { |
||||
/* Don't add any cost for the last zero run, since these are encoded |
||||
only implicitly. */ |
||||
break |
||||
} |
||||
|
||||
if reps < 3 { |
||||
depth_histo[0] += reps |
||||
} else { |
||||
reps -= 2 |
||||
for reps > 0 { |
||||
depth_histo[repeatZeroCodeLength]++ |
||||
|
||||
/* Add the 3 extra bits for the 17 code length code. */ |
||||
bits += 3 |
||||
|
||||
reps >>= 3 |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
/* Add the estimated encoding cost of the code length code histogram. */ |
||||
bits += float64(18 + 2*max_depth) |
||||
|
||||
/* Add the entropy of the code length code histogram. */ |
||||
bits += bitsEntropy(depth_histo[:], codeLengthCodes) |
||||
} |
||||
|
||||
return bits |
||||
} |
||||
|
||||
func populationCostDistance(histogram *histogramDistance) float64 { |
||||
var data_size uint = histogramDataSizeDistance() |
||||
var count int = 0 |
||||
var s [5]uint |
||||
var bits float64 = 0.0 |
||||
var i uint |
||||
if histogram.total_count_ == 0 { |
||||
return kOneSymbolHistogramCost |
||||
} |
||||
|
||||
for i = 0; i < data_size; i++ { |
||||
if histogram.data_[i] > 0 { |
||||
s[count] = i |
||||
count++ |
||||
if count > 4 { |
||||
break |
||||
} |
||||
} |
||||
} |
||||
|
||||
if count == 1 { |
||||
return kOneSymbolHistogramCost |
||||
} |
||||
|
||||
if count == 2 { |
||||
return kTwoSymbolHistogramCost + float64(histogram.total_count_) |
||||
} |
||||
|
||||
if count == 3 { |
||||
var histo0 uint32 = histogram.data_[s[0]] |
||||
var histo1 uint32 = histogram.data_[s[1]] |
||||
var histo2 uint32 = histogram.data_[s[2]] |
||||
var histomax uint32 = brotli_max_uint32_t(histo0, brotli_max_uint32_t(histo1, histo2)) |
||||
return kThreeSymbolHistogramCost + 2*(float64(histo0)+float64(histo1)+float64(histo2)) - float64(histomax) |
||||
} |
||||
|
||||
if count == 4 { |
||||
var histo [4]uint32 |
||||
var h23 uint32 |
||||
var histomax uint32 |
||||
for i = 0; i < 4; i++ { |
||||
histo[i] = histogram.data_[s[i]] |
||||
} |
||||
|
||||
/* Sort */ |
||||
for i = 0; i < 4; i++ { |
||||
var j uint |
||||
for j = i + 1; j < 4; j++ { |
||||
if histo[j] > histo[i] { |
||||
var tmp uint32 = histo[j] |
||||
histo[j] = histo[i] |
||||
histo[i] = tmp |
||||
} |
||||
} |
||||
} |
||||
|
||||
h23 = histo[2] + histo[3] |
||||
histomax = brotli_max_uint32_t(h23, histo[0]) |
||||
return kFourSymbolHistogramCost + 3*float64(h23) + 2*(float64(histo[0])+float64(histo[1])) - float64(histomax) |
||||
} |
||||
{ |
||||
var max_depth uint = 1 |
||||
var depth_histo = [codeLengthCodes]uint32{0} |
||||
/* In this loop we compute the entropy of the histogram and simultaneously |
||||
build a simplified histogram of the code length codes where we use the |
||||
zero repeat code 17, but we don't use the non-zero repeat code 16. */ |
||||
|
||||
var log2total float64 = fastLog2(histogram.total_count_) |
||||
for i = 0; i < data_size; { |
||||
if histogram.data_[i] > 0 { |
||||
var log2p float64 = log2total - fastLog2(uint(histogram.data_[i])) |
||||
/* Compute -log2(P(symbol)) = -log2(count(symbol)/total_count) = |
||||
= log2(total_count) - log2(count(symbol)) */ |
||||
|
||||
var depth uint = uint(log2p + 0.5) |
||||
/* Approximate the bit depth by round(-log2(P(symbol))) */ |
||||
bits += float64(histogram.data_[i]) * log2p |
||||
|
||||
if depth > 15 { |
||||
depth = 15 |
||||
} |
||||
|
||||
if depth > max_depth { |
||||
max_depth = depth |
||||
} |
||||
|
||||
depth_histo[depth]++ |
||||
i++ |
||||
} else { |
||||
var reps uint32 = 1 |
||||
/* Compute the run length of zeros and add the appropriate number of 0 |
||||
and 17 code length codes to the code length code histogram. */ |
||||
|
||||
var k uint |
||||
for k = i + 1; k < data_size && histogram.data_[k] == 0; k++ { |
||||
reps++ |
||||
} |
||||
|
||||
i += uint(reps) |
||||
if i == data_size { |
||||
/* Don't add any cost for the last zero run, since these are encoded |
||||
only implicitly. */ |
||||
break |
||||
} |
||||
|
||||
if reps < 3 { |
||||
depth_histo[0] += reps |
||||
} else { |
||||
reps -= 2 |
||||
for reps > 0 { |
||||
depth_histo[repeatZeroCodeLength]++ |
||||
|
||||
/* Add the 3 extra bits for the 17 code length code. */ |
||||
bits += 3 |
||||
|
||||
reps >>= 3 |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
/* Add the estimated encoding cost of the code length code histogram. */ |
||||
bits += float64(18 + 2*max_depth) |
||||
|
||||
/* Add the entropy of the code length code histogram. */ |
||||
bits += bitsEntropy(depth_histo[:], codeLengthCodes) |
||||
} |
||||
|
||||
return bits |
||||
} |
||||
@ -0,0 +1,266 @@ |
||||
package brotli |
||||
|
||||
import "encoding/binary" |
||||
|
||||
/* Copyright 2013 Google Inc. All Rights Reserved. |
||||
|
||||
Distributed under MIT license. |
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/ |
||||
|
||||
/* Bit reading helpers */ |
||||
|
||||
const shortFillBitWindowRead = (8 >> 1) |
||||
|
||||
var kBitMask = [33]uint32{ |
||||
0x00000000, |
||||
0x00000001, |
||||
0x00000003, |
||||
0x00000007, |
||||
0x0000000F, |
||||
0x0000001F, |
||||
0x0000003F, |
||||
0x0000007F, |
||||
0x000000FF, |
||||
0x000001FF, |
||||
0x000003FF, |
||||
0x000007FF, |
||||
0x00000FFF, |
||||
0x00001FFF, |
||||
0x00003FFF, |
||||
0x00007FFF, |
||||
0x0000FFFF, |
||||
0x0001FFFF, |
||||
0x0003FFFF, |
||||
0x0007FFFF, |
||||
0x000FFFFF, |
||||
0x001FFFFF, |
||||
0x003FFFFF, |
||||
0x007FFFFF, |
||||
0x00FFFFFF, |
||||
0x01FFFFFF, |
||||
0x03FFFFFF, |
||||
0x07FFFFFF, |
||||
0x0FFFFFFF, |
||||
0x1FFFFFFF, |
||||
0x3FFFFFFF, |
||||
0x7FFFFFFF, |
||||
0xFFFFFFFF, |
||||
} |
||||
|
||||
func bitMask(n uint32) uint32 { |
||||
return kBitMask[n] |
||||
} |
||||
|
||||
type bitReader struct { |
||||
val_ uint64 |
||||
bit_pos_ uint32 |
||||
input []byte |
||||
input_len uint |
||||
byte_pos uint |
||||
} |
||||
|
||||
type bitReaderState struct { |
||||
val_ uint64 |
||||
bit_pos_ uint32 |
||||
input []byte |
||||
input_len uint |
||||
byte_pos uint |
||||
} |
||||
|
||||
/* Initializes the BrotliBitReader fields. */ |
||||
|
||||
/* Ensures that accumulator is not empty. |
||||
May consume up to sizeof(brotli_reg_t) - 1 bytes of input. |
||||
Returns false if data is required but there is no input available. |
||||
For BROTLI_ALIGNED_READ this function also prepares bit reader for aligned |
||||
reading. */ |
||||
func bitReaderSaveState(from *bitReader, to *bitReaderState) { |
||||
to.val_ = from.val_ |
||||
to.bit_pos_ = from.bit_pos_ |
||||
to.input = from.input |
||||
to.input_len = from.input_len |
||||
to.byte_pos = from.byte_pos |
||||
} |
||||
|
||||
func bitReaderRestoreState(to *bitReader, from *bitReaderState) { |
||||
to.val_ = from.val_ |
||||
to.bit_pos_ = from.bit_pos_ |
||||
to.input = from.input |
||||
to.input_len = from.input_len |
||||
to.byte_pos = from.byte_pos |
||||
} |
||||
|
||||
func getAvailableBits(br *bitReader) uint32 { |
||||
return 64 - br.bit_pos_ |
||||
} |
||||
|
||||
/* Returns amount of unread bytes the bit reader still has buffered from the |
||||
BrotliInput, including whole bytes in br->val_. */ |
||||
func getRemainingBytes(br *bitReader) uint { |
||||
return uint(uint32(br.input_len-br.byte_pos) + (getAvailableBits(br) >> 3)) |
||||
} |
||||
|
||||
/* Checks if there is at least |num| bytes left in the input ring-buffer |
||||
(excluding the bits remaining in br->val_). */ |
||||
func checkInputAmount(br *bitReader, num uint) bool { |
||||
return br.input_len-br.byte_pos >= num |
||||
} |
||||
|
||||
/* Guarantees that there are at least |n_bits| + 1 bits in accumulator. |
||||
Precondition: accumulator contains at least 1 bit. |
||||
|n_bits| should be in the range [1..24] for regular build. For portable |
||||
non-64-bit little-endian build only 16 bits are safe to request. */ |
||||
func fillBitWindow(br *bitReader, n_bits uint32) { |
||||
if br.bit_pos_ >= 32 { |
||||
br.val_ >>= 32 |
||||
br.bit_pos_ ^= 32 /* here same as -= 32 because of the if condition */ |
||||
br.val_ |= (uint64(binary.LittleEndian.Uint32(br.input[br.byte_pos:]))) << 32 |
||||
br.byte_pos += 4 |
||||
} |
||||
} |
||||
|
||||
/* Mostly like BrotliFillBitWindow, but guarantees only 16 bits and reads no |
||||
more than BROTLI_SHORT_FILL_BIT_WINDOW_READ bytes of input. */ |
||||
func fillBitWindow16(br *bitReader) { |
||||
fillBitWindow(br, 17) |
||||
} |
||||
|
||||
/* Tries to pull one byte of input to accumulator. |
||||
Returns false if there is no input available. */ |
||||
func pullByte(br *bitReader) bool { |
||||
if br.byte_pos == br.input_len { |
||||
return false |
||||
} |
||||
|
||||
br.val_ >>= 8 |
||||
br.val_ |= (uint64(br.input[br.byte_pos])) << 56 |
||||
br.bit_pos_ -= 8 |
||||
br.byte_pos++ |
||||
return true |
||||
} |
||||
|
||||
/* Returns currently available bits. |
||||
The number of valid bits could be calculated by BrotliGetAvailableBits. */ |
||||
func getBitsUnmasked(br *bitReader) uint64 { |
||||
return br.val_ >> br.bit_pos_ |
||||
} |
||||
|
||||
/* Like BrotliGetBits, but does not mask the result. |
||||
The result contains at least 16 valid bits. */ |
||||
func get16BitsUnmasked(br *bitReader) uint32 { |
||||
fillBitWindow(br, 16) |
||||
return uint32(getBitsUnmasked(br)) |
||||
} |
||||
|
||||
/* Returns the specified number of bits from |br| without advancing bit |
||||
position. */ |
||||
func getBits(br *bitReader, n_bits uint32) uint32 { |
||||
fillBitWindow(br, n_bits) |
||||
return uint32(getBitsUnmasked(br)) & bitMask(n_bits) |
||||
} |
||||
|
||||
/* Tries to peek the specified amount of bits. Returns false, if there |
||||
is not enough input. */ |
||||
func safeGetBits(br *bitReader, n_bits uint32, val *uint32) bool { |
||||
for getAvailableBits(br) < n_bits { |
||||
if !pullByte(br) { |
||||
return false |
||||
} |
||||
} |
||||
|
||||
*val = uint32(getBitsUnmasked(br)) & bitMask(n_bits) |
||||
return true |
||||
} |
||||
|
||||
/* Advances the bit pos by |n_bits|. */ |
||||
func dropBits(br *bitReader, n_bits uint32) { |
||||
br.bit_pos_ += n_bits |
||||
} |
||||
|
||||
func bitReaderUnload(br *bitReader) { |
||||
var unused_bytes uint32 = getAvailableBits(br) >> 3 |
||||
var unused_bits uint32 = unused_bytes << 3 |
||||
br.byte_pos -= uint(unused_bytes) |
||||
if unused_bits == 64 { |
||||
br.val_ = 0 |
||||
} else { |
||||
br.val_ <<= unused_bits |
||||
} |
||||
|
||||
br.bit_pos_ += unused_bits |
||||
} |
||||
|
||||
/* Reads the specified number of bits from |br| and advances the bit pos. |
||||
Precondition: accumulator MUST contain at least |n_bits|. */ |
||||
func takeBits(br *bitReader, n_bits uint32, val *uint32) { |
||||
*val = uint32(getBitsUnmasked(br)) & bitMask(n_bits) |
||||
dropBits(br, n_bits) |
||||
} |
||||
|
||||
/* Reads the specified number of bits from |br| and advances the bit pos. |
||||
Assumes that there is enough input to perform BrotliFillBitWindow. */ |
||||
func readBits(br *bitReader, n_bits uint32) uint32 { |
||||
var val uint32 |
||||
fillBitWindow(br, n_bits) |
||||
takeBits(br, n_bits, &val) |
||||
return val |
||||
} |
||||
|
||||
/* Tries to read the specified amount of bits. Returns false, if there |
||||
is not enough input. |n_bits| MUST be positive. */ |
||||
func safeReadBits(br *bitReader, n_bits uint32, val *uint32) bool { |
||||
for getAvailableBits(br) < n_bits { |
||||
if !pullByte(br) { |
||||
return false |
||||
} |
||||
} |
||||
|
||||
takeBits(br, n_bits, val) |
||||
return true |
||||
} |
||||
|
||||
/* Advances the bit reader position to the next byte boundary and verifies |
||||
that any skipped bits are set to zero. */ |
||||
func bitReaderJumpToByteBoundary(br *bitReader) bool { |
||||
var pad_bits_count uint32 = getAvailableBits(br) & 0x7 |
||||
var pad_bits uint32 = 0 |
||||
if pad_bits_count != 0 { |
||||
takeBits(br, pad_bits_count, &pad_bits) |
||||
} |
||||
|
||||
return pad_bits == 0 |
||||
} |
||||
|
||||
/* Copies remaining input bytes stored in the bit reader to the output. Value |
||||
|num| may not be larger than BrotliGetRemainingBytes. The bit reader must be |
||||
warmed up again after this. */ |
||||
func copyBytes(dest []byte, br *bitReader, num uint) { |
||||
for getAvailableBits(br) >= 8 && num > 0 { |
||||
dest[0] = byte(getBitsUnmasked(br)) |
||||
dropBits(br, 8) |
||||
dest = dest[1:] |
||||
num-- |
||||
} |
||||
|
||||
copy(dest, br.input[br.byte_pos:][:num]) |
||||
br.byte_pos += num |
||||
} |
||||
|
||||
func initBitReader(br *bitReader) { |
||||
br.val_ = 0 |
||||
br.bit_pos_ = 64 |
||||
} |
||||
|
||||
func warmupBitReader(br *bitReader) bool { |
||||
/* Fixing alignment after unaligned BrotliFillWindow would result accumulator |
||||
overflow. If unalignment is caused by BrotliSafeReadBits, then there is |
||||
enough space in accumulator to fix alignment. */ |
||||
if getAvailableBits(br) == 0 { |
||||
if !pullByte(br) { |
||||
return false |
||||
} |
||||
} |
||||
|
||||
return true |
||||
} |
||||
@ -0,0 +1,153 @@ |
||||
package brotli |
||||
|
||||
/* Copyright 2013 Google Inc. All Rights Reserved. |
||||
|
||||
Distributed under MIT license. |
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/ |
||||
|
||||
/* Block split point selection utilities. */ |
||||
|
||||
type blockSplit struct { |
||||
num_types uint |
||||
num_blocks uint |
||||
types []byte |
||||
lengths []uint32 |
||||
types_alloc_size uint |
||||
lengths_alloc_size uint |
||||
} |
||||
|
||||
const ( |
||||
kMaxLiteralHistograms uint = 100 |
||||
kMaxCommandHistograms uint = 50 |
||||
kLiteralBlockSwitchCost float64 = 28.1 |
||||
kCommandBlockSwitchCost float64 = 13.5 |
||||
kDistanceBlockSwitchCost float64 = 14.6 |
||||
kLiteralStrideLength uint = 70 |
||||
kCommandStrideLength uint = 40 |
||||
kSymbolsPerLiteralHistogram uint = 544 |
||||
kSymbolsPerCommandHistogram uint = 530 |
||||
kSymbolsPerDistanceHistogram uint = 544 |
||||
kMinLengthForBlockSplitting uint = 128 |
||||
kIterMulForRefining uint = 2 |
||||
kMinItersForRefining uint = 100 |
||||
) |
||||
|
||||
func countLiterals(cmds []command, num_commands uint) uint { |
||||
var total_length uint = 0 |
||||
/* Count how many we have. */ |
||||
|
||||
var i uint |
||||
for i = 0; i < num_commands; i++ { |
||||
total_length += uint(cmds[i].insert_len_) |
||||
} |
||||
|
||||
return total_length |
||||
} |
||||
|
||||
func copyLiteralsToByteArray(cmds []command, num_commands uint, data []byte, offset uint, mask uint, literals []byte) { |
||||
var pos uint = 0 |
||||
var from_pos uint = offset & mask |
||||
var i uint |
||||
for i = 0; i < num_commands; i++ { |
||||
var insert_len uint = uint(cmds[i].insert_len_) |
||||
if from_pos+insert_len > mask { |
||||
var head_size uint = mask + 1 - from_pos |
||||
copy(literals[pos:], data[from_pos:][:head_size]) |
||||
from_pos = 0 |
||||
pos += head_size |
||||
insert_len -= head_size |
||||
} |
||||
|
||||
if insert_len > 0 { |
||||
copy(literals[pos:], data[from_pos:][:insert_len]) |
||||
pos += insert_len |
||||
} |
||||
|
||||
from_pos = uint((uint32(from_pos+insert_len) + commandCopyLen(&cmds[i])) & uint32(mask)) |
||||
} |
||||
} |
||||
|
||||
func myRand(seed *uint32) uint32 { |
||||
/* Initial seed should be 7. In this case, loop length is (1 << 29). */ |
||||
*seed *= 16807 |
||||
|
||||
return *seed |
||||
} |
||||
|
||||
func bitCost(count uint) float64 { |
||||
if count == 0 { |
||||
return -2.0 |
||||
} else { |
||||
return fastLog2(count) |
||||
} |
||||
} |
||||
|
||||
const histogramsPerBatch = 64 |
||||
|
||||
const clustersPerBatch = 16 |
||||
|
||||
func initBlockSplit(self *blockSplit) { |
||||
self.num_types = 0 |
||||
self.num_blocks = 0 |
||||
self.types = nil |
||||
self.lengths = nil |
||||
self.types_alloc_size = 0 |
||||
self.lengths_alloc_size = 0 |
||||
} |
||||
|
||||
func destroyBlockSplit(self *blockSplit) { |
||||
self.types = nil |
||||
self.lengths = nil |
||||
} |
||||
|
||||
func splitBlock(cmds []command, num_commands uint, data []byte, pos uint, mask uint, params *encoderParams, literal_split *blockSplit, insert_and_copy_split *blockSplit, dist_split *blockSplit) { |
||||
{ |
||||
var literals_count uint = countLiterals(cmds, num_commands) |
||||
var literals []byte = make([]byte, literals_count) |
||||
|
||||
/* Create a continuous array of literals. */ |
||||
copyLiteralsToByteArray(cmds, num_commands, data, pos, mask, literals) |
||||
|
||||
/* Create the block split on the array of literals. |
||||
Literal histograms have alphabet size 256. */ |
||||
splitByteVectorLiteral(literals, literals_count, kSymbolsPerLiteralHistogram, kMaxLiteralHistograms, kLiteralStrideLength, kLiteralBlockSwitchCost, params, literal_split) |
||||
|
||||
literals = nil |
||||
} |
||||
{ |
||||
var insert_and_copy_codes []uint16 = make([]uint16, num_commands) |
||||
/* Compute prefix codes for commands. */ |
||||
|
||||
var i uint |
||||
for i = 0; i < num_commands; i++ { |
||||
insert_and_copy_codes[i] = cmds[i].cmd_prefix_ |
||||
} |
||||
|
||||
/* Create the block split on the array of command prefixes. */ |
||||
splitByteVectorCommand(insert_and_copy_codes, num_commands, kSymbolsPerCommandHistogram, kMaxCommandHistograms, kCommandStrideLength, kCommandBlockSwitchCost, params, insert_and_copy_split) |
||||
|
||||
/* TODO: reuse for distances? */ |
||||
|
||||
insert_and_copy_codes = nil |
||||
} |
||||
{ |
||||
var distance_prefixes []uint16 = make([]uint16, num_commands) |
||||
var j uint = 0 |
||||
/* Create a continuous array of distance prefixes. */ |
||||
|
||||
var i uint |
||||
for i = 0; i < num_commands; i++ { |
||||
var cmd *command = &cmds[i] |
||||
if commandCopyLen(cmd) != 0 && cmd.cmd_prefix_ >= 128 { |
||||
distance_prefixes[j] = cmd.dist_prefix_ & 0x3FF |
||||
j++ |
||||
} |
||||
} |
||||
|
||||
/* Create the block split on the array of distance prefixes. */ |
||||
splitByteVectorDistance(distance_prefixes, j, kSymbolsPerDistanceHistogram, kMaxCommandHistograms, kCommandStrideLength, kDistanceBlockSwitchCost, params, dist_split) |
||||
|
||||
distance_prefixes = nil |
||||
} |
||||
} |
||||
@ -0,0 +1,433 @@ |
||||
package brotli |
||||
|
||||
import "math" |
||||
|
||||
/* Copyright 2013 Google Inc. All Rights Reserved. |
||||
|
||||
Distributed under MIT license. |
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/ |
||||
|
||||
func initialEntropyCodesCommand(data []uint16, length uint, stride uint, num_histograms uint, histograms []histogramCommand) { |
||||
var seed uint32 = 7 |
||||
var block_length uint = length / num_histograms |
||||
var i uint |
||||
clearHistogramsCommand(histograms, num_histograms) |
||||
for i = 0; i < num_histograms; i++ { |
||||
var pos uint = length * i / num_histograms |
||||
if i != 0 { |
||||
pos += uint(myRand(&seed) % uint32(block_length)) |
||||
} |
||||
|
||||
if pos+stride >= length { |
||||
pos = length - stride - 1 |
||||
} |
||||
|
||||
histogramAddVectorCommand(&histograms[i], data[pos:], stride) |
||||
} |
||||
} |
||||
|
||||
func randomSampleCommand(seed *uint32, data []uint16, length uint, stride uint, sample *histogramCommand) { |
||||
var pos uint = 0 |
||||
if stride >= length { |
||||
stride = length |
||||
} else { |
||||
pos = uint(myRand(seed) % uint32(length-stride+1)) |
||||
} |
||||
|
||||
histogramAddVectorCommand(sample, data[pos:], stride) |
||||
} |
||||
|
||||
func refineEntropyCodesCommand(data []uint16, length uint, stride uint, num_histograms uint, histograms []histogramCommand) { |
||||
var iters uint = kIterMulForRefining*length/stride + kMinItersForRefining |
||||
var seed uint32 = 7 |
||||
var iter uint |
||||
iters = ((iters + num_histograms - 1) / num_histograms) * num_histograms |
||||
for iter = 0; iter < iters; iter++ { |
||||
var sample histogramCommand |
||||
histogramClearCommand(&sample) |
||||
randomSampleCommand(&seed, data, length, stride, &sample) |
||||
histogramAddHistogramCommand(&histograms[iter%num_histograms], &sample) |
||||
} |
||||
} |
||||
|
||||
/* Assigns a block id from the range [0, num_histograms) to each data element |
||||
in data[0..length) and fills in block_id[0..length) with the assigned values. |
||||
Returns the number of blocks, i.e. one plus the number of block switches. */ |
||||
func findBlocksCommand(data []uint16, length uint, block_switch_bitcost float64, num_histograms uint, histograms []histogramCommand, insert_cost []float64, cost []float64, switch_signal []byte, block_id []byte) uint { |
||||
var data_size uint = histogramDataSizeCommand() |
||||
var bitmaplen uint = (num_histograms + 7) >> 3 |
||||
var num_blocks uint = 1 |
||||
var i uint |
||||
var j uint |
||||
assert(num_histograms <= 256) |
||||
if num_histograms <= 1 { |
||||
for i = 0; i < length; i++ { |
||||
block_id[i] = 0 |
||||
} |
||||
|
||||
return 1 |
||||
} |
||||
|
||||
for i := 0; i < int(data_size*num_histograms); i++ { |
||||
insert_cost[i] = 0 |
||||
} |
||||
for i = 0; i < num_histograms; i++ { |
||||
insert_cost[i] = fastLog2(uint(uint32(histograms[i].total_count_))) |
||||
} |
||||
|
||||
for i = data_size; i != 0; { |
||||
i-- |
||||
for j = 0; j < num_histograms; j++ { |
||||
insert_cost[i*num_histograms+j] = insert_cost[j] - bitCost(uint(histograms[j].data_[i])) |
||||
} |
||||
} |
||||
|
||||
for i := 0; i < int(num_histograms); i++ { |
||||
cost[i] = 0 |
||||
} |
||||
for i := 0; i < int(length*bitmaplen); i++ { |
||||
switch_signal[i] = 0 |
||||
} |
||||
|
||||
/* After each iteration of this loop, cost[k] will contain the difference |
||||
between the minimum cost of arriving at the current byte position using |
||||
entropy code k, and the minimum cost of arriving at the current byte |
||||
position. This difference is capped at the block switch cost, and if it |
||||
reaches block switch cost, it means that when we trace back from the last |
||||
position, we need to switch here. */ |
||||
for i = 0; i < length; i++ { |
||||
var byte_ix uint = i |
||||
var ix uint = byte_ix * bitmaplen |
||||
var insert_cost_ix uint = uint(data[byte_ix]) * num_histograms |
||||
var min_cost float64 = 1e99 |
||||
var block_switch_cost float64 = block_switch_bitcost |
||||
var k uint |
||||
for k = 0; k < num_histograms; k++ { |
||||
/* We are coding the symbol in data[byte_ix] with entropy code k. */ |
||||
cost[k] += insert_cost[insert_cost_ix+k] |
||||
|
||||
if cost[k] < min_cost { |
||||
min_cost = cost[k] |
||||
block_id[byte_ix] = byte(k) |
||||
} |
||||
} |
||||
|
||||
/* More blocks for the beginning. */ |
||||
if byte_ix < 2000 { |
||||
block_switch_cost *= 0.77 + 0.07*float64(byte_ix)/2000 |
||||
} |
||||
|
||||
for k = 0; k < num_histograms; k++ { |
||||
cost[k] -= min_cost |
||||
if cost[k] >= block_switch_cost { |
||||
var mask byte = byte(1 << (k & 7)) |
||||
cost[k] = block_switch_cost |
||||
assert(k>>3 < bitmaplen) |
||||
switch_signal[ix+(k>>3)] |= mask |
||||
/* Trace back from the last position and switch at the marked places. */ |
||||
} |
||||
} |
||||
} |
||||
{ |
||||
var byte_ix uint = length - 1 |
||||
var ix uint = byte_ix * bitmaplen |
||||
var cur_id byte = block_id[byte_ix] |
||||
for byte_ix > 0 { |
||||
var mask byte = byte(1 << (cur_id & 7)) |
||||
assert(uint(cur_id)>>3 < bitmaplen) |
||||
byte_ix-- |
||||
ix -= bitmaplen |
||||
if switch_signal[ix+uint(cur_id>>3)]&mask != 0 { |
||||
if cur_id != block_id[byte_ix] { |
||||
cur_id = block_id[byte_ix] |
||||
num_blocks++ |
||||
} |
||||
} |
||||
|
||||
block_id[byte_ix] = cur_id |
||||
} |
||||
} |
||||
|
||||
return num_blocks |
||||
} |
||||
|
||||
var remapBlockIdsCommand_kInvalidId uint16 = 256 |
||||
|
||||
func remapBlockIdsCommand(block_ids []byte, length uint, new_id []uint16, num_histograms uint) uint { |
||||
var next_id uint16 = 0 |
||||
var i uint |
||||
for i = 0; i < num_histograms; i++ { |
||||
new_id[i] = remapBlockIdsCommand_kInvalidId |
||||
} |
||||
|
||||
for i = 0; i < length; i++ { |
||||
assert(uint(block_ids[i]) < num_histograms) |
||||
if new_id[block_ids[i]] == remapBlockIdsCommand_kInvalidId { |
||||
new_id[block_ids[i]] = next_id |
||||
next_id++ |
||||
} |
||||
} |
||||
|
||||
for i = 0; i < length; i++ { |
||||
block_ids[i] = byte(new_id[block_ids[i]]) |
||||
assert(uint(block_ids[i]) < num_histograms) |
||||
} |
||||
|
||||
assert(uint(next_id) <= num_histograms) |
||||
return uint(next_id) |
||||
} |
||||
|
||||
func buildBlockHistogramsCommand(data []uint16, length uint, block_ids []byte, num_histograms uint, histograms []histogramCommand) { |
||||
var i uint |
||||
clearHistogramsCommand(histograms, num_histograms) |
||||
for i = 0; i < length; i++ { |
||||
histogramAddCommand(&histograms[block_ids[i]], uint(data[i])) |
||||
} |
||||
} |
||||
|
||||
var clusterBlocksCommand_kInvalidIndex uint32 = math.MaxUint32 |
||||
|
||||
func clusterBlocksCommand(data []uint16, length uint, num_blocks uint, block_ids []byte, split *blockSplit) { |
||||
var histogram_symbols []uint32 = make([]uint32, num_blocks) |
||||
var block_lengths []uint32 = make([]uint32, num_blocks) |
||||
var expected_num_clusters uint = clustersPerBatch * (num_blocks + histogramsPerBatch - 1) / histogramsPerBatch |
||||
var all_histograms_size uint = 0 |
||||
var all_histograms_capacity uint = expected_num_clusters |
||||
var all_histograms []histogramCommand = make([]histogramCommand, all_histograms_capacity) |
||||
var cluster_size_size uint = 0 |
||||
var cluster_size_capacity uint = expected_num_clusters |
||||
var cluster_size []uint32 = make([]uint32, cluster_size_capacity) |
||||
var num_clusters uint = 0 |
||||
var histograms []histogramCommand = make([]histogramCommand, brotli_min_size_t(num_blocks, histogramsPerBatch)) |
||||
var max_num_pairs uint = histogramsPerBatch * histogramsPerBatch / 2 |
||||
var pairs_capacity uint = max_num_pairs + 1 |
||||
var pairs []histogramPair = make([]histogramPair, pairs_capacity) |
||||
var pos uint = 0 |
||||
var clusters []uint32 |
||||
var num_final_clusters uint |
||||
var new_index []uint32 |
||||
var i uint |
||||
var sizes = [histogramsPerBatch]uint32{0} |
||||
var new_clusters = [histogramsPerBatch]uint32{0} |
||||
var symbols = [histogramsPerBatch]uint32{0} |
||||
var remap = [histogramsPerBatch]uint32{0} |
||||
|
||||
for i := 0; i < int(num_blocks); i++ { |
||||
block_lengths[i] = 0 |
||||
} |
||||
{ |
||||
var block_idx uint = 0 |
||||
for i = 0; i < length; i++ { |
||||
assert(block_idx < num_blocks) |
||||
block_lengths[block_idx]++ |
||||
if i+1 == length || block_ids[i] != block_ids[i+1] { |
||||
block_idx++ |
||||
} |
||||
} |
||||
|
||||
assert(block_idx == num_blocks) |
||||
} |
||||
|
||||
for i = 0; i < num_blocks; i += histogramsPerBatch { |
||||
var num_to_combine uint = brotli_min_size_t(num_blocks-i, histogramsPerBatch) |
||||
var num_new_clusters uint |
||||
var j uint |
||||
for j = 0; j < num_to_combine; j++ { |
||||
var k uint |
||||
histogramClearCommand(&histograms[j]) |
||||
for k = 0; uint32(k) < block_lengths[i+j]; k++ { |
||||
histogramAddCommand(&histograms[j], uint(data[pos])) |
||||
pos++ |
||||
} |
||||
|
||||
histograms[j].bit_cost_ = populationCostCommand(&histograms[j]) |
||||
new_clusters[j] = uint32(j) |
||||
symbols[j] = uint32(j) |
||||
sizes[j] = 1 |
||||
} |
||||
|
||||
num_new_clusters = histogramCombineCommand(histograms, sizes[:], symbols[:], new_clusters[:], []histogramPair(pairs), num_to_combine, num_to_combine, histogramsPerBatch, max_num_pairs) |
||||
if all_histograms_capacity < (all_histograms_size + num_new_clusters) { |
||||
var _new_size uint |
||||
if all_histograms_capacity == 0 { |
||||
_new_size = all_histograms_size + num_new_clusters |
||||
} else { |
||||
_new_size = all_histograms_capacity |
||||
} |
||||
var new_array []histogramCommand |
||||
for _new_size < (all_histograms_size + num_new_clusters) { |
||||
_new_size *= 2 |
||||
} |
||||
new_array = make([]histogramCommand, _new_size) |
||||
if all_histograms_capacity != 0 { |
||||
copy(new_array, all_histograms[:all_histograms_capacity]) |
||||
} |
||||
|
||||
all_histograms = new_array |
||||
all_histograms_capacity = _new_size |
||||
} |
||||
|
||||
brotli_ensure_capacity_uint32_t(&cluster_size, &cluster_size_capacity, cluster_size_size+num_new_clusters) |
||||
for j = 0; j < num_new_clusters; j++ { |
||||
all_histograms[all_histograms_size] = histograms[new_clusters[j]] |
||||
all_histograms_size++ |
||||
cluster_size[cluster_size_size] = sizes[new_clusters[j]] |
||||
cluster_size_size++ |
||||
remap[new_clusters[j]] = uint32(j) |
||||
} |
||||
|
||||
for j = 0; j < num_to_combine; j++ { |
||||
histogram_symbols[i+j] = uint32(num_clusters) + remap[symbols[j]] |
||||
} |
||||
|
||||
num_clusters += num_new_clusters |
||||
assert(num_clusters == cluster_size_size) |
||||
assert(num_clusters == all_histograms_size) |
||||
} |
||||
|
||||
histograms = nil |
||||
|
||||
max_num_pairs = brotli_min_size_t(64*num_clusters, (num_clusters/2)*num_clusters) |
||||
if pairs_capacity < max_num_pairs+1 { |
||||
pairs = nil |
||||
pairs = make([]histogramPair, (max_num_pairs + 1)) |
||||
} |
||||
|
||||
clusters = make([]uint32, num_clusters) |
||||
for i = 0; i < num_clusters; i++ { |
||||
clusters[i] = uint32(i) |
||||
} |
||||
|
||||
num_final_clusters = histogramCombineCommand(all_histograms, cluster_size, histogram_symbols, clusters, pairs, num_clusters, num_blocks, maxNumberOfBlockTypes, max_num_pairs) |
||||
pairs = nil |
||||
cluster_size = nil |
||||
|
||||
new_index = make([]uint32, num_clusters) |
||||
for i = 0; i < num_clusters; i++ { |
||||
new_index[i] = clusterBlocksCommand_kInvalidIndex |
||||
} |
||||
pos = 0 |
||||
{ |
||||
var next_index uint32 = 0 |
||||
for i = 0; i < num_blocks; i++ { |
||||
var histo histogramCommand |
||||
var j uint |
||||
var best_out uint32 |
||||
var best_bits float64 |
||||
histogramClearCommand(&histo) |
||||
for j = 0; uint32(j) < block_lengths[i]; j++ { |
||||
histogramAddCommand(&histo, uint(data[pos])) |
||||
pos++ |
||||
} |
||||
|
||||
if i == 0 { |
||||
best_out = histogram_symbols[0] |
||||
} else { |
||||
best_out = histogram_symbols[i-1] |
||||
} |
||||
best_bits = histogramBitCostDistanceCommand(&histo, &all_histograms[best_out]) |
||||
for j = 0; j < num_final_clusters; j++ { |
||||
var cur_bits float64 = histogramBitCostDistanceCommand(&histo, &all_histograms[clusters[j]]) |
||||
if cur_bits < best_bits { |
||||
best_bits = cur_bits |
||||
best_out = clusters[j] |
||||
} |
||||
} |
||||
|
||||
histogram_symbols[i] = best_out |
||||
if new_index[best_out] == clusterBlocksCommand_kInvalidIndex { |
||||
new_index[best_out] = next_index |
||||
next_index++ |
||||
} |
||||
} |
||||
} |
||||
|
||||
clusters = nil |
||||
all_histograms = nil |
||||
brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, num_blocks) |
||||
brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, num_blocks) |
||||
{ |
||||
var cur_length uint32 = 0 |
||||
var block_idx uint = 0 |
||||
var max_type byte = 0 |
||||
for i = 0; i < num_blocks; i++ { |
||||
cur_length += block_lengths[i] |
||||
if i+1 == num_blocks || histogram_symbols[i] != histogram_symbols[i+1] { |
||||
var id byte = byte(new_index[histogram_symbols[i]]) |
||||
split.types[block_idx] = id |
||||
split.lengths[block_idx] = cur_length |
||||
max_type = brotli_max_uint8_t(max_type, id) |
||||
cur_length = 0 |
||||
block_idx++ |
||||
} |
||||
} |
||||
|
||||
split.num_blocks = block_idx |
||||
split.num_types = uint(max_type) + 1 |
||||
} |
||||
|
||||
new_index = nil |
||||
block_lengths = nil |
||||
histogram_symbols = nil |
||||
} |
||||
|
||||
func splitByteVectorCommand(data []uint16, length uint, literals_per_histogram uint, max_histograms uint, sampling_stride_length uint, block_switch_cost float64, params *encoderParams, split *blockSplit) { |
||||
var data_size uint = histogramDataSizeCommand() |
||||
var num_histograms uint = length/literals_per_histogram + 1 |
||||
var histograms []histogramCommand |
||||
if num_histograms > max_histograms { |
||||
num_histograms = max_histograms |
||||
} |
||||
|
||||
if length == 0 { |
||||
split.num_types = 1 |
||||
return |
||||
} else if length < kMinLengthForBlockSplitting { |
||||
brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, split.num_blocks+1) |
||||
brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, split.num_blocks+1) |
||||
split.num_types = 1 |
||||
split.types[split.num_blocks] = 0 |
||||
split.lengths[split.num_blocks] = uint32(length) |
||||
split.num_blocks++ |
||||
return |
||||
} |
||||
|
||||
histograms = make([]histogramCommand, num_histograms) |
||||
|
||||
/* Find good entropy codes. */ |
||||
initialEntropyCodesCommand(data, length, sampling_stride_length, num_histograms, histograms) |
||||
|
||||
refineEntropyCodesCommand(data, length, sampling_stride_length, num_histograms, histograms) |
||||
{ |
||||
var block_ids []byte = make([]byte, length) |
||||
var num_blocks uint = 0 |
||||
var bitmaplen uint = (num_histograms + 7) >> 3 |
||||
var insert_cost []float64 = make([]float64, (data_size * num_histograms)) |
||||
var cost []float64 = make([]float64, num_histograms) |
||||
var switch_signal []byte = make([]byte, (length * bitmaplen)) |
||||
var new_id []uint16 = make([]uint16, num_histograms) |
||||
var iters uint |
||||
if params.quality < hqZopflificationQuality { |
||||
iters = 3 |
||||
} else { |
||||
iters = 10 |
||||
} |
||||
/* Find a good path through literals with the good entropy codes. */ |
||||
|
||||
var i uint |
||||
for i = 0; i < iters; i++ { |
||||
num_blocks = findBlocksCommand(data, length, block_switch_cost, num_histograms, histograms, insert_cost, cost, switch_signal, block_ids) |
||||
num_histograms = remapBlockIdsCommand(block_ids, length, new_id, num_histograms) |
||||
buildBlockHistogramsCommand(data, length, block_ids, num_histograms, histograms) |
||||
} |
||||
|
||||
insert_cost = nil |
||||
cost = nil |
||||
switch_signal = nil |
||||
new_id = nil |
||||
histograms = nil |
||||
clusterBlocksCommand(data, length, num_blocks, block_ids, split) |
||||
block_ids = nil |
||||
} |
||||
} |
||||
@ -0,0 +1,433 @@ |
||||
package brotli |
||||
|
||||
import "math" |
||||
|
||||
/* Copyright 2013 Google Inc. All Rights Reserved. |
||||
|
||||
Distributed under MIT license. |
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/ |
||||
|
||||
func initialEntropyCodesDistance(data []uint16, length uint, stride uint, num_histograms uint, histograms []histogramDistance) { |
||||
var seed uint32 = 7 |
||||
var block_length uint = length / num_histograms |
||||
var i uint |
||||
clearHistogramsDistance(histograms, num_histograms) |
||||
for i = 0; i < num_histograms; i++ { |
||||
var pos uint = length * i / num_histograms |
||||
if i != 0 { |
||||
pos += uint(myRand(&seed) % uint32(block_length)) |
||||
} |
||||
|
||||
if pos+stride >= length { |
||||
pos = length - stride - 1 |
||||
} |
||||
|
||||
histogramAddVectorDistance(&histograms[i], data[pos:], stride) |
||||
} |
||||
} |
||||
|
||||
func randomSampleDistance(seed *uint32, data []uint16, length uint, stride uint, sample *histogramDistance) { |
||||
var pos uint = 0 |
||||
if stride >= length { |
||||
stride = length |
||||
} else { |
||||
pos = uint(myRand(seed) % uint32(length-stride+1)) |
||||
} |
||||
|
||||
histogramAddVectorDistance(sample, data[pos:], stride) |
||||
} |
||||
|
||||
func refineEntropyCodesDistance(data []uint16, length uint, stride uint, num_histograms uint, histograms []histogramDistance) { |
||||
var iters uint = kIterMulForRefining*length/stride + kMinItersForRefining |
||||
var seed uint32 = 7 |
||||
var iter uint |
||||
iters = ((iters + num_histograms - 1) / num_histograms) * num_histograms |
||||
for iter = 0; iter < iters; iter++ { |
||||
var sample histogramDistance |
||||
histogramClearDistance(&sample) |
||||
randomSampleDistance(&seed, data, length, stride, &sample) |
||||
histogramAddHistogramDistance(&histograms[iter%num_histograms], &sample) |
||||
} |
||||
} |
||||
|
||||
/* Assigns a block id from the range [0, num_histograms) to each data element |
||||
in data[0..length) and fills in block_id[0..length) with the assigned values. |
||||
Returns the number of blocks, i.e. one plus the number of block switches. */ |
||||
func findBlocksDistance(data []uint16, length uint, block_switch_bitcost float64, num_histograms uint, histograms []histogramDistance, insert_cost []float64, cost []float64, switch_signal []byte, block_id []byte) uint { |
||||
var data_size uint = histogramDataSizeDistance() |
||||
var bitmaplen uint = (num_histograms + 7) >> 3 |
||||
var num_blocks uint = 1 |
||||
var i uint |
||||
var j uint |
||||
assert(num_histograms <= 256) |
||||
if num_histograms <= 1 { |
||||
for i = 0; i < length; i++ { |
||||
block_id[i] = 0 |
||||
} |
||||
|
||||
return 1 |
||||
} |
||||
|
||||
for i := 0; i < int(data_size*num_histograms); i++ { |
||||
insert_cost[i] = 0 |
||||
} |
||||
for i = 0; i < num_histograms; i++ { |
||||
insert_cost[i] = fastLog2(uint(uint32(histograms[i].total_count_))) |
||||
} |
||||
|
||||
for i = data_size; i != 0; { |
||||
i-- |
||||
for j = 0; j < num_histograms; j++ { |
||||
insert_cost[i*num_histograms+j] = insert_cost[j] - bitCost(uint(histograms[j].data_[i])) |
||||
} |
||||
} |
||||
|
||||
for i := 0; i < int(num_histograms); i++ { |
||||
cost[i] = 0 |
||||
} |
||||
for i := 0; i < int(length*bitmaplen); i++ { |
||||
switch_signal[i] = 0 |
||||
} |
||||
|
||||
/* After each iteration of this loop, cost[k] will contain the difference |
||||
between the minimum cost of arriving at the current byte position using |
||||
entropy code k, and the minimum cost of arriving at the current byte |
||||
position. This difference is capped at the block switch cost, and if it |
||||
reaches block switch cost, it means that when we trace back from the last |
||||
position, we need to switch here. */ |
||||
for i = 0; i < length; i++ { |
||||
var byte_ix uint = i |
||||
var ix uint = byte_ix * bitmaplen |
||||
var insert_cost_ix uint = uint(data[byte_ix]) * num_histograms |
||||
var min_cost float64 = 1e99 |
||||
var block_switch_cost float64 = block_switch_bitcost |
||||
var k uint |
||||
for k = 0; k < num_histograms; k++ { |
||||
/* We are coding the symbol in data[byte_ix] with entropy code k. */ |
||||
cost[k] += insert_cost[insert_cost_ix+k] |
||||
|
||||
if cost[k] < min_cost { |
||||
min_cost = cost[k] |
||||
block_id[byte_ix] = byte(k) |
||||
} |
||||
} |
||||
|
||||
/* More blocks for the beginning. */ |
||||
if byte_ix < 2000 { |
||||
block_switch_cost *= 0.77 + 0.07*float64(byte_ix)/2000 |
||||
} |
||||
|
||||
for k = 0; k < num_histograms; k++ { |
||||
cost[k] -= min_cost |
||||
if cost[k] >= block_switch_cost { |
||||
var mask byte = byte(1 << (k & 7)) |
||||
cost[k] = block_switch_cost |
||||
assert(k>>3 < bitmaplen) |
||||
switch_signal[ix+(k>>3)] |= mask |
||||
/* Trace back from the last position and switch at the marked places. */ |
||||
} |
||||
} |
||||
} |
||||
{ |
||||
var byte_ix uint = length - 1 |
||||
var ix uint = byte_ix * bitmaplen |
||||
var cur_id byte = block_id[byte_ix] |
||||
for byte_ix > 0 { |
||||
var mask byte = byte(1 << (cur_id & 7)) |
||||
assert(uint(cur_id)>>3 < bitmaplen) |
||||
byte_ix-- |
||||
ix -= bitmaplen |
||||
if switch_signal[ix+uint(cur_id>>3)]&mask != 0 { |
||||
if cur_id != block_id[byte_ix] { |
||||
cur_id = block_id[byte_ix] |
||||
num_blocks++ |
||||
} |
||||
} |
||||
|
||||
block_id[byte_ix] = cur_id |
||||
} |
||||
} |
||||
|
||||
return num_blocks |
||||
} |
||||
|
||||
var remapBlockIdsDistance_kInvalidId uint16 = 256 |
||||
|
||||
func remapBlockIdsDistance(block_ids []byte, length uint, new_id []uint16, num_histograms uint) uint { |
||||
var next_id uint16 = 0 |
||||
var i uint |
||||
for i = 0; i < num_histograms; i++ { |
||||
new_id[i] = remapBlockIdsDistance_kInvalidId |
||||
} |
||||
|
||||
for i = 0; i < length; i++ { |
||||
assert(uint(block_ids[i]) < num_histograms) |
||||
if new_id[block_ids[i]] == remapBlockIdsDistance_kInvalidId { |
||||
new_id[block_ids[i]] = next_id |
||||
next_id++ |
||||
} |
||||
} |
||||
|
||||
for i = 0; i < length; i++ { |
||||
block_ids[i] = byte(new_id[block_ids[i]]) |
||||
assert(uint(block_ids[i]) < num_histograms) |
||||
} |
||||
|
||||
assert(uint(next_id) <= num_histograms) |
||||
return uint(next_id) |
||||
} |
||||
|
||||
func buildBlockHistogramsDistance(data []uint16, length uint, block_ids []byte, num_histograms uint, histograms []histogramDistance) { |
||||
var i uint |
||||
clearHistogramsDistance(histograms, num_histograms) |
||||
for i = 0; i < length; i++ { |
||||
histogramAddDistance(&histograms[block_ids[i]], uint(data[i])) |
||||
} |
||||
} |
||||
|
||||
var clusterBlocksDistance_kInvalidIndex uint32 = math.MaxUint32 |
||||
|
||||
func clusterBlocksDistance(data []uint16, length uint, num_blocks uint, block_ids []byte, split *blockSplit) { |
||||
var histogram_symbols []uint32 = make([]uint32, num_blocks) |
||||
var block_lengths []uint32 = make([]uint32, num_blocks) |
||||
var expected_num_clusters uint = clustersPerBatch * (num_blocks + histogramsPerBatch - 1) / histogramsPerBatch |
||||
var all_histograms_size uint = 0 |
||||
var all_histograms_capacity uint = expected_num_clusters |
||||
var all_histograms []histogramDistance = make([]histogramDistance, all_histograms_capacity) |
||||
var cluster_size_size uint = 0 |
||||
var cluster_size_capacity uint = expected_num_clusters |
||||
var cluster_size []uint32 = make([]uint32, cluster_size_capacity) |
||||
var num_clusters uint = 0 |
||||
var histograms []histogramDistance = make([]histogramDistance, brotli_min_size_t(num_blocks, histogramsPerBatch)) |
||||
var max_num_pairs uint = histogramsPerBatch * histogramsPerBatch / 2 |
||||
var pairs_capacity uint = max_num_pairs + 1 |
||||
var pairs []histogramPair = make([]histogramPair, pairs_capacity) |
||||
var pos uint = 0 |
||||
var clusters []uint32 |
||||
var num_final_clusters uint |
||||
var new_index []uint32 |
||||
var i uint |
||||
var sizes = [histogramsPerBatch]uint32{0} |
||||
var new_clusters = [histogramsPerBatch]uint32{0} |
||||
var symbols = [histogramsPerBatch]uint32{0} |
||||
var remap = [histogramsPerBatch]uint32{0} |
||||
|
||||
for i := 0; i < int(num_blocks); i++ { |
||||
block_lengths[i] = 0 |
||||
} |
||||
{ |
||||
var block_idx uint = 0 |
||||
for i = 0; i < length; i++ { |
||||
assert(block_idx < num_blocks) |
||||
block_lengths[block_idx]++ |
||||
if i+1 == length || block_ids[i] != block_ids[i+1] { |
||||
block_idx++ |
||||
} |
||||
} |
||||
|
||||
assert(block_idx == num_blocks) |
||||
} |
||||
|
||||
for i = 0; i < num_blocks; i += histogramsPerBatch { |
||||
var num_to_combine uint = brotli_min_size_t(num_blocks-i, histogramsPerBatch) |
||||
var num_new_clusters uint |
||||
var j uint |
||||
for j = 0; j < num_to_combine; j++ { |
||||
var k uint |
||||
histogramClearDistance(&histograms[j]) |
||||
for k = 0; uint32(k) < block_lengths[i+j]; k++ { |
||||
histogramAddDistance(&histograms[j], uint(data[pos])) |
||||
pos++ |
||||
} |
||||
|
||||
histograms[j].bit_cost_ = populationCostDistance(&histograms[j]) |
||||
new_clusters[j] = uint32(j) |
||||
symbols[j] = uint32(j) |
||||
sizes[j] = 1 |
||||
} |
||||
|
||||
num_new_clusters = histogramCombineDistance(histograms, sizes[:], symbols[:], new_clusters[:], []histogramPair(pairs), num_to_combine, num_to_combine, histogramsPerBatch, max_num_pairs) |
||||
if all_histograms_capacity < (all_histograms_size + num_new_clusters) { |
||||
var _new_size uint |
||||
if all_histograms_capacity == 0 { |
||||
_new_size = all_histograms_size + num_new_clusters |
||||
} else { |
||||
_new_size = all_histograms_capacity |
||||
} |
||||
var new_array []histogramDistance |
||||
for _new_size < (all_histograms_size + num_new_clusters) { |
||||
_new_size *= 2 |
||||
} |
||||
new_array = make([]histogramDistance, _new_size) |
||||
if all_histograms_capacity != 0 { |
||||
copy(new_array, all_histograms[:all_histograms_capacity]) |
||||
} |
||||
|
||||
all_histograms = new_array |
||||
all_histograms_capacity = _new_size |
||||
} |
||||
|
||||
brotli_ensure_capacity_uint32_t(&cluster_size, &cluster_size_capacity, cluster_size_size+num_new_clusters) |
||||
for j = 0; j < num_new_clusters; j++ { |
||||
all_histograms[all_histograms_size] = histograms[new_clusters[j]] |
||||
all_histograms_size++ |
||||
cluster_size[cluster_size_size] = sizes[new_clusters[j]] |
||||
cluster_size_size++ |
||||
remap[new_clusters[j]] = uint32(j) |
||||
} |
||||
|
||||
for j = 0; j < num_to_combine; j++ { |
||||
histogram_symbols[i+j] = uint32(num_clusters) + remap[symbols[j]] |
||||
} |
||||
|
||||
num_clusters += num_new_clusters |
||||
assert(num_clusters == cluster_size_size) |
||||
assert(num_clusters == all_histograms_size) |
||||
} |
||||
|
||||
histograms = nil |
||||
|
||||
max_num_pairs = brotli_min_size_t(64*num_clusters, (num_clusters/2)*num_clusters) |
||||
if pairs_capacity < max_num_pairs+1 { |
||||
pairs = nil |
||||
pairs = make([]histogramPair, (max_num_pairs + 1)) |
||||
} |
||||
|
||||
clusters = make([]uint32, num_clusters) |
||||
for i = 0; i < num_clusters; i++ { |
||||
clusters[i] = uint32(i) |
||||
} |
||||
|
||||
num_final_clusters = histogramCombineDistance(all_histograms, cluster_size, histogram_symbols, clusters, pairs, num_clusters, num_blocks, maxNumberOfBlockTypes, max_num_pairs) |
||||
pairs = nil |
||||
cluster_size = nil |
||||
|
||||
new_index = make([]uint32, num_clusters) |
||||
for i = 0; i < num_clusters; i++ { |
||||
new_index[i] = clusterBlocksDistance_kInvalidIndex |
||||
} |
||||
pos = 0 |
||||
{ |
||||
var next_index uint32 = 0 |
||||
for i = 0; i < num_blocks; i++ { |
||||
var histo histogramDistance |
||||
var j uint |
||||
var best_out uint32 |
||||
var best_bits float64 |
||||
histogramClearDistance(&histo) |
||||
for j = 0; uint32(j) < block_lengths[i]; j++ { |
||||
histogramAddDistance(&histo, uint(data[pos])) |
||||
pos++ |
||||
} |
||||
|
||||
if i == 0 { |
||||
best_out = histogram_symbols[0] |
||||
} else { |
||||
best_out = histogram_symbols[i-1] |
||||
} |
||||
best_bits = histogramBitCostDistanceDistance(&histo, &all_histograms[best_out]) |
||||
for j = 0; j < num_final_clusters; j++ { |
||||
var cur_bits float64 = histogramBitCostDistanceDistance(&histo, &all_histograms[clusters[j]]) |
||||
if cur_bits < best_bits { |
||||
best_bits = cur_bits |
||||
best_out = clusters[j] |
||||
} |
||||
} |
||||
|
||||
histogram_symbols[i] = best_out |
||||
if new_index[best_out] == clusterBlocksDistance_kInvalidIndex { |
||||
new_index[best_out] = next_index |
||||
next_index++ |
||||
} |
||||
} |
||||
} |
||||
|
||||
clusters = nil |
||||
all_histograms = nil |
||||
brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, num_blocks) |
||||
brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, num_blocks) |
||||
{ |
||||
var cur_length uint32 = 0 |
||||
var block_idx uint = 0 |
||||
var max_type byte = 0 |
||||
for i = 0; i < num_blocks; i++ { |
||||
cur_length += block_lengths[i] |
||||
if i+1 == num_blocks || histogram_symbols[i] != histogram_symbols[i+1] { |
||||
var id byte = byte(new_index[histogram_symbols[i]]) |
||||
split.types[block_idx] = id |
||||
split.lengths[block_idx] = cur_length |
||||
max_type = brotli_max_uint8_t(max_type, id) |
||||
cur_length = 0 |
||||
block_idx++ |
||||
} |
||||
} |
||||
|
||||
split.num_blocks = block_idx |
||||
split.num_types = uint(max_type) + 1 |
||||
} |
||||
|
||||
new_index = nil |
||||
block_lengths = nil |
||||
histogram_symbols = nil |
||||
} |
||||
|
||||
func splitByteVectorDistance(data []uint16, length uint, literals_per_histogram uint, max_histograms uint, sampling_stride_length uint, block_switch_cost float64, params *encoderParams, split *blockSplit) { |
||||
var data_size uint = histogramDataSizeDistance() |
||||
var num_histograms uint = length/literals_per_histogram + 1 |
||||
var histograms []histogramDistance |
||||
if num_histograms > max_histograms { |
||||
num_histograms = max_histograms |
||||
} |
||||
|
||||
if length == 0 { |
||||
split.num_types = 1 |
||||
return |
||||
} else if length < kMinLengthForBlockSplitting { |
||||
brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, split.num_blocks+1) |
||||
brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, split.num_blocks+1) |
||||
split.num_types = 1 |
||||
split.types[split.num_blocks] = 0 |
||||
split.lengths[split.num_blocks] = uint32(length) |
||||
split.num_blocks++ |
||||
return |
||||
} |
||||
|
||||
histograms = make([]histogramDistance, num_histograms) |
||||
|
||||
/* Find good entropy codes. */ |
||||
initialEntropyCodesDistance(data, length, sampling_stride_length, num_histograms, histograms) |
||||
|
||||
refineEntropyCodesDistance(data, length, sampling_stride_length, num_histograms, histograms) |
||||
{ |
||||
var block_ids []byte = make([]byte, length) |
||||
var num_blocks uint = 0 |
||||
var bitmaplen uint = (num_histograms + 7) >> 3 |
||||
var insert_cost []float64 = make([]float64, (data_size * num_histograms)) |
||||
var cost []float64 = make([]float64, num_histograms) |
||||
var switch_signal []byte = make([]byte, (length * bitmaplen)) |
||||
var new_id []uint16 = make([]uint16, num_histograms) |
||||
var iters uint |
||||
if params.quality < hqZopflificationQuality { |
||||
iters = 3 |
||||
} else { |
||||
iters = 10 |
||||
} |
||||
/* Find a good path through literals with the good entropy codes. */ |
||||
|
||||
var i uint |
||||
for i = 0; i < iters; i++ { |
||||
num_blocks = findBlocksDistance(data, length, block_switch_cost, num_histograms, histograms, insert_cost, cost, switch_signal, block_ids) |
||||
num_histograms = remapBlockIdsDistance(block_ids, length, new_id, num_histograms) |
||||
buildBlockHistogramsDistance(data, length, block_ids, num_histograms, histograms) |
||||
} |
||||
|
||||
insert_cost = nil |
||||
cost = nil |
||||
switch_signal = nil |
||||
new_id = nil |
||||
histograms = nil |
||||
clusterBlocksDistance(data, length, num_blocks, block_ids, split) |
||||
block_ids = nil |
||||
} |
||||
} |
||||
@ -0,0 +1,433 @@ |
||||
package brotli |
||||
|
||||
import "math" |
||||
|
||||
/* Copyright 2013 Google Inc. All Rights Reserved. |
||||
|
||||
Distributed under MIT license. |
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/ |
||||
|
||||
func initialEntropyCodesLiteral(data []byte, length uint, stride uint, num_histograms uint, histograms []histogramLiteral) { |
||||
var seed uint32 = 7 |
||||
var block_length uint = length / num_histograms |
||||
var i uint |
||||
clearHistogramsLiteral(histograms, num_histograms) |
||||
for i = 0; i < num_histograms; i++ { |
||||
var pos uint = length * i / num_histograms |
||||
if i != 0 { |
||||
pos += uint(myRand(&seed) % uint32(block_length)) |
||||
} |
||||
|
||||
if pos+stride >= length { |
||||
pos = length - stride - 1 |
||||
} |
||||
|
||||
histogramAddVectorLiteral(&histograms[i], data[pos:], stride) |
||||
} |
||||
} |
||||
|
||||
func randomSampleLiteral(seed *uint32, data []byte, length uint, stride uint, sample *histogramLiteral) { |
||||
var pos uint = 0 |
||||
if stride >= length { |
||||
stride = length |
||||
} else { |
||||
pos = uint(myRand(seed) % uint32(length-stride+1)) |
||||
} |
||||
|
||||
histogramAddVectorLiteral(sample, data[pos:], stride) |
||||
} |
||||
|
||||
func refineEntropyCodesLiteral(data []byte, length uint, stride uint, num_histograms uint, histograms []histogramLiteral) { |
||||
var iters uint = kIterMulForRefining*length/stride + kMinItersForRefining |
||||
var seed uint32 = 7 |
||||
var iter uint |
||||
iters = ((iters + num_histograms - 1) / num_histograms) * num_histograms |
||||
for iter = 0; iter < iters; iter++ { |
||||
var sample histogramLiteral |
||||
histogramClearLiteral(&sample) |
||||
randomSampleLiteral(&seed, data, length, stride, &sample) |
||||
histogramAddHistogramLiteral(&histograms[iter%num_histograms], &sample) |
||||
} |
||||
} |
||||
|
||||
/* Assigns a block id from the range [0, num_histograms) to each data element |
||||
in data[0..length) and fills in block_id[0..length) with the assigned values. |
||||
Returns the number of blocks, i.e. one plus the number of block switches. */ |
||||
func findBlocksLiteral(data []byte, length uint, block_switch_bitcost float64, num_histograms uint, histograms []histogramLiteral, insert_cost []float64, cost []float64, switch_signal []byte, block_id []byte) uint { |
||||
var data_size uint = histogramDataSizeLiteral() |
||||
var bitmaplen uint = (num_histograms + 7) >> 3 |
||||
var num_blocks uint = 1 |
||||
var i uint |
||||
var j uint |
||||
assert(num_histograms <= 256) |
||||
if num_histograms <= 1 { |
||||
for i = 0; i < length; i++ { |
||||
block_id[i] = 0 |
||||
} |
||||
|
||||
return 1 |
||||
} |
||||
|
||||
for i := 0; i < int(data_size*num_histograms); i++ { |
||||
insert_cost[i] = 0 |
||||
} |
||||
for i = 0; i < num_histograms; i++ { |
||||
insert_cost[i] = fastLog2(uint(uint32(histograms[i].total_count_))) |
||||
} |
||||
|
||||
for i = data_size; i != 0; { |
||||
i-- |
||||
for j = 0; j < num_histograms; j++ { |
||||
insert_cost[i*num_histograms+j] = insert_cost[j] - bitCost(uint(histograms[j].data_[i])) |
||||
} |
||||
} |
||||
|
||||
for i := 0; i < int(num_histograms); i++ { |
||||
cost[i] = 0 |
||||
} |
||||
for i := 0; i < int(length*bitmaplen); i++ { |
||||
switch_signal[i] = 0 |
||||
} |
||||
|
||||
/* After each iteration of this loop, cost[k] will contain the difference |
||||
between the minimum cost of arriving at the current byte position using |
||||
entropy code k, and the minimum cost of arriving at the current byte |
||||
position. This difference is capped at the block switch cost, and if it |
||||
reaches block switch cost, it means that when we trace back from the last |
||||
position, we need to switch here. */ |
||||
for i = 0; i < length; i++ { |
||||
var byte_ix uint = i |
||||
var ix uint = byte_ix * bitmaplen |
||||
var insert_cost_ix uint = uint(data[byte_ix]) * num_histograms |
||||
var min_cost float64 = 1e99 |
||||
var block_switch_cost float64 = block_switch_bitcost |
||||
var k uint |
||||
for k = 0; k < num_histograms; k++ { |
||||
/* We are coding the symbol in data[byte_ix] with entropy code k. */ |
||||
cost[k] += insert_cost[insert_cost_ix+k] |
||||
|
||||
if cost[k] < min_cost { |
||||
min_cost = cost[k] |
||||
block_id[byte_ix] = byte(k) |
||||
} |
||||
} |
||||
|
||||
/* More blocks for the beginning. */ |
||||
if byte_ix < 2000 { |
||||
block_switch_cost *= 0.77 + 0.07*float64(byte_ix)/2000 |
||||
} |
||||
|
||||
for k = 0; k < num_histograms; k++ { |
||||
cost[k] -= min_cost |
||||
if cost[k] >= block_switch_cost { |
||||
var mask byte = byte(1 << (k & 7)) |
||||
cost[k] = block_switch_cost |
||||
assert(k>>3 < bitmaplen) |
||||
switch_signal[ix+(k>>3)] |= mask |
||||
/* Trace back from the last position and switch at the marked places. */ |
||||
} |
||||
} |
||||
} |
||||
{ |
||||
var byte_ix uint = length - 1 |
||||
var ix uint = byte_ix * bitmaplen |
||||
var cur_id byte = block_id[byte_ix] |
||||
for byte_ix > 0 { |
||||
var mask byte = byte(1 << (cur_id & 7)) |
||||
assert(uint(cur_id)>>3 < bitmaplen) |
||||
byte_ix-- |
||||
ix -= bitmaplen |
||||
if switch_signal[ix+uint(cur_id>>3)]&mask != 0 { |
||||
if cur_id != block_id[byte_ix] { |
||||
cur_id = block_id[byte_ix] |
||||
num_blocks++ |
||||
} |
||||
} |
||||
|
||||
block_id[byte_ix] = cur_id |
||||
} |
||||
} |
||||
|
||||
return num_blocks |
||||
} |
||||
|
||||
var remapBlockIdsLiteral_kInvalidId uint16 = 256 |
||||
|
||||
func remapBlockIdsLiteral(block_ids []byte, length uint, new_id []uint16, num_histograms uint) uint { |
||||
var next_id uint16 = 0 |
||||
var i uint |
||||
for i = 0; i < num_histograms; i++ { |
||||
new_id[i] = remapBlockIdsLiteral_kInvalidId |
||||
} |
||||
|
||||
for i = 0; i < length; i++ { |
||||
assert(uint(block_ids[i]) < num_histograms) |
||||
if new_id[block_ids[i]] == remapBlockIdsLiteral_kInvalidId { |
||||
new_id[block_ids[i]] = next_id |
||||
next_id++ |
||||
} |
||||
} |
||||
|
||||
for i = 0; i < length; i++ { |
||||
block_ids[i] = byte(new_id[block_ids[i]]) |
||||
assert(uint(block_ids[i]) < num_histograms) |
||||
} |
||||
|
||||
assert(uint(next_id) <= num_histograms) |
||||
return uint(next_id) |
||||
} |
||||
|
||||
func buildBlockHistogramsLiteral(data []byte, length uint, block_ids []byte, num_histograms uint, histograms []histogramLiteral) { |
||||
var i uint |
||||
clearHistogramsLiteral(histograms, num_histograms) |
||||
for i = 0; i < length; i++ { |
||||
histogramAddLiteral(&histograms[block_ids[i]], uint(data[i])) |
||||
} |
||||
} |
||||
|
||||
var clusterBlocksLiteral_kInvalidIndex uint32 = math.MaxUint32 |
||||
|
||||
func clusterBlocksLiteral(data []byte, length uint, num_blocks uint, block_ids []byte, split *blockSplit) { |
||||
var histogram_symbols []uint32 = make([]uint32, num_blocks) |
||||
var block_lengths []uint32 = make([]uint32, num_blocks) |
||||
var expected_num_clusters uint = clustersPerBatch * (num_blocks + histogramsPerBatch - 1) / histogramsPerBatch |
||||
var all_histograms_size uint = 0 |
||||
var all_histograms_capacity uint = expected_num_clusters |
||||
var all_histograms []histogramLiteral = make([]histogramLiteral, all_histograms_capacity) |
||||
var cluster_size_size uint = 0 |
||||
var cluster_size_capacity uint = expected_num_clusters |
||||
var cluster_size []uint32 = make([]uint32, cluster_size_capacity) |
||||
var num_clusters uint = 0 |
||||
var histograms []histogramLiteral = make([]histogramLiteral, brotli_min_size_t(num_blocks, histogramsPerBatch)) |
||||
var max_num_pairs uint = histogramsPerBatch * histogramsPerBatch / 2 |
||||
var pairs_capacity uint = max_num_pairs + 1 |
||||
var pairs []histogramPair = make([]histogramPair, pairs_capacity) |
||||
var pos uint = 0 |
||||
var clusters []uint32 |
||||
var num_final_clusters uint |
||||
var new_index []uint32 |
||||
var i uint |
||||
var sizes = [histogramsPerBatch]uint32{0} |
||||
var new_clusters = [histogramsPerBatch]uint32{0} |
||||
var symbols = [histogramsPerBatch]uint32{0} |
||||
var remap = [histogramsPerBatch]uint32{0} |
||||
|
||||
for i := 0; i < int(num_blocks); i++ { |
||||
block_lengths[i] = 0 |
||||
} |
||||
{ |
||||
var block_idx uint = 0 |
||||
for i = 0; i < length; i++ { |
||||
assert(block_idx < num_blocks) |
||||
block_lengths[block_idx]++ |
||||
if i+1 == length || block_ids[i] != block_ids[i+1] { |
||||
block_idx++ |
||||
} |
||||
} |
||||
|
||||
assert(block_idx == num_blocks) |
||||
} |
||||
|
||||
for i = 0; i < num_blocks; i += histogramsPerBatch { |
||||
var num_to_combine uint = brotli_min_size_t(num_blocks-i, histogramsPerBatch) |
||||
var num_new_clusters uint |
||||
var j uint |
||||
for j = 0; j < num_to_combine; j++ { |
||||
var k uint |
||||
histogramClearLiteral(&histograms[j]) |
||||
for k = 0; uint32(k) < block_lengths[i+j]; k++ { |
||||
histogramAddLiteral(&histograms[j], uint(data[pos])) |
||||
pos++ |
||||
} |
||||
|
||||
histograms[j].bit_cost_ = populationCostLiteral(&histograms[j]) |
||||
new_clusters[j] = uint32(j) |
||||
symbols[j] = uint32(j) |
||||
sizes[j] = 1 |
||||
} |
||||
|
||||
num_new_clusters = histogramCombineLiteral(histograms, sizes[:], symbols[:], new_clusters[:], []histogramPair(pairs), num_to_combine, num_to_combine, histogramsPerBatch, max_num_pairs) |
||||
if all_histograms_capacity < (all_histograms_size + num_new_clusters) { |
||||
var _new_size uint |
||||
if all_histograms_capacity == 0 { |
||||
_new_size = all_histograms_size + num_new_clusters |
||||
} else { |
||||
_new_size = all_histograms_capacity |
||||
} |
||||
var new_array []histogramLiteral |
||||
for _new_size < (all_histograms_size + num_new_clusters) { |
||||
_new_size *= 2 |
||||
} |
||||
new_array = make([]histogramLiteral, _new_size) |
||||
if all_histograms_capacity != 0 { |
||||
copy(new_array, all_histograms[:all_histograms_capacity]) |
||||
} |
||||
|
||||
all_histograms = new_array |
||||
all_histograms_capacity = _new_size |
||||
} |
||||
|
||||
brotli_ensure_capacity_uint32_t(&cluster_size, &cluster_size_capacity, cluster_size_size+num_new_clusters) |
||||
for j = 0; j < num_new_clusters; j++ { |
||||
all_histograms[all_histograms_size] = histograms[new_clusters[j]] |
||||
all_histograms_size++ |
||||
cluster_size[cluster_size_size] = sizes[new_clusters[j]] |
||||
cluster_size_size++ |
||||
remap[new_clusters[j]] = uint32(j) |
||||
} |
||||
|
||||
for j = 0; j < num_to_combine; j++ { |
||||
histogram_symbols[i+j] = uint32(num_clusters) + remap[symbols[j]] |
||||
} |
||||
|
||||
num_clusters += num_new_clusters |
||||
assert(num_clusters == cluster_size_size) |
||||
assert(num_clusters == all_histograms_size) |
||||
} |
||||
|
||||
histograms = nil |
||||
|
||||
max_num_pairs = brotli_min_size_t(64*num_clusters, (num_clusters/2)*num_clusters) |
||||
if pairs_capacity < max_num_pairs+1 { |
||||
pairs = nil |
||||
pairs = make([]histogramPair, (max_num_pairs + 1)) |
||||
} |
||||
|
||||
clusters = make([]uint32, num_clusters) |
||||
for i = 0; i < num_clusters; i++ { |
||||
clusters[i] = uint32(i) |
||||
} |
||||
|
||||
num_final_clusters = histogramCombineLiteral(all_histograms, cluster_size, histogram_symbols, clusters, pairs, num_clusters, num_blocks, maxNumberOfBlockTypes, max_num_pairs) |
||||
pairs = nil |
||||
cluster_size = nil |
||||
|
||||
new_index = make([]uint32, num_clusters) |
||||
for i = 0; i < num_clusters; i++ { |
||||
new_index[i] = clusterBlocksLiteral_kInvalidIndex |
||||
} |
||||
pos = 0 |
||||
{ |
||||
var next_index uint32 = 0 |
||||
for i = 0; i < num_blocks; i++ { |
||||
var histo histogramLiteral |
||||
var j uint |
||||
var best_out uint32 |
||||
var best_bits float64 |
||||
histogramClearLiteral(&histo) |
||||
for j = 0; uint32(j) < block_lengths[i]; j++ { |
||||
histogramAddLiteral(&histo, uint(data[pos])) |
||||
pos++ |
||||
} |
||||
|
||||
if i == 0 { |
||||
best_out = histogram_symbols[0] |
||||
} else { |
||||
best_out = histogram_symbols[i-1] |
||||
} |
||||
best_bits = histogramBitCostDistanceLiteral(&histo, &all_histograms[best_out]) |
||||
for j = 0; j < num_final_clusters; j++ { |
||||
var cur_bits float64 = histogramBitCostDistanceLiteral(&histo, &all_histograms[clusters[j]]) |
||||
if cur_bits < best_bits { |
||||
best_bits = cur_bits |
||||
best_out = clusters[j] |
||||
} |
||||
} |
||||
|
||||
histogram_symbols[i] = best_out |
||||
if new_index[best_out] == clusterBlocksLiteral_kInvalidIndex { |
||||
new_index[best_out] = next_index |
||||
next_index++ |
||||
} |
||||
} |
||||
} |
||||
|
||||
clusters = nil |
||||
all_histograms = nil |
||||
brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, num_blocks) |
||||
brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, num_blocks) |
||||
{ |
||||
var cur_length uint32 = 0 |
||||
var block_idx uint = 0 |
||||
var max_type byte = 0 |
||||
for i = 0; i < num_blocks; i++ { |
||||
cur_length += block_lengths[i] |
||||
if i+1 == num_blocks || histogram_symbols[i] != histogram_symbols[i+1] { |
||||
var id byte = byte(new_index[histogram_symbols[i]]) |
||||
split.types[block_idx] = id |
||||
split.lengths[block_idx] = cur_length |
||||
max_type = brotli_max_uint8_t(max_type, id) |
||||
cur_length = 0 |
||||
block_idx++ |
||||
} |
||||
} |
||||
|
||||
split.num_blocks = block_idx |
||||
split.num_types = uint(max_type) + 1 |
||||
} |
||||
|
||||
new_index = nil |
||||
block_lengths = nil |
||||
histogram_symbols = nil |
||||
} |
||||
|
||||
func splitByteVectorLiteral(data []byte, length uint, literals_per_histogram uint, max_histograms uint, sampling_stride_length uint, block_switch_cost float64, params *encoderParams, split *blockSplit) { |
||||
var data_size uint = histogramDataSizeLiteral() |
||||
var num_histograms uint = length/literals_per_histogram + 1 |
||||
var histograms []histogramLiteral |
||||
if num_histograms > max_histograms { |
||||
num_histograms = max_histograms |
||||
} |
||||
|
||||
if length == 0 { |
||||
split.num_types = 1 |
||||
return |
||||
} else if length < kMinLengthForBlockSplitting { |
||||
brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, split.num_blocks+1) |
||||
brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, split.num_blocks+1) |
||||
split.num_types = 1 |
||||
split.types[split.num_blocks] = 0 |
||||
split.lengths[split.num_blocks] = uint32(length) |
||||
split.num_blocks++ |
||||
return |
||||
} |
||||
|
||||
histograms = make([]histogramLiteral, num_histograms) |
||||
|
||||
/* Find good entropy codes. */ |
||||
initialEntropyCodesLiteral(data, length, sampling_stride_length, num_histograms, histograms) |
||||
|
||||
refineEntropyCodesLiteral(data, length, sampling_stride_length, num_histograms, histograms) |
||||
{ |
||||
var block_ids []byte = make([]byte, length) |
||||
var num_blocks uint = 0 |
||||
var bitmaplen uint = (num_histograms + 7) >> 3 |
||||
var insert_cost []float64 = make([]float64, (data_size * num_histograms)) |
||||
var cost []float64 = make([]float64, num_histograms) |
||||
var switch_signal []byte = make([]byte, (length * bitmaplen)) |
||||
var new_id []uint16 = make([]uint16, num_histograms) |
||||
var iters uint |
||||
if params.quality < hqZopflificationQuality { |
||||
iters = 3 |
||||
} else { |
||||
iters = 10 |
||||
} |
||||
/* Find a good path through literals with the good entropy codes. */ |
||||
|
||||
var i uint |
||||
for i = 0; i < iters; i++ { |
||||
num_blocks = findBlocksLiteral(data, length, block_switch_cost, num_histograms, histograms, insert_cost, cost, switch_signal, block_ids) |
||||
num_histograms = remapBlockIdsLiteral(block_ids, length, new_id, num_histograms) |
||||
buildBlockHistogramsLiteral(data, length, block_ids, num_histograms, histograms) |
||||
} |
||||
|
||||
insert_cost = nil |
||||
cost = nil |
||||
switch_signal = nil |
||||
new_id = nil |
||||
histograms = nil |
||||
clusterBlocksLiteral(data, length, num_blocks, block_ids, split) |
||||
block_ids = nil |
||||
} |
||||
} |
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,30 @@ |
||||
package brotli |
||||
|
||||
/* Copyright 2013 Google Inc. All Rights Reserved. |
||||
|
||||
Distributed under MIT license. |
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/ |
||||
|
||||
/* Functions for clustering similar histograms together. */ |
||||
|
||||
type histogramPair struct { |
||||
idx1 uint32 |
||||
idx2 uint32 |
||||
cost_combo float64 |
||||
cost_diff float64 |
||||
} |
||||
|
||||
func histogramPairIsLess(p1 *histogramPair, p2 *histogramPair) bool { |
||||
if p1.cost_diff != p2.cost_diff { |
||||
return p1.cost_diff > p2.cost_diff |
||||
} |
||||
|
||||
return (p1.idx2 - p1.idx1) > (p2.idx2 - p2.idx1) |
||||
} |
||||
|
||||
/* Returns entropy reduction of the context map when we combine two clusters. */ |
||||
func clusterCostDiff(size_a uint, size_b uint) float64 { |
||||
var size_c uint = size_a + size_b |
||||
return float64(size_a)*fastLog2(size_a) + float64(size_b)*fastLog2(size_b) - float64(size_c)*fastLog2(size_c) |
||||
} |
||||
@ -0,0 +1,326 @@ |
||||
package brotli |
||||
|
||||
import "math" |
||||
|
||||
/* Copyright 2013 Google Inc. All Rights Reserved. |
||||
|
||||
Distributed under MIT license. |
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/ |
||||
|
||||
/* Computes the bit cost reduction by combining out[idx1] and out[idx2] and if |
||||
it is below a threshold, stores the pair (idx1, idx2) in the *pairs queue. */ |
||||
func compareAndPushToQueueCommand(out []histogramCommand, cluster_size []uint32, idx1 uint32, idx2 uint32, max_num_pairs uint, pairs []histogramPair, num_pairs *uint) { |
||||
var is_good_pair bool = false |
||||
var p histogramPair |
||||
p.idx2 = 0 |
||||
p.idx1 = p.idx2 |
||||
p.cost_combo = 0 |
||||
p.cost_diff = p.cost_combo |
||||
if idx1 == idx2 { |
||||
return |
||||
} |
||||
|
||||
if idx2 < idx1 { |
||||
var t uint32 = idx2 |
||||
idx2 = idx1 |
||||
idx1 = t |
||||
} |
||||
|
||||
p.idx1 = idx1 |
||||
p.idx2 = idx2 |
||||
p.cost_diff = 0.5 * clusterCostDiff(uint(cluster_size[idx1]), uint(cluster_size[idx2])) |
||||
p.cost_diff -= out[idx1].bit_cost_ |
||||
p.cost_diff -= out[idx2].bit_cost_ |
||||
|
||||
if out[idx1].total_count_ == 0 { |
||||
p.cost_combo = out[idx2].bit_cost_ |
||||
is_good_pair = true |
||||
} else if out[idx2].total_count_ == 0 { |
||||
p.cost_combo = out[idx1].bit_cost_ |
||||
is_good_pair = true |
||||
} else { |
||||
var threshold float64 |
||||
if *num_pairs == 0 { |
||||
threshold = 1e99 |
||||
} else { |
||||
threshold = brotli_max_double(0.0, pairs[0].cost_diff) |
||||
} |
||||
var combo histogramCommand = out[idx1] |
||||
var cost_combo float64 |
||||
histogramAddHistogramCommand(&combo, &out[idx2]) |
||||
cost_combo = populationCostCommand(&combo) |
||||
if cost_combo < threshold-p.cost_diff { |
||||
p.cost_combo = cost_combo |
||||
is_good_pair = true |
||||
} |
||||
} |
||||
|
||||
if is_good_pair { |
||||
p.cost_diff += p.cost_combo |
||||
if *num_pairs > 0 && histogramPairIsLess(&pairs[0], &p) { |
||||
/* Replace the top of the queue if needed. */ |
||||
if *num_pairs < max_num_pairs { |
||||
pairs[*num_pairs] = pairs[0] |
||||
(*num_pairs)++ |
||||
} |
||||
|
||||
pairs[0] = p |
||||
} else if *num_pairs < max_num_pairs { |
||||
pairs[*num_pairs] = p |
||||
(*num_pairs)++ |
||||
} |
||||
} |
||||
} |
||||
|
||||
func histogramCombineCommand(out []histogramCommand, cluster_size []uint32, symbols []uint32, clusters []uint32, pairs []histogramPair, num_clusters uint, symbols_size uint, max_clusters uint, max_num_pairs uint) uint { |
||||
var cost_diff_threshold float64 = 0.0 |
||||
var min_cluster_size uint = 1 |
||||
var num_pairs uint = 0 |
||||
{ |
||||
/* We maintain a vector of histogram pairs, with the property that the pair |
||||
with the maximum bit cost reduction is the first. */ |
||||
var idx1 uint |
||||
for idx1 = 0; idx1 < num_clusters; idx1++ { |
||||
var idx2 uint |
||||
for idx2 = idx1 + 1; idx2 < num_clusters; idx2++ { |
||||
compareAndPushToQueueCommand(out, cluster_size, clusters[idx1], clusters[idx2], max_num_pairs, pairs[0:], &num_pairs) |
||||
} |
||||
} |
||||
} |
||||
|
||||
for num_clusters > min_cluster_size { |
||||
var best_idx1 uint32 |
||||
var best_idx2 uint32 |
||||
var i uint |
||||
if pairs[0].cost_diff >= cost_diff_threshold { |
||||
cost_diff_threshold = 1e99 |
||||
min_cluster_size = max_clusters |
||||
continue |
||||
} |
||||
|
||||
/* Take the best pair from the top of heap. */ |
||||
best_idx1 = pairs[0].idx1 |
||||
|
||||
best_idx2 = pairs[0].idx2 |
||||
histogramAddHistogramCommand(&out[best_idx1], &out[best_idx2]) |
||||
out[best_idx1].bit_cost_ = pairs[0].cost_combo |
||||
cluster_size[best_idx1] += cluster_size[best_idx2] |
||||
for i = 0; i < symbols_size; i++ { |
||||
if symbols[i] == best_idx2 { |
||||
symbols[i] = best_idx1 |
||||
} |
||||
} |
||||
|
||||
for i = 0; i < num_clusters; i++ { |
||||
if clusters[i] == best_idx2 { |
||||
copy(clusters[i:], clusters[i+1:][:num_clusters-i-1]) |
||||
break |
||||
} |
||||
} |
||||
|
||||
num_clusters-- |
||||
{ |
||||
/* Remove pairs intersecting the just combined best pair. */ |
||||
var copy_to_idx uint = 0 |
||||
for i = 0; i < num_pairs; i++ { |
||||
var p *histogramPair = &pairs[i] |
||||
if p.idx1 == best_idx1 || p.idx2 == best_idx1 || p.idx1 == best_idx2 || p.idx2 == best_idx2 { |
||||
/* Remove invalid pair from the queue. */ |
||||
continue |
||||
} |
||||
|
||||
if histogramPairIsLess(&pairs[0], p) { |
||||
/* Replace the top of the queue if needed. */ |
||||
var front histogramPair = pairs[0] |
||||
pairs[0] = *p |
||||
pairs[copy_to_idx] = front |
||||
} else { |
||||
pairs[copy_to_idx] = *p |
||||
} |
||||
|
||||
copy_to_idx++ |
||||
} |
||||
|
||||
num_pairs = copy_to_idx |
||||
} |
||||
|
||||
/* Push new pairs formed with the combined histogram to the heap. */ |
||||
for i = 0; i < num_clusters; i++ { |
||||
compareAndPushToQueueCommand(out, cluster_size, best_idx1, clusters[i], max_num_pairs, pairs[0:], &num_pairs) |
||||
} |
||||
} |
||||
|
||||
return num_clusters |
||||
} |
||||
|
||||
/* What is the bit cost of moving histogram from cur_symbol to candidate. */ |
||||
func histogramBitCostDistanceCommand(histogram *histogramCommand, candidate *histogramCommand) float64 { |
||||
if histogram.total_count_ == 0 { |
||||
return 0.0 |
||||
} else { |
||||
var tmp histogramCommand = *histogram |
||||
histogramAddHistogramCommand(&tmp, candidate) |
||||
return populationCostCommand(&tmp) - candidate.bit_cost_ |
||||
} |
||||
} |
||||
|
||||
/* Find the best 'out' histogram for each of the 'in' histograms. |
||||
When called, clusters[0..num_clusters) contains the unique values from |
||||
symbols[0..in_size), but this property is not preserved in this function. |
||||
Note: we assume that out[]->bit_cost_ is already up-to-date. */ |
||||
func histogramRemapCommand(in []histogramCommand, in_size uint, clusters []uint32, num_clusters uint, out []histogramCommand, symbols []uint32) { |
||||
var i uint |
||||
for i = 0; i < in_size; i++ { |
||||
var best_out uint32 |
||||
if i == 0 { |
||||
best_out = symbols[0] |
||||
} else { |
||||
best_out = symbols[i-1] |
||||
} |
||||
var best_bits float64 = histogramBitCostDistanceCommand(&in[i], &out[best_out]) |
||||
var j uint |
||||
for j = 0; j < num_clusters; j++ { |
||||
var cur_bits float64 = histogramBitCostDistanceCommand(&in[i], &out[clusters[j]]) |
||||
if cur_bits < best_bits { |
||||
best_bits = cur_bits |
||||
best_out = clusters[j] |
||||
} |
||||
} |
||||
|
||||
symbols[i] = best_out |
||||
} |
||||
|
||||
/* Recompute each out based on raw and symbols. */ |
||||
for i = 0; i < num_clusters; i++ { |
||||
histogramClearCommand(&out[clusters[i]]) |
||||
} |
||||
|
||||
for i = 0; i < in_size; i++ { |
||||
histogramAddHistogramCommand(&out[symbols[i]], &in[i]) |
||||
} |
||||
} |
||||
|
||||
/* Reorders elements of the out[0..length) array and changes values in |
||||
symbols[0..length) array in the following way: |
||||
* when called, symbols[] contains indexes into out[], and has N unique |
||||
values (possibly N < length) |
||||
* on return, symbols'[i] = f(symbols[i]) and |
||||
out'[symbols'[i]] = out[symbols[i]], for each 0 <= i < length, |
||||
where f is a bijection between the range of symbols[] and [0..N), and |
||||
the first occurrences of values in symbols'[i] come in consecutive |
||||
increasing order. |
||||
Returns N, the number of unique values in symbols[]. */ |
||||
|
||||
var histogramReindexCommand_kInvalidIndex uint32 = math.MaxUint32 |
||||
|
||||
func histogramReindexCommand(out []histogramCommand, symbols []uint32, length uint) uint { |
||||
var new_index []uint32 = make([]uint32, length) |
||||
var next_index uint32 |
||||
var tmp []histogramCommand |
||||
var i uint |
||||
for i = 0; i < length; i++ { |
||||
new_index[i] = histogramReindexCommand_kInvalidIndex |
||||
} |
||||
|
||||
next_index = 0 |
||||
for i = 0; i < length; i++ { |
||||
if new_index[symbols[i]] == histogramReindexCommand_kInvalidIndex { |
||||
new_index[symbols[i]] = next_index |
||||
next_index++ |
||||
} |
||||
} |
||||
|
||||
/* TODO: by using idea of "cycle-sort" we can avoid allocation of |
||||
tmp and reduce the number of copying by the factor of 2. */ |
||||
tmp = make([]histogramCommand, next_index) |
||||
|
||||
next_index = 0 |
||||
for i = 0; i < length; i++ { |
||||
if new_index[symbols[i]] == next_index { |
||||
tmp[next_index] = out[symbols[i]] |
||||
next_index++ |
||||
} |
||||
|
||||
symbols[i] = new_index[symbols[i]] |
||||
} |
||||
|
||||
new_index = nil |
||||
for i = 0; uint32(i) < next_index; i++ { |
||||
out[i] = tmp[i] |
||||
} |
||||
|
||||
tmp = nil |
||||
return uint(next_index) |
||||
} |
||||
|
||||
func clusterHistogramsCommand(in []histogramCommand, in_size uint, max_histograms uint, out []histogramCommand, out_size *uint, histogram_symbols []uint32) { |
||||
var cluster_size []uint32 = make([]uint32, in_size) |
||||
var clusters []uint32 = make([]uint32, in_size) |
||||
var num_clusters uint = 0 |
||||
var max_input_histograms uint = 64 |
||||
var pairs_capacity uint = max_input_histograms * max_input_histograms / 2 |
||||
var pairs []histogramPair = make([]histogramPair, (pairs_capacity + 1)) |
||||
var i uint |
||||
|
||||
/* For the first pass of clustering, we allow all pairs. */ |
||||
for i = 0; i < in_size; i++ { |
||||
cluster_size[i] = 1 |
||||
} |
||||
|
||||
for i = 0; i < in_size; i++ { |
||||
out[i] = in[i] |
||||
out[i].bit_cost_ = populationCostCommand(&in[i]) |
||||
histogram_symbols[i] = uint32(i) |
||||
} |
||||
|
||||
for i = 0; i < in_size; i += max_input_histograms { |
||||
var num_to_combine uint = brotli_min_size_t(in_size-i, max_input_histograms) |
||||
var num_new_clusters uint |
||||
var j uint |
||||
for j = 0; j < num_to_combine; j++ { |
||||
clusters[num_clusters+j] = uint32(i + j) |
||||
} |
||||
|
||||
num_new_clusters = histogramCombineCommand(out, cluster_size, histogram_symbols[i:], clusters[num_clusters:], pairs, num_to_combine, num_to_combine, max_histograms, pairs_capacity) |
||||
num_clusters += num_new_clusters |
||||
} |
||||
{ |
||||
/* For the second pass, we limit the total number of histogram pairs. |
||||
After this limit is reached, we only keep searching for the best pair. */ |
||||
var max_num_pairs uint = brotli_min_size_t(64*num_clusters, (num_clusters/2)*num_clusters) |
||||
if pairs_capacity < (max_num_pairs + 1) { |
||||
var _new_size uint |
||||
if pairs_capacity == 0 { |
||||
_new_size = max_num_pairs + 1 |
||||
} else { |
||||
_new_size = pairs_capacity |
||||
} |
||||
var new_array []histogramPair |
||||
for _new_size < (max_num_pairs + 1) { |
||||
_new_size *= 2 |
||||
} |
||||
new_array = make([]histogramPair, _new_size) |
||||
if pairs_capacity != 0 { |
||||
copy(new_array, pairs[:pairs_capacity]) |
||||
} |
||||
|
||||
pairs = new_array |
||||
pairs_capacity = _new_size |
||||
} |
||||
|
||||
/* Collapse similar histograms. */ |
||||
num_clusters = histogramCombineCommand(out, cluster_size, histogram_symbols, clusters, pairs, num_clusters, in_size, max_histograms, max_num_pairs) |
||||
} |
||||
|
||||
pairs = nil |
||||
cluster_size = nil |
||||
|
||||
/* Find the optimal map from original histograms to the final ones. */ |
||||
histogramRemapCommand(in, in_size, clusters, num_clusters, out, histogram_symbols) |
||||
|
||||
clusters = nil |
||||
|
||||
/* Convert the context map to a canonical form. */ |
||||
*out_size = histogramReindexCommand(out, histogram_symbols, in_size) |
||||
} |
||||
@ -0,0 +1,326 @@ |
||||
package brotli |
||||
|
||||
import "math" |
||||
|
||||
/* Copyright 2013 Google Inc. All Rights Reserved. |
||||
|
||||
Distributed under MIT license. |
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/ |
||||
|
||||
/* Computes the bit cost reduction by combining out[idx1] and out[idx2] and if |
||||
it is below a threshold, stores the pair (idx1, idx2) in the *pairs queue. */ |
||||
func compareAndPushToQueueDistance(out []histogramDistance, cluster_size []uint32, idx1 uint32, idx2 uint32, max_num_pairs uint, pairs []histogramPair, num_pairs *uint) { |
||||
var is_good_pair bool = false |
||||
var p histogramPair |
||||
p.idx2 = 0 |
||||
p.idx1 = p.idx2 |
||||
p.cost_combo = 0 |
||||
p.cost_diff = p.cost_combo |
||||
if idx1 == idx2 { |
||||
return |
||||
} |
||||
|
||||
if idx2 < idx1 { |
||||
var t uint32 = idx2 |
||||
idx2 = idx1 |
||||
idx1 = t |
||||
} |
||||
|
||||
p.idx1 = idx1 |
||||
p.idx2 = idx2 |
||||
p.cost_diff = 0.5 * clusterCostDiff(uint(cluster_size[idx1]), uint(cluster_size[idx2])) |
||||
p.cost_diff -= out[idx1].bit_cost_ |
||||
p.cost_diff -= out[idx2].bit_cost_ |
||||
|
||||
if out[idx1].total_count_ == 0 { |
||||
p.cost_combo = out[idx2].bit_cost_ |
||||
is_good_pair = true |
||||
} else if out[idx2].total_count_ == 0 { |
||||
p.cost_combo = out[idx1].bit_cost_ |
||||
is_good_pair = true |
||||
} else { |
||||
var threshold float64 |
||||
if *num_pairs == 0 { |
||||
threshold = 1e99 |
||||
} else { |
||||
threshold = brotli_max_double(0.0, pairs[0].cost_diff) |
||||
} |
||||
var combo histogramDistance = out[idx1] |
||||
var cost_combo float64 |
||||
histogramAddHistogramDistance(&combo, &out[idx2]) |
||||
cost_combo = populationCostDistance(&combo) |
||||
if cost_combo < threshold-p.cost_diff { |
||||
p.cost_combo = cost_combo |
||||
is_good_pair = true |
||||
} |
||||
} |
||||
|
||||
if is_good_pair { |
||||
p.cost_diff += p.cost_combo |
||||
if *num_pairs > 0 && histogramPairIsLess(&pairs[0], &p) { |
||||
/* Replace the top of the queue if needed. */ |
||||
if *num_pairs < max_num_pairs { |
||||
pairs[*num_pairs] = pairs[0] |
||||
(*num_pairs)++ |
||||
} |
||||
|
||||
pairs[0] = p |
||||
} else if *num_pairs < max_num_pairs { |
||||
pairs[*num_pairs] = p |
||||
(*num_pairs)++ |
||||
} |
||||
} |
||||
} |
||||
|
||||
func histogramCombineDistance(out []histogramDistance, cluster_size []uint32, symbols []uint32, clusters []uint32, pairs []histogramPair, num_clusters uint, symbols_size uint, max_clusters uint, max_num_pairs uint) uint { |
||||
var cost_diff_threshold float64 = 0.0 |
||||
var min_cluster_size uint = 1 |
||||
var num_pairs uint = 0 |
||||
{ |
||||
/* We maintain a vector of histogram pairs, with the property that the pair |
||||
with the maximum bit cost reduction is the first. */ |
||||
var idx1 uint |
||||
for idx1 = 0; idx1 < num_clusters; idx1++ { |
||||
var idx2 uint |
||||
for idx2 = idx1 + 1; idx2 < num_clusters; idx2++ { |
||||
compareAndPushToQueueDistance(out, cluster_size, clusters[idx1], clusters[idx2], max_num_pairs, pairs[0:], &num_pairs) |
||||
} |
||||
} |
||||
} |
||||
|
||||
for num_clusters > min_cluster_size { |
||||
var best_idx1 uint32 |
||||
var best_idx2 uint32 |
||||
var i uint |
||||
if pairs[0].cost_diff >= cost_diff_threshold { |
||||
cost_diff_threshold = 1e99 |
||||
min_cluster_size = max_clusters |
||||
continue |
||||
} |
||||
|
||||
/* Take the best pair from the top of heap. */ |
||||
best_idx1 = pairs[0].idx1 |
||||
|
||||
best_idx2 = pairs[0].idx2 |
||||
histogramAddHistogramDistance(&out[best_idx1], &out[best_idx2]) |
||||
out[best_idx1].bit_cost_ = pairs[0].cost_combo |
||||
cluster_size[best_idx1] += cluster_size[best_idx2] |
||||
for i = 0; i < symbols_size; i++ { |
||||
if symbols[i] == best_idx2 { |
||||
symbols[i] = best_idx1 |
||||
} |
||||
} |
||||
|
||||
for i = 0; i < num_clusters; i++ { |
||||
if clusters[i] == best_idx2 { |
||||
copy(clusters[i:], clusters[i+1:][:num_clusters-i-1]) |
||||
break |
||||
} |
||||
} |
||||
|
||||
num_clusters-- |
||||
{ |
||||
/* Remove pairs intersecting the just combined best pair. */ |
||||
var copy_to_idx uint = 0 |
||||
for i = 0; i < num_pairs; i++ { |
||||
var p *histogramPair = &pairs[i] |
||||
if p.idx1 == best_idx1 || p.idx2 == best_idx1 || p.idx1 == best_idx2 || p.idx2 == best_idx2 { |
||||
/* Remove invalid pair from the queue. */ |
||||
continue |
||||
} |
||||
|
||||
if histogramPairIsLess(&pairs[0], p) { |
||||
/* Replace the top of the queue if needed. */ |
||||
var front histogramPair = pairs[0] |
||||
pairs[0] = *p |
||||
pairs[copy_to_idx] = front |
||||
} else { |
||||
pairs[copy_to_idx] = *p |
||||
} |
||||
|
||||
copy_to_idx++ |
||||
} |
||||
|
||||
num_pairs = copy_to_idx |
||||
} |
||||
|
||||
/* Push new pairs formed with the combined histogram to the heap. */ |
||||
for i = 0; i < num_clusters; i++ { |
||||
compareAndPushToQueueDistance(out, cluster_size, best_idx1, clusters[i], max_num_pairs, pairs[0:], &num_pairs) |
||||
} |
||||
} |
||||
|
||||
return num_clusters |
||||
} |
||||
|
||||
/* What is the bit cost of moving histogram from cur_symbol to candidate. */ |
||||
func histogramBitCostDistanceDistance(histogram *histogramDistance, candidate *histogramDistance) float64 { |
||||
if histogram.total_count_ == 0 { |
||||
return 0.0 |
||||
} else { |
||||
var tmp histogramDistance = *histogram |
||||
histogramAddHistogramDistance(&tmp, candidate) |
||||
return populationCostDistance(&tmp) - candidate.bit_cost_ |
||||
} |
||||
} |
||||
|
||||
/* Find the best 'out' histogram for each of the 'in' histograms. |
||||
When called, clusters[0..num_clusters) contains the unique values from |
||||
symbols[0..in_size), but this property is not preserved in this function. |
||||
Note: we assume that out[]->bit_cost_ is already up-to-date. */ |
||||
func histogramRemapDistance(in []histogramDistance, in_size uint, clusters []uint32, num_clusters uint, out []histogramDistance, symbols []uint32) { |
||||
var i uint |
||||
for i = 0; i < in_size; i++ { |
||||
var best_out uint32 |
||||
if i == 0 { |
||||
best_out = symbols[0] |
||||
} else { |
||||
best_out = symbols[i-1] |
||||
} |
||||
var best_bits float64 = histogramBitCostDistanceDistance(&in[i], &out[best_out]) |
||||
var j uint |
||||
for j = 0; j < num_clusters; j++ { |
||||
var cur_bits float64 = histogramBitCostDistanceDistance(&in[i], &out[clusters[j]]) |
||||
if cur_bits < best_bits { |
||||
best_bits = cur_bits |
||||
best_out = clusters[j] |
||||
} |
||||
} |
||||
|
||||
symbols[i] = best_out |
||||
} |
||||
|
||||
/* Recompute each out based on raw and symbols. */ |
||||
for i = 0; i < num_clusters; i++ { |
||||
histogramClearDistance(&out[clusters[i]]) |
||||
} |
||||
|
||||
for i = 0; i < in_size; i++ { |
||||
histogramAddHistogramDistance(&out[symbols[i]], &in[i]) |
||||
} |
||||
} |
||||
|
||||
/* Reorders elements of the out[0..length) array and changes values in |
||||
symbols[0..length) array in the following way: |
||||
* when called, symbols[] contains indexes into out[], and has N unique |
||||
values (possibly N < length) |
||||
* on return, symbols'[i] = f(symbols[i]) and |
||||
out'[symbols'[i]] = out[symbols[i]], for each 0 <= i < length, |
||||
where f is a bijection between the range of symbols[] and [0..N), and |
||||
the first occurrences of values in symbols'[i] come in consecutive |
||||
increasing order. |
||||
Returns N, the number of unique values in symbols[]. */ |
||||
|
||||
var histogramReindexDistance_kInvalidIndex uint32 = math.MaxUint32 |
||||
|
||||
func histogramReindexDistance(out []histogramDistance, symbols []uint32, length uint) uint { |
||||
var new_index []uint32 = make([]uint32, length) |
||||
var next_index uint32 |
||||
var tmp []histogramDistance |
||||
var i uint |
||||
for i = 0; i < length; i++ { |
||||
new_index[i] = histogramReindexDistance_kInvalidIndex |
||||
} |
||||
|
||||
next_index = 0 |
||||
for i = 0; i < length; i++ { |
||||
if new_index[symbols[i]] == histogramReindexDistance_kInvalidIndex { |
||||
new_index[symbols[i]] = next_index |
||||
next_index++ |
||||
} |
||||
} |
||||
|
||||
/* TODO: by using idea of "cycle-sort" we can avoid allocation of |
||||
tmp and reduce the number of copying by the factor of 2. */ |
||||
tmp = make([]histogramDistance, next_index) |
||||
|
||||
next_index = 0 |
||||
for i = 0; i < length; i++ { |
||||
if new_index[symbols[i]] == next_index { |
||||
tmp[next_index] = out[symbols[i]] |
||||
next_index++ |
||||
} |
||||
|
||||
symbols[i] = new_index[symbols[i]] |
||||
} |
||||
|
||||
new_index = nil |
||||
for i = 0; uint32(i) < next_index; i++ { |
||||
out[i] = tmp[i] |
||||
} |
||||
|
||||
tmp = nil |
||||
return uint(next_index) |
||||
} |
||||
|
||||
func clusterHistogramsDistance(in []histogramDistance, in_size uint, max_histograms uint, out []histogramDistance, out_size *uint, histogram_symbols []uint32) { |
||||
var cluster_size []uint32 = make([]uint32, in_size) |
||||
var clusters []uint32 = make([]uint32, in_size) |
||||
var num_clusters uint = 0 |
||||
var max_input_histograms uint = 64 |
||||
var pairs_capacity uint = max_input_histograms * max_input_histograms / 2 |
||||
var pairs []histogramPair = make([]histogramPair, (pairs_capacity + 1)) |
||||
var i uint |
||||
|
||||
/* For the first pass of clustering, we allow all pairs. */ |
||||
for i = 0; i < in_size; i++ { |
||||
cluster_size[i] = 1 |
||||
} |
||||
|
||||
for i = 0; i < in_size; i++ { |
||||
out[i] = in[i] |
||||
out[i].bit_cost_ = populationCostDistance(&in[i]) |
||||
histogram_symbols[i] = uint32(i) |
||||
} |
||||
|
||||
for i = 0; i < in_size; i += max_input_histograms { |
||||
var num_to_combine uint = brotli_min_size_t(in_size-i, max_input_histograms) |
||||
var num_new_clusters uint |
||||
var j uint |
||||
for j = 0; j < num_to_combine; j++ { |
||||
clusters[num_clusters+j] = uint32(i + j) |
||||
} |
||||
|
||||
num_new_clusters = histogramCombineDistance(out, cluster_size, histogram_symbols[i:], clusters[num_clusters:], pairs, num_to_combine, num_to_combine, max_histograms, pairs_capacity) |
||||
num_clusters += num_new_clusters |
||||
} |
||||
{ |
||||
/* For the second pass, we limit the total number of histogram pairs. |
||||
After this limit is reached, we only keep searching for the best pair. */ |
||||
var max_num_pairs uint = brotli_min_size_t(64*num_clusters, (num_clusters/2)*num_clusters) |
||||
if pairs_capacity < (max_num_pairs + 1) { |
||||
var _new_size uint |
||||
if pairs_capacity == 0 { |
||||
_new_size = max_num_pairs + 1 |
||||
} else { |
||||
_new_size = pairs_capacity |
||||
} |
||||
var new_array []histogramPair |
||||
for _new_size < (max_num_pairs + 1) { |
||||
_new_size *= 2 |
||||
} |
||||
new_array = make([]histogramPair, _new_size) |
||||
if pairs_capacity != 0 { |
||||
copy(new_array, pairs[:pairs_capacity]) |
||||
} |
||||
|
||||
pairs = new_array |
||||
pairs_capacity = _new_size |
||||
} |
||||
|
||||
/* Collapse similar histograms. */ |
||||
num_clusters = histogramCombineDistance(out, cluster_size, histogram_symbols, clusters, pairs, num_clusters, in_size, max_histograms, max_num_pairs) |
||||
} |
||||
|
||||
pairs = nil |
||||
cluster_size = nil |
||||
|
||||
/* Find the optimal map from original histograms to the final ones. */ |
||||
histogramRemapDistance(in, in_size, clusters, num_clusters, out, histogram_symbols) |
||||
|
||||
clusters = nil |
||||
|
||||
/* Convert the context map to a canonical form. */ |
||||
*out_size = histogramReindexDistance(out, histogram_symbols, in_size) |
||||
} |
||||
@ -0,0 +1,326 @@ |
||||
package brotli |
||||
|
||||
import "math" |
||||
|
||||
/* Copyright 2013 Google Inc. All Rights Reserved. |
||||
|
||||
Distributed under MIT license. |
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/ |
||||
|
||||
/* Computes the bit cost reduction by combining out[idx1] and out[idx2] and if |
||||
it is below a threshold, stores the pair (idx1, idx2) in the *pairs queue. */ |
||||
func compareAndPushToQueueLiteral(out []histogramLiteral, cluster_size []uint32, idx1 uint32, idx2 uint32, max_num_pairs uint, pairs []histogramPair, num_pairs *uint) { |
||||
var is_good_pair bool = false |
||||
var p histogramPair |
||||
p.idx2 = 0 |
||||
p.idx1 = p.idx2 |
||||
p.cost_combo = 0 |
||||
p.cost_diff = p.cost_combo |
||||
if idx1 == idx2 { |
||||
return |
||||
} |
||||
|
||||
if idx2 < idx1 { |
||||
var t uint32 = idx2 |
||||
idx2 = idx1 |
||||
idx1 = t |
||||
} |
||||
|
||||
p.idx1 = idx1 |
||||
p.idx2 = idx2 |
||||
p.cost_diff = 0.5 * clusterCostDiff(uint(cluster_size[idx1]), uint(cluster_size[idx2])) |
||||
p.cost_diff -= out[idx1].bit_cost_ |
||||
p.cost_diff -= out[idx2].bit_cost_ |
||||
|
||||
if out[idx1].total_count_ == 0 { |
||||
p.cost_combo = out[idx2].bit_cost_ |
||||
is_good_pair = true |
||||
} else if out[idx2].total_count_ == 0 { |
||||
p.cost_combo = out[idx1].bit_cost_ |
||||
is_good_pair = true |
||||
} else { |
||||
var threshold float64 |
||||
if *num_pairs == 0 { |
||||
threshold = 1e99 |
||||
} else { |
||||
threshold = brotli_max_double(0.0, pairs[0].cost_diff) |
||||
} |
||||
var combo histogramLiteral = out[idx1] |
||||
var cost_combo float64 |
||||
histogramAddHistogramLiteral(&combo, &out[idx2]) |
||||
cost_combo = populationCostLiteral(&combo) |
||||
if cost_combo < threshold-p.cost_diff { |
||||
p.cost_combo = cost_combo |
||||
is_good_pair = true |
||||
} |
||||
} |
||||
|
||||
if is_good_pair { |
||||
p.cost_diff += p.cost_combo |
||||
if *num_pairs > 0 && histogramPairIsLess(&pairs[0], &p) { |
||||
/* Replace the top of the queue if needed. */ |
||||
if *num_pairs < max_num_pairs { |
||||
pairs[*num_pairs] = pairs[0] |
||||
(*num_pairs)++ |
||||
} |
||||
|
||||
pairs[0] = p |
||||
} else if *num_pairs < max_num_pairs { |
||||
pairs[*num_pairs] = p |
||||
(*num_pairs)++ |
||||
} |
||||
} |
||||
} |
||||
|
||||
func histogramCombineLiteral(out []histogramLiteral, cluster_size []uint32, symbols []uint32, clusters []uint32, pairs []histogramPair, num_clusters uint, symbols_size uint, max_clusters uint, max_num_pairs uint) uint { |
||||
var cost_diff_threshold float64 = 0.0 |
||||
var min_cluster_size uint = 1 |
||||
var num_pairs uint = 0 |
||||
{ |
||||
/* We maintain a vector of histogram pairs, with the property that the pair |
||||
with the maximum bit cost reduction is the first. */ |
||||
var idx1 uint |
||||
for idx1 = 0; idx1 < num_clusters; idx1++ { |
||||
var idx2 uint |
||||
for idx2 = idx1 + 1; idx2 < num_clusters; idx2++ { |
||||
compareAndPushToQueueLiteral(out, cluster_size, clusters[idx1], clusters[idx2], max_num_pairs, pairs[0:], &num_pairs) |
||||
} |
||||
} |
||||
} |
||||
|
||||
for num_clusters > min_cluster_size { |
||||
var best_idx1 uint32 |
||||
var best_idx2 uint32 |
||||
var i uint |
||||
if pairs[0].cost_diff >= cost_diff_threshold { |
||||
cost_diff_threshold = 1e99 |
||||
min_cluster_size = max_clusters |
||||
continue |
||||
} |
||||
|
||||
/* Take the best pair from the top of heap. */ |
||||
best_idx1 = pairs[0].idx1 |
||||
|
||||
best_idx2 = pairs[0].idx2 |
||||
histogramAddHistogramLiteral(&out[best_idx1], &out[best_idx2]) |
||||
out[best_idx1].bit_cost_ = pairs[0].cost_combo |
||||
cluster_size[best_idx1] += cluster_size[best_idx2] |
||||
for i = 0; i < symbols_size; i++ { |
||||
if symbols[i] == best_idx2 { |
||||
symbols[i] = best_idx1 |
||||
} |
||||
} |
||||
|
||||
for i = 0; i < num_clusters; i++ { |
||||
if clusters[i] == best_idx2 { |
||||
copy(clusters[i:], clusters[i+1:][:num_clusters-i-1]) |
||||
break |
||||
} |
||||
} |
||||
|
||||
num_clusters-- |
||||
{ |
||||
/* Remove pairs intersecting the just combined best pair. */ |
||||
var copy_to_idx uint = 0 |
||||
for i = 0; i < num_pairs; i++ { |
||||
var p *histogramPair = &pairs[i] |
||||
if p.idx1 == best_idx1 || p.idx2 == best_idx1 || p.idx1 == best_idx2 || p.idx2 == best_idx2 { |
||||
/* Remove invalid pair from the queue. */ |
||||
continue |
||||
} |
||||
|
||||
if histogramPairIsLess(&pairs[0], p) { |
||||
/* Replace the top of the queue if needed. */ |
||||
var front histogramPair = pairs[0] |
||||
pairs[0] = *p |
||||
pairs[copy_to_idx] = front |
||||
} else { |
||||
pairs[copy_to_idx] = *p |
||||
} |
||||
|
||||
copy_to_idx++ |
||||
} |
||||
|
||||
num_pairs = copy_to_idx |
||||
} |
||||
|
||||
/* Push new pairs formed with the combined histogram to the heap. */ |
||||
for i = 0; i < num_clusters; i++ { |
||||
compareAndPushToQueueLiteral(out, cluster_size, best_idx1, clusters[i], max_num_pairs, pairs[0:], &num_pairs) |
||||
} |
||||
} |
||||
|
||||
return num_clusters |
||||
} |
||||
|
||||
/* What is the bit cost of moving histogram from cur_symbol to candidate. */ |
||||
func histogramBitCostDistanceLiteral(histogram *histogramLiteral, candidate *histogramLiteral) float64 { |
||||
if histogram.total_count_ == 0 { |
||||
return 0.0 |
||||
} else { |
||||
var tmp histogramLiteral = *histogram |
||||
histogramAddHistogramLiteral(&tmp, candidate) |
||||
return populationCostLiteral(&tmp) - candidate.bit_cost_ |
||||
} |
||||
} |
||||
|
||||
/* Find the best 'out' histogram for each of the 'in' histograms. |
||||
When called, clusters[0..num_clusters) contains the unique values from |
||||
symbols[0..in_size), but this property is not preserved in this function. |
||||
Note: we assume that out[]->bit_cost_ is already up-to-date. */ |
||||
func histogramRemapLiteral(in []histogramLiteral, in_size uint, clusters []uint32, num_clusters uint, out []histogramLiteral, symbols []uint32) { |
||||
var i uint |
||||
for i = 0; i < in_size; i++ { |
||||
var best_out uint32 |
||||
if i == 0 { |
||||
best_out = symbols[0] |
||||
} else { |
||||
best_out = symbols[i-1] |
||||
} |
||||
var best_bits float64 = histogramBitCostDistanceLiteral(&in[i], &out[best_out]) |
||||
var j uint |
||||
for j = 0; j < num_clusters; j++ { |
||||
var cur_bits float64 = histogramBitCostDistanceLiteral(&in[i], &out[clusters[j]]) |
||||
if cur_bits < best_bits { |
||||
best_bits = cur_bits |
||||
best_out = clusters[j] |
||||
} |
||||
} |
||||
|
||||
symbols[i] = best_out |
||||
} |
||||
|
||||
/* Recompute each out based on raw and symbols. */ |
||||
for i = 0; i < num_clusters; i++ { |
||||
histogramClearLiteral(&out[clusters[i]]) |
||||
} |
||||
|
||||
for i = 0; i < in_size; i++ { |
||||
histogramAddHistogramLiteral(&out[symbols[i]], &in[i]) |
||||
} |
||||
} |
||||
|
||||
/* Reorders elements of the out[0..length) array and changes values in |
||||
symbols[0..length) array in the following way: |
||||
* when called, symbols[] contains indexes into out[], and has N unique |
||||
values (possibly N < length) |
||||
* on return, symbols'[i] = f(symbols[i]) and |
||||
out'[symbols'[i]] = out[symbols[i]], for each 0 <= i < length, |
||||
where f is a bijection between the range of symbols[] and [0..N), and |
||||
the first occurrences of values in symbols'[i] come in consecutive |
||||
increasing order. |
||||
Returns N, the number of unique values in symbols[]. */ |
||||
|
||||
var histogramReindexLiteral_kInvalidIndex uint32 = math.MaxUint32 |
||||
|
||||
func histogramReindexLiteral(out []histogramLiteral, symbols []uint32, length uint) uint { |
||||
var new_index []uint32 = make([]uint32, length) |
||||
var next_index uint32 |
||||
var tmp []histogramLiteral |
||||
var i uint |
||||
for i = 0; i < length; i++ { |
||||
new_index[i] = histogramReindexLiteral_kInvalidIndex |
||||
} |
||||
|
||||
next_index = 0 |
||||
for i = 0; i < length; i++ { |
||||
if new_index[symbols[i]] == histogramReindexLiteral_kInvalidIndex { |
||||
new_index[symbols[i]] = next_index |
||||
next_index++ |
||||
} |
||||
} |
||||
|
||||
/* TODO: by using idea of "cycle-sort" we can avoid allocation of |
||||
tmp and reduce the number of copying by the factor of 2. */ |
||||
tmp = make([]histogramLiteral, next_index) |
||||
|
||||
next_index = 0 |
||||
for i = 0; i < length; i++ { |
||||
if new_index[symbols[i]] == next_index { |
||||
tmp[next_index] = out[symbols[i]] |
||||
next_index++ |
||||
} |
||||
|
||||
symbols[i] = new_index[symbols[i]] |
||||
} |
||||
|
||||
new_index = nil |
||||
for i = 0; uint32(i) < next_index; i++ { |
||||
out[i] = tmp[i] |
||||
} |
||||
|
||||
tmp = nil |
||||
return uint(next_index) |
||||
} |
||||
|
||||
func clusterHistogramsLiteral(in []histogramLiteral, in_size uint, max_histograms uint, out []histogramLiteral, out_size *uint, histogram_symbols []uint32) { |
||||
var cluster_size []uint32 = make([]uint32, in_size) |
||||
var clusters []uint32 = make([]uint32, in_size) |
||||
var num_clusters uint = 0 |
||||
var max_input_histograms uint = 64 |
||||
var pairs_capacity uint = max_input_histograms * max_input_histograms / 2 |
||||
var pairs []histogramPair = make([]histogramPair, (pairs_capacity + 1)) |
||||
var i uint |
||||
|
||||
/* For the first pass of clustering, we allow all pairs. */ |
||||
for i = 0; i < in_size; i++ { |
||||
cluster_size[i] = 1 |
||||
} |
||||
|
||||
for i = 0; i < in_size; i++ { |
||||
out[i] = in[i] |
||||
out[i].bit_cost_ = populationCostLiteral(&in[i]) |
||||
histogram_symbols[i] = uint32(i) |
||||
} |
||||
|
||||
for i = 0; i < in_size; i += max_input_histograms { |
||||
var num_to_combine uint = brotli_min_size_t(in_size-i, max_input_histograms) |
||||
var num_new_clusters uint |
||||
var j uint |
||||
for j = 0; j < num_to_combine; j++ { |
||||
clusters[num_clusters+j] = uint32(i + j) |
||||
} |
||||
|
||||
num_new_clusters = histogramCombineLiteral(out, cluster_size, histogram_symbols[i:], clusters[num_clusters:], pairs, num_to_combine, num_to_combine, max_histograms, pairs_capacity) |
||||
num_clusters += num_new_clusters |
||||
} |
||||
{ |
||||
/* For the second pass, we limit the total number of histogram pairs. |
||||
After this limit is reached, we only keep searching for the best pair. */ |
||||
var max_num_pairs uint = brotli_min_size_t(64*num_clusters, (num_clusters/2)*num_clusters) |
||||
if pairs_capacity < (max_num_pairs + 1) { |
||||
var _new_size uint |
||||
if pairs_capacity == 0 { |
||||
_new_size = max_num_pairs + 1 |
||||
} else { |
||||
_new_size = pairs_capacity |
||||
} |
||||
var new_array []histogramPair |
||||
for _new_size < (max_num_pairs + 1) { |
||||
_new_size *= 2 |
||||
} |
||||
new_array = make([]histogramPair, _new_size) |
||||
if pairs_capacity != 0 { |
||||
copy(new_array, pairs[:pairs_capacity]) |
||||
} |
||||
|
||||
pairs = new_array |
||||
pairs_capacity = _new_size |
||||
} |
||||
|
||||
/* Collapse similar histograms. */ |
||||
num_clusters = histogramCombineLiteral(out, cluster_size, histogram_symbols, clusters, pairs, num_clusters, in_size, max_histograms, max_num_pairs) |
||||
} |
||||
|
||||
pairs = nil |
||||
cluster_size = nil |
||||
|
||||
/* Find the optimal map from original histograms to the final ones. */ |
||||
histogramRemapLiteral(in, in_size, clusters, num_clusters, out, histogram_symbols) |
||||
|
||||
clusters = nil |
||||
|
||||
/* Convert the context map to a canonical form. */ |
||||
*out_size = histogramReindexLiteral(out, histogram_symbols, in_size) |
||||
} |
||||
@ -0,0 +1,252 @@ |
||||
package brotli |
||||
|
||||
var kInsBase = []uint32{ |
||||
0, |
||||
1, |
||||
2, |
||||
3, |
||||
4, |
||||
5, |
||||
6, |
||||
8, |
||||
10, |
||||
14, |
||||
18, |
||||
26, |
||||
34, |
||||
50, |
||||
66, |
||||
98, |
||||
130, |
||||
194, |
||||
322, |
||||
578, |
||||
1090, |
||||
2114, |
||||
6210, |
||||
22594, |
||||
} |
||||
|
||||
var kInsExtra = []uint32{ |
||||
0, |
||||
0, |
||||
0, |
||||
0, |
||||
0, |
||||
0, |
||||
1, |
||||
1, |
||||
2, |
||||
2, |
||||
3, |
||||
3, |
||||
4, |
||||
4, |
||||
5, |
||||
5, |
||||
6, |
||||
7, |
||||
8, |
||||
9, |
||||
10, |
||||
12, |
||||
14, |
||||
24, |
||||
} |
||||
|
||||
var kCopyBase = []uint32{ |
||||
2, |
||||
3, |
||||
4, |
||||
5, |
||||
6, |
||||
7, |
||||
8, |
||||
9, |
||||
10, |
||||
12, |
||||
14, |
||||
18, |
||||
22, |
||||
30, |
||||
38, |
||||
54, |
||||
70, |
||||
102, |
||||
134, |
||||
198, |
||||
326, |
||||
582, |
||||
1094, |
||||
2118, |
||||
} |
||||
|
||||
var kCopyExtra = []uint32{ |
||||
0, |
||||
0, |
||||
0, |
||||
0, |
||||
0, |
||||
0, |
||||
0, |
||||
0, |
||||
1, |
||||
1, |
||||
2, |
||||
2, |
||||
3, |
||||
3, |
||||
4, |
||||
4, |
||||
5, |
||||
5, |
||||
6, |
||||
7, |
||||
8, |
||||
9, |
||||
10, |
||||
24, |
||||
} |
||||
|
||||
func getInsertLengthCode(insertlen uint) uint16 { |
||||
if insertlen < 6 { |
||||
return uint16(insertlen) |
||||
} else if insertlen < 130 { |
||||
var nbits uint32 = log2FloorNonZero(insertlen-2) - 1 |
||||
return uint16((nbits << 1) + uint32((insertlen-2)>>nbits) + 2) |
||||
} else if insertlen < 2114 { |
||||
return uint16(log2FloorNonZero(insertlen-66) + 10) |
||||
} else if insertlen < 6210 { |
||||
return 21 |
||||
} else if insertlen < 22594 { |
||||
return 22 |
||||
} else { |
||||
return 23 |
||||
} |
||||
} |
||||
|
||||
func getCopyLengthCode(copylen uint) uint16 { |
||||
if copylen < 10 { |
||||
return uint16(copylen - 2) |
||||
} else if copylen < 134 { |
||||
var nbits uint32 = log2FloorNonZero(copylen-6) - 1 |
||||
return uint16((nbits << 1) + uint32((copylen-6)>>nbits) + 4) |
||||
} else if copylen < 2118 { |
||||
return uint16(log2FloorNonZero(copylen-70) + 12) |
||||
} else { |
||||
return 23 |
||||
} |
||||
} |
||||
|
||||
func combineLengthCodes(inscode uint16, copycode uint16, use_last_distance bool) uint16 { |
||||
var bits64 uint16 = uint16(copycode&0x7 | (inscode&0x7)<<3) |
||||
if use_last_distance && inscode < 8 && copycode < 16 { |
||||
if copycode < 8 { |
||||
return bits64 |
||||
} else { |
||||
return bits64 | 64 |
||||
} |
||||
} else { |
||||
/* Specification: 5 Encoding of ... (last table) */ |
||||
/* offset = 2 * index, where index is in range [0..8] */ |
||||
var offset uint32 = 2 * ((uint32(copycode) >> 3) + 3*(uint32(inscode)>>3)) |
||||
|
||||
/* All values in specification are K * 64, |
||||
where K = [2, 3, 6, 4, 5, 8, 7, 9, 10], |
||||
i + 1 = [1, 2, 3, 4, 5, 6, 7, 8, 9], |
||||
K - i - 1 = [1, 1, 3, 0, 0, 2, 0, 1, 2] = D. |
||||
All values in D require only 2 bits to encode. |
||||
Magic constant is shifted 6 bits left, to avoid final multiplication. */ |
||||
offset = (offset << 5) + 0x40 + ((0x520D40 >> offset) & 0xC0) |
||||
|
||||
return uint16(offset | uint32(bits64)) |
||||
} |
||||
} |
||||
|
||||
func getLengthCode(insertlen uint, copylen uint, use_last_distance bool, code *uint16) { |
||||
var inscode uint16 = getInsertLengthCode(insertlen) |
||||
var copycode uint16 = getCopyLengthCode(copylen) |
||||
*code = combineLengthCodes(inscode, copycode, use_last_distance) |
||||
} |
||||
|
||||
func getInsertBase(inscode uint16) uint32 { |
||||
return kInsBase[inscode] |
||||
} |
||||
|
||||
func getInsertExtra(inscode uint16) uint32 { |
||||
return kInsExtra[inscode] |
||||
} |
||||
|
||||
func getCopyBase(copycode uint16) uint32 { |
||||
return kCopyBase[copycode] |
||||
} |
||||
|
||||
func getCopyExtra(copycode uint16) uint32 { |
||||
return kCopyExtra[copycode] |
||||
} |
||||
|
||||
type command struct { |
||||
insert_len_ uint32 |
||||
copy_len_ uint32 |
||||
dist_extra_ uint32 |
||||
cmd_prefix_ uint16 |
||||
dist_prefix_ uint16 |
||||
} |
||||
|
||||
/* distance_code is e.g. 0 for same-as-last short code, or 16 for offset 1. */ |
||||
func initCommand(self *command, dist *distanceParams, insertlen uint, copylen uint, copylen_code_delta int, distance_code uint) { |
||||
/* Don't rely on signed int representation, use honest casts. */ |
||||
var delta uint32 = uint32(byte(int8(copylen_code_delta))) |
||||
self.insert_len_ = uint32(insertlen) |
||||
self.copy_len_ = uint32(uint32(copylen) | delta<<25) |
||||
|
||||
/* The distance prefix and extra bits are stored in this Command as if |
||||
npostfix and ndirect were 0, they are only recomputed later after the |
||||
clustering if needed. */ |
||||
prefixEncodeCopyDistance(distance_code, uint(dist.num_direct_distance_codes), uint(dist.distance_postfix_bits), &self.dist_prefix_, &self.dist_extra_) |
||||
|
||||
getLengthCode(insertlen, uint(int(copylen)+copylen_code_delta), (self.dist_prefix_&0x3FF == 0), &self.cmd_prefix_) |
||||
} |
||||
|
||||
func initInsertCommand(self *command, insertlen uint) { |
||||
self.insert_len_ = uint32(insertlen) |
||||
self.copy_len_ = 4 << 25 |
||||
self.dist_extra_ = 0 |
||||
self.dist_prefix_ = numDistanceShortCodes |
||||
getLengthCode(insertlen, 4, false, &self.cmd_prefix_) |
||||
} |
||||
|
||||
func commandRestoreDistanceCode(self *command, dist *distanceParams) uint32 { |
||||
if uint32(self.dist_prefix_&0x3FF) < numDistanceShortCodes+dist.num_direct_distance_codes { |
||||
return uint32(self.dist_prefix_) & 0x3FF |
||||
} else { |
||||
var dcode uint32 = uint32(self.dist_prefix_) & 0x3FF |
||||
var nbits uint32 = uint32(self.dist_prefix_) >> 10 |
||||
var extra uint32 = self.dist_extra_ |
||||
var postfix_mask uint32 = (1 << dist.distance_postfix_bits) - 1 |
||||
var hcode uint32 = (dcode - dist.num_direct_distance_codes - numDistanceShortCodes) >> dist.distance_postfix_bits |
||||
var lcode uint32 = (dcode - dist.num_direct_distance_codes - numDistanceShortCodes) & postfix_mask |
||||
var offset uint32 = ((2 + (hcode & 1)) << nbits) - 4 |
||||
return ((offset + extra) << dist.distance_postfix_bits) + lcode + dist.num_direct_distance_codes + numDistanceShortCodes |
||||
} |
||||
} |
||||
|
||||
func commandDistanceContext(self *command) uint32 { |
||||
var r uint32 = uint32(self.cmd_prefix_) >> 6 |
||||
var c uint32 = uint32(self.cmd_prefix_) & 7 |
||||
if (r == 0 || r == 2 || r == 4 || r == 7) && (c <= 2) { |
||||
return c |
||||
} |
||||
|
||||
return 3 |
||||
} |
||||
|
||||
func commandCopyLen(self *command) uint32 { |
||||
return self.copy_len_ & 0x1FFFFFF |
||||
} |
||||
|
||||
func commandCopyLenCode(self *command) uint32 { |
||||
var modifier uint32 = self.copy_len_ >> 25 |
||||
var delta int32 = int32(int8(byte(modifier | (modifier&0x40)<<1))) |
||||
return uint32(int32(self.copy_len_&0x1FFFFFF) + delta) |
||||
} |
||||
@ -0,0 +1,840 @@ |
||||
package brotli |
||||
|
||||
import "encoding/binary" |
||||
|
||||
/* Copyright 2015 Google Inc. All Rights Reserved. |
||||
|
||||
Distributed under MIT license. |
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/ |
||||
|
||||
/* Function for fast encoding of an input fragment, independently from the input |
||||
history. This function uses one-pass processing: when we find a backward |
||||
match, we immediately emit the corresponding command and literal codes to |
||||
the bit stream. |
||||
|
||||
Adapted from the CompressFragment() function in |
||||
https://github.com/google/snappy/blob/master/snappy.cc */
|
||||
|
||||
const maxDistance_compress_fragment = 262128 |
||||
|
||||
func hash5(p []byte, shift uint) uint32 { |
||||
var h uint64 = (binary.LittleEndian.Uint64(p) << 24) * uint64(kHashMul32) |
||||
return uint32(h >> shift) |
||||
} |
||||
|
||||
func hashBytesAtOffset5(v uint64, offset int, shift uint) uint32 { |
||||
assert(offset >= 0) |
||||
assert(offset <= 3) |
||||
{ |
||||
var h uint64 = ((v >> uint(8*offset)) << 24) * uint64(kHashMul32) |
||||
return uint32(h >> shift) |
||||
} |
||||
} |
||||
|
||||
func isMatch5(p1 []byte, p2 []byte) bool { |
||||
var i int |
||||
for i = 0; i < 5; i++ { |
||||
if p1[i] != p2[i] { |
||||
return false |
||||
} |
||||
} |
||||
|
||||
return true |
||||
} |
||||
|
||||
/* Builds a literal prefix code into "depths" and "bits" based on the statistics |
||||
of the "input" string and stores it into the bit stream. |
||||
Note that the prefix code here is built from the pre-LZ77 input, therefore |
||||
we can only approximate the statistics of the actual literal stream. |
||||
Moreover, for long inputs we build a histogram from a sample of the input |
||||
and thus have to assign a non-zero depth for each literal. |
||||
Returns estimated compression ratio millibytes/char for encoding given input |
||||
with generated code. */ |
||||
func buildAndStoreLiteralPrefixCode(input []byte, input_size uint, depths []byte, bits []uint16, storage_ix *uint, storage []byte) uint { |
||||
var histogram = [256]uint32{0} |
||||
var histogram_total uint |
||||
var i uint |
||||
if input_size < 1<<15 { |
||||
for i = 0; i < input_size; i++ { |
||||
histogram[input[i]]++ |
||||
} |
||||
|
||||
histogram_total = input_size |
||||
for i = 0; i < 256; i++ { |
||||
/* We weigh the first 11 samples with weight 3 to account for the |
||||
balancing effect of the LZ77 phase on the histogram. */ |
||||
var adjust uint32 = 2 * brotli_min_uint32_t(histogram[i], 11) |
||||
histogram[i] += adjust |
||||
histogram_total += uint(adjust) |
||||
} |
||||
} else { |
||||
const kSampleRate uint = 29 |
||||
for i = 0; i < input_size; i += kSampleRate { |
||||
histogram[input[i]]++ |
||||
} |
||||
|
||||
histogram_total = (input_size + kSampleRate - 1) / kSampleRate |
||||
for i = 0; i < 256; i++ { |
||||
/* We add 1 to each population count to avoid 0 bit depths (since this is |
||||
only a sample and we don't know if the symbol appears or not), and we |
||||
weigh the first 11 samples with weight 3 to account for the balancing |
||||
effect of the LZ77 phase on the histogram (more frequent symbols are |
||||
more likely to be in backward references instead as literals). */ |
||||
var adjust uint32 = 1 + 2*brotli_min_uint32_t(histogram[i], 11) |
||||
histogram[i] += adjust |
||||
histogram_total += uint(adjust) |
||||
} |
||||
} |
||||
|
||||
buildAndStoreHuffmanTreeFast(histogram[:], histogram_total, /* max_bits = */ |
||||
8, depths, bits, storage_ix, storage) |
||||
{ |
||||
var literal_ratio uint = 0 |
||||
for i = 0; i < 256; i++ { |
||||
if histogram[i] != 0 { |
||||
literal_ratio += uint(histogram[i] * uint32(depths[i])) |
||||
} |
||||
} |
||||
|
||||
/* Estimated encoding ratio, millibytes per symbol. */ |
||||
return (literal_ratio * 125) / histogram_total |
||||
} |
||||
} |
||||
|
||||
/* Builds a command and distance prefix code (each 64 symbols) into "depth" and |
||||
"bits" based on "histogram" and stores it into the bit stream. */ |
||||
func buildAndStoreCommandPrefixCode1(histogram []uint32, depth []byte, bits []uint16, storage_ix *uint, storage []byte) { |
||||
var tree [129]huffmanTree |
||||
var cmd_depth = [numCommandSymbols]byte{0} |
||||
/* Tree size for building a tree over 64 symbols is 2 * 64 + 1. */ |
||||
|
||||
var cmd_bits [64]uint16 |
||||
|
||||
createHuffmanTree(histogram, 64, 15, tree[:], depth) |
||||
createHuffmanTree(histogram[64:], 64, 14, tree[:], depth[64:]) |
||||
|
||||
/* We have to jump through a few hoops here in order to compute |
||||
the command bits because the symbols are in a different order than in |
||||
the full alphabet. This looks complicated, but having the symbols |
||||
in this order in the command bits saves a few branches in the Emit* |
||||
functions. */ |
||||
copy(cmd_depth[:], depth[:24]) |
||||
|
||||
copy(cmd_depth[24:][:], depth[40:][:8]) |
||||
copy(cmd_depth[32:][:], depth[24:][:8]) |
||||
copy(cmd_depth[40:][:], depth[48:][:8]) |
||||
copy(cmd_depth[48:][:], depth[32:][:8]) |
||||
copy(cmd_depth[56:][:], depth[56:][:8]) |
||||
convertBitDepthsToSymbols(cmd_depth[:], 64, cmd_bits[:]) |
||||
copy(bits, cmd_bits[:24]) |
||||
copy(bits[24:], cmd_bits[32:][:8]) |
||||
copy(bits[32:], cmd_bits[48:][:8]) |
||||
copy(bits[40:], cmd_bits[24:][:8]) |
||||
copy(bits[48:], cmd_bits[40:][:8]) |
||||
copy(bits[56:], cmd_bits[56:][:8]) |
||||
convertBitDepthsToSymbols(depth[64:], 64, bits[64:]) |
||||
{ |
||||
/* Create the bit length array for the full command alphabet. */ |
||||
var i uint |
||||
for i := 0; i < int(64); i++ { |
||||
cmd_depth[i] = 0 |
||||
} /* only 64 first values were used */ |
||||
copy(cmd_depth[:], depth[:8]) |
||||
copy(cmd_depth[64:][:], depth[8:][:8]) |
||||
copy(cmd_depth[128:][:], depth[16:][:8]) |
||||
copy(cmd_depth[192:][:], depth[24:][:8]) |
||||
copy(cmd_depth[384:][:], depth[32:][:8]) |
||||
for i = 0; i < 8; i++ { |
||||
cmd_depth[128+8*i] = depth[40+i] |
||||
cmd_depth[256+8*i] = depth[48+i] |
||||
cmd_depth[448+8*i] = depth[56+i] |
||||
} |
||||
|
||||
storeHuffmanTree(cmd_depth[:], numCommandSymbols, tree[:], storage_ix, storage) |
||||
} |
||||
|
||||
storeHuffmanTree(depth[64:], 64, tree[:], storage_ix, storage) |
||||
} |
||||
|
||||
/* REQUIRES: insertlen < 6210 */ |
||||
func emitInsertLen1(insertlen uint, depth []byte, bits []uint16, histo []uint32, storage_ix *uint, storage []byte) { |
||||
if insertlen < 6 { |
||||
var code uint = insertlen + 40 |
||||
writeBits(uint(depth[code]), uint64(bits[code]), storage_ix, storage) |
||||
histo[code]++ |
||||
} else if insertlen < 130 { |
||||
var tail uint = insertlen - 2 |
||||
var nbits uint32 = log2FloorNonZero(tail) - 1 |
||||
var prefix uint = tail >> nbits |
||||
var inscode uint = uint((nbits << 1) + uint32(prefix) + 42) |
||||
writeBits(uint(depth[inscode]), uint64(bits[inscode]), storage_ix, storage) |
||||
writeBits(uint(nbits), uint64(tail)-(uint64(prefix)<<nbits), storage_ix, storage) |
||||
histo[inscode]++ |
||||
} else if insertlen < 2114 { |
||||
var tail uint = insertlen - 66 |
||||
var nbits uint32 = log2FloorNonZero(tail) |
||||
var code uint = uint(nbits + 50) |
||||
writeBits(uint(depth[code]), uint64(bits[code]), storage_ix, storage) |
||||
writeBits(uint(nbits), uint64(tail)-(uint64(uint(1))<<nbits), storage_ix, storage) |
||||
histo[code]++ |
||||
} else { |
||||
writeBits(uint(depth[61]), uint64(bits[61]), storage_ix, storage) |
||||
writeBits(12, uint64(insertlen)-2114, storage_ix, storage) |
||||
histo[61]++ |
||||
} |
||||
} |
||||
|
||||
func emitLongInsertLen(insertlen uint, depth []byte, bits []uint16, histo []uint32, storage_ix *uint, storage []byte) { |
||||
if insertlen < 22594 { |
||||
writeBits(uint(depth[62]), uint64(bits[62]), storage_ix, storage) |
||||
writeBits(14, uint64(insertlen)-6210, storage_ix, storage) |
||||
histo[62]++ |
||||
} else { |
||||
writeBits(uint(depth[63]), uint64(bits[63]), storage_ix, storage) |
||||
writeBits(24, uint64(insertlen)-22594, storage_ix, storage) |
||||
histo[63]++ |
||||
} |
||||
} |
||||
|
||||
func emitCopyLen1(copylen uint, depth []byte, bits []uint16, histo []uint32, storage_ix *uint, storage []byte) { |
||||
if copylen < 10 { |
||||
writeBits(uint(depth[copylen+14]), uint64(bits[copylen+14]), storage_ix, storage) |
||||
histo[copylen+14]++ |
||||
} else if copylen < 134 { |
||||
var tail uint = copylen - 6 |
||||
var nbits uint32 = log2FloorNonZero(tail) - 1 |
||||
var prefix uint = tail >> nbits |
||||
var code uint = uint((nbits << 1) + uint32(prefix) + 20) |
||||
writeBits(uint(depth[code]), uint64(bits[code]), storage_ix, storage) |
||||
writeBits(uint(nbits), uint64(tail)-(uint64(prefix)<<nbits), storage_ix, storage) |
||||
histo[code]++ |
||||
} else if copylen < 2118 { |
||||
var tail uint = copylen - 70 |
||||
var nbits uint32 = log2FloorNonZero(tail) |
||||
var code uint = uint(nbits + 28) |
||||
writeBits(uint(depth[code]), uint64(bits[code]), storage_ix, storage) |
||||
writeBits(uint(nbits), uint64(tail)-(uint64(uint(1))<<nbits), storage_ix, storage) |
||||
histo[code]++ |
||||
} else { |
||||
writeBits(uint(depth[39]), uint64(bits[39]), storage_ix, storage) |
||||
writeBits(24, uint64(copylen)-2118, storage_ix, storage) |
||||
histo[39]++ |
||||
} |
||||
} |
||||
|
||||
func emitCopyLenLastDistance1(copylen uint, depth []byte, bits []uint16, histo []uint32, storage_ix *uint, storage []byte) { |
||||
if copylen < 12 { |
||||
writeBits(uint(depth[copylen-4]), uint64(bits[copylen-4]), storage_ix, storage) |
||||
histo[copylen-4]++ |
||||
} else if copylen < 72 { |
||||
var tail uint = copylen - 8 |
||||
var nbits uint32 = log2FloorNonZero(tail) - 1 |
||||
var prefix uint = tail >> nbits |
||||
var code uint = uint((nbits << 1) + uint32(prefix) + 4) |
||||
writeBits(uint(depth[code]), uint64(bits[code]), storage_ix, storage) |
||||
writeBits(uint(nbits), uint64(tail)-(uint64(prefix)<<nbits), storage_ix, storage) |
||||
histo[code]++ |
||||
} else if copylen < 136 { |
||||
var tail uint = copylen - 8 |
||||
var code uint = (tail >> 5) + 30 |
||||
writeBits(uint(depth[code]), uint64(bits[code]), storage_ix, storage) |
||||
writeBits(5, uint64(tail)&31, storage_ix, storage) |
||||
writeBits(uint(depth[64]), uint64(bits[64]), storage_ix, storage) |
||||
histo[code]++ |
||||
histo[64]++ |
||||
} else if copylen < 2120 { |
||||
var tail uint = copylen - 72 |
||||
var nbits uint32 = log2FloorNonZero(tail) |
||||
var code uint = uint(nbits + 28) |
||||
writeBits(uint(depth[code]), uint64(bits[code]), storage_ix, storage) |
||||
writeBits(uint(nbits), uint64(tail)-(uint64(uint(1))<<nbits), storage_ix, storage) |
||||
writeBits(uint(depth[64]), uint64(bits[64]), storage_ix, storage) |
||||
histo[code]++ |
||||
histo[64]++ |
||||
} else { |
||||
writeBits(uint(depth[39]), uint64(bits[39]), storage_ix, storage) |
||||
writeBits(24, uint64(copylen)-2120, storage_ix, storage) |
||||
writeBits(uint(depth[64]), uint64(bits[64]), storage_ix, storage) |
||||
histo[39]++ |
||||
histo[64]++ |
||||
} |
||||
} |
||||
|
||||
func emitDistance1(distance uint, depth []byte, bits []uint16, histo []uint32, storage_ix *uint, storage []byte) { |
||||
var d uint = distance + 3 |
||||
var nbits uint32 = log2FloorNonZero(d) - 1 |
||||
var prefix uint = (d >> nbits) & 1 |
||||
var offset uint = (2 + prefix) << nbits |
||||
var distcode uint = uint(2*(nbits-1) + uint32(prefix) + 80) |
||||
writeBits(uint(depth[distcode]), uint64(bits[distcode]), storage_ix, storage) |
||||
writeBits(uint(nbits), uint64(d)-uint64(offset), storage_ix, storage) |
||||
histo[distcode]++ |
||||
} |
||||
|
||||
func emitLiterals(input []byte, len uint, depth []byte, bits []uint16, storage_ix *uint, storage []byte) { |
||||
var j uint |
||||
for j = 0; j < len; j++ { |
||||
var lit byte = input[j] |
||||
writeBits(uint(depth[lit]), uint64(bits[lit]), storage_ix, storage) |
||||
} |
||||
} |
||||
|
||||
/* REQUIRES: len <= 1 << 24. */ |
||||
func storeMetaBlockHeader1(len uint, is_uncompressed bool, storage_ix *uint, storage []byte) { |
||||
var nibbles uint = 6 |
||||
|
||||
/* ISLAST */ |
||||
writeBits(1, 0, storage_ix, storage) |
||||
|
||||
if len <= 1<<16 { |
||||
nibbles = 4 |
||||
} else if len <= 1<<20 { |
||||
nibbles = 5 |
||||
} |
||||
|
||||
writeBits(2, uint64(nibbles)-4, storage_ix, storage) |
||||
writeBits(nibbles*4, uint64(len)-1, storage_ix, storage) |
||||
|
||||
/* ISUNCOMPRESSED */ |
||||
writeSingleBit(is_uncompressed, storage_ix, storage) |
||||
} |
||||
|
||||
func updateBits(n_bits uint, bits uint32, pos uint, array []byte) { |
||||
for n_bits > 0 { |
||||
var byte_pos uint = pos >> 3 |
||||
var n_unchanged_bits uint = pos & 7 |
||||
var n_changed_bits uint = brotli_min_size_t(n_bits, 8-n_unchanged_bits) |
||||
var total_bits uint = n_unchanged_bits + n_changed_bits |
||||
var mask uint32 = (^((1 << total_bits) - 1)) | ((1 << n_unchanged_bits) - 1) |
||||
var unchanged_bits uint32 = uint32(array[byte_pos]) & mask |
||||
var changed_bits uint32 = bits & ((1 << n_changed_bits) - 1) |
||||
array[byte_pos] = byte(changed_bits<<n_unchanged_bits | unchanged_bits) |
||||
n_bits -= n_changed_bits |
||||
bits >>= n_changed_bits |
||||
pos += n_changed_bits |
||||
} |
||||
} |
||||
|
||||
func rewindBitPosition1(new_storage_ix uint, storage_ix *uint, storage []byte) { |
||||
var bitpos uint = new_storage_ix & 7 |
||||
var mask uint = (1 << bitpos) - 1 |
||||
storage[new_storage_ix>>3] &= byte(mask) |
||||
*storage_ix = new_storage_ix |
||||
} |
||||
|
||||
var shouldMergeBlock_kSampleRate uint = 43 |
||||
|
||||
func shouldMergeBlock(data []byte, len uint, depths []byte) bool { |
||||
var histo = [256]uint{0} |
||||
var i uint |
||||
for i = 0; i < len; i += shouldMergeBlock_kSampleRate { |
||||
histo[data[i]]++ |
||||
} |
||||
{ |
||||
var total uint = (len + shouldMergeBlock_kSampleRate - 1) / shouldMergeBlock_kSampleRate |
||||
var r float64 = (fastLog2(total)+0.5)*float64(total) + 200 |
||||
for i = 0; i < 256; i++ { |
||||
r -= float64(histo[i]) * (float64(depths[i]) + fastLog2(histo[i])) |
||||
} |
||||
|
||||
return r >= 0.0 |
||||
} |
||||
} |
||||
|
||||
func shouldUseUncompressedMode(metablock_start []byte, next_emit []byte, insertlen uint, literal_ratio uint) bool { |
||||
var compressed uint = uint(-cap(next_emit) + cap(metablock_start)) |
||||
if compressed*50 > insertlen { |
||||
return false |
||||
} else { |
||||
return literal_ratio > 980 |
||||
} |
||||
} |
||||
|
||||
func emitUncompressedMetaBlock1(begin []byte, end []byte, storage_ix_start uint, storage_ix *uint, storage []byte) { |
||||
var len uint = uint(-cap(end) + cap(begin)) |
||||
rewindBitPosition1(storage_ix_start, storage_ix, storage) |
||||
storeMetaBlockHeader1(uint(len), true, storage_ix, storage) |
||||
*storage_ix = (*storage_ix + 7) &^ 7 |
||||
copy(storage[*storage_ix>>3:], begin[:len]) |
||||
*storage_ix += uint(len << 3) |
||||
storage[*storage_ix>>3] = 0 |
||||
} |
||||
|
||||
var kCmdHistoSeed = [128]uint32{ |
||||
0, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
0, |
||||
0, |
||||
0, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
0, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
0, |
||||
0, |
||||
0, |
||||
0, |
||||
0, |
||||
0, |
||||
0, |
||||
0, |
||||
0, |
||||
0, |
||||
0, |
||||
0, |
||||
0, |
||||
0, |
||||
0, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
1, |
||||
0, |
||||
0, |
||||
0, |
||||
0, |
||||
} |
||||
|
||||
var compressFragmentFastImpl_kFirstBlockSize uint = 3 << 15 |
||||
var compressFragmentFastImpl_kMergeBlockSize uint = 1 << 16 |
||||
|
||||
func compressFragmentFastImpl(in []byte, input_size uint, is_last bool, table []int, table_bits uint, cmd_depth []byte, cmd_bits []uint16, cmd_code_numbits *uint, cmd_code []byte, storage_ix *uint, storage []byte) { |
||||
var cmd_histo [128]uint32 |
||||
var ip_end int |
||||
var next_emit int = 0 |
||||
var base_ip int = 0 |
||||
var input int = 0 |
||||
const kInputMarginBytes uint = windowGap |
||||
const kMinMatchLen uint = 5 |
||||
var metablock_start int = input |
||||
var block_size uint = brotli_min_size_t(input_size, compressFragmentFastImpl_kFirstBlockSize) |
||||
var total_block_size uint = block_size |
||||
var mlen_storage_ix uint = *storage_ix + 3 |
||||
var lit_depth [256]byte |
||||
var lit_bits [256]uint16 |
||||
var literal_ratio uint |
||||
var ip int |
||||
var last_distance int |
||||
var shift uint = 64 - table_bits |
||||
|
||||
/* "next_emit" is a pointer to the first byte that is not covered by a |
||||
previous copy. Bytes between "next_emit" and the start of the next copy or |
||||
the end of the input will be emitted as literal bytes. */ |
||||
|
||||
/* Save the start of the first block for position and distance computations. |
||||
*/ |
||||
|
||||
/* Save the bit position of the MLEN field of the meta-block header, so that |
||||
we can update it later if we decide to extend this meta-block. */ |
||||
storeMetaBlockHeader1(block_size, false, storage_ix, storage) |
||||
|
||||
/* No block splits, no contexts. */ |
||||
writeBits(13, 0, storage_ix, storage) |
||||
|
||||
literal_ratio = buildAndStoreLiteralPrefixCode(in[input:], block_size, lit_depth[:], lit_bits[:], storage_ix, storage) |
||||
{ |
||||
/* Store the pre-compressed command and distance prefix codes. */ |
||||
var i uint |
||||
for i = 0; i+7 < *cmd_code_numbits; i += 8 { |
||||
writeBits(8, uint64(cmd_code[i>>3]), storage_ix, storage) |
||||
} |
||||
} |
||||
|
||||
writeBits(*cmd_code_numbits&7, uint64(cmd_code[*cmd_code_numbits>>3]), storage_ix, storage) |
||||
|
||||
/* Initialize the command and distance histograms. We will gather |
||||
statistics of command and distance codes during the processing |
||||
of this block and use it to update the command and distance |
||||
prefix codes for the next block. */ |
||||
emit_commands: |
||||
copy(cmd_histo[:], kCmdHistoSeed[:]) |
||||
|
||||
/* "ip" is the input pointer. */ |
||||
ip = input |
||||
|
||||
last_distance = -1 |
||||
ip_end = int(uint(input) + block_size) |
||||
|
||||
if block_size >= kInputMarginBytes { |
||||
var len_limit uint = brotli_min_size_t(block_size-kMinMatchLen, input_size-kInputMarginBytes) |
||||
var ip_limit int = int(uint(input) + len_limit) |
||||
/* For the last block, we need to keep a 16 bytes margin so that we can be |
||||
sure that all distances are at most window size - 16. |
||||
For all other blocks, we only need to keep a margin of 5 bytes so that |
||||
we don't go over the block size with a copy. */ |
||||
|
||||
var next_hash uint32 |
||||
ip++ |
||||
for next_hash = hash5(in[ip:], shift); ; { |
||||
var skip uint32 = 32 |
||||
var next_ip int = ip |
||||
/* Step 1: Scan forward in the input looking for a 5-byte-long match. |
||||
If we get close to exhausting the input then goto emit_remainder. |
||||
|
||||
Heuristic match skipping: If 32 bytes are scanned with no matches |
||||
found, start looking only at every other byte. If 32 more bytes are |
||||
scanned, look at every third byte, etc.. When a match is found, |
||||
immediately go back to looking at every byte. This is a small loss |
||||
(~5% performance, ~0.1% density) for compressible data due to more |
||||
bookkeeping, but for non-compressible data (such as JPEG) it's a huge |
||||
win since the compressor quickly "realizes" the data is incompressible |
||||
and doesn't bother looking for matches everywhere. |
||||
|
||||
The "skip" variable keeps track of how many bytes there are since the |
||||
last match; dividing it by 32 (i.e. right-shifting by five) gives the |
||||
number of bytes to move ahead for each iteration. */ |
||||
|
||||
var candidate int |
||||
assert(next_emit < ip) |
||||
|
||||
trawl: |
||||
for { |
||||
var hash uint32 = next_hash |
||||
var bytes_between_hash_lookups uint32 = skip >> 5 |
||||
skip++ |
||||
assert(hash == hash5(in[next_ip:], shift)) |
||||
ip = next_ip |
||||
next_ip = int(uint32(ip) + bytes_between_hash_lookups) |
||||
if next_ip > ip_limit { |
||||
goto emit_remainder |
||||
} |
||||
|
||||
next_hash = hash5(in[next_ip:], shift) |
||||
candidate = ip - last_distance |
||||
if isMatch5(in[ip:], in[candidate:]) { |
||||
if candidate < ip { |
||||
table[hash] = int(ip - base_ip) |
||||
break |
||||
} |
||||
} |
||||
|
||||
candidate = base_ip + table[hash] |
||||
assert(candidate >= base_ip) |
||||
assert(candidate < ip) |
||||
|
||||
table[hash] = int(ip - base_ip) |
||||
if !(!isMatch5(in[ip:], in[candidate:])) { |
||||
break |
||||
} |
||||
} |
||||
|
||||
/* Check copy distance. If candidate is not feasible, continue search. |
||||
Checking is done outside of hot loop to reduce overhead. */ |
||||
if ip-candidate > maxDistance_compress_fragment { |
||||
goto trawl |
||||
} |
||||
|
||||
/* Step 2: Emit the found match together with the literal bytes from |
||||
"next_emit" to the bit stream, and then see if we can find a next match |
||||
immediately afterwards. Repeat until we find no match for the input |
||||
without emitting some literal bytes. */ |
||||
{ |
||||
var base int = ip |
||||
/* > 0 */ |
||||
var matched uint = 5 + findMatchLengthWithLimit(in[candidate+5:], in[ip+5:], uint(ip_end-ip)-5) |
||||
var distance int = int(base - candidate) |
||||
/* We have a 5-byte match at ip, and we need to emit bytes in |
||||
[next_emit, ip). */ |
||||
|
||||
var insert uint = uint(base - next_emit) |
||||
ip += int(matched) |
||||
if insert < 6210 { |
||||
emitInsertLen1(insert, cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage) |
||||
} else if shouldUseUncompressedMode(in[metablock_start:], in[next_emit:], insert, literal_ratio) { |
||||
emitUncompressedMetaBlock1(in[metablock_start:], in[base:], mlen_storage_ix-3, storage_ix, storage) |
||||
input_size -= uint(base - input) |
||||
input = base |
||||
next_emit = input |
||||
goto next_block |
||||
} else { |
||||
emitLongInsertLen(insert, cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage) |
||||
} |
||||
|
||||
emitLiterals(in[next_emit:], insert, lit_depth[:], lit_bits[:], storage_ix, storage) |
||||
if distance == last_distance { |
||||
writeBits(uint(cmd_depth[64]), uint64(cmd_bits[64]), storage_ix, storage) |
||||
cmd_histo[64]++ |
||||
} else { |
||||
emitDistance1(uint(distance), cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage) |
||||
last_distance = distance |
||||
} |
||||
|
||||
emitCopyLenLastDistance1(matched, cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage) |
||||
|
||||
next_emit = ip |
||||
if ip >= ip_limit { |
||||
goto emit_remainder |
||||
} |
||||
|
||||
/* We could immediately start working at ip now, but to improve |
||||
compression we first update "table" with the hashes of some positions |
||||
within the last copy. */ |
||||
{ |
||||
var input_bytes uint64 = binary.LittleEndian.Uint64(in[ip-3:]) |
||||
var prev_hash uint32 = hashBytesAtOffset5(input_bytes, 0, shift) |
||||
var cur_hash uint32 = hashBytesAtOffset5(input_bytes, 3, shift) |
||||
table[prev_hash] = int(ip - base_ip - 3) |
||||
prev_hash = hashBytesAtOffset5(input_bytes, 1, shift) |
||||
table[prev_hash] = int(ip - base_ip - 2) |
||||
prev_hash = hashBytesAtOffset5(input_bytes, 2, shift) |
||||
table[prev_hash] = int(ip - base_ip - 1) |
||||
|
||||
candidate = base_ip + table[cur_hash] |
||||
table[cur_hash] = int(ip - base_ip) |
||||
} |
||||
} |
||||
|
||||
for isMatch5(in[ip:], in[candidate:]) { |
||||
var base int = ip |
||||
/* We have a 5-byte match at ip, and no need to emit any literal bytes |
||||
prior to ip. */ |
||||
|
||||
var matched uint = 5 + findMatchLengthWithLimit(in[candidate+5:], in[ip+5:], uint(ip_end-ip)-5) |
||||
if ip-candidate > maxDistance_compress_fragment { |
||||
break |
||||
} |
||||
ip += int(matched) |
||||
last_distance = int(base - candidate) /* > 0 */ |
||||
emitCopyLen1(matched, cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage) |
||||
emitDistance1(uint(last_distance), cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage) |
||||
|
||||
next_emit = ip |
||||
if ip >= ip_limit { |
||||
goto emit_remainder |
||||
} |
||||
|
||||
/* We could immediately start working at ip now, but to improve |
||||
compression we first update "table" with the hashes of some positions |
||||
within the last copy. */ |
||||
{ |
||||
var input_bytes uint64 = binary.LittleEndian.Uint64(in[ip-3:]) |
||||
var prev_hash uint32 = hashBytesAtOffset5(input_bytes, 0, shift) |
||||
var cur_hash uint32 = hashBytesAtOffset5(input_bytes, 3, shift) |
||||
table[prev_hash] = int(ip - base_ip - 3) |
||||
prev_hash = hashBytesAtOffset5(input_bytes, 1, shift) |
||||
table[prev_hash] = int(ip - base_ip - 2) |
||||
prev_hash = hashBytesAtOffset5(input_bytes, 2, shift) |
||||
table[prev_hash] = int(ip - base_ip - 1) |
||||
|
||||
candidate = base_ip + table[cur_hash] |
||||
table[cur_hash] = int(ip - base_ip) |
||||
} |
||||
} |
||||
|
||||
ip++ |
||||
next_hash = hash5(in[ip:], shift) |
||||
} |
||||
} |
||||
|
||||
emit_remainder: |
||||
assert(next_emit <= ip_end) |
||||
input += int(block_size) |
||||
input_size -= block_size |
||||
block_size = brotli_min_size_t(input_size, compressFragmentFastImpl_kMergeBlockSize) |
||||
|
||||
/* Decide if we want to continue this meta-block instead of emitting the |
||||
last insert-only command. */ |
||||
if input_size > 0 && total_block_size+block_size <= 1<<20 && shouldMergeBlock(in[input:], block_size, lit_depth[:]) { |
||||
assert(total_block_size > 1<<16) |
||||
|
||||
/* Update the size of the current meta-block and continue emitting commands. |
||||
We can do this because the current size and the new size both have 5 |
||||
nibbles. */ |
||||
total_block_size += block_size |
||||
|
||||
updateBits(20, uint32(total_block_size-1), mlen_storage_ix, storage) |
||||
goto emit_commands |
||||
} |
||||
|
||||
/* Emit the remaining bytes as literals. */ |
||||
if next_emit < ip_end { |
||||
var insert uint = uint(ip_end - next_emit) |
||||
if insert < 6210 { |
||||
emitInsertLen1(insert, cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage) |
||||
emitLiterals(in[next_emit:], insert, lit_depth[:], lit_bits[:], storage_ix, storage) |
||||
} else if shouldUseUncompressedMode(in[metablock_start:], in[next_emit:], insert, literal_ratio) { |
||||
emitUncompressedMetaBlock1(in[metablock_start:], in[ip_end:], mlen_storage_ix-3, storage_ix, storage) |
||||
} else { |
||||
emitLongInsertLen(insert, cmd_depth, cmd_bits, cmd_histo[:], storage_ix, storage) |
||||
emitLiterals(in[next_emit:], insert, lit_depth[:], lit_bits[:], storage_ix, storage) |
||||
} |
||||
} |
||||
|
||||
next_emit = ip_end |
||||
|
||||
/* If we have more data, write a new meta-block header and prefix codes and |
||||
then continue emitting commands. */ |
||||
next_block: |
||||
if input_size > 0 { |
||||
metablock_start = input |
||||
block_size = brotli_min_size_t(input_size, compressFragmentFastImpl_kFirstBlockSize) |
||||
total_block_size = block_size |
||||
|
||||
/* Save the bit position of the MLEN field of the meta-block header, so that |
||||
we can update it later if we decide to extend this meta-block. */ |
||||
mlen_storage_ix = *storage_ix + 3 |
||||
|
||||
storeMetaBlockHeader1(block_size, false, storage_ix, storage) |
||||
|
||||
/* No block splits, no contexts. */ |
||||
writeBits(13, 0, storage_ix, storage) |
||||
|
||||
literal_ratio = buildAndStoreLiteralPrefixCode(in[input:], block_size, lit_depth[:], lit_bits[:], storage_ix, storage) |
||||
buildAndStoreCommandPrefixCode1(cmd_histo[:], cmd_depth, cmd_bits, storage_ix, storage) |
||||
goto emit_commands |
||||
} |
||||
|
||||
if !is_last { |
||||
/* If this is not the last block, update the command and distance prefix |
||||
codes for the next block and store the compressed forms. */ |
||||
cmd_code[0] = 0 |
||||
|
||||
*cmd_code_numbits = 0 |
||||
buildAndStoreCommandPrefixCode1(cmd_histo[:], cmd_depth, cmd_bits, cmd_code_numbits, cmd_code) |
||||
} |
||||
} |
||||
|
||||
/* Compresses "input" string to the "*storage" buffer as one or more complete |
||||
meta-blocks, and updates the "*storage_ix" bit position. |
||||
|
||||
If "is_last" is 1, emits an additional empty last meta-block. |
||||
|
||||
"cmd_depth" and "cmd_bits" contain the command and distance prefix codes |
||||
(see comment in encode.h) used for the encoding of this input fragment. |
||||
If "is_last" is 0, they are updated to reflect the statistics |
||||
of this input fragment, to be used for the encoding of the next fragment. |
||||
|
||||
"*cmd_code_numbits" is the number of bits of the compressed representation |
||||
of the command and distance prefix codes, and "cmd_code" is an array of |
||||
at least "(*cmd_code_numbits + 7) >> 3" size that contains the compressed |
||||
command and distance prefix codes. If "is_last" is 0, these are also |
||||
updated to represent the updated "cmd_depth" and "cmd_bits". |
||||
|
||||
REQUIRES: "input_size" is greater than zero, or "is_last" is 1. |
||||
REQUIRES: "input_size" is less or equal to maximal metablock size (1 << 24). |
||||
REQUIRES: All elements in "table[0..table_size-1]" are initialized to zero. |
||||
REQUIRES: "table_size" is an odd (9, 11, 13, 15) power of two |
||||
OUTPUT: maximal copy distance <= |input_size| |
||||
OUTPUT: maximal copy distance <= BROTLI_MAX_BACKWARD_LIMIT(18) */ |
||||
func compressFragmentFast(input []byte, input_size uint, is_last bool, table []int, table_size uint, cmd_depth []byte, cmd_bits []uint16, cmd_code_numbits *uint, cmd_code []byte, storage_ix *uint, storage []byte) { |
||||
var initial_storage_ix uint = *storage_ix |
||||
var table_bits uint = uint(log2FloorNonZero(table_size)) |
||||
|
||||
if input_size == 0 { |
||||
assert(is_last) |
||||
writeBits(1, 1, storage_ix, storage) /* islast */ |
||||
writeBits(1, 1, storage_ix, storage) /* isempty */ |
||||
*storage_ix = (*storage_ix + 7) &^ 7 |
||||
return |
||||
} |
||||
|
||||
compressFragmentFastImpl(input, input_size, is_last, table, table_bits, cmd_depth, cmd_bits, cmd_code_numbits, cmd_code, storage_ix, storage) |
||||
|
||||
/* If output is larger than single uncompressed block, rewrite it. */ |
||||
if *storage_ix-initial_storage_ix > 31+(input_size<<3) { |
||||
emitUncompressedMetaBlock1(input, input[input_size:], initial_storage_ix, storage_ix, storage) |
||||
} |
||||
|
||||
if is_last { |
||||
writeBits(1, 1, storage_ix, storage) /* islast */ |
||||
writeBits(1, 1, storage_ix, storage) /* isempty */ |
||||
*storage_ix = (*storage_ix + 7) &^ 7 |
||||
} |
||||
} |
||||
@ -0,0 +1,749 @@ |
||||
package brotli |
||||
|
||||
import "encoding/binary" |
||||
|
||||
/* Copyright 2015 Google Inc. All Rights Reserved. |
||||
|
||||
Distributed under MIT license. |
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/ |
||||
|
||||
/* Function for fast encoding of an input fragment, independently from the input |
||||
history. This function uses two-pass processing: in the first pass we save |
||||
the found backward matches and literal bytes into a buffer, and in the |
||||
second pass we emit them into the bit stream using prefix codes built based |
||||
on the actual command and literal byte histograms. */ |
||||
|
||||
const kCompressFragmentTwoPassBlockSize uint = 1 << 17 |
||||
|
||||
func hash1(p []byte, shift uint, length uint) uint32 { |
||||
var h uint64 = (binary.LittleEndian.Uint64(p) << ((8 - length) * 8)) * uint64(kHashMul32) |
||||
return uint32(h >> shift) |
||||
} |
||||
|
||||
func hashBytesAtOffset(v uint64, offset uint, shift uint, length uint) uint32 { |
||||
assert(offset <= 8-length) |
||||
{ |
||||
var h uint64 = ((v >> (8 * offset)) << ((8 - length) * 8)) * uint64(kHashMul32) |
||||
return uint32(h >> shift) |
||||
} |
||||
} |
||||
|
||||
func isMatch1(p1 []byte, p2 []byte, length uint) bool { |
||||
var i uint |
||||
for i = 0; i < length && i < 6; i++ { |
||||
if p1[i] != p2[i] { |
||||
return false |
||||
} |
||||
} |
||||
|
||||
return true |
||||
} |
||||
|
||||
/* Builds a command and distance prefix code (each 64 symbols) into "depth" and |
||||
"bits" based on "histogram" and stores it into the bit stream. */ |
||||
func buildAndStoreCommandPrefixCode(histogram []uint32, depth []byte, bits []uint16, storage_ix *uint, storage []byte) { |
||||
var tree [129]huffmanTree |
||||
var cmd_depth = [numCommandSymbols]byte{0} |
||||
/* Tree size for building a tree over 64 symbols is 2 * 64 + 1. */ |
||||
|
||||
var cmd_bits [64]uint16 |
||||
createHuffmanTree(histogram, 64, 15, tree[:], depth) |
||||
createHuffmanTree(histogram[64:], 64, 14, tree[:], depth[64:]) |
||||
|
||||
/* We have to jump through a few hoops here in order to compute |
||||
the command bits because the symbols are in a different order than in |
||||
the full alphabet. This looks complicated, but having the symbols |
||||
in this order in the command bits saves a few branches in the Emit* |
||||
functions. */ |
||||
copy(cmd_depth[:], depth[24:][:24]) |
||||
|
||||
copy(cmd_depth[24:][:], depth[:8]) |
||||
copy(cmd_depth[32:][:], depth[48:][:8]) |
||||
copy(cmd_depth[40:][:], depth[8:][:8]) |
||||
copy(cmd_depth[48:][:], depth[56:][:8]) |
||||
copy(cmd_depth[56:][:], depth[16:][:8]) |
||||
convertBitDepthsToSymbols(cmd_depth[:], 64, cmd_bits[:]) |
||||
copy(bits, cmd_bits[24:][:8]) |
||||
copy(bits[8:], cmd_bits[40:][:8]) |
||||
copy(bits[16:], cmd_bits[56:][:8]) |
||||
copy(bits[24:], cmd_bits[:24]) |
||||
copy(bits[48:], cmd_bits[32:][:8]) |
||||
copy(bits[56:], cmd_bits[48:][:8]) |
||||
convertBitDepthsToSymbols(depth[64:], 64, bits[64:]) |
||||
{ |
||||
/* Create the bit length array for the full command alphabet. */ |
||||
var i uint |
||||
for i := 0; i < int(64); i++ { |
||||
cmd_depth[i] = 0 |
||||
} /* only 64 first values were used */ |
||||
copy(cmd_depth[:], depth[24:][:8]) |
||||
copy(cmd_depth[64:][:], depth[32:][:8]) |
||||
copy(cmd_depth[128:][:], depth[40:][:8]) |
||||
copy(cmd_depth[192:][:], depth[48:][:8]) |
||||
copy(cmd_depth[384:][:], depth[56:][:8]) |
||||
for i = 0; i < 8; i++ { |
||||
cmd_depth[128+8*i] = depth[i] |
||||
cmd_depth[256+8*i] = depth[8+i] |
||||
cmd_depth[448+8*i] = depth[16+i] |
||||
} |
||||
|
||||
storeHuffmanTree(cmd_depth[:], numCommandSymbols, tree[:], storage_ix, storage) |
||||
} |
||||
|
||||
storeHuffmanTree(depth[64:], 64, tree[:], storage_ix, storage) |
||||
} |
||||
|
||||
func emitInsertLen(insertlen uint32, commands *[]uint32) { |
||||
if insertlen < 6 { |
||||
(*commands)[0] = insertlen |
||||
} else if insertlen < 130 { |
||||
var tail uint32 = insertlen - 2 |
||||
var nbits uint32 = log2FloorNonZero(uint(tail)) - 1 |
||||
var prefix uint32 = tail >> nbits |
||||
var inscode uint32 = (nbits << 1) + prefix + 2 |
||||
var extra uint32 = tail - (prefix << nbits) |
||||
(*commands)[0] = inscode | extra<<8 |
||||
} else if insertlen < 2114 { |
||||
var tail uint32 = insertlen - 66 |
||||
var nbits uint32 = log2FloorNonZero(uint(tail)) |
||||
var code uint32 = nbits + 10 |
||||
var extra uint32 = tail - (1 << nbits) |
||||
(*commands)[0] = code | extra<<8 |
||||
} else if insertlen < 6210 { |
||||
var extra uint32 = insertlen - 2114 |
||||
(*commands)[0] = 21 | extra<<8 |
||||
} else if insertlen < 22594 { |
||||
var extra uint32 = insertlen - 6210 |
||||
(*commands)[0] = 22 | extra<<8 |
||||
} else { |
||||
var extra uint32 = insertlen - 22594 |
||||
(*commands)[0] = 23 | extra<<8 |
||||
} |
||||
|
||||
*commands = (*commands)[1:] |
||||
} |
||||
|
||||
func emitCopyLen(copylen uint, commands *[]uint32) { |
||||
if copylen < 10 { |
||||
(*commands)[0] = uint32(copylen + 38) |
||||
} else if copylen < 134 { |
||||
var tail uint = copylen - 6 |
||||
var nbits uint = uint(log2FloorNonZero(tail) - 1) |
||||
var prefix uint = tail >> nbits |
||||
var code uint = (nbits << 1) + prefix + 44 |
||||
var extra uint = tail - (prefix << nbits) |
||||
(*commands)[0] = uint32(code | extra<<8) |
||||
} else if copylen < 2118 { |
||||
var tail uint = copylen - 70 |
||||
var nbits uint = uint(log2FloorNonZero(tail)) |
||||
var code uint = nbits + 52 |
||||
var extra uint = tail - (uint(1) << nbits) |
||||
(*commands)[0] = uint32(code | extra<<8) |
||||
} else { |
||||
var extra uint = copylen - 2118 |
||||
(*commands)[0] = uint32(63 | extra<<8) |
||||
} |
||||
|
||||
*commands = (*commands)[1:] |
||||
} |
||||
|
||||
func emitCopyLenLastDistance(copylen uint, commands *[]uint32) { |
||||
if copylen < 12 { |
||||
(*commands)[0] = uint32(copylen + 20) |
||||
*commands = (*commands)[1:] |
||||
} else if copylen < 72 { |
||||
var tail uint = copylen - 8 |
||||
var nbits uint = uint(log2FloorNonZero(tail) - 1) |
||||
var prefix uint = tail >> nbits |
||||
var code uint = (nbits << 1) + prefix + 28 |
||||
var extra uint = tail - (prefix << nbits) |
||||
(*commands)[0] = uint32(code | extra<<8) |
||||
*commands = (*commands)[1:] |
||||
} else if copylen < 136 { |
||||
var tail uint = copylen - 8 |
||||
var code uint = (tail >> 5) + 54 |
||||
var extra uint = tail & 31 |
||||
(*commands)[0] = uint32(code | extra<<8) |
||||
*commands = (*commands)[1:] |
||||
(*commands)[0] = 64 |
||||
*commands = (*commands)[1:] |
||||
} else if copylen < 2120 { |
||||
var tail uint = copylen - 72 |
||||
var nbits uint = uint(log2FloorNonZero(tail)) |
||||
var code uint = nbits + 52 |
||||
var extra uint = tail - (uint(1) << nbits) |
||||
(*commands)[0] = uint32(code | extra<<8) |
||||
*commands = (*commands)[1:] |
||||
(*commands)[0] = 64 |
||||
*commands = (*commands)[1:] |
||||
} else { |
||||
var extra uint = copylen - 2120 |
||||
(*commands)[0] = uint32(63 | extra<<8) |
||||
*commands = (*commands)[1:] |
||||
(*commands)[0] = 64 |
||||
*commands = (*commands)[1:] |
||||
} |
||||
} |
||||
|
||||
func emitDistance(distance uint32, commands *[]uint32) { |
||||
var d uint32 = distance + 3 |
||||
var nbits uint32 = log2FloorNonZero(uint(d)) - 1 |
||||
var prefix uint32 = (d >> nbits) & 1 |
||||
var offset uint32 = (2 + prefix) << nbits |
||||
var distcode uint32 = 2*(nbits-1) + prefix + 80 |
||||
var extra uint32 = d - offset |
||||
(*commands)[0] = distcode | extra<<8 |
||||
*commands = (*commands)[1:] |
||||
} |
||||
|
||||
/* REQUIRES: len <= 1 << 24. */ |
||||
func storeMetaBlockHeader(len uint, is_uncompressed bool, storage_ix *uint, storage []byte) { |
||||
var nibbles uint = 6 |
||||
|
||||
/* ISLAST */ |
||||
writeBits(1, 0, storage_ix, storage) |
||||
|
||||
if len <= 1<<16 { |
||||
nibbles = 4 |
||||
} else if len <= 1<<20 { |
||||
nibbles = 5 |
||||
} |
||||
|
||||
writeBits(2, uint64(nibbles)-4, storage_ix, storage) |
||||
writeBits(nibbles*4, uint64(len)-1, storage_ix, storage) |
||||
|
||||
/* ISUNCOMPRESSED */ |
||||
writeSingleBit(is_uncompressed, storage_ix, storage) |
||||
} |
||||
|
||||
func createCommands(input []byte, block_size uint, input_size uint, base_ip_ptr []byte, table []int, table_bits uint, min_match uint, literals *[]byte, commands *[]uint32) { |
||||
var ip int = 0 |
||||
var shift uint = 64 - table_bits |
||||
var ip_end int = int(block_size) |
||||
var base_ip int = -cap(base_ip_ptr) + cap(input) |
||||
var next_emit int = 0 |
||||
var last_distance int = -1 |
||||
/* "ip" is the input pointer. */ |
||||
|
||||
const kInputMarginBytes uint = windowGap |
||||
|
||||
/* "next_emit" is a pointer to the first byte that is not covered by a |
||||
previous copy. Bytes between "next_emit" and the start of the next copy or |
||||
the end of the input will be emitted as literal bytes. */ |
||||
if block_size >= kInputMarginBytes { |
||||
var len_limit uint = brotli_min_size_t(block_size-min_match, input_size-kInputMarginBytes) |
||||
var ip_limit int = int(len_limit) |
||||
/* For the last block, we need to keep a 16 bytes margin so that we can be |
||||
sure that all distances are at most window size - 16. |
||||
For all other blocks, we only need to keep a margin of 5 bytes so that |
||||
we don't go over the block size with a copy. */ |
||||
|
||||
var next_hash uint32 |
||||
ip++ |
||||
for next_hash = hash1(input[ip:], shift, min_match); ; { |
||||
var skip uint32 = 32 |
||||
var next_ip int = ip |
||||
/* Step 1: Scan forward in the input looking for a 6-byte-long match. |
||||
If we get close to exhausting the input then goto emit_remainder. |
||||
|
||||
Heuristic match skipping: If 32 bytes are scanned with no matches |
||||
found, start looking only at every other byte. If 32 more bytes are |
||||
scanned, look at every third byte, etc.. When a match is found, |
||||
immediately go back to looking at every byte. This is a small loss |
||||
(~5% performance, ~0.1% density) for compressible data due to more |
||||
bookkeeping, but for non-compressible data (such as JPEG) it's a huge |
||||
win since the compressor quickly "realizes" the data is incompressible |
||||
and doesn't bother looking for matches everywhere. |
||||
|
||||
The "skip" variable keeps track of how many bytes there are since the |
||||
last match; dividing it by 32 (ie. right-shifting by five) gives the |
||||
number of bytes to move ahead for each iteration. */ |
||||
|
||||
var candidate int |
||||
|
||||
assert(next_emit < ip) |
||||
|
||||
trawl: |
||||
for { |
||||
var hash uint32 = next_hash |
||||
var bytes_between_hash_lookups uint32 = skip >> 5 |
||||
skip++ |
||||
ip = next_ip |
||||
assert(hash == hash1(input[ip:], shift, min_match)) |
||||
next_ip = int(uint32(ip) + bytes_between_hash_lookups) |
||||
if next_ip > ip_limit { |
||||
goto emit_remainder |
||||
} |
||||
|
||||
next_hash = hash1(input[next_ip:], shift, min_match) |
||||
candidate = ip - last_distance |
||||
if isMatch1(input[ip:], base_ip_ptr[candidate-base_ip:], min_match) { |
||||
if candidate < ip { |
||||
table[hash] = int(ip - base_ip) |
||||
break |
||||
} |
||||
} |
||||
|
||||
candidate = base_ip + table[hash] |
||||
assert(candidate >= base_ip) |
||||
assert(candidate < ip) |
||||
|
||||
table[hash] = int(ip - base_ip) |
||||
if isMatch1(input[ip:], base_ip_ptr[candidate-base_ip:], min_match) { |
||||
break |
||||
} |
||||
} |
||||
|
||||
/* Check copy distance. If candidate is not feasible, continue search. |
||||
Checking is done outside of hot loop to reduce overhead. */ |
||||
if ip-candidate > maxDistance_compress_fragment { |
||||
goto trawl |
||||
} |
||||
|
||||
/* Step 2: Emit the found match together with the literal bytes from |
||||
"next_emit", and then see if we can find a next match immediately |
||||
afterwards. Repeat until we find no match for the input |
||||
without emitting some literal bytes. */ |
||||
{ |
||||
var base int = ip |
||||
/* > 0 */ |
||||
var matched uint = min_match + findMatchLengthWithLimit(base_ip_ptr[uint(candidate-base_ip)+min_match:], input[uint(ip)+min_match:], uint(ip_end-ip)-min_match) |
||||
var distance int = int(base - candidate) |
||||
/* We have a 6-byte match at ip, and we need to emit bytes in |
||||
[next_emit, ip). */ |
||||
|
||||
var insert int = int(base - next_emit) |
||||
ip += int(matched) |
||||
emitInsertLen(uint32(insert), commands) |
||||
copy(*literals, input[next_emit:][:uint(insert)]) |
||||
*literals = (*literals)[insert:] |
||||
if distance == last_distance { |
||||
(*commands)[0] = 64 |
||||
*commands = (*commands)[1:] |
||||
} else { |
||||
emitDistance(uint32(distance), commands) |
||||
last_distance = distance |
||||
} |
||||
|
||||
emitCopyLenLastDistance(matched, commands) |
||||
|
||||
next_emit = ip |
||||
if ip >= ip_limit { |
||||
goto emit_remainder |
||||
} |
||||
{ |
||||
var input_bytes uint64 |
||||
var cur_hash uint32 |
||||
/* We could immediately start working at ip now, but to improve |
||||
compression we first update "table" with the hashes of some |
||||
positions within the last copy. */ |
||||
|
||||
var prev_hash uint32 |
||||
if min_match == 4 { |
||||
input_bytes = binary.LittleEndian.Uint64(input[ip-3:]) |
||||
cur_hash = hashBytesAtOffset(input_bytes, 3, shift, min_match) |
||||
prev_hash = hashBytesAtOffset(input_bytes, 0, shift, min_match) |
||||
table[prev_hash] = int(ip - base_ip - 3) |
||||
prev_hash = hashBytesAtOffset(input_bytes, 1, shift, min_match) |
||||
table[prev_hash] = int(ip - base_ip - 2) |
||||
prev_hash = hashBytesAtOffset(input_bytes, 0, shift, min_match) |
||||
table[prev_hash] = int(ip - base_ip - 1) |
||||
} else { |
||||
input_bytes = binary.LittleEndian.Uint64(input[ip-5:]) |
||||
prev_hash = hashBytesAtOffset(input_bytes, 0, shift, min_match) |
||||
table[prev_hash] = int(ip - base_ip - 5) |
||||
prev_hash = hashBytesAtOffset(input_bytes, 1, shift, min_match) |
||||
table[prev_hash] = int(ip - base_ip - 4) |
||||
prev_hash = hashBytesAtOffset(input_bytes, 2, shift, min_match) |
||||
table[prev_hash] = int(ip - base_ip - 3) |
||||
input_bytes = binary.LittleEndian.Uint64(input[ip-2:]) |
||||
cur_hash = hashBytesAtOffset(input_bytes, 2, shift, min_match) |
||||
prev_hash = hashBytesAtOffset(input_bytes, 0, shift, min_match) |
||||
table[prev_hash] = int(ip - base_ip - 2) |
||||
prev_hash = hashBytesAtOffset(input_bytes, 1, shift, min_match) |
||||
table[prev_hash] = int(ip - base_ip - 1) |
||||
} |
||||
|
||||
candidate = base_ip + table[cur_hash] |
||||
table[cur_hash] = int(ip - base_ip) |
||||
} |
||||
} |
||||
|
||||
for ip-candidate <= maxDistance_compress_fragment && isMatch1(input[ip:], base_ip_ptr[candidate-base_ip:], min_match) { |
||||
var base int = ip |
||||
/* We have a 6-byte match at ip, and no need to emit any |
||||
literal bytes prior to ip. */ |
||||
|
||||
var matched uint = min_match + findMatchLengthWithLimit(base_ip_ptr[uint(candidate-base_ip)+min_match:], input[uint(ip)+min_match:], uint(ip_end-ip)-min_match) |
||||
ip += int(matched) |
||||
last_distance = int(base - candidate) /* > 0 */ |
||||
emitCopyLen(matched, commands) |
||||
emitDistance(uint32(last_distance), commands) |
||||
|
||||
next_emit = ip |
||||
if ip >= ip_limit { |
||||
goto emit_remainder |
||||
} |
||||
{ |
||||
var input_bytes uint64 |
||||
var cur_hash uint32 |
||||
/* We could immediately start working at ip now, but to improve |
||||
compression we first update "table" with the hashes of some |
||||
positions within the last copy. */ |
||||
|
||||
var prev_hash uint32 |
||||
if min_match == 4 { |
||||
input_bytes = binary.LittleEndian.Uint64(input[ip-3:]) |
||||
cur_hash = hashBytesAtOffset(input_bytes, 3, shift, min_match) |
||||
prev_hash = hashBytesAtOffset(input_bytes, 0, shift, min_match) |
||||
table[prev_hash] = int(ip - base_ip - 3) |
||||
prev_hash = hashBytesAtOffset(input_bytes, 1, shift, min_match) |
||||
table[prev_hash] = int(ip - base_ip - 2) |
||||
prev_hash = hashBytesAtOffset(input_bytes, 2, shift, min_match) |
||||
table[prev_hash] = int(ip - base_ip - 1) |
||||
} else { |
||||
input_bytes = binary.LittleEndian.Uint64(input[ip-5:]) |
||||
prev_hash = hashBytesAtOffset(input_bytes, 0, shift, min_match) |
||||
table[prev_hash] = int(ip - base_ip - 5) |
||||
prev_hash = hashBytesAtOffset(input_bytes, 1, shift, min_match) |
||||
table[prev_hash] = int(ip - base_ip - 4) |
||||
prev_hash = hashBytesAtOffset(input_bytes, 2, shift, min_match) |
||||
table[prev_hash] = int(ip - base_ip - 3) |
||||
input_bytes = binary.LittleEndian.Uint64(input[ip-2:]) |
||||
cur_hash = hashBytesAtOffset(input_bytes, 2, shift, min_match) |
||||
prev_hash = hashBytesAtOffset(input_bytes, 0, shift, min_match) |
||||
table[prev_hash] = int(ip - base_ip - 2) |
||||
prev_hash = hashBytesAtOffset(input_bytes, 1, shift, min_match) |
||||
table[prev_hash] = int(ip - base_ip - 1) |
||||
} |
||||
|
||||
candidate = base_ip + table[cur_hash] |
||||
table[cur_hash] = int(ip - base_ip) |
||||
} |
||||
} |
||||
|
||||
ip++ |
||||
next_hash = hash1(input[ip:], shift, min_match) |
||||
} |
||||
} |
||||
|
||||
emit_remainder: |
||||
assert(next_emit <= ip_end) |
||||
|
||||
/* Emit the remaining bytes as literals. */ |
||||
if next_emit < ip_end { |
||||
var insert uint32 = uint32(ip_end - next_emit) |
||||
emitInsertLen(insert, commands) |
||||
copy(*literals, input[next_emit:][:insert]) |
||||
*literals = (*literals)[insert:] |
||||
} |
||||
} |
||||
|
||||
var storeCommands_kNumExtraBits = [128]uint32{ |
||||
0, |
||||
0, |
||||
0, |
||||
0, |
||||
0, |
||||
0, |
||||
1, |
||||
1, |
||||
2, |
||||
2, |
||||
3, |
||||
3, |
||||
4, |
||||
4, |
||||
5, |
||||
5, |
||||
6, |
||||
7, |
||||
8, |
||||
9, |
||||
10, |
||||
12, |
||||
14, |
||||
24, |
||||
0, |
||||
0, |
||||
0, |
||||
0, |
||||
0, |
||||
0, |
||||
0, |
||||
0, |
||||
1, |
||||
1, |
||||
2, |
||||
2, |
||||
3, |
||||
3, |
||||
4, |
||||
4, |
||||
0, |
||||
0, |
||||
0, |
||||
0, |
||||
0, |
||||
0, |
||||
0, |
||||
0, |
||||
1, |
||||
1, |
||||
2, |
||||
2, |
||||
3, |
||||
3, |
||||
4, |
||||
4, |
||||
5, |
||||
5, |
||||
6, |
||||
7, |
||||
8, |
||||
9, |
||||
10, |
||||
24, |
||||
0, |
||||
0, |
||||
0, |
||||
0, |
||||
0, |
||||
0, |
||||
0, |
||||
0, |
||||
0, |
||||
0, |
||||
0, |
||||
0, |
||||
0, |
||||
0, |
||||
0, |
||||
0, |
||||
1, |
||||
1, |
||||
2, |
||||
2, |
||||
3, |
||||
3, |
||||
4, |
||||
4, |
||||
5, |
||||
5, |
||||
6, |
||||
6, |
||||
7, |
||||
7, |
||||
8, |
||||
8, |
||||
9, |
||||
9, |
||||
10, |
||||
10, |
||||
11, |
||||
11, |
||||
12, |
||||
12, |
||||
13, |
||||
13, |
||||
14, |
||||
14, |
||||
15, |
||||
15, |
||||
16, |
||||
16, |
||||
17, |
||||
17, |
||||
18, |
||||
18, |
||||
19, |
||||
19, |
||||
20, |
||||
20, |
||||
21, |
||||
21, |
||||
22, |
||||
22, |
||||
23, |
||||
23, |
||||
24, |
||||
24, |
||||
} |
||||
var storeCommands_kInsertOffset = [24]uint32{ |
||||
0, |
||||
1, |
||||
2, |
||||
3, |
||||
4, |
||||
5, |
||||
6, |
||||
8, |
||||
10, |
||||
14, |
||||
18, |
||||
26, |
||||
34, |
||||
50, |
||||
66, |
||||
98, |
||||
130, |
||||
194, |
||||
322, |
||||
578, |
||||
1090, |
||||
2114, |
||||
6210, |
||||
22594, |
||||
} |
||||
|
||||
func storeCommands(literals []byte, num_literals uint, commands []uint32, num_commands uint, storage_ix *uint, storage []byte) { |
||||
var lit_depths [256]byte |
||||
var lit_bits [256]uint16 |
||||
var lit_histo = [256]uint32{0} |
||||
var cmd_depths = [128]byte{0} |
||||
var cmd_bits = [128]uint16{0} |
||||
var cmd_histo = [128]uint32{0} |
||||
var i uint |
||||
for i = 0; i < num_literals; i++ { |
||||
lit_histo[literals[i]]++ |
||||
} |
||||
|
||||
buildAndStoreHuffmanTreeFast(lit_histo[:], num_literals, /* max_bits = */ |
||||
8, lit_depths[:], lit_bits[:], storage_ix, storage) |
||||
|
||||
for i = 0; i < num_commands; i++ { |
||||
var code uint32 = commands[i] & 0xFF |
||||
assert(code < 128) |
||||
cmd_histo[code]++ |
||||
} |
||||
|
||||
cmd_histo[1] += 1 |
||||
cmd_histo[2] += 1 |
||||
cmd_histo[64] += 1 |
||||
cmd_histo[84] += 1 |
||||
buildAndStoreCommandPrefixCode(cmd_histo[:], cmd_depths[:], cmd_bits[:], storage_ix, storage) |
||||
|
||||
for i = 0; i < num_commands; i++ { |
||||
var cmd uint32 = commands[i] |
||||
var code uint32 = cmd & 0xFF |
||||
var extra uint32 = cmd >> 8 |
||||
assert(code < 128) |
||||
writeBits(uint(cmd_depths[code]), uint64(cmd_bits[code]), storage_ix, storage) |
||||
writeBits(uint(storeCommands_kNumExtraBits[code]), uint64(extra), storage_ix, storage) |
||||
if code < 24 { |
||||
var insert uint32 = storeCommands_kInsertOffset[code] + extra |
||||
var j uint32 |
||||
for j = 0; j < insert; j++ { |
||||
var lit byte = literals[0] |
||||
writeBits(uint(lit_depths[lit]), uint64(lit_bits[lit]), storage_ix, storage) |
||||
literals = literals[1:] |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
/* Acceptable loss for uncompressible speedup is 2% */ |
||||
const minRatio = 0.98 |
||||
|
||||
const sampleRate = 43 |
||||
|
||||
func shouldCompress(input []byte, input_size uint, num_literals uint) bool { |
||||
var corpus_size float64 = float64(input_size) |
||||
if float64(num_literals) < minRatio*corpus_size { |
||||
return true |
||||
} else { |
||||
var literal_histo = [256]uint32{0} |
||||
var max_total_bit_cost float64 = corpus_size * 8 * minRatio / sampleRate |
||||
var i uint |
||||
for i = 0; i < input_size; i += sampleRate { |
||||
literal_histo[input[i]]++ |
||||
} |
||||
|
||||
return bitsEntropy(literal_histo[:], 256) < max_total_bit_cost |
||||
} |
||||
} |
||||
|
||||
func rewindBitPosition(new_storage_ix uint, storage_ix *uint, storage []byte) { |
||||
var bitpos uint = new_storage_ix & 7 |
||||
var mask uint = (1 << bitpos) - 1 |
||||
storage[new_storage_ix>>3] &= byte(mask) |
||||
*storage_ix = new_storage_ix |
||||
} |
||||
|
||||
func emitUncompressedMetaBlock(input []byte, input_size uint, storage_ix *uint, storage []byte) { |
||||
storeMetaBlockHeader(input_size, true, storage_ix, storage) |
||||
*storage_ix = (*storage_ix + 7) &^ 7 |
||||
copy(storage[*storage_ix>>3:], input[:input_size]) |
||||
*storage_ix += input_size << 3 |
||||
storage[*storage_ix>>3] = 0 |
||||
} |
||||
|
||||
func compressFragmentTwoPassImpl(input []byte, input_size uint, is_last bool, command_buf []uint32, literal_buf []byte, table []int, table_bits uint, min_match uint, storage_ix *uint, storage []byte) { |
||||
/* Save the start of the first block for position and distance computations. |
||||
*/ |
||||
var base_ip []byte = input |
||||
|
||||
for input_size > 0 { |
||||
var block_size uint = brotli_min_size_t(input_size, kCompressFragmentTwoPassBlockSize) |
||||
var commands []uint32 = command_buf |
||||
var literals []byte = literal_buf |
||||
var num_literals uint |
||||
createCommands(input, block_size, input_size, base_ip, table, table_bits, min_match, &literals, &commands) |
||||
num_literals = uint(-cap(literals) + cap(literal_buf)) |
||||
if shouldCompress(input, block_size, num_literals) { |
||||
var num_commands uint = uint(-cap(commands) + cap(command_buf)) |
||||
storeMetaBlockHeader(block_size, false, storage_ix, storage) |
||||
|
||||
/* No block splits, no contexts. */ |
||||
writeBits(13, 0, storage_ix, storage) |
||||
|
||||
storeCommands(literal_buf, num_literals, command_buf, num_commands, storage_ix, storage) |
||||
} else { |
||||
/* Since we did not find many backward references and the entropy of |
||||
the data is close to 8 bits, we can simply emit an uncompressed block. |
||||
This makes compression speed of uncompressible data about 3x faster. */ |
||||
emitUncompressedMetaBlock(input, block_size, storage_ix, storage) |
||||
} |
||||
|
||||
input = input[block_size:] |
||||
input_size -= block_size |
||||
} |
||||
} |
||||
|
||||
/* Compresses "input" string to the "*storage" buffer as one or more complete |
||||
meta-blocks, and updates the "*storage_ix" bit position. |
||||
|
||||
If "is_last" is 1, emits an additional empty last meta-block. |
||||
|
||||
REQUIRES: "input_size" is greater than zero, or "is_last" is 1. |
||||
REQUIRES: "input_size" is less or equal to maximal metablock size (1 << 24). |
||||
REQUIRES: "command_buf" and "literal_buf" point to at least |
||||
kCompressFragmentTwoPassBlockSize long arrays. |
||||
REQUIRES: All elements in "table[0..table_size-1]" are initialized to zero. |
||||
REQUIRES: "table_size" is a power of two |
||||
OUTPUT: maximal copy distance <= |input_size| |
||||
OUTPUT: maximal copy distance <= BROTLI_MAX_BACKWARD_LIMIT(18) */ |
||||
func compressFragmentTwoPass(input []byte, input_size uint, is_last bool, command_buf []uint32, literal_buf []byte, table []int, table_size uint, storage_ix *uint, storage []byte) { |
||||
var initial_storage_ix uint = *storage_ix |
||||
var table_bits uint = uint(log2FloorNonZero(table_size)) |
||||
var min_match uint |
||||
if table_bits <= 15 { |
||||
min_match = 4 |
||||
} else { |
||||
min_match = 6 |
||||
} |
||||
compressFragmentTwoPassImpl(input, input_size, is_last, command_buf, literal_buf, table, table_bits, min_match, storage_ix, storage) |
||||
|
||||
/* If output is larger than single uncompressed block, rewrite it. */ |
||||
if *storage_ix-initial_storage_ix > 31+(input_size<<3) { |
||||
rewindBitPosition(initial_storage_ix, storage_ix, storage) |
||||
emitUncompressedMetaBlock(input, input_size, storage_ix, storage) |
||||
} |
||||
|
||||
if is_last { |
||||
writeBits(1, 1, storage_ix, storage) /* islast */ |
||||
writeBits(1, 1, storage_ix, storage) /* isempty */ |
||||
*storage_ix = (*storage_ix + 7) &^ 7 |
||||
} |
||||
} |
||||
@ -0,0 +1,77 @@ |
||||
package brotli |
||||
|
||||
/* Copyright 2016 Google Inc. All Rights Reserved. |
||||
|
||||
Distributed under MIT license. |
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/ |
||||
|
||||
/* Specification: 7.3. Encoding of the context map */ |
||||
const contextMapMaxRle = 16 |
||||
|
||||
/* Specification: 2. Compressed representation overview */ |
||||
const maxNumberOfBlockTypes = 256 |
||||
|
||||
/* Specification: 3.3. Alphabet sizes: insert-and-copy length */ |
||||
const numLiteralSymbols = 256 |
||||
|
||||
const numCommandSymbols = 704 |
||||
|
||||
const numBlockLenSymbols = 26 |
||||
|
||||
const maxContextMapSymbols = (maxNumberOfBlockTypes + contextMapMaxRle) |
||||
|
||||
const maxBlockTypeSymbols = (maxNumberOfBlockTypes + 2) |
||||
|
||||
/* Specification: 3.5. Complex prefix codes */ |
||||
const repeatPreviousCodeLength = 16 |
||||
|
||||
const repeatZeroCodeLength = 17 |
||||
|
||||
const codeLengthCodes = (repeatZeroCodeLength + 1) |
||||
|
||||
/* "code length of 8 is repeated" */ |
||||
const initialRepeatedCodeLength = 8 |
||||
|
||||
/* "Large Window Brotli" */ |
||||
const largeMaxDistanceBits = 62 |
||||
|
||||
const largeMinWbits = 10 |
||||
|
||||
const largeMaxWbits = 30 |
||||
|
||||
/* Specification: 4. Encoding of distances */ |
||||
const numDistanceShortCodes = 16 |
||||
|
||||
const maxNpostfix = 3 |
||||
|
||||
const maxNdirect = 120 |
||||
|
||||
const maxDistanceBits = 24 |
||||
|
||||
func distanceAlphabetSize(NPOSTFIX uint, NDIRECT uint, MAXNBITS uint) uint { |
||||
return numDistanceShortCodes + NDIRECT + uint(MAXNBITS<<(NPOSTFIX+1)) |
||||
} |
||||
|
||||
/* numDistanceSymbols == 1128 */ |
||||
const numDistanceSymbols = 1128 |
||||
|
||||
const maxDistance = 0x3FFFFFC |
||||
|
||||
const maxAllowedDistance = 0x7FFFFFFC |
||||
|
||||
/* 7.1. Context modes and context ID lookup for literals */ |
||||
/* "context IDs for literals are in the range of 0..63" */ |
||||
const literalContextBits = 6 |
||||
|
||||
/* 7.2. Context ID for distances */ |
||||
const distanceContextBits = 2 |
||||
|
||||
/* 9.1. Format of the Stream Header */ |
||||
/* Number of slack bytes for window size. Don't confuse |
||||
with BROTLI_NUM_DISTANCE_SHORT_CODES. */ |
||||
const windowGap = 16 |
||||
|
||||
func maxBackwardLimit(W uint) uint { |
||||
return (uint(1) << W) - windowGap |
||||
} |
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,22 @@ |
||||
package brotli |
||||
|
||||
/* Dictionary data (words and transforms) for 1 possible context */ |
||||
type encoderDictionary struct { |
||||
words *dictionary |
||||
cutoffTransformsCount uint32 |
||||
cutoffTransforms uint64 |
||||
hash_table []uint16 |
||||
buckets []uint16 |
||||
dict_words []dictWord |
||||
} |
||||
|
||||
func initEncoderDictionary(dict *encoderDictionary) { |
||||
dict.words = getDictionary() |
||||
|
||||
dict.hash_table = kStaticDictionaryHash[:] |
||||
dict.buckets = kStaticDictionaryBuckets[:] |
||||
dict.dict_words = kStaticDictionaryWords[:] |
||||
|
||||
dict.cutoffTransformsCount = kCutoffTransformsCount |
||||
dict.cutoffTransforms = kCutoffTransforms |
||||
} |
||||
@ -0,0 +1,593 @@ |
||||
package brotli |
||||
|
||||
import "math" |
||||
|
||||
/* Copyright 2010 Google Inc. All Rights Reserved. |
||||
|
||||
Distributed under MIT license. |
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/ |
||||
|
||||
/* Entropy encoding (Huffman) utilities. */ |
||||
|
||||
/* A node of a Huffman tree. */ |
||||
type huffmanTree struct { |
||||
total_count_ uint32 |
||||
index_left_ int16 |
||||
index_right_or_value_ int16 |
||||
} |
||||
|
||||
func initHuffmanTree(self *huffmanTree, count uint32, left int16, right int16) { |
||||
self.total_count_ = count |
||||
self.index_left_ = left |
||||
self.index_right_or_value_ = right |
||||
} |
||||
|
||||
/* Input size optimized Shell sort. */ |
||||
type huffmanTreeComparator func(*huffmanTree, *huffmanTree) bool |
||||
|
||||
var sortHuffmanTreeItems_gaps = []uint{132, 57, 23, 10, 4, 1} |
||||
|
||||
func sortHuffmanTreeItems(items []huffmanTree, n uint, comparator huffmanTreeComparator) { |
||||
if n < 13 { |
||||
/* Insertion sort. */ |
||||
var i uint |
||||
for i = 1; i < n; i++ { |
||||
var tmp huffmanTree = items[i] |
||||
var k uint = i |
||||
var j uint = i - 1 |
||||
for comparator(&tmp, &items[j]) { |
||||
items[k] = items[j] |
||||
k = j |
||||
tmp10 := j |
||||
j-- |
||||
if tmp10 == 0 { |
||||
break |
||||
} |
||||
} |
||||
|
||||
items[k] = tmp |
||||
} |
||||
|
||||
return |
||||
} else { |
||||
var g int |
||||
if n < 57 { |
||||
g = 2 |
||||
} else { |
||||
g = 0 |
||||
} |
||||
for ; g < 6; g++ { |
||||
var gap uint = sortHuffmanTreeItems_gaps[g] |
||||
var i uint |
||||
for i = gap; i < n; i++ { |
||||
var j uint = i |
||||
var tmp huffmanTree = items[i] |
||||
for ; j >= gap && comparator(&tmp, &items[j-gap]); j -= gap { |
||||
items[j] = items[j-gap] |
||||
} |
||||
|
||||
items[j] = tmp |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
/* Returns 1 if assignment of depths succeeded, otherwise 0. */ |
||||
func setDepth(p0 int, pool []huffmanTree, depth []byte, max_depth int) bool { |
||||
var stack [16]int |
||||
var level int = 0 |
||||
var p int = p0 |
||||
assert(max_depth <= 15) |
||||
stack[0] = -1 |
||||
for { |
||||
if pool[p].index_left_ >= 0 { |
||||
level++ |
||||
if level > max_depth { |
||||
return false |
||||
} |
||||
stack[level] = int(pool[p].index_right_or_value_) |
||||
p = int(pool[p].index_left_) |
||||
continue |
||||
} else { |
||||
depth[pool[p].index_right_or_value_] = byte(level) |
||||
} |
||||
|
||||
for level >= 0 && stack[level] == -1 { |
||||
level-- |
||||
} |
||||
if level < 0 { |
||||
return true |
||||
} |
||||
p = stack[level] |
||||
stack[level] = -1 |
||||
} |
||||
} |
||||
|
||||
/* Sort the root nodes, least popular first. */ |
||||
func sortHuffmanTree(v0 *huffmanTree, v1 *huffmanTree) bool { |
||||
if v0.total_count_ != v1.total_count_ { |
||||
return v0.total_count_ < v1.total_count_ |
||||
} |
||||
|
||||
return v0.index_right_or_value_ > v1.index_right_or_value_ |
||||
} |
||||
|
||||
/* This function will create a Huffman tree. |
||||
|
||||
The catch here is that the tree cannot be arbitrarily deep. |
||||
Brotli specifies a maximum depth of 15 bits for "code trees" |
||||
and 7 bits for "code length code trees." |
||||
|
||||
count_limit is the value that is to be faked as the minimum value |
||||
and this minimum value is raised until the tree matches the |
||||
maximum length requirement. |
||||
|
||||
This algorithm is not of excellent performance for very long data blocks, |
||||
especially when population counts are longer than 2**tree_limit, but |
||||
we are not planning to use this with extremely long blocks. |
||||
|
||||
See http://en.wikipedia.org/wiki/Huffman_coding */
|
||||
func createHuffmanTree(data []uint32, length uint, tree_limit int, tree []huffmanTree, depth []byte) { |
||||
var count_limit uint32 |
||||
var sentinel huffmanTree |
||||
initHuffmanTree(&sentinel, math.MaxUint32, -1, -1) |
||||
|
||||
/* For block sizes below 64 kB, we never need to do a second iteration |
||||
of this loop. Probably all of our block sizes will be smaller than |
||||
that, so this loop is mostly of academic interest. If we actually |
||||
would need this, we would be better off with the Katajainen algorithm. */ |
||||
for count_limit = 1; ; count_limit *= 2 { |
||||
var n uint = 0 |
||||
var i uint |
||||
var j uint |
||||
var k uint |
||||
for i = length; i != 0; { |
||||
i-- |
||||
if data[i] != 0 { |
||||
var count uint32 = brotli_max_uint32_t(data[i], count_limit) |
||||
initHuffmanTree(&tree[n], count, -1, int16(i)) |
||||
n++ |
||||
} |
||||
} |
||||
|
||||
if n == 1 { |
||||
depth[tree[0].index_right_or_value_] = 1 /* Only one element. */ |
||||
break |
||||
} |
||||
|
||||
sortHuffmanTreeItems(tree, n, huffmanTreeComparator(sortHuffmanTree)) |
||||
|
||||
/* The nodes are: |
||||
[0, n): the sorted leaf nodes that we start with. |
||||
[n]: we add a sentinel here. |
||||
[n + 1, 2n): new parent nodes are added here, starting from |
||||
(n+1). These are naturally in ascending order. |
||||
[2n]: we add a sentinel at the end as well. |
||||
There will be (2n+1) elements at the end. */ |
||||
tree[n] = sentinel |
||||
|
||||
tree[n+1] = sentinel |
||||
|
||||
i = 0 /* Points to the next leaf node. */ |
||||
j = n + 1 /* Points to the next non-leaf node. */ |
||||
for k = n - 1; k != 0; k-- { |
||||
var left uint |
||||
var right uint |
||||
if tree[i].total_count_ <= tree[j].total_count_ { |
||||
left = i |
||||
i++ |
||||
} else { |
||||
left = j |
||||
j++ |
||||
} |
||||
|
||||
if tree[i].total_count_ <= tree[j].total_count_ { |
||||
right = i |
||||
i++ |
||||
} else { |
||||
right = j |
||||
j++ |
||||
} |
||||
{ |
||||
/* The sentinel node becomes the parent node. */ |
||||
var j_end uint = 2*n - k |
||||
tree[j_end].total_count_ = tree[left].total_count_ + tree[right].total_count_ |
||||
tree[j_end].index_left_ = int16(left) |
||||
tree[j_end].index_right_or_value_ = int16(right) |
||||
|
||||
/* Add back the last sentinel node. */ |
||||
tree[j_end+1] = sentinel |
||||
} |
||||
} |
||||
|
||||
if setDepth(int(2*n-1), tree[0:], depth, tree_limit) { |
||||
/* We need to pack the Huffman tree in tree_limit bits. If this was not |
||||
successful, add fake entities to the lowest values and retry. */ |
||||
break |
||||
} |
||||
} |
||||
} |
||||
|
||||
func reverse(v []byte, start uint, end uint) { |
||||
end-- |
||||
for start < end { |
||||
var tmp byte = v[start] |
||||
v[start] = v[end] |
||||
v[end] = tmp |
||||
start++ |
||||
end-- |
||||
} |
||||
} |
||||
|
||||
func writeHuffmanTreeRepetitions(previous_value byte, value byte, repetitions uint, tree_size *uint, tree []byte, extra_bits_data []byte) { |
||||
assert(repetitions > 0) |
||||
if previous_value != value { |
||||
tree[*tree_size] = value |
||||
extra_bits_data[*tree_size] = 0 |
||||
(*tree_size)++ |
||||
repetitions-- |
||||
} |
||||
|
||||
if repetitions == 7 { |
||||
tree[*tree_size] = value |
||||
extra_bits_data[*tree_size] = 0 |
||||
(*tree_size)++ |
||||
repetitions-- |
||||
} |
||||
|
||||
if repetitions < 3 { |
||||
var i uint |
||||
for i = 0; i < repetitions; i++ { |
||||
tree[*tree_size] = value |
||||
extra_bits_data[*tree_size] = 0 |
||||
(*tree_size)++ |
||||
} |
||||
} else { |
||||
var start uint = *tree_size |
||||
repetitions -= 3 |
||||
for { |
||||
tree[*tree_size] = repeatPreviousCodeLength |
||||
extra_bits_data[*tree_size] = byte(repetitions & 0x3) |
||||
(*tree_size)++ |
||||
repetitions >>= 2 |
||||
if repetitions == 0 { |
||||
break |
||||
} |
||||
|
||||
repetitions-- |
||||
} |
||||
|
||||
reverse(tree, start, *tree_size) |
||||
reverse(extra_bits_data, start, *tree_size) |
||||
} |
||||
} |
||||
|
||||
func writeHuffmanTreeRepetitionsZeros(repetitions uint, tree_size *uint, tree []byte, extra_bits_data []byte) { |
||||
if repetitions == 11 { |
||||
tree[*tree_size] = 0 |
||||
extra_bits_data[*tree_size] = 0 |
||||
(*tree_size)++ |
||||
repetitions-- |
||||
} |
||||
|
||||
if repetitions < 3 { |
||||
var i uint |
||||
for i = 0; i < repetitions; i++ { |
||||
tree[*tree_size] = 0 |
||||
extra_bits_data[*tree_size] = 0 |
||||
(*tree_size)++ |
||||
} |
||||
} else { |
||||
var start uint = *tree_size |
||||
repetitions -= 3 |
||||
for { |
||||
tree[*tree_size] = repeatZeroCodeLength |
||||
extra_bits_data[*tree_size] = byte(repetitions & 0x7) |
||||
(*tree_size)++ |
||||
repetitions >>= 3 |
||||
if repetitions == 0 { |
||||
break |
||||
} |
||||
|
||||
repetitions-- |
||||
} |
||||
|
||||
reverse(tree, start, *tree_size) |
||||
reverse(extra_bits_data, start, *tree_size) |
||||
} |
||||
} |
||||
|
||||
/* Change the population counts in a way that the consequent |
||||
Huffman tree compression, especially its RLE-part will be more |
||||
likely to compress this data more efficiently. |
||||
|
||||
length contains the size of the histogram. |
||||
counts contains the population counts. |
||||
good_for_rle is a buffer of at least length size */ |
||||
func optimizeHuffmanCountsForRLE(length uint, counts []uint32, good_for_rle []byte) { |
||||
var nonzero_count uint = 0 |
||||
var stride uint |
||||
var limit uint |
||||
var sum uint |
||||
var streak_limit uint = 1240 |
||||
var i uint |
||||
/* Let's make the Huffman code more compatible with RLE encoding. */ |
||||
for i = 0; i < length; i++ { |
||||
if counts[i] != 0 { |
||||
nonzero_count++ |
||||
} |
||||
} |
||||
|
||||
if nonzero_count < 16 { |
||||
return |
||||
} |
||||
|
||||
for length != 0 && counts[length-1] == 0 { |
||||
length-- |
||||
} |
||||
|
||||
if length == 0 { |
||||
return /* All zeros. */ |
||||
} |
||||
|
||||
/* Now counts[0..length - 1] does not have trailing zeros. */ |
||||
{ |
||||
var nonzeros uint = 0 |
||||
var smallest_nonzero uint32 = 1 << 30 |
||||
for i = 0; i < length; i++ { |
||||
if counts[i] != 0 { |
||||
nonzeros++ |
||||
if smallest_nonzero > counts[i] { |
||||
smallest_nonzero = counts[i] |
||||
} |
||||
} |
||||
} |
||||
|
||||
if nonzeros < 5 { |
||||
/* Small histogram will model it well. */ |
||||
return |
||||
} |
||||
|
||||
if smallest_nonzero < 4 { |
||||
var zeros uint = length - nonzeros |
||||
if zeros < 6 { |
||||
for i = 1; i < length-1; i++ { |
||||
if counts[i-1] != 0 && counts[i] == 0 && counts[i+1] != 0 { |
||||
counts[i] = 1 |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
if nonzeros < 28 { |
||||
return |
||||
} |
||||
} |
||||
|
||||
/* 2) Let's mark all population counts that already can be encoded |
||||
with an RLE code. */ |
||||
for i := 0; i < int(length); i++ { |
||||
good_for_rle[i] = 0 |
||||
} |
||||
{ |
||||
var symbol uint32 = counts[0] |
||||
/* Let's not spoil any of the existing good RLE codes. |
||||
Mark any seq of 0's that is longer as 5 as a good_for_rle. |
||||
Mark any seq of non-0's that is longer as 7 as a good_for_rle. */ |
||||
|
||||
var step uint = 0 |
||||
for i = 0; i <= length; i++ { |
||||
if i == length || counts[i] != symbol { |
||||
if (symbol == 0 && step >= 5) || (symbol != 0 && step >= 7) { |
||||
var k uint |
||||
for k = 0; k < step; k++ { |
||||
good_for_rle[i-k-1] = 1 |
||||
} |
||||
} |
||||
|
||||
step = 1 |
||||
if i != length { |
||||
symbol = counts[i] |
||||
} |
||||
} else { |
||||
step++ |
||||
} |
||||
} |
||||
} |
||||
|
||||
/* 3) Let's replace those population counts that lead to more RLE codes. |
||||
Math here is in 24.8 fixed point representation. */ |
||||
stride = 0 |
||||
|
||||
limit = uint(256*(counts[0]+counts[1]+counts[2])/3 + 420) |
||||
sum = 0 |
||||
for i = 0; i <= length; i++ { |
||||
if i == length || good_for_rle[i] != 0 || (i != 0 && good_for_rle[i-1] != 0) || (256*counts[i]-uint32(limit)+uint32(streak_limit)) >= uint32(2*streak_limit) { |
||||
if stride >= 4 || (stride >= 3 && sum == 0) { |
||||
var k uint |
||||
var count uint = (sum + stride/2) / stride |
||||
/* The stride must end, collapse what we have, if we have enough (4). */ |
||||
if count == 0 { |
||||
count = 1 |
||||
} |
||||
|
||||
if sum == 0 { |
||||
/* Don't make an all zeros stride to be upgraded to ones. */ |
||||
count = 0 |
||||
} |
||||
|
||||
for k = 0; k < stride; k++ { |
||||
/* We don't want to change value at counts[i], |
||||
that is already belonging to the next stride. Thus - 1. */ |
||||
counts[i-k-1] = uint32(count) |
||||
} |
||||
} |
||||
|
||||
stride = 0 |
||||
sum = 0 |
||||
if i < length-2 { |
||||
/* All interesting strides have a count of at least 4, */ |
||||
/* at least when non-zeros. */ |
||||
limit = uint(256*(counts[i]+counts[i+1]+counts[i+2])/3 + 420) |
||||
} else if i < length { |
||||
limit = uint(256 * counts[i]) |
||||
} else { |
||||
limit = 0 |
||||
} |
||||
} |
||||
|
||||
stride++ |
||||
if i != length { |
||||
sum += uint(counts[i]) |
||||
if stride >= 4 { |
||||
limit = (256*sum + stride/2) / stride |
||||
} |
||||
|
||||
if stride == 4 { |
||||
limit += 120 |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
func decideOverRLEUse(depth []byte, length uint, use_rle_for_non_zero *bool, use_rle_for_zero *bool) { |
||||
var total_reps_zero uint = 0 |
||||
var total_reps_non_zero uint = 0 |
||||
var count_reps_zero uint = 1 |
||||
var count_reps_non_zero uint = 1 |
||||
var i uint |
||||
for i = 0; i < length; { |
||||
var value byte = depth[i] |
||||
var reps uint = 1 |
||||
var k uint |
||||
for k = i + 1; k < length && depth[k] == value; k++ { |
||||
reps++ |
||||
} |
||||
|
||||
if reps >= 3 && value == 0 { |
||||
total_reps_zero += reps |
||||
count_reps_zero++ |
||||
} |
||||
|
||||
if reps >= 4 && value != 0 { |
||||
total_reps_non_zero += reps |
||||
count_reps_non_zero++ |
||||
} |
||||
|
||||
i += reps |
||||
} |
||||
|
||||
*use_rle_for_non_zero = total_reps_non_zero > count_reps_non_zero*2 |
||||
*use_rle_for_zero = total_reps_zero > count_reps_zero*2 |
||||
} |
||||
|
||||
/* Write a Huffman tree from bit depths into the bit-stream representation |
||||
of a Huffman tree. The generated Huffman tree is to be compressed once |
||||
more using a Huffman tree */ |
||||
func writeHuffmanTree(depth []byte, length uint, tree_size *uint, tree []byte, extra_bits_data []byte) { |
||||
var previous_value byte = initialRepeatedCodeLength |
||||
var i uint |
||||
var use_rle_for_non_zero bool = false |
||||
var use_rle_for_zero bool = false |
||||
var new_length uint = length |
||||
/* Throw away trailing zeros. */ |
||||
for i = 0; i < length; i++ { |
||||
if depth[length-i-1] == 0 { |
||||
new_length-- |
||||
} else { |
||||
break |
||||
} |
||||
} |
||||
|
||||
/* First gather statistics on if it is a good idea to do RLE. */ |
||||
if length > 50 { |
||||
/* Find RLE coding for longer codes. |
||||
Shorter codes seem not to benefit from RLE. */ |
||||
decideOverRLEUse(depth, new_length, &use_rle_for_non_zero, &use_rle_for_zero) |
||||
} |
||||
|
||||
/* Actual RLE coding. */ |
||||
for i = 0; i < new_length; { |
||||
var value byte = depth[i] |
||||
var reps uint = 1 |
||||
if (value != 0 && use_rle_for_non_zero) || (value == 0 && use_rle_for_zero) { |
||||
var k uint |
||||
for k = i + 1; k < new_length && depth[k] == value; k++ { |
||||
reps++ |
||||
} |
||||
} |
||||
|
||||
if value == 0 { |
||||
writeHuffmanTreeRepetitionsZeros(reps, tree_size, tree, extra_bits_data) |
||||
} else { |
||||
writeHuffmanTreeRepetitions(previous_value, value, reps, tree_size, tree, extra_bits_data) |
||||
previous_value = value |
||||
} |
||||
|
||||
i += reps |
||||
} |
||||
} |
||||
|
||||
var reverseBits_kLut = [16]uint{ |
||||
0x00, |
||||
0x08, |
||||
0x04, |
||||
0x0C, |
||||
0x02, |
||||
0x0A, |
||||
0x06, |
||||
0x0E, |
||||
0x01, |
||||
0x09, |
||||
0x05, |
||||
0x0D, |
||||
0x03, |
||||
0x0B, |
||||
0x07, |
||||
0x0F, |
||||
} |
||||
|
||||
func reverseBits(num_bits uint, bits uint16) uint16 { |
||||
var retval uint = reverseBits_kLut[bits&0x0F] |
||||
var i uint |
||||
for i = 4; i < num_bits; i += 4 { |
||||
retval <<= 4 |
||||
bits = uint16(bits >> 4) |
||||
retval |= reverseBits_kLut[bits&0x0F] |
||||
} |
||||
|
||||
retval >>= ((0 - num_bits) & 0x03) |
||||
return uint16(retval) |
||||
} |
||||
|
||||
/* 0..15 are values for bits */ |
||||
const maxHuffmanBits = 16 |
||||
|
||||
/* Get the actual bit values for a tree of bit depths. */ |
||||
func convertBitDepthsToSymbols(depth []byte, len uint, bits []uint16) { |
||||
var bl_count = [maxHuffmanBits]uint16{0} |
||||
var next_code [maxHuffmanBits]uint16 |
||||
var i uint |
||||
/* In Brotli, all bit depths are [1..15] |
||||
0 bit depth means that the symbol does not exist. */ |
||||
|
||||
var code int = 0 |
||||
for i = 0; i < len; i++ { |
||||
bl_count[depth[i]]++ |
||||
} |
||||
|
||||
bl_count[0] = 0 |
||||
next_code[0] = 0 |
||||
for i = 1; i < maxHuffmanBits; i++ { |
||||
code = (code + int(bl_count[i-1])) << 1 |
||||
next_code[i] = uint16(code) |
||||
} |
||||
|
||||
for i = 0; i < len; i++ { |
||||
if depth[i] != 0 { |
||||
bits[i] = reverseBits(uint(depth[i]), next_code[depth[i]]) |
||||
next_code[depth[i]]++ |
||||
} |
||||
} |
||||
} |
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,296 @@ |
||||
package brotli |
||||
|
||||
import "math" |
||||
|
||||
/* Copyright 2013 Google Inc. All Rights Reserved. |
||||
|
||||
Distributed under MIT license. |
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/ |
||||
|
||||
/* Utilities for fast computation of logarithms. */ |
||||
|
||||
func log2FloorNonZero(n uint) uint32 { |
||||
/* TODO: generalize and move to platform.h */ |
||||
var result uint32 = 0 |
||||
for { |
||||
n >>= 1 |
||||
if n == 0 { |
||||
break |
||||
} |
||||
result++ |
||||
} |
||||
return result |
||||
} |
||||
|
||||
/* A lookup table for small values of log2(int) to be used in entropy |
||||
computation. |
||||
|
||||
", ".join(["%.16ff" % x for x in [0.0]+[log2(x) for x in range(1, 256)]]) */ |
||||
var kLog2Table = []float32{ |
||||
0.0000000000000000, |
||||
0.0000000000000000, |
||||
1.0000000000000000, |
||||
1.5849625007211563, |
||||
2.0000000000000000, |
||||
2.3219280948873622, |
||||
2.5849625007211561, |
||||
2.8073549220576042, |
||||
3.0000000000000000, |
||||
3.1699250014423126, |
||||
3.3219280948873626, |
||||
3.4594316186372978, |
||||
3.5849625007211565, |
||||
3.7004397181410922, |
||||
3.8073549220576037, |
||||
3.9068905956085187, |
||||
4.0000000000000000, |
||||
4.0874628412503400, |
||||
4.1699250014423122, |
||||
4.2479275134435852, |
||||
4.3219280948873626, |
||||
4.3923174227787607, |
||||
4.4594316186372973, |
||||
4.5235619560570131, |
||||
4.5849625007211570, |
||||
4.6438561897747244, |
||||
4.7004397181410926, |
||||
4.7548875021634691, |
||||
4.8073549220576037, |
||||
4.8579809951275728, |
||||
4.9068905956085187, |
||||
4.9541963103868758, |
||||
5.0000000000000000, |
||||
5.0443941193584534, |
||||
5.0874628412503400, |
||||
5.1292830169449664, |
||||
5.1699250014423122, |
||||
5.2094533656289501, |
||||
5.2479275134435852, |
||||
5.2854022188622487, |
||||
5.3219280948873626, |
||||
5.3575520046180838, |
||||
5.3923174227787607, |
||||
5.4262647547020979, |
||||
5.4594316186372973, |
||||
5.4918530963296748, |
||||
5.5235619560570131, |
||||
5.5545888516776376, |
||||
5.5849625007211570, |
||||
5.6147098441152083, |
||||
5.6438561897747244, |
||||
5.6724253419714961, |
||||
5.7004397181410926, |
||||
5.7279204545631996, |
||||
5.7548875021634691, |
||||
5.7813597135246599, |
||||
5.8073549220576046, |
||||
5.8328900141647422, |
||||
5.8579809951275719, |
||||
5.8826430493618416, |
||||
5.9068905956085187, |
||||
5.9307373375628867, |
||||
5.9541963103868758, |
||||
5.9772799234999168, |
||||
6.0000000000000000, |
||||
6.0223678130284544, |
||||
6.0443941193584534, |
||||
6.0660891904577721, |
||||
6.0874628412503400, |
||||
6.1085244567781700, |
||||
6.1292830169449672, |
||||
6.1497471195046822, |
||||
6.1699250014423122, |
||||
6.1898245588800176, |
||||
6.2094533656289510, |
||||
6.2288186904958804, |
||||
6.2479275134435861, |
||||
6.2667865406949019, |
||||
6.2854022188622487, |
||||
6.3037807481771031, |
||||
6.3219280948873617, |
||||
6.3398500028846252, |
||||
6.3575520046180847, |
||||
6.3750394313469254, |
||||
6.3923174227787598, |
||||
6.4093909361377026, |
||||
6.4262647547020979, |
||||
6.4429434958487288, |
||||
6.4594316186372982, |
||||
6.4757334309663976, |
||||
6.4918530963296748, |
||||
6.5077946401986964, |
||||
6.5235619560570131, |
||||
6.5391588111080319, |
||||
6.5545888516776376, |
||||
6.5698556083309478, |
||||
6.5849625007211561, |
||||
6.5999128421871278, |
||||
6.6147098441152092, |
||||
6.6293566200796095, |
||||
6.6438561897747253, |
||||
6.6582114827517955, |
||||
6.6724253419714952, |
||||
6.6865005271832185, |
||||
6.7004397181410917, |
||||
6.7142455176661224, |
||||
6.7279204545631988, |
||||
6.7414669864011465, |
||||
6.7548875021634691, |
||||
6.7681843247769260, |
||||
6.7813597135246599, |
||||
6.7944158663501062, |
||||
6.8073549220576037, |
||||
6.8201789624151887, |
||||
6.8328900141647422, |
||||
6.8454900509443757, |
||||
6.8579809951275719, |
||||
6.8703647195834048, |
||||
6.8826430493618416, |
||||
6.8948177633079437, |
||||
6.9068905956085187, |
||||
6.9188632372745955, |
||||
6.9307373375628867, |
||||
6.9425145053392399, |
||||
6.9541963103868758, |
||||
6.9657842846620879, |
||||
6.9772799234999168, |
||||
6.9886846867721664, |
||||
7.0000000000000000, |
||||
7.0112272554232540, |
||||
7.0223678130284544, |
||||
7.0334230015374501, |
||||
7.0443941193584534, |
||||
7.0552824355011898, |
||||
7.0660891904577721, |
||||
7.0768155970508317, |
||||
7.0874628412503400, |
||||
7.0980320829605272, |
||||
7.1085244567781700, |
||||
7.1189410727235076, |
||||
7.1292830169449664, |
||||
7.1395513523987937, |
||||
7.1497471195046822, |
||||
7.1598713367783891, |
||||
7.1699250014423130, |
||||
7.1799090900149345, |
||||
7.1898245588800176, |
||||
7.1996723448363644, |
||||
7.2094533656289492, |
||||
7.2191685204621621, |
||||
7.2288186904958804, |
||||
7.2384047393250794, |
||||
7.2479275134435861, |
||||
7.2573878426926521, |
||||
7.2667865406949019, |
||||
7.2761244052742384, |
||||
7.2854022188622487, |
||||
7.2946207488916270, |
||||
7.3037807481771031, |
||||
7.3128829552843557, |
||||
7.3219280948873617, |
||||
7.3309168781146177, |
||||
7.3398500028846243, |
||||
7.3487281542310781, |
||||
7.3575520046180847, |
||||
7.3663222142458151, |
||||
7.3750394313469254, |
||||
7.3837042924740528, |
||||
7.3923174227787607, |
||||
7.4008794362821844, |
||||
7.4093909361377026, |
||||
7.4178525148858991, |
||||
7.4262647547020979, |
||||
7.4346282276367255, |
||||
7.4429434958487288, |
||||
7.4512111118323299, |
||||
7.4594316186372973, |
||||
7.4676055500829976, |
||||
7.4757334309663976, |
||||
7.4838157772642564, |
||||
7.4918530963296748, |
||||
7.4998458870832057, |
||||
7.5077946401986964, |
||||
7.5156998382840436, |
||||
7.5235619560570131, |
||||
7.5313814605163119, |
||||
7.5391588111080319, |
||||
7.5468944598876373, |
||||
7.5545888516776376, |
||||
7.5622424242210728, |
||||
7.5698556083309478, |
||||
7.5774288280357487, |
||||
7.5849625007211561, |
||||
7.5924570372680806, |
||||
7.5999128421871278, |
||||
7.6073303137496113, |
||||
7.6147098441152075, |
||||
7.6220518194563764, |
||||
7.6293566200796095, |
||||
7.6366246205436488, |
||||
7.6438561897747244, |
||||
7.6510516911789290, |
||||
7.6582114827517955, |
||||
7.6653359171851765, |
||||
7.6724253419714952, |
||||
7.6794800995054464, |
||||
7.6865005271832185, |
||||
7.6934869574993252, |
||||
7.7004397181410926, |
||||
7.7073591320808825, |
||||
7.7142455176661224, |
||||
7.7210991887071856, |
||||
7.7279204545631996, |
||||
7.7347096202258392, |
||||
7.7414669864011465, |
||||
7.7481928495894596, |
||||
7.7548875021634691, |
||||
7.7615512324444795, |
||||
7.7681843247769260, |
||||
7.7747870596011737, |
||||
7.7813597135246608, |
||||
7.7879025593914317, |
||||
7.7944158663501062, |
||||
7.8008998999203047, |
||||
7.8073549220576037, |
||||
7.8137811912170374, |
||||
7.8201789624151887, |
||||
7.8265484872909159, |
||||
7.8328900141647422, |
||||
7.8392037880969445, |
||||
7.8454900509443757, |
||||
7.8517490414160571, |
||||
7.8579809951275719, |
||||
7.8641861446542798, |
||||
7.8703647195834048, |
||||
7.8765169465650002, |
||||
7.8826430493618425, |
||||
7.8887432488982601, |
||||
7.8948177633079446, |
||||
7.9008668079807496, |
||||
7.9068905956085187, |
||||
7.9128893362299619, |
||||
7.9188632372745955, |
||||
7.9248125036057813, |
||||
7.9307373375628867, |
||||
7.9366379390025719, |
||||
7.9425145053392399, |
||||
7.9483672315846778, |
||||
7.9541963103868758, |
||||
7.9600019320680806, |
||||
7.9657842846620870, |
||||
7.9715435539507720, |
||||
7.9772799234999168, |
||||
7.9829935746943104, |
||||
7.9886846867721664, |
||||
7.9943534368588578, |
||||
} |
||||
|
||||
/* Faster logarithm for small integers, with the property of log2(0) == 0. */ |
||||
func fastLog2(v uint) float64 { |
||||
if v < uint(len(kLog2Table)) { |
||||
return float64(kLog2Table[v]) |
||||
} |
||||
|
||||
return math.Log2(float64(v)) |
||||
} |
||||
@ -0,0 +1,16 @@ |
||||
package brotli |
||||
|
||||
/* Copyright 2010 Google Inc. All Rights Reserved. |
||||
|
||||
Distributed under MIT license. |
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/ |
||||
|
||||
/* Function to find maximal matching prefixes of strings. */ |
||||
func findMatchLengthWithLimit(s1 []byte, s2 []byte, limit uint) uint { |
||||
var matched uint = 0 |
||||
for matched < limit && s1[matched] == s2[matched] { |
||||
matched++ |
||||
} |
||||
return matched |
||||
} |
||||
@ -0,0 +1,5 @@ |
||||
module github.com/andybalholm/brotli |
||||
|
||||
go 1.12 |
||||
|
||||
require github.com/golang/gddo v0.0.0-20190419222130-af0f2af80721 |
||||
@ -0,0 +1,2 @@ |
||||
github.com/golang/gddo v0.0.0-20190419222130-af0f2af80721 h1:KRMr9A3qfbVM7iV/WcLY/rL5LICqwMHLhwRXKu99fXw= |
||||
github.com/golang/gddo v0.0.0-20190419222130-af0f2af80721/go.mod h1:xEhNfoBDX1hzLm2Nf80qUvZ2sVwoMZ8d6IE2SrsQfh4= |
||||
@ -0,0 +1,287 @@ |
||||
package brotli |
||||
|
||||
import "encoding/binary" |
||||
|
||||
/* Copyright 2016 Google Inc. All Rights Reserved. |
||||
|
||||
Distributed under MIT license. |
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/ |
||||
|
||||
func (*h10) HashTypeLength() uint { |
||||
return 4 |
||||
} |
||||
|
||||
func (*h10) StoreLookahead() uint { |
||||
return 128 |
||||
} |
||||
|
||||
func hashBytesH10(data []byte) uint32 { |
||||
var h uint32 = binary.LittleEndian.Uint32(data) * kHashMul32 |
||||
|
||||
/* The higher bits contain more mixture from the multiplication, |
||||
so we take our results from there. */ |
||||
return h >> (32 - 17) |
||||
} |
||||
|
||||
/* A (forgetful) hash table where each hash bucket contains a binary tree of |
||||
sequences whose first 4 bytes share the same hash code. |
||||
Each sequence is 128 long and is identified by its starting |
||||
position in the input data. The binary tree is sorted by the lexicographic |
||||
order of the sequences, and it is also a max-heap with respect to the |
||||
starting positions. */ |
||||
type h10 struct { |
||||
hasherCommon |
||||
window_mask_ uint |
||||
buckets_ [1 << 17]uint32 |
||||
invalid_pos_ uint32 |
||||
forest []uint32 |
||||
} |
||||
|
||||
func (h *h10) Initialize(params *encoderParams) { |
||||
h.window_mask_ = (1 << params.lgwin) - 1 |
||||
h.invalid_pos_ = uint32(0 - h.window_mask_) |
||||
var num_nodes uint = uint(1) << params.lgwin |
||||
h.forest = make([]uint32, 2*num_nodes) |
||||
} |
||||
|
||||
func (h *h10) Prepare(one_shot bool, input_size uint, data []byte) { |
||||
var invalid_pos uint32 = h.invalid_pos_ |
||||
var i uint32 |
||||
for i = 0; i < 1<<17; i++ { |
||||
h.buckets_[i] = invalid_pos |
||||
} |
||||
} |
||||
|
||||
func leftChildIndexH10(self *h10, pos uint) uint { |
||||
return 2 * (pos & self.window_mask_) |
||||
} |
||||
|
||||
func rightChildIndexH10(self *h10, pos uint) uint { |
||||
return 2*(pos&self.window_mask_) + 1 |
||||
} |
||||
|
||||
/* Stores the hash of the next 4 bytes and in a single tree-traversal, the |
||||
hash bucket's binary tree is searched for matches and is re-rooted at the |
||||
current position. |
||||
|
||||
If less than 128 data is available, the hash bucket of the |
||||
current position is searched for matches, but the state of the hash table |
||||
is not changed, since we can not know the final sorting order of the |
||||
current (incomplete) sequence. |
||||
|
||||
This function must be called with increasing cur_ix positions. */ |
||||
func storeAndFindMatchesH10(self *h10, data []byte, cur_ix uint, ring_buffer_mask uint, max_length uint, max_backward uint, best_len *uint, matches []backwardMatch) []backwardMatch { |
||||
var cur_ix_masked uint = cur_ix & ring_buffer_mask |
||||
var max_comp_len uint = brotli_min_size_t(max_length, 128) |
||||
var should_reroot_tree bool = (max_length >= 128) |
||||
var key uint32 = hashBytesH10(data[cur_ix_masked:]) |
||||
var forest []uint32 = self.forest |
||||
var prev_ix uint = uint(self.buckets_[key]) |
||||
var node_left uint = leftChildIndexH10(self, cur_ix) |
||||
var node_right uint = rightChildIndexH10(self, cur_ix) |
||||
var best_len_left uint = 0 |
||||
var best_len_right uint = 0 |
||||
var depth_remaining uint |
||||
/* The forest index of the rightmost node of the left subtree of the new |
||||
root, updated as we traverse and re-root the tree of the hash bucket. */ |
||||
|
||||
/* The forest index of the leftmost node of the right subtree of the new |
||||
root, updated as we traverse and re-root the tree of the hash bucket. */ |
||||
|
||||
/* The match length of the rightmost node of the left subtree of the new |
||||
root, updated as we traverse and re-root the tree of the hash bucket. */ |
||||
|
||||
/* The match length of the leftmost node of the right subtree of the new |
||||
root, updated as we traverse and re-root the tree of the hash bucket. */ |
||||
if should_reroot_tree { |
||||
self.buckets_[key] = uint32(cur_ix) |
||||
} |
||||
|
||||
for depth_remaining = 64; ; depth_remaining-- { |
||||
var backward uint = cur_ix - prev_ix |
||||
var prev_ix_masked uint = prev_ix & ring_buffer_mask |
||||
if backward == 0 || backward > max_backward || depth_remaining == 0 { |
||||
if should_reroot_tree { |
||||
forest[node_left] = self.invalid_pos_ |
||||
forest[node_right] = self.invalid_pos_ |
||||
} |
||||
|
||||
break |
||||
} |
||||
{ |
||||
var cur_len uint = brotli_min_size_t(best_len_left, best_len_right) |
||||
var len uint |
||||
assert(cur_len <= 128) |
||||
len = cur_len + findMatchLengthWithLimit(data[cur_ix_masked+cur_len:], data[prev_ix_masked+cur_len:], max_length-cur_len) |
||||
if matches != nil && len > *best_len { |
||||
*best_len = uint(len) |
||||
initBackwardMatch(&matches[0], backward, uint(len)) |
||||
matches = matches[1:] |
||||
} |
||||
|
||||
if len >= max_comp_len { |
||||
if should_reroot_tree { |
||||
forest[node_left] = forest[leftChildIndexH10(self, prev_ix)] |
||||
forest[node_right] = forest[rightChildIndexH10(self, prev_ix)] |
||||
} |
||||
|
||||
break |
||||
} |
||||
|
||||
if data[cur_ix_masked+len] > data[prev_ix_masked+len] { |
||||
best_len_left = uint(len) |
||||
if should_reroot_tree { |
||||
forest[node_left] = uint32(prev_ix) |
||||
} |
||||
|
||||
node_left = rightChildIndexH10(self, prev_ix) |
||||
prev_ix = uint(forest[node_left]) |
||||
} else { |
||||
best_len_right = uint(len) |
||||
if should_reroot_tree { |
||||
forest[node_right] = uint32(prev_ix) |
||||
} |
||||
|
||||
node_right = leftChildIndexH10(self, prev_ix) |
||||
prev_ix = uint(forest[node_right]) |
||||
} |
||||
} |
||||
} |
||||
|
||||
return matches |
||||
} |
||||
|
||||
/* Finds all backward matches of &data[cur_ix & ring_buffer_mask] up to the |
||||
length of max_length and stores the position cur_ix in the hash table. |
||||
|
||||
Sets *num_matches to the number of matches found, and stores the found |
||||
matches in matches[0] to matches[*num_matches - 1]. The matches will be |
||||
sorted by strictly increasing length and (non-strictly) increasing |
||||
distance. */ |
||||
func findAllMatchesH10(handle *h10, dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, cur_ix uint, max_length uint, max_backward uint, gap uint, params *encoderParams, matches []backwardMatch) uint { |
||||
var orig_matches []backwardMatch = matches |
||||
var cur_ix_masked uint = cur_ix & ring_buffer_mask |
||||
var best_len uint = 1 |
||||
var short_match_max_backward uint |
||||
if params.quality != hqZopflificationQuality { |
||||
short_match_max_backward = 16 |
||||
} else { |
||||
short_match_max_backward = 64 |
||||
} |
||||
var stop uint = cur_ix - short_match_max_backward |
||||
var dict_matches [maxStaticDictionaryMatchLen + 1]uint32 |
||||
var i uint |
||||
if cur_ix < short_match_max_backward { |
||||
stop = 0 |
||||
} |
||||
for i = cur_ix - 1; i > stop && best_len <= 2; i-- { |
||||
var prev_ix uint = i |
||||
var backward uint = cur_ix - prev_ix |
||||
if backward > max_backward { |
||||
break |
||||
} |
||||
|
||||
prev_ix &= ring_buffer_mask |
||||
if data[cur_ix_masked] != data[prev_ix] || data[cur_ix_masked+1] != data[prev_ix+1] { |
||||
continue |
||||
} |
||||
{ |
||||
var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length) |
||||
if len > best_len { |
||||
best_len = uint(len) |
||||
initBackwardMatch(&matches[0], backward, uint(len)) |
||||
matches = matches[1:] |
||||
} |
||||
} |
||||
} |
||||
|
||||
if best_len < max_length { |
||||
matches = storeAndFindMatchesH10(handle, data, cur_ix, ring_buffer_mask, max_length, max_backward, &best_len, matches) |
||||
} |
||||
|
||||
for i = 0; i <= maxStaticDictionaryMatchLen; i++ { |
||||
dict_matches[i] = kInvalidMatch |
||||
} |
||||
{ |
||||
var minlen uint = brotli_max_size_t(4, best_len+1) |
||||
if findAllStaticDictionaryMatches(dictionary, data[cur_ix_masked:], minlen, max_length, dict_matches[0:]) { |
||||
var maxlen uint = brotli_min_size_t(maxStaticDictionaryMatchLen, max_length) |
||||
var l uint |
||||
for l = minlen; l <= maxlen; l++ { |
||||
var dict_id uint32 = dict_matches[l] |
||||
if dict_id < kInvalidMatch { |
||||
var distance uint = max_backward + gap + uint(dict_id>>5) + 1 |
||||
if distance <= params.dist.max_distance { |
||||
initDictionaryBackwardMatch(&matches[0], distance, l, uint(dict_id&31)) |
||||
matches = matches[1:] |
||||
} |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
return uint(-cap(matches) + cap(orig_matches)) |
||||
} |
||||
|
||||
/* Stores the hash of the next 4 bytes and re-roots the binary tree at the |
||||
current sequence, without returning any matches. |
||||
REQUIRES: ix + 128 <= end-of-current-block */ |
||||
func (h *h10) Store(data []byte, mask uint, ix uint) { |
||||
var max_backward uint = h.window_mask_ - windowGap + 1 |
||||
/* Maximum distance is window size - 16, see section 9.1. of the spec. */ |
||||
storeAndFindMatchesH10(h, data, ix, mask, 128, max_backward, nil, nil) |
||||
} |
||||
|
||||
func (h *h10) StoreRange(data []byte, mask uint, ix_start uint, ix_end uint) { |
||||
var i uint = ix_start |
||||
var j uint = ix_start |
||||
if ix_start+63 <= ix_end { |
||||
i = ix_end - 63 |
||||
} |
||||
|
||||
if ix_start+512 <= i { |
||||
for ; j < i; j += 8 { |
||||
h.Store(data, mask, j) |
||||
} |
||||
} |
||||
|
||||
for ; i < ix_end; i++ { |
||||
h.Store(data, mask, i) |
||||
} |
||||
} |
||||
|
||||
func (h *h10) StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint) { |
||||
if num_bytes >= h.HashTypeLength()-1 && position >= 128 { |
||||
var i_start uint = position - 128 + 1 |
||||
var i_end uint = brotli_min_size_t(position, i_start+num_bytes) |
||||
/* Store the last `128 - 1` positions in the hasher. |
||||
These could not be calculated before, since they require knowledge |
||||
of both the previous and the current block. */ |
||||
|
||||
var i uint |
||||
for i = i_start; i < i_end; i++ { |
||||
/* Maximum distance is window size - 16, see section 9.1. of the spec. |
||||
Furthermore, we have to make sure that we don't look further back |
||||
from the start of the next block than the window size, otherwise we |
||||
could access already overwritten areas of the ring-buffer. */ |
||||
var max_backward uint = h.window_mask_ - brotli_max_size_t(windowGap-1, position-i) |
||||
|
||||
/* We know that i + 128 <= position + num_bytes, i.e. the |
||||
end of the current block and that we have at least |
||||
128 tail in the ring-buffer. */ |
||||
storeAndFindMatchesH10(h, ringbuffer, i, ringbuffer_mask, 128, max_backward, nil, nil) |
||||
} |
||||
} |
||||
} |
||||
|
||||
/* MAX_NUM_MATCHES == 64 + MAX_TREE_SEARCH_DEPTH */ |
||||
const maxNumMatchesH10 = 128 |
||||
|
||||
func (*h10) FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult) { |
||||
panic("unimplemented") |
||||
} |
||||
|
||||
func (*h10) PrepareDistanceCache(distance_cache []int) { |
||||
panic("unimplemented") |
||||
} |
||||
@ -0,0 +1,214 @@ |
||||
package brotli |
||||
|
||||
import "encoding/binary" |
||||
|
||||
/* Copyright 2010 Google Inc. All Rights Reserved. |
||||
|
||||
Distributed under MIT license. |
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/ |
||||
|
||||
/* A (forgetful) hash table to the data seen by the compressor, to |
||||
help create backward references to previous data. |
||||
|
||||
This is a hash map of fixed size (bucket_size_) to a ring buffer of |
||||
fixed size (block_size_). The ring buffer contains the last block_size_ |
||||
index positions of the given hash key in the compressed data. */ |
||||
func (*h5) HashTypeLength() uint { |
||||
return 4 |
||||
} |
||||
|
||||
func (*h5) StoreLookahead() uint { |
||||
return 4 |
||||
} |
||||
|
||||
/* HashBytes is the function that chooses the bucket to place the address in. */ |
||||
func hashBytesH5(data []byte, shift int) uint32 { |
||||
var h uint32 = binary.LittleEndian.Uint32(data) * kHashMul32 |
||||
|
||||
/* The higher bits contain more mixture from the multiplication, |
||||
so we take our results from there. */ |
||||
return uint32(h >> uint(shift)) |
||||
} |
||||
|
||||
type h5 struct { |
||||
hasherCommon |
||||
bucket_size_ uint |
||||
block_size_ uint |
||||
hash_shift_ int |
||||
block_mask_ uint32 |
||||
num []uint16 |
||||
buckets []uint32 |
||||
} |
||||
|
||||
func (h *h5) Initialize(params *encoderParams) { |
||||
h.hash_shift_ = 32 - h.params.bucket_bits |
||||
h.bucket_size_ = uint(1) << uint(h.params.bucket_bits) |
||||
h.block_size_ = uint(1) << uint(h.params.block_bits) |
||||
h.block_mask_ = uint32(h.block_size_ - 1) |
||||
h.num = make([]uint16, h.bucket_size_) |
||||
h.buckets = make([]uint32, h.block_size_*h.bucket_size_) |
||||
} |
||||
|
||||
func (h *h5) Prepare(one_shot bool, input_size uint, data []byte) { |
||||
var num []uint16 = h.num |
||||
var partial_prepare_threshold uint = h.bucket_size_ >> 6 |
||||
/* Partial preparation is 100 times slower (per socket). */ |
||||
if one_shot && input_size <= partial_prepare_threshold { |
||||
var i uint |
||||
for i = 0; i < input_size; i++ { |
||||
var key uint32 = hashBytesH5(data[i:], h.hash_shift_) |
||||
num[key] = 0 |
||||
} |
||||
} else { |
||||
for i := 0; i < int(h.bucket_size_); i++ { |
||||
num[i] = 0 |
||||
} |
||||
} |
||||
} |
||||
|
||||
/* Look at 4 bytes at &data[ix & mask]. |
||||
Compute a hash from these, and store the value of ix at that position. */ |
||||
func (h *h5) Store(data []byte, mask uint, ix uint) { |
||||
var num []uint16 = h.num |
||||
var key uint32 = hashBytesH5(data[ix&mask:], h.hash_shift_) |
||||
var minor_ix uint = uint(num[key]) & uint(h.block_mask_) |
||||
var offset uint = minor_ix + uint(key<<uint(h.params.block_bits)) |
||||
h.buckets[offset] = uint32(ix) |
||||
num[key]++ |
||||
} |
||||
|
||||
func (h *h5) StoreRange(data []byte, mask uint, ix_start uint, ix_end uint) { |
||||
var i uint |
||||
for i = ix_start; i < ix_end; i++ { |
||||
h.Store(data, mask, i) |
||||
} |
||||
} |
||||
|
||||
func (h *h5) StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint) { |
||||
if num_bytes >= h.HashTypeLength()-1 && position >= 3 { |
||||
/* Prepare the hashes for three last bytes of the last write. |
||||
These could not be calculated before, since they require knowledge |
||||
of both the previous and the current block. */ |
||||
h.Store(ringbuffer, ringbuffer_mask, position-3) |
||||
h.Store(ringbuffer, ringbuffer_mask, position-2) |
||||
h.Store(ringbuffer, ringbuffer_mask, position-1) |
||||
} |
||||
} |
||||
|
||||
func (h *h5) PrepareDistanceCache(distance_cache []int) { |
||||
prepareDistanceCache(distance_cache, h.params.num_last_distances_to_check) |
||||
} |
||||
|
||||
/* Find a longest backward match of &data[cur_ix] up to the length of |
||||
max_length and stores the position cur_ix in the hash table. |
||||
|
||||
REQUIRES: PrepareDistanceCacheH5 must be invoked for current distance cache |
||||
values; if this method is invoked repeatedly with the same distance |
||||
cache values, it is enough to invoke PrepareDistanceCacheH5 once. |
||||
|
||||
Does not look for matches longer than max_length. |
||||
Does not look for matches further away than max_backward. |
||||
Writes the best match into |out|. |
||||
|out|->score is updated only if a better match is found. */ |
||||
func (h *h5) FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult) { |
||||
var num []uint16 = h.num |
||||
var buckets []uint32 = h.buckets |
||||
var cur_ix_masked uint = cur_ix & ring_buffer_mask |
||||
var min_score uint = out.score |
||||
var best_score uint = out.score |
||||
var best_len uint = out.len |
||||
var i uint |
||||
var bucket []uint32 |
||||
/* Don't accept a short copy from far away. */ |
||||
out.len = 0 |
||||
|
||||
out.len_code_delta = 0 |
||||
|
||||
/* Try last distance first. */ |
||||
for i = 0; i < uint(h.params.num_last_distances_to_check); i++ { |
||||
var backward uint = uint(distance_cache[i]) |
||||
var prev_ix uint = uint(cur_ix - backward) |
||||
if prev_ix >= cur_ix { |
||||
continue |
||||
} |
||||
|
||||
if backward > max_backward { |
||||
continue |
||||
} |
||||
|
||||
prev_ix &= ring_buffer_mask |
||||
|
||||
if cur_ix_masked+best_len > ring_buffer_mask || prev_ix+best_len > ring_buffer_mask || data[cur_ix_masked+best_len] != data[prev_ix+best_len] { |
||||
continue |
||||
} |
||||
{ |
||||
var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length) |
||||
if len >= 3 || (len == 2 && i < 2) { |
||||
/* Comparing for >= 2 does not change the semantics, but just saves for |
||||
a few unnecessary binary logarithms in backward reference score, |
||||
since we are not interested in such short matches. */ |
||||
var score uint = backwardReferenceScoreUsingLastDistance(uint(len)) |
||||
if best_score < score { |
||||
if i != 0 { |
||||
score -= backwardReferencePenaltyUsingLastDistance(i) |
||||
} |
||||
if best_score < score { |
||||
best_score = score |
||||
best_len = uint(len) |
||||
out.len = best_len |
||||
out.distance = backward |
||||
out.score = best_score |
||||
} |
||||
} |
||||
} |
||||
} |
||||
} |
||||
{ |
||||
var key uint32 = hashBytesH5(data[cur_ix_masked:], h.hash_shift_) |
||||
bucket = buckets[key<<uint(h.params.block_bits):] |
||||
var down uint |
||||
if uint(num[key]) > h.block_size_ { |
||||
down = uint(num[key]) - h.block_size_ |
||||
} else { |
||||
down = 0 |
||||
} |
||||
for i = uint(num[key]); i > down; { |
||||
var prev_ix uint |
||||
i-- |
||||
prev_ix = uint(bucket[uint32(i)&h.block_mask_]) |
||||
var backward uint = cur_ix - prev_ix |
||||
if backward > max_backward { |
||||
break |
||||
} |
||||
|
||||
prev_ix &= ring_buffer_mask |
||||
if cur_ix_masked+best_len > ring_buffer_mask || prev_ix+best_len > ring_buffer_mask || data[cur_ix_masked+best_len] != data[prev_ix+best_len] { |
||||
continue |
||||
} |
||||
{ |
||||
var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length) |
||||
if len >= 4 { |
||||
/* Comparing for >= 3 does not change the semantics, but just saves |
||||
for a few unnecessary binary logarithms in backward reference |
||||
score, since we are not interested in such short matches. */ |
||||
var score uint = backwardReferenceScore(uint(len), backward) |
||||
if best_score < score { |
||||
best_score = score |
||||
best_len = uint(len) |
||||
out.len = best_len |
||||
out.distance = backward |
||||
out.score = best_score |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
bucket[uint32(num[key])&h.block_mask_] = uint32(cur_ix) |
||||
num[key]++ |
||||
} |
||||
|
||||
if min_score == out.score { |
||||
searchInStaticDictionary(dictionary, h, data[cur_ix_masked:], max_length, max_backward+gap, max_distance, out, false) |
||||
} |
||||
} |
||||
@ -0,0 +1,216 @@ |
||||
package brotli |
||||
|
||||
import "encoding/binary" |
||||
|
||||
/* Copyright 2010 Google Inc. All Rights Reserved. |
||||
|
||||
Distributed under MIT license. |
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/ |
||||
|
||||
/* A (forgetful) hash table to the data seen by the compressor, to |
||||
help create backward references to previous data. |
||||
|
||||
This is a hash map of fixed size (bucket_size_) to a ring buffer of |
||||
fixed size (block_size_). The ring buffer contains the last block_size_ |
||||
index positions of the given hash key in the compressed data. */ |
||||
func (*h6) HashTypeLength() uint { |
||||
return 8 |
||||
} |
||||
|
||||
func (*h6) StoreLookahead() uint { |
||||
return 8 |
||||
} |
||||
|
||||
/* HashBytes is the function that chooses the bucket to place the address in. */ |
||||
func hashBytesH6(data []byte, mask uint64, shift int) uint32 { |
||||
var h uint64 = (binary.LittleEndian.Uint64(data) & mask) * kHashMul64Long |
||||
|
||||
/* The higher bits contain more mixture from the multiplication, |
||||
so we take our results from there. */ |
||||
return uint32(h >> uint(shift)) |
||||
} |
||||
|
||||
type h6 struct { |
||||
hasherCommon |
||||
bucket_size_ uint |
||||
block_size_ uint |
||||
hash_shift_ int |
||||
hash_mask_ uint64 |
||||
block_mask_ uint32 |
||||
num []uint16 |
||||
buckets []uint32 |
||||
} |
||||
|
||||
func (h *h6) Initialize(params *encoderParams) { |
||||
h.hash_shift_ = 64 - h.params.bucket_bits |
||||
h.hash_mask_ = (^(uint64(0))) >> uint(64-8*h.params.hash_len) |
||||
h.bucket_size_ = uint(1) << uint(h.params.bucket_bits) |
||||
h.block_size_ = uint(1) << uint(h.params.block_bits) |
||||
h.block_mask_ = uint32(h.block_size_ - 1) |
||||
h.num = make([]uint16, h.bucket_size_) |
||||
h.buckets = make([]uint32, h.block_size_*h.bucket_size_) |
||||
} |
||||
|
||||
func (h *h6) Prepare(one_shot bool, input_size uint, data []byte) { |
||||
var num []uint16 = h.num |
||||
var partial_prepare_threshold uint = h.bucket_size_ >> 6 |
||||
/* Partial preparation is 100 times slower (per socket). */ |
||||
if one_shot && input_size <= partial_prepare_threshold { |
||||
var i uint |
||||
for i = 0; i < input_size; i++ { |
||||
var key uint32 = hashBytesH6(data[i:], h.hash_mask_, h.hash_shift_) |
||||
num[key] = 0 |
||||
} |
||||
} else { |
||||
for i := 0; i < int(h.bucket_size_); i++ { |
||||
num[i] = 0 |
||||
} |
||||
} |
||||
} |
||||
|
||||
/* Look at 4 bytes at &data[ix & mask]. |
||||
Compute a hash from these, and store the value of ix at that position. */ |
||||
func (h *h6) Store(data []byte, mask uint, ix uint) { |
||||
var num []uint16 = h.num |
||||
var key uint32 = hashBytesH6(data[ix&mask:], h.hash_mask_, h.hash_shift_) |
||||
var minor_ix uint = uint(num[key]) & uint(h.block_mask_) |
||||
var offset uint = minor_ix + uint(key<<uint(h.params.block_bits)) |
||||
h.buckets[offset] = uint32(ix) |
||||
num[key]++ |
||||
} |
||||
|
||||
func (h *h6) StoreRange(data []byte, mask uint, ix_start uint, ix_end uint) { |
||||
var i uint |
||||
for i = ix_start; i < ix_end; i++ { |
||||
h.Store(data, mask, i) |
||||
} |
||||
} |
||||
|
||||
func (h *h6) StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint) { |
||||
if num_bytes >= h.HashTypeLength()-1 && position >= 3 { |
||||
/* Prepare the hashes for three last bytes of the last write. |
||||
These could not be calculated before, since they require knowledge |
||||
of both the previous and the current block. */ |
||||
h.Store(ringbuffer, ringbuffer_mask, position-3) |
||||
h.Store(ringbuffer, ringbuffer_mask, position-2) |
||||
h.Store(ringbuffer, ringbuffer_mask, position-1) |
||||
} |
||||
} |
||||
|
||||
func (h *h6) PrepareDistanceCache(distance_cache []int) { |
||||
prepareDistanceCache(distance_cache, h.params.num_last_distances_to_check) |
||||
} |
||||
|
||||
/* Find a longest backward match of &data[cur_ix] up to the length of |
||||
max_length and stores the position cur_ix in the hash table. |
||||
|
||||
REQUIRES: PrepareDistanceCacheH6 must be invoked for current distance cache |
||||
values; if this method is invoked repeatedly with the same distance |
||||
cache values, it is enough to invoke PrepareDistanceCacheH6 once. |
||||
|
||||
Does not look for matches longer than max_length. |
||||
Does not look for matches further away than max_backward. |
||||
Writes the best match into |out|. |
||||
|out|->score is updated only if a better match is found. */ |
||||
func (h *h6) FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult) { |
||||
var num []uint16 = h.num |
||||
var buckets []uint32 = h.buckets |
||||
var cur_ix_masked uint = cur_ix & ring_buffer_mask |
||||
var min_score uint = out.score |
||||
var best_score uint = out.score |
||||
var best_len uint = out.len |
||||
var i uint |
||||
var bucket []uint32 |
||||
/* Don't accept a short copy from far away. */ |
||||
out.len = 0 |
||||
|
||||
out.len_code_delta = 0 |
||||
|
||||
/* Try last distance first. */ |
||||
for i = 0; i < uint(h.params.num_last_distances_to_check); i++ { |
||||
var backward uint = uint(distance_cache[i]) |
||||
var prev_ix uint = uint(cur_ix - backward) |
||||
if prev_ix >= cur_ix { |
||||
continue |
||||
} |
||||
|
||||
if backward > max_backward { |
||||
continue |
||||
} |
||||
|
||||
prev_ix &= ring_buffer_mask |
||||
|
||||
if cur_ix_masked+best_len > ring_buffer_mask || prev_ix+best_len > ring_buffer_mask || data[cur_ix_masked+best_len] != data[prev_ix+best_len] { |
||||
continue |
||||
} |
||||
{ |
||||
var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length) |
||||
if len >= 3 || (len == 2 && i < 2) { |
||||
/* Comparing for >= 2 does not change the semantics, but just saves for |
||||
a few unnecessary binary logarithms in backward reference score, |
||||
since we are not interested in such short matches. */ |
||||
var score uint = backwardReferenceScoreUsingLastDistance(uint(len)) |
||||
if best_score < score { |
||||
if i != 0 { |
||||
score -= backwardReferencePenaltyUsingLastDistance(i) |
||||
} |
||||
if best_score < score { |
||||
best_score = score |
||||
best_len = uint(len) |
||||
out.len = best_len |
||||
out.distance = backward |
||||
out.score = best_score |
||||
} |
||||
} |
||||
} |
||||
} |
||||
} |
||||
{ |
||||
var key uint32 = hashBytesH6(data[cur_ix_masked:], h.hash_mask_, h.hash_shift_) |
||||
bucket = buckets[key<<uint(h.params.block_bits):] |
||||
var down uint |
||||
if uint(num[key]) > h.block_size_ { |
||||
down = uint(num[key]) - h.block_size_ |
||||
} else { |
||||
down = 0 |
||||
} |
||||
for i = uint(num[key]); i > down; { |
||||
var prev_ix uint |
||||
i-- |
||||
prev_ix = uint(bucket[uint32(i)&h.block_mask_]) |
||||
var backward uint = cur_ix - prev_ix |
||||
if backward > max_backward { |
||||
break |
||||
} |
||||
|
||||
prev_ix &= ring_buffer_mask |
||||
if cur_ix_masked+best_len > ring_buffer_mask || prev_ix+best_len > ring_buffer_mask || data[cur_ix_masked+best_len] != data[prev_ix+best_len] { |
||||
continue |
||||
} |
||||
{ |
||||
var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length) |
||||
if len >= 4 { |
||||
/* Comparing for >= 3 does not change the semantics, but just saves |
||||
for a few unnecessary binary logarithms in backward reference |
||||
score, since we are not interested in such short matches. */ |
||||
var score uint = backwardReferenceScore(uint(len), backward) |
||||
if best_score < score { |
||||
best_score = score |
||||
best_len = uint(len) |
||||
out.len = best_len |
||||
out.distance = backward |
||||
out.score = best_score |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
bucket[uint32(num[key])&h.block_mask_] = uint32(cur_ix) |
||||
num[key]++ |
||||
} |
||||
|
||||
if min_score == out.score { |
||||
searchInStaticDictionary(dictionary, h, data[cur_ix_masked:], max_length, max_backward+gap, max_distance, out, false) |
||||
} |
||||
} |
||||
@ -0,0 +1,344 @@ |
||||
package brotli |
||||
|
||||
import ( |
||||
"encoding/binary" |
||||
"fmt" |
||||
) |
||||
|
||||
type hasherCommon struct { |
||||
params hasherParams |
||||
is_prepared_ bool |
||||
dict_num_lookups uint |
||||
dict_num_matches uint |
||||
} |
||||
|
||||
func (h *hasherCommon) Common() *hasherCommon { |
||||
return h |
||||
} |
||||
|
||||
type hasherHandle interface { |
||||
Common() *hasherCommon |
||||
Initialize(params *encoderParams) |
||||
Prepare(one_shot bool, input_size uint, data []byte) |
||||
StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint) |
||||
HashTypeLength() uint |
||||
StoreLookahead() uint |
||||
PrepareDistanceCache(distance_cache []int) |
||||
FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult) |
||||
StoreRange(data []byte, mask uint, ix_start uint, ix_end uint) |
||||
Store(data []byte, mask uint, ix uint) |
||||
} |
||||
|
||||
type score_t uint |
||||
|
||||
const kCutoffTransformsCount uint32 = 10 |
||||
|
||||
/* 0, 12, 27, 23, 42, 63, 56, 48, 59, 64 */ |
||||
/* 0+0, 4+8, 8+19, 12+11, 16+26, 20+43, 24+32, 28+20, 32+27, 36+28 */ |
||||
const kCutoffTransforms uint64 = 0x071B520ADA2D3200 |
||||
|
||||
type hasherSearchResult struct { |
||||
len uint |
||||
distance uint |
||||
score uint |
||||
len_code_delta int |
||||
} |
||||
|
||||
/* kHashMul32 multiplier has these properties: |
||||
* The multiplier must be odd. Otherwise we may lose the highest bit. |
||||
* No long streaks of ones or zeros. |
||||
* There is no effort to ensure that it is a prime, the oddity is enough |
||||
for this use. |
||||
* The number has been tuned heuristically against compression benchmarks. */ |
||||
const kHashMul32 uint32 = 0x1E35A7BD |
||||
|
||||
const kHashMul64 uint64 = 0x1E35A7BD1E35A7BD |
||||
|
||||
const kHashMul64Long uint64 = 0x1FE35A7BD3579BD3 |
||||
|
||||
func hash14(data []byte) uint32 { |
||||
var h uint32 = binary.LittleEndian.Uint32(data) * kHashMul32 |
||||
|
||||
/* The higher bits contain more mixture from the multiplication, |
||||
so we take our results from there. */ |
||||
return h >> (32 - 14) |
||||
} |
||||
|
||||
func prepareDistanceCache(distance_cache []int, num_distances int) { |
||||
if num_distances > 4 { |
||||
var last_distance int = distance_cache[0] |
||||
distance_cache[4] = last_distance - 1 |
||||
distance_cache[5] = last_distance + 1 |
||||
distance_cache[6] = last_distance - 2 |
||||
distance_cache[7] = last_distance + 2 |
||||
distance_cache[8] = last_distance - 3 |
||||
distance_cache[9] = last_distance + 3 |
||||
if num_distances > 10 { |
||||
var next_last_distance int = distance_cache[1] |
||||
distance_cache[10] = next_last_distance - 1 |
||||
distance_cache[11] = next_last_distance + 1 |
||||
distance_cache[12] = next_last_distance - 2 |
||||
distance_cache[13] = next_last_distance + 2 |
||||
distance_cache[14] = next_last_distance - 3 |
||||
distance_cache[15] = next_last_distance + 3 |
||||
} |
||||
} |
||||
} |
||||
|
||||
const literalByteScore = 135 |
||||
|
||||
const distanceBitPenalty = 30 |
||||
|
||||
/* Score must be positive after applying maximal penalty. */ |
||||
const scoreBase = (distanceBitPenalty * 8 * 8) |
||||
|
||||
/* Usually, we always choose the longest backward reference. This function |
||||
allows for the exception of that rule. |
||||
|
||||
If we choose a backward reference that is further away, it will |
||||
usually be coded with more bits. We approximate this by assuming |
||||
log2(distance). If the distance can be expressed in terms of the |
||||
last four distances, we use some heuristic constants to estimate |
||||
the bits cost. For the first up to four literals we use the bit |
||||
cost of the literals from the literal cost model, after that we |
||||
use the average bit cost of the cost model. |
||||
|
||||
This function is used to sometimes discard a longer backward reference |
||||
when it is not much longer and the bit cost for encoding it is more |
||||
than the saved literals. |
||||
|
||||
backward_reference_offset MUST be positive. */ |
||||
func backwardReferenceScore(copy_length uint, backward_reference_offset uint) uint { |
||||
return scoreBase + literalByteScore*uint(copy_length) - distanceBitPenalty*uint(log2FloorNonZero(backward_reference_offset)) |
||||
} |
||||
|
||||
func backwardReferenceScoreUsingLastDistance(copy_length uint) uint { |
||||
return literalByteScore*uint(copy_length) + scoreBase + 15 |
||||
} |
||||
|
||||
func backwardReferencePenaltyUsingLastDistance(distance_short_code uint) uint { |
||||
return uint(39) + ((0x1CA10 >> (distance_short_code & 0xE)) & 0xE) |
||||
} |
||||
|
||||
func testStaticDictionaryItem(dictionary *encoderDictionary, item uint, data []byte, max_length uint, max_backward uint, max_distance uint, out *hasherSearchResult) bool { |
||||
var len uint |
||||
var word_idx uint |
||||
var offset uint |
||||
var matchlen uint |
||||
var backward uint |
||||
var score uint |
||||
len = item & 0x1F |
||||
word_idx = item >> 5 |
||||
offset = uint(dictionary.words.offsets_by_length[len]) + len*word_idx |
||||
if len > max_length { |
||||
return false |
||||
} |
||||
|
||||
matchlen = findMatchLengthWithLimit(data, dictionary.words.data[offset:], uint(len)) |
||||
if matchlen+uint(dictionary.cutoffTransformsCount) <= len || matchlen == 0 { |
||||
return false |
||||
} |
||||
{ |
||||
var cut uint = len - matchlen |
||||
var transform_id uint = (cut << 2) + uint((dictionary.cutoffTransforms>>(cut*6))&0x3F) |
||||
backward = max_backward + 1 + word_idx + (transform_id << dictionary.words.size_bits_by_length[len]) |
||||
} |
||||
|
||||
if backward > max_distance { |
||||
return false |
||||
} |
||||
|
||||
score = backwardReferenceScore(matchlen, backward) |
||||
if score < out.score { |
||||
return false |
||||
} |
||||
|
||||
out.len = matchlen |
||||
out.len_code_delta = int(len) - int(matchlen) |
||||
out.distance = backward |
||||
out.score = score |
||||
return true |
||||
} |
||||
|
||||
func searchInStaticDictionary(dictionary *encoderDictionary, handle hasherHandle, data []byte, max_length uint, max_backward uint, max_distance uint, out *hasherSearchResult, shallow bool) { |
||||
var key uint |
||||
var i uint |
||||
var self *hasherCommon = handle.Common() |
||||
if self.dict_num_matches < self.dict_num_lookups>>7 { |
||||
return |
||||
} |
||||
|
||||
key = uint(hash14(data) << 1) |
||||
for i = 0; ; (func() { i++; key++ })() { |
||||
var tmp uint |
||||
if shallow { |
||||
tmp = 1 |
||||
} else { |
||||
tmp = 2 |
||||
} |
||||
if i >= tmp { |
||||
break |
||||
} |
||||
var item uint = uint(dictionary.hash_table[key]) |
||||
self.dict_num_lookups++ |
||||
if item != 0 { |
||||
var item_matches bool = testStaticDictionaryItem(dictionary, item, data, max_length, max_backward, max_distance, out) |
||||
if item_matches { |
||||
self.dict_num_matches++ |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
type backwardMatch struct { |
||||
distance uint32 |
||||
length_and_code uint32 |
||||
} |
||||
|
||||
func initBackwardMatch(self *backwardMatch, dist uint, len uint) { |
||||
self.distance = uint32(dist) |
||||
self.length_and_code = uint32(len << 5) |
||||
} |
||||
|
||||
func initDictionaryBackwardMatch(self *backwardMatch, dist uint, len uint, len_code uint) { |
||||
self.distance = uint32(dist) |
||||
var tmp uint |
||||
if len == len_code { |
||||
tmp = 0 |
||||
} else { |
||||
tmp = len_code |
||||
} |
||||
self.length_and_code = uint32(len<<5 | tmp) |
||||
} |
||||
|
||||
func backwardMatchLength(self *backwardMatch) uint { |
||||
return uint(self.length_and_code >> 5) |
||||
} |
||||
|
||||
func backwardMatchLengthCode(self *backwardMatch) uint { |
||||
var code uint = uint(self.length_and_code) & 31 |
||||
if code != 0 { |
||||
return code |
||||
} else { |
||||
return backwardMatchLength(self) |
||||
} |
||||
} |
||||
|
||||
func hasherReset(handle hasherHandle) { |
||||
if handle == nil { |
||||
return |
||||
} |
||||
handle.Common().is_prepared_ = false |
||||
} |
||||
|
||||
func newHasher(typ int) hasherHandle { |
||||
switch typ { |
||||
case 2: |
||||
return &hashLongestMatchQuickly{ |
||||
bucketBits: 16, |
||||
bucketSweep: 1, |
||||
hashLen: 5, |
||||
useDictionary: true, |
||||
} |
||||
case 3: |
||||
return &hashLongestMatchQuickly{ |
||||
bucketBits: 16, |
||||
bucketSweep: 2, |
||||
hashLen: 5, |
||||
useDictionary: false, |
||||
} |
||||
case 4: |
||||
return &hashLongestMatchQuickly{ |
||||
bucketBits: 17, |
||||
bucketSweep: 4, |
||||
hashLen: 5, |
||||
useDictionary: true, |
||||
} |
||||
case 5: |
||||
return new(h5) |
||||
case 6: |
||||
return new(h6) |
||||
case 10: |
||||
return new(h10) |
||||
case 35: |
||||
return &hashComposite{ |
||||
ha: newHasher(3), |
||||
hb: &hashRolling{jump: 4}, |
||||
} |
||||
case 40: |
||||
return &hashForgetfulChain{ |
||||
bucketBits: 15, |
||||
numBanks: 1, |
||||
bankBits: 16, |
||||
numLastDistancesToCheck: 4, |
||||
} |
||||
case 41: |
||||
return &hashForgetfulChain{ |
||||
bucketBits: 15, |
||||
numBanks: 1, |
||||
bankBits: 16, |
||||
numLastDistancesToCheck: 10, |
||||
} |
||||
case 42: |
||||
return &hashForgetfulChain{ |
||||
bucketBits: 15, |
||||
numBanks: 512, |
||||
bankBits: 9, |
||||
numLastDistancesToCheck: 16, |
||||
} |
||||
case 54: |
||||
return &hashLongestMatchQuickly{ |
||||
bucketBits: 20, |
||||
bucketSweep: 4, |
||||
hashLen: 7, |
||||
useDictionary: false, |
||||
} |
||||
case 55: |
||||
return &hashComposite{ |
||||
ha: newHasher(54), |
||||
hb: &hashRolling{jump: 4}, |
||||
} |
||||
case 65: |
||||
return &hashComposite{ |
||||
ha: newHasher(6), |
||||
hb: &hashRolling{jump: 1}, |
||||
} |
||||
} |
||||
|
||||
panic(fmt.Sprintf("unknown hasher type: %d", typ)) |
||||
} |
||||
|
||||
func hasherSetup(handle *hasherHandle, params *encoderParams, data []byte, position uint, input_size uint, is_last bool) { |
||||
var self hasherHandle = nil |
||||
var common *hasherCommon = nil |
||||
var one_shot bool = (position == 0 && is_last) |
||||
if *handle == nil { |
||||
chooseHasher(params, ¶ms.hasher) |
||||
self = newHasher(params.hasher.type_) |
||||
|
||||
*handle = self |
||||
common = self.Common() |
||||
common.params = params.hasher |
||||
self.Initialize(params) |
||||
} |
||||
|
||||
self = *handle |
||||
common = self.Common() |
||||
if !common.is_prepared_ { |
||||
self.Prepare(one_shot, input_size, data) |
||||
|
||||
if position == 0 { |
||||
common.dict_num_lookups = 0 |
||||
common.dict_num_matches = 0 |
||||
} |
||||
|
||||
common.is_prepared_ = true |
||||
} |
||||
} |
||||
|
||||
func initOrStitchToPreviousBlock(handle *hasherHandle, data []byte, mask uint, params *encoderParams, position uint, input_size uint, is_last bool) { |
||||
var self hasherHandle |
||||
hasherSetup(handle, params, data, position, input_size, is_last) |
||||
self = *handle |
||||
self.StitchToPreviousBlock(input_size, position, data, mask) |
||||
} |
||||
@ -0,0 +1,93 @@ |
||||
package brotli |
||||
|
||||
/* Copyright 2018 Google Inc. All Rights Reserved. |
||||
|
||||
Distributed under MIT license. |
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/ |
||||
|
||||
func (h *hashComposite) HashTypeLength() uint { |
||||
var a uint = h.ha.HashTypeLength() |
||||
var b uint = h.hb.HashTypeLength() |
||||
if a > b { |
||||
return a |
||||
} else { |
||||
return b |
||||
} |
||||
} |
||||
|
||||
func (h *hashComposite) StoreLookahead() uint { |
||||
var a uint = h.ha.StoreLookahead() |
||||
var b uint = h.hb.StoreLookahead() |
||||
if a > b { |
||||
return a |
||||
} else { |
||||
return b |
||||
} |
||||
} |
||||
|
||||
/* Composite hasher: This hasher allows to combine two other hashers, HASHER_A |
||||
and HASHER_B. */ |
||||
type hashComposite struct { |
||||
hasherCommon |
||||
ha hasherHandle |
||||
hb hasherHandle |
||||
params *encoderParams |
||||
} |
||||
|
||||
func (h *hashComposite) Initialize(params *encoderParams) { |
||||
h.params = params |
||||
} |
||||
|
||||
/* TODO: Initialize of the hashers is defered to Prepare (and params |
||||
remembered here) because we don't get the one_shot and input_size params |
||||
here that are needed to know the memory size of them. Instead provide |
||||
those params to all hashers InitializehashComposite */ |
||||
func (h *hashComposite) Prepare(one_shot bool, input_size uint, data []byte) { |
||||
if h.ha == nil { |
||||
var common_a *hasherCommon |
||||
var common_b *hasherCommon |
||||
|
||||
common_a = h.ha.Common() |
||||
common_a.params = h.params.hasher |
||||
common_a.is_prepared_ = false |
||||
common_a.dict_num_lookups = 0 |
||||
common_a.dict_num_matches = 0 |
||||
h.ha.Initialize(h.params) |
||||
|
||||
common_b = h.hb.Common() |
||||
common_b.params = h.params.hasher |
||||
common_b.is_prepared_ = false |
||||
common_b.dict_num_lookups = 0 |
||||
common_b.dict_num_matches = 0 |
||||
h.hb.Initialize(h.params) |
||||
} |
||||
|
||||
h.ha.Prepare(one_shot, input_size, data) |
||||
h.hb.Prepare(one_shot, input_size, data) |
||||
} |
||||
|
||||
func (h *hashComposite) Store(data []byte, mask uint, ix uint) { |
||||
h.ha.Store(data, mask, ix) |
||||
h.hb.Store(data, mask, ix) |
||||
} |
||||
|
||||
func (h *hashComposite) StoreRange(data []byte, mask uint, ix_start uint, ix_end uint) { |
||||
h.ha.StoreRange(data, mask, ix_start, ix_end) |
||||
h.hb.StoreRange(data, mask, ix_start, ix_end) |
||||
} |
||||
|
||||
func (h *hashComposite) StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []byte, ring_buffer_mask uint) { |
||||
h.ha.StitchToPreviousBlock(num_bytes, position, ringbuffer, ring_buffer_mask) |
||||
h.hb.StitchToPreviousBlock(num_bytes, position, ringbuffer, ring_buffer_mask) |
||||
} |
||||
|
||||
func (h *hashComposite) PrepareDistanceCache(distance_cache []int) { |
||||
h.ha.PrepareDistanceCache(distance_cache) |
||||
h.hb.PrepareDistanceCache(distance_cache) |
||||
} |
||||
|
||||
func (h *hashComposite) FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult) { |
||||
h.ha.FindLongestMatch(dictionary, data, ring_buffer_mask, distance_cache, cur_ix, max_length, max_backward, gap, max_distance, out) |
||||
h.hb.FindLongestMatch(dictionary, data, ring_buffer_mask, distance_cache, cur_ix, max_length, max_backward, gap, max_distance, out) |
||||
} |
||||
@ -0,0 +1,253 @@ |
||||
package brotli |
||||
|
||||
import "encoding/binary" |
||||
|
||||
/* Copyright 2016 Google Inc. All Rights Reserved. |
||||
|
||||
Distributed under MIT license. |
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/ |
||||
|
||||
func (*hashForgetfulChain) HashTypeLength() uint { |
||||
return 4 |
||||
} |
||||
|
||||
func (*hashForgetfulChain) StoreLookahead() uint { |
||||
return 4 |
||||
} |
||||
|
||||
/* HashBytes is the function that chooses the bucket to place the address in.*/ |
||||
func (h *hashForgetfulChain) HashBytes(data []byte) uint { |
||||
var hash uint32 = binary.LittleEndian.Uint32(data) * kHashMul32 |
||||
|
||||
/* The higher bits contain more mixture from the multiplication, |
||||
so we take our results from there. */ |
||||
return uint(hash >> (32 - h.bucketBits)) |
||||
} |
||||
|
||||
type slot struct { |
||||
delta uint16 |
||||
next uint16 |
||||
} |
||||
|
||||
/* A (forgetful) hash table to the data seen by the compressor, to |
||||
help create backward references to previous data. |
||||
|
||||
Hashes are stored in chains which are bucketed to groups. Group of chains |
||||
share a storage "bank". When more than "bank size" chain nodes are added, |
||||
oldest nodes are replaced; this way several chains may share a tail. */ |
||||
type hashForgetfulChain struct { |
||||
hasherCommon |
||||
|
||||
bucketBits uint |
||||
numBanks uint |
||||
bankBits uint |
||||
numLastDistancesToCheck int |
||||
|
||||
addr []uint32 |
||||
head []uint16 |
||||
tiny_hash [65536]byte |
||||
banks [][]slot |
||||
free_slot_idx []uint16 |
||||
max_hops uint |
||||
} |
||||
|
||||
func (h *hashForgetfulChain) Initialize(params *encoderParams) { |
||||
var q uint |
||||
if params.quality > 6 { |
||||
q = 7 |
||||
} else { |
||||
q = 8 |
||||
} |
||||
h.max_hops = q << uint(params.quality-4) |
||||
|
||||
bankSize := 1 << h.bankBits |
||||
bucketSize := 1 << h.bucketBits |
||||
|
||||
h.addr = make([]uint32, bucketSize) |
||||
h.head = make([]uint16, bucketSize) |
||||
h.banks = make([][]slot, h.numBanks) |
||||
for i := range h.banks { |
||||
h.banks[i] = make([]slot, bankSize) |
||||
} |
||||
h.free_slot_idx = make([]uint16, h.numBanks) |
||||
} |
||||
|
||||
func (h *hashForgetfulChain) Prepare(one_shot bool, input_size uint, data []byte) { |
||||
var partial_prepare_threshold uint = (1 << h.bucketBits) >> 6 |
||||
/* Partial preparation is 100 times slower (per socket). */ |
||||
if one_shot && input_size <= partial_prepare_threshold { |
||||
var i uint |
||||
for i = 0; i < input_size; i++ { |
||||
var bucket uint = h.HashBytes(data[i:]) |
||||
|
||||
/* See InitEmpty comment. */ |
||||
h.addr[bucket] = 0xCCCCCCCC |
||||
|
||||
h.head[bucket] = 0xCCCC |
||||
} |
||||
} else { |
||||
/* Fill |addr| array with 0xCCCCCCCC value. Because of wrapping, position |
||||
processed by hasher never reaches 3GB + 64M; this makes all new chains |
||||
to be terminated after the first node. */ |
||||
for i := range h.addr { |
||||
h.addr[i] = 0xCCCCCCCC |
||||
} |
||||
|
||||
for i := range h.head { |
||||
h.head[i] = 0 |
||||
} |
||||
} |
||||
|
||||
h.tiny_hash = [65536]byte{} |
||||
for i := range h.free_slot_idx { |
||||
h.free_slot_idx[i] = 0 |
||||
} |
||||
} |
||||
|
||||
/* Look at 4 bytes at &data[ix & mask]. Compute a hash from these, and prepend |
||||
node to corresponding chain; also update tiny_hash for current position. */ |
||||
func (h *hashForgetfulChain) Store(data []byte, mask uint, ix uint) { |
||||
var key uint = h.HashBytes(data[ix&mask:]) |
||||
var bank uint = key & (h.numBanks - 1) |
||||
var idx uint |
||||
idx = uint(h.free_slot_idx[bank]) & ((1 << h.bankBits) - 1) |
||||
h.free_slot_idx[bank]++ |
||||
var delta uint = ix - uint(h.addr[key]) |
||||
h.tiny_hash[uint16(ix)] = byte(key) |
||||
if delta > 0xFFFF { |
||||
delta = 0xFFFF |
||||
} |
||||
h.banks[bank][idx].delta = uint16(delta) |
||||
h.banks[bank][idx].next = h.head[key] |
||||
h.addr[key] = uint32(ix) |
||||
h.head[key] = uint16(idx) |
||||
} |
||||
|
||||
func (h *hashForgetfulChain) StoreRange(data []byte, mask uint, ix_start uint, ix_end uint) { |
||||
var i uint |
||||
for i = ix_start; i < ix_end; i++ { |
||||
h.Store(data, mask, i) |
||||
} |
||||
} |
||||
|
||||
func (h *hashForgetfulChain) StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []byte, ring_buffer_mask uint) { |
||||
if num_bytes >= h.HashTypeLength()-1 && position >= 3 { |
||||
/* Prepare the hashes for three last bytes of the last write. |
||||
These could not be calculated before, since they require knowledge |
||||
of both the previous and the current block. */ |
||||
h.Store(ringbuffer, ring_buffer_mask, position-3) |
||||
h.Store(ringbuffer, ring_buffer_mask, position-2) |
||||
h.Store(ringbuffer, ring_buffer_mask, position-1) |
||||
} |
||||
} |
||||
|
||||
func (h *hashForgetfulChain) PrepareDistanceCache(distance_cache []int) { |
||||
prepareDistanceCache(distance_cache, h.numLastDistancesToCheck) |
||||
} |
||||
|
||||
/* Find a longest backward match of &data[cur_ix] up to the length of |
||||
max_length and stores the position cur_ix in the hash table. |
||||
|
||||
REQUIRES: PrepareDistanceCachehashForgetfulChain must be invoked for current distance cache |
||||
values; if this method is invoked repeatedly with the same distance |
||||
cache values, it is enough to invoke PrepareDistanceCachehashForgetfulChain once. |
||||
|
||||
Does not look for matches longer than max_length. |
||||
Does not look for matches further away than max_backward. |
||||
Writes the best match into |out|. |
||||
|out|->score is updated only if a better match is found. */ |
||||
func (h *hashForgetfulChain) FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult) { |
||||
var cur_ix_masked uint = cur_ix & ring_buffer_mask |
||||
var min_score uint = out.score |
||||
var best_score uint = out.score |
||||
var best_len uint = out.len |
||||
var key uint = h.HashBytes(data[cur_ix_masked:]) |
||||
var tiny_hash byte = byte(key) |
||||
/* Don't accept a short copy from far away. */ |
||||
out.len = 0 |
||||
|
||||
out.len_code_delta = 0 |
||||
|
||||
/* Try last distance first. */ |
||||
for i := 0; i < h.numLastDistancesToCheck; i++ { |
||||
var backward uint = uint(distance_cache[i]) |
||||
var prev_ix uint = (cur_ix - backward) |
||||
|
||||
/* For distance code 0 we want to consider 2-byte matches. */ |
||||
if i > 0 && h.tiny_hash[uint16(prev_ix)] != tiny_hash { |
||||
continue |
||||
} |
||||
if prev_ix >= cur_ix || backward > max_backward { |
||||
continue |
||||
} |
||||
|
||||
prev_ix &= ring_buffer_mask |
||||
{ |
||||
var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length) |
||||
if len >= 2 { |
||||
var score uint = backwardReferenceScoreUsingLastDistance(uint(len)) |
||||
if best_score < score { |
||||
if i != 0 { |
||||
score -= backwardReferencePenaltyUsingLastDistance(uint(i)) |
||||
} |
||||
if best_score < score { |
||||
best_score = score |
||||
best_len = uint(len) |
||||
out.len = best_len |
||||
out.distance = backward |
||||
out.score = best_score |
||||
} |
||||
} |
||||
} |
||||
} |
||||
} |
||||
{ |
||||
var bank uint = key & (h.numBanks - 1) |
||||
var backward uint = 0 |
||||
var hops uint = h.max_hops |
||||
var delta uint = cur_ix - uint(h.addr[key]) |
||||
var slot uint = uint(h.head[key]) |
||||
for { |
||||
tmp6 := hops |
||||
hops-- |
||||
if tmp6 == 0 { |
||||
break |
||||
} |
||||
var prev_ix uint |
||||
var last uint = slot |
||||
backward += delta |
||||
if backward > max_backward { |
||||
break |
||||
} |
||||
prev_ix = (cur_ix - backward) & ring_buffer_mask |
||||
slot = uint(h.banks[bank][last].next) |
||||
delta = uint(h.banks[bank][last].delta) |
||||
if cur_ix_masked+best_len > ring_buffer_mask || prev_ix+best_len > ring_buffer_mask || data[cur_ix_masked+best_len] != data[prev_ix+best_len] { |
||||
continue |
||||
} |
||||
{ |
||||
var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length) |
||||
if len >= 4 { |
||||
/* Comparing for >= 3 does not change the semantics, but just saves |
||||
for a few unnecessary binary logarithms in backward reference |
||||
score, since we are not interested in such short matches. */ |
||||
var score uint = backwardReferenceScore(uint(len), backward) |
||||
if best_score < score { |
||||
best_score = score |
||||
best_len = uint(len) |
||||
out.len = best_len |
||||
out.distance = backward |
||||
out.score = best_score |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
h.Store(data, ring_buffer_mask, cur_ix) |
||||
} |
||||
|
||||
if out.score == min_score { |
||||
searchInStaticDictionary(dictionary, h, data[cur_ix_masked:], max_length, max_backward+gap, max_distance, out, false) |
||||
} |
||||
} |
||||
@ -0,0 +1,214 @@ |
||||
package brotli |
||||
|
||||
import "encoding/binary" |
||||
|
||||
/* Copyright 2010 Google Inc. All Rights Reserved. |
||||
|
||||
Distributed under MIT license. |
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/ |
||||
|
||||
/* For BUCKET_SWEEP == 1, enabling the dictionary lookup makes compression |
||||
a little faster (0.5% - 1%) and it compresses 0.15% better on small text |
||||
and HTML inputs. */ |
||||
|
||||
func (*hashLongestMatchQuickly) HashTypeLength() uint { |
||||
return 8 |
||||
} |
||||
|
||||
func (*hashLongestMatchQuickly) StoreLookahead() uint { |
||||
return 8 |
||||
} |
||||
|
||||
/* HashBytes is the function that chooses the bucket to place |
||||
the address in. The HashLongestMatch and hashLongestMatchQuickly |
||||
classes have separate, different implementations of hashing. */ |
||||
func (h *hashLongestMatchQuickly) HashBytes(data []byte) uint32 { |
||||
var hash uint64 = ((binary.LittleEndian.Uint64(data) << (64 - 8*h.hashLen)) * kHashMul64) |
||||
|
||||
/* The higher bits contain more mixture from the multiplication, |
||||
so we take our results from there. */ |
||||
return uint32(hash >> (64 - h.bucketBits)) |
||||
} |
||||
|
||||
/* A (forgetful) hash table to the data seen by the compressor, to |
||||
help create backward references to previous data. |
||||
|
||||
This is a hash map of fixed size (1 << 16). Starting from the |
||||
given index, 1 buckets are used to store values of a key. */ |
||||
type hashLongestMatchQuickly struct { |
||||
hasherCommon |
||||
|
||||
bucketBits uint |
||||
bucketSweep int |
||||
hashLen uint |
||||
useDictionary bool |
||||
|
||||
buckets []uint32 |
||||
} |
||||
|
||||
func (h *hashLongestMatchQuickly) Initialize(params *encoderParams) { |
||||
h.buckets = make([]uint32, 1<<h.bucketBits+h.bucketSweep) |
||||
} |
||||
|
||||
func (h *hashLongestMatchQuickly) Prepare(one_shot bool, input_size uint, data []byte) { |
||||
var partial_prepare_threshold uint = (4 << h.bucketBits) >> 7 |
||||
/* Partial preparation is 100 times slower (per socket). */ |
||||
if one_shot && input_size <= partial_prepare_threshold { |
||||
var i uint |
||||
for i = 0; i < input_size; i++ { |
||||
var key uint32 = h.HashBytes(data[i:]) |
||||
for j := 0; j < h.bucketSweep; j++ { |
||||
h.buckets[key+uint32(j)] = 0 |
||||
} |
||||
} |
||||
} else { |
||||
/* It is not strictly necessary to fill this buffer here, but |
||||
not filling will make the results of the compression stochastic |
||||
(but correct). This is because random data would cause the |
||||
system to find accidentally good backward references here and there. */ |
||||
for i := range h.buckets { |
||||
h.buckets[i] = 0 |
||||
} |
||||
} |
||||
} |
||||
|
||||
/* Look at 5 bytes at &data[ix & mask]. |
||||
Compute a hash from these, and store the value somewhere within |
||||
[ix .. ix+3]. */ |
||||
func (h *hashLongestMatchQuickly) Store(data []byte, mask uint, ix uint) { |
||||
var key uint32 = h.HashBytes(data[ix&mask:]) |
||||
var off uint32 = uint32(ix>>3) % uint32(h.bucketSweep) |
||||
/* Wiggle the value with the bucket sweep range. */ |
||||
h.buckets[key+off] = uint32(ix) |
||||
} |
||||
|
||||
func (h *hashLongestMatchQuickly) StoreRange(data []byte, mask uint, ix_start uint, ix_end uint) { |
||||
var i uint |
||||
for i = ix_start; i < ix_end; i++ { |
||||
h.Store(data, mask, i) |
||||
} |
||||
} |
||||
|
||||
func (h *hashLongestMatchQuickly) StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []byte, ringbuffer_mask uint) { |
||||
if num_bytes >= h.HashTypeLength()-1 && position >= 3 { |
||||
/* Prepare the hashes for three last bytes of the last write. |
||||
These could not be calculated before, since they require knowledge |
||||
of both the previous and the current block. */ |
||||
h.Store(ringbuffer, ringbuffer_mask, position-3) |
||||
h.Store(ringbuffer, ringbuffer_mask, position-2) |
||||
h.Store(ringbuffer, ringbuffer_mask, position-1) |
||||
} |
||||
} |
||||
|
||||
func (*hashLongestMatchQuickly) PrepareDistanceCache(distance_cache []int) { |
||||
} |
||||
|
||||
/* Find a longest backward match of &data[cur_ix & ring_buffer_mask] |
||||
up to the length of max_length and stores the position cur_ix in the |
||||
hash table. |
||||
|
||||
Does not look for matches longer than max_length. |
||||
Does not look for matches further away than max_backward. |
||||
Writes the best match into |out|. |
||||
|out|->score is updated only if a better match is found. */ |
||||
func (h *hashLongestMatchQuickly) FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult) { |
||||
var best_len_in uint = out.len |
||||
var cur_ix_masked uint = cur_ix & ring_buffer_mask |
||||
var key uint32 = h.HashBytes(data[cur_ix_masked:]) |
||||
var compare_char int = int(data[cur_ix_masked+best_len_in]) |
||||
var min_score uint = out.score |
||||
var best_score uint = out.score |
||||
var best_len uint = best_len_in |
||||
var cached_backward uint = uint(distance_cache[0]) |
||||
var prev_ix uint = cur_ix - cached_backward |
||||
var bucket []uint32 |
||||
out.len_code_delta = 0 |
||||
if prev_ix < cur_ix { |
||||
prev_ix &= uint(uint32(ring_buffer_mask)) |
||||
if compare_char == int(data[prev_ix+best_len]) { |
||||
var len uint = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length) |
||||
if len >= 4 { |
||||
var score uint = backwardReferenceScoreUsingLastDistance(uint(len)) |
||||
if best_score < score { |
||||
best_score = score |
||||
best_len = uint(len) |
||||
out.len = uint(len) |
||||
out.distance = cached_backward |
||||
out.score = best_score |
||||
compare_char = int(data[cur_ix_masked+best_len]) |
||||
if h.bucketSweep == 1 { |
||||
h.buckets[key] = uint32(cur_ix) |
||||
return |
||||
} |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
if h.bucketSweep == 1 { |
||||
var backward uint |
||||
var len uint |
||||
|
||||
/* Only one to look for, don't bother to prepare for a loop. */ |
||||
prev_ix = uint(h.buckets[key]) |
||||
|
||||
h.buckets[key] = uint32(cur_ix) |
||||
backward = cur_ix - prev_ix |
||||
prev_ix &= uint(uint32(ring_buffer_mask)) |
||||
if compare_char != int(data[prev_ix+best_len_in]) { |
||||
return |
||||
} |
||||
|
||||
if backward == 0 || backward > max_backward { |
||||
return |
||||
} |
||||
|
||||
len = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length) |
||||
if len >= 4 { |
||||
var score uint = backwardReferenceScore(uint(len), backward) |
||||
if best_score < score { |
||||
out.len = uint(len) |
||||
out.distance = backward |
||||
out.score = score |
||||
return |
||||
} |
||||
} |
||||
} else { |
||||
bucket = h.buckets[key:] |
||||
var i int |
||||
prev_ix = uint(bucket[0]) |
||||
bucket = bucket[1:] |
||||
for i = 0; i < h.bucketSweep; (func() { i++; tmp3 := bucket; bucket = bucket[1:]; prev_ix = uint(tmp3[0]) })() { |
||||
var backward uint = cur_ix - prev_ix |
||||
var len uint |
||||
prev_ix &= uint(uint32(ring_buffer_mask)) |
||||
if compare_char != int(data[prev_ix+best_len]) { |
||||
continue |
||||
} |
||||
|
||||
if backward == 0 || backward > max_backward { |
||||
continue |
||||
} |
||||
|
||||
len = findMatchLengthWithLimit(data[prev_ix:], data[cur_ix_masked:], max_length) |
||||
if len >= 4 { |
||||
var score uint = backwardReferenceScore(uint(len), backward) |
||||
if best_score < score { |
||||
best_score = score |
||||
best_len = uint(len) |
||||
out.len = best_len |
||||
out.distance = backward |
||||
out.score = score |
||||
compare_char = int(data[cur_ix_masked+best_len]) |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
if h.useDictionary && min_score == out.score { |
||||
searchInStaticDictionary(dictionary, h, data[cur_ix_masked:], max_length, max_backward+gap, max_distance, out, true) |
||||
} |
||||
|
||||
h.buckets[key+uint32((cur_ix>>3)%uint(h.bucketSweep))] = uint32(cur_ix) |
||||
} |
||||
@ -0,0 +1,169 @@ |
||||
package brotli |
||||
|
||||
/* Copyright 2018 Google Inc. All Rights Reserved. |
||||
|
||||
Distributed under MIT license. |
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/ |
||||
|
||||
/* NOTE: this hasher does not search in the dictionary. It is used as |
||||
backup-hasher, the main hasher already searches in it. */ |
||||
|
||||
const kRollingHashMul32 uint32 = 69069 |
||||
|
||||
const kInvalidPosHashRolling uint32 = 0xffffffff |
||||
|
||||
/* This hasher uses a longer forward length, but returning a higher value here |
||||
will hurt compression by the main hasher when combined with a composite |
||||
hasher. The hasher tests for forward itself instead. */ |
||||
func (*hashRolling) HashTypeLength() uint { |
||||
return 4 |
||||
} |
||||
|
||||
func (*hashRolling) StoreLookahead() uint { |
||||
return 4 |
||||
} |
||||
|
||||
/* Computes a code from a single byte. A lookup table of 256 values could be |
||||
used, but simply adding 1 works about as good. */ |
||||
func (*hashRolling) HashByte(b byte) uint32 { |
||||
return uint32(b) + 1 |
||||
} |
||||
|
||||
func (h *hashRolling) HashRollingFunctionInitial(state uint32, add byte, factor uint32) uint32 { |
||||
return uint32(factor*state + h.HashByte(add)) |
||||
} |
||||
|
||||
func (h *hashRolling) HashRollingFunction(state uint32, add byte, rem byte, factor uint32, factor_remove uint32) uint32 { |
||||
return uint32(factor*state + h.HashByte(add) - factor_remove*h.HashByte(rem)) |
||||
} |
||||
|
||||
/* Rolling hash for long distance long string matches. Stores one position |
||||
per bucket, bucket key is computed over a long region. */ |
||||
type hashRolling struct { |
||||
hasherCommon |
||||
|
||||
jump int |
||||
|
||||
state uint32 |
||||
table []uint32 |
||||
next_ix uint |
||||
chunk_len uint32 |
||||
factor uint32 |
||||
factor_remove uint32 |
||||
} |
||||
|
||||
func (h *hashRolling) Initialize(params *encoderParams) { |
||||
h.state = 0 |
||||
h.next_ix = 0 |
||||
|
||||
h.factor = kRollingHashMul32 |
||||
|
||||
/* Compute the factor of the oldest byte to remove: factor**steps modulo |
||||
0xffffffff (the multiplications rely on 32-bit overflow) */ |
||||
h.factor_remove = 1 |
||||
|
||||
for i := 0; i < 32; i += h.jump { |
||||
h.factor_remove *= h.factor |
||||
} |
||||
|
||||
h.table = make([]uint32, 16777216) |
||||
for i := 0; i < 16777216; i++ { |
||||
h.table[i] = kInvalidPosHashRolling |
||||
} |
||||
} |
||||
|
||||
func (h *hashRolling) Prepare(one_shot bool, input_size uint, data []byte) { |
||||
/* Too small size, cannot use this hasher. */ |
||||
if input_size < 32 { |
||||
return |
||||
} |
||||
h.state = 0 |
||||
for i := 0; i < 32; i += h.jump { |
||||
h.state = h.HashRollingFunctionInitial(h.state, data[i], h.factor) |
||||
} |
||||
} |
||||
|
||||
func (*hashRolling) Store(data []byte, mask uint, ix uint) { |
||||
} |
||||
|
||||
func (*hashRolling) StoreRange(data []byte, mask uint, ix_start uint, ix_end uint) { |
||||
} |
||||
|
||||
func (h *hashRolling) StitchToPreviousBlock(num_bytes uint, position uint, ringbuffer []byte, ring_buffer_mask uint) { |
||||
var position_masked uint |
||||
/* In this case we must re-initialize the hasher from scratch from the |
||||
current position. */ |
||||
|
||||
var available uint = num_bytes |
||||
if position&uint(h.jump-1) != 0 { |
||||
var diff uint = uint(h.jump) - (position & uint(h.jump-1)) |
||||
if diff > available { |
||||
available = 0 |
||||
} else { |
||||
available = available - diff |
||||
} |
||||
position += diff |
||||
} |
||||
|
||||
position_masked = position & ring_buffer_mask |
||||
|
||||
/* wrapping around ringbuffer not handled. */ |
||||
if available > ring_buffer_mask-position_masked { |
||||
available = ring_buffer_mask - position_masked |
||||
} |
||||
|
||||
h.Prepare(false, available, ringbuffer[position&ring_buffer_mask:]) |
||||
h.next_ix = position |
||||
} |
||||
|
||||
func (*hashRolling) PrepareDistanceCache(distance_cache []int) { |
||||
} |
||||
|
||||
func (h *hashRolling) FindLongestMatch(dictionary *encoderDictionary, data []byte, ring_buffer_mask uint, distance_cache []int, cur_ix uint, max_length uint, max_backward uint, gap uint, max_distance uint, out *hasherSearchResult) { |
||||
var cur_ix_masked uint = cur_ix & ring_buffer_mask |
||||
var pos uint = h.next_ix |
||||
|
||||
if cur_ix&uint(h.jump-1) != 0 { |
||||
return |
||||
} |
||||
|
||||
/* Not enough lookahead */ |
||||
if max_length < 32 { |
||||
return |
||||
} |
||||
|
||||
for pos = h.next_ix; pos <= cur_ix; pos += uint(h.jump) { |
||||
var code uint32 = h.state & ((16777216 * 64) - 1) |
||||
var rem byte = data[pos&ring_buffer_mask] |
||||
var add byte = data[(pos+32)&ring_buffer_mask] |
||||
var found_ix uint = uint(kInvalidPosHashRolling) |
||||
|
||||
h.state = h.HashRollingFunction(h.state, add, rem, h.factor, h.factor_remove) |
||||
|
||||
if code < 16777216 { |
||||
found_ix = uint(h.table[code]) |
||||
h.table[code] = uint32(pos) |
||||
if pos == cur_ix && uint32(found_ix) != kInvalidPosHashRolling { |
||||
/* The cast to 32-bit makes backward distances up to 4GB work even |
||||
if cur_ix is above 4GB, despite using 32-bit values in the table. */ |
||||
var backward uint = uint(uint32(cur_ix - found_ix)) |
||||
if backward <= max_backward { |
||||
var found_ix_masked uint = found_ix & ring_buffer_mask |
||||
var len uint = findMatchLengthWithLimit(data[found_ix_masked:], data[cur_ix_masked:], max_length) |
||||
if len >= 4 && len > out.len { |
||||
var score uint = backwardReferenceScore(uint(len), backward) |
||||
if score > out.score { |
||||
out.len = uint(len) |
||||
out.distance = backward |
||||
out.score = score |
||||
out.len_code_delta = 0 |
||||
} |
||||
} |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
h.next_ix = cur_ix + uint(h.jump) |
||||
} |
||||
@ -0,0 +1,227 @@ |
||||
package brotli |
||||
|
||||
import "math" |
||||
|
||||
/* The distance symbols effectively used by "Large Window Brotli" (32-bit). */ |
||||
const numHistogramDistanceSymbols = 544 |
||||
|
||||
type histogramLiteral struct { |
||||
data_ [numLiteralSymbols]uint32 |
||||
total_count_ uint |
||||
bit_cost_ float64 |
||||
} |
||||
|
||||
func histogramClearLiteral(self *histogramLiteral) { |
||||
self.data_ = [numLiteralSymbols]uint32{} |
||||
self.total_count_ = 0 |
||||
self.bit_cost_ = math.MaxFloat64 |
||||
} |
||||
|
||||
func clearHistogramsLiteral(array []histogramLiteral, length uint) { |
||||
var i uint |
||||
for i = 0; i < length; i++ { |
||||
histogramClearLiteral(&array[i:][0]) |
||||
} |
||||
} |
||||
|
||||
func histogramAddLiteral(self *histogramLiteral, val uint) { |
||||
self.data_[val]++ |
||||
self.total_count_++ |
||||
} |
||||
|
||||
func histogramAddVectorLiteral(self *histogramLiteral, p []byte, n uint) { |
||||
self.total_count_ += n |
||||
n += 1 |
||||
for { |
||||
n-- |
||||
if n == 0 { |
||||
break |
||||
} |
||||
self.data_[p[0]]++ |
||||
p = p[1:] |
||||
} |
||||
} |
||||
|
||||
func histogramAddHistogramLiteral(self *histogramLiteral, v *histogramLiteral) { |
||||
var i uint |
||||
self.total_count_ += v.total_count_ |
||||
for i = 0; i < numLiteralSymbols; i++ { |
||||
self.data_[i] += v.data_[i] |
||||
} |
||||
} |
||||
|
||||
func histogramDataSizeLiteral() uint { |
||||
return numLiteralSymbols |
||||
} |
||||
|
||||
type histogramCommand struct { |
||||
data_ [numCommandSymbols]uint32 |
||||
total_count_ uint |
||||
bit_cost_ float64 |
||||
} |
||||
|
||||
func histogramClearCommand(self *histogramCommand) { |
||||
self.data_ = [numCommandSymbols]uint32{} |
||||
self.total_count_ = 0 |
||||
self.bit_cost_ = math.MaxFloat64 |
||||
} |
||||
|
||||
func clearHistogramsCommand(array []histogramCommand, length uint) { |
||||
var i uint |
||||
for i = 0; i < length; i++ { |
||||
histogramClearCommand(&array[i:][0]) |
||||
} |
||||
} |
||||
|
||||
func histogramAddCommand(self *histogramCommand, val uint) { |
||||
self.data_[val]++ |
||||
self.total_count_++ |
||||
} |
||||
|
||||
func histogramAddVectorCommand(self *histogramCommand, p []uint16, n uint) { |
||||
self.total_count_ += n |
||||
n += 1 |
||||
for { |
||||
n-- |
||||
if n == 0 { |
||||
break |
||||
} |
||||
self.data_[p[0]]++ |
||||
p = p[1:] |
||||
} |
||||
} |
||||
|
||||
func histogramAddHistogramCommand(self *histogramCommand, v *histogramCommand) { |
||||
var i uint |
||||
self.total_count_ += v.total_count_ |
||||
for i = 0; i < numCommandSymbols; i++ { |
||||
self.data_[i] += v.data_[i] |
||||
} |
||||
} |
||||
|
||||
func histogramDataSizeCommand() uint { |
||||
return numCommandSymbols |
||||
} |
||||
|
||||
type histogramDistance struct { |
||||
data_ [numDistanceSymbols]uint32 |
||||
total_count_ uint |
||||
bit_cost_ float64 |
||||
} |
||||
|
||||
func histogramClearDistance(self *histogramDistance) { |
||||
self.data_ = [numDistanceSymbols]uint32{} |
||||
self.total_count_ = 0 |
||||
self.bit_cost_ = math.MaxFloat64 |
||||
} |
||||
|
||||
func clearHistogramsDistance(array []histogramDistance, length uint) { |
||||
var i uint |
||||
for i = 0; i < length; i++ { |
||||
histogramClearDistance(&array[i:][0]) |
||||
} |
||||
} |
||||
|
||||
func histogramAddDistance(self *histogramDistance, val uint) { |
||||
self.data_[val]++ |
||||
self.total_count_++ |
||||
} |
||||
|
||||
func histogramAddVectorDistance(self *histogramDistance, p []uint16, n uint) { |
||||
self.total_count_ += n |
||||
n += 1 |
||||
for { |
||||
n-- |
||||
if n == 0 { |
||||
break |
||||
} |
||||
self.data_[p[0]]++ |
||||
p = p[1:] |
||||
} |
||||
} |
||||
|
||||
func histogramAddHistogramDistance(self *histogramDistance, v *histogramDistance) { |
||||
var i uint |
||||
self.total_count_ += v.total_count_ |
||||
for i = 0; i < numDistanceSymbols; i++ { |
||||
self.data_[i] += v.data_[i] |
||||
} |
||||
} |
||||
|
||||
func histogramDataSizeDistance() uint { |
||||
return numDistanceSymbols |
||||
} |
||||
|
||||
type blockSplitIterator struct { |
||||
split_ *blockSplit |
||||
idx_ uint |
||||
type_ uint |
||||
length_ uint |
||||
} |
||||
|
||||
func initBlockSplitIterator(self *blockSplitIterator, split *blockSplit) { |
||||
self.split_ = split |
||||
self.idx_ = 0 |
||||
self.type_ = 0 |
||||
if split.lengths != nil { |
||||
self.length_ = uint(split.lengths[0]) |
||||
} else { |
||||
self.length_ = 0 |
||||
} |
||||
} |
||||
|
||||
func blockSplitIteratorNext(self *blockSplitIterator) { |
||||
if self.length_ == 0 { |
||||
self.idx_++ |
||||
self.type_ = uint(self.split_.types[self.idx_]) |
||||
self.length_ = uint(self.split_.lengths[self.idx_]) |
||||
} |
||||
|
||||
self.length_-- |
||||
} |
||||
|
||||
func buildHistogramsWithContext(cmds []command, num_commands uint, literal_split *blockSplit, insert_and_copy_split *blockSplit, dist_split *blockSplit, ringbuffer []byte, start_pos uint, mask uint, prev_byte byte, prev_byte2 byte, context_modes []int, literal_histograms []histogramLiteral, insert_and_copy_histograms []histogramCommand, copy_dist_histograms []histogramDistance) { |
||||
var pos uint = start_pos |
||||
var literal_it blockSplitIterator |
||||
var insert_and_copy_it blockSplitIterator |
||||
var dist_it blockSplitIterator |
||||
var i uint |
||||
|
||||
initBlockSplitIterator(&literal_it, literal_split) |
||||
initBlockSplitIterator(&insert_and_copy_it, insert_and_copy_split) |
||||
initBlockSplitIterator(&dist_it, dist_split) |
||||
for i = 0; i < num_commands; i++ { |
||||
var cmd *command = &cmds[i] |
||||
var j uint |
||||
blockSplitIteratorNext(&insert_and_copy_it) |
||||
histogramAddCommand(&insert_and_copy_histograms[insert_and_copy_it.type_], uint(cmd.cmd_prefix_)) |
||||
|
||||
/* TODO: unwrap iterator blocks. */ |
||||
for j = uint(cmd.insert_len_); j != 0; j-- { |
||||
var context uint |
||||
blockSplitIteratorNext(&literal_it) |
||||
context = literal_it.type_ |
||||
if context_modes != nil { |
||||
var lut contextLUT = getContextLUT(context_modes[context]) |
||||
context = (context << literalContextBits) + uint(getContext(prev_byte, prev_byte2, lut)) |
||||
} |
||||
|
||||
histogramAddLiteral(&literal_histograms[context], uint(ringbuffer[pos&mask])) |
||||
prev_byte2 = prev_byte |
||||
prev_byte = ringbuffer[pos&mask] |
||||
pos++ |
||||
} |
||||
|
||||
pos += uint(commandCopyLen(cmd)) |
||||
if commandCopyLen(cmd) != 0 { |
||||
prev_byte2 = ringbuffer[(pos-2)&mask] |
||||
prev_byte = ringbuffer[(pos-1)&mask] |
||||
if cmd.cmd_prefix_ >= 128 { |
||||
var context uint |
||||
blockSplitIteratorNext(&dist_it) |
||||
context = uint(uint32(dist_it.type_<<distanceContextBits) + commandDistanceContext(cmd)) |
||||
histogramAddDistance(©_dist_histograms[context], uint(cmd.dist_prefix_)&0x3FF) |
||||
} |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,653 @@ |
||||
package brotli |
||||
|
||||
/* Copyright 2013 Google Inc. All Rights Reserved. |
||||
|
||||
Distributed under MIT license. |
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/ |
||||
|
||||
/* Utilities for building Huffman decoding tables. */ |
||||
|
||||
const huffmanMaxCodeLength = 15 |
||||
|
||||
/* Maximum possible Huffman table size for an alphabet size of (index * 32), |
||||
max code length 15 and root table bits 8. */ |
||||
var kMaxHuffmanTableSize = []uint16{ |
||||
256, |
||||
402, |
||||
436, |
||||
468, |
||||
500, |
||||
534, |
||||
566, |
||||
598, |
||||
630, |
||||
662, |
||||
694, |
||||
726, |
||||
758, |
||||
790, |
||||
822, |
||||
854, |
||||
886, |
||||
920, |
||||
952, |
||||
984, |
||||
1016, |
||||
1048, |
||||
1080, |
||||
1112, |
||||
1144, |
||||
1176, |
||||
1208, |
||||
1240, |
||||
1272, |
||||
1304, |
||||
1336, |
||||
1368, |
||||
1400, |
||||
1432, |
||||
1464, |
||||
1496, |
||||
1528, |
||||
} |
||||
|
||||
/* BROTLI_NUM_BLOCK_LEN_SYMBOLS == 26 */ |
||||
const huffmanMaxSize26 = 396 |
||||
|
||||
/* BROTLI_MAX_BLOCK_TYPE_SYMBOLS == 258 */ |
||||
const huffmanMaxSize258 = 632 |
||||
|
||||
/* BROTLI_MAX_CONTEXT_MAP_SYMBOLS == 272 */ |
||||
const huffmanMaxSize272 = 646 |
||||
|
||||
const huffmanMaxCodeLengthCodeLength = 5 |
||||
|
||||
/* Do not create this struct directly - use the ConstructHuffmanCode |
||||
* constructor below! */ |
||||
type huffmanCode struct { |
||||
bits byte |
||||
value uint16 |
||||
} |
||||
|
||||
func constructHuffmanCode(bits byte, value uint16) huffmanCode { |
||||
var h huffmanCode |
||||
h.bits = bits |
||||
h.value = value |
||||
return h |
||||
} |
||||
|
||||
/* Builds Huffman lookup table assuming code lengths are in symbol order. */ |
||||
|
||||
/* Builds Huffman lookup table assuming code lengths are in symbol order. |
||||
Returns size of resulting table. */ |
||||
|
||||
/* Builds a simple Huffman table. The |num_symbols| parameter is to be |
||||
interpreted as follows: 0 means 1 symbol, 1 means 2 symbols, |
||||
2 means 3 symbols, 3 means 4 symbols with lengths [2, 2, 2, 2], |
||||
4 means 4 symbols with lengths [1, 2, 3, 3]. */ |
||||
|
||||
/* Contains a collection of Huffman trees with the same alphabet size. */ |
||||
/* max_symbol is needed due to simple codes since log2(alphabet_size) could be |
||||
greater than log2(max_symbol). */ |
||||
type huffmanTreeGroup struct { |
||||
htrees [][]huffmanCode |
||||
codes []huffmanCode |
||||
alphabet_size uint16 |
||||
max_symbol uint16 |
||||
num_htrees uint16 |
||||
} |
||||
|
||||
const reverseBitsMax = 8 |
||||
|
||||
const reverseBitsBase = 0 |
||||
|
||||
var kReverseBits = [1 << reverseBitsMax]byte{ |
||||
0x00, |
||||
0x80, |
||||
0x40, |
||||
0xC0, |
||||
0x20, |
||||
0xA0, |
||||
0x60, |
||||
0xE0, |
||||
0x10, |
||||
0x90, |
||||
0x50, |
||||
0xD0, |
||||
0x30, |
||||
0xB0, |
||||
0x70, |
||||
0xF0, |
||||
0x08, |
||||
0x88, |
||||
0x48, |
||||
0xC8, |
||||
0x28, |
||||
0xA8, |
||||
0x68, |
||||
0xE8, |
||||
0x18, |
||||
0x98, |
||||
0x58, |
||||
0xD8, |
||||
0x38, |
||||
0xB8, |
||||
0x78, |
||||
0xF8, |
||||
0x04, |
||||
0x84, |
||||
0x44, |
||||
0xC4, |
||||
0x24, |
||||
0xA4, |
||||
0x64, |
||||
0xE4, |
||||
0x14, |
||||
0x94, |
||||
0x54, |
||||
0xD4, |
||||
0x34, |
||||
0xB4, |
||||
0x74, |
||||
0xF4, |
||||
0x0C, |
||||
0x8C, |
||||
0x4C, |
||||
0xCC, |
||||
0x2C, |
||||
0xAC, |
||||
0x6C, |
||||
0xEC, |
||||
0x1C, |
||||
0x9C, |
||||
0x5C, |
||||
0xDC, |
||||
0x3C, |
||||
0xBC, |
||||
0x7C, |
||||
0xFC, |
||||
0x02, |
||||
0x82, |
||||
0x42, |
||||
0xC2, |
||||
0x22, |
||||
0xA2, |
||||
0x62, |
||||
0xE2, |
||||
0x12, |
||||
0x92, |
||||
0x52, |
||||
0xD2, |
||||
0x32, |
||||
0xB2, |
||||
0x72, |
||||
0xF2, |
||||
0x0A, |
||||
0x8A, |
||||
0x4A, |
||||
0xCA, |
||||
0x2A, |
||||
0xAA, |
||||
0x6A, |
||||
0xEA, |
||||
0x1A, |
||||
0x9A, |
||||
0x5A, |
||||
0xDA, |
||||
0x3A, |
||||
0xBA, |
||||
0x7A, |
||||
0xFA, |
||||
0x06, |
||||
0x86, |
||||
0x46, |
||||
0xC6, |
||||
0x26, |
||||
0xA6, |
||||
0x66, |
||||
0xE6, |
||||
0x16, |
||||
0x96, |
||||
0x56, |
||||
0xD6, |
||||
0x36, |
||||
0xB6, |
||||
0x76, |
||||
0xF6, |
||||
0x0E, |
||||
0x8E, |
||||
0x4E, |
||||
0xCE, |
||||
0x2E, |
||||
0xAE, |
||||
0x6E, |
||||
0xEE, |
||||
0x1E, |
||||
0x9E, |
||||
0x5E, |
||||
0xDE, |
||||
0x3E, |
||||
0xBE, |
||||
0x7E, |
||||
0xFE, |
||||
0x01, |
||||
0x81, |
||||
0x41, |
||||
0xC1, |
||||
0x21, |
||||
0xA1, |
||||
0x61, |
||||
0xE1, |
||||
0x11, |
||||
0x91, |
||||
0x51, |
||||
0xD1, |
||||
0x31, |
||||
0xB1, |
||||
0x71, |
||||
0xF1, |
||||
0x09, |
||||
0x89, |
||||
0x49, |
||||
0xC9, |
||||
0x29, |
||||
0xA9, |
||||
0x69, |
||||
0xE9, |
||||
0x19, |
||||
0x99, |
||||
0x59, |
||||
0xD9, |
||||
0x39, |
||||
0xB9, |
||||
0x79, |
||||
0xF9, |
||||
0x05, |
||||
0x85, |
||||
0x45, |
||||
0xC5, |
||||
0x25, |
||||
0xA5, |
||||
0x65, |
||||
0xE5, |
||||
0x15, |
||||
0x95, |
||||
0x55, |
||||
0xD5, |
||||
0x35, |
||||
0xB5, |
||||
0x75, |
||||
0xF5, |
||||
0x0D, |
||||
0x8D, |
||||
0x4D, |
||||
0xCD, |
||||
0x2D, |
||||
0xAD, |
||||
0x6D, |
||||
0xED, |
||||
0x1D, |
||||
0x9D, |
||||
0x5D, |
||||
0xDD, |
||||
0x3D, |
||||
0xBD, |
||||
0x7D, |
||||
0xFD, |
||||
0x03, |
||||
0x83, |
||||
0x43, |
||||
0xC3, |
||||
0x23, |
||||
0xA3, |
||||
0x63, |
||||
0xE3, |
||||
0x13, |
||||
0x93, |
||||
0x53, |
||||
0xD3, |
||||
0x33, |
||||
0xB3, |
||||
0x73, |
||||
0xF3, |
||||
0x0B, |
||||
0x8B, |
||||
0x4B, |
||||
0xCB, |
||||
0x2B, |
||||
0xAB, |
||||
0x6B, |
||||
0xEB, |
||||
0x1B, |
||||
0x9B, |
||||
0x5B, |
||||
0xDB, |
||||
0x3B, |
||||
0xBB, |
||||
0x7B, |
||||
0xFB, |
||||
0x07, |
||||
0x87, |
||||
0x47, |
||||
0xC7, |
||||
0x27, |
||||
0xA7, |
||||
0x67, |
||||
0xE7, |
||||
0x17, |
||||
0x97, |
||||
0x57, |
||||
0xD7, |
||||
0x37, |
||||
0xB7, |
||||
0x77, |
||||
0xF7, |
||||
0x0F, |
||||
0x8F, |
||||
0x4F, |
||||
0xCF, |
||||
0x2F, |
||||
0xAF, |
||||
0x6F, |
||||
0xEF, |
||||
0x1F, |
||||
0x9F, |
||||
0x5F, |
||||
0xDF, |
||||
0x3F, |
||||
0xBF, |
||||
0x7F, |
||||
0xFF, |
||||
} |
||||
|
||||
const reverseBitsLowest = (uint64(1) << (reverseBitsMax - 1 + reverseBitsBase)) |
||||
|
||||
/* Returns reverse(num >> BROTLI_REVERSE_BITS_BASE, BROTLI_REVERSE_BITS_MAX), |
||||
where reverse(value, len) is the bit-wise reversal of the len least |
||||
significant bits of value. */ |
||||
func reverseBits8(num uint64) uint64 { |
||||
return uint64(kReverseBits[num]) |
||||
} |
||||
|
||||
/* Stores code in table[0], table[step], table[2*step], ..., table[end] */ |
||||
/* Assumes that end is an integer multiple of step */ |
||||
func replicateValue(table []huffmanCode, step int, end int, code huffmanCode) { |
||||
for { |
||||
end -= step |
||||
table[end] = code |
||||
if end <= 0 { |
||||
break |
||||
} |
||||
} |
||||
} |
||||
|
||||
/* Returns the table width of the next 2nd level table. |count| is the histogram |
||||
of bit lengths for the remaining symbols, |len| is the code length of the |
||||
next processed symbol. */ |
||||
func nextTableBitSize(count []uint16, len int, root_bits int) int { |
||||
var left int = 1 << uint(len-root_bits) |
||||
for len < huffmanMaxCodeLength { |
||||
left -= int(count[len]) |
||||
if left <= 0 { |
||||
break |
||||
} |
||||
len++ |
||||
left <<= 1 |
||||
} |
||||
|
||||
return len - root_bits |
||||
} |
||||
|
||||
func buildCodeLengthsHuffmanTable(table []huffmanCode, code_lengths []byte, count []uint16) { |
||||
var code huffmanCode /* current table entry */ /* symbol index in original or sorted table */ /* prefix code */ /* prefix code addend */ /* step size to replicate values in current table */ /* size of current table */ /* symbols sorted by code length */ |
||||
var symbol int |
||||
var key uint64 |
||||
var key_step uint64 |
||||
var step int |
||||
var table_size int |
||||
var sorted [codeLengthCodes]int |
||||
var offset [huffmanMaxCodeLengthCodeLength + 1]int |
||||
var bits int |
||||
var bits_count int |
||||
/* offsets in sorted table for each length */ |
||||
assert(huffmanMaxCodeLengthCodeLength <= reverseBitsMax) |
||||
|
||||
/* Generate offsets into sorted symbol table by code length. */ |
||||
symbol = -1 |
||||
|
||||
bits = 1 |
||||
var i int |
||||
for i = 0; i < huffmanMaxCodeLengthCodeLength; i++ { |
||||
symbol += int(count[bits]) |
||||
offset[bits] = symbol |
||||
bits++ |
||||
} |
||||
|
||||
/* Symbols with code length 0 are placed after all other symbols. */ |
||||
offset[0] = codeLengthCodes - 1 |
||||
|
||||
/* Sort symbols by length, by symbol order within each length. */ |
||||
symbol = codeLengthCodes |
||||
|
||||
for { |
||||
var i int |
||||
for i = 0; i < 6; i++ { |
||||
symbol-- |
||||
sorted[offset[code_lengths[symbol]]] = symbol |
||||
offset[code_lengths[symbol]]-- |
||||
} |
||||
if symbol == 0 { |
||||
break |
||||
} |
||||
} |
||||
|
||||
table_size = 1 << huffmanMaxCodeLengthCodeLength |
||||
|
||||
/* Special case: all symbols but one have 0 code length. */ |
||||
if offset[0] == 0 { |
||||
code = constructHuffmanCode(0, uint16(sorted[0])) |
||||
for key = 0; key < uint64(table_size); key++ { |
||||
table[key] = code |
||||
} |
||||
|
||||
return |
||||
} |
||||
|
||||
/* Fill in table. */ |
||||
key = 0 |
||||
|
||||
key_step = reverseBitsLowest |
||||
symbol = 0 |
||||
bits = 1 |
||||
step = 2 |
||||
for { |
||||
for bits_count = int(count[bits]); bits_count != 0; bits_count-- { |
||||
code = constructHuffmanCode(byte(bits), uint16(sorted[symbol])) |
||||
symbol++ |
||||
replicateValue(table[reverseBits8(key):], step, table_size, code) |
||||
key += key_step |
||||
} |
||||
|
||||
step <<= 1 |
||||
key_step >>= 1 |
||||
bits++ |
||||
if bits > huffmanMaxCodeLengthCodeLength { |
||||
break |
||||
} |
||||
} |
||||
} |
||||
|
||||
func buildHuffmanTable(root_table []huffmanCode, root_bits int, symbol_lists symbolList, count []uint16) uint32 { |
||||
var code huffmanCode /* current table entry */ /* next available space in table */ /* current code length */ /* symbol index in original or sorted table */ /* prefix code */ /* prefix code addend */ /* 2nd level table prefix code */ /* 2nd level table prefix code addend */ /* step size to replicate values in current table */ /* key length of current table */ /* size of current table */ /* sum of root table size and 2nd level table sizes */ |
||||
var table []huffmanCode |
||||
var len int |
||||
var symbol int |
||||
var key uint64 |
||||
var key_step uint64 |
||||
var sub_key uint64 |
||||
var sub_key_step uint64 |
||||
var step int |
||||
var table_bits int |
||||
var table_size int |
||||
var total_size int |
||||
var max_length int = -1 |
||||
var bits int |
||||
var bits_count int |
||||
|
||||
assert(root_bits <= reverseBitsMax) |
||||
assert(huffmanMaxCodeLength-root_bits <= reverseBitsMax) |
||||
|
||||
for symbolListGet(symbol_lists, max_length) == 0xFFFF { |
||||
max_length-- |
||||
} |
||||
max_length += huffmanMaxCodeLength + 1 |
||||
|
||||
table = root_table |
||||
table_bits = root_bits |
||||
table_size = 1 << uint(table_bits) |
||||
total_size = table_size |
||||
|
||||
/* Fill in the root table. Reduce the table size to if possible, |
||||
and create the repetitions by memcpy. */ |
||||
if table_bits > max_length { |
||||
table_bits = max_length |
||||
table_size = 1 << uint(table_bits) |
||||
} |
||||
|
||||
key = 0 |
||||
key_step = reverseBitsLowest |
||||
bits = 1 |
||||
step = 2 |
||||
for { |
||||
symbol = bits - (huffmanMaxCodeLength + 1) |
||||
for bits_count = int(count[bits]); bits_count != 0; bits_count-- { |
||||
symbol = int(symbolListGet(symbol_lists, symbol)) |
||||
code = constructHuffmanCode(byte(bits), uint16(symbol)) |
||||
replicateValue(table[reverseBits8(key):], step, table_size, code) |
||||
key += key_step |
||||
} |
||||
|
||||
step <<= 1 |
||||
key_step >>= 1 |
||||
bits++ |
||||
if bits > table_bits { |
||||
break |
||||
} |
||||
} |
||||
|
||||
/* If root_bits != table_bits then replicate to fill the remaining slots. */ |
||||
for total_size != table_size { |
||||
copy(table[table_size:], table[:uint(table_size)]) |
||||
table_size <<= 1 |
||||
} |
||||
|
||||
/* Fill in 2nd level tables and add pointers to root table. */ |
||||
key_step = reverseBitsLowest >> uint(root_bits-1) |
||||
|
||||
sub_key = reverseBitsLowest << 1 |
||||
sub_key_step = reverseBitsLowest |
||||
len = root_bits + 1 |
||||
step = 2 |
||||
for ; len <= max_length; len++ { |
||||
symbol = len - (huffmanMaxCodeLength + 1) |
||||
for ; count[len] != 0; count[len]-- { |
||||
if sub_key == reverseBitsLowest<<1 { |
||||
table = table[table_size:] |
||||
table_bits = nextTableBitSize(count, int(len), root_bits) |
||||
table_size = 1 << uint(table_bits) |
||||
total_size += table_size |
||||
sub_key = reverseBits8(key) |
||||
key += key_step |
||||
root_table[sub_key] = constructHuffmanCode(byte(table_bits+root_bits), uint16(uint64(uint(-cap(table)+cap(root_table)))-sub_key)) |
||||
sub_key = 0 |
||||
} |
||||
|
||||
symbol = int(symbolListGet(symbol_lists, symbol)) |
||||
code = constructHuffmanCode(byte(len-root_bits), uint16(symbol)) |
||||
replicateValue(table[reverseBits8(sub_key):], step, table_size, code) |
||||
sub_key += sub_key_step |
||||
} |
||||
|
||||
step <<= 1 |
||||
sub_key_step >>= 1 |
||||
} |
||||
|
||||
return uint32(total_size) |
||||
} |
||||
|
||||
func buildSimpleHuffmanTable(table []huffmanCode, root_bits int, val []uint16, num_symbols uint32) uint32 { |
||||
var table_size uint32 = 1 |
||||
var goal_size uint32 = 1 << uint(root_bits) |
||||
switch num_symbols { |
||||
case 0: |
||||
table[0] = constructHuffmanCode(0, val[0]) |
||||
|
||||
case 1: |
||||
if val[1] > val[0] { |
||||
table[0] = constructHuffmanCode(1, val[0]) |
||||
table[1] = constructHuffmanCode(1, val[1]) |
||||
} else { |
||||
table[0] = constructHuffmanCode(1, val[1]) |
||||
table[1] = constructHuffmanCode(1, val[0]) |
||||
} |
||||
|
||||
table_size = 2 |
||||
|
||||
case 2: |
||||
table[0] = constructHuffmanCode(1, val[0]) |
||||
table[2] = constructHuffmanCode(1, val[0]) |
||||
if val[2] > val[1] { |
||||
table[1] = constructHuffmanCode(2, val[1]) |
||||
table[3] = constructHuffmanCode(2, val[2]) |
||||
} else { |
||||
table[1] = constructHuffmanCode(2, val[2]) |
||||
table[3] = constructHuffmanCode(2, val[1]) |
||||
} |
||||
|
||||
table_size = 4 |
||||
|
||||
case 3: |
||||
var i int |
||||
var k int |
||||
for i = 0; i < 3; i++ { |
||||
for k = i + 1; k < 4; k++ { |
||||
if val[k] < val[i] { |
||||
var t uint16 = val[k] |
||||
val[k] = val[i] |
||||
val[i] = t |
||||
} |
||||
} |
||||
} |
||||
|
||||
table[0] = constructHuffmanCode(2, val[0]) |
||||
table[2] = constructHuffmanCode(2, val[1]) |
||||
table[1] = constructHuffmanCode(2, val[2]) |
||||
table[3] = constructHuffmanCode(2, val[3]) |
||||
table_size = 4 |
||||
|
||||
case 4: |
||||
if val[3] < val[2] { |
||||
var t uint16 = val[3] |
||||
val[3] = val[2] |
||||
val[2] = t |
||||
} |
||||
|
||||
table[0] = constructHuffmanCode(1, val[0]) |
||||
table[1] = constructHuffmanCode(2, val[1]) |
||||
table[2] = constructHuffmanCode(1, val[0]) |
||||
table[3] = constructHuffmanCode(3, val[2]) |
||||
table[4] = constructHuffmanCode(1, val[0]) |
||||
table[5] = constructHuffmanCode(2, val[1]) |
||||
table[6] = constructHuffmanCode(1, val[0]) |
||||
table[7] = constructHuffmanCode(3, val[3]) |
||||
table_size = 8 |
||||
} |
||||
|
||||
for table_size != goal_size { |
||||
copy(table[table_size:], table[:uint(table_size)]) |
||||
table_size <<= 1 |
||||
} |
||||
|
||||
return goal_size |
||||
} |
||||
@ -0,0 +1,182 @@ |
||||
package brotli |
||||
|
||||
func utf8Position(last uint, c uint, clamp uint) uint { |
||||
if c < 128 { |
||||
return 0 /* Next one is the 'Byte 1' again. */ |
||||
} else if c >= 192 { /* Next one is the 'Byte 2' of utf-8 encoding. */ |
||||
return brotli_min_size_t(1, clamp) |
||||
} else { |
||||
/* Let's decide over the last byte if this ends the sequence. */ |
||||
if last < 0xE0 { |
||||
return 0 /* Completed two or three byte coding. */ /* Next one is the 'Byte 3' of utf-8 encoding. */ |
||||
} else { |
||||
return brotli_min_size_t(2, clamp) |
||||
} |
||||
} |
||||
} |
||||
|
||||
func decideMultiByteStatsLevel(pos uint, len uint, mask uint, data []byte) uint { |
||||
var counts = [3]uint{0} /* should be 2, but 1 compresses better. */ |
||||
var max_utf8 uint = 1 |
||||
var last_c uint = 0 |
||||
var i uint |
||||
for i = 0; i < len; i++ { |
||||
var c uint = uint(data[(pos+i)&mask]) |
||||
counts[utf8Position(last_c, c, 2)]++ |
||||
last_c = c |
||||
} |
||||
|
||||
if counts[2] < 500 { |
||||
max_utf8 = 1 |
||||
} |
||||
|
||||
if counts[1]+counts[2] < 25 { |
||||
max_utf8 = 0 |
||||
} |
||||
|
||||
return max_utf8 |
||||
} |
||||
|
||||
func estimateBitCostsForLiteralsUTF8(pos uint, len uint, mask uint, data []byte, cost []float32) { |
||||
var max_utf8 uint = decideMultiByteStatsLevel(pos, uint(len), mask, data) |
||||
/* Bootstrap histograms. */ |
||||
var histogram = [3][256]uint{[256]uint{0}} |
||||
var window_half uint = 495 |
||||
var in_window uint = brotli_min_size_t(window_half, uint(len)) |
||||
var in_window_utf8 = [3]uint{0} |
||||
/* max_utf8 is 0 (normal ASCII single byte modeling), |
||||
1 (for 2-byte UTF-8 modeling), or 2 (for 3-byte UTF-8 modeling). */ |
||||
|
||||
var i uint |
||||
{ |
||||
var last_c uint = 0 |
||||
var utf8_pos uint = 0 |
||||
for i = 0; i < in_window; i++ { |
||||
var c uint = uint(data[(pos+i)&mask]) |
||||
histogram[utf8_pos][c]++ |
||||
in_window_utf8[utf8_pos]++ |
||||
utf8_pos = utf8Position(last_c, c, max_utf8) |
||||
last_c = c |
||||
} |
||||
} |
||||
|
||||
/* Compute bit costs with sliding window. */ |
||||
for i = 0; i < len; i++ { |
||||
if i >= window_half { |
||||
var c uint |
||||
var last_c uint |
||||
if i < window_half+1 { |
||||
c = 0 |
||||
} else { |
||||
c = uint(data[(pos+i-window_half-1)&mask]) |
||||
} |
||||
if i < window_half+2 { |
||||
last_c = 0 |
||||
} else { |
||||
last_c = uint(data[(pos+i-window_half-2)&mask]) |
||||
} |
||||
/* Remove a byte in the past. */ |
||||
|
||||
var utf8_pos2 uint = utf8Position(last_c, c, max_utf8) |
||||
histogram[utf8_pos2][data[(pos+i-window_half)&mask]]-- |
||||
in_window_utf8[utf8_pos2]-- |
||||
} |
||||
|
||||
if i+window_half < len { |
||||
var c uint = uint(data[(pos+i+window_half-1)&mask]) |
||||
var last_c uint = uint(data[(pos+i+window_half-2)&mask]) |
||||
/* Add a byte in the future. */ |
||||
|
||||
var utf8_pos2 uint = utf8Position(last_c, c, max_utf8) |
||||
histogram[utf8_pos2][data[(pos+i+window_half)&mask]]++ |
||||
in_window_utf8[utf8_pos2]++ |
||||
} |
||||
{ |
||||
var c uint |
||||
var last_c uint |
||||
if i < 1 { |
||||
c = 0 |
||||
} else { |
||||
c = uint(data[(pos+i-1)&mask]) |
||||
} |
||||
if i < 2 { |
||||
last_c = 0 |
||||
} else { |
||||
last_c = uint(data[(pos+i-2)&mask]) |
||||
} |
||||
var utf8_pos uint = utf8Position(last_c, c, max_utf8) |
||||
var masked_pos uint = (pos + i) & mask |
||||
var histo uint = histogram[utf8_pos][data[masked_pos]] |
||||
var lit_cost float64 |
||||
if histo == 0 { |
||||
histo = 1 |
||||
} |
||||
|
||||
lit_cost = fastLog2(in_window_utf8[utf8_pos]) - fastLog2(histo) |
||||
lit_cost += 0.02905 |
||||
if lit_cost < 1.0 { |
||||
lit_cost *= 0.5 |
||||
lit_cost += 0.5 |
||||
} |
||||
|
||||
/* Make the first bytes more expensive -- seems to help, not sure why. |
||||
Perhaps because the entropy source is changing its properties |
||||
rapidly in the beginning of the file, perhaps because the beginning |
||||
of the data is a statistical "anomaly". */ |
||||
if i < 2000 { |
||||
lit_cost += 0.7 - (float64(2000-i) / 2000.0 * 0.35) |
||||
} |
||||
|
||||
cost[i] = float32(lit_cost) |
||||
} |
||||
} |
||||
} |
||||
|
||||
func estimateBitCostsForLiterals(pos uint, len uint, mask uint, data []byte, cost []float32) { |
||||
if isMostlyUTF8(data, pos, mask, uint(len), kMinUTF8Ratio) { |
||||
estimateBitCostsForLiteralsUTF8(pos, uint(len), mask, data, cost) |
||||
return |
||||
} else { |
||||
var histogram = [256]uint{0} |
||||
var window_half uint = 2000 |
||||
var in_window uint = brotli_min_size_t(window_half, uint(len)) |
||||
var i uint |
||||
/* Bootstrap histogram. */ |
||||
for i = 0; i < in_window; i++ { |
||||
histogram[data[(pos+i)&mask]]++ |
||||
} |
||||
|
||||
/* Compute bit costs with sliding window. */ |
||||
for i = 0; i < len; i++ { |
||||
var histo uint |
||||
if i >= window_half { |
||||
/* Remove a byte in the past. */ |
||||
histogram[data[(pos+i-window_half)&mask]]-- |
||||
|
||||
in_window-- |
||||
} |
||||
|
||||
if i+window_half < len { |
||||
/* Add a byte in the future. */ |
||||
histogram[data[(pos+i+window_half)&mask]]++ |
||||
|
||||
in_window++ |
||||
} |
||||
|
||||
histo = histogram[data[(pos+i)&mask]] |
||||
if histo == 0 { |
||||
histo = 1 |
||||
} |
||||
{ |
||||
var lit_cost float64 = fastLog2(in_window) - fastLog2(histo) |
||||
lit_cost += 0.029 |
||||
if lit_cost < 1.0 { |
||||
lit_cost *= 0.5 |
||||
lit_cost += 0.5 |
||||
} |
||||
|
||||
cost[i] = float32(lit_cost) |
||||
} |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,56 @@ |
||||
package brotli |
||||
|
||||
/* Copyright 2016 Google Inc. All Rights Reserved. |
||||
|
||||
Distributed under MIT license. |
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/ |
||||
|
||||
/* |
||||
Dynamically grows array capacity to at least the requested size |
||||
T: data type |
||||
A: array |
||||
C: capacity |
||||
R: requested size |
||||
*/ |
||||
func brotli_ensure_capacity_uint8_t(a *[]byte, c *uint, r uint) { |
||||
if *c < r { |
||||
var new_size uint = *c |
||||
if new_size == 0 { |
||||
new_size = r |
||||
} |
||||
|
||||
for new_size < r { |
||||
new_size *= 2 |
||||
} |
||||
var new_array []byte = make([]byte, new_size) |
||||
if *c != 0 { |
||||
copy(new_array, (*a)[:*c]) |
||||
} |
||||
|
||||
*a = new_array |
||||
*c = new_size |
||||
} |
||||
} |
||||
|
||||
func brotli_ensure_capacity_uint32_t(a *[]uint32, c *uint, r uint) { |
||||
var new_array []uint32 |
||||
if *c < r { |
||||
var new_size uint = *c |
||||
if new_size == 0 { |
||||
new_size = r |
||||
} |
||||
|
||||
for new_size < r { |
||||
new_size *= 2 |
||||
} |
||||
|
||||
new_array = make([]uint32, new_size) |
||||
if *c != 0 { |
||||
copy(new_array, (*a)[:*c]) |
||||
} |
||||
|
||||
*a = new_array |
||||
*c = new_size |
||||
} |
||||
} |
||||
@ -0,0 +1,555 @@ |
||||
package brotli |
||||
|
||||
/* Copyright 2014 Google Inc. All Rights Reserved. |
||||
|
||||
Distributed under MIT license. |
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/ |
||||
|
||||
/* Algorithms for distributing the literals and commands of a metablock between |
||||
block types and contexts. */ |
||||
|
||||
type metaBlockSplit struct { |
||||
literal_split blockSplit |
||||
command_split blockSplit |
||||
distance_split blockSplit |
||||
literal_context_map []uint32 |
||||
literal_context_map_size uint |
||||
distance_context_map []uint32 |
||||
distance_context_map_size uint |
||||
literal_histograms []histogramLiteral |
||||
literal_histograms_size uint |
||||
command_histograms []histogramCommand |
||||
command_histograms_size uint |
||||
distance_histograms []histogramDistance |
||||
distance_histograms_size uint |
||||
} |
||||
|
||||
func initMetaBlockSplit(mb *metaBlockSplit) { |
||||
initBlockSplit(&mb.literal_split) |
||||
initBlockSplit(&mb.command_split) |
||||
initBlockSplit(&mb.distance_split) |
||||
mb.literal_context_map = nil |
||||
mb.literal_context_map_size = 0 |
||||
mb.distance_context_map = nil |
||||
mb.distance_context_map_size = 0 |
||||
mb.literal_histograms = nil |
||||
mb.literal_histograms_size = 0 |
||||
mb.command_histograms = nil |
||||
mb.command_histograms_size = 0 |
||||
mb.distance_histograms = nil |
||||
mb.distance_histograms_size = 0 |
||||
} |
||||
|
||||
func destroyMetaBlockSplit(mb *metaBlockSplit) { |
||||
destroyBlockSplit(&mb.literal_split) |
||||
destroyBlockSplit(&mb.command_split) |
||||
destroyBlockSplit(&mb.distance_split) |
||||
mb.literal_context_map = nil |
||||
mb.distance_context_map = nil |
||||
mb.literal_histograms = nil |
||||
mb.command_histograms = nil |
||||
mb.distance_histograms = nil |
||||
} |
||||
|
||||
func initDistanceParams(params *encoderParams, npostfix uint32, ndirect uint32) { |
||||
var dist_params *distanceParams = ¶ms.dist |
||||
var alphabet_size uint32 |
||||
var max_distance uint32 |
||||
|
||||
dist_params.distance_postfix_bits = npostfix |
||||
dist_params.num_direct_distance_codes = ndirect |
||||
|
||||
alphabet_size = uint32(distanceAlphabetSize(uint(npostfix), uint(ndirect), maxDistanceBits)) |
||||
max_distance = ndirect + (1 << (maxDistanceBits + npostfix + 2)) - (1 << (npostfix + 2)) |
||||
|
||||
if params.large_window { |
||||
var bound = [maxNpostfix + 1]uint32{0, 4, 12, 28} |
||||
var postfix uint32 = 1 << npostfix |
||||
alphabet_size = uint32(distanceAlphabetSize(uint(npostfix), uint(ndirect), largeMaxDistanceBits)) |
||||
|
||||
/* The maximum distance is set so that no distance symbol used can encode |
||||
a distance larger than BROTLI_MAX_ALLOWED_DISTANCE with all |
||||
its extra bits set. */ |
||||
if ndirect < bound[npostfix] { |
||||
max_distance = maxAllowedDistance - (bound[npostfix] - ndirect) |
||||
} else if ndirect >= bound[npostfix]+postfix { |
||||
max_distance = (3 << 29) - 4 + (ndirect - bound[npostfix]) |
||||
} else { |
||||
max_distance = maxAllowedDistance |
||||
} |
||||
} |
||||
|
||||
dist_params.alphabet_size = alphabet_size |
||||
dist_params.max_distance = uint(max_distance) |
||||
} |
||||
|
||||
func recomputeDistancePrefixes(cmds []command, num_commands uint, orig_params *distanceParams, new_params *distanceParams) { |
||||
var i uint |
||||
|
||||
if orig_params.distance_postfix_bits == new_params.distance_postfix_bits && orig_params.num_direct_distance_codes == new_params.num_direct_distance_codes { |
||||
return |
||||
} |
||||
|
||||
for i = 0; i < num_commands; i++ { |
||||
var cmd *command = &cmds[i] |
||||
if commandCopyLen(cmd) != 0 && cmd.cmd_prefix_ >= 128 { |
||||
prefixEncodeCopyDistance(uint(commandRestoreDistanceCode(cmd, orig_params)), uint(new_params.num_direct_distance_codes), uint(new_params.distance_postfix_bits), &cmd.dist_prefix_, &cmd.dist_extra_) |
||||
} |
||||
} |
||||
} |
||||
|
||||
func computeDistanceCost(cmds []command, num_commands uint, orig_params *distanceParams, new_params *distanceParams, cost *float64) bool { |
||||
var i uint |
||||
var equal_params bool = false |
||||
var dist_prefix uint16 |
||||
var dist_extra uint32 |
||||
var extra_bits float64 = 0.0 |
||||
var histo histogramDistance |
||||
histogramClearDistance(&histo) |
||||
|
||||
if orig_params.distance_postfix_bits == new_params.distance_postfix_bits && orig_params.num_direct_distance_codes == new_params.num_direct_distance_codes { |
||||
equal_params = true |
||||
} |
||||
|
||||
for i = 0; i < num_commands; i++ { |
||||
var cmd *command = &cmds[i] |
||||
if commandCopyLen(cmd) != 0 && cmd.cmd_prefix_ >= 128 { |
||||
if equal_params { |
||||
dist_prefix = cmd.dist_prefix_ |
||||
} else { |
||||
var distance uint32 = commandRestoreDistanceCode(cmd, orig_params) |
||||
if distance > uint32(new_params.max_distance) { |
||||
return false |
||||
} |
||||
|
||||
prefixEncodeCopyDistance(uint(distance), uint(new_params.num_direct_distance_codes), uint(new_params.distance_postfix_bits), &dist_prefix, &dist_extra) |
||||
} |
||||
|
||||
histogramAddDistance(&histo, uint(dist_prefix)&0x3FF) |
||||
extra_bits += float64(dist_prefix >> 10) |
||||
} |
||||
} |
||||
|
||||
*cost = populationCostDistance(&histo) + extra_bits |
||||
return true |
||||
} |
||||
|
||||
var buildMetaBlock_kMaxNumberOfHistograms uint = 256 |
||||
|
||||
func buildMetaBlock(ringbuffer []byte, pos uint, mask uint, params *encoderParams, prev_byte byte, prev_byte2 byte, cmds []command, num_commands uint, literal_context_mode int, mb *metaBlockSplit) { |
||||
var distance_histograms []histogramDistance |
||||
var literal_histograms []histogramLiteral |
||||
var literal_context_modes []int = nil |
||||
var literal_histograms_size uint |
||||
var distance_histograms_size uint |
||||
var i uint |
||||
var literal_context_multiplier uint = 1 |
||||
var npostfix uint32 |
||||
var ndirect_msb uint32 = 0 |
||||
var check_orig bool = true |
||||
var best_dist_cost float64 = 1e99 |
||||
var orig_params encoderParams = *params |
||||
/* Histogram ids need to fit in one byte. */ |
||||
|
||||
var new_params encoderParams = *params |
||||
|
||||
for npostfix = 0; npostfix <= maxNpostfix; npostfix++ { |
||||
for ; ndirect_msb < 16; ndirect_msb++ { |
||||
var ndirect uint32 = ndirect_msb << npostfix |
||||
var skip bool |
||||
var dist_cost float64 |
||||
initDistanceParams(&new_params, npostfix, ndirect) |
||||
if npostfix == orig_params.dist.distance_postfix_bits && ndirect == orig_params.dist.num_direct_distance_codes { |
||||
check_orig = false |
||||
} |
||||
|
||||
skip = !computeDistanceCost(cmds, num_commands, &orig_params.dist, &new_params.dist, &dist_cost) |
||||
if skip || (dist_cost > best_dist_cost) { |
||||
break |
||||
} |
||||
|
||||
best_dist_cost = dist_cost |
||||
params.dist = new_params.dist |
||||
} |
||||
|
||||
if ndirect_msb > 0 { |
||||
ndirect_msb-- |
||||
} |
||||
ndirect_msb /= 2 |
||||
} |
||||
|
||||
if check_orig { |
||||
var dist_cost float64 |
||||
computeDistanceCost(cmds, num_commands, &orig_params.dist, &orig_params.dist, &dist_cost) |
||||
if dist_cost < best_dist_cost { |
||||
/* NB: currently unused; uncomment when more param tuning is added. */ |
||||
/* best_dist_cost = dist_cost; */ |
||||
params.dist = orig_params.dist |
||||
} |
||||
} |
||||
|
||||
recomputeDistancePrefixes(cmds, num_commands, &orig_params.dist, ¶ms.dist) |
||||
|
||||
splitBlock(cmds, num_commands, ringbuffer, pos, mask, params, &mb.literal_split, &mb.command_split, &mb.distance_split) |
||||
|
||||
if !params.disable_literal_context_modeling { |
||||
literal_context_multiplier = 1 << literalContextBits |
||||
literal_context_modes = make([]int, (mb.literal_split.num_types)) |
||||
for i = 0; i < mb.literal_split.num_types; i++ { |
||||
literal_context_modes[i] = literal_context_mode |
||||
} |
||||
} |
||||
|
||||
literal_histograms_size = mb.literal_split.num_types * literal_context_multiplier |
||||
literal_histograms = make([]histogramLiteral, literal_histograms_size) |
||||
clearHistogramsLiteral(literal_histograms, literal_histograms_size) |
||||
|
||||
distance_histograms_size = mb.distance_split.num_types << distanceContextBits |
||||
distance_histograms = make([]histogramDistance, distance_histograms_size) |
||||
clearHistogramsDistance(distance_histograms, distance_histograms_size) |
||||
|
||||
assert(mb.command_histograms == nil) |
||||
mb.command_histograms_size = mb.command_split.num_types |
||||
mb.command_histograms = make([]histogramCommand, (mb.command_histograms_size)) |
||||
clearHistogramsCommand(mb.command_histograms, mb.command_histograms_size) |
||||
|
||||
buildHistogramsWithContext(cmds, num_commands, &mb.literal_split, &mb.command_split, &mb.distance_split, ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_modes, literal_histograms, mb.command_histograms, distance_histograms) |
||||
literal_context_modes = nil |
||||
|
||||
assert(mb.literal_context_map == nil) |
||||
mb.literal_context_map_size = mb.literal_split.num_types << literalContextBits |
||||
mb.literal_context_map = make([]uint32, (mb.literal_context_map_size)) |
||||
|
||||
assert(mb.literal_histograms == nil) |
||||
mb.literal_histograms_size = mb.literal_context_map_size |
||||
mb.literal_histograms = make([]histogramLiteral, (mb.literal_histograms_size)) |
||||
|
||||
clusterHistogramsLiteral(literal_histograms, literal_histograms_size, buildMetaBlock_kMaxNumberOfHistograms, mb.literal_histograms, &mb.literal_histograms_size, mb.literal_context_map) |
||||
literal_histograms = nil |
||||
|
||||
if params.disable_literal_context_modeling { |
||||
/* Distribute assignment to all contexts. */ |
||||
for i = mb.literal_split.num_types; i != 0; { |
||||
var j uint = 0 |
||||
i-- |
||||
for ; j < 1<<literalContextBits; j++ { |
||||
mb.literal_context_map[(i<<literalContextBits)+j] = mb.literal_context_map[i] |
||||
} |
||||
} |
||||
} |
||||
|
||||
assert(mb.distance_context_map == nil) |
||||
mb.distance_context_map_size = mb.distance_split.num_types << distanceContextBits |
||||
mb.distance_context_map = make([]uint32, (mb.distance_context_map_size)) |
||||
|
||||
assert(mb.distance_histograms == nil) |
||||
mb.distance_histograms_size = mb.distance_context_map_size |
||||
mb.distance_histograms = make([]histogramDistance, (mb.distance_histograms_size)) |
||||
|
||||
clusterHistogramsDistance(distance_histograms, mb.distance_context_map_size, buildMetaBlock_kMaxNumberOfHistograms, mb.distance_histograms, &mb.distance_histograms_size, mb.distance_context_map) |
||||
distance_histograms = nil |
||||
} |
||||
|
||||
const maxStaticContexts = 13 |
||||
|
||||
/* Greedy block splitter for one block category (literal, command or distance). |
||||
Gathers histograms for all context buckets. */ |
||||
type contextBlockSplitter struct { |
||||
alphabet_size_ uint |
||||
num_contexts_ uint |
||||
max_block_types_ uint |
||||
min_block_size_ uint |
||||
split_threshold_ float64 |
||||
num_blocks_ uint |
||||
split_ *blockSplit |
||||
histograms_ []histogramLiteral |
||||
histograms_size_ *uint |
||||
target_block_size_ uint |
||||
block_size_ uint |
||||
curr_histogram_ix_ uint |
||||
last_histogram_ix_ [2]uint |
||||
last_entropy_ [2 * maxStaticContexts]float64 |
||||
merge_last_count_ uint |
||||
} |
||||
|
||||
func initContextBlockSplitter(self *contextBlockSplitter, alphabet_size uint, num_contexts uint, min_block_size uint, split_threshold float64, num_symbols uint, split *blockSplit, histograms *[]histogramLiteral, histograms_size *uint) { |
||||
var max_num_blocks uint = num_symbols/min_block_size + 1 |
||||
var max_num_types uint |
||||
assert(num_contexts <= maxStaticContexts) |
||||
|
||||
self.alphabet_size_ = alphabet_size |
||||
self.num_contexts_ = num_contexts |
||||
self.max_block_types_ = maxNumberOfBlockTypes / num_contexts |
||||
self.min_block_size_ = min_block_size |
||||
self.split_threshold_ = split_threshold |
||||
self.num_blocks_ = 0 |
||||
self.split_ = split |
||||
self.histograms_size_ = histograms_size |
||||
self.target_block_size_ = min_block_size |
||||
self.block_size_ = 0 |
||||
self.curr_histogram_ix_ = 0 |
||||
self.merge_last_count_ = 0 |
||||
|
||||
/* We have to allocate one more histogram than the maximum number of block |
||||
types for the current histogram when the meta-block is too big. */ |
||||
max_num_types = brotli_min_size_t(max_num_blocks, self.max_block_types_+1) |
||||
|
||||
brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, max_num_blocks) |
||||
brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, max_num_blocks) |
||||
split.num_blocks = max_num_blocks |
||||
assert(*histograms == nil) |
||||
*histograms_size = max_num_types * num_contexts |
||||
*histograms = make([]histogramLiteral, (*histograms_size)) |
||||
self.histograms_ = *histograms |
||||
|
||||
/* Clear only current histogram. */ |
||||
clearHistogramsLiteral(self.histograms_[0:], num_contexts) |
||||
|
||||
self.last_histogram_ix_[1] = 0 |
||||
self.last_histogram_ix_[0] = self.last_histogram_ix_[1] |
||||
} |
||||
|
||||
/* Does either of three things: |
||||
(1) emits the current block with a new block type; |
||||
(2) emits the current block with the type of the second last block; |
||||
(3) merges the current block with the last block. */ |
||||
func contextBlockSplitterFinishBlock(self *contextBlockSplitter, is_final bool) { |
||||
var split *blockSplit = self.split_ |
||||
var num_contexts uint = self.num_contexts_ |
||||
var last_entropy []float64 = self.last_entropy_[:] |
||||
var histograms []histogramLiteral = self.histograms_ |
||||
|
||||
if self.block_size_ < self.min_block_size_ { |
||||
self.block_size_ = self.min_block_size_ |
||||
} |
||||
|
||||
if self.num_blocks_ == 0 { |
||||
var i uint |
||||
|
||||
/* Create first block. */ |
||||
split.lengths[0] = uint32(self.block_size_) |
||||
|
||||
split.types[0] = 0 |
||||
|
||||
for i = 0; i < num_contexts; i++ { |
||||
last_entropy[i] = bitsEntropy(histograms[i].data_[:], self.alphabet_size_) |
||||
last_entropy[num_contexts+i] = last_entropy[i] |
||||
} |
||||
|
||||
self.num_blocks_++ |
||||
split.num_types++ |
||||
self.curr_histogram_ix_ += num_contexts |
||||
if self.curr_histogram_ix_ < *self.histograms_size_ { |
||||
clearHistogramsLiteral(self.histograms_[self.curr_histogram_ix_:], self.num_contexts_) |
||||
} |
||||
|
||||
self.block_size_ = 0 |
||||
} else if self.block_size_ > 0 { |
||||
var entropy [maxStaticContexts]float64 |
||||
var combined_histo []histogramLiteral = make([]histogramLiteral, (2 * num_contexts)) |
||||
var combined_entropy [2 * maxStaticContexts]float64 |
||||
var diff = [2]float64{0.0} |
||||
/* Try merging the set of histograms for the current block type with the |
||||
respective set of histograms for the last and second last block types. |
||||
Decide over the split based on the total reduction of entropy across |
||||
all contexts. */ |
||||
|
||||
var i uint |
||||
for i = 0; i < num_contexts; i++ { |
||||
var curr_histo_ix uint = self.curr_histogram_ix_ + i |
||||
var j uint |
||||
entropy[i] = bitsEntropy(histograms[curr_histo_ix].data_[:], self.alphabet_size_) |
||||
for j = 0; j < 2; j++ { |
||||
var jx uint = j*num_contexts + i |
||||
var last_histogram_ix uint = self.last_histogram_ix_[j] + i |
||||
combined_histo[jx] = histograms[curr_histo_ix] |
||||
histogramAddHistogramLiteral(&combined_histo[jx], &histograms[last_histogram_ix]) |
||||
combined_entropy[jx] = bitsEntropy(combined_histo[jx].data_[0:], self.alphabet_size_) |
||||
diff[j] += combined_entropy[jx] - entropy[i] - last_entropy[jx] |
||||
} |
||||
} |
||||
|
||||
if split.num_types < self.max_block_types_ && diff[0] > self.split_threshold_ && diff[1] > self.split_threshold_ { |
||||
/* Create new block. */ |
||||
split.lengths[self.num_blocks_] = uint32(self.block_size_) |
||||
|
||||
split.types[self.num_blocks_] = byte(split.num_types) |
||||
self.last_histogram_ix_[1] = self.last_histogram_ix_[0] |
||||
self.last_histogram_ix_[0] = split.num_types * num_contexts |
||||
for i = 0; i < num_contexts; i++ { |
||||
last_entropy[num_contexts+i] = last_entropy[i] |
||||
last_entropy[i] = entropy[i] |
||||
} |
||||
|
||||
self.num_blocks_++ |
||||
split.num_types++ |
||||
self.curr_histogram_ix_ += num_contexts |
||||
if self.curr_histogram_ix_ < *self.histograms_size_ { |
||||
clearHistogramsLiteral(self.histograms_[self.curr_histogram_ix_:], self.num_contexts_) |
||||
} |
||||
|
||||
self.block_size_ = 0 |
||||
self.merge_last_count_ = 0 |
||||
self.target_block_size_ = self.min_block_size_ |
||||
} else if diff[1] < diff[0]-20.0 { |
||||
split.lengths[self.num_blocks_] = uint32(self.block_size_) |
||||
split.types[self.num_blocks_] = split.types[self.num_blocks_-2] |
||||
/* Combine this block with second last block. */ |
||||
|
||||
var tmp uint = self.last_histogram_ix_[0] |
||||
self.last_histogram_ix_[0] = self.last_histogram_ix_[1] |
||||
self.last_histogram_ix_[1] = tmp |
||||
for i = 0; i < num_contexts; i++ { |
||||
histograms[self.last_histogram_ix_[0]+i] = combined_histo[num_contexts+i] |
||||
last_entropy[num_contexts+i] = last_entropy[i] |
||||
last_entropy[i] = combined_entropy[num_contexts+i] |
||||
histogramClearLiteral(&histograms[self.curr_histogram_ix_+i]) |
||||
} |
||||
|
||||
self.num_blocks_++ |
||||
self.block_size_ = 0 |
||||
self.merge_last_count_ = 0 |
||||
self.target_block_size_ = self.min_block_size_ |
||||
} else { |
||||
/* Combine this block with last block. */ |
||||
split.lengths[self.num_blocks_-1] += uint32(self.block_size_) |
||||
|
||||
for i = 0; i < num_contexts; i++ { |
||||
histograms[self.last_histogram_ix_[0]+i] = combined_histo[i] |
||||
last_entropy[i] = combined_entropy[i] |
||||
if split.num_types == 1 { |
||||
last_entropy[num_contexts+i] = last_entropy[i] |
||||
} |
||||
|
||||
histogramClearLiteral(&histograms[self.curr_histogram_ix_+i]) |
||||
} |
||||
|
||||
self.block_size_ = 0 |
||||
self.merge_last_count_++ |
||||
if self.merge_last_count_ > 1 { |
||||
self.target_block_size_ += self.min_block_size_ |
||||
} |
||||
} |
||||
|
||||
combined_histo = nil |
||||
} |
||||
|
||||
if is_final { |
||||
*self.histograms_size_ = split.num_types * num_contexts |
||||
split.num_blocks = self.num_blocks_ |
||||
} |
||||
} |
||||
|
||||
/* Adds the next symbol to the current block type and context. When the |
||||
current block reaches the target size, decides on merging the block. */ |
||||
func contextBlockSplitterAddSymbol(self *contextBlockSplitter, symbol uint, context uint) { |
||||
histogramAddLiteral(&self.histograms_[self.curr_histogram_ix_+context], symbol) |
||||
self.block_size_++ |
||||
if self.block_size_ == self.target_block_size_ { |
||||
contextBlockSplitterFinishBlock(self, false) /* is_final = */ |
||||
} |
||||
} |
||||
|
||||
func mapStaticContexts(num_contexts uint, static_context_map []uint32, mb *metaBlockSplit) { |
||||
var i uint |
||||
assert(mb.literal_context_map == nil) |
||||
mb.literal_context_map_size = mb.literal_split.num_types << literalContextBits |
||||
mb.literal_context_map = make([]uint32, (mb.literal_context_map_size)) |
||||
|
||||
for i = 0; i < mb.literal_split.num_types; i++ { |
||||
var offset uint32 = uint32(i * num_contexts) |
||||
var j uint |
||||
for j = 0; j < 1<<literalContextBits; j++ { |
||||
mb.literal_context_map[(i<<literalContextBits)+j] = offset + static_context_map[j] |
||||
} |
||||
} |
||||
} |
||||
|
||||
func buildMetaBlockGreedyInternal(ringbuffer []byte, pos uint, mask uint, prev_byte byte, prev_byte2 byte, literal_context_lut contextLUT, num_contexts uint, static_context_map []uint32, commands []command, n_commands uint, mb *metaBlockSplit) { |
||||
var lit_blocks struct { |
||||
plain blockSplitterLiteral |
||||
ctx contextBlockSplitter |
||||
} |
||||
var cmd_blocks blockSplitterCommand |
||||
var dist_blocks blockSplitterDistance |
||||
var num_literals uint = 0 |
||||
var i uint |
||||
for i = 0; i < n_commands; i++ { |
||||
num_literals += uint(commands[i].insert_len_) |
||||
} |
||||
|
||||
if num_contexts == 1 { |
||||
initBlockSplitterLiteral(&lit_blocks.plain, 256, 512, 400.0, num_literals, &mb.literal_split, &mb.literal_histograms, &mb.literal_histograms_size) |
||||
} else { |
||||
initContextBlockSplitter(&lit_blocks.ctx, 256, num_contexts, 512, 400.0, num_literals, &mb.literal_split, &mb.literal_histograms, &mb.literal_histograms_size) |
||||
} |
||||
|
||||
initBlockSplitterCommand(&cmd_blocks, numCommandSymbols, 1024, 500.0, n_commands, &mb.command_split, &mb.command_histograms, &mb.command_histograms_size) |
||||
initBlockSplitterDistance(&dist_blocks, 64, 512, 100.0, n_commands, &mb.distance_split, &mb.distance_histograms, &mb.distance_histograms_size) |
||||
|
||||
for i = 0; i < n_commands; i++ { |
||||
var cmd command = commands[i] |
||||
var j uint |
||||
blockSplitterAddSymbolCommand(&cmd_blocks, uint(cmd.cmd_prefix_)) |
||||
for j = uint(cmd.insert_len_); j != 0; j-- { |
||||
var literal byte = ringbuffer[pos&mask] |
||||
if num_contexts == 1 { |
||||
blockSplitterAddSymbolLiteral(&lit_blocks.plain, uint(literal)) |
||||
} else { |
||||
var context uint = uint(getContext(prev_byte, prev_byte2, literal_context_lut)) |
||||
contextBlockSplitterAddSymbol(&lit_blocks.ctx, uint(literal), uint(static_context_map[context])) |
||||
} |
||||
|
||||
prev_byte2 = prev_byte |
||||
prev_byte = literal |
||||
pos++ |
||||
} |
||||
|
||||
pos += uint(commandCopyLen(&cmd)) |
||||
if commandCopyLen(&cmd) != 0 { |
||||
prev_byte2 = ringbuffer[(pos-2)&mask] |
||||
prev_byte = ringbuffer[(pos-1)&mask] |
||||
if cmd.cmd_prefix_ >= 128 { |
||||
blockSplitterAddSymbolDistance(&dist_blocks, uint(cmd.dist_prefix_)&0x3FF) |
||||
} |
||||
} |
||||
} |
||||
|
||||
if num_contexts == 1 { |
||||
blockSplitterFinishBlockLiteral(&lit_blocks.plain, true) /* is_final = */ |
||||
} else { |
||||
contextBlockSplitterFinishBlock(&lit_blocks.ctx, true) /* is_final = */ |
||||
} |
||||
|
||||
blockSplitterFinishBlockCommand(&cmd_blocks, true) /* is_final = */ |
||||
blockSplitterFinishBlockDistance(&dist_blocks, true) /* is_final = */ |
||||
|
||||
if num_contexts > 1 { |
||||
mapStaticContexts(num_contexts, static_context_map, mb) |
||||
} |
||||
} |
||||
|
||||
func buildMetaBlockGreedy(ringbuffer []byte, pos uint, mask uint, prev_byte byte, prev_byte2 byte, literal_context_lut contextLUT, num_contexts uint, static_context_map []uint32, commands []command, n_commands uint, mb *metaBlockSplit) { |
||||
if num_contexts == 1 { |
||||
buildMetaBlockGreedyInternal(ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_lut, 1, nil, commands, n_commands, mb) |
||||
} else { |
||||
buildMetaBlockGreedyInternal(ringbuffer, pos, mask, prev_byte, prev_byte2, literal_context_lut, num_contexts, static_context_map, commands, n_commands, mb) |
||||
} |
||||
} |
||||
|
||||
func optimizeHistograms(num_distance_codes uint32, mb *metaBlockSplit) { |
||||
var good_for_rle [numCommandSymbols]byte |
||||
var i uint |
||||
for i = 0; i < mb.literal_histograms_size; i++ { |
||||
optimizeHuffmanCountsForRLE(256, mb.literal_histograms[i].data_[:], good_for_rle[:]) |
||||
} |
||||
|
||||
for i = 0; i < mb.command_histograms_size; i++ { |
||||
optimizeHuffmanCountsForRLE(numCommandSymbols, mb.command_histograms[i].data_[:], good_for_rle[:]) |
||||
} |
||||
|
||||
for i = 0; i < mb.distance_histograms_size; i++ { |
||||
optimizeHuffmanCountsForRLE(uint(num_distance_codes), mb.distance_histograms[i].data_[:], good_for_rle[:]) |
||||
} |
||||
} |
||||
@ -0,0 +1,162 @@ |
||||
package brotli |
||||
|
||||
/* Copyright 2015 Google Inc. All Rights Reserved. |
||||
|
||||
Distributed under MIT license. |
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/ |
||||
|
||||
/* Greedy block splitter for one block category (literal, command or distance). |
||||
*/ |
||||
type blockSplitterCommand struct { |
||||
alphabet_size_ uint |
||||
min_block_size_ uint |
||||
split_threshold_ float64 |
||||
num_blocks_ uint |
||||
split_ *blockSplit |
||||
histograms_ []histogramCommand |
||||
histograms_size_ *uint |
||||
target_block_size_ uint |
||||
block_size_ uint |
||||
curr_histogram_ix_ uint |
||||
last_histogram_ix_ [2]uint |
||||
last_entropy_ [2]float64 |
||||
merge_last_count_ uint |
||||
} |
||||
|
||||
func initBlockSplitterCommand(self *blockSplitterCommand, alphabet_size uint, min_block_size uint, split_threshold float64, num_symbols uint, split *blockSplit, histograms *[]histogramCommand, histograms_size *uint) { |
||||
var max_num_blocks uint = num_symbols/min_block_size + 1 |
||||
var max_num_types uint = brotli_min_size_t(max_num_blocks, maxNumberOfBlockTypes+1) |
||||
/* We have to allocate one more histogram than the maximum number of block |
||||
types for the current histogram when the meta-block is too big. */ |
||||
self.alphabet_size_ = alphabet_size |
||||
|
||||
self.min_block_size_ = min_block_size |
||||
self.split_threshold_ = split_threshold |
||||
self.num_blocks_ = 0 |
||||
self.split_ = split |
||||
self.histograms_size_ = histograms_size |
||||
self.target_block_size_ = min_block_size |
||||
self.block_size_ = 0 |
||||
self.curr_histogram_ix_ = 0 |
||||
self.merge_last_count_ = 0 |
||||
brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, max_num_blocks) |
||||
brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, max_num_blocks) |
||||
self.split_.num_blocks = max_num_blocks |
||||
assert(*histograms == nil) |
||||
*histograms_size = max_num_types |
||||
*histograms = make([]histogramCommand, (*histograms_size)) |
||||
self.histograms_ = *histograms |
||||
|
||||
/* Clear only current histogram. */ |
||||
histogramClearCommand(&self.histograms_[0]) |
||||
|
||||
self.last_histogram_ix_[1] = 0 |
||||
self.last_histogram_ix_[0] = self.last_histogram_ix_[1] |
||||
} |
||||
|
||||
/* Does either of three things: |
||||
(1) emits the current block with a new block type; |
||||
(2) emits the current block with the type of the second last block; |
||||
(3) merges the current block with the last block. */ |
||||
func blockSplitterFinishBlockCommand(self *blockSplitterCommand, is_final bool) { |
||||
var split *blockSplit = self.split_ |
||||
var last_entropy []float64 = self.last_entropy_[:] |
||||
var histograms []histogramCommand = self.histograms_ |
||||
self.block_size_ = brotli_max_size_t(self.block_size_, self.min_block_size_) |
||||
if self.num_blocks_ == 0 { |
||||
/* Create first block. */ |
||||
split.lengths[0] = uint32(self.block_size_) |
||||
|
||||
split.types[0] = 0 |
||||
last_entropy[0] = bitsEntropy(histograms[0].data_[:], self.alphabet_size_) |
||||
last_entropy[1] = last_entropy[0] |
||||
self.num_blocks_++ |
||||
split.num_types++ |
||||
self.curr_histogram_ix_++ |
||||
if self.curr_histogram_ix_ < *self.histograms_size_ { |
||||
histogramClearCommand(&histograms[self.curr_histogram_ix_]) |
||||
} |
||||
self.block_size_ = 0 |
||||
} else if self.block_size_ > 0 { |
||||
var entropy float64 = bitsEntropy(histograms[self.curr_histogram_ix_].data_[:], self.alphabet_size_) |
||||
var combined_histo [2]histogramCommand |
||||
var combined_entropy [2]float64 |
||||
var diff [2]float64 |
||||
var j uint |
||||
for j = 0; j < 2; j++ { |
||||
var last_histogram_ix uint = self.last_histogram_ix_[j] |
||||
combined_histo[j] = histograms[self.curr_histogram_ix_] |
||||
histogramAddHistogramCommand(&combined_histo[j], &histograms[last_histogram_ix]) |
||||
combined_entropy[j] = bitsEntropy(combined_histo[j].data_[0:], self.alphabet_size_) |
||||
diff[j] = combined_entropy[j] - entropy - last_entropy[j] |
||||
} |
||||
|
||||
if split.num_types < maxNumberOfBlockTypes && diff[0] > self.split_threshold_ && diff[1] > self.split_threshold_ { |
||||
/* Create new block. */ |
||||
split.lengths[self.num_blocks_] = uint32(self.block_size_) |
||||
|
||||
split.types[self.num_blocks_] = byte(split.num_types) |
||||
self.last_histogram_ix_[1] = self.last_histogram_ix_[0] |
||||
self.last_histogram_ix_[0] = uint(byte(split.num_types)) |
||||
last_entropy[1] = last_entropy[0] |
||||
last_entropy[0] = entropy |
||||
self.num_blocks_++ |
||||
split.num_types++ |
||||
self.curr_histogram_ix_++ |
||||
if self.curr_histogram_ix_ < *self.histograms_size_ { |
||||
histogramClearCommand(&histograms[self.curr_histogram_ix_]) |
||||
} |
||||
self.block_size_ = 0 |
||||
self.merge_last_count_ = 0 |
||||
self.target_block_size_ = self.min_block_size_ |
||||
} else if diff[1] < diff[0]-20.0 { |
||||
split.lengths[self.num_blocks_] = uint32(self.block_size_) |
||||
split.types[self.num_blocks_] = split.types[self.num_blocks_-2] |
||||
/* Combine this block with second last block. */ |
||||
|
||||
var tmp uint = self.last_histogram_ix_[0] |
||||
self.last_histogram_ix_[0] = self.last_histogram_ix_[1] |
||||
self.last_histogram_ix_[1] = tmp |
||||
histograms[self.last_histogram_ix_[0]] = combined_histo[1] |
||||
last_entropy[1] = last_entropy[0] |
||||
last_entropy[0] = combined_entropy[1] |
||||
self.num_blocks_++ |
||||
self.block_size_ = 0 |
||||
histogramClearCommand(&histograms[self.curr_histogram_ix_]) |
||||
self.merge_last_count_ = 0 |
||||
self.target_block_size_ = self.min_block_size_ |
||||
} else { |
||||
/* Combine this block with last block. */ |
||||
split.lengths[self.num_blocks_-1] += uint32(self.block_size_) |
||||
|
||||
histograms[self.last_histogram_ix_[0]] = combined_histo[0] |
||||
last_entropy[0] = combined_entropy[0] |
||||
if split.num_types == 1 { |
||||
last_entropy[1] = last_entropy[0] |
||||
} |
||||
|
||||
self.block_size_ = 0 |
||||
histogramClearCommand(&histograms[self.curr_histogram_ix_]) |
||||
self.merge_last_count_++ |
||||
if self.merge_last_count_ > 1 { |
||||
self.target_block_size_ += self.min_block_size_ |
||||
} |
||||
} |
||||
} |
||||
|
||||
if is_final { |
||||
*self.histograms_size_ = split.num_types |
||||
split.num_blocks = self.num_blocks_ |
||||
} |
||||
} |
||||
|
||||
/* Adds the next symbol to the current histogram. When the current histogram |
||||
reaches the target size, decides on merging the block. */ |
||||
func blockSplitterAddSymbolCommand(self *blockSplitterCommand, symbol uint) { |
||||
histogramAddCommand(&self.histograms_[self.curr_histogram_ix_], symbol) |
||||
self.block_size_++ |
||||
if self.block_size_ == self.target_block_size_ { |
||||
blockSplitterFinishBlockCommand(self, false) /* is_final = */ |
||||
} |
||||
} |
||||
@ -0,0 +1,162 @@ |
||||
package brotli |
||||
|
||||
/* Copyright 2015 Google Inc. All Rights Reserved. |
||||
|
||||
Distributed under MIT license. |
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/ |
||||
|
||||
/* Greedy block splitter for one block category (literal, command or distance). |
||||
*/ |
||||
type blockSplitterDistance struct { |
||||
alphabet_size_ uint |
||||
min_block_size_ uint |
||||
split_threshold_ float64 |
||||
num_blocks_ uint |
||||
split_ *blockSplit |
||||
histograms_ []histogramDistance |
||||
histograms_size_ *uint |
||||
target_block_size_ uint |
||||
block_size_ uint |
||||
curr_histogram_ix_ uint |
||||
last_histogram_ix_ [2]uint |
||||
last_entropy_ [2]float64 |
||||
merge_last_count_ uint |
||||
} |
||||
|
||||
func initBlockSplitterDistance(self *blockSplitterDistance, alphabet_size uint, min_block_size uint, split_threshold float64, num_symbols uint, split *blockSplit, histograms *[]histogramDistance, histograms_size *uint) { |
||||
var max_num_blocks uint = num_symbols/min_block_size + 1 |
||||
var max_num_types uint = brotli_min_size_t(max_num_blocks, maxNumberOfBlockTypes+1) |
||||
/* We have to allocate one more histogram than the maximum number of block |
||||
types for the current histogram when the meta-block is too big. */ |
||||
self.alphabet_size_ = alphabet_size |
||||
|
||||
self.min_block_size_ = min_block_size |
||||
self.split_threshold_ = split_threshold |
||||
self.num_blocks_ = 0 |
||||
self.split_ = split |
||||
self.histograms_size_ = histograms_size |
||||
self.target_block_size_ = min_block_size |
||||
self.block_size_ = 0 |
||||
self.curr_histogram_ix_ = 0 |
||||
self.merge_last_count_ = 0 |
||||
brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, max_num_blocks) |
||||
brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, max_num_blocks) |
||||
self.split_.num_blocks = max_num_blocks |
||||
assert(*histograms == nil) |
||||
*histograms_size = max_num_types |
||||
*histograms = make([]histogramDistance, (*histograms_size)) |
||||
self.histograms_ = *histograms |
||||
|
||||
/* Clear only current histogram. */ |
||||
histogramClearDistance(&self.histograms_[0]) |
||||
|
||||
self.last_histogram_ix_[1] = 0 |
||||
self.last_histogram_ix_[0] = self.last_histogram_ix_[1] |
||||
} |
||||
|
||||
/* Does either of three things: |
||||
(1) emits the current block with a new block type; |
||||
(2) emits the current block with the type of the second last block; |
||||
(3) merges the current block with the last block. */ |
||||
func blockSplitterFinishBlockDistance(self *blockSplitterDistance, is_final bool) { |
||||
var split *blockSplit = self.split_ |
||||
var last_entropy []float64 = self.last_entropy_[:] |
||||
var histograms []histogramDistance = self.histograms_ |
||||
self.block_size_ = brotli_max_size_t(self.block_size_, self.min_block_size_) |
||||
if self.num_blocks_ == 0 { |
||||
/* Create first block. */ |
||||
split.lengths[0] = uint32(self.block_size_) |
||||
|
||||
split.types[0] = 0 |
||||
last_entropy[0] = bitsEntropy(histograms[0].data_[:], self.alphabet_size_) |
||||
last_entropy[1] = last_entropy[0] |
||||
self.num_blocks_++ |
||||
split.num_types++ |
||||
self.curr_histogram_ix_++ |
||||
if self.curr_histogram_ix_ < *self.histograms_size_ { |
||||
histogramClearDistance(&histograms[self.curr_histogram_ix_]) |
||||
} |
||||
self.block_size_ = 0 |
||||
} else if self.block_size_ > 0 { |
||||
var entropy float64 = bitsEntropy(histograms[self.curr_histogram_ix_].data_[:], self.alphabet_size_) |
||||
var combined_histo [2]histogramDistance |
||||
var combined_entropy [2]float64 |
||||
var diff [2]float64 |
||||
var j uint |
||||
for j = 0; j < 2; j++ { |
||||
var last_histogram_ix uint = self.last_histogram_ix_[j] |
||||
combined_histo[j] = histograms[self.curr_histogram_ix_] |
||||
histogramAddHistogramDistance(&combined_histo[j], &histograms[last_histogram_ix]) |
||||
combined_entropy[j] = bitsEntropy(combined_histo[j].data_[0:], self.alphabet_size_) |
||||
diff[j] = combined_entropy[j] - entropy - last_entropy[j] |
||||
} |
||||
|
||||
if split.num_types < maxNumberOfBlockTypes && diff[0] > self.split_threshold_ && diff[1] > self.split_threshold_ { |
||||
/* Create new block. */ |
||||
split.lengths[self.num_blocks_] = uint32(self.block_size_) |
||||
|
||||
split.types[self.num_blocks_] = byte(split.num_types) |
||||
self.last_histogram_ix_[1] = self.last_histogram_ix_[0] |
||||
self.last_histogram_ix_[0] = uint(byte(split.num_types)) |
||||
last_entropy[1] = last_entropy[0] |
||||
last_entropy[0] = entropy |
||||
self.num_blocks_++ |
||||
split.num_types++ |
||||
self.curr_histogram_ix_++ |
||||
if self.curr_histogram_ix_ < *self.histograms_size_ { |
||||
histogramClearDistance(&histograms[self.curr_histogram_ix_]) |
||||
} |
||||
self.block_size_ = 0 |
||||
self.merge_last_count_ = 0 |
||||
self.target_block_size_ = self.min_block_size_ |
||||
} else if diff[1] < diff[0]-20.0 { |
||||
split.lengths[self.num_blocks_] = uint32(self.block_size_) |
||||
split.types[self.num_blocks_] = split.types[self.num_blocks_-2] |
||||
/* Combine this block with second last block. */ |
||||
|
||||
var tmp uint = self.last_histogram_ix_[0] |
||||
self.last_histogram_ix_[0] = self.last_histogram_ix_[1] |
||||
self.last_histogram_ix_[1] = tmp |
||||
histograms[self.last_histogram_ix_[0]] = combined_histo[1] |
||||
last_entropy[1] = last_entropy[0] |
||||
last_entropy[0] = combined_entropy[1] |
||||
self.num_blocks_++ |
||||
self.block_size_ = 0 |
||||
histogramClearDistance(&histograms[self.curr_histogram_ix_]) |
||||
self.merge_last_count_ = 0 |
||||
self.target_block_size_ = self.min_block_size_ |
||||
} else { |
||||
/* Combine this block with last block. */ |
||||
split.lengths[self.num_blocks_-1] += uint32(self.block_size_) |
||||
|
||||
histograms[self.last_histogram_ix_[0]] = combined_histo[0] |
||||
last_entropy[0] = combined_entropy[0] |
||||
if split.num_types == 1 { |
||||
last_entropy[1] = last_entropy[0] |
||||
} |
||||
|
||||
self.block_size_ = 0 |
||||
histogramClearDistance(&histograms[self.curr_histogram_ix_]) |
||||
self.merge_last_count_++ |
||||
if self.merge_last_count_ > 1 { |
||||
self.target_block_size_ += self.min_block_size_ |
||||
} |
||||
} |
||||
} |
||||
|
||||
if is_final { |
||||
*self.histograms_size_ = split.num_types |
||||
split.num_blocks = self.num_blocks_ |
||||
} |
||||
} |
||||
|
||||
/* Adds the next symbol to the current histogram. When the current histogram |
||||
reaches the target size, decides on merging the block. */ |
||||
func blockSplitterAddSymbolDistance(self *blockSplitterDistance, symbol uint) { |
||||
histogramAddDistance(&self.histograms_[self.curr_histogram_ix_], symbol) |
||||
self.block_size_++ |
||||
if self.block_size_ == self.target_block_size_ { |
||||
blockSplitterFinishBlockDistance(self, false) /* is_final = */ |
||||
} |
||||
} |
||||
@ -0,0 +1,162 @@ |
||||
package brotli |
||||
|
||||
/* Copyright 2015 Google Inc. All Rights Reserved. |
||||
|
||||
Distributed under MIT license. |
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/ |
||||
|
||||
/* Greedy block splitter for one block category (literal, command or distance). |
||||
*/ |
||||
type blockSplitterLiteral struct { |
||||
alphabet_size_ uint |
||||
min_block_size_ uint |
||||
split_threshold_ float64 |
||||
num_blocks_ uint |
||||
split_ *blockSplit |
||||
histograms_ []histogramLiteral |
||||
histograms_size_ *uint |
||||
target_block_size_ uint |
||||
block_size_ uint |
||||
curr_histogram_ix_ uint |
||||
last_histogram_ix_ [2]uint |
||||
last_entropy_ [2]float64 |
||||
merge_last_count_ uint |
||||
} |
||||
|
||||
func initBlockSplitterLiteral(self *blockSplitterLiteral, alphabet_size uint, min_block_size uint, split_threshold float64, num_symbols uint, split *blockSplit, histograms *[]histogramLiteral, histograms_size *uint) { |
||||
var max_num_blocks uint = num_symbols/min_block_size + 1 |
||||
var max_num_types uint = brotli_min_size_t(max_num_blocks, maxNumberOfBlockTypes+1) |
||||
/* We have to allocate one more histogram than the maximum number of block |
||||
types for the current histogram when the meta-block is too big. */ |
||||
self.alphabet_size_ = alphabet_size |
||||
|
||||
self.min_block_size_ = min_block_size |
||||
self.split_threshold_ = split_threshold |
||||
self.num_blocks_ = 0 |
||||
self.split_ = split |
||||
self.histograms_size_ = histograms_size |
||||
self.target_block_size_ = min_block_size |
||||
self.block_size_ = 0 |
||||
self.curr_histogram_ix_ = 0 |
||||
self.merge_last_count_ = 0 |
||||
brotli_ensure_capacity_uint8_t(&split.types, &split.types_alloc_size, max_num_blocks) |
||||
brotli_ensure_capacity_uint32_t(&split.lengths, &split.lengths_alloc_size, max_num_blocks) |
||||
self.split_.num_blocks = max_num_blocks |
||||
assert(*histograms == nil) |
||||
*histograms_size = max_num_types |
||||
*histograms = make([]histogramLiteral, (*histograms_size)) |
||||
self.histograms_ = *histograms |
||||
|
||||
/* Clear only current histogram. */ |
||||
histogramClearLiteral(&self.histograms_[0]) |
||||
|
||||
self.last_histogram_ix_[1] = 0 |
||||
self.last_histogram_ix_[0] = self.last_histogram_ix_[1] |
||||
} |
||||
|
||||
/* Does either of three things: |
||||
(1) emits the current block with a new block type; |
||||
(2) emits the current block with the type of the second last block; |
||||
(3) merges the current block with the last block. */ |
||||
func blockSplitterFinishBlockLiteral(self *blockSplitterLiteral, is_final bool) { |
||||
var split *blockSplit = self.split_ |
||||
var last_entropy []float64 = self.last_entropy_[:] |
||||
var histograms []histogramLiteral = self.histograms_ |
||||
self.block_size_ = brotli_max_size_t(self.block_size_, self.min_block_size_) |
||||
if self.num_blocks_ == 0 { |
||||
/* Create first block. */ |
||||
split.lengths[0] = uint32(self.block_size_) |
||||
|
||||
split.types[0] = 0 |
||||
last_entropy[0] = bitsEntropy(histograms[0].data_[:], self.alphabet_size_) |
||||
last_entropy[1] = last_entropy[0] |
||||
self.num_blocks_++ |
||||
split.num_types++ |
||||
self.curr_histogram_ix_++ |
||||
if self.curr_histogram_ix_ < *self.histograms_size_ { |
||||
histogramClearLiteral(&histograms[self.curr_histogram_ix_]) |
||||
} |
||||
self.block_size_ = 0 |
||||
} else if self.block_size_ > 0 { |
||||
var entropy float64 = bitsEntropy(histograms[self.curr_histogram_ix_].data_[:], self.alphabet_size_) |
||||
var combined_histo [2]histogramLiteral |
||||
var combined_entropy [2]float64 |
||||
var diff [2]float64 |
||||
var j uint |
||||
for j = 0; j < 2; j++ { |
||||
var last_histogram_ix uint = self.last_histogram_ix_[j] |
||||
combined_histo[j] = histograms[self.curr_histogram_ix_] |
||||
histogramAddHistogramLiteral(&combined_histo[j], &histograms[last_histogram_ix]) |
||||
combined_entropy[j] = bitsEntropy(combined_histo[j].data_[0:], self.alphabet_size_) |
||||
diff[j] = combined_entropy[j] - entropy - last_entropy[j] |
||||
} |
||||
|
||||
if split.num_types < maxNumberOfBlockTypes && diff[0] > self.split_threshold_ && diff[1] > self.split_threshold_ { |
||||
/* Create new block. */ |
||||
split.lengths[self.num_blocks_] = uint32(self.block_size_) |
||||
|
||||
split.types[self.num_blocks_] = byte(split.num_types) |
||||
self.last_histogram_ix_[1] = self.last_histogram_ix_[0] |
||||
self.last_histogram_ix_[0] = uint(byte(split.num_types)) |
||||
last_entropy[1] = last_entropy[0] |
||||
last_entropy[0] = entropy |
||||
self.num_blocks_++ |
||||
split.num_types++ |
||||
self.curr_histogram_ix_++ |
||||
if self.curr_histogram_ix_ < *self.histograms_size_ { |
||||
histogramClearLiteral(&histograms[self.curr_histogram_ix_]) |
||||
} |
||||
self.block_size_ = 0 |
||||
self.merge_last_count_ = 0 |
||||
self.target_block_size_ = self.min_block_size_ |
||||
} else if diff[1] < diff[0]-20.0 { |
||||
split.lengths[self.num_blocks_] = uint32(self.block_size_) |
||||
split.types[self.num_blocks_] = split.types[self.num_blocks_-2] |
||||
/* Combine this block with second last block. */ |
||||
|
||||
var tmp uint = self.last_histogram_ix_[0] |
||||
self.last_histogram_ix_[0] = self.last_histogram_ix_[1] |
||||
self.last_histogram_ix_[1] = tmp |
||||
histograms[self.last_histogram_ix_[0]] = combined_histo[1] |
||||
last_entropy[1] = last_entropy[0] |
||||
last_entropy[0] = combined_entropy[1] |
||||
self.num_blocks_++ |
||||
self.block_size_ = 0 |
||||
histogramClearLiteral(&histograms[self.curr_histogram_ix_]) |
||||
self.merge_last_count_ = 0 |
||||
self.target_block_size_ = self.min_block_size_ |
||||
} else { |
||||
/* Combine this block with last block. */ |
||||
split.lengths[self.num_blocks_-1] += uint32(self.block_size_) |
||||
|
||||
histograms[self.last_histogram_ix_[0]] = combined_histo[0] |
||||
last_entropy[0] = combined_entropy[0] |
||||
if split.num_types == 1 { |
||||
last_entropy[1] = last_entropy[0] |
||||
} |
||||
|
||||
self.block_size_ = 0 |
||||
histogramClearLiteral(&histograms[self.curr_histogram_ix_]) |
||||
self.merge_last_count_++ |
||||
if self.merge_last_count_ > 1 { |
||||
self.target_block_size_ += self.min_block_size_ |
||||
} |
||||
} |
||||
} |
||||
|
||||
if is_final { |
||||
*self.histograms_size_ = split.num_types |
||||
split.num_blocks = self.num_blocks_ |
||||
} |
||||
} |
||||
|
||||
/* Adds the next symbol to the current histogram. When the current histogram |
||||
reaches the target size, decides on merging the block. */ |
||||
func blockSplitterAddSymbolLiteral(self *blockSplitterLiteral, symbol uint) { |
||||
histogramAddLiteral(&self.histograms_[self.curr_histogram_ix_], symbol) |
||||
self.block_size_++ |
||||
if self.block_size_ == self.target_block_size_ { |
||||
blockSplitterFinishBlockLiteral(self, false) /* is_final = */ |
||||
} |
||||
} |
||||
@ -0,0 +1,37 @@ |
||||
package brotli |
||||
|
||||
/* Copyright 2017 Google Inc. All Rights Reserved. |
||||
|
||||
Distributed under MIT license. |
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/ |
||||
|
||||
/* Parameters for the Brotli encoder with chosen quality levels. */ |
||||
type hasherParams struct { |
||||
type_ int |
||||
bucket_bits int |
||||
block_bits int |
||||
hash_len int |
||||
num_last_distances_to_check int |
||||
} |
||||
|
||||
type distanceParams struct { |
||||
distance_postfix_bits uint32 |
||||
num_direct_distance_codes uint32 |
||||
alphabet_size uint32 |
||||
max_distance uint |
||||
} |
||||
|
||||
/* Encoding parameters */ |
||||
type encoderParams struct { |
||||
mode int |
||||
quality int |
||||
lgwin uint |
||||
lgblock int |
||||
size_hint uint |
||||
disable_literal_context_modeling bool |
||||
large_window bool |
||||
hasher hasherParams |
||||
dist distanceParams |
||||
dictionary encoderDictionary |
||||
} |
||||
@ -0,0 +1,103 @@ |
||||
package brotli |
||||
|
||||
/* Copyright 2013 Google Inc. All Rights Reserved. |
||||
|
||||
Distributed under MIT license. |
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/ |
||||
|
||||
func brotli_min_double(a float64, b float64) float64 { |
||||
if a < b { |
||||
return a |
||||
} else { |
||||
return b |
||||
} |
||||
} |
||||
|
||||
func brotli_max_double(a float64, b float64) float64 { |
||||
if a > b { |
||||
return a |
||||
} else { |
||||
return b |
||||
} |
||||
} |
||||
|
||||
func brotli_min_float(a float32, b float32) float32 { |
||||
if a < b { |
||||
return a |
||||
} else { |
||||
return b |
||||
} |
||||
} |
||||
|
||||
func brotli_max_float(a float32, b float32) float32 { |
||||
if a > b { |
||||
return a |
||||
} else { |
||||
return b |
||||
} |
||||
} |
||||
|
||||
func brotli_min_int(a int, b int) int { |
||||
if a < b { |
||||
return a |
||||
} else { |
||||
return b |
||||
} |
||||
} |
||||
|
||||
func brotli_max_int(a int, b int) int { |
||||
if a > b { |
||||
return a |
||||
} else { |
||||
return b |
||||
} |
||||
} |
||||
|
||||
func brotli_min_size_t(a uint, b uint) uint { |
||||
if a < b { |
||||
return a |
||||
} else { |
||||
return b |
||||
} |
||||
} |
||||
|
||||
func brotli_max_size_t(a uint, b uint) uint { |
||||
if a > b { |
||||
return a |
||||
} else { |
||||
return b |
||||
} |
||||
} |
||||
|
||||
func brotli_min_uint32_t(a uint32, b uint32) uint32 { |
||||
if a < b { |
||||
return a |
||||
} else { |
||||
return b |
||||
} |
||||
} |
||||
|
||||
func brotli_max_uint32_t(a uint32, b uint32) uint32 { |
||||
if a > b { |
||||
return a |
||||
} else { |
||||
return b |
||||
} |
||||
} |
||||
|
||||
func brotli_min_uint8_t(a byte, b byte) byte { |
||||
if a < b { |
||||
return a |
||||
} else { |
||||
return b |
||||
} |
||||
} |
||||
|
||||
func brotli_max_uint8_t(a byte, b byte) byte { |
||||
if a > b { |
||||
return a |
||||
} else { |
||||
return b |
||||
} |
||||
} |
||||
@ -0,0 +1,30 @@ |
||||
package brotli |
||||
|
||||
/* Copyright 2013 Google Inc. All Rights Reserved. |
||||
|
||||
Distributed under MIT license. |
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/ |
||||
|
||||
/* Functions for encoding of integers into prefix codes the amount of extra |
||||
bits, and the actual values of the extra bits. */ |
||||
|
||||
/* Here distance_code is an intermediate code, i.e. one of the special codes or |
||||
the actual distance increased by BROTLI_NUM_DISTANCE_SHORT_CODES - 1. */ |
||||
func prefixEncodeCopyDistance(distance_code uint, num_direct_codes uint, postfix_bits uint, code *uint16, extra_bits *uint32) { |
||||
if distance_code < numDistanceShortCodes+num_direct_codes { |
||||
*code = uint16(distance_code) |
||||
*extra_bits = 0 |
||||
return |
||||
} else { |
||||
var dist uint = (uint(1) << (postfix_bits + 2)) + (distance_code - numDistanceShortCodes - num_direct_codes) |
||||
var bucket uint = uint(log2FloorNonZero(dist) - 1) |
||||
var postfix_mask uint = (1 << postfix_bits) - 1 |
||||
var postfix uint = dist & postfix_mask |
||||
var prefix uint = (dist >> bucket) & 1 |
||||
var offset uint = (2 + prefix) << bucket |
||||
var nbits uint = bucket - postfix_bits |
||||
*code = uint16(nbits<<10 | (numDistanceShortCodes + num_direct_codes + ((2*(nbits-1) + prefix) << postfix_bits) + postfix)) |
||||
*extra_bits = uint32((dist - offset) >> postfix_bits) |
||||
} |
||||
} |
||||
@ -0,0 +1,723 @@ |
||||
package brotli |
||||
|
||||
/* Copyright 2013 Google Inc. All Rights Reserved. |
||||
|
||||
Distributed under MIT license. |
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/ |
||||
|
||||
type cmdLutElement struct { |
||||
insert_len_extra_bits byte |
||||
copy_len_extra_bits byte |
||||
distance_code int8 |
||||
context byte |
||||
insert_len_offset uint16 |
||||
copy_len_offset uint16 |
||||
} |
||||
|
||||
var kCmdLut = [numCommandSymbols]cmdLutElement{ |
||||
cmdLutElement{0x00, 0x00, 0, 0x00, 0x0000, 0x0002}, |
||||
cmdLutElement{0x00, 0x00, 0, 0x01, 0x0000, 0x0003}, |
||||
cmdLutElement{0x00, 0x00, 0, 0x02, 0x0000, 0x0004}, |
||||
cmdLutElement{0x00, 0x00, 0, 0x03, 0x0000, 0x0005}, |
||||
cmdLutElement{0x00, 0x00, 0, 0x03, 0x0000, 0x0006}, |
||||
cmdLutElement{0x00, 0x00, 0, 0x03, 0x0000, 0x0007}, |
||||
cmdLutElement{0x00, 0x00, 0, 0x03, 0x0000, 0x0008}, |
||||
cmdLutElement{0x00, 0x00, 0, 0x03, 0x0000, 0x0009}, |
||||
cmdLutElement{0x00, 0x00, 0, 0x00, 0x0001, 0x0002}, |
||||
cmdLutElement{0x00, 0x00, 0, 0x01, 0x0001, 0x0003}, |
||||
cmdLutElement{0x00, 0x00, 0, 0x02, 0x0001, 0x0004}, |
||||
cmdLutElement{0x00, 0x00, 0, 0x03, 0x0001, 0x0005}, |
||||
cmdLutElement{0x00, 0x00, 0, 0x03, 0x0001, 0x0006}, |
||||
cmdLutElement{0x00, 0x00, 0, 0x03, 0x0001, 0x0007}, |
||||
cmdLutElement{0x00, 0x00, 0, 0x03, 0x0001, 0x0008}, |
||||
cmdLutElement{0x00, 0x00, 0, 0x03, 0x0001, 0x0009}, |
||||
cmdLutElement{0x00, 0x00, 0, 0x00, 0x0002, 0x0002}, |
||||
cmdLutElement{0x00, 0x00, 0, 0x01, 0x0002, 0x0003}, |
||||
cmdLutElement{0x00, 0x00, 0, 0x02, 0x0002, 0x0004}, |
||||
cmdLutElement{0x00, 0x00, 0, 0x03, 0x0002, 0x0005}, |
||||
cmdLutElement{0x00, 0x00, 0, 0x03, 0x0002, 0x0006}, |
||||
cmdLutElement{0x00, 0x00, 0, 0x03, 0x0002, 0x0007}, |
||||
cmdLutElement{0x00, 0x00, 0, 0x03, 0x0002, 0x0008}, |
||||
cmdLutElement{0x00, 0x00, 0, 0x03, 0x0002, 0x0009}, |
||||
cmdLutElement{0x00, 0x00, 0, 0x00, 0x0003, 0x0002}, |
||||
cmdLutElement{0x00, 0x00, 0, 0x01, 0x0003, 0x0003}, |
||||
cmdLutElement{0x00, 0x00, 0, 0x02, 0x0003, 0x0004}, |
||||
cmdLutElement{0x00, 0x00, 0, 0x03, 0x0003, 0x0005}, |
||||
cmdLutElement{0x00, 0x00, 0, 0x03, 0x0003, 0x0006}, |
||||
cmdLutElement{0x00, 0x00, 0, 0x03, 0x0003, 0x0007}, |
||||
cmdLutElement{0x00, 0x00, 0, 0x03, 0x0003, 0x0008}, |
||||
cmdLutElement{0x00, 0x00, 0, 0x03, 0x0003, 0x0009}, |
||||
cmdLutElement{0x00, 0x00, 0, 0x00, 0x0004, 0x0002}, |
||||
cmdLutElement{0x00, 0x00, 0, 0x01, 0x0004, 0x0003}, |
||||
cmdLutElement{0x00, 0x00, 0, 0x02, 0x0004, 0x0004}, |
||||
cmdLutElement{0x00, 0x00, 0, 0x03, 0x0004, 0x0005}, |
||||
cmdLutElement{0x00, 0x00, 0, 0x03, 0x0004, 0x0006}, |
||||
cmdLutElement{0x00, 0x00, 0, 0x03, 0x0004, 0x0007}, |
||||
cmdLutElement{0x00, 0x00, 0, 0x03, 0x0004, 0x0008}, |
||||
cmdLutElement{0x00, 0x00, 0, 0x03, 0x0004, 0x0009}, |
||||
cmdLutElement{0x00, 0x00, 0, 0x00, 0x0005, 0x0002}, |
||||
cmdLutElement{0x00, 0x00, 0, 0x01, 0x0005, 0x0003}, |
||||
cmdLutElement{0x00, 0x00, 0, 0x02, 0x0005, 0x0004}, |
||||
cmdLutElement{0x00, 0x00, 0, 0x03, 0x0005, 0x0005}, |
||||
cmdLutElement{0x00, 0x00, 0, 0x03, 0x0005, 0x0006}, |
||||
cmdLutElement{0x00, 0x00, 0, 0x03, 0x0005, 0x0007}, |
||||
cmdLutElement{0x00, 0x00, 0, 0x03, 0x0005, 0x0008}, |
||||
cmdLutElement{0x00, 0x00, 0, 0x03, 0x0005, 0x0009}, |
||||
cmdLutElement{0x01, 0x00, 0, 0x00, 0x0006, 0x0002}, |
||||
cmdLutElement{0x01, 0x00, 0, 0x01, 0x0006, 0x0003}, |
||||
cmdLutElement{0x01, 0x00, 0, 0x02, 0x0006, 0x0004}, |
||||
cmdLutElement{0x01, 0x00, 0, 0x03, 0x0006, 0x0005}, |
||||
cmdLutElement{0x01, 0x00, 0, 0x03, 0x0006, 0x0006}, |
||||
cmdLutElement{0x01, 0x00, 0, 0x03, 0x0006, 0x0007}, |
||||
cmdLutElement{0x01, 0x00, 0, 0x03, 0x0006, 0x0008}, |
||||
cmdLutElement{0x01, 0x00, 0, 0x03, 0x0006, 0x0009}, |
||||
cmdLutElement{0x01, 0x00, 0, 0x00, 0x0008, 0x0002}, |
||||
cmdLutElement{0x01, 0x00, 0, 0x01, 0x0008, 0x0003}, |
||||
cmdLutElement{0x01, 0x00, 0, 0x02, 0x0008, 0x0004}, |
||||
cmdLutElement{0x01, 0x00, 0, 0x03, 0x0008, 0x0005}, |
||||
cmdLutElement{0x01, 0x00, 0, 0x03, 0x0008, 0x0006}, |
||||
cmdLutElement{0x01, 0x00, 0, 0x03, 0x0008, 0x0007}, |
||||
cmdLutElement{0x01, 0x00, 0, 0x03, 0x0008, 0x0008}, |
||||
cmdLutElement{0x01, 0x00, 0, 0x03, 0x0008, 0x0009}, |
||||
cmdLutElement{0x00, 0x01, 0, 0x03, 0x0000, 0x000a}, |
||||
cmdLutElement{0x00, 0x01, 0, 0x03, 0x0000, 0x000c}, |
||||
cmdLutElement{0x00, 0x02, 0, 0x03, 0x0000, 0x000e}, |
||||
cmdLutElement{0x00, 0x02, 0, 0x03, 0x0000, 0x0012}, |
||||
cmdLutElement{0x00, 0x03, 0, 0x03, 0x0000, 0x0016}, |
||||
cmdLutElement{0x00, 0x03, 0, 0x03, 0x0000, 0x001e}, |
||||
cmdLutElement{0x00, 0x04, 0, 0x03, 0x0000, 0x0026}, |
||||
cmdLutElement{0x00, 0x04, 0, 0x03, 0x0000, 0x0036}, |
||||
cmdLutElement{0x00, 0x01, 0, 0x03, 0x0001, 0x000a}, |
||||
cmdLutElement{0x00, 0x01, 0, 0x03, 0x0001, 0x000c}, |
||||
cmdLutElement{0x00, 0x02, 0, 0x03, 0x0001, 0x000e}, |
||||
cmdLutElement{0x00, 0x02, 0, 0x03, 0x0001, 0x0012}, |
||||
cmdLutElement{0x00, 0x03, 0, 0x03, 0x0001, 0x0016}, |
||||
cmdLutElement{0x00, 0x03, 0, 0x03, 0x0001, 0x001e}, |
||||
cmdLutElement{0x00, 0x04, 0, 0x03, 0x0001, 0x0026}, |
||||
cmdLutElement{0x00, 0x04, 0, 0x03, 0x0001, 0x0036}, |
||||
cmdLutElement{0x00, 0x01, 0, 0x03, 0x0002, 0x000a}, |
||||
cmdLutElement{0x00, 0x01, 0, 0x03, 0x0002, 0x000c}, |
||||
cmdLutElement{0x00, 0x02, 0, 0x03, 0x0002, 0x000e}, |
||||
cmdLutElement{0x00, 0x02, 0, 0x03, 0x0002, 0x0012}, |
||||
cmdLutElement{0x00, 0x03, 0, 0x03, 0x0002, 0x0016}, |
||||
cmdLutElement{0x00, 0x03, 0, 0x03, 0x0002, 0x001e}, |
||||
cmdLutElement{0x00, 0x04, 0, 0x03, 0x0002, 0x0026}, |
||||
cmdLutElement{0x00, 0x04, 0, 0x03, 0x0002, 0x0036}, |
||||
cmdLutElement{0x00, 0x01, 0, 0x03, 0x0003, 0x000a}, |
||||
cmdLutElement{0x00, 0x01, 0, 0x03, 0x0003, 0x000c}, |
||||
cmdLutElement{0x00, 0x02, 0, 0x03, 0x0003, 0x000e}, |
||||
cmdLutElement{0x00, 0x02, 0, 0x03, 0x0003, 0x0012}, |
||||
cmdLutElement{0x00, 0x03, 0, 0x03, 0x0003, 0x0016}, |
||||
cmdLutElement{0x00, 0x03, 0, 0x03, 0x0003, 0x001e}, |
||||
cmdLutElement{0x00, 0x04, 0, 0x03, 0x0003, 0x0026}, |
||||
cmdLutElement{0x00, 0x04, 0, 0x03, 0x0003, 0x0036}, |
||||
cmdLutElement{0x00, 0x01, 0, 0x03, 0x0004, 0x000a}, |
||||
cmdLutElement{0x00, 0x01, 0, 0x03, 0x0004, 0x000c}, |
||||
cmdLutElement{0x00, 0x02, 0, 0x03, 0x0004, 0x000e}, |
||||
cmdLutElement{0x00, 0x02, 0, 0x03, 0x0004, 0x0012}, |
||||
cmdLutElement{0x00, 0x03, 0, 0x03, 0x0004, 0x0016}, |
||||
cmdLutElement{0x00, 0x03, 0, 0x03, 0x0004, 0x001e}, |
||||
cmdLutElement{0x00, 0x04, 0, 0x03, 0x0004, 0x0026}, |
||||
cmdLutElement{0x00, 0x04, 0, 0x03, 0x0004, 0x0036}, |
||||
cmdLutElement{0x00, 0x01, 0, 0x03, 0x0005, 0x000a}, |
||||
cmdLutElement{0x00, 0x01, 0, 0x03, 0x0005, 0x000c}, |
||||
cmdLutElement{0x00, 0x02, 0, 0x03, 0x0005, 0x000e}, |
||||
cmdLutElement{0x00, 0x02, 0, 0x03, 0x0005, 0x0012}, |
||||
cmdLutElement{0x00, 0x03, 0, 0x03, 0x0005, 0x0016}, |
||||
cmdLutElement{0x00, 0x03, 0, 0x03, 0x0005, 0x001e}, |
||||
cmdLutElement{0x00, 0x04, 0, 0x03, 0x0005, 0x0026}, |
||||
cmdLutElement{0x00, 0x04, 0, 0x03, 0x0005, 0x0036}, |
||||
cmdLutElement{0x01, 0x01, 0, 0x03, 0x0006, 0x000a}, |
||||
cmdLutElement{0x01, 0x01, 0, 0x03, 0x0006, 0x000c}, |
||||
cmdLutElement{0x01, 0x02, 0, 0x03, 0x0006, 0x000e}, |
||||
cmdLutElement{0x01, 0x02, 0, 0x03, 0x0006, 0x0012}, |
||||
cmdLutElement{0x01, 0x03, 0, 0x03, 0x0006, 0x0016}, |
||||
cmdLutElement{0x01, 0x03, 0, 0x03, 0x0006, 0x001e}, |
||||
cmdLutElement{0x01, 0x04, 0, 0x03, 0x0006, 0x0026}, |
||||
cmdLutElement{0x01, 0x04, 0, 0x03, 0x0006, 0x0036}, |
||||
cmdLutElement{0x01, 0x01, 0, 0x03, 0x0008, 0x000a}, |
||||
cmdLutElement{0x01, 0x01, 0, 0x03, 0x0008, 0x000c}, |
||||
cmdLutElement{0x01, 0x02, 0, 0x03, 0x0008, 0x000e}, |
||||
cmdLutElement{0x01, 0x02, 0, 0x03, 0x0008, 0x0012}, |
||||
cmdLutElement{0x01, 0x03, 0, 0x03, 0x0008, 0x0016}, |
||||
cmdLutElement{0x01, 0x03, 0, 0x03, 0x0008, 0x001e}, |
||||
cmdLutElement{0x01, 0x04, 0, 0x03, 0x0008, 0x0026}, |
||||
cmdLutElement{0x01, 0x04, 0, 0x03, 0x0008, 0x0036}, |
||||
cmdLutElement{0x00, 0x00, -1, 0x00, 0x0000, 0x0002}, |
||||
cmdLutElement{0x00, 0x00, -1, 0x01, 0x0000, 0x0003}, |
||||
cmdLutElement{0x00, 0x00, -1, 0x02, 0x0000, 0x0004}, |
||||
cmdLutElement{0x00, 0x00, -1, 0x03, 0x0000, 0x0005}, |
||||
cmdLutElement{0x00, 0x00, -1, 0x03, 0x0000, 0x0006}, |
||||
cmdLutElement{0x00, 0x00, -1, 0x03, 0x0000, 0x0007}, |
||||
cmdLutElement{0x00, 0x00, -1, 0x03, 0x0000, 0x0008}, |
||||
cmdLutElement{0x00, 0x00, -1, 0x03, 0x0000, 0x0009}, |
||||
cmdLutElement{0x00, 0x00, -1, 0x00, 0x0001, 0x0002}, |
||||
cmdLutElement{0x00, 0x00, -1, 0x01, 0x0001, 0x0003}, |
||||
cmdLutElement{0x00, 0x00, -1, 0x02, 0x0001, 0x0004}, |
||||
cmdLutElement{0x00, 0x00, -1, 0x03, 0x0001, 0x0005}, |
||||
cmdLutElement{0x00, 0x00, -1, 0x03, 0x0001, 0x0006}, |
||||
cmdLutElement{0x00, 0x00, -1, 0x03, 0x0001, 0x0007}, |
||||
cmdLutElement{0x00, 0x00, -1, 0x03, 0x0001, 0x0008}, |
||||
cmdLutElement{0x00, 0x00, -1, 0x03, 0x0001, 0x0009}, |
||||
cmdLutElement{0x00, 0x00, -1, 0x00, 0x0002, 0x0002}, |
||||
cmdLutElement{0x00, 0x00, -1, 0x01, 0x0002, 0x0003}, |
||||
cmdLutElement{0x00, 0x00, -1, 0x02, 0x0002, 0x0004}, |
||||
cmdLutElement{0x00, 0x00, -1, 0x03, 0x0002, 0x0005}, |
||||
cmdLutElement{0x00, 0x00, -1, 0x03, 0x0002, 0x0006}, |
||||
cmdLutElement{0x00, 0x00, -1, 0x03, 0x0002, 0x0007}, |
||||
cmdLutElement{0x00, 0x00, -1, 0x03, 0x0002, 0x0008}, |
||||
cmdLutElement{0x00, 0x00, -1, 0x03, 0x0002, 0x0009}, |
||||
cmdLutElement{0x00, 0x00, -1, 0x00, 0x0003, 0x0002}, |
||||
cmdLutElement{0x00, 0x00, -1, 0x01, 0x0003, 0x0003}, |
||||
cmdLutElement{0x00, 0x00, -1, 0x02, 0x0003, 0x0004}, |
||||
cmdLutElement{0x00, 0x00, -1, 0x03, 0x0003, 0x0005}, |
||||
cmdLutElement{0x00, 0x00, -1, 0x03, 0x0003, 0x0006}, |
||||
cmdLutElement{0x00, 0x00, -1, 0x03, 0x0003, 0x0007}, |
||||
cmdLutElement{0x00, 0x00, -1, 0x03, 0x0003, 0x0008}, |
||||
cmdLutElement{0x00, 0x00, -1, 0x03, 0x0003, 0x0009}, |
||||
cmdLutElement{0x00, 0x00, -1, 0x00, 0x0004, 0x0002}, |
||||
cmdLutElement{0x00, 0x00, -1, 0x01, 0x0004, 0x0003}, |
||||
cmdLutElement{0x00, 0x00, -1, 0x02, 0x0004, 0x0004}, |
||||
cmdLutElement{0x00, 0x00, -1, 0x03, 0x0004, 0x0005}, |
||||
cmdLutElement{0x00, 0x00, -1, 0x03, 0x0004, 0x0006}, |
||||
cmdLutElement{0x00, 0x00, -1, 0x03, 0x0004, 0x0007}, |
||||
cmdLutElement{0x00, 0x00, -1, 0x03, 0x0004, 0x0008}, |
||||
cmdLutElement{0x00, 0x00, -1, 0x03, 0x0004, 0x0009}, |
||||
cmdLutElement{0x00, 0x00, -1, 0x00, 0x0005, 0x0002}, |
||||
cmdLutElement{0x00, 0x00, -1, 0x01, 0x0005, 0x0003}, |
||||
cmdLutElement{0x00, 0x00, -1, 0x02, 0x0005, 0x0004}, |
||||
cmdLutElement{0x00, 0x00, -1, 0x03, 0x0005, 0x0005}, |
||||
cmdLutElement{0x00, 0x00, -1, 0x03, 0x0005, 0x0006}, |
||||
cmdLutElement{0x00, 0x00, -1, 0x03, 0x0005, 0x0007}, |
||||
cmdLutElement{0x00, 0x00, -1, 0x03, 0x0005, 0x0008}, |
||||
cmdLutElement{0x00, 0x00, -1, 0x03, 0x0005, 0x0009}, |
||||
cmdLutElement{0x01, 0x00, -1, 0x00, 0x0006, 0x0002}, |
||||
cmdLutElement{0x01, 0x00, -1, 0x01, 0x0006, 0x0003}, |
||||
cmdLutElement{0x01, 0x00, -1, 0x02, 0x0006, 0x0004}, |
||||
cmdLutElement{0x01, 0x00, -1, 0x03, 0x0006, 0x0005}, |
||||
cmdLutElement{0x01, 0x00, -1, 0x03, 0x0006, 0x0006}, |
||||
cmdLutElement{0x01, 0x00, -1, 0x03, 0x0006, 0x0007}, |
||||
cmdLutElement{0x01, 0x00, -1, 0x03, 0x0006, 0x0008}, |
||||
cmdLutElement{0x01, 0x00, -1, 0x03, 0x0006, 0x0009}, |
||||
cmdLutElement{0x01, 0x00, -1, 0x00, 0x0008, 0x0002}, |
||||
cmdLutElement{0x01, 0x00, -1, 0x01, 0x0008, 0x0003}, |
||||
cmdLutElement{0x01, 0x00, -1, 0x02, 0x0008, 0x0004}, |
||||
cmdLutElement{0x01, 0x00, -1, 0x03, 0x0008, 0x0005}, |
||||
cmdLutElement{0x01, 0x00, -1, 0x03, 0x0008, 0x0006}, |
||||
cmdLutElement{0x01, 0x00, -1, 0x03, 0x0008, 0x0007}, |
||||
cmdLutElement{0x01, 0x00, -1, 0x03, 0x0008, 0x0008}, |
||||
cmdLutElement{0x01, 0x00, -1, 0x03, 0x0008, 0x0009}, |
||||
cmdLutElement{0x00, 0x01, -1, 0x03, 0x0000, 0x000a}, |
||||
cmdLutElement{0x00, 0x01, -1, 0x03, 0x0000, 0x000c}, |
||||
cmdLutElement{0x00, 0x02, -1, 0x03, 0x0000, 0x000e}, |
||||
cmdLutElement{0x00, 0x02, -1, 0x03, 0x0000, 0x0012}, |
||||
cmdLutElement{0x00, 0x03, -1, 0x03, 0x0000, 0x0016}, |
||||
cmdLutElement{0x00, 0x03, -1, 0x03, 0x0000, 0x001e}, |
||||
cmdLutElement{0x00, 0x04, -1, 0x03, 0x0000, 0x0026}, |
||||
cmdLutElement{0x00, 0x04, -1, 0x03, 0x0000, 0x0036}, |
||||
cmdLutElement{0x00, 0x01, -1, 0x03, 0x0001, 0x000a}, |
||||
cmdLutElement{0x00, 0x01, -1, 0x03, 0x0001, 0x000c}, |
||||
cmdLutElement{0x00, 0x02, -1, 0x03, 0x0001, 0x000e}, |
||||
cmdLutElement{0x00, 0x02, -1, 0x03, 0x0001, 0x0012}, |
||||
cmdLutElement{0x00, 0x03, -1, 0x03, 0x0001, 0x0016}, |
||||
cmdLutElement{0x00, 0x03, -1, 0x03, 0x0001, 0x001e}, |
||||
cmdLutElement{0x00, 0x04, -1, 0x03, 0x0001, 0x0026}, |
||||
cmdLutElement{0x00, 0x04, -1, 0x03, 0x0001, 0x0036}, |
||||
cmdLutElement{0x00, 0x01, -1, 0x03, 0x0002, 0x000a}, |
||||
cmdLutElement{0x00, 0x01, -1, 0x03, 0x0002, 0x000c}, |
||||
cmdLutElement{0x00, 0x02, -1, 0x03, 0x0002, 0x000e}, |
||||
cmdLutElement{0x00, 0x02, -1, 0x03, 0x0002, 0x0012}, |
||||
cmdLutElement{0x00, 0x03, -1, 0x03, 0x0002, 0x0016}, |
||||
cmdLutElement{0x00, 0x03, -1, 0x03, 0x0002, 0x001e}, |
||||
cmdLutElement{0x00, 0x04, -1, 0x03, 0x0002, 0x0026}, |
||||
cmdLutElement{0x00, 0x04, -1, 0x03, 0x0002, 0x0036}, |
||||
cmdLutElement{0x00, 0x01, -1, 0x03, 0x0003, 0x000a}, |
||||
cmdLutElement{0x00, 0x01, -1, 0x03, 0x0003, 0x000c}, |
||||
cmdLutElement{0x00, 0x02, -1, 0x03, 0x0003, 0x000e}, |
||||
cmdLutElement{0x00, 0x02, -1, 0x03, 0x0003, 0x0012}, |
||||
cmdLutElement{0x00, 0x03, -1, 0x03, 0x0003, 0x0016}, |
||||
cmdLutElement{0x00, 0x03, -1, 0x03, 0x0003, 0x001e}, |
||||
cmdLutElement{0x00, 0x04, -1, 0x03, 0x0003, 0x0026}, |
||||
cmdLutElement{0x00, 0x04, -1, 0x03, 0x0003, 0x0036}, |
||||
cmdLutElement{0x00, 0x01, -1, 0x03, 0x0004, 0x000a}, |
||||
cmdLutElement{0x00, 0x01, -1, 0x03, 0x0004, 0x000c}, |
||||
cmdLutElement{0x00, 0x02, -1, 0x03, 0x0004, 0x000e}, |
||||
cmdLutElement{0x00, 0x02, -1, 0x03, 0x0004, 0x0012}, |
||||
cmdLutElement{0x00, 0x03, -1, 0x03, 0x0004, 0x0016}, |
||||
cmdLutElement{0x00, 0x03, -1, 0x03, 0x0004, 0x001e}, |
||||
cmdLutElement{0x00, 0x04, -1, 0x03, 0x0004, 0x0026}, |
||||
cmdLutElement{0x00, 0x04, -1, 0x03, 0x0004, 0x0036}, |
||||
cmdLutElement{0x00, 0x01, -1, 0x03, 0x0005, 0x000a}, |
||||
cmdLutElement{0x00, 0x01, -1, 0x03, 0x0005, 0x000c}, |
||||
cmdLutElement{0x00, 0x02, -1, 0x03, 0x0005, 0x000e}, |
||||
cmdLutElement{0x00, 0x02, -1, 0x03, 0x0005, 0x0012}, |
||||
cmdLutElement{0x00, 0x03, -1, 0x03, 0x0005, 0x0016}, |
||||
cmdLutElement{0x00, 0x03, -1, 0x03, 0x0005, 0x001e}, |
||||
cmdLutElement{0x00, 0x04, -1, 0x03, 0x0005, 0x0026}, |
||||
cmdLutElement{0x00, 0x04, -1, 0x03, 0x0005, 0x0036}, |
||||
cmdLutElement{0x01, 0x01, -1, 0x03, 0x0006, 0x000a}, |
||||
cmdLutElement{0x01, 0x01, -1, 0x03, 0x0006, 0x000c}, |
||||
cmdLutElement{0x01, 0x02, -1, 0x03, 0x0006, 0x000e}, |
||||
cmdLutElement{0x01, 0x02, -1, 0x03, 0x0006, 0x0012}, |
||||
cmdLutElement{0x01, 0x03, -1, 0x03, 0x0006, 0x0016}, |
||||
cmdLutElement{0x01, 0x03, -1, 0x03, 0x0006, 0x001e}, |
||||
cmdLutElement{0x01, 0x04, -1, 0x03, 0x0006, 0x0026}, |
||||
cmdLutElement{0x01, 0x04, -1, 0x03, 0x0006, 0x0036}, |
||||
cmdLutElement{0x01, 0x01, -1, 0x03, 0x0008, 0x000a}, |
||||
cmdLutElement{0x01, 0x01, -1, 0x03, 0x0008, 0x000c}, |
||||
cmdLutElement{0x01, 0x02, -1, 0x03, 0x0008, 0x000e}, |
||||
cmdLutElement{0x01, 0x02, -1, 0x03, 0x0008, 0x0012}, |
||||
cmdLutElement{0x01, 0x03, -1, 0x03, 0x0008, 0x0016}, |
||||
cmdLutElement{0x01, 0x03, -1, 0x03, 0x0008, 0x001e}, |
||||
cmdLutElement{0x01, 0x04, -1, 0x03, 0x0008, 0x0026}, |
||||
cmdLutElement{0x01, 0x04, -1, 0x03, 0x0008, 0x0036}, |
||||
cmdLutElement{0x02, 0x00, -1, 0x00, 0x000a, 0x0002}, |
||||
cmdLutElement{0x02, 0x00, -1, 0x01, 0x000a, 0x0003}, |
||||
cmdLutElement{0x02, 0x00, -1, 0x02, 0x000a, 0x0004}, |
||||
cmdLutElement{0x02, 0x00, -1, 0x03, 0x000a, 0x0005}, |
||||
cmdLutElement{0x02, 0x00, -1, 0x03, 0x000a, 0x0006}, |
||||
cmdLutElement{0x02, 0x00, -1, 0x03, 0x000a, 0x0007}, |
||||
cmdLutElement{0x02, 0x00, -1, 0x03, 0x000a, 0x0008}, |
||||
cmdLutElement{0x02, 0x00, -1, 0x03, 0x000a, 0x0009}, |
||||
cmdLutElement{0x02, 0x00, -1, 0x00, 0x000e, 0x0002}, |
||||
cmdLutElement{0x02, 0x00, -1, 0x01, 0x000e, 0x0003}, |
||||
cmdLutElement{0x02, 0x00, -1, 0x02, 0x000e, 0x0004}, |
||||
cmdLutElement{0x02, 0x00, -1, 0x03, 0x000e, 0x0005}, |
||||
cmdLutElement{0x02, 0x00, -1, 0x03, 0x000e, 0x0006}, |
||||
cmdLutElement{0x02, 0x00, -1, 0x03, 0x000e, 0x0007}, |
||||
cmdLutElement{0x02, 0x00, -1, 0x03, 0x000e, 0x0008}, |
||||
cmdLutElement{0x02, 0x00, -1, 0x03, 0x000e, 0x0009}, |
||||
cmdLutElement{0x03, 0x00, -1, 0x00, 0x0012, 0x0002}, |
||||
cmdLutElement{0x03, 0x00, -1, 0x01, 0x0012, 0x0003}, |
||||
cmdLutElement{0x03, 0x00, -1, 0x02, 0x0012, 0x0004}, |
||||
cmdLutElement{0x03, 0x00, -1, 0x03, 0x0012, 0x0005}, |
||||
cmdLutElement{0x03, 0x00, -1, 0x03, 0x0012, 0x0006}, |
||||
cmdLutElement{0x03, 0x00, -1, 0x03, 0x0012, 0x0007}, |
||||
cmdLutElement{0x03, 0x00, -1, 0x03, 0x0012, 0x0008}, |
||||
cmdLutElement{0x03, 0x00, -1, 0x03, 0x0012, 0x0009}, |
||||
cmdLutElement{0x03, 0x00, -1, 0x00, 0x001a, 0x0002}, |
||||
cmdLutElement{0x03, 0x00, -1, 0x01, 0x001a, 0x0003}, |
||||
cmdLutElement{0x03, 0x00, -1, 0x02, 0x001a, 0x0004}, |
||||
cmdLutElement{0x03, 0x00, -1, 0x03, 0x001a, 0x0005}, |
||||
cmdLutElement{0x03, 0x00, -1, 0x03, 0x001a, 0x0006}, |
||||
cmdLutElement{0x03, 0x00, -1, 0x03, 0x001a, 0x0007}, |
||||
cmdLutElement{0x03, 0x00, -1, 0x03, 0x001a, 0x0008}, |
||||
cmdLutElement{0x03, 0x00, -1, 0x03, 0x001a, 0x0009}, |
||||
cmdLutElement{0x04, 0x00, -1, 0x00, 0x0022, 0x0002}, |
||||
cmdLutElement{0x04, 0x00, -1, 0x01, 0x0022, 0x0003}, |
||||
cmdLutElement{0x04, 0x00, -1, 0x02, 0x0022, 0x0004}, |
||||
cmdLutElement{0x04, 0x00, -1, 0x03, 0x0022, 0x0005}, |
||||
cmdLutElement{0x04, 0x00, -1, 0x03, 0x0022, 0x0006}, |
||||
cmdLutElement{0x04, 0x00, -1, 0x03, 0x0022, 0x0007}, |
||||
cmdLutElement{0x04, 0x00, -1, 0x03, 0x0022, 0x0008}, |
||||
cmdLutElement{0x04, 0x00, -1, 0x03, 0x0022, 0x0009}, |
||||
cmdLutElement{0x04, 0x00, -1, 0x00, 0x0032, 0x0002}, |
||||
cmdLutElement{0x04, 0x00, -1, 0x01, 0x0032, 0x0003}, |
||||
cmdLutElement{0x04, 0x00, -1, 0x02, 0x0032, 0x0004}, |
||||
cmdLutElement{0x04, 0x00, -1, 0x03, 0x0032, 0x0005}, |
||||
cmdLutElement{0x04, 0x00, -1, 0x03, 0x0032, 0x0006}, |
||||
cmdLutElement{0x04, 0x00, -1, 0x03, 0x0032, 0x0007}, |
||||
cmdLutElement{0x04, 0x00, -1, 0x03, 0x0032, 0x0008}, |
||||
cmdLutElement{0x04, 0x00, -1, 0x03, 0x0032, 0x0009}, |
||||
cmdLutElement{0x05, 0x00, -1, 0x00, 0x0042, 0x0002}, |
||||
cmdLutElement{0x05, 0x00, -1, 0x01, 0x0042, 0x0003}, |
||||
cmdLutElement{0x05, 0x00, -1, 0x02, 0x0042, 0x0004}, |
||||
cmdLutElement{0x05, 0x00, -1, 0x03, 0x0042, 0x0005}, |
||||
cmdLutElement{0x05, 0x00, -1, 0x03, 0x0042, 0x0006}, |
||||
cmdLutElement{0x05, 0x00, -1, 0x03, 0x0042, 0x0007}, |
||||
cmdLutElement{0x05, 0x00, -1, 0x03, 0x0042, 0x0008}, |
||||
cmdLutElement{0x05, 0x00, -1, 0x03, 0x0042, 0x0009}, |
||||
cmdLutElement{0x05, 0x00, -1, 0x00, 0x0062, 0x0002}, |
||||
cmdLutElement{0x05, 0x00, -1, 0x01, 0x0062, 0x0003}, |
||||
cmdLutElement{0x05, 0x00, -1, 0x02, 0x0062, 0x0004}, |
||||
cmdLutElement{0x05, 0x00, -1, 0x03, 0x0062, 0x0005}, |
||||
cmdLutElement{0x05, 0x00, -1, 0x03, 0x0062, 0x0006}, |
||||
cmdLutElement{0x05, 0x00, -1, 0x03, 0x0062, 0x0007}, |
||||
cmdLutElement{0x05, 0x00, -1, 0x03, 0x0062, 0x0008}, |
||||
cmdLutElement{0x05, 0x00, -1, 0x03, 0x0062, 0x0009}, |
||||
cmdLutElement{0x02, 0x01, -1, 0x03, 0x000a, 0x000a}, |
||||
cmdLutElement{0x02, 0x01, -1, 0x03, 0x000a, 0x000c}, |
||||
cmdLutElement{0x02, 0x02, -1, 0x03, 0x000a, 0x000e}, |
||||
cmdLutElement{0x02, 0x02, -1, 0x03, 0x000a, 0x0012}, |
||||
cmdLutElement{0x02, 0x03, -1, 0x03, 0x000a, 0x0016}, |
||||
cmdLutElement{0x02, 0x03, -1, 0x03, 0x000a, 0x001e}, |
||||
cmdLutElement{0x02, 0x04, -1, 0x03, 0x000a, 0x0026}, |
||||
cmdLutElement{0x02, 0x04, -1, 0x03, 0x000a, 0x0036}, |
||||
cmdLutElement{0x02, 0x01, -1, 0x03, 0x000e, 0x000a}, |
||||
cmdLutElement{0x02, 0x01, -1, 0x03, 0x000e, 0x000c}, |
||||
cmdLutElement{0x02, 0x02, -1, 0x03, 0x000e, 0x000e}, |
||||
cmdLutElement{0x02, 0x02, -1, 0x03, 0x000e, 0x0012}, |
||||
cmdLutElement{0x02, 0x03, -1, 0x03, 0x000e, 0x0016}, |
||||
cmdLutElement{0x02, 0x03, -1, 0x03, 0x000e, 0x001e}, |
||||
cmdLutElement{0x02, 0x04, -1, 0x03, 0x000e, 0x0026}, |
||||
cmdLutElement{0x02, 0x04, -1, 0x03, 0x000e, 0x0036}, |
||||
cmdLutElement{0x03, 0x01, -1, 0x03, 0x0012, 0x000a}, |
||||
cmdLutElement{0x03, 0x01, -1, 0x03, 0x0012, 0x000c}, |
||||
cmdLutElement{0x03, 0x02, -1, 0x03, 0x0012, 0x000e}, |
||||
cmdLutElement{0x03, 0x02, -1, 0x03, 0x0012, 0x0012}, |
||||
cmdLutElement{0x03, 0x03, -1, 0x03, 0x0012, 0x0016}, |
||||
cmdLutElement{0x03, 0x03, -1, 0x03, 0x0012, 0x001e}, |
||||
cmdLutElement{0x03, 0x04, -1, 0x03, 0x0012, 0x0026}, |
||||
cmdLutElement{0x03, 0x04, -1, 0x03, 0x0012, 0x0036}, |
||||
cmdLutElement{0x03, 0x01, -1, 0x03, 0x001a, 0x000a}, |
||||
cmdLutElement{0x03, 0x01, -1, 0x03, 0x001a, 0x000c}, |
||||
cmdLutElement{0x03, 0x02, -1, 0x03, 0x001a, 0x000e}, |
||||
cmdLutElement{0x03, 0x02, -1, 0x03, 0x001a, 0x0012}, |
||||
cmdLutElement{0x03, 0x03, -1, 0x03, 0x001a, 0x0016}, |
||||
cmdLutElement{0x03, 0x03, -1, 0x03, 0x001a, 0x001e}, |
||||
cmdLutElement{0x03, 0x04, -1, 0x03, 0x001a, 0x0026}, |
||||
cmdLutElement{0x03, 0x04, -1, 0x03, 0x001a, 0x0036}, |
||||
cmdLutElement{0x04, 0x01, -1, 0x03, 0x0022, 0x000a}, |
||||
cmdLutElement{0x04, 0x01, -1, 0x03, 0x0022, 0x000c}, |
||||
cmdLutElement{0x04, 0x02, -1, 0x03, 0x0022, 0x000e}, |
||||
cmdLutElement{0x04, 0x02, -1, 0x03, 0x0022, 0x0012}, |
||||
cmdLutElement{0x04, 0x03, -1, 0x03, 0x0022, 0x0016}, |
||||
cmdLutElement{0x04, 0x03, -1, 0x03, 0x0022, 0x001e}, |
||||
cmdLutElement{0x04, 0x04, -1, 0x03, 0x0022, 0x0026}, |
||||
cmdLutElement{0x04, 0x04, -1, 0x03, 0x0022, 0x0036}, |
||||
cmdLutElement{0x04, 0x01, -1, 0x03, 0x0032, 0x000a}, |
||||
cmdLutElement{0x04, 0x01, -1, 0x03, 0x0032, 0x000c}, |
||||
cmdLutElement{0x04, 0x02, -1, 0x03, 0x0032, 0x000e}, |
||||
cmdLutElement{0x04, 0x02, -1, 0x03, 0x0032, 0x0012}, |
||||
cmdLutElement{0x04, 0x03, -1, 0x03, 0x0032, 0x0016}, |
||||
cmdLutElement{0x04, 0x03, -1, 0x03, 0x0032, 0x001e}, |
||||
cmdLutElement{0x04, 0x04, -1, 0x03, 0x0032, 0x0026}, |
||||
cmdLutElement{0x04, 0x04, -1, 0x03, 0x0032, 0x0036}, |
||||
cmdLutElement{0x05, 0x01, -1, 0x03, 0x0042, 0x000a}, |
||||
cmdLutElement{0x05, 0x01, -1, 0x03, 0x0042, 0x000c}, |
||||
cmdLutElement{0x05, 0x02, -1, 0x03, 0x0042, 0x000e}, |
||||
cmdLutElement{0x05, 0x02, -1, 0x03, 0x0042, 0x0012}, |
||||
cmdLutElement{0x05, 0x03, -1, 0x03, 0x0042, 0x0016}, |
||||
cmdLutElement{0x05, 0x03, -1, 0x03, 0x0042, 0x001e}, |
||||
cmdLutElement{0x05, 0x04, -1, 0x03, 0x0042, 0x0026}, |
||||
cmdLutElement{0x05, 0x04, -1, 0x03, 0x0042, 0x0036}, |
||||
cmdLutElement{0x05, 0x01, -1, 0x03, 0x0062, 0x000a}, |
||||
cmdLutElement{0x05, 0x01, -1, 0x03, 0x0062, 0x000c}, |
||||
cmdLutElement{0x05, 0x02, -1, 0x03, 0x0062, 0x000e}, |
||||
cmdLutElement{0x05, 0x02, -1, 0x03, 0x0062, 0x0012}, |
||||
cmdLutElement{0x05, 0x03, -1, 0x03, 0x0062, 0x0016}, |
||||
cmdLutElement{0x05, 0x03, -1, 0x03, 0x0062, 0x001e}, |
||||
cmdLutElement{0x05, 0x04, -1, 0x03, 0x0062, 0x0026}, |
||||
cmdLutElement{0x05, 0x04, -1, 0x03, 0x0062, 0x0036}, |
||||
cmdLutElement{0x00, 0x05, -1, 0x03, 0x0000, 0x0046}, |
||||
cmdLutElement{0x00, 0x05, -1, 0x03, 0x0000, 0x0066}, |
||||
cmdLutElement{0x00, 0x06, -1, 0x03, 0x0000, 0x0086}, |
||||
cmdLutElement{0x00, 0x07, -1, 0x03, 0x0000, 0x00c6}, |
||||
cmdLutElement{0x00, 0x08, -1, 0x03, 0x0000, 0x0146}, |
||||
cmdLutElement{0x00, 0x09, -1, 0x03, 0x0000, 0x0246}, |
||||
cmdLutElement{0x00, 0x0a, -1, 0x03, 0x0000, 0x0446}, |
||||
cmdLutElement{0x00, 0x18, -1, 0x03, 0x0000, 0x0846}, |
||||
cmdLutElement{0x00, 0x05, -1, 0x03, 0x0001, 0x0046}, |
||||
cmdLutElement{0x00, 0x05, -1, 0x03, 0x0001, 0x0066}, |
||||
cmdLutElement{0x00, 0x06, -1, 0x03, 0x0001, 0x0086}, |
||||
cmdLutElement{0x00, 0x07, -1, 0x03, 0x0001, 0x00c6}, |
||||
cmdLutElement{0x00, 0x08, -1, 0x03, 0x0001, 0x0146}, |
||||
cmdLutElement{0x00, 0x09, -1, 0x03, 0x0001, 0x0246}, |
||||
cmdLutElement{0x00, 0x0a, -1, 0x03, 0x0001, 0x0446}, |
||||
cmdLutElement{0x00, 0x18, -1, 0x03, 0x0001, 0x0846}, |
||||
cmdLutElement{0x00, 0x05, -1, 0x03, 0x0002, 0x0046}, |
||||
cmdLutElement{0x00, 0x05, -1, 0x03, 0x0002, 0x0066}, |
||||
cmdLutElement{0x00, 0x06, -1, 0x03, 0x0002, 0x0086}, |
||||
cmdLutElement{0x00, 0x07, -1, 0x03, 0x0002, 0x00c6}, |
||||
cmdLutElement{0x00, 0x08, -1, 0x03, 0x0002, 0x0146}, |
||||
cmdLutElement{0x00, 0x09, -1, 0x03, 0x0002, 0x0246}, |
||||
cmdLutElement{0x00, 0x0a, -1, 0x03, 0x0002, 0x0446}, |
||||
cmdLutElement{0x00, 0x18, -1, 0x03, 0x0002, 0x0846}, |
||||
cmdLutElement{0x00, 0x05, -1, 0x03, 0x0003, 0x0046}, |
||||
cmdLutElement{0x00, 0x05, -1, 0x03, 0x0003, 0x0066}, |
||||
cmdLutElement{0x00, 0x06, -1, 0x03, 0x0003, 0x0086}, |
||||
cmdLutElement{0x00, 0x07, -1, 0x03, 0x0003, 0x00c6}, |
||||
cmdLutElement{0x00, 0x08, -1, 0x03, 0x0003, 0x0146}, |
||||
cmdLutElement{0x00, 0x09, -1, 0x03, 0x0003, 0x0246}, |
||||
cmdLutElement{0x00, 0x0a, -1, 0x03, 0x0003, 0x0446}, |
||||
cmdLutElement{0x00, 0x18, -1, 0x03, 0x0003, 0x0846}, |
||||
cmdLutElement{0x00, 0x05, -1, 0x03, 0x0004, 0x0046}, |
||||
cmdLutElement{0x00, 0x05, -1, 0x03, 0x0004, 0x0066}, |
||||
cmdLutElement{0x00, 0x06, -1, 0x03, 0x0004, 0x0086}, |
||||
cmdLutElement{0x00, 0x07, -1, 0x03, 0x0004, 0x00c6}, |
||||
cmdLutElement{0x00, 0x08, -1, 0x03, 0x0004, 0x0146}, |
||||
cmdLutElement{0x00, 0x09, -1, 0x03, 0x0004, 0x0246}, |
||||
cmdLutElement{0x00, 0x0a, -1, 0x03, 0x0004, 0x0446}, |
||||
cmdLutElement{0x00, 0x18, -1, 0x03, 0x0004, 0x0846}, |
||||
cmdLutElement{0x00, 0x05, -1, 0x03, 0x0005, 0x0046}, |
||||
cmdLutElement{0x00, 0x05, -1, 0x03, 0x0005, 0x0066}, |
||||
cmdLutElement{0x00, 0x06, -1, 0x03, 0x0005, 0x0086}, |
||||
cmdLutElement{0x00, 0x07, -1, 0x03, 0x0005, 0x00c6}, |
||||
cmdLutElement{0x00, 0x08, -1, 0x03, 0x0005, 0x0146}, |
||||
cmdLutElement{0x00, 0x09, -1, 0x03, 0x0005, 0x0246}, |
||||
cmdLutElement{0x00, 0x0a, -1, 0x03, 0x0005, 0x0446}, |
||||
cmdLutElement{0x00, 0x18, -1, 0x03, 0x0005, 0x0846}, |
||||
cmdLutElement{0x01, 0x05, -1, 0x03, 0x0006, 0x0046}, |
||||
cmdLutElement{0x01, 0x05, -1, 0x03, 0x0006, 0x0066}, |
||||
cmdLutElement{0x01, 0x06, -1, 0x03, 0x0006, 0x0086}, |
||||
cmdLutElement{0x01, 0x07, -1, 0x03, 0x0006, 0x00c6}, |
||||
cmdLutElement{0x01, 0x08, -1, 0x03, 0x0006, 0x0146}, |
||||
cmdLutElement{0x01, 0x09, -1, 0x03, 0x0006, 0x0246}, |
||||
cmdLutElement{0x01, 0x0a, -1, 0x03, 0x0006, 0x0446}, |
||||
cmdLutElement{0x01, 0x18, -1, 0x03, 0x0006, 0x0846}, |
||||
cmdLutElement{0x01, 0x05, -1, 0x03, 0x0008, 0x0046}, |
||||
cmdLutElement{0x01, 0x05, -1, 0x03, 0x0008, 0x0066}, |
||||
cmdLutElement{0x01, 0x06, -1, 0x03, 0x0008, 0x0086}, |
||||
cmdLutElement{0x01, 0x07, -1, 0x03, 0x0008, 0x00c6}, |
||||
cmdLutElement{0x01, 0x08, -1, 0x03, 0x0008, 0x0146}, |
||||
cmdLutElement{0x01, 0x09, -1, 0x03, 0x0008, 0x0246}, |
||||
cmdLutElement{0x01, 0x0a, -1, 0x03, 0x0008, 0x0446}, |
||||
cmdLutElement{0x01, 0x18, -1, 0x03, 0x0008, 0x0846}, |
||||
cmdLutElement{0x06, 0x00, -1, 0x00, 0x0082, 0x0002}, |
||||
cmdLutElement{0x06, 0x00, -1, 0x01, 0x0082, 0x0003}, |
||||
cmdLutElement{0x06, 0x00, -1, 0x02, 0x0082, 0x0004}, |
||||
cmdLutElement{0x06, 0x00, -1, 0x03, 0x0082, 0x0005}, |
||||
cmdLutElement{0x06, 0x00, -1, 0x03, 0x0082, 0x0006}, |
||||
cmdLutElement{0x06, 0x00, -1, 0x03, 0x0082, 0x0007}, |
||||
cmdLutElement{0x06, 0x00, -1, 0x03, 0x0082, 0x0008}, |
||||
cmdLutElement{0x06, 0x00, -1, 0x03, 0x0082, 0x0009}, |
||||
cmdLutElement{0x07, 0x00, -1, 0x00, 0x00c2, 0x0002}, |
||||
cmdLutElement{0x07, 0x00, -1, 0x01, 0x00c2, 0x0003}, |
||||
cmdLutElement{0x07, 0x00, -1, 0x02, 0x00c2, 0x0004}, |
||||
cmdLutElement{0x07, 0x00, -1, 0x03, 0x00c2, 0x0005}, |
||||
cmdLutElement{0x07, 0x00, -1, 0x03, 0x00c2, 0x0006}, |
||||
cmdLutElement{0x07, 0x00, -1, 0x03, 0x00c2, 0x0007}, |
||||
cmdLutElement{0x07, 0x00, -1, 0x03, 0x00c2, 0x0008}, |
||||
cmdLutElement{0x07, 0x00, -1, 0x03, 0x00c2, 0x0009}, |
||||
cmdLutElement{0x08, 0x00, -1, 0x00, 0x0142, 0x0002}, |
||||
cmdLutElement{0x08, 0x00, -1, 0x01, 0x0142, 0x0003}, |
||||
cmdLutElement{0x08, 0x00, -1, 0x02, 0x0142, 0x0004}, |
||||
cmdLutElement{0x08, 0x00, -1, 0x03, 0x0142, 0x0005}, |
||||
cmdLutElement{0x08, 0x00, -1, 0x03, 0x0142, 0x0006}, |
||||
cmdLutElement{0x08, 0x00, -1, 0x03, 0x0142, 0x0007}, |
||||
cmdLutElement{0x08, 0x00, -1, 0x03, 0x0142, 0x0008}, |
||||
cmdLutElement{0x08, 0x00, -1, 0x03, 0x0142, 0x0009}, |
||||
cmdLutElement{0x09, 0x00, -1, 0x00, 0x0242, 0x0002}, |
||||
cmdLutElement{0x09, 0x00, -1, 0x01, 0x0242, 0x0003}, |
||||
cmdLutElement{0x09, 0x00, -1, 0x02, 0x0242, 0x0004}, |
||||
cmdLutElement{0x09, 0x00, -1, 0x03, 0x0242, 0x0005}, |
||||
cmdLutElement{0x09, 0x00, -1, 0x03, 0x0242, 0x0006}, |
||||
cmdLutElement{0x09, 0x00, -1, 0x03, 0x0242, 0x0007}, |
||||
cmdLutElement{0x09, 0x00, -1, 0x03, 0x0242, 0x0008}, |
||||
cmdLutElement{0x09, 0x00, -1, 0x03, 0x0242, 0x0009}, |
||||
cmdLutElement{0x0a, 0x00, -1, 0x00, 0x0442, 0x0002}, |
||||
cmdLutElement{0x0a, 0x00, -1, 0x01, 0x0442, 0x0003}, |
||||
cmdLutElement{0x0a, 0x00, -1, 0x02, 0x0442, 0x0004}, |
||||
cmdLutElement{0x0a, 0x00, -1, 0x03, 0x0442, 0x0005}, |
||||
cmdLutElement{0x0a, 0x00, -1, 0x03, 0x0442, 0x0006}, |
||||
cmdLutElement{0x0a, 0x00, -1, 0x03, 0x0442, 0x0007}, |
||||
cmdLutElement{0x0a, 0x00, -1, 0x03, 0x0442, 0x0008}, |
||||
cmdLutElement{0x0a, 0x00, -1, 0x03, 0x0442, 0x0009}, |
||||
cmdLutElement{0x0c, 0x00, -1, 0x00, 0x0842, 0x0002}, |
||||
cmdLutElement{0x0c, 0x00, -1, 0x01, 0x0842, 0x0003}, |
||||
cmdLutElement{0x0c, 0x00, -1, 0x02, 0x0842, 0x0004}, |
||||
cmdLutElement{0x0c, 0x00, -1, 0x03, 0x0842, 0x0005}, |
||||
cmdLutElement{0x0c, 0x00, -1, 0x03, 0x0842, 0x0006}, |
||||
cmdLutElement{0x0c, 0x00, -1, 0x03, 0x0842, 0x0007}, |
||||
cmdLutElement{0x0c, 0x00, -1, 0x03, 0x0842, 0x0008}, |
||||
cmdLutElement{0x0c, 0x00, -1, 0x03, 0x0842, 0x0009}, |
||||
cmdLutElement{0x0e, 0x00, -1, 0x00, 0x1842, 0x0002}, |
||||
cmdLutElement{0x0e, 0x00, -1, 0x01, 0x1842, 0x0003}, |
||||
cmdLutElement{0x0e, 0x00, -1, 0x02, 0x1842, 0x0004}, |
||||
cmdLutElement{0x0e, 0x00, -1, 0x03, 0x1842, 0x0005}, |
||||
cmdLutElement{0x0e, 0x00, -1, 0x03, 0x1842, 0x0006}, |
||||
cmdLutElement{0x0e, 0x00, -1, 0x03, 0x1842, 0x0007}, |
||||
cmdLutElement{0x0e, 0x00, -1, 0x03, 0x1842, 0x0008}, |
||||
cmdLutElement{0x0e, 0x00, -1, 0x03, 0x1842, 0x0009}, |
||||
cmdLutElement{0x18, 0x00, -1, 0x00, 0x5842, 0x0002}, |
||||
cmdLutElement{0x18, 0x00, -1, 0x01, 0x5842, 0x0003}, |
||||
cmdLutElement{0x18, 0x00, -1, 0x02, 0x5842, 0x0004}, |
||||
cmdLutElement{0x18, 0x00, -1, 0x03, 0x5842, 0x0005}, |
||||
cmdLutElement{0x18, 0x00, -1, 0x03, 0x5842, 0x0006}, |
||||
cmdLutElement{0x18, 0x00, -1, 0x03, 0x5842, 0x0007}, |
||||
cmdLutElement{0x18, 0x00, -1, 0x03, 0x5842, 0x0008}, |
||||
cmdLutElement{0x18, 0x00, -1, 0x03, 0x5842, 0x0009}, |
||||
cmdLutElement{0x02, 0x05, -1, 0x03, 0x000a, 0x0046}, |
||||
cmdLutElement{0x02, 0x05, -1, 0x03, 0x000a, 0x0066}, |
||||
cmdLutElement{0x02, 0x06, -1, 0x03, 0x000a, 0x0086}, |
||||
cmdLutElement{0x02, 0x07, -1, 0x03, 0x000a, 0x00c6}, |
||||
cmdLutElement{0x02, 0x08, -1, 0x03, 0x000a, 0x0146}, |
||||
cmdLutElement{0x02, 0x09, -1, 0x03, 0x000a, 0x0246}, |
||||
cmdLutElement{0x02, 0x0a, -1, 0x03, 0x000a, 0x0446}, |
||||
cmdLutElement{0x02, 0x18, -1, 0x03, 0x000a, 0x0846}, |
||||
cmdLutElement{0x02, 0x05, -1, 0x03, 0x000e, 0x0046}, |
||||
cmdLutElement{0x02, 0x05, -1, 0x03, 0x000e, 0x0066}, |
||||
cmdLutElement{0x02, 0x06, -1, 0x03, 0x000e, 0x0086}, |
||||
cmdLutElement{0x02, 0x07, -1, 0x03, 0x000e, 0x00c6}, |
||||
cmdLutElement{0x02, 0x08, -1, 0x03, 0x000e, 0x0146}, |
||||
cmdLutElement{0x02, 0x09, -1, 0x03, 0x000e, 0x0246}, |
||||
cmdLutElement{0x02, 0x0a, -1, 0x03, 0x000e, 0x0446}, |
||||
cmdLutElement{0x02, 0x18, -1, 0x03, 0x000e, 0x0846}, |
||||
cmdLutElement{0x03, 0x05, -1, 0x03, 0x0012, 0x0046}, |
||||
cmdLutElement{0x03, 0x05, -1, 0x03, 0x0012, 0x0066}, |
||||
cmdLutElement{0x03, 0x06, -1, 0x03, 0x0012, 0x0086}, |
||||
cmdLutElement{0x03, 0x07, -1, 0x03, 0x0012, 0x00c6}, |
||||
cmdLutElement{0x03, 0x08, -1, 0x03, 0x0012, 0x0146}, |
||||
cmdLutElement{0x03, 0x09, -1, 0x03, 0x0012, 0x0246}, |
||||
cmdLutElement{0x03, 0x0a, -1, 0x03, 0x0012, 0x0446}, |
||||
cmdLutElement{0x03, 0x18, -1, 0x03, 0x0012, 0x0846}, |
||||
cmdLutElement{0x03, 0x05, -1, 0x03, 0x001a, 0x0046}, |
||||
cmdLutElement{0x03, 0x05, -1, 0x03, 0x001a, 0x0066}, |
||||
cmdLutElement{0x03, 0x06, -1, 0x03, 0x001a, 0x0086}, |
||||
cmdLutElement{0x03, 0x07, -1, 0x03, 0x001a, 0x00c6}, |
||||
cmdLutElement{0x03, 0x08, -1, 0x03, 0x001a, 0x0146}, |
||||
cmdLutElement{0x03, 0x09, -1, 0x03, 0x001a, 0x0246}, |
||||
cmdLutElement{0x03, 0x0a, -1, 0x03, 0x001a, 0x0446}, |
||||
cmdLutElement{0x03, 0x18, -1, 0x03, 0x001a, 0x0846}, |
||||
cmdLutElement{0x04, 0x05, -1, 0x03, 0x0022, 0x0046}, |
||||
cmdLutElement{0x04, 0x05, -1, 0x03, 0x0022, 0x0066}, |
||||
cmdLutElement{0x04, 0x06, -1, 0x03, 0x0022, 0x0086}, |
||||
cmdLutElement{0x04, 0x07, -1, 0x03, 0x0022, 0x00c6}, |
||||
cmdLutElement{0x04, 0x08, -1, 0x03, 0x0022, 0x0146}, |
||||
cmdLutElement{0x04, 0x09, -1, 0x03, 0x0022, 0x0246}, |
||||
cmdLutElement{0x04, 0x0a, -1, 0x03, 0x0022, 0x0446}, |
||||
cmdLutElement{0x04, 0x18, -1, 0x03, 0x0022, 0x0846}, |
||||
cmdLutElement{0x04, 0x05, -1, 0x03, 0x0032, 0x0046}, |
||||
cmdLutElement{0x04, 0x05, -1, 0x03, 0x0032, 0x0066}, |
||||
cmdLutElement{0x04, 0x06, -1, 0x03, 0x0032, 0x0086}, |
||||
cmdLutElement{0x04, 0x07, -1, 0x03, 0x0032, 0x00c6}, |
||||
cmdLutElement{0x04, 0x08, -1, 0x03, 0x0032, 0x0146}, |
||||
cmdLutElement{0x04, 0x09, -1, 0x03, 0x0032, 0x0246}, |
||||
cmdLutElement{0x04, 0x0a, -1, 0x03, 0x0032, 0x0446}, |
||||
cmdLutElement{0x04, 0x18, -1, 0x03, 0x0032, 0x0846}, |
||||
cmdLutElement{0x05, 0x05, -1, 0x03, 0x0042, 0x0046}, |
||||
cmdLutElement{0x05, 0x05, -1, 0x03, 0x0042, 0x0066}, |
||||
cmdLutElement{0x05, 0x06, -1, 0x03, 0x0042, 0x0086}, |
||||
cmdLutElement{0x05, 0x07, -1, 0x03, 0x0042, 0x00c6}, |
||||
cmdLutElement{0x05, 0x08, -1, 0x03, 0x0042, 0x0146}, |
||||
cmdLutElement{0x05, 0x09, -1, 0x03, 0x0042, 0x0246}, |
||||
cmdLutElement{0x05, 0x0a, -1, 0x03, 0x0042, 0x0446}, |
||||
cmdLutElement{0x05, 0x18, -1, 0x03, 0x0042, 0x0846}, |
||||
cmdLutElement{0x05, 0x05, -1, 0x03, 0x0062, 0x0046}, |
||||
cmdLutElement{0x05, 0x05, -1, 0x03, 0x0062, 0x0066}, |
||||
cmdLutElement{0x05, 0x06, -1, 0x03, 0x0062, 0x0086}, |
||||
cmdLutElement{0x05, 0x07, -1, 0x03, 0x0062, 0x00c6}, |
||||
cmdLutElement{0x05, 0x08, -1, 0x03, 0x0062, 0x0146}, |
||||
cmdLutElement{0x05, 0x09, -1, 0x03, 0x0062, 0x0246}, |
||||
cmdLutElement{0x05, 0x0a, -1, 0x03, 0x0062, 0x0446}, |
||||
cmdLutElement{0x05, 0x18, -1, 0x03, 0x0062, 0x0846}, |
||||
cmdLutElement{0x06, 0x01, -1, 0x03, 0x0082, 0x000a}, |
||||
cmdLutElement{0x06, 0x01, -1, 0x03, 0x0082, 0x000c}, |
||||
cmdLutElement{0x06, 0x02, -1, 0x03, 0x0082, 0x000e}, |
||||
cmdLutElement{0x06, 0x02, -1, 0x03, 0x0082, 0x0012}, |
||||
cmdLutElement{0x06, 0x03, -1, 0x03, 0x0082, 0x0016}, |
||||
cmdLutElement{0x06, 0x03, -1, 0x03, 0x0082, 0x001e}, |
||||
cmdLutElement{0x06, 0x04, -1, 0x03, 0x0082, 0x0026}, |
||||
cmdLutElement{0x06, 0x04, -1, 0x03, 0x0082, 0x0036}, |
||||
cmdLutElement{0x07, 0x01, -1, 0x03, 0x00c2, 0x000a}, |
||||
cmdLutElement{0x07, 0x01, -1, 0x03, 0x00c2, 0x000c}, |
||||
cmdLutElement{0x07, 0x02, -1, 0x03, 0x00c2, 0x000e}, |
||||
cmdLutElement{0x07, 0x02, -1, 0x03, 0x00c2, 0x0012}, |
||||
cmdLutElement{0x07, 0x03, -1, 0x03, 0x00c2, 0x0016}, |
||||
cmdLutElement{0x07, 0x03, -1, 0x03, 0x00c2, 0x001e}, |
||||
cmdLutElement{0x07, 0x04, -1, 0x03, 0x00c2, 0x0026}, |
||||
cmdLutElement{0x07, 0x04, -1, 0x03, 0x00c2, 0x0036}, |
||||
cmdLutElement{0x08, 0x01, -1, 0x03, 0x0142, 0x000a}, |
||||
cmdLutElement{0x08, 0x01, -1, 0x03, 0x0142, 0x000c}, |
||||
cmdLutElement{0x08, 0x02, -1, 0x03, 0x0142, 0x000e}, |
||||
cmdLutElement{0x08, 0x02, -1, 0x03, 0x0142, 0x0012}, |
||||
cmdLutElement{0x08, 0x03, -1, 0x03, 0x0142, 0x0016}, |
||||
cmdLutElement{0x08, 0x03, -1, 0x03, 0x0142, 0x001e}, |
||||
cmdLutElement{0x08, 0x04, -1, 0x03, 0x0142, 0x0026}, |
||||
cmdLutElement{0x08, 0x04, -1, 0x03, 0x0142, 0x0036}, |
||||
cmdLutElement{0x09, 0x01, -1, 0x03, 0x0242, 0x000a}, |
||||
cmdLutElement{0x09, 0x01, -1, 0x03, 0x0242, 0x000c}, |
||||
cmdLutElement{0x09, 0x02, -1, 0x03, 0x0242, 0x000e}, |
||||
cmdLutElement{0x09, 0x02, -1, 0x03, 0x0242, 0x0012}, |
||||
cmdLutElement{0x09, 0x03, -1, 0x03, 0x0242, 0x0016}, |
||||
cmdLutElement{0x09, 0x03, -1, 0x03, 0x0242, 0x001e}, |
||||
cmdLutElement{0x09, 0x04, -1, 0x03, 0x0242, 0x0026}, |
||||
cmdLutElement{0x09, 0x04, -1, 0x03, 0x0242, 0x0036}, |
||||
cmdLutElement{0x0a, 0x01, -1, 0x03, 0x0442, 0x000a}, |
||||
cmdLutElement{0x0a, 0x01, -1, 0x03, 0x0442, 0x000c}, |
||||
cmdLutElement{0x0a, 0x02, -1, 0x03, 0x0442, 0x000e}, |
||||
cmdLutElement{0x0a, 0x02, -1, 0x03, 0x0442, 0x0012}, |
||||
cmdLutElement{0x0a, 0x03, -1, 0x03, 0x0442, 0x0016}, |
||||
cmdLutElement{0x0a, 0x03, -1, 0x03, 0x0442, 0x001e}, |
||||
cmdLutElement{0x0a, 0x04, -1, 0x03, 0x0442, 0x0026}, |
||||
cmdLutElement{0x0a, 0x04, -1, 0x03, 0x0442, 0x0036}, |
||||
cmdLutElement{0x0c, 0x01, -1, 0x03, 0x0842, 0x000a}, |
||||
cmdLutElement{0x0c, 0x01, -1, 0x03, 0x0842, 0x000c}, |
||||
cmdLutElement{0x0c, 0x02, -1, 0x03, 0x0842, 0x000e}, |
||||
cmdLutElement{0x0c, 0x02, -1, 0x03, 0x0842, 0x0012}, |
||||
cmdLutElement{0x0c, 0x03, -1, 0x03, 0x0842, 0x0016}, |
||||
cmdLutElement{0x0c, 0x03, -1, 0x03, 0x0842, 0x001e}, |
||||
cmdLutElement{0x0c, 0x04, -1, 0x03, 0x0842, 0x0026}, |
||||
cmdLutElement{0x0c, 0x04, -1, 0x03, 0x0842, 0x0036}, |
||||
cmdLutElement{0x0e, 0x01, -1, 0x03, 0x1842, 0x000a}, |
||||
cmdLutElement{0x0e, 0x01, -1, 0x03, 0x1842, 0x000c}, |
||||
cmdLutElement{0x0e, 0x02, -1, 0x03, 0x1842, 0x000e}, |
||||
cmdLutElement{0x0e, 0x02, -1, 0x03, 0x1842, 0x0012}, |
||||
cmdLutElement{0x0e, 0x03, -1, 0x03, 0x1842, 0x0016}, |
||||
cmdLutElement{0x0e, 0x03, -1, 0x03, 0x1842, 0x001e}, |
||||
cmdLutElement{0x0e, 0x04, -1, 0x03, 0x1842, 0x0026}, |
||||
cmdLutElement{0x0e, 0x04, -1, 0x03, 0x1842, 0x0036}, |
||||
cmdLutElement{0x18, 0x01, -1, 0x03, 0x5842, 0x000a}, |
||||
cmdLutElement{0x18, 0x01, -1, 0x03, 0x5842, 0x000c}, |
||||
cmdLutElement{0x18, 0x02, -1, 0x03, 0x5842, 0x000e}, |
||||
cmdLutElement{0x18, 0x02, -1, 0x03, 0x5842, 0x0012}, |
||||
cmdLutElement{0x18, 0x03, -1, 0x03, 0x5842, 0x0016}, |
||||
cmdLutElement{0x18, 0x03, -1, 0x03, 0x5842, 0x001e}, |
||||
cmdLutElement{0x18, 0x04, -1, 0x03, 0x5842, 0x0026}, |
||||
cmdLutElement{0x18, 0x04, -1, 0x03, 0x5842, 0x0036}, |
||||
cmdLutElement{0x06, 0x05, -1, 0x03, 0x0082, 0x0046}, |
||||
cmdLutElement{0x06, 0x05, -1, 0x03, 0x0082, 0x0066}, |
||||
cmdLutElement{0x06, 0x06, -1, 0x03, 0x0082, 0x0086}, |
||||
cmdLutElement{0x06, 0x07, -1, 0x03, 0x0082, 0x00c6}, |
||||
cmdLutElement{0x06, 0x08, -1, 0x03, 0x0082, 0x0146}, |
||||
cmdLutElement{0x06, 0x09, -1, 0x03, 0x0082, 0x0246}, |
||||
cmdLutElement{0x06, 0x0a, -1, 0x03, 0x0082, 0x0446}, |
||||
cmdLutElement{0x06, 0x18, -1, 0x03, 0x0082, 0x0846}, |
||||
cmdLutElement{0x07, 0x05, -1, 0x03, 0x00c2, 0x0046}, |
||||
cmdLutElement{0x07, 0x05, -1, 0x03, 0x00c2, 0x0066}, |
||||
cmdLutElement{0x07, 0x06, -1, 0x03, 0x00c2, 0x0086}, |
||||
cmdLutElement{0x07, 0x07, -1, 0x03, 0x00c2, 0x00c6}, |
||||
cmdLutElement{0x07, 0x08, -1, 0x03, 0x00c2, 0x0146}, |
||||
cmdLutElement{0x07, 0x09, -1, 0x03, 0x00c2, 0x0246}, |
||||
cmdLutElement{0x07, 0x0a, -1, 0x03, 0x00c2, 0x0446}, |
||||
cmdLutElement{0x07, 0x18, -1, 0x03, 0x00c2, 0x0846}, |
||||
cmdLutElement{0x08, 0x05, -1, 0x03, 0x0142, 0x0046}, |
||||
cmdLutElement{0x08, 0x05, -1, 0x03, 0x0142, 0x0066}, |
||||
cmdLutElement{0x08, 0x06, -1, 0x03, 0x0142, 0x0086}, |
||||
cmdLutElement{0x08, 0x07, -1, 0x03, 0x0142, 0x00c6}, |
||||
cmdLutElement{0x08, 0x08, -1, 0x03, 0x0142, 0x0146}, |
||||
cmdLutElement{0x08, 0x09, -1, 0x03, 0x0142, 0x0246}, |
||||
cmdLutElement{0x08, 0x0a, -1, 0x03, 0x0142, 0x0446}, |
||||
cmdLutElement{0x08, 0x18, -1, 0x03, 0x0142, 0x0846}, |
||||
cmdLutElement{0x09, 0x05, -1, 0x03, 0x0242, 0x0046}, |
||||
cmdLutElement{0x09, 0x05, -1, 0x03, 0x0242, 0x0066}, |
||||
cmdLutElement{0x09, 0x06, -1, 0x03, 0x0242, 0x0086}, |
||||
cmdLutElement{0x09, 0x07, -1, 0x03, 0x0242, 0x00c6}, |
||||
cmdLutElement{0x09, 0x08, -1, 0x03, 0x0242, 0x0146}, |
||||
cmdLutElement{0x09, 0x09, -1, 0x03, 0x0242, 0x0246}, |
||||
cmdLutElement{0x09, 0x0a, -1, 0x03, 0x0242, 0x0446}, |
||||
cmdLutElement{0x09, 0x18, -1, 0x03, 0x0242, 0x0846}, |
||||
cmdLutElement{0x0a, 0x05, -1, 0x03, 0x0442, 0x0046}, |
||||
cmdLutElement{0x0a, 0x05, -1, 0x03, 0x0442, 0x0066}, |
||||
cmdLutElement{0x0a, 0x06, -1, 0x03, 0x0442, 0x0086}, |
||||
cmdLutElement{0x0a, 0x07, -1, 0x03, 0x0442, 0x00c6}, |
||||
cmdLutElement{0x0a, 0x08, -1, 0x03, 0x0442, 0x0146}, |
||||
cmdLutElement{0x0a, 0x09, -1, 0x03, 0x0442, 0x0246}, |
||||
cmdLutElement{0x0a, 0x0a, -1, 0x03, 0x0442, 0x0446}, |
||||
cmdLutElement{0x0a, 0x18, -1, 0x03, 0x0442, 0x0846}, |
||||
cmdLutElement{0x0c, 0x05, -1, 0x03, 0x0842, 0x0046}, |
||||
cmdLutElement{0x0c, 0x05, -1, 0x03, 0x0842, 0x0066}, |
||||
cmdLutElement{0x0c, 0x06, -1, 0x03, 0x0842, 0x0086}, |
||||
cmdLutElement{0x0c, 0x07, -1, 0x03, 0x0842, 0x00c6}, |
||||
cmdLutElement{0x0c, 0x08, -1, 0x03, 0x0842, 0x0146}, |
||||
cmdLutElement{0x0c, 0x09, -1, 0x03, 0x0842, 0x0246}, |
||||
cmdLutElement{0x0c, 0x0a, -1, 0x03, 0x0842, 0x0446}, |
||||
cmdLutElement{0x0c, 0x18, -1, 0x03, 0x0842, 0x0846}, |
||||
cmdLutElement{0x0e, 0x05, -1, 0x03, 0x1842, 0x0046}, |
||||
cmdLutElement{0x0e, 0x05, -1, 0x03, 0x1842, 0x0066}, |
||||
cmdLutElement{0x0e, 0x06, -1, 0x03, 0x1842, 0x0086}, |
||||
cmdLutElement{0x0e, 0x07, -1, 0x03, 0x1842, 0x00c6}, |
||||
cmdLutElement{0x0e, 0x08, -1, 0x03, 0x1842, 0x0146}, |
||||
cmdLutElement{0x0e, 0x09, -1, 0x03, 0x1842, 0x0246}, |
||||
cmdLutElement{0x0e, 0x0a, -1, 0x03, 0x1842, 0x0446}, |
||||
cmdLutElement{0x0e, 0x18, -1, 0x03, 0x1842, 0x0846}, |
||||
cmdLutElement{0x18, 0x05, -1, 0x03, 0x5842, 0x0046}, |
||||
cmdLutElement{0x18, 0x05, -1, 0x03, 0x5842, 0x0066}, |
||||
cmdLutElement{0x18, 0x06, -1, 0x03, 0x5842, 0x0086}, |
||||
cmdLutElement{0x18, 0x07, -1, 0x03, 0x5842, 0x00c6}, |
||||
cmdLutElement{0x18, 0x08, -1, 0x03, 0x5842, 0x0146}, |
||||
cmdLutElement{0x18, 0x09, -1, 0x03, 0x5842, 0x0246}, |
||||
cmdLutElement{0x18, 0x0a, -1, 0x03, 0x5842, 0x0446}, |
||||
cmdLutElement{0x18, 0x18, -1, 0x03, 0x5842, 0x0846}, |
||||
} |
||||
@ -0,0 +1,196 @@ |
||||
package brotli |
||||
|
||||
const fastOnePassCompressionQuality = 0 |
||||
|
||||
const fastTwoPassCompressionQuality = 1 |
||||
|
||||
const zopflificationQuality = 10 |
||||
|
||||
const hqZopflificationQuality = 11 |
||||
|
||||
const maxQualityForStaticEntropyCodes = 2 |
||||
|
||||
const minQualityForBlockSplit = 4 |
||||
|
||||
const minQualityForNonzeroDistanceParams = 4 |
||||
|
||||
const minQualityForOptimizeHistograms = 4 |
||||
|
||||
const minQualityForExtensiveReferenceSearch = 5 |
||||
|
||||
const minQualityForContextModeling = 5 |
||||
|
||||
const minQualityForHqContextModeling = 7 |
||||
|
||||
const minQualityForHqBlockSplitting = 10 |
||||
|
||||
/* For quality below MIN_QUALITY_FOR_BLOCK_SPLIT there is no block splitting, |
||||
so we buffer at most this much literals and commands. */ |
||||
const maxNumDelayedSymbols = 0x2FFF |
||||
|
||||
/* Returns hash-table size for quality levels 0 and 1. */ |
||||
func maxHashTableSize(quality int) uint { |
||||
if quality == fastOnePassCompressionQuality { |
||||
return 1 << 15 |
||||
} else { |
||||
return 1 << 17 |
||||
} |
||||
} |
||||
|
||||
/* The maximum length for which the zopflification uses distinct distances. */ |
||||
const maxZopfliLenQuality10 = 150 |
||||
|
||||
const maxZopfliLenQuality11 = 325 |
||||
|
||||
/* Do not thoroughly search when a long copy is found. */ |
||||
const longCopyQuickStep = 16384 |
||||
|
||||
func maxZopfliLen(params *encoderParams) uint { |
||||
if params.quality <= 10 { |
||||
return maxZopfliLenQuality10 |
||||
} else { |
||||
return maxZopfliLenQuality11 |
||||
} |
||||
} |
||||
|
||||
/* Number of best candidates to evaluate to expand Zopfli chain. */ |
||||
func maxZopfliCandidates(params *encoderParams) uint { |
||||
if params.quality <= 10 { |
||||
return 1 |
||||
} else { |
||||
return 5 |
||||
} |
||||
} |
||||
|
||||
func sanitizeParams(params *encoderParams) { |
||||
params.quality = brotli_min_int(maxQuality, brotli_max_int(minQuality, params.quality)) |
||||
if params.quality <= maxQualityForStaticEntropyCodes { |
||||
params.large_window = false |
||||
} |
||||
|
||||
if params.lgwin < minWindowBits { |
||||
params.lgwin = minWindowBits |
||||
} else { |
||||
var max_lgwin int |
||||
if params.large_window { |
||||
max_lgwin = largeMaxWindowBits |
||||
} else { |
||||
max_lgwin = maxWindowBits |
||||
} |
||||
if params.lgwin > uint(max_lgwin) { |
||||
params.lgwin = uint(max_lgwin) |
||||
} |
||||
} |
||||
} |
||||
|
||||
/* Returns optimized lg_block value. */ |
||||
func computeLgBlock(params *encoderParams) int { |
||||
var lgblock int = params.lgblock |
||||
if params.quality == fastOnePassCompressionQuality || params.quality == fastTwoPassCompressionQuality { |
||||
lgblock = int(params.lgwin) |
||||
} else if params.quality < minQualityForBlockSplit { |
||||
lgblock = 14 |
||||
} else if lgblock == 0 { |
||||
lgblock = 16 |
||||
if params.quality >= 9 && params.lgwin > uint(lgblock) { |
||||
lgblock = brotli_min_int(18, int(params.lgwin)) |
||||
} |
||||
} else { |
||||
lgblock = brotli_min_int(maxInputBlockBits, brotli_max_int(minInputBlockBits, lgblock)) |
||||
} |
||||
|
||||
return lgblock |
||||
} |
||||
|
||||
/* Returns log2 of the size of main ring buffer area. |
||||
Allocate at least lgwin + 1 bits for the ring buffer so that the newly |
||||
added block fits there completely and we still get lgwin bits and at least |
||||
read_block_size_bits + 1 bits because the copy tail length needs to be |
||||
smaller than ring-buffer size. */ |
||||
func computeRbBits(params *encoderParams) int { |
||||
return 1 + brotli_max_int(int(params.lgwin), params.lgblock) |
||||
} |
||||
|
||||
func maxMetablockSize(params *encoderParams) uint { |
||||
var bits int = brotli_min_int(computeRbBits(params), maxInputBlockBits) |
||||
return uint(1) << uint(bits) |
||||
} |
||||
|
||||
/* When searching for backward references and have not seen matches for a long |
||||
time, we can skip some match lookups. Unsuccessful match lookups are very |
||||
expensive and this kind of a heuristic speeds up compression quite a lot. |
||||
At first 8 byte strides are taken and every second byte is put to hasher. |
||||
After 4x more literals stride by 16 bytes, every put 4-th byte to hasher. |
||||
Applied only to qualities 2 to 9. */ |
||||
func literalSpreeLengthForSparseSearch(params *encoderParams) uint { |
||||
if params.quality < 9 { |
||||
return 64 |
||||
} else { |
||||
return 512 |
||||
} |
||||
} |
||||
|
||||
func chooseHasher(params *encoderParams, hparams *hasherParams) { |
||||
if params.quality > 9 { |
||||
hparams.type_ = 10 |
||||
} else if params.quality == 4 && params.size_hint >= 1<<20 { |
||||
hparams.type_ = 54 |
||||
} else if params.quality < 5 { |
||||
hparams.type_ = params.quality |
||||
} else if params.lgwin <= 16 { |
||||
if params.quality < 7 { |
||||
hparams.type_ = 40 |
||||
} else if params.quality < 9 { |
||||
hparams.type_ = 41 |
||||
} else { |
||||
hparams.type_ = 42 |
||||
} |
||||
} else if params.size_hint >= 1<<20 && params.lgwin >= 19 { |
||||
hparams.type_ = 6 |
||||
hparams.block_bits = params.quality - 1 |
||||
hparams.bucket_bits = 15 |
||||
hparams.hash_len = 5 |
||||
if params.quality < 7 { |
||||
hparams.num_last_distances_to_check = 4 |
||||
} else if params.quality < 9 { |
||||
hparams.num_last_distances_to_check = 10 |
||||
} else { |
||||
hparams.num_last_distances_to_check = 16 |
||||
} |
||||
} else { |
||||
hparams.type_ = 5 |
||||
hparams.block_bits = params.quality - 1 |
||||
if params.quality < 7 { |
||||
hparams.bucket_bits = 14 |
||||
} else { |
||||
hparams.bucket_bits = 15 |
||||
} |
||||
if params.quality < 7 { |
||||
hparams.num_last_distances_to_check = 4 |
||||
} else if params.quality < 9 { |
||||
hparams.num_last_distances_to_check = 10 |
||||
} else { |
||||
hparams.num_last_distances_to_check = 16 |
||||
} |
||||
} |
||||
|
||||
if params.lgwin > 24 { |
||||
/* Different hashers for large window brotli: not for qualities <= 2, |
||||
these are too fast for large window. Not for qualities >= 10: their |
||||
hasher already works well with large window. So the changes are: |
||||
H3 --> H35: for quality 3. |
||||
H54 --> H55: for quality 4 with size hint > 1MB |
||||
H6 --> H65: for qualities 5, 6, 7, 8, 9. */ |
||||
if hparams.type_ == 3 { |
||||
hparams.type_ = 35 |
||||
} |
||||
|
||||
if hparams.type_ == 54 { |
||||
hparams.type_ = 55 |
||||
} |
||||
|
||||
if hparams.type_ == 6 { |
||||
hparams.type_ = 65 |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,100 @@ |
||||
package brotli |
||||
|
||||
import ( |
||||
"errors" |
||||
"io" |
||||
) |
||||
|
||||
type decodeError int |
||||
|
||||
func (err decodeError) Error() string { |
||||
return "brotli: " + string(decoderErrorString(int(err))) |
||||
} |
||||
|
||||
var errExcessiveInput = errors.New("brotli: excessive input") |
||||
var errInvalidState = errors.New("brotli: invalid state") |
||||
|
||||
// readBufSize is a "good" buffer size that avoids excessive round-trips
|
||||
// between C and Go but doesn't waste too much memory on buffering.
|
||||
// It is arbitrarily chosen to be equal to the constant used in io.Copy.
|
||||
const readBufSize = 32 * 1024 |
||||
|
||||
// NewReader creates a new Reader reading the given reader.
|
||||
func NewReader(src io.Reader) *Reader { |
||||
r := new(Reader) |
||||
r.Reset(src) |
||||
return r |
||||
} |
||||
|
||||
// Reset discards the Reader's state and makes it equivalent to the result of
|
||||
// its original state from NewReader, but writing to src instead.
|
||||
// This permits reusing a Reader rather than allocating a new one.
|
||||
// Error is always nil
|
||||
func (r *Reader) Reset(src io.Reader) error { |
||||
decoderStateInit(r) |
||||
r.src = src |
||||
r.buf = make([]byte, readBufSize) |
||||
return nil |
||||
} |
||||
|
||||
func (r *Reader) Read(p []byte) (n int, err error) { |
||||
if !decoderHasMoreOutput(r) && len(r.in) == 0 { |
||||
m, readErr := r.src.Read(r.buf) |
||||
if m == 0 { |
||||
// If readErr is `nil`, we just proxy underlying stream behavior.
|
||||
return 0, readErr |
||||
} |
||||
r.in = r.buf[:m] |
||||
} |
||||
|
||||
if len(p) == 0 { |
||||
return 0, nil |
||||
} |
||||
|
||||
for { |
||||
var written uint |
||||
in_len := uint(len(r.in)) |
||||
out_len := uint(len(p)) |
||||
in_remaining := in_len |
||||
out_remaining := out_len |
||||
result := decoderDecompressStream(r, &in_remaining, &r.in, &out_remaining, &p) |
||||
written = out_len - out_remaining |
||||
n = int(written) |
||||
|
||||
switch result { |
||||
case decoderResultSuccess: |
||||
if len(r.in) > 0 { |
||||
return n, errExcessiveInput |
||||
} |
||||
return n, nil |
||||
case decoderResultError: |
||||
return n, decodeError(decoderGetErrorCode(r)) |
||||
case decoderResultNeedsMoreOutput: |
||||
if n == 0 { |
||||
return 0, io.ErrShortBuffer |
||||
} |
||||
return n, nil |
||||
case decoderNeedsMoreInput: |
||||
} |
||||
|
||||
if len(r.in) != 0 { |
||||
return 0, errInvalidState |
||||
} |
||||
|
||||
// Calling r.src.Read may block. Don't block if we have data to return.
|
||||
if n > 0 { |
||||
return n, nil |
||||
} |
||||
|
||||
// Top off the buffer.
|
||||
encN, err := r.src.Read(r.buf) |
||||
if encN == 0 { |
||||
// Not enough data to complete decoding.
|
||||
if err == io.EOF { |
||||
return 0, io.ErrUnexpectedEOF |
||||
} |
||||
return 0, err |
||||
} |
||||
r.in = r.buf[:encN] |
||||
} |
||||
} |
||||
@ -0,0 +1,132 @@ |
||||
package brotli |
||||
|
||||
/* Copyright 2013 Google Inc. All Rights Reserved. |
||||
|
||||
Distributed under MIT license. |
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/ |
||||
|
||||
/* A ringBuffer(window_bits, tail_bits) contains `1 << window_bits' bytes of |
||||
data in a circular manner: writing a byte writes it to: |
||||
`position() % (1 << window_bits)'. |
||||
For convenience, the ringBuffer array contains another copy of the |
||||
first `1 << tail_bits' bytes: |
||||
buffer_[i] == buffer_[i + (1 << window_bits)], if i < (1 << tail_bits), |
||||
and another copy of the last two bytes: |
||||
buffer_[-1] == buffer_[(1 << window_bits) - 1] and |
||||
buffer_[-2] == buffer_[(1 << window_bits) - 2]. */ |
||||
type ringBuffer struct { |
||||
size_ uint32 |
||||
mask_ uint32 |
||||
tail_size_ uint32 |
||||
total_size_ uint32 |
||||
cur_size_ uint32 |
||||
pos_ uint32 |
||||
data_ []byte |
||||
buffer_ []byte |
||||
} |
||||
|
||||
func ringBufferInit(rb *ringBuffer) { |
||||
rb.cur_size_ = 0 |
||||
rb.pos_ = 0 |
||||
rb.data_ = nil |
||||
rb.buffer_ = nil |
||||
} |
||||
|
||||
func ringBufferSetup(params *encoderParams, rb *ringBuffer) { |
||||
var window_bits int = computeRbBits(params) |
||||
var tail_bits int = params.lgblock |
||||
*(*uint32)(&rb.size_) = 1 << uint(window_bits) |
||||
*(*uint32)(&rb.mask_) = (1 << uint(window_bits)) - 1 |
||||
*(*uint32)(&rb.tail_size_) = 1 << uint(tail_bits) |
||||
*(*uint32)(&rb.total_size_) = rb.size_ + rb.tail_size_ |
||||
} |
||||
|
||||
const kSlackForEightByteHashingEverywhere uint = 7 |
||||
|
||||
/* Allocates or re-allocates data_ to the given length + plus some slack |
||||
region before and after. Fills the slack regions with zeros. */ |
||||
func ringBufferInitBuffer(buflen uint32, rb *ringBuffer) { |
||||
var new_data []byte = make([]byte, (2 + uint(buflen) + kSlackForEightByteHashingEverywhere)) |
||||
var i uint |
||||
if rb.data_ != nil { |
||||
copy(new_data, rb.data_[:2+rb.cur_size_+uint32(kSlackForEightByteHashingEverywhere)]) |
||||
rb.data_ = nil |
||||
} |
||||
|
||||
rb.data_ = new_data |
||||
rb.cur_size_ = buflen |
||||
rb.buffer_ = rb.data_[2:] |
||||
rb.data_[1] = 0 |
||||
rb.data_[0] = rb.data_[1] |
||||
for i = 0; i < kSlackForEightByteHashingEverywhere; i++ { |
||||
rb.buffer_[rb.cur_size_+uint32(i)] = 0 |
||||
} |
||||
} |
||||
|
||||
func ringBufferWriteTail(bytes []byte, n uint, rb *ringBuffer) { |
||||
var masked_pos uint = uint(rb.pos_ & rb.mask_) |
||||
if uint32(masked_pos) < rb.tail_size_ { |
||||
/* Just fill the tail buffer with the beginning data. */ |
||||
var p uint = uint(rb.size_ + uint32(masked_pos)) |
||||
copy(rb.buffer_[p:], bytes[:brotli_min_size_t(n, uint(rb.tail_size_-uint32(masked_pos)))]) |
||||
} |
||||
} |
||||
|
||||
/* Push bytes into the ring buffer. */ |
||||
func ringBufferWrite(bytes []byte, n uint, rb *ringBuffer) { |
||||
if rb.pos_ == 0 && uint32(n) < rb.tail_size_ { |
||||
/* Special case for the first write: to process the first block, we don't |
||||
need to allocate the whole ring-buffer and we don't need the tail |
||||
either. However, we do this memory usage optimization only if the |
||||
first write is less than the tail size, which is also the input block |
||||
size, otherwise it is likely that other blocks will follow and we |
||||
will need to reallocate to the full size anyway. */ |
||||
rb.pos_ = uint32(n) |
||||
|
||||
ringBufferInitBuffer(rb.pos_, rb) |
||||
copy(rb.buffer_, bytes[:n]) |
||||
return |
||||
} |
||||
|
||||
if rb.cur_size_ < rb.total_size_ { |
||||
/* Lazily allocate the full buffer. */ |
||||
ringBufferInitBuffer(rb.total_size_, rb) |
||||
|
||||
/* Initialize the last two bytes to zero, so that we don't have to worry |
||||
later when we copy the last two bytes to the first two positions. */ |
||||
rb.buffer_[rb.size_-2] = 0 |
||||
|
||||
rb.buffer_[rb.size_-1] = 0 |
||||
} |
||||
{ |
||||
var masked_pos uint = uint(rb.pos_ & rb.mask_) |
||||
|
||||
/* The length of the writes is limited so that we do not need to worry |
||||
about a write */ |
||||
ringBufferWriteTail(bytes, n, rb) |
||||
|
||||
if uint32(masked_pos+n) <= rb.size_ { |
||||
/* A single write fits. */ |
||||
copy(rb.buffer_[masked_pos:], bytes[:n]) |
||||
} else { |
||||
/* Split into two writes. |
||||
Copy into the end of the buffer, including the tail buffer. */ |
||||
copy(rb.buffer_[masked_pos:], bytes[:brotli_min_size_t(n, uint(rb.total_size_-uint32(masked_pos)))]) |
||||
|
||||
/* Copy into the beginning of the buffer */ |
||||
copy(rb.buffer_, bytes[rb.size_-uint32(masked_pos):][:uint32(n)-(rb.size_-uint32(masked_pos))]) |
||||
} |
||||
} |
||||
{ |
||||
var not_first_lap bool = rb.pos_&(1<<31) != 0 |
||||
var rb_pos_mask uint32 = (1 << 31) - 1 |
||||
rb.data_[0] = rb.buffer_[rb.size_-2] |
||||
rb.data_[1] = rb.buffer_[rb.size_-1] |
||||
rb.pos_ = (rb.pos_ & rb_pos_mask) + uint32(uint32(n)&rb_pos_mask) |
||||
if not_first_lap { |
||||
/* Wrap, but preserve not-a-first-lap feature. */ |
||||
rb.pos_ |= 1 << 31 |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,295 @@ |
||||
package brotli |
||||
|
||||
import "io" |
||||
|
||||
/* Copyright 2015 Google Inc. All Rights Reserved. |
||||
|
||||
Distributed under MIT license. |
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/ |
||||
|
||||
/* Brotli state for partial streaming decoding. */ |
||||
const ( |
||||
stateUninited = iota |
||||
stateLargeWindowBits |
||||
stateInitialize |
||||
stateMetablockBegin |
||||
stateMetablockHeader |
||||
stateMetablockHeader2 |
||||
stateContextModes |
||||
stateCommandBegin |
||||
stateCommandInner |
||||
stateCommandPostDecodeLiterals |
||||
stateCommandPostWrapCopy |
||||
stateUncompressed |
||||
stateMetadata |
||||
stateCommandInnerWrite |
||||
stateMetablockDone |
||||
stateCommandPostWrite1 |
||||
stateCommandPostWrite2 |
||||
stateHuffmanCode0 |
||||
stateHuffmanCode1 |
||||
stateHuffmanCode2 |
||||
stateHuffmanCode3 |
||||
stateContextMap1 |
||||
stateContextMap2 |
||||
stateTreeGroup |
||||
stateDone |
||||
) |
||||
|
||||
const ( |
||||
stateMetablockHeaderNone = iota |
||||
stateMetablockHeaderEmpty |
||||
stateMetablockHeaderNibbles |
||||
stateMetablockHeaderSize |
||||
stateMetablockHeaderUncompressed |
||||
stateMetablockHeaderReserved |
||||
stateMetablockHeaderBytes |
||||
stateMetablockHeaderMetadata |
||||
) |
||||
|
||||
const ( |
||||
stateUncompressedNone = iota |
||||
stateUncompressedWrite |
||||
) |
||||
|
||||
const ( |
||||
stateTreeGroupNone = iota |
||||
stateTreeGroupLoop |
||||
) |
||||
|
||||
const ( |
||||
stateContextMapNone = iota |
||||
stateContextMapReadPrefix |
||||
stateContextMapHuffman |
||||
stateContextMapDecode |
||||
stateContextMapTransform |
||||
) |
||||
|
||||
const ( |
||||
stateHuffmanNone = iota |
||||
stateHuffmanSimpleSize |
||||
stateHuffmanSimpleRead |
||||
stateHuffmanSimpleBuild |
||||
stateHuffmanComplex |
||||
stateHuffmanLengthSymbols |
||||
) |
||||
|
||||
const ( |
||||
stateDecodeUint8None = iota |
||||
stateDecodeUint8Short |
||||
stateDecodeUint8Long |
||||
) |
||||
|
||||
const ( |
||||
stateReadBlockLengthNone = iota |
||||
stateReadBlockLengthSuffix |
||||
) |
||||
|
||||
type Reader struct { |
||||
src io.Reader |
||||
buf []byte // scratch space for reading from src
|
||||
in []byte // current chunk to decode; usually aliases buf
|
||||
|
||||
state int |
||||
loop_counter int |
||||
br bitReader |
||||
buffer struct { |
||||
u64 uint64 |
||||
u8 [8]byte |
||||
} |
||||
buffer_length uint32 |
||||
pos int |
||||
max_backward_distance int |
||||
max_distance int |
||||
ringbuffer_size int |
||||
ringbuffer_mask int |
||||
dist_rb_idx int |
||||
dist_rb [4]int |
||||
error_code int |
||||
sub_loop_counter uint32 |
||||
ringbuffer []byte |
||||
ringbuffer_end []byte |
||||
htree_command []huffmanCode |
||||
context_lookup []byte |
||||
context_map_slice []byte |
||||
dist_context_map_slice []byte |
||||
literal_hgroup huffmanTreeGroup |
||||
insert_copy_hgroup huffmanTreeGroup |
||||
distance_hgroup huffmanTreeGroup |
||||
block_type_trees []huffmanCode |
||||
block_len_trees []huffmanCode |
||||
trivial_literal_context int |
||||
distance_context int |
||||
meta_block_remaining_len int |
||||
block_length_index uint32 |
||||
block_length [3]uint32 |
||||
num_block_types [3]uint32 |
||||
block_type_rb [6]uint32 |
||||
distance_postfix_bits uint32 |
||||
num_direct_distance_codes uint32 |
||||
distance_postfix_mask int |
||||
num_dist_htrees uint32 |
||||
dist_context_map []byte |
||||
literal_htree []huffmanCode |
||||
dist_htree_index byte |
||||
repeat_code_len uint32 |
||||
prev_code_len uint32 |
||||
copy_length int |
||||
distance_code int |
||||
rb_roundtrips uint |
||||
partial_pos_out uint |
||||
symbol uint32 |
||||
repeat uint32 |
||||
space uint32 |
||||
table [32]huffmanCode |
||||
symbol_lists symbolList |
||||
symbols_lists_array [huffmanMaxCodeLength + 1 + numCommandSymbols]uint16 |
||||
next_symbol [32]int |
||||
code_length_code_lengths [codeLengthCodes]byte |
||||
code_length_histo [16]uint16 |
||||
htree_index int |
||||
next []huffmanCode |
||||
context_index uint32 |
||||
max_run_length_prefix uint32 |
||||
code uint32 |
||||
context_map_table [huffmanMaxSize272]huffmanCode |
||||
substate_metablock_header int |
||||
substate_tree_group int |
||||
substate_context_map int |
||||
substate_uncompressed int |
||||
substate_huffman int |
||||
substate_decode_uint8 int |
||||
substate_read_block_length int |
||||
is_last_metablock uint |
||||
is_uncompressed uint |
||||
is_metadata uint |
||||
should_wrap_ringbuffer uint |
||||
canny_ringbuffer_allocation uint |
||||
large_window bool |
||||
size_nibbles uint |
||||
window_bits uint32 |
||||
new_ringbuffer_size int |
||||
num_literal_htrees uint32 |
||||
context_map []byte |
||||
context_modes []byte |
||||
dictionary *dictionary |
||||
transforms *transforms |
||||
trivial_literal_contexts [8]uint32 |
||||
} |
||||
|
||||
func decoderStateInit(s *Reader) bool { |
||||
s.error_code = 0 /* BROTLI_DECODER_NO_ERROR */ |
||||
|
||||
initBitReader(&s.br) |
||||
s.state = stateUninited |
||||
s.large_window = false |
||||
s.substate_metablock_header = stateMetablockHeaderNone |
||||
s.substate_tree_group = stateTreeGroupNone |
||||
s.substate_context_map = stateContextMapNone |
||||
s.substate_uncompressed = stateUncompressedNone |
||||
s.substate_huffman = stateHuffmanNone |
||||
s.substate_decode_uint8 = stateDecodeUint8None |
||||
s.substate_read_block_length = stateReadBlockLengthNone |
||||
|
||||
s.buffer_length = 0 |
||||
s.loop_counter = 0 |
||||
s.pos = 0 |
||||
s.rb_roundtrips = 0 |
||||
s.partial_pos_out = 0 |
||||
|
||||
s.block_type_trees = nil |
||||
s.block_len_trees = nil |
||||
s.ringbuffer = nil |
||||
s.ringbuffer_size = 0 |
||||
s.new_ringbuffer_size = 0 |
||||
s.ringbuffer_mask = 0 |
||||
|
||||
s.context_map = nil |
||||
s.context_modes = nil |
||||
s.dist_context_map = nil |
||||
s.context_map_slice = nil |
||||
s.dist_context_map_slice = nil |
||||
|
||||
s.sub_loop_counter = 0 |
||||
|
||||
s.literal_hgroup.codes = nil |
||||
s.literal_hgroup.htrees = nil |
||||
s.insert_copy_hgroup.codes = nil |
||||
s.insert_copy_hgroup.htrees = nil |
||||
s.distance_hgroup.codes = nil |
||||
s.distance_hgroup.htrees = nil |
||||
|
||||
s.is_last_metablock = 0 |
||||
s.is_uncompressed = 0 |
||||
s.is_metadata = 0 |
||||
s.should_wrap_ringbuffer = 0 |
||||
s.canny_ringbuffer_allocation = 1 |
||||
|
||||
s.window_bits = 0 |
||||
s.max_distance = 0 |
||||
s.dist_rb[0] = 16 |
||||
s.dist_rb[1] = 15 |
||||
s.dist_rb[2] = 11 |
||||
s.dist_rb[3] = 4 |
||||
s.dist_rb_idx = 0 |
||||
s.block_type_trees = nil |
||||
s.block_len_trees = nil |
||||
|
||||
s.symbol_lists.storage = s.symbols_lists_array[:] |
||||
s.symbol_lists.offset = huffmanMaxCodeLength + 1 |
||||
|
||||
s.dictionary = getDictionary() |
||||
s.transforms = getTransforms() |
||||
|
||||
return true |
||||
} |
||||
|
||||
func decoderStateMetablockBegin(s *Reader) { |
||||
s.meta_block_remaining_len = 0 |
||||
s.block_length[0] = 1 << 24 |
||||
s.block_length[1] = 1 << 24 |
||||
s.block_length[2] = 1 << 24 |
||||
s.num_block_types[0] = 1 |
||||
s.num_block_types[1] = 1 |
||||
s.num_block_types[2] = 1 |
||||
s.block_type_rb[0] = 1 |
||||
s.block_type_rb[1] = 0 |
||||
s.block_type_rb[2] = 1 |
||||
s.block_type_rb[3] = 0 |
||||
s.block_type_rb[4] = 1 |
||||
s.block_type_rb[5] = 0 |
||||
s.context_map = nil |
||||
s.context_modes = nil |
||||
s.dist_context_map = nil |
||||
s.context_map_slice = nil |
||||
s.literal_htree = nil |
||||
s.dist_context_map_slice = nil |
||||
s.dist_htree_index = 0 |
||||
s.context_lookup = nil |
||||
s.literal_hgroup.codes = nil |
||||
s.literal_hgroup.htrees = nil |
||||
s.insert_copy_hgroup.codes = nil |
||||
s.insert_copy_hgroup.htrees = nil |
||||
s.distance_hgroup.codes = nil |
||||
s.distance_hgroup.htrees = nil |
||||
} |
||||
|
||||
func decoderStateCleanupAfterMetablock(s *Reader) { |
||||
s.context_modes = nil |
||||
s.context_map = nil |
||||
s.dist_context_map = nil |
||||
s.literal_hgroup.htrees = nil |
||||
s.insert_copy_hgroup.htrees = nil |
||||
s.distance_hgroup.htrees = nil |
||||
} |
||||
|
||||
func decoderHuffmanTreeGroupInit(s *Reader, group *huffmanTreeGroup, alphabet_size uint32, max_symbol uint32, ntrees uint32) bool { |
||||
var max_table_size uint = uint(kMaxHuffmanTableSize[(alphabet_size+31)>>5]) |
||||
group.alphabet_size = uint16(alphabet_size) |
||||
group.max_symbol = uint16(max_symbol) |
||||
group.num_htrees = uint16(ntrees) |
||||
group.htrees = make([][]huffmanCode, ntrees) |
||||
group.codes = make([]huffmanCode, (uint(ntrees) * max_table_size)) |
||||
return !(group.codes == nil) |
||||
} |
||||
@ -0,0 +1,666 @@ |
||||
package brotli |
||||
|
||||
import "encoding/binary" |
||||
|
||||
/* Copyright 2013 Google Inc. All Rights Reserved. |
||||
|
||||
Distributed under MIT license. |
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/ |
||||
|
||||
/* Class to model the static dictionary. */ |
||||
|
||||
const maxStaticDictionaryMatchLen = 37 |
||||
|
||||
const kInvalidMatch uint32 = 0xFFFFFFF |
||||
|
||||
/* Copyright 2013 Google Inc. All Rights Reserved. |
||||
|
||||
Distributed under MIT license. |
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/ |
||||
func hash(data []byte) uint32 { |
||||
var h uint32 = binary.LittleEndian.Uint32(data) * kDictHashMul32 |
||||
|
||||
/* The higher bits contain more mixture from the multiplication, |
||||
so we take our results from there. */ |
||||
return h >> uint(32-kDictNumBits) |
||||
} |
||||
|
||||
func addMatch(distance uint, len uint, len_code uint, matches []uint32) { |
||||
var match uint32 = uint32((distance << 5) + len_code) |
||||
matches[len] = brotli_min_uint32_t(matches[len], match) |
||||
} |
||||
|
||||
func dictMatchLength(dict *dictionary, data []byte, id uint, len uint, maxlen uint) uint { |
||||
var offset uint = uint(dict.offsets_by_length[len]) + len*id |
||||
return findMatchLengthWithLimit(dict.data[offset:], data, brotli_min_size_t(uint(len), maxlen)) |
||||
} |
||||
|
||||
func isMatch(d *dictionary, w dictWord, data []byte, max_length uint) bool { |
||||
if uint(w.len) > max_length { |
||||
return false |
||||
} else { |
||||
var offset uint = uint(d.offsets_by_length[w.len]) + uint(w.len)*uint(w.idx) |
||||
var dict []byte = d.data[offset:] |
||||
if w.transform == 0 { |
||||
/* Match against base dictionary word. */ |
||||
return findMatchLengthWithLimit(dict, data, uint(w.len)) == uint(w.len) |
||||
} else if w.transform == 10 { |
||||
/* Match against uppercase first transform. |
||||
Note that there are only ASCII uppercase words in the lookup table. */ |
||||
return dict[0] >= 'a' && dict[0] <= 'z' && (dict[0]^32) == data[0] && findMatchLengthWithLimit(dict[1:], data[1:], uint(w.len)-1) == uint(w.len-1) |
||||
} else { |
||||
/* Match against uppercase all transform. |
||||
Note that there are only ASCII uppercase words in the lookup table. */ |
||||
var i uint |
||||
for i = 0; i < uint(w.len); i++ { |
||||
if dict[i] >= 'a' && dict[i] <= 'z' { |
||||
if (dict[i] ^ 32) != data[i] { |
||||
return false |
||||
} |
||||
} else { |
||||
if dict[i] != data[i] { |
||||
return false |
||||
} |
||||
} |
||||
} |
||||
|
||||
return true |
||||
} |
||||
} |
||||
} |
||||
|
||||
func findAllStaticDictionaryMatches(dict *encoderDictionary, data []byte, min_length uint, max_length uint, matches []uint32) bool { |
||||
var has_found_match bool = false |
||||
{ |
||||
var offset uint = uint(dict.buckets[hash(data)]) |
||||
var end bool = offset == 0 |
||||
for !end { |
||||
var w dictWord |
||||
w = dict.dict_words[offset] |
||||
offset++ |
||||
var l uint = uint(w.len) & 0x1F |
||||
var n uint = uint(1) << dict.words.size_bits_by_length[l] |
||||
var id uint = uint(w.idx) |
||||
end = !(w.len&0x80 == 0) |
||||
w.len = byte(l) |
||||
if w.transform == 0 { |
||||
var matchlen uint = dictMatchLength(dict.words, data, id, l, max_length) |
||||
var s []byte |
||||
var minlen uint |
||||
var maxlen uint |
||||
var len uint |
||||
|
||||
/* Transform "" + BROTLI_TRANSFORM_IDENTITY + "" */ |
||||
if matchlen == l { |
||||
addMatch(id, l, l, matches) |
||||
has_found_match = true |
||||
} |
||||
|
||||
/* Transforms "" + BROTLI_TRANSFORM_OMIT_LAST_1 + "" and |
||||
"" + BROTLI_TRANSFORM_OMIT_LAST_1 + "ing " */ |
||||
if matchlen >= l-1 { |
||||
addMatch(id+12*n, l-1, l, matches) |
||||
if l+2 < max_length && data[l-1] == 'i' && data[l] == 'n' && data[l+1] == 'g' && data[l+2] == ' ' { |
||||
addMatch(id+49*n, l+3, l, matches) |
||||
} |
||||
|
||||
has_found_match = true |
||||
} |
||||
|
||||
/* Transform "" + BROTLI_TRANSFORM_OMIT_LAST_# + "" (# = 2 .. 9) */ |
||||
minlen = min_length |
||||
|
||||
if l > 9 { |
||||
minlen = brotli_max_size_t(minlen, l-9) |
||||
} |
||||
maxlen = brotli_min_size_t(matchlen, l-2) |
||||
for len = minlen; len <= maxlen; len++ { |
||||
var cut uint = l - len |
||||
var transform_id uint = (cut << 2) + uint((dict.cutoffTransforms>>(cut*6))&0x3F) |
||||
addMatch(id+transform_id*n, uint(len), l, matches) |
||||
has_found_match = true |
||||
} |
||||
|
||||
if matchlen < l || l+6 >= max_length { |
||||
continue |
||||
} |
||||
|
||||
s = data[l:] |
||||
|
||||
/* Transforms "" + BROTLI_TRANSFORM_IDENTITY + <suffix> */ |
||||
if s[0] == ' ' { |
||||
addMatch(id+n, l+1, l, matches) |
||||
if s[1] == 'a' { |
||||
if s[2] == ' ' { |
||||
addMatch(id+28*n, l+3, l, matches) |
||||
} else if s[2] == 's' { |
||||
if s[3] == ' ' { |
||||
addMatch(id+46*n, l+4, l, matches) |
||||
} |
||||
} else if s[2] == 't' { |
||||
if s[3] == ' ' { |
||||
addMatch(id+60*n, l+4, l, matches) |
||||
} |
||||
} else if s[2] == 'n' { |
||||
if s[3] == 'd' && s[4] == ' ' { |
||||
addMatch(id+10*n, l+5, l, matches) |
||||
} |
||||
} |
||||
} else if s[1] == 'b' { |
||||
if s[2] == 'y' && s[3] == ' ' { |
||||
addMatch(id+38*n, l+4, l, matches) |
||||
} |
||||
} else if s[1] == 'i' { |
||||
if s[2] == 'n' { |
||||
if s[3] == ' ' { |
||||
addMatch(id+16*n, l+4, l, matches) |
||||
} |
||||
} else if s[2] == 's' { |
||||
if s[3] == ' ' { |
||||
addMatch(id+47*n, l+4, l, matches) |
||||
} |
||||
} |
||||
} else if s[1] == 'f' { |
||||
if s[2] == 'o' { |
||||
if s[3] == 'r' && s[4] == ' ' { |
||||
addMatch(id+25*n, l+5, l, matches) |
||||
} |
||||
} else if s[2] == 'r' { |
||||
if s[3] == 'o' && s[4] == 'm' && s[5] == ' ' { |
||||
addMatch(id+37*n, l+6, l, matches) |
||||
} |
||||
} |
||||
} else if s[1] == 'o' { |
||||
if s[2] == 'f' { |
||||
if s[3] == ' ' { |
||||
addMatch(id+8*n, l+4, l, matches) |
||||
} |
||||
} else if s[2] == 'n' { |
||||
if s[3] == ' ' { |
||||
addMatch(id+45*n, l+4, l, matches) |
||||
} |
||||
} |
||||
} else if s[1] == 'n' { |
||||
if s[2] == 'o' && s[3] == 't' && s[4] == ' ' { |
||||
addMatch(id+80*n, l+5, l, matches) |
||||
} |
||||
} else if s[1] == 't' { |
||||
if s[2] == 'h' { |
||||
if s[3] == 'e' { |
||||
if s[4] == ' ' { |
||||
addMatch(id+5*n, l+5, l, matches) |
||||
} |
||||
} else if s[3] == 'a' { |
||||
if s[4] == 't' && s[5] == ' ' { |
||||
addMatch(id+29*n, l+6, l, matches) |
||||
} |
||||
} |
||||
} else if s[2] == 'o' { |
||||
if s[3] == ' ' { |
||||
addMatch(id+17*n, l+4, l, matches) |
||||
} |
||||
} |
||||
} else if s[1] == 'w' { |
||||
if s[2] == 'i' && s[3] == 't' && s[4] == 'h' && s[5] == ' ' { |
||||
addMatch(id+35*n, l+6, l, matches) |
||||
} |
||||
} |
||||
} else if s[0] == '"' { |
||||
addMatch(id+19*n, l+1, l, matches) |
||||
if s[1] == '>' { |
||||
addMatch(id+21*n, l+2, l, matches) |
||||
} |
||||
} else if s[0] == '.' { |
||||
addMatch(id+20*n, l+1, l, matches) |
||||
if s[1] == ' ' { |
||||
addMatch(id+31*n, l+2, l, matches) |
||||
if s[2] == 'T' && s[3] == 'h' { |
||||
if s[4] == 'e' { |
||||
if s[5] == ' ' { |
||||
addMatch(id+43*n, l+6, l, matches) |
||||
} |
||||
} else if s[4] == 'i' { |
||||
if s[5] == 's' && s[6] == ' ' { |
||||
addMatch(id+75*n, l+7, l, matches) |
||||
} |
||||
} |
||||
} |
||||
} |
||||
} else if s[0] == ',' { |
||||
addMatch(id+76*n, l+1, l, matches) |
||||
if s[1] == ' ' { |
||||
addMatch(id+14*n, l+2, l, matches) |
||||
} |
||||
} else if s[0] == '\n' { |
||||
addMatch(id+22*n, l+1, l, matches) |
||||
if s[1] == '\t' { |
||||
addMatch(id+50*n, l+2, l, matches) |
||||
} |
||||
} else if s[0] == ']' { |
||||
addMatch(id+24*n, l+1, l, matches) |
||||
} else if s[0] == '\'' { |
||||
addMatch(id+36*n, l+1, l, matches) |
||||
} else if s[0] == ':' { |
||||
addMatch(id+51*n, l+1, l, matches) |
||||
} else if s[0] == '(' { |
||||
addMatch(id+57*n, l+1, l, matches) |
||||
} else if s[0] == '=' { |
||||
if s[1] == '"' { |
||||
addMatch(id+70*n, l+2, l, matches) |
||||
} else if s[1] == '\'' { |
||||
addMatch(id+86*n, l+2, l, matches) |
||||
} |
||||
} else if s[0] == 'a' { |
||||
if s[1] == 'l' && s[2] == ' ' { |
||||
addMatch(id+84*n, l+3, l, matches) |
||||
} |
||||
} else if s[0] == 'e' { |
||||
if s[1] == 'd' { |
||||
if s[2] == ' ' { |
||||
addMatch(id+53*n, l+3, l, matches) |
||||
} |
||||
} else if s[1] == 'r' { |
||||
if s[2] == ' ' { |
||||
addMatch(id+82*n, l+3, l, matches) |
||||
} |
||||
} else if s[1] == 's' { |
||||
if s[2] == 't' && s[3] == ' ' { |
||||
addMatch(id+95*n, l+4, l, matches) |
||||
} |
||||
} |
||||
} else if s[0] == 'f' { |
||||
if s[1] == 'u' && s[2] == 'l' && s[3] == ' ' { |
||||
addMatch(id+90*n, l+4, l, matches) |
||||
} |
||||
} else if s[0] == 'i' { |
||||
if s[1] == 'v' { |
||||
if s[2] == 'e' && s[3] == ' ' { |
||||
addMatch(id+92*n, l+4, l, matches) |
||||
} |
||||
} else if s[1] == 'z' { |
||||
if s[2] == 'e' && s[3] == ' ' { |
||||
addMatch(id+100*n, l+4, l, matches) |
||||
} |
||||
} |
||||
} else if s[0] == 'l' { |
||||
if s[1] == 'e' { |
||||
if s[2] == 's' && s[3] == 's' && s[4] == ' ' { |
||||
addMatch(id+93*n, l+5, l, matches) |
||||
} |
||||
} else if s[1] == 'y' { |
||||
if s[2] == ' ' { |
||||
addMatch(id+61*n, l+3, l, matches) |
||||
} |
||||
} |
||||
} else if s[0] == 'o' { |
||||
if s[1] == 'u' && s[2] == 's' && s[3] == ' ' { |
||||
addMatch(id+106*n, l+4, l, matches) |
||||
} |
||||
} |
||||
} else { |
||||
var is_all_caps bool = (w.transform != transformUppercaseFirst) |
||||
/* Set is_all_caps=0 for BROTLI_TRANSFORM_UPPERCASE_FIRST and |
||||
is_all_caps=1 otherwise (BROTLI_TRANSFORM_UPPERCASE_ALL) |
||||
transform. */ |
||||
|
||||
var s []byte |
||||
if !isMatch(dict.words, w, data, max_length) { |
||||
continue |
||||
} |
||||
|
||||
/* Transform "" + kUppercase{First,All} + "" */ |
||||
var tmp int |
||||
if is_all_caps { |
||||
tmp = 44 |
||||
} else { |
||||
tmp = 9 |
||||
} |
||||
addMatch(id+uint(tmp)*n, l, l, matches) |
||||
|
||||
has_found_match = true |
||||
if l+1 >= max_length { |
||||
continue |
||||
} |
||||
|
||||
/* Transforms "" + kUppercase{First,All} + <suffix> */ |
||||
s = data[l:] |
||||
|
||||
if s[0] == ' ' { |
||||
var tmp int |
||||
if is_all_caps { |
||||
tmp = 68 |
||||
} else { |
||||
tmp = 4 |
||||
} |
||||
addMatch(id+uint(tmp)*n, l+1, l, matches) |
||||
} else if s[0] == '"' { |
||||
var tmp int |
||||
if is_all_caps { |
||||
tmp = 87 |
||||
} else { |
||||
tmp = 66 |
||||
} |
||||
addMatch(id+uint(tmp)*n, l+1, l, matches) |
||||
if s[1] == '>' { |
||||
var tmp int |
||||
if is_all_caps { |
||||
tmp = 97 |
||||
} else { |
||||
tmp = 69 |
||||
} |
||||
addMatch(id+uint(tmp)*n, l+2, l, matches) |
||||
} |
||||
} else if s[0] == '.' { |
||||
var tmp int |
||||
if is_all_caps { |
||||
tmp = 101 |
||||
} else { |
||||
tmp = 79 |
||||
} |
||||
addMatch(id+uint(tmp)*n, l+1, l, matches) |
||||
if s[1] == ' ' { |
||||
var tmp int |
||||
if is_all_caps { |
||||
tmp = 114 |
||||
} else { |
||||
tmp = 88 |
||||
} |
||||
addMatch(id+uint(tmp)*n, l+2, l, matches) |
||||
} |
||||
} else if s[0] == ',' { |
||||
var tmp int |
||||
if is_all_caps { |
||||
tmp = 112 |
||||
} else { |
||||
tmp = 99 |
||||
} |
||||
addMatch(id+uint(tmp)*n, l+1, l, matches) |
||||
if s[1] == ' ' { |
||||
var tmp int |
||||
if is_all_caps { |
||||
tmp = 107 |
||||
} else { |
||||
tmp = 58 |
||||
} |
||||
addMatch(id+uint(tmp)*n, l+2, l, matches) |
||||
} |
||||
} else if s[0] == '\'' { |
||||
var tmp int |
||||
if is_all_caps { |
||||
tmp = 94 |
||||
} else { |
||||
tmp = 74 |
||||
} |
||||
addMatch(id+uint(tmp)*n, l+1, l, matches) |
||||
} else if s[0] == '(' { |
||||
var tmp int |
||||
if is_all_caps { |
||||
tmp = 113 |
||||
} else { |
||||
tmp = 78 |
||||
} |
||||
addMatch(id+uint(tmp)*n, l+1, l, matches) |
||||
} else if s[0] == '=' { |
||||
if s[1] == '"' { |
||||
var tmp int |
||||
if is_all_caps { |
||||
tmp = 105 |
||||
} else { |
||||
tmp = 104 |
||||
} |
||||
addMatch(id+uint(tmp)*n, l+2, l, matches) |
||||
} else if s[1] == '\'' { |
||||
var tmp int |
||||
if is_all_caps { |
||||
tmp = 116 |
||||
} else { |
||||
tmp = 108 |
||||
} |
||||
addMatch(id+uint(tmp)*n, l+2, l, matches) |
||||
} |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
/* Transforms with prefixes " " and "." */ |
||||
if max_length >= 5 && (data[0] == ' ' || data[0] == '.') { |
||||
var is_space bool = (data[0] == ' ') |
||||
var offset uint = uint(dict.buckets[hash(data[1:])]) |
||||
var end bool = offset == 0 |
||||
for !end { |
||||
var w dictWord |
||||
w = dict.dict_words[offset] |
||||
offset++ |
||||
var l uint = uint(w.len) & 0x1F |
||||
var n uint = uint(1) << dict.words.size_bits_by_length[l] |
||||
var id uint = uint(w.idx) |
||||
end = !(w.len&0x80 == 0) |
||||
w.len = byte(l) |
||||
if w.transform == 0 { |
||||
var s []byte |
||||
if !isMatch(dict.words, w, data[1:], max_length-1) { |
||||
continue |
||||
} |
||||
|
||||
/* Transforms " " + BROTLI_TRANSFORM_IDENTITY + "" and |
||||
"." + BROTLI_TRANSFORM_IDENTITY + "" */ |
||||
var tmp int |
||||
if is_space { |
||||
tmp = 6 |
||||
} else { |
||||
tmp = 32 |
||||
} |
||||
addMatch(id+uint(tmp)*n, l+1, l, matches) |
||||
|
||||
has_found_match = true |
||||
if l+2 >= max_length { |
||||
continue |
||||
} |
||||
|
||||
/* Transforms " " + BROTLI_TRANSFORM_IDENTITY + <suffix> and |
||||
"." + BROTLI_TRANSFORM_IDENTITY + <suffix> |
||||
*/ |
||||
s = data[l+1:] |
||||
|
||||
if s[0] == ' ' { |
||||
var tmp int |
||||
if is_space { |
||||
tmp = 2 |
||||
} else { |
||||
tmp = 77 |
||||
} |
||||
addMatch(id+uint(tmp)*n, l+2, l, matches) |
||||
} else if s[0] == '(' { |
||||
var tmp int |
||||
if is_space { |
||||
tmp = 89 |
||||
} else { |
||||
tmp = 67 |
||||
} |
||||
addMatch(id+uint(tmp)*n, l+2, l, matches) |
||||
} else if is_space { |
||||
if s[0] == ',' { |
||||
addMatch(id+103*n, l+2, l, matches) |
||||
if s[1] == ' ' { |
||||
addMatch(id+33*n, l+3, l, matches) |
||||
} |
||||
} else if s[0] == '.' { |
||||
addMatch(id+71*n, l+2, l, matches) |
||||
if s[1] == ' ' { |
||||
addMatch(id+52*n, l+3, l, matches) |
||||
} |
||||
} else if s[0] == '=' { |
||||
if s[1] == '"' { |
||||
addMatch(id+81*n, l+3, l, matches) |
||||
} else if s[1] == '\'' { |
||||
addMatch(id+98*n, l+3, l, matches) |
||||
} |
||||
} |
||||
} |
||||
} else if is_space { |
||||
var is_all_caps bool = (w.transform != transformUppercaseFirst) |
||||
/* Set is_all_caps=0 for BROTLI_TRANSFORM_UPPERCASE_FIRST and |
||||
is_all_caps=1 otherwise (BROTLI_TRANSFORM_UPPERCASE_ALL) |
||||
transform. */ |
||||
|
||||
var s []byte |
||||
if !isMatch(dict.words, w, data[1:], max_length-1) { |
||||
continue |
||||
} |
||||
|
||||
/* Transforms " " + kUppercase{First,All} + "" */ |
||||
var tmp int |
||||
if is_all_caps { |
||||
tmp = 85 |
||||
} else { |
||||
tmp = 30 |
||||
} |
||||
addMatch(id+uint(tmp)*n, l+1, l, matches) |
||||
|
||||
has_found_match = true |
||||
if l+2 >= max_length { |
||||
continue |
||||
} |
||||
|
||||
/* Transforms " " + kUppercase{First,All} + <suffix> */ |
||||
s = data[l+1:] |
||||
|
||||
if s[0] == ' ' { |
||||
var tmp int |
||||
if is_all_caps { |
||||
tmp = 83 |
||||
} else { |
||||
tmp = 15 |
||||
} |
||||
addMatch(id+uint(tmp)*n, l+2, l, matches) |
||||
} else if s[0] == ',' { |
||||
if !is_all_caps { |
||||
addMatch(id+109*n, l+2, l, matches) |
||||
} |
||||
|
||||
if s[1] == ' ' { |
||||
var tmp int |
||||
if is_all_caps { |
||||
tmp = 111 |
||||
} else { |
||||
tmp = 65 |
||||
} |
||||
addMatch(id+uint(tmp)*n, l+3, l, matches) |
||||
} |
||||
} else if s[0] == '.' { |
||||
var tmp int |
||||
if is_all_caps { |
||||
tmp = 115 |
||||
} else { |
||||
tmp = 96 |
||||
} |
||||
addMatch(id+uint(tmp)*n, l+2, l, matches) |
||||
if s[1] == ' ' { |
||||
var tmp int |
||||
if is_all_caps { |
||||
tmp = 117 |
||||
} else { |
||||
tmp = 91 |
||||
} |
||||
addMatch(id+uint(tmp)*n, l+3, l, matches) |
||||
} |
||||
} else if s[0] == '=' { |
||||
if s[1] == '"' { |
||||
var tmp int |
||||
if is_all_caps { |
||||
tmp = 110 |
||||
} else { |
||||
tmp = 118 |
||||
} |
||||
addMatch(id+uint(tmp)*n, l+3, l, matches) |
||||
} else if s[1] == '\'' { |
||||
var tmp int |
||||
if is_all_caps { |
||||
tmp = 119 |
||||
} else { |
||||
tmp = 120 |
||||
} |
||||
addMatch(id+uint(tmp)*n, l+3, l, matches) |
||||
} |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
if max_length >= 6 { |
||||
/* Transforms with prefixes "e ", "s ", ", " and "\xC2\xA0" */ |
||||
if (data[1] == ' ' && (data[0] == 'e' || data[0] == 's' || data[0] == ',')) || (data[0] == 0xC2 && data[1] == 0xA0) { |
||||
var offset uint = uint(dict.buckets[hash(data[2:])]) |
||||
var end bool = offset == 0 |
||||
for !end { |
||||
var w dictWord |
||||
w = dict.dict_words[offset] |
||||
offset++ |
||||
var l uint = uint(w.len) & 0x1F |
||||
var n uint = uint(1) << dict.words.size_bits_by_length[l] |
||||
var id uint = uint(w.idx) |
||||
end = !(w.len&0x80 == 0) |
||||
w.len = byte(l) |
||||
if w.transform == 0 && isMatch(dict.words, w, data[2:], max_length-2) { |
||||
if data[0] == 0xC2 { |
||||
addMatch(id+102*n, l+2, l, matches) |
||||
has_found_match = true |
||||
} else if l+2 < max_length && data[l+2] == ' ' { |
||||
var t uint = 13 |
||||
if data[0] == 'e' { |
||||
t = 18 |
||||
} else if data[0] == 's' { |
||||
t = 7 |
||||
} |
||||
addMatch(id+t*n, l+3, l, matches) |
||||
has_found_match = true |
||||
} |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
if max_length >= 9 { |
||||
/* Transforms with prefixes " the " and ".com/" */ |
||||
if (data[0] == ' ' && data[1] == 't' && data[2] == 'h' && data[3] == 'e' && data[4] == ' ') || (data[0] == '.' && data[1] == 'c' && data[2] == 'o' && data[3] == 'm' && data[4] == '/') { |
||||
var offset uint = uint(dict.buckets[hash(data[5:])]) |
||||
var end bool = offset == 0 |
||||
for !end { |
||||
var w dictWord |
||||
w = dict.dict_words[offset] |
||||
offset++ |
||||
var l uint = uint(w.len) & 0x1F |
||||
var n uint = uint(1) << dict.words.size_bits_by_length[l] |
||||
var id uint = uint(w.idx) |
||||
end = !(w.len&0x80 == 0) |
||||
w.len = byte(l) |
||||
if w.transform == 0 && isMatch(dict.words, w, data[5:], max_length-5) { |
||||
var tmp int |
||||
if data[0] == ' ' { |
||||
tmp = 41 |
||||
} else { |
||||
tmp = 72 |
||||
} |
||||
addMatch(id+uint(tmp)*n, l+5, l, matches) |
||||
has_found_match = true |
||||
if l+5 < max_length { |
||||
var s []byte = data[l+5:] |
||||
if data[0] == ' ' { |
||||
if l+8 < max_length && s[0] == ' ' && s[1] == 'o' && s[2] == 'f' && s[3] == ' ' { |
||||
addMatch(id+62*n, l+9, l, matches) |
||||
if l+12 < max_length && s[4] == 't' && s[5] == 'h' && s[6] == 'e' && s[7] == ' ' { |
||||
addMatch(id+73*n, l+13, l, matches) |
||||
} |
||||
} |
||||
} |
||||
} |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
return has_found_match |
||||
} |
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,22 @@ |
||||
package brotli |
||||
|
||||
/* Copyright 2013 Google Inc. All Rights Reserved. |
||||
|
||||
Distributed under MIT license. |
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/ |
||||
|
||||
/* Utilities for building Huffman decoding tables. */ |
||||
|
||||
type symbolList struct { |
||||
storage []uint16 |
||||
offset int |
||||
} |
||||
|
||||
func symbolListGet(sl symbolList, i int) uint16 { |
||||
return sl.storage[i+sl.offset] |
||||
} |
||||
|
||||
func symbolListPut(sl symbolList, i int, val uint16) { |
||||
sl.storage[i+sl.offset] = val |
||||
} |
||||
@ -0,0 +1,641 @@ |
||||
package brotli |
||||
|
||||
const ( |
||||
transformIdentity = 0 |
||||
transformOmitLast1 = 1 |
||||
transformOmitLast2 = 2 |
||||
transformOmitLast3 = 3 |
||||
transformOmitLast4 = 4 |
||||
transformOmitLast5 = 5 |
||||
transformOmitLast6 = 6 |
||||
transformOmitLast7 = 7 |
||||
transformOmitLast8 = 8 |
||||
transformOmitLast9 = 9 |
||||
transformUppercaseFirst = 10 |
||||
transformUppercaseAll = 11 |
||||
transformOmitFirst1 = 12 |
||||
transformOmitFirst2 = 13 |
||||
transformOmitFirst3 = 14 |
||||
transformOmitFirst4 = 15 |
||||
transformOmitFirst5 = 16 |
||||
transformOmitFirst6 = 17 |
||||
transformOmitFirst7 = 18 |
||||
transformOmitFirst8 = 19 |
||||
transformOmitFirst9 = 20 |
||||
transformShiftFirst = 21 |
||||
transformShiftAll = 22 + iota - 22 |
||||
numTransformTypes |
||||
) |
||||
|
||||
const transformsMaxCutOff = transformOmitLast9 |
||||
|
||||
type transforms struct { |
||||
prefix_suffix_size uint16 |
||||
prefix_suffix []byte |
||||
prefix_suffix_map []uint16 |
||||
num_transforms uint32 |
||||
transforms []byte |
||||
params []byte |
||||
cutOffTransforms [transformsMaxCutOff + 1]int16 |
||||
} |
||||
|
||||
func transformPrefixId(t *transforms, I int) byte { |
||||
return t.transforms[(I*3)+0] |
||||
} |
||||
|
||||
func transformType(t *transforms, I int) byte { |
||||
return t.transforms[(I*3)+1] |
||||
} |
||||
|
||||
func transformSuffixId(t *transforms, I int) byte { |
||||
return t.transforms[(I*3)+2] |
||||
} |
||||
|
||||
func transformPrefix(t *transforms, I int) []byte { |
||||
return t.prefix_suffix[t.prefix_suffix_map[transformPrefixId(t, I)]:] |
||||
} |
||||
|
||||
func transformSuffix(t *transforms, I int) []byte { |
||||
return t.prefix_suffix[t.prefix_suffix_map[transformSuffixId(t, I)]:] |
||||
} |
||||
|
||||
/* RFC 7932 transforms string data */ |
||||
const kPrefixSuffix string = "\001 \002, \010 of the \004 of \002s \001.\005 and \004 " + "in \001\"\004 to \002\">\001\n\002. \001]\005 for \003 a \006 " + "that \001'\006 with \006 from \004 by \001(\006. T" + "he \004 on \004 as \004 is \004ing \002\n\t\001:\003ed " + "\002=\"\004 at \003ly \001,\002='\005.com/\007. This \005" + " not \003er \003al \004ful \004ive \005less \004es" + "t \004ize \002\xc2\xa0\004ous \005 the \002e \000" |
||||
|
||||
var kPrefixSuffixMap = [50]uint16{ |
||||
0x00, |
||||
0x02, |
||||
0x05, |
||||
0x0E, |
||||
0x13, |
||||
0x16, |
||||
0x18, |
||||
0x1E, |
||||
0x23, |
||||
0x25, |
||||
0x2A, |
||||
0x2D, |
||||
0x2F, |
||||
0x32, |
||||
0x34, |
||||
0x3A, |
||||
0x3E, |
||||
0x45, |
||||
0x47, |
||||
0x4E, |
||||
0x55, |
||||
0x5A, |
||||
0x5C, |
||||
0x63, |
||||
0x68, |
||||
0x6D, |
||||
0x72, |
||||
0x77, |
||||
0x7A, |
||||
0x7C, |
||||
0x80, |
||||
0x83, |
||||
0x88, |
||||
0x8C, |
||||
0x8E, |
||||
0x91, |
||||
0x97, |
||||
0x9F, |
||||
0xA5, |
||||
0xA9, |
||||
0xAD, |
||||
0xB2, |
||||
0xB7, |
||||
0xBD, |
||||
0xC2, |
||||
0xC7, |
||||
0xCA, |
||||
0xCF, |
||||
0xD5, |
||||
0xD8, |
||||
} |
||||
|
||||
/* RFC 7932 transforms */ |
||||
var kTransformsData = []byte{ |
||||
49, |
||||
transformIdentity, |
||||
49, |
||||
49, |
||||
transformIdentity, |
||||
0, |
||||
0, |
||||
transformIdentity, |
||||
0, |
||||
49, |
||||
transformOmitFirst1, |
||||
49, |
||||
49, |
||||
transformUppercaseFirst, |
||||
0, |
||||
49, |
||||
transformIdentity, |
||||
47, |
||||
0, |
||||
transformIdentity, |
||||
49, |
||||
4, |
||||
transformIdentity, |
||||
0, |
||||
49, |
||||
transformIdentity, |
||||
3, |
||||
49, |
||||
transformUppercaseFirst, |
||||
49, |
||||
49, |
||||
transformIdentity, |
||||
6, |
||||
49, |
||||
transformOmitFirst2, |
||||
49, |
||||
49, |
||||
transformOmitLast1, |
||||
49, |
||||
1, |
||||
transformIdentity, |
||||
0, |
||||
49, |
||||
transformIdentity, |
||||
1, |
||||
0, |
||||
transformUppercaseFirst, |
||||
0, |
||||
49, |
||||
transformIdentity, |
||||
7, |
||||
49, |
||||
transformIdentity, |
||||
9, |
||||
48, |
||||
transformIdentity, |
||||
0, |
||||
49, |
||||
transformIdentity, |
||||
8, |
||||
49, |
||||
transformIdentity, |
||||
5, |
||||
49, |
||||
transformIdentity, |
||||
10, |
||||
49, |
||||
transformIdentity, |
||||
11, |
||||
49, |
||||
transformOmitLast3, |
||||
49, |
||||
49, |
||||
transformIdentity, |
||||
13, |
||||
49, |
||||
transformIdentity, |
||||
14, |
||||
49, |
||||
transformOmitFirst3, |
||||
49, |
||||
49, |
||||
transformOmitLast2, |
||||
49, |
||||
49, |
||||
transformIdentity, |
||||
15, |
||||
49, |
||||
transformIdentity, |
||||
16, |
||||
0, |
||||
transformUppercaseFirst, |
||||
49, |
||||
49, |
||||
transformIdentity, |
||||
12, |
||||
5, |
||||
transformIdentity, |
||||
49, |
||||
0, |
||||
transformIdentity, |
||||
1, |
||||
49, |
||||
transformOmitFirst4, |
||||
49, |
||||
49, |
||||
transformIdentity, |
||||
18, |
||||
49, |
||||
transformIdentity, |
||||
17, |
||||
49, |
||||
transformIdentity, |
||||
19, |
||||
49, |
||||
transformIdentity, |
||||
20, |
||||
49, |
||||
transformOmitFirst5, |
||||
49, |
||||
49, |
||||
transformOmitFirst6, |
||||
49, |
||||
47, |
||||
transformIdentity, |
||||
49, |
||||
49, |
||||
transformOmitLast4, |
||||
49, |
||||
49, |
||||
transformIdentity, |
||||
22, |
||||
49, |
||||
transformUppercaseAll, |
||||
49, |
||||
49, |
||||
transformIdentity, |
||||
23, |
||||
49, |
||||
transformIdentity, |
||||
24, |
||||
49, |
||||
transformIdentity, |
||||
25, |
||||
49, |
||||
transformOmitLast7, |
||||
49, |
||||
49, |
||||
transformOmitLast1, |
||||
26, |
||||
49, |
||||
transformIdentity, |
||||
27, |
||||
49, |
||||
transformIdentity, |
||||
28, |
||||
0, |
||||
transformIdentity, |
||||
12, |
||||
49, |
||||
transformIdentity, |
||||
29, |
||||
49, |
||||
transformOmitFirst9, |
||||
49, |
||||
49, |
||||
transformOmitFirst7, |
||||
49, |
||||
49, |
||||
transformOmitLast6, |
||||
49, |
||||
49, |
||||
transformIdentity, |
||||
21, |
||||
49, |
||||
transformUppercaseFirst, |
||||
1, |
||||
49, |
||||
transformOmitLast8, |
||||
49, |
||||
49, |
||||
transformIdentity, |
||||
31, |
||||
49, |
||||
transformIdentity, |
||||
32, |
||||
47, |
||||
transformIdentity, |
||||
3, |
||||
49, |
||||
transformOmitLast5, |
||||
49, |
||||
49, |
||||
transformOmitLast9, |
||||
49, |
||||
0, |
||||
transformUppercaseFirst, |
||||
1, |
||||
49, |
||||
transformUppercaseFirst, |
||||
8, |
||||
5, |
||||
transformIdentity, |
||||
21, |
||||
49, |
||||
transformUppercaseAll, |
||||
0, |
||||
49, |
||||
transformUppercaseFirst, |
||||
10, |
||||
49, |
||||
transformIdentity, |
||||
30, |
||||
0, |
||||
transformIdentity, |
||||
5, |
||||
35, |
||||
transformIdentity, |
||||
49, |
||||
47, |
||||
transformIdentity, |
||||
2, |
||||
49, |
||||
transformUppercaseFirst, |
||||
17, |
||||
49, |
||||
transformIdentity, |
||||
36, |
||||
49, |
||||
transformIdentity, |
||||
33, |
||||
5, |
||||
transformIdentity, |
||||
0, |
||||
49, |
||||
transformUppercaseFirst, |
||||
21, |
||||
49, |
||||
transformUppercaseFirst, |
||||
5, |
||||
49, |
||||
transformIdentity, |
||||
37, |
||||
0, |
||||
transformIdentity, |
||||
30, |
||||
49, |
||||
transformIdentity, |
||||
38, |
||||
0, |
||||
transformUppercaseAll, |
||||
0, |
||||
49, |
||||
transformIdentity, |
||||
39, |
||||
0, |
||||
transformUppercaseAll, |
||||
49, |
||||
49, |
||||
transformIdentity, |
||||
34, |
||||
49, |
||||
transformUppercaseAll, |
||||
8, |
||||
49, |
||||
transformUppercaseFirst, |
||||
12, |
||||
0, |
||||
transformIdentity, |
||||
21, |
||||
49, |
||||
transformIdentity, |
||||
40, |
||||
0, |
||||
transformUppercaseFirst, |
||||
12, |
||||
49, |
||||
transformIdentity, |
||||
41, |
||||
49, |
||||
transformIdentity, |
||||
42, |
||||
49, |
||||
transformUppercaseAll, |
||||
17, |
||||
49, |
||||
transformIdentity, |
||||
43, |
||||
0, |
||||
transformUppercaseFirst, |
||||
5, |
||||
49, |
||||
transformUppercaseAll, |
||||
10, |
||||
0, |
||||
transformIdentity, |
||||
34, |
||||
49, |
||||
transformUppercaseFirst, |
||||
33, |
||||
49, |
||||
transformIdentity, |
||||
44, |
||||
49, |
||||
transformUppercaseAll, |
||||
5, |
||||
45, |
||||
transformIdentity, |
||||
49, |
||||
0, |
||||
transformIdentity, |
||||
33, |
||||
49, |
||||
transformUppercaseFirst, |
||||
30, |
||||
49, |
||||
transformUppercaseAll, |
||||
30, |
||||
49, |
||||
transformIdentity, |
||||
46, |
||||
49, |
||||
transformUppercaseAll, |
||||
1, |
||||
49, |
||||
transformUppercaseFirst, |
||||
34, |
||||
0, |
||||
transformUppercaseFirst, |
||||
33, |
||||
0, |
||||
transformUppercaseAll, |
||||
30, |
||||
0, |
||||
transformUppercaseAll, |
||||
1, |
||||
49, |
||||
transformUppercaseAll, |
||||
33, |
||||
49, |
||||
transformUppercaseAll, |
||||
21, |
||||
49, |
||||
transformUppercaseAll, |
||||
12, |
||||
0, |
||||
transformUppercaseAll, |
||||
5, |
||||
49, |
||||
transformUppercaseAll, |
||||
34, |
||||
0, |
||||
transformUppercaseAll, |
||||
12, |
||||
0, |
||||
transformUppercaseFirst, |
||||
30, |
||||
0, |
||||
transformUppercaseAll, |
||||
34, |
||||
0, |
||||
transformUppercaseFirst, |
||||
34, |
||||
} |
||||
|
||||
var kBrotliTransforms = transforms{ |
||||
217, |
||||
[]byte(kPrefixSuffix), |
||||
kPrefixSuffixMap[:], |
||||
121, |
||||
kTransformsData, |
||||
nil, /* no extra parameters */ |
||||
[transformsMaxCutOff + 1]int16{0, 12, 27, 23, 42, 63, 56, 48, 59, 64}, |
||||
} |
||||
|
||||
func getTransforms() *transforms { |
||||
return &kBrotliTransforms |
||||
} |
||||
|
||||
func toUpperCase(p []byte) int { |
||||
if p[0] < 0xC0 { |
||||
if p[0] >= 'a' && p[0] <= 'z' { |
||||
p[0] ^= 32 |
||||
} |
||||
|
||||
return 1 |
||||
} |
||||
|
||||
/* An overly simplified uppercasing model for UTF-8. */ |
||||
if p[0] < 0xE0 { |
||||
p[1] ^= 32 |
||||
return 2 |
||||
} |
||||
|
||||
/* An arbitrary transform for three byte characters. */ |
||||
p[2] ^= 5 |
||||
|
||||
return 3 |
||||
} |
||||
|
||||
func shiftTransform(word []byte, word_len int, parameter uint16) int { |
||||
/* Limited sign extension: scalar < (1 << 24). */ |
||||
var scalar uint32 = (uint32(parameter) & 0x7FFF) + (0x1000000 - (uint32(parameter) & 0x8000)) |
||||
if word[0] < 0x80 { |
||||
/* 1-byte rune / 0sssssss / 7 bit scalar (ASCII). */ |
||||
scalar += uint32(word[0]) |
||||
|
||||
word[0] = byte(scalar & 0x7F) |
||||
return 1 |
||||
} else if word[0] < 0xC0 { |
||||
/* Continuation / 10AAAAAA. */ |
||||
return 1 |
||||
} else if word[0] < 0xE0 { |
||||
/* 2-byte rune / 110sssss AAssssss / 11 bit scalar. */ |
||||
if word_len < 2 { |
||||
return 1 |
||||
} |
||||
scalar += uint32(word[1]&0x3F | (word[0]&0x1F)<<6) |
||||
word[0] = byte(0xC0 | (scalar>>6)&0x1F) |
||||
word[1] = byte(uint32(word[1]&0xC0) | scalar&0x3F) |
||||
return 2 |
||||
} else if word[0] < 0xF0 { |
||||
/* 3-byte rune / 1110ssss AAssssss BBssssss / 16 bit scalar. */ |
||||
if word_len < 3 { |
||||
return word_len |
||||
} |
||||
scalar += uint32(word[2])&0x3F | uint32(word[1]&0x3F)<<6 | uint32(word[0]&0x0F)<<12 |
||||
word[0] = byte(0xE0 | (scalar>>12)&0x0F) |
||||
word[1] = byte(uint32(word[1]&0xC0) | (scalar>>6)&0x3F) |
||||
word[2] = byte(uint32(word[2]&0xC0) | scalar&0x3F) |
||||
return 3 |
||||
} else if word[0] < 0xF8 { |
||||
/* 4-byte rune / 11110sss AAssssss BBssssss CCssssss / 21 bit scalar. */ |
||||
if word_len < 4 { |
||||
return word_len |
||||
} |
||||
scalar += uint32(word[3])&0x3F | uint32(word[2]&0x3F)<<6 | uint32(word[1]&0x3F)<<12 | uint32(word[0]&0x07)<<18 |
||||
word[0] = byte(0xF0 | (scalar>>18)&0x07) |
||||
word[1] = byte(uint32(word[1]&0xC0) | (scalar>>12)&0x3F) |
||||
word[2] = byte(uint32(word[2]&0xC0) | (scalar>>6)&0x3F) |
||||
word[3] = byte(uint32(word[3]&0xC0) | scalar&0x3F) |
||||
return 4 |
||||
} |
||||
|
||||
return 1 |
||||
} |
||||
|
||||
func transformDictionaryWord(dst []byte, word []byte, len int, trans *transforms, transform_idx int) int { |
||||
var idx int = 0 |
||||
var prefix []byte = transformPrefix(trans, transform_idx) |
||||
var type_ byte = transformType(trans, transform_idx) |
||||
var suffix []byte = transformSuffix(trans, transform_idx) |
||||
{ |
||||
var prefix_len int = int(prefix[0]) |
||||
prefix = prefix[1:] |
||||
for { |
||||
tmp1 := prefix_len |
||||
prefix_len-- |
||||
if tmp1 == 0 { |
||||
break |
||||
} |
||||
dst[idx] = prefix[0] |
||||
idx++ |
||||
prefix = prefix[1:] |
||||
} |
||||
} |
||||
{ |
||||
var t int = int(type_) |
||||
var i int = 0 |
||||
if t <= transformOmitLast9 { |
||||
len -= t |
||||
} else if t >= transformOmitFirst1 && t <= transformOmitFirst9 { |
||||
var skip int = t - (transformOmitFirst1 - 1) |
||||
word = word[skip:] |
||||
len -= skip |
||||
} |
||||
|
||||
for i < len { |
||||
dst[idx] = word[i] |
||||
idx++ |
||||
i++ |
||||
} |
||||
if t == transformUppercaseFirst { |
||||
toUpperCase(dst[idx-len:]) |
||||
} else if t == transformUppercaseAll { |
||||
var uppercase []byte = dst |
||||
uppercase = uppercase[idx-len:] |
||||
for len > 0 { |
||||
var step int = toUpperCase(uppercase) |
||||
uppercase = uppercase[step:] |
||||
len -= step |
||||
} |
||||
} else if t == transformShiftFirst { |
||||
var param uint16 = uint16(trans.params[transform_idx*2]) + uint16(trans.params[transform_idx*2+1])<<8 |
||||
shiftTransform(dst[idx-len:], int(len), param) |
||||
} else if t == transformShiftAll { |
||||
var param uint16 = uint16(trans.params[transform_idx*2]) + uint16(trans.params[transform_idx*2+1])<<8 |
||||
var shift []byte = dst |
||||
shift = shift[idx-len:] |
||||
for len > 0 { |
||||
var step int = shiftTransform(shift, int(len), param) |
||||
shift = shift[step:] |
||||
len -= step |
||||
} |
||||
} |
||||
} |
||||
{ |
||||
var suffix_len int = int(suffix[0]) |
||||
suffix = suffix[1:] |
||||
for { |
||||
tmp2 := suffix_len |
||||
suffix_len-- |
||||
if tmp2 == 0 { |
||||
break |
||||
} |
||||
dst[idx] = suffix[0] |
||||
idx++ |
||||
suffix = suffix[1:] |
||||
} |
||||
return idx |
||||
} |
||||
} |
||||
@ -0,0 +1,71 @@ |
||||
package brotli |
||||
|
||||
/* Copyright 2013 Google Inc. All Rights Reserved. |
||||
|
||||
Distributed under MIT license. |
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/ |
||||
|
||||
/* Heuristics for deciding about the UTF8-ness of strings. */ |
||||
|
||||
const kMinUTF8Ratio float64 = 0.75 |
||||
|
||||
/* Returns 1 if at least min_fraction of the bytes between pos and |
||||
pos + length in the (data, mask) ring-buffer is UTF8-encoded, otherwise |
||||
returns 0. */ |
||||
func parseAsUTF8(symbol *int, input []byte, size uint) uint { |
||||
/* ASCII */ |
||||
if input[0]&0x80 == 0 { |
||||
*symbol = int(input[0]) |
||||
if *symbol > 0 { |
||||
return 1 |
||||
} |
||||
} |
||||
|
||||
/* 2-byte UTF8 */ |
||||
if size > 1 && input[0]&0xE0 == 0xC0 && input[1]&0xC0 == 0x80 { |
||||
*symbol = (int(input[0])&0x1F)<<6 | int(input[1])&0x3F |
||||
if *symbol > 0x7F { |
||||
return 2 |
||||
} |
||||
} |
||||
|
||||
/* 3-byte UFT8 */ |
||||
if size > 2 && input[0]&0xF0 == 0xE0 && input[1]&0xC0 == 0x80 && input[2]&0xC0 == 0x80 { |
||||
*symbol = (int(input[0])&0x0F)<<12 | (int(input[1])&0x3F)<<6 | int(input[2])&0x3F |
||||
if *symbol > 0x7FF { |
||||
return 3 |
||||
} |
||||
} |
||||
|
||||
/* 4-byte UFT8 */ |
||||
if size > 3 && input[0]&0xF8 == 0xF0 && input[1]&0xC0 == 0x80 && input[2]&0xC0 == 0x80 && input[3]&0xC0 == 0x80 { |
||||
*symbol = (int(input[0])&0x07)<<18 | (int(input[1])&0x3F)<<12 | (int(input[2])&0x3F)<<6 | int(input[3])&0x3F |
||||
if *symbol > 0xFFFF && *symbol <= 0x10FFFF { |
||||
return 4 |
||||
} |
||||
} |
||||
|
||||
/* Not UTF8, emit a special symbol above the UTF8-code space */ |
||||
*symbol = 0x110000 | int(input[0]) |
||||
|
||||
return 1 |
||||
} |
||||
|
||||
/* Returns 1 if at least min_fraction of the data is UTF8-encoded.*/ |
||||
func isMostlyUTF8(data []byte, pos uint, mask uint, length uint, min_fraction float64) bool { |
||||
var size_utf8 uint = 0 |
||||
var i uint = 0 |
||||
for i < length { |
||||
var symbol int |
||||
var current_data []byte |
||||
current_data = data[(pos+i)&mask:] |
||||
var bytes_read uint = parseAsUTF8(&symbol, current_data, length-i) |
||||
i += bytes_read |
||||
if symbol < 0x110000 { |
||||
size_utf8 += bytes_read |
||||
} |
||||
} |
||||
|
||||
return float64(size_utf8) > min_fraction*float64(length) |
||||
} |
||||
@ -0,0 +1,7 @@ |
||||
package brotli |
||||
|
||||
func assert(cond bool) { |
||||
if !cond { |
||||
panic("assertion failure") |
||||
} |
||||
} |
||||
@ -0,0 +1,56 @@ |
||||
package brotli |
||||
|
||||
/* Copyright 2010 Google Inc. All Rights Reserved. |
||||
|
||||
Distributed under MIT license. |
||||
See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
|
||||
*/ |
||||
|
||||
/* Write bits into a byte array. */ |
||||
|
||||
/* This function writes bits into bytes in increasing addresses, and within |
||||
a byte least-significant-bit first. |
||||
|
||||
The function can write up to 56 bits in one go with WriteBits |
||||
Example: let's assume that 3 bits (Rs below) have been written already: |
||||
|
||||
BYTE-0 BYTE+1 BYTE+2 |
||||
|
||||
0000 0RRR 0000 0000 0000 0000 |
||||
|
||||
Now, we could write 5 or less bits in MSB by just sifting by 3 |
||||
and OR'ing to BYTE-0. |
||||
|
||||
For n bits, we take the last 5 bits, OR that with high bits in BYTE-0, |
||||
and locate the rest in BYTE+1, BYTE+2, etc. */ |
||||
func writeBits(n_bits uint, bits uint64, pos *uint, array []byte) { |
||||
var array_pos []byte = array[*pos>>3:] |
||||
var bits_reserved_in_first_byte uint = (*pos & 7) |
||||
/* implicit & 0xFF is assumed for uint8_t arithmetics */ |
||||
|
||||
var bits_left_to_write uint |
||||
bits <<= bits_reserved_in_first_byte |
||||
array_pos[0] |= byte(bits) |
||||
array_pos = array_pos[1:] |
||||
for bits_left_to_write = n_bits + bits_reserved_in_first_byte; bits_left_to_write >= 9; bits_left_to_write -= 8 { |
||||
bits >>= 8 |
||||
array_pos[0] = byte(bits) |
||||
array_pos = array_pos[1:] |
||||
} |
||||
|
||||
array_pos[0] = 0 |
||||
*pos += n_bits |
||||
} |
||||
|
||||
func writeSingleBit(bit bool, pos *uint, array []byte) { |
||||
if bit { |
||||
writeBits(1, 1, pos, array) |
||||
} else { |
||||
writeBits(1, 0, pos, array) |
||||
} |
||||
} |
||||
|
||||
func writeBitsPrepareStorage(pos uint, array []byte) { |
||||
assert(pos&7 == 0) |
||||
array[pos>>3] = 0 |
||||
} |
||||
@ -0,0 +1,155 @@ |
||||
package brotli |
||||
|
||||
import ( |
||||
"compress/gzip" |
||||
"errors" |
||||
"io" |
||||
"net/http" |
||||
|
||||
"github.com/golang/gddo/httputil" |
||||
) |
||||
|
||||
const ( |
||||
BestSpeed = 0 |
||||
BestCompression = 11 |
||||
DefaultCompression = 6 |
||||
) |
||||
|
||||
// WriterOptions configures Writer.
|
||||
type WriterOptions struct { |
||||
// Quality controls the compression-speed vs compression-density trade-offs.
|
||||
// The higher the quality, the slower the compression. Range is 0 to 11.
|
||||
Quality int |
||||
// LGWin is the base 2 logarithm of the sliding window size.
|
||||
// Range is 10 to 24. 0 indicates automatic configuration based on Quality.
|
||||
LGWin int |
||||
} |
||||
|
||||
var ( |
||||
errEncode = errors.New("brotli: encode error") |
||||
errWriterClosed = errors.New("brotli: Writer is closed") |
||||
) |
||||
|
||||
// Writes to the returned writer are compressed and written to dst.
|
||||
// It is the caller's responsibility to call Close on the Writer when done.
|
||||
// Writes may be buffered and not flushed until Close.
|
||||
func NewWriter(dst io.Writer) *Writer { |
||||
return NewWriterLevel(dst, DefaultCompression) |
||||
} |
||||
|
||||
// NewWriterLevel is like NewWriter but specifies the compression level instead
|
||||
// of assuming DefaultCompression.
|
||||
// The compression level can be DefaultCompression or any integer value between
|
||||
// BestSpeed and BestCompression inclusive.
|
||||
func NewWriterLevel(dst io.Writer, level int) *Writer { |
||||
return NewWriterOptions(dst, WriterOptions{ |
||||
Quality: level, |
||||
}) |
||||
} |
||||
|
||||
// NewWriterOptions is like NewWriter but specifies WriterOptions
|
||||
func NewWriterOptions(dst io.Writer, options WriterOptions) *Writer { |
||||
w := new(Writer) |
||||
w.Reset(dst) |
||||
w.params.quality = options.Quality |
||||
if options.LGWin > 0 { |
||||
w.params.lgwin = uint(options.LGWin) |
||||
} |
||||
return w |
||||
} |
||||
|
||||
// Reset discards the Writer's state and makes it equivalent to the result of
|
||||
// its original state from NewWriter or NewWriterLevel, but writing to dst
|
||||
// instead. This permits reusing a Writer rather than allocating a new one.
|
||||
func (w *Writer) Reset(dst io.Writer) { |
||||
encoderInitState(w) |
||||
w.dst = dst |
||||
} |
||||
|
||||
func (w *Writer) writeChunk(p []byte, op int) (n int, err error) { |
||||
if w.dst == nil { |
||||
return 0, errWriterClosed |
||||
} |
||||
|
||||
for { |
||||
availableIn := uint(len(p)) |
||||
nextIn := p |
||||
success := encoderCompressStream(w, op, &availableIn, &nextIn) |
||||
bytesConsumed := len(p) - int(availableIn) |
||||
p = p[bytesConsumed:] |
||||
n += bytesConsumed |
||||
if !success { |
||||
return n, errEncode |
||||
} |
||||
|
||||
outputData := encoderTakeOutput(w) |
||||
|
||||
if len(outputData) > 0 { |
||||
_, err = w.dst.Write(outputData) |
||||
if err != nil { |
||||
return n, err |
||||
} |
||||
} |
||||
if len(p) == 0 { |
||||
return n, nil |
||||
} |
||||
} |
||||
} |
||||
|
||||
// Flush outputs encoded data for all input provided to Write. The resulting
|
||||
// output can be decoded to match all input before Flush, but the stream is
|
||||
// not yet complete until after Close.
|
||||
// Flush has a negative impact on compression.
|
||||
func (w *Writer) Flush() error { |
||||
_, err := w.writeChunk(nil, operationFlush) |
||||
return err |
||||
} |
||||
|
||||
// Close flushes remaining data to the decorated writer.
|
||||
func (w *Writer) Close() error { |
||||
// If stream is already closed, it is reported by `writeChunk`.
|
||||
_, err := w.writeChunk(nil, operationFinish) |
||||
w.dst = nil |
||||
return err |
||||
} |
||||
|
||||
// Write implements io.Writer. Flush or Close must be called to ensure that the
|
||||
// encoded bytes are actually flushed to the underlying Writer.
|
||||
func (w *Writer) Write(p []byte) (n int, err error) { |
||||
return w.writeChunk(p, operationProcess) |
||||
} |
||||
|
||||
type nopCloser struct { |
||||
io.Writer |
||||
} |
||||
|
||||
func (nopCloser) Close() error { return nil } |
||||
|
||||
// HTTPCompressor chooses a compression method (brotli, gzip, or none) based on
|
||||
// the Accept-Encoding header, sets the Content-Encoding header, and returns a
|
||||
// WriteCloser that implements that compression. The Close method must be called
|
||||
// before the current HTTP handler returns.
|
||||
//
|
||||
// Due to https://github.com/golang/go/issues/31753, the response will not be
|
||||
// compressed unless you set a Content-Type header before you call
|
||||
// HTTPCompressor.
|
||||
func HTTPCompressor(w http.ResponseWriter, r *http.Request) io.WriteCloser { |
||||
if w.Header().Get("Content-Type") == "" { |
||||
return nopCloser{w} |
||||
} |
||||
|
||||
if w.Header().Get("Vary") == "" { |
||||
w.Header().Set("Vary", "Accept-Encoding") |
||||
} |
||||
|
||||
encoding := httputil.NegotiateContentEncoding(r, []string{"br", "gzip"}) |
||||
switch encoding { |
||||
case "br": |
||||
w.Header().Set("Content-Encoding", "br") |
||||
return NewWriter(w) |
||||
case "gzip": |
||||
w.Header().Set("Content-Encoding", "gzip") |
||||
return gzip.NewWriter(w) |
||||
} |
||||
return nopCloser{w} |
||||
} |
||||
@ -0,0 +1,36 @@ |
||||
sudo: false |
||||
language: go |
||||
before_install: |
||||
- curl -L https://github.com/google/brotli/archive/v1.0.2.tar.gz | tar -zxv |
||||
- (cd brotli-1.0.2 && mkdir out && cd out && ../configure-cmake && make && sudo make install) |
||||
- rm -rf brotli-1.0.2 |
||||
- curl -L https://github.com/facebook/zstd/archive/v1.3.2.tar.gz | tar -zxv |
||||
- (cd zstd-1.3.2 && sudo make install) |
||||
- rm -rf zstd-1.3.2 |
||||
- sudo ldconfig |
||||
- mkdir /tmp/go1.12 |
||||
- curl -L -s https://dl.google.com/go/go1.12.linux-amd64.tar.gz | tar -zxf - -C /tmp/go1.12 --strip-components 1 |
||||
- unset GOROOT |
||||
- (GO111MODULE=on /tmp/go1.12/bin/go mod vendor) |
||||
- (cd /tmp && GO111MODULE=on /tmp/go1.12/bin/go get golang.org/x/lint/golint@8f45f776aaf18cebc8d65861cc70c33c60471952) |
||||
- (cd /tmp && GO111MODULE=on /tmp/go1.12/bin/go get honnef.co/go/tools/cmd/staticcheck@2019.1) |
||||
matrix: |
||||
include: |
||||
- go: 1.9.x |
||||
script: |
||||
- go test -v -race ./... |
||||
- go: 1.10.x |
||||
script: |
||||
- go test -v -race ./... |
||||
- go: 1.11.x |
||||
script: |
||||
- go test -v -race ./... |
||||
- go: 1.12.x |
||||
script: |
||||
- ./ztest.sh |
||||
- go: master |
||||
script: |
||||
- go test -v -race ./... |
||||
allow_failures: |
||||
- go: master |
||||
fast_finish: true |
||||
@ -0,0 +1,24 @@ |
||||
Copyright © 2015, Joe Tsai and The Go Authors. All rights reserved. |
||||
|
||||
Redistribution and use in source and binary forms, with or without |
||||
modification, are permitted provided that the following conditions are met: |
||||
|
||||
* Redistributions of source code must retain the above copyright notice, this |
||||
list of conditions and the following disclaimer. |
||||
* Redistributions in binary form must reproduce the above copyright notice, |
||||
this list of conditions and the following disclaimer in the documentation and/or |
||||
other materials provided with the distribution. |
||||
* Neither the copyright holder nor the names of its contributors may be used to |
||||
endorse or promote products derived from this software without specific prior |
||||
written permission. |
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND |
||||
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
||||
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE |
||||
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE FOR ANY |
||||
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES |
||||
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
||||
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND |
||||
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS |
||||
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
||||
@ -0,0 +1,75 @@ |
||||
# Collection of compression libraries for Go # |
||||
|
||||
[](https://godoc.org/github.com/dsnet/compress) |
||||
[](https://travis-ci.org/dsnet/compress) |
||||
[](https://goreportcard.com/report/github.com/dsnet/compress) |
||||
|
||||
## Introduction ## |
||||
|
||||
**NOTE: This library is in active development. As such, there are no guarantees about the stability of the API. The author reserves the right to arbitrarily break the API for any reason.** |
||||
|
||||
This repository hosts a collection of compression related libraries. The goal of this project is to provide pure Go implementations for popular compression algorithms beyond what the Go standard library provides. The goals for these packages are as follows: |
||||
* Maintainable: That the code remains well documented, well tested, readable, easy to maintain, and easy to verify that it conforms to the specification for the format being implemented. |
||||
* Performant: To be able to compress and decompress within at least 80% of the rates that the C implementations are able to achieve. |
||||
* Flexible: That the code provides low-level and fine granularity control over the compression streams similar to what the C APIs would provide. |
||||
|
||||
Of these three, the first objective is often at odds with the other two objectives and provides interesting challenges. Higher performance can often be achieved by muddling abstraction layers or using non-intuitive low-level primitives. Also, more features and functionality, while useful in some situations, often complicates the API. Thus, this package will attempt to satisfy all the goals, but will defer to favoring maintainability when the performance or flexibility benefits are not significant enough. |
||||
|
||||
|
||||
## Library Status ## |
||||
|
||||
For the packages available, only some features are currently implemented: |
||||
|
||||
| Package | Reader | Writer | |
||||
| ------- | :----: | :----: | |
||||
| brotli | :white_check_mark: | | |
||||
| bzip2 | :white_check_mark: | :white_check_mark: | |
||||
| flate | :white_check_mark: | | |
||||
| xflate | :white_check_mark: | :white_check_mark: | |
||||
|
||||
This library is in active development. As such, there are no guarantees about the stability of the API. The author reserves the right to arbitrarily break the API for any reason. When the library becomes more mature, it is planned to eventually conform to some strict versioning scheme like [Semantic Versioning](http://semver.org/). |
||||
|
||||
However, in the meanwhile, this library does provide some basic API guarantees. For the types defined below, the method signatures are guaranteed to not change. Note that the author still reserves the right to change the fields within each ```Reader``` and ```Writer``` structs. |
||||
```go |
||||
type ReaderConfig struct { ... } |
||||
type Reader struct { ... } |
||||
func NewReader(io.Reader, *ReaderConfig) (*Reader, error) { ... } |
||||
func (*Reader) Read([]byte) (int, error) { ... } |
||||
func (*Reader) Close() error { ... } |
||||
|
||||
type WriterConfig struct { ... } |
||||
type Writer struct { ... } |
||||
func NewWriter(io.Writer, *WriterConfig) (*Writer, error) { ... } |
||||
func (*Writer) Write([]byte) (int, error) { ... } |
||||
func (*Writer) Close() error { ... } |
||||
``` |
||||
|
||||
To see what work still remains, see the [Task List](https://github.com/dsnet/compress/wiki/Task-List). |
||||
|
||||
## Performance ## |
||||
|
||||
See [Performance Metrics](https://github.com/dsnet/compress/wiki/Performance-Metrics). |
||||
|
||||
|
||||
## Frequently Asked Questions ## |
||||
|
||||
See [Frequently Asked Questions](https://github.com/dsnet/compress/wiki/Frequently-Asked-Questions). |
||||
|
||||
|
||||
## Installation ## |
||||
|
||||
Run the command: |
||||
|
||||
```go get -u github.com/dsnet/compress``` |
||||
|
||||
This library requires `Go1.9` or higher in order to build. |
||||
|
||||
|
||||
## Packages ## |
||||
|
||||
| Package | Description | |
||||
| :------ | :---------- | |
||||
| [brotli](http://godoc.org/github.com/dsnet/compress/brotli) | Package brotli implements the Brotli format, described in RFC 7932. | |
||||
| [bzip2](http://godoc.org/github.com/dsnet/compress/bzip2) | Package bzip2 implements the BZip2 compressed data format. | |
||||
| [flate](http://godoc.org/github.com/dsnet/compress/flate) | Package flate implements the DEFLATE format, described in RFC 1951. | |
||||
| [xflate](http://godoc.org/github.com/dsnet/compress/xflate) | Package xflate implements the XFLATE format, an random-access extension to DEFLATE. | |
||||
@ -0,0 +1,74 @@ |
||||
// Copyright 2015, Joe Tsai. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE.md file.
|
||||
|
||||
// Package compress is a collection of compression libraries.
|
||||
package compress |
||||
|
||||
import ( |
||||
"bufio" |
||||
"io" |
||||
|
||||
"github.com/dsnet/compress/internal/errors" |
||||
) |
||||
|
||||
// The Error interface identifies all compression related errors.
|
||||
type Error interface { |
||||
error |
||||
CompressError() |
||||
|
||||
// IsDeprecated reports the use of a deprecated and unsupported feature.
|
||||
IsDeprecated() bool |
||||
|
||||
// IsCorrupted reports whether the input stream was corrupted.
|
||||
IsCorrupted() bool |
||||
} |
||||
|
||||
var _ Error = errors.Error{} |
||||
|
||||
// ByteReader is an interface accepted by all decompression Readers.
|
||||
// It guarantees that the decompressor never reads more data than is necessary
|
||||
// from the underlying io.Reader.
|
||||
type ByteReader interface { |
||||
io.Reader |
||||
io.ByteReader |
||||
} |
||||
|
||||
var _ ByteReader = (*bufio.Reader)(nil) |
||||
|
||||
// BufferedReader is an interface accepted by all decompression Readers.
|
||||
// It guarantees that the decompressor never reads more data than is necessary
|
||||
// from the underlying io.Reader. Since BufferedReader allows a decompressor
|
||||
// to peek at bytes further along in the stream without advancing the read
|
||||
// pointer, decompression can experience a significant performance gain when
|
||||
// provided a reader that satisfies this interface. Thus, a decompressor will
|
||||
// prefer this interface over ByteReader for performance reasons.
|
||||
//
|
||||
// The bufio.Reader satisfies this interface.
|
||||
type BufferedReader interface { |
||||
io.Reader |
||||
|
||||
// Buffered returns the number of bytes currently buffered.
|
||||
//
|
||||
// This value becomes invalid following the next Read/Discard operation.
|
||||
Buffered() int |
||||
|
||||
// Peek returns the next n bytes without advancing the reader.
|
||||
//
|
||||
// If Peek returns fewer than n bytes, it also returns an error explaining
|
||||
// why the peek is short. Peek must support peeking of at least 8 bytes.
|
||||
// If 0 <= n <= Buffered(), Peek is guaranteed to succeed without reading
|
||||
// from the underlying io.Reader.
|
||||
//
|
||||
// This result becomes invalid following the next Read/Discard operation.
|
||||
Peek(n int) ([]byte, error) |
||||
|
||||
// Discard skips the next n bytes, returning the number of bytes discarded.
|
||||
//
|
||||
// If Discard skips fewer than n bytes, it also returns an error.
|
||||
// If 0 <= n <= Buffered(), Discard is guaranteed to succeed without reading
|
||||
// from the underlying io.Reader.
|
||||
Discard(n int) (int, error) |
||||
} |
||||
|
||||
var _ BufferedReader = (*bufio.Reader)(nil) |
||||
@ -0,0 +1,110 @@ |
||||
// Copyright 2015, Joe Tsai. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE.md file.
|
||||
|
||||
package bzip2 |
||||
|
||||
import "github.com/dsnet/compress/bzip2/internal/sais" |
||||
|
||||
// The Burrows-Wheeler Transform implementation used here is based on the
|
||||
// Suffix Array by Induced Sorting (SA-IS) methodology by Nong, Zhang, and Chan.
|
||||
// This implementation uses the sais algorithm originally written by Yuta Mori.
|
||||
//
|
||||
// The SA-IS algorithm runs in O(n) and outputs a Suffix Array. There is a
|
||||
// mathematical relationship between Suffix Arrays and the Burrows-Wheeler
|
||||
// Transform, such that a SA can be converted to a BWT in O(n) time.
|
||||
//
|
||||
// References:
|
||||
// http://www.hpl.hp.com/techreports/Compaq-DEC/SRC-RR-124.pdf
|
||||
// https://github.com/cscott/compressjs/blob/master/lib/BWT.js
|
||||
// https://www.quora.com/How-can-I-optimize-burrows-wheeler-transform-and-inverse-transform-to-work-in-O-n-time-O-n-space
|
||||
type burrowsWheelerTransform struct { |
||||
buf []byte |
||||
sa []int |
||||
perm []uint32 |
||||
} |
||||
|
||||
func (bwt *burrowsWheelerTransform) Encode(buf []byte) (ptr int) { |
||||
if len(buf) == 0 { |
||||
return -1 |
||||
} |
||||
|
||||
// TODO(dsnet): Find a way to avoid the duplicate input string method.
|
||||
// We only need to do this because suffix arrays (by definition) only
|
||||
// operate non-wrapped suffixes of a string. On the other hand,
|
||||
// the BWT specifically used in bzip2 operate on a strings that wrap-around
|
||||
// when being sorted.
|
||||
|
||||
// Step 1: Concatenate the input string to itself so that we can use the
|
||||
// suffix array algorithm for bzip2's variant of BWT.
|
||||
n := len(buf) |
||||
bwt.buf = append(append(bwt.buf[:0], buf...), buf...) |
||||
if cap(bwt.sa) < 2*n { |
||||
bwt.sa = make([]int, 2*n) |
||||
} |
||||
t := bwt.buf[:2*n] |
||||
sa := bwt.sa[:2*n] |
||||
|
||||
// Step 2: Compute the suffix array (SA). The input string, t, will not be
|
||||
// modified, while the results will be written to the output, sa.
|
||||
sais.ComputeSA(t, sa) |
||||
|
||||
// Step 3: Convert the SA to a BWT. Since ComputeSA does not mutate the
|
||||
// input, we have two copies of the input; in buf and buf2. Thus, we write
|
||||
// the transformation to buf, while using buf2.
|
||||
var j int |
||||
buf2 := t[n:] |
||||
for _, i := range sa { |
||||
if i < n { |
||||
if i == 0 { |
||||
ptr = j |
||||
i = n |
||||
} |
||||
buf[j] = buf2[i-1] |
||||
j++ |
||||
} |
||||
} |
||||
return ptr |
||||
} |
||||
|
||||
func (bwt *burrowsWheelerTransform) Decode(buf []byte, ptr int) { |
||||
if len(buf) == 0 { |
||||
return |
||||
} |
||||
|
||||
// Step 1: Compute cumm, where cumm[ch] reports the total number of
|
||||
// characters that precede the character ch in the alphabet.
|
||||
var cumm [256]int |
||||
for _, v := range buf { |
||||
cumm[v]++ |
||||
} |
||||
var sum int |
||||
for i, v := range cumm { |
||||
cumm[i] = sum |
||||
sum += v |
||||
} |
||||
|
||||
// Step 2: Compute perm, where perm[ptr] contains a pointer to the next
|
||||
// byte in buf and the next pointer in perm itself.
|
||||
if cap(bwt.perm) < len(buf) { |
||||
bwt.perm = make([]uint32, len(buf)) |
||||
} |
||||
perm := bwt.perm[:len(buf)] |
||||
for i, b := range buf { |
||||
perm[cumm[b]] = uint32(i) |
||||
cumm[b]++ |
||||
} |
||||
|
||||
// Step 3: Follow each pointer in perm to the next byte, starting with the
|
||||
// origin pointer.
|
||||
if cap(bwt.buf) < len(buf) { |
||||
bwt.buf = make([]byte, len(buf)) |
||||
} |
||||
buf2 := bwt.buf[:len(buf)] |
||||
i := perm[ptr] |
||||
for j := range buf2 { |
||||
buf2[j] = buf[i] |
||||
i = perm[i] |
||||
} |
||||
copy(buf, buf2) |
||||
} |
||||
@ -0,0 +1,110 @@ |
||||
// Copyright 2015, Joe Tsai. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE.md file.
|
||||
|
||||
// Package bzip2 implements the BZip2 compressed data format.
|
||||
//
|
||||
// Canonical C implementation:
|
||||
// http://bzip.org
|
||||
//
|
||||
// Unofficial format specification:
|
||||
// https://github.com/dsnet/compress/blob/master/doc/bzip2-format.pdf
|
||||
package bzip2 |
||||
|
||||
import ( |
||||
"fmt" |
||||
"hash/crc32" |
||||
|
||||
"github.com/dsnet/compress/internal" |
||||
"github.com/dsnet/compress/internal/errors" |
||||
) |
||||
|
||||
// There does not exist a formal specification of the BZip2 format. As such,
|
||||
// much of this work is derived by either reverse engineering the original C
|
||||
// source code or using secondary sources.
|
||||
//
|
||||
// Significant amounts of fuzz testing is done to ensure that outputs from
|
||||
// this package is properly decoded by the C library. Furthermore, we test that
|
||||
// both this package and the C library agree about what inputs are invalid.
|
||||
//
|
||||
// Compression stack:
|
||||
// Run-length encoding 1 (RLE1)
|
||||
// Burrows-Wheeler transform (BWT)
|
||||
// Move-to-front transform (MTF)
|
||||
// Run-length encoding 2 (RLE2)
|
||||
// Prefix encoding (PE)
|
||||
//
|
||||
// References:
|
||||
// http://bzip.org/
|
||||
// https://en.wikipedia.org/wiki/Bzip2
|
||||
// https://code.google.com/p/jbzip2/
|
||||
|
||||
const ( |
||||
BestSpeed = 1 |
||||
BestCompression = 9 |
||||
DefaultCompression = 6 |
||||
) |
||||
|
||||
const ( |
||||
hdrMagic = 0x425a // Hex of "BZ"
|
||||
blkMagic = 0x314159265359 // BCD of PI
|
||||
endMagic = 0x177245385090 // BCD of sqrt(PI)
|
||||
|
||||
blockSize = 100000 |
||||
) |
||||
|
||||
func errorf(c int, f string, a ...interface{}) error { |
||||
return errors.Error{Code: c, Pkg: "bzip2", Msg: fmt.Sprintf(f, a...)} |
||||
} |
||||
|
||||
func panicf(c int, f string, a ...interface{}) { |
||||
errors.Panic(errorf(c, f, a...)) |
||||
} |
||||
|
||||
// errWrap converts a lower-level errors.Error to be one from this package.
|
||||
// The replaceCode passed in will be used to replace the code for any errors
|
||||
// with the errors.Invalid code.
|
||||
//
|
||||
// For the Reader, set this to errors.Corrupted.
|
||||
// For the Writer, set this to errors.Internal.
|
||||
func errWrap(err error, replaceCode int) error { |
||||
if cerr, ok := err.(errors.Error); ok { |
||||
if errors.IsInvalid(cerr) { |
||||
cerr.Code = replaceCode |
||||
} |
||||
err = errorf(cerr.Code, "%s", cerr.Msg) |
||||
} |
||||
return err |
||||
} |
||||
|
||||
var errClosed = errorf(errors.Closed, "") |
||||
|
||||
// crc computes the CRC-32 used by BZip2.
|
||||
//
|
||||
// The CRC-32 computation in bzip2 treats bytes as having bits in big-endian
|
||||
// order. That is, the MSB is read before the LSB. Thus, we can use the
|
||||
// standard library version of CRC-32 IEEE with some minor adjustments.
|
||||
//
|
||||
// The byte array is used as an intermediate buffer to swap the bits of every
|
||||
// byte of the input.
|
||||
type crc struct { |
||||
val uint32 |
||||
buf [256]byte |
||||
} |
||||
|
||||
// update computes the CRC-32 of appending buf to c.
|
||||
func (c *crc) update(buf []byte) { |
||||
cval := internal.ReverseUint32(c.val) |
||||
for len(buf) > 0 { |
||||
n := len(buf) |
||||
if n > len(c.buf) { |
||||
n = len(c.buf) |
||||
} |
||||
for i, b := range buf[:n] { |
||||
c.buf[i] = internal.ReverseLUT[b] |
||||
} |
||||
cval = crc32.Update(cval, crc32.IEEETable, c.buf[:n]) |
||||
buf = buf[n:] |
||||
} |
||||
c.val = internal.ReverseUint32(cval) |
||||
} |
||||
@ -0,0 +1,13 @@ |
||||
// Copyright 2016, Joe Tsai. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE.md file.
|
||||
|
||||
// +build !gofuzz
|
||||
|
||||
// This file exists to suppress fuzzing details from release builds.
|
||||
|
||||
package bzip2 |
||||
|
||||
type fuzzReader struct{} |
||||
|
||||
func (*fuzzReader) updateChecksum(int64, uint32) {} |
||||
@ -0,0 +1,77 @@ |
||||
// Copyright 2016, Joe Tsai. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE.md file.
|
||||
|
||||
// +build gofuzz
|
||||
|
||||
// This file exists to export internal implementation details for fuzz testing.
|
||||
|
||||
package bzip2 |
||||
|
||||
func ForwardBWT(buf []byte) (ptr int) { |
||||
var bwt burrowsWheelerTransform |
||||
return bwt.Encode(buf) |
||||
} |
||||
|
||||
func ReverseBWT(buf []byte, ptr int) { |
||||
var bwt burrowsWheelerTransform |
||||
bwt.Decode(buf, ptr) |
||||
} |
||||
|
||||
type fuzzReader struct { |
||||
Checksums Checksums |
||||
} |
||||
|
||||
// updateChecksum updates Checksums.
|
||||
//
|
||||
// If a valid pos is provided, it appends the (pos, val) pair to the slice.
|
||||
// Otherwise, it will update the last record with the new value.
|
||||
func (fr *fuzzReader) updateChecksum(pos int64, val uint32) { |
||||
if pos >= 0 { |
||||
fr.Checksums = append(fr.Checksums, Checksum{pos, val}) |
||||
} else { |
||||
fr.Checksums[len(fr.Checksums)-1].Value = val |
||||
} |
||||
} |
||||
|
||||
type Checksum struct { |
||||
Offset int64 // Bit offset of the checksum
|
||||
Value uint32 // Checksum value
|
||||
} |
||||
|
||||
type Checksums []Checksum |
||||
|
||||
// Apply overwrites all checksum fields in d with the ones in cs.
|
||||
func (cs Checksums) Apply(d []byte) []byte { |
||||
d = append([]byte(nil), d...) |
||||
for _, c := range cs { |
||||
setU32(d, c.Offset, c.Value) |
||||
} |
||||
return d |
||||
} |
||||
|
||||
func setU32(d []byte, pos int64, val uint32) { |
||||
for i := uint(0); i < 32; i++ { |
||||
bpos := uint64(pos) + uint64(i) |
||||
d[bpos/8] &= ^byte(1 << (7 - bpos%8)) |
||||
d[bpos/8] |= byte(val>>(31-i)) << (7 - bpos%8) |
||||
} |
||||
} |
||||
|
||||
// Verify checks that all checksum fields in d matches those in cs.
|
||||
func (cs Checksums) Verify(d []byte) bool { |
||||
for _, c := range cs { |
||||
if getU32(d, c.Offset) != c.Value { |
||||
return false |
||||
} |
||||
} |
||||
return true |
||||
} |
||||
|
||||
func getU32(d []byte, pos int64) (val uint32) { |
||||
for i := uint(0); i < 32; i++ { |
||||
bpos := uint64(pos) + uint64(i) |
||||
val |= (uint32(d[bpos/8] >> (7 - bpos%8))) << (31 - i) |
||||
} |
||||
return val |
||||
} |
||||
@ -0,0 +1,28 @@ |
||||
// Copyright 2015, Joe Tsai. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE.md file.
|
||||
|
||||
// Package sais implements a linear time suffix array algorithm.
|
||||
package sais |
||||
|
||||
//go:generate go run sais_gen.go byte sais_byte.go
|
||||
//go:generate go run sais_gen.go int sais_int.go
|
||||
|
||||
// This package ports the C sais implementation by Yuta Mori. The ports are
|
||||
// located in sais_byte.go and sais_int.go, which are identical to each other
|
||||
// except for the types. Since Go does not support generics, we use generators to
|
||||
// create the two files.
|
||||
//
|
||||
// References:
|
||||
// https://sites.google.com/site/yuta256/sais
|
||||
// https://www.researchgate.net/publication/221313676_Linear_Time_Suffix_Array_Construction_Using_D-Critical_Substrings
|
||||
// https://www.researchgate.net/publication/224176324_Two_Efficient_Algorithms_for_Linear_Time_Suffix_Array_Construction
|
||||
|
||||
// ComputeSA computes the suffix array of t and places the result in sa.
|
||||
// Both t and sa must be the same length.
|
||||
func ComputeSA(t []byte, sa []int) { |
||||
if len(sa) != len(t) { |
||||
panic("mismatching sizes") |
||||
} |
||||
computeSA_byte(t, sa, 0, len(t), 256) |
||||
} |
||||
@ -0,0 +1,661 @@ |
||||
// Copyright 2015, Joe Tsai. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE.md file.
|
||||
|
||||
// Code generated by sais_gen.go. DO NOT EDIT.
|
||||
|
||||
// ====================================================
|
||||
// Copyright (c) 2008-2010 Yuta Mori All Rights Reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the "Software"), to deal in the Software without
|
||||
// restriction, including without limitation the rights to use,
|
||||
// copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following
|
||||
// conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be
|
||||
// included in all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||
// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
// OTHER DEALINGS IN THE SOFTWARE.
|
||||
// ====================================================
|
||||
|
||||
package sais |
||||
|
||||
func getCounts_byte(T []byte, C []int, n, k int) { |
||||
var i int |
||||
for i = 0; i < k; i++ { |
||||
C[i] = 0 |
||||
} |
||||
for i = 0; i < n; i++ { |
||||
C[T[i]]++ |
||||
} |
||||
} |
||||
|
||||
func getBuckets_byte(C, B []int, k int, end bool) { |
||||
var i, sum int |
||||
if end { |
||||
for i = 0; i < k; i++ { |
||||
sum += C[i] |
||||
B[i] = sum |
||||
} |
||||
} else { |
||||
for i = 0; i < k; i++ { |
||||
sum += C[i] |
||||
B[i] = sum - C[i] |
||||
} |
||||
} |
||||
} |
||||
|
||||
func sortLMS1_byte(T []byte, SA, C, B []int, n, k int) { |
||||
var b, i, j int |
||||
var c0, c1 int |
||||
|
||||
// Compute SAl.
|
||||
if &C[0] == &B[0] { |
||||
getCounts_byte(T, C, n, k) |
||||
} |
||||
getBuckets_byte(C, B, k, false) // Find starts of buckets
|
||||
j = n - 1 |
||||
c1 = int(T[j]) |
||||
b = B[c1] |
||||
j-- |
||||
if int(T[j]) < c1 { |
||||
SA[b] = ^j |
||||
} else { |
||||
SA[b] = j |
||||
} |
||||
b++ |
||||
for i = 0; i < n; i++ { |
||||
if j = SA[i]; j > 0 { |
||||
if c0 = int(T[j]); c0 != c1 { |
||||
B[c1] = b |
||||
c1 = c0 |
||||
b = B[c1] |
||||
} |
||||
j-- |
||||
if int(T[j]) < c1 { |
||||
SA[b] = ^j |
||||
} else { |
||||
SA[b] = j |
||||
} |
||||
b++ |
||||
SA[i] = 0 |
||||
} else if j < 0 { |
||||
SA[i] = ^j |
||||
} |
||||
} |
||||
|
||||
// Compute SAs.
|
||||
if &C[0] == &B[0] { |
||||
getCounts_byte(T, C, n, k) |
||||
} |
||||
getBuckets_byte(C, B, k, true) // Find ends of buckets
|
||||
c1 = 0 |
||||
b = B[c1] |
||||
for i = n - 1; i >= 0; i-- { |
||||
if j = SA[i]; j > 0 { |
||||
if c0 = int(T[j]); c0 != c1 { |
||||
B[c1] = b |
||||
c1 = c0 |
||||
b = B[c1] |
||||
} |
||||
j-- |
||||
b-- |
||||
if int(T[j]) > c1 { |
||||
SA[b] = ^(j + 1) |
||||
} else { |
||||
SA[b] = j |
||||
} |
||||
SA[i] = 0 |
||||
} |
||||
} |
||||
} |
||||
|
||||
func postProcLMS1_byte(T []byte, SA []int, n, m int) int { |
||||
var i, j, p, q, plen, qlen, name int |
||||
var c0, c1 int |
||||
var diff bool |
||||
|
||||
// Compact all the sorted substrings into the first m items of SA.
|
||||
// 2*m must be not larger than n (provable).
|
||||
for i = 0; SA[i] < 0; i++ { |
||||
SA[i] = ^SA[i] |
||||
} |
||||
if i < m { |
||||
for j, i = i, i+1; ; i++ { |
||||
if p = SA[i]; p < 0 { |
||||
SA[j] = ^p |
||||
j++ |
||||
SA[i] = 0 |
||||
if j == m { |
||||
break |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
// Store the length of all substrings.
|
||||
i = n - 1 |
||||
j = n - 1 |
||||
c0 = int(T[n-1]) |
||||
for { |
||||
c1 = c0 |
||||
if i--; i < 0 { |
||||
break |
||||
} |
||||
if c0 = int(T[i]); c0 < c1 { |
||||
break |
||||
} |
||||
} |
||||
for i >= 0 { |
||||
for { |
||||
c1 = c0 |
||||
if i--; i < 0 { |
||||
break |
||||
} |
||||
if c0 = int(T[i]); c0 > c1 { |
||||
break |
||||
} |
||||
} |
||||
if i >= 0 { |
||||
SA[m+((i+1)>>1)] = j - i |
||||
j = i + 1 |
||||
for { |
||||
c1 = c0 |
||||
if i--; i < 0 { |
||||
break |
||||
} |
||||
if c0 = int(T[i]); c0 < c1 { |
||||
break |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
// Find the lexicographic names of all substrings.
|
||||
name = 0 |
||||
qlen = 0 |
||||
for i, q = 0, n; i < m; i++ { |
||||
p = SA[i] |
||||
plen = SA[m+(p>>1)] |
||||
diff = true |
||||
if (plen == qlen) && ((q + plen) < n) { |
||||
for j = 0; (j < plen) && (T[p+j] == T[q+j]); j++ { |
||||
} |
||||
if j == plen { |
||||
diff = false |
||||
} |
||||
} |
||||
if diff { |
||||
name++ |
||||
q = p |
||||
qlen = plen |
||||
} |
||||
SA[m+(p>>1)] = name |
||||
} |
||||
return name |
||||
} |
||||
|
||||
func sortLMS2_byte(T []byte, SA, C, B, D []int, n, k int) { |
||||
var b, i, j, t, d int |
||||
var c0, c1 int |
||||
|
||||
// Compute SAl.
|
||||
getBuckets_byte(C, B, k, false) // Find starts of buckets
|
||||
j = n - 1 |
||||
c1 = int(T[j]) |
||||
b = B[c1] |
||||
j-- |
||||
if int(T[j]) < c1 { |
||||
t = 1 |
||||
} else { |
||||
t = 0 |
||||
} |
||||
j += n |
||||
if t&1 > 0 { |
||||
SA[b] = ^j |
||||
} else { |
||||
SA[b] = j |
||||
} |
||||
b++ |
||||
for i, d = 0, 0; i < n; i++ { |
||||
if j = SA[i]; j > 0 { |
||||
if n <= j { |
||||
d += 1 |
||||
j -= n |
||||
} |
||||
if c0 = int(T[j]); c0 != c1 { |
||||
B[c1] = b |
||||
c1 = c0 |
||||
b = B[c1] |
||||
} |
||||
j-- |
||||
t = int(c0) << 1 |
||||
if int(T[j]) < c1 { |
||||
t |= 1 |
||||
} |
||||
if D[t] != d { |
||||
j += n |
||||
D[t] = d |
||||
} |
||||
if t&1 > 0 { |
||||
SA[b] = ^j |
||||
} else { |
||||
SA[b] = j |
||||
} |
||||
b++ |
||||
SA[i] = 0 |
||||
} else if j < 0 { |
||||
SA[i] = ^j |
||||
} |
||||
} |
||||
for i = n - 1; 0 <= i; i-- { |
||||
if SA[i] > 0 { |
||||
if SA[i] < n { |
||||
SA[i] += n |
||||
for j = i - 1; SA[j] < n; j-- { |
||||
} |
||||
SA[j] -= n |
||||
i = j |
||||
} |
||||
} |
||||
} |
||||
|
||||
// Compute SAs.
|
||||
getBuckets_byte(C, B, k, true) // Find ends of buckets
|
||||
c1 = 0 |
||||
b = B[c1] |
||||
for i, d = n-1, d+1; i >= 0; i-- { |
||||
if j = SA[i]; j > 0 { |
||||
if n <= j { |
||||
d += 1 |
||||
j -= n |
||||
} |
||||
if c0 = int(T[j]); c0 != c1 { |
||||
B[c1] = b |
||||
c1 = c0 |
||||
b = B[c1] |
||||
} |
||||
j-- |
||||
t = int(c0) << 1 |
||||
if int(T[j]) > c1 { |
||||
t |= 1 |
||||
} |
||||
if D[t] != d { |
||||
j += n |
||||
D[t] = d |
||||
} |
||||
b-- |
||||
if t&1 > 0 { |
||||
SA[b] = ^(j + 1) |
||||
} else { |
||||
SA[b] = j |
||||
} |
||||
SA[i] = 0 |
||||
} |
||||
} |
||||
} |
||||
|
||||
func postProcLMS2_byte(SA []int, n, m int) int { |
||||
var i, j, d, name int |
||||
|
||||
// Compact all the sorted LMS substrings into the first m items of SA.
|
||||
name = 0 |
||||
for i = 0; SA[i] < 0; i++ { |
||||
j = ^SA[i] |
||||
if n <= j { |
||||
name += 1 |
||||
} |
||||
SA[i] = j |
||||
} |
||||
if i < m { |
||||
for d, i = i, i+1; ; i++ { |
||||
if j = SA[i]; j < 0 { |
||||
j = ^j |
||||
if n <= j { |
||||
name += 1 |
||||
} |
||||
SA[d] = j |
||||
d++ |
||||
SA[i] = 0 |
||||
if d == m { |
||||
break |
||||
} |
||||
} |
||||
} |
||||
} |
||||
if name < m { |
||||
// Store the lexicographic names.
|
||||
for i, d = m-1, name+1; 0 <= i; i-- { |
||||
if j = SA[i]; n <= j { |
||||
j -= n |
||||
d-- |
||||
} |
||||
SA[m+(j>>1)] = d |
||||
} |
||||
} else { |
||||
// Unset flags.
|
||||
for i = 0; i < m; i++ { |
||||
if j = SA[i]; n <= j { |
||||
j -= n |
||||
SA[i] = j |
||||
} |
||||
} |
||||
} |
||||
return name |
||||
} |
||||
|
||||
func induceSA_byte(T []byte, SA, C, B []int, n, k int) { |
||||
var b, i, j int |
||||
var c0, c1 int |
||||
|
||||
// Compute SAl.
|
||||
if &C[0] == &B[0] { |
||||
getCounts_byte(T, C, n, k) |
||||
} |
||||
getBuckets_byte(C, B, k, false) // Find starts of buckets
|
||||
j = n - 1 |
||||
c1 = int(T[j]) |
||||
b = B[c1] |
||||
if j > 0 && int(T[j-1]) < c1 { |
||||
SA[b] = ^j |
||||
} else { |
||||
SA[b] = j |
||||
} |
||||
b++ |
||||
for i = 0; i < n; i++ { |
||||
j = SA[i] |
||||
SA[i] = ^j |
||||
if j > 0 { |
||||
j-- |
||||
if c0 = int(T[j]); c0 != c1 { |
||||
B[c1] = b |
||||
c1 = c0 |
||||
b = B[c1] |
||||
} |
||||
if j > 0 && int(T[j-1]) < c1 { |
||||
SA[b] = ^j |
||||
} else { |
||||
SA[b] = j |
||||
} |
||||
b++ |
||||
} |
||||
} |
||||
|
||||
// Compute SAs.
|
||||
if &C[0] == &B[0] { |
||||
getCounts_byte(T, C, n, k) |
||||
} |
||||
getBuckets_byte(C, B, k, true) // Find ends of buckets
|
||||
c1 = 0 |
||||
b = B[c1] |
||||
for i = n - 1; i >= 0; i-- { |
||||
if j = SA[i]; j > 0 { |
||||
j-- |
||||
if c0 = int(T[j]); c0 != c1 { |
||||
B[c1] = b |
||||
c1 = c0 |
||||
b = B[c1] |
||||
} |
||||
b-- |
||||
if (j == 0) || (int(T[j-1]) > c1) { |
||||
SA[b] = ^j |
||||
} else { |
||||
SA[b] = j |
||||
} |
||||
} else { |
||||
SA[i] = ^j |
||||
} |
||||
} |
||||
} |
||||
|
||||
func computeSA_byte(T []byte, SA []int, fs, n, k int) { |
||||
const ( |
||||
minBucketSize = 512 |
||||
sortLMS2Limit = 0x3fffffff |
||||
) |
||||
|
||||
var C, B, D, RA []int |
||||
var bo int // Offset of B relative to SA
|
||||
var b, i, j, m, p, q, name, newfs int |
||||
var c0, c1 int |
||||
var flags uint |
||||
|
||||
if k <= minBucketSize { |
||||
C = make([]int, k) |
||||
if k <= fs { |
||||
bo = n + fs - k |
||||
B = SA[bo:] |
||||
flags = 1 |
||||
} else { |
||||
B = make([]int, k) |
||||
flags = 3 |
||||
} |
||||
} else if k <= fs { |
||||
C = SA[n+fs-k:] |
||||
if k <= fs-k { |
||||
bo = n + fs - 2*k |
||||
B = SA[bo:] |
||||
flags = 0 |
||||
} else if k <= 4*minBucketSize { |
||||
B = make([]int, k) |
||||
flags = 2 |
||||
} else { |
||||
B = C |
||||
flags = 8 |
||||
} |
||||
} else { |
||||
C = make([]int, k) |
||||
B = C |
||||
flags = 4 | 8 |
||||
} |
||||
if n <= sortLMS2Limit && 2 <= (n/k) { |
||||
if flags&1 > 0 { |
||||
if 2*k <= fs-k { |
||||
flags |= 32 |
||||
} else { |
||||
flags |= 16 |
||||
} |
||||
} else if flags == 0 && 2*k <= (fs-2*k) { |
||||
flags |= 32 |
||||
} |
||||
} |
||||
|
||||
// Stage 1: Reduce the problem by at least 1/2.
|
||||
// Sort all the LMS-substrings.
|
||||
getCounts_byte(T, C, n, k) |
||||
getBuckets_byte(C, B, k, true) // Find ends of buckets
|
||||
for i = 0; i < n; i++ { |
||||
SA[i] = 0 |
||||
} |
||||
b = -1 |
||||
i = n - 1 |
||||
j = n |
||||
m = 0 |
||||
c0 = int(T[n-1]) |
||||
for { |
||||
c1 = c0 |
||||
if i--; i < 0 { |
||||
break |
||||
} |
||||
if c0 = int(T[i]); c0 < c1 { |
||||
break |
||||
} |
||||
} |
||||
for i >= 0 { |
||||
for { |
||||
c1 = c0 |
||||
if i--; i < 0 { |
||||
break |
||||
} |
||||
if c0 = int(T[i]); c0 > c1 { |
||||
break |
||||
} |
||||
} |
||||
if i >= 0 { |
||||
if b >= 0 { |
||||
SA[b] = j |
||||
} |
||||
B[c1]-- |
||||
b = B[c1] |
||||
j = i |
||||
m++ |
||||
for { |
||||
c1 = c0 |
||||
if i--; i < 0 { |
||||
break |
||||
} |
||||
if c0 = int(T[i]); c0 < c1 { |
||||
break |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
if m > 1 { |
||||
if flags&(16|32) > 0 { |
||||
if flags&16 > 0 { |
||||
D = make([]int, 2*k) |
||||
} else { |
||||
D = SA[bo-2*k:] |
||||
} |
||||
B[T[j+1]]++ |
||||
for i, j = 0, 0; i < k; i++ { |
||||
j += C[i] |
||||
if B[i] != j { |
||||
SA[B[i]] += n |
||||
} |
||||
D[i] = 0 |
||||
D[i+k] = 0 |
||||
} |
||||
sortLMS2_byte(T, SA, C, B, D, n, k) |
||||
name = postProcLMS2_byte(SA, n, m) |
||||
} else { |
||||
sortLMS1_byte(T, SA, C, B, n, k) |
||||
name = postProcLMS1_byte(T, SA, n, m) |
||||
} |
||||
} else if m == 1 { |
||||
SA[b] = j + 1 |
||||
name = 1 |
||||
} else { |
||||
name = 0 |
||||
} |
||||
|
||||
// Stage 2: Solve the reduced problem.
|
||||
// Recurse if names are not yet unique.
|
||||
if name < m { |
||||
newfs = n + fs - 2*m |
||||
if flags&(1|4|8) == 0 { |
||||
if k+name <= newfs { |
||||
newfs -= k |
||||
} else { |
||||
flags |= 8 |
||||
} |
||||
} |
||||
RA = SA[m+newfs:] |
||||
for i, j = m+(n>>1)-1, m-1; m <= i; i-- { |
||||
if SA[i] != 0 { |
||||
RA[j] = SA[i] - 1 |
||||
j-- |
||||
} |
||||
} |
||||
computeSA_int(RA, SA, newfs, m, name) |
||||
|
||||
i = n - 1 |
||||
j = m - 1 |
||||
c0 = int(T[n-1]) |
||||
for { |
||||
c1 = c0 |
||||
if i--; i < 0 { |
||||
break |
||||
} |
||||
if c0 = int(T[i]); c0 < c1 { |
||||
break |
||||
} |
||||
} |
||||
for i >= 0 { |
||||
for { |
||||
c1 = c0 |
||||
if i--; i < 0 { |
||||
break |
||||
} |
||||
if c0 = int(T[i]); c0 > c1 { |
||||
break |
||||
} |
||||
} |
||||
if i >= 0 { |
||||
RA[j] = i + 1 |
||||
j-- |
||||
for { |
||||
c1 = c0 |
||||
if i--; i < 0 { |
||||
break |
||||
} |
||||
if c0 = int(T[i]); c0 < c1 { |
||||
break |
||||
} |
||||
} |
||||
} |
||||
} |
||||
for i = 0; i < m; i++ { |
||||
SA[i] = RA[SA[i]] |
||||
} |
||||
if flags&4 > 0 { |
||||
B = make([]int, k) |
||||
C = B |
||||
} |
||||
if flags&2 > 0 { |
||||
B = make([]int, k) |
||||
} |
||||
} |
||||
|
||||
// Stage 3: Induce the result for the original problem.
|
||||
if flags&8 > 0 { |
||||
getCounts_byte(T, C, n, k) |
||||
} |
||||
// Put all left-most S characters into their buckets.
|
||||
if m > 1 { |
||||
getBuckets_byte(C, B, k, true) // Find ends of buckets
|
||||
i = m - 1 |
||||
j = n |
||||
p = SA[m-1] |
||||
c1 = int(T[p]) |
||||
for { |
||||
c0 = c1 |
||||
q = B[c0] |
||||
for q < j { |
||||
j-- |
||||
SA[j] = 0 |
||||
} |
||||
for { |
||||
j-- |
||||
SA[j] = p |
||||
if i--; i < 0 { |
||||
break |
||||
} |
||||
p = SA[i] |
||||
if c1 = int(T[p]); c1 != c0 { |
||||
break |
||||
} |
||||
} |
||||
if i < 0 { |
||||
break |
||||
} |
||||
} |
||||
for j > 0 { |
||||
j-- |
||||
SA[j] = 0 |
||||
} |
||||
} |
||||
induceSA_byte(T, SA, C, B, n, k) |
||||
} |
||||
@ -0,0 +1,661 @@ |
||||
// Copyright 2015, Joe Tsai. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE.md file.
|
||||
|
||||
// Code generated by sais_gen.go. DO NOT EDIT.
|
||||
|
||||
// ====================================================
|
||||
// Copyright (c) 2008-2010 Yuta Mori All Rights Reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person
|
||||
// obtaining a copy of this software and associated documentation
|
||||
// files (the "Software"), to deal in the Software without
|
||||
// restriction, including without limitation the rights to use,
|
||||
// copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following
|
||||
// conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be
|
||||
// included in all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||
// OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
// HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
// WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
// OTHER DEALINGS IN THE SOFTWARE.
|
||||
// ====================================================
|
||||
|
||||
package sais |
||||
|
||||
func getCounts_int(T []int, C []int, n, k int) { |
||||
var i int |
||||
for i = 0; i < k; i++ { |
||||
C[i] = 0 |
||||
} |
||||
for i = 0; i < n; i++ { |
||||
C[T[i]]++ |
||||
} |
||||
} |
||||
|
||||
func getBuckets_int(C, B []int, k int, end bool) { |
||||
var i, sum int |
||||
if end { |
||||
for i = 0; i < k; i++ { |
||||
sum += C[i] |
||||
B[i] = sum |
||||
} |
||||
} else { |
||||
for i = 0; i < k; i++ { |
||||
sum += C[i] |
||||
B[i] = sum - C[i] |
||||
} |
||||
} |
||||
} |
||||
|
||||
func sortLMS1_int(T []int, SA, C, B []int, n, k int) { |
||||
var b, i, j int |
||||
var c0, c1 int |
||||
|
||||
// Compute SAl.
|
||||
if &C[0] == &B[0] { |
||||
getCounts_int(T, C, n, k) |
||||
} |
||||
getBuckets_int(C, B, k, false) // Find starts of buckets
|
||||
j = n - 1 |
||||
c1 = int(T[j]) |
||||
b = B[c1] |
||||
j-- |
||||
if int(T[j]) < c1 { |
||||
SA[b] = ^j |
||||
} else { |
||||
SA[b] = j |
||||
} |
||||
b++ |
||||
for i = 0; i < n; i++ { |
||||
if j = SA[i]; j > 0 { |
||||
if c0 = int(T[j]); c0 != c1 { |
||||
B[c1] = b |
||||
c1 = c0 |
||||
b = B[c1] |
||||
} |
||||
j-- |
||||
if int(T[j]) < c1 { |
||||
SA[b] = ^j |
||||
} else { |
||||
SA[b] = j |
||||
} |
||||
b++ |
||||
SA[i] = 0 |
||||
} else if j < 0 { |
||||
SA[i] = ^j |
||||
} |
||||
} |
||||
|
||||
// Compute SAs.
|
||||
if &C[0] == &B[0] { |
||||
getCounts_int(T, C, n, k) |
||||
} |
||||
getBuckets_int(C, B, k, true) // Find ends of buckets
|
||||
c1 = 0 |
||||
b = B[c1] |
||||
for i = n - 1; i >= 0; i-- { |
||||
if j = SA[i]; j > 0 { |
||||
if c0 = int(T[j]); c0 != c1 { |
||||
B[c1] = b |
||||
c1 = c0 |
||||
b = B[c1] |
||||
} |
||||
j-- |
||||
b-- |
||||
if int(T[j]) > c1 { |
||||
SA[b] = ^(j + 1) |
||||
} else { |
||||
SA[b] = j |
||||
} |
||||
SA[i] = 0 |
||||
} |
||||
} |
||||
} |
||||
|
||||
func postProcLMS1_int(T []int, SA []int, n, m int) int { |
||||
var i, j, p, q, plen, qlen, name int |
||||
var c0, c1 int |
||||
var diff bool |
||||
|
||||
// Compact all the sorted substrings into the first m items of SA.
|
||||
// 2*m must be not larger than n (provable).
|
||||
for i = 0; SA[i] < 0; i++ { |
||||
SA[i] = ^SA[i] |
||||
} |
||||
if i < m { |
||||
for j, i = i, i+1; ; i++ { |
||||
if p = SA[i]; p < 0 { |
||||
SA[j] = ^p |
||||
j++ |
||||
SA[i] = 0 |
||||
if j == m { |
||||
break |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
// Store the length of all substrings.
|
||||
i = n - 1 |
||||
j = n - 1 |
||||
c0 = int(T[n-1]) |
||||
for { |
||||
c1 = c0 |
||||
if i--; i < 0 { |
||||
break |
||||
} |
||||
if c0 = int(T[i]); c0 < c1 { |
||||
break |
||||
} |
||||
} |
||||
for i >= 0 { |
||||
for { |
||||
c1 = c0 |
||||
if i--; i < 0 { |
||||
break |
||||
} |
||||
if c0 = int(T[i]); c0 > c1 { |
||||
break |
||||
} |
||||
} |
||||
if i >= 0 { |
||||
SA[m+((i+1)>>1)] = j - i |
||||
j = i + 1 |
||||
for { |
||||
c1 = c0 |
||||
if i--; i < 0 { |
||||
break |
||||
} |
||||
if c0 = int(T[i]); c0 < c1 { |
||||
break |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
// Find the lexicographic names of all substrings.
|
||||
name = 0 |
||||
qlen = 0 |
||||
for i, q = 0, n; i < m; i++ { |
||||
p = SA[i] |
||||
plen = SA[m+(p>>1)] |
||||
diff = true |
||||
if (plen == qlen) && ((q + plen) < n) { |
||||
for j = 0; (j < plen) && (T[p+j] == T[q+j]); j++ { |
||||
} |
||||
if j == plen { |
||||
diff = false |
||||
} |
||||
} |
||||
if diff { |
||||
name++ |
||||
q = p |
||||
qlen = plen |
||||
} |
||||
SA[m+(p>>1)] = name |
||||
} |
||||
return name |
||||
} |
||||
|
||||
func sortLMS2_int(T []int, SA, C, B, D []int, n, k int) { |
||||
var b, i, j, t, d int |
||||
var c0, c1 int |
||||
|
||||
// Compute SAl.
|
||||
getBuckets_int(C, B, k, false) // Find starts of buckets
|
||||
j = n - 1 |
||||
c1 = int(T[j]) |
||||
b = B[c1] |
||||
j-- |
||||
if int(T[j]) < c1 { |
||||
t = 1 |
||||
} else { |
||||
t = 0 |
||||
} |
||||
j += n |
||||
if t&1 > 0 { |
||||
SA[b] = ^j |
||||
} else { |
||||
SA[b] = j |
||||
} |
||||
b++ |
||||
for i, d = 0, 0; i < n; i++ { |
||||
if j = SA[i]; j > 0 { |
||||
if n <= j { |
||||
d += 1 |
||||
j -= n |
||||
} |
||||
if c0 = int(T[j]); c0 != c1 { |
||||
B[c1] = b |
||||
c1 = c0 |
||||
b = B[c1] |
||||
} |
||||
j-- |
||||
t = int(c0) << 1 |
||||
if int(T[j]) < c1 { |
||||
t |= 1 |
||||
} |
||||
if D[t] != d { |
||||
j += n |
||||
D[t] = d |
||||
} |
||||
if t&1 > 0 { |
||||
SA[b] = ^j |
||||
} else { |
||||
SA[b] = j |
||||
} |
||||
b++ |
||||
SA[i] = 0 |
||||
} else if j < 0 { |
||||
SA[i] = ^j |
||||
} |
||||
} |
||||
for i = n - 1; 0 <= i; i-- { |
||||
if SA[i] > 0 { |
||||
if SA[i] < n { |
||||
SA[i] += n |
||||
for j = i - 1; SA[j] < n; j-- { |
||||
} |
||||
SA[j] -= n |
||||
i = j |
||||
} |
||||
} |
||||
} |
||||
|
||||
// Compute SAs.
|
||||
getBuckets_int(C, B, k, true) // Find ends of buckets
|
||||
c1 = 0 |
||||
b = B[c1] |
||||
for i, d = n-1, d+1; i >= 0; i-- { |
||||
if j = SA[i]; j > 0 { |
||||
if n <= j { |
||||
d += 1 |
||||
j -= n |
||||
} |
||||
if c0 = int(T[j]); c0 != c1 { |
||||
B[c1] = b |
||||
c1 = c0 |
||||
b = B[c1] |
||||
} |
||||
j-- |
||||
t = int(c0) << 1 |
||||
if int(T[j]) > c1 { |
||||
t |= 1 |
||||
} |
||||
if D[t] != d { |
||||
j += n |
||||
D[t] = d |
||||
} |
||||
b-- |
||||
if t&1 > 0 { |
||||
SA[b] = ^(j + 1) |
||||
} else { |
||||
SA[b] = j |
||||
} |
||||
SA[i] = 0 |
||||
} |
||||
} |
||||
} |
||||
|
||||
func postProcLMS2_int(SA []int, n, m int) int { |
||||
var i, j, d, name int |
||||
|
||||
// Compact all the sorted LMS substrings into the first m items of SA.
|
||||
name = 0 |
||||
for i = 0; SA[i] < 0; i++ { |
||||
j = ^SA[i] |
||||
if n <= j { |
||||
name += 1 |
||||
} |
||||
SA[i] = j |
||||
} |
||||
if i < m { |
||||
for d, i = i, i+1; ; i++ { |
||||
if j = SA[i]; j < 0 { |
||||
j = ^j |
||||
if n <= j { |
||||
name += 1 |
||||
} |
||||
SA[d] = j |
||||
d++ |
||||
SA[i] = 0 |
||||
if d == m { |
||||
break |
||||
} |
||||
} |
||||
} |
||||
} |
||||
if name < m { |
||||
// Store the lexicographic names.
|
||||
for i, d = m-1, name+1; 0 <= i; i-- { |
||||
if j = SA[i]; n <= j { |
||||
j -= n |
||||
d-- |
||||
} |
||||
SA[m+(j>>1)] = d |
||||
} |
||||
} else { |
||||
// Unset flags.
|
||||
for i = 0; i < m; i++ { |
||||
if j = SA[i]; n <= j { |
||||
j -= n |
||||
SA[i] = j |
||||
} |
||||
} |
||||
} |
||||
return name |
||||
} |
||||
|
||||
func induceSA_int(T []int, SA, C, B []int, n, k int) { |
||||
var b, i, j int |
||||
var c0, c1 int |
||||
|
||||
// Compute SAl.
|
||||
if &C[0] == &B[0] { |
||||
getCounts_int(T, C, n, k) |
||||
} |
||||
getBuckets_int(C, B, k, false) // Find starts of buckets
|
||||
j = n - 1 |
||||
c1 = int(T[j]) |
||||
b = B[c1] |
||||
if j > 0 && int(T[j-1]) < c1 { |
||||
SA[b] = ^j |
||||
} else { |
||||
SA[b] = j |
||||
} |
||||
b++ |
||||
for i = 0; i < n; i++ { |
||||
j = SA[i] |
||||
SA[i] = ^j |
||||
if j > 0 { |
||||
j-- |
||||
if c0 = int(T[j]); c0 != c1 { |
||||
B[c1] = b |
||||
c1 = c0 |
||||
b = B[c1] |
||||
} |
||||
if j > 0 && int(T[j-1]) < c1 { |
||||
SA[b] = ^j |
||||
} else { |
||||
SA[b] = j |
||||
} |
||||
b++ |
||||
} |
||||
} |
||||
|
||||
// Compute SAs.
|
||||
if &C[0] == &B[0] { |
||||
getCounts_int(T, C, n, k) |
||||
} |
||||
getBuckets_int(C, B, k, true) // Find ends of buckets
|
||||
c1 = 0 |
||||
b = B[c1] |
||||
for i = n - 1; i >= 0; i-- { |
||||
if j = SA[i]; j > 0 { |
||||
j-- |
||||
if c0 = int(T[j]); c0 != c1 { |
||||
B[c1] = b |
||||
c1 = c0 |
||||
b = B[c1] |
||||
} |
||||
b-- |
||||
if (j == 0) || (int(T[j-1]) > c1) { |
||||
SA[b] = ^j |
||||
} else { |
||||
SA[b] = j |
||||
} |
||||
} else { |
||||
SA[i] = ^j |
||||
} |
||||
} |
||||
} |
||||
|
||||
func computeSA_int(T []int, SA []int, fs, n, k int) { |
||||
const ( |
||||
minBucketSize = 512 |
||||
sortLMS2Limit = 0x3fffffff |
||||
) |
||||
|
||||
var C, B, D, RA []int |
||||
var bo int // Offset of B relative to SA
|
||||
var b, i, j, m, p, q, name, newfs int |
||||
var c0, c1 int |
||||
var flags uint |
||||
|
||||
if k <= minBucketSize { |
||||
C = make([]int, k) |
||||
if k <= fs { |
||||
bo = n + fs - k |
||||
B = SA[bo:] |
||||
flags = 1 |
||||
} else { |
||||
B = make([]int, k) |
||||
flags = 3 |
||||
} |
||||
} else if k <= fs { |
||||
C = SA[n+fs-k:] |
||||
if k <= fs-k { |
||||
bo = n + fs - 2*k |
||||
B = SA[bo:] |
||||
flags = 0 |
||||
} else if k <= 4*minBucketSize { |
||||
B = make([]int, k) |
||||
flags = 2 |
||||
} else { |
||||
B = C |
||||
flags = 8 |
||||
} |
||||
} else { |
||||
C = make([]int, k) |
||||
B = C |
||||
flags = 4 | 8 |
||||
} |
||||
if n <= sortLMS2Limit && 2 <= (n/k) { |
||||
if flags&1 > 0 { |
||||
if 2*k <= fs-k { |
||||
flags |= 32 |
||||
} else { |
||||
flags |= 16 |
||||
} |
||||
} else if flags == 0 && 2*k <= (fs-2*k) { |
||||
flags |= 32 |
||||
} |
||||
} |
||||
|
||||
// Stage 1: Reduce the problem by at least 1/2.
|
||||
// Sort all the LMS-substrings.
|
||||
getCounts_int(T, C, n, k) |
||||
getBuckets_int(C, B, k, true) // Find ends of buckets
|
||||
for i = 0; i < n; i++ { |
||||
SA[i] = 0 |
||||
} |
||||
b = -1 |
||||
i = n - 1 |
||||
j = n |
||||
m = 0 |
||||
c0 = int(T[n-1]) |
||||
for { |
||||
c1 = c0 |
||||
if i--; i < 0 { |
||||
break |
||||
} |
||||
if c0 = int(T[i]); c0 < c1 { |
||||
break |
||||
} |
||||
} |
||||
for i >= 0 { |
||||
for { |
||||
c1 = c0 |
||||
if i--; i < 0 { |
||||
break |
||||
} |
||||
if c0 = int(T[i]); c0 > c1 { |
||||
break |
||||
} |
||||
} |
||||
if i >= 0 { |
||||
if b >= 0 { |
||||
SA[b] = j |
||||
} |
||||
B[c1]-- |
||||
b = B[c1] |
||||
j = i |
||||
m++ |
||||
for { |
||||
c1 = c0 |
||||
if i--; i < 0 { |
||||
break |
||||
} |
||||
if c0 = int(T[i]); c0 < c1 { |
||||
break |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
if m > 1 { |
||||
if flags&(16|32) > 0 { |
||||
if flags&16 > 0 { |
||||
D = make([]int, 2*k) |
||||
} else { |
||||
D = SA[bo-2*k:] |
||||
} |
||||
B[T[j+1]]++ |
||||
for i, j = 0, 0; i < k; i++ { |
||||
j += C[i] |
||||
if B[i] != j { |
||||
SA[B[i]] += n |
||||
} |
||||
D[i] = 0 |
||||
D[i+k] = 0 |
||||
} |
||||
sortLMS2_int(T, SA, C, B, D, n, k) |
||||
name = postProcLMS2_int(SA, n, m) |
||||
} else { |
||||
sortLMS1_int(T, SA, C, B, n, k) |
||||
name = postProcLMS1_int(T, SA, n, m) |
||||
} |
||||
} else if m == 1 { |
||||
SA[b] = j + 1 |
||||
name = 1 |
||||
} else { |
||||
name = 0 |
||||
} |
||||
|
||||
// Stage 2: Solve the reduced problem.
|
||||
// Recurse if names are not yet unique.
|
||||
if name < m { |
||||
newfs = n + fs - 2*m |
||||
if flags&(1|4|8) == 0 { |
||||
if k+name <= newfs { |
||||
newfs -= k |
||||
} else { |
||||
flags |= 8 |
||||
} |
||||
} |
||||
RA = SA[m+newfs:] |
||||
for i, j = m+(n>>1)-1, m-1; m <= i; i-- { |
||||
if SA[i] != 0 { |
||||
RA[j] = SA[i] - 1 |
||||
j-- |
||||
} |
||||
} |
||||
computeSA_int(RA, SA, newfs, m, name) |
||||
|
||||
i = n - 1 |
||||
j = m - 1 |
||||
c0 = int(T[n-1]) |
||||
for { |
||||
c1 = c0 |
||||
if i--; i < 0 { |
||||
break |
||||
} |
||||
if c0 = int(T[i]); c0 < c1 { |
||||
break |
||||
} |
||||
} |
||||
for i >= 0 { |
||||
for { |
||||
c1 = c0 |
||||
if i--; i < 0 { |
||||
break |
||||
} |
||||
if c0 = int(T[i]); c0 > c1 { |
||||
break |
||||
} |
||||
} |
||||
if i >= 0 { |
||||
RA[j] = i + 1 |
||||
j-- |
||||
for { |
||||
c1 = c0 |
||||
if i--; i < 0 { |
||||
break |
||||
} |
||||
if c0 = int(T[i]); c0 < c1 { |
||||
break |
||||
} |
||||
} |
||||
} |
||||
} |
||||
for i = 0; i < m; i++ { |
||||
SA[i] = RA[SA[i]] |
||||
} |
||||
if flags&4 > 0 { |
||||
B = make([]int, k) |
||||
C = B |
||||
} |
||||
if flags&2 > 0 { |
||||
B = make([]int, k) |
||||
} |
||||
} |
||||
|
||||
// Stage 3: Induce the result for the original problem.
|
||||
if flags&8 > 0 { |
||||
getCounts_int(T, C, n, k) |
||||
} |
||||
// Put all left-most S characters into their buckets.
|
||||
if m > 1 { |
||||
getBuckets_int(C, B, k, true) // Find ends of buckets
|
||||
i = m - 1 |
||||
j = n |
||||
p = SA[m-1] |
||||
c1 = int(T[p]) |
||||
for { |
||||
c0 = c1 |
||||
q = B[c0] |
||||
for q < j { |
||||
j-- |
||||
SA[j] = 0 |
||||
} |
||||
for { |
||||
j-- |
||||
SA[j] = p |
||||
if i--; i < 0 { |
||||
break |
||||
} |
||||
p = SA[i] |
||||
if c1 = int(T[p]); c1 != c0 { |
||||
break |
||||
} |
||||
} |
||||
if i < 0 { |
||||
break |
||||
} |
||||
} |
||||
for j > 0 { |
||||
j-- |
||||
SA[j] = 0 |
||||
} |
||||
} |
||||
induceSA_int(T, SA, C, B, n, k) |
||||
} |
||||
@ -0,0 +1,131 @@ |
||||
// Copyright 2015, Joe Tsai. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE.md file.
|
||||
|
||||
package bzip2 |
||||
|
||||
import "github.com/dsnet/compress/internal/errors" |
||||
|
||||
// moveToFront implements both the MTF and RLE stages of bzip2 at the same time.
|
||||
// Any runs of zeros in the encoded output will be replaced by a sequence of
|
||||
// RUNA and RUNB symbols are encode the length of the run.
|
||||
//
|
||||
// The RLE encoding used can actually be encoded to and decoded from using
|
||||
// normal two's complement arithmetic. The methodology for doing so is below.
|
||||
//
|
||||
// Assuming the following:
|
||||
// num: The value being encoded by RLE encoding.
|
||||
// run: A sequence of RUNA and RUNB symbols represented as a binary integer,
|
||||
// where RUNA is the 0 bit, RUNB is the 1 bit, and least-significant RUN
|
||||
// symbols are at the least-significant bit positions.
|
||||
// cnt: The number of RUNA and RUNB symbols.
|
||||
//
|
||||
// Then the RLE encoding used by bzip2 has this mathematical property:
|
||||
// num+1 == (1<<cnt) | run
|
||||
type moveToFront struct { |
||||
dictBuf [256]uint8 |
||||
dictLen int |
||||
|
||||
vals []byte |
||||
syms []uint16 |
||||
blkSize int |
||||
} |
||||
|
||||
func (mtf *moveToFront) Init(dict []uint8, blkSize int) { |
||||
if len(dict) > len(mtf.dictBuf) { |
||||
panicf(errors.Internal, "alphabet too large") |
||||
} |
||||
copy(mtf.dictBuf[:], dict) |
||||
mtf.dictLen = len(dict) |
||||
mtf.blkSize = blkSize |
||||
} |
||||
|
||||
func (mtf *moveToFront) Encode(vals []byte) (syms []uint16) { |
||||
dict := mtf.dictBuf[:mtf.dictLen] |
||||
syms = mtf.syms[:0] |
||||
|
||||
if len(vals) > mtf.blkSize { |
||||
panicf(errors.Internal, "exceeded block size") |
||||
} |
||||
|
||||
var lastNum uint32 |
||||
for _, val := range vals { |
||||
// Normal move-to-front transform.
|
||||
var idx uint8 // Reverse lookup idx in dict
|
||||
for di, dv := range dict { |
||||
if dv == val { |
||||
idx = uint8(di) |
||||
break |
||||
} |
||||
} |
||||
copy(dict[1:], dict[:idx]) |
||||
dict[0] = val |
||||
|
||||
// Run-length encoding augmentation.
|
||||
if idx == 0 { |
||||
lastNum++ |
||||
continue |
||||
} |
||||
if lastNum > 0 { |
||||
for rc := lastNum + 1; rc != 1; rc >>= 1 { |
||||
syms = append(syms, uint16(rc&1)) |
||||
} |
||||
lastNum = 0 |
||||
} |
||||
syms = append(syms, uint16(idx)+1) |
||||
} |
||||
if lastNum > 0 { |
||||
for rc := lastNum + 1; rc != 1; rc >>= 1 { |
||||
syms = append(syms, uint16(rc&1)) |
||||
} |
||||
} |
||||
mtf.syms = syms |
||||
return syms |
||||
} |
||||
|
||||
func (mtf *moveToFront) Decode(syms []uint16) (vals []byte) { |
||||
dict := mtf.dictBuf[:mtf.dictLen] |
||||
vals = mtf.vals[:0] |
||||
|
||||
var lastCnt uint |
||||
var lastRun uint32 |
||||
for _, sym := range syms { |
||||
// Run-length encoding augmentation.
|
||||
if sym < 2 { |
||||
lastRun |= uint32(sym) << lastCnt |
||||
lastCnt++ |
||||
continue |
||||
} |
||||
if lastCnt > 0 { |
||||
cnt := int((1<<lastCnt)|lastRun) - 1 |
||||
if len(vals)+cnt > mtf.blkSize || lastCnt > 24 { |
||||
panicf(errors.Corrupted, "run-length decoding exceeded block size") |
||||
} |
||||
for i := cnt; i > 0; i-- { |
||||
vals = append(vals, dict[0]) |
||||
} |
||||
lastCnt, lastRun = 0, 0 |
||||
} |
||||
|
||||
// Normal move-to-front transform.
|
||||
val := dict[sym-1] // Forward lookup val in dict
|
||||
copy(dict[1:], dict[:sym-1]) |
||||
dict[0] = val |
||||
|
||||
if len(vals) >= mtf.blkSize { |
||||
panicf(errors.Corrupted, "run-length decoding exceeded block size") |
||||
} |
||||
vals = append(vals, val) |
||||
} |
||||
if lastCnt > 0 { |
||||
cnt := int((1<<lastCnt)|lastRun) - 1 |
||||
if len(vals)+cnt > mtf.blkSize || lastCnt > 24 { |
||||
panicf(errors.Corrupted, "run-length decoding exceeded block size") |
||||
} |
||||
for i := cnt; i > 0; i-- { |
||||
vals = append(vals, dict[0]) |
||||
} |
||||
} |
||||
mtf.vals = vals |
||||
return vals |
||||
} |
||||
@ -0,0 +1,374 @@ |
||||
// Copyright 2015, Joe Tsai. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE.md file.
|
||||
|
||||
package bzip2 |
||||
|
||||
import ( |
||||
"io" |
||||
|
||||
"github.com/dsnet/compress/internal" |
||||
"github.com/dsnet/compress/internal/errors" |
||||
"github.com/dsnet/compress/internal/prefix" |
||||
) |
||||
|
||||
const ( |
||||
minNumTrees = 2 |
||||
maxNumTrees = 6 |
||||
|
||||
maxPrefixBits = 20 // Maximum bit-width of a prefix code
|
||||
maxNumSyms = 256 + 2 // Maximum number of symbols in the alphabet
|
||||
numBlockSyms = 50 // Number of bytes in a block
|
||||
) |
||||
|
||||
// encSel and decSel are used to handle the prefix encoding for tree selectors.
|
||||
// The prefix encoding is as follows:
|
||||
//
|
||||
// Code TreeIdx
|
||||
// 0 <=> 0
|
||||
// 10 <=> 1
|
||||
// 110 <=> 2
|
||||
// 1110 <=> 3
|
||||
// 11110 <=> 4
|
||||
// 111110 <=> 5
|
||||
// 111111 <=> 6 Invalid tree index, so should fail
|
||||
//
|
||||
var encSel, decSel = func() (e prefix.Encoder, d prefix.Decoder) { |
||||
var selCodes [maxNumTrees + 1]prefix.PrefixCode |
||||
for i := range selCodes { |
||||
selCodes[i] = prefix.PrefixCode{Sym: uint32(i), Len: uint32(i + 1)} |
||||
} |
||||
selCodes[maxNumTrees] = prefix.PrefixCode{Sym: maxNumTrees, Len: maxNumTrees} |
||||
prefix.GeneratePrefixes(selCodes[:]) |
||||
e.Init(selCodes[:]) |
||||
d.Init(selCodes[:]) |
||||
return |
||||
}() |
||||
|
||||
type prefixReader struct{ prefix.Reader } |
||||
|
||||
func (pr *prefixReader) Init(r io.Reader) { |
||||
pr.Reader.Init(r, true) |
||||
} |
||||
|
||||
func (pr *prefixReader) ReadBitsBE64(nb uint) uint64 { |
||||
if nb <= 32 { |
||||
v := uint32(pr.ReadBits(nb)) |
||||
return uint64(internal.ReverseUint32N(v, nb)) |
||||
} |
||||
v0 := internal.ReverseUint32(uint32(pr.ReadBits(32))) |
||||
v1 := internal.ReverseUint32(uint32(pr.ReadBits(nb - 32))) |
||||
v := uint64(v0)<<32 | uint64(v1) |
||||
return v >> (64 - nb) |
||||
} |
||||
|
||||
func (pr *prefixReader) ReadPrefixCodes(codes []prefix.PrefixCodes, trees []prefix.Decoder) { |
||||
for i, pc := range codes { |
||||
clen := int(pr.ReadBitsBE64(5)) |
||||
sum := 1 << maxPrefixBits |
||||
for sym := range pc { |
||||
for { |
||||
if clen < 1 || clen > maxPrefixBits { |
||||
panicf(errors.Corrupted, "invalid prefix bit-length: %d", clen) |
||||
} |
||||
|
||||
b, ok := pr.TryReadBits(1) |
||||
if !ok { |
||||
b = pr.ReadBits(1) |
||||
} |
||||
if b == 0 { |
||||
break |
||||
} |
||||
|
||||
b, ok = pr.TryReadBits(1) |
||||
if !ok { |
||||
b = pr.ReadBits(1) |
||||
} |
||||
clen -= int(b*2) - 1 // +1 or -1
|
||||
} |
||||
pc[sym] = prefix.PrefixCode{Sym: uint32(sym), Len: uint32(clen)} |
||||
sum -= (1 << maxPrefixBits) >> uint(clen) |
||||
} |
||||
|
||||
if sum == 0 { |
||||
// Fast path, but only handles complete trees.
|
||||
if err := prefix.GeneratePrefixes(pc); err != nil { |
||||
errors.Panic(err) // Using complete trees; should never fail
|
||||
} |
||||
} else { |
||||
// Slow path, but handles anything.
|
||||
pc = handleDegenerateCodes(pc) // Never fails, but may fail later
|
||||
codes[i] = pc |
||||
} |
||||
trees[i].Init(pc) |
||||
} |
||||
} |
||||
|
||||
type prefixWriter struct{ prefix.Writer } |
||||
|
||||
func (pw *prefixWriter) Init(w io.Writer) { |
||||
pw.Writer.Init(w, true) |
||||
} |
||||
|
||||
func (pw *prefixWriter) WriteBitsBE64(v uint64, nb uint) { |
||||
if nb <= 32 { |
||||
v := internal.ReverseUint32N(uint32(v), nb) |
||||
pw.WriteBits(uint(v), nb) |
||||
return |
||||
} |
||||
v <<= (64 - nb) |
||||
v0 := internal.ReverseUint32(uint32(v >> 32)) |
||||
v1 := internal.ReverseUint32(uint32(v)) |
||||
pw.WriteBits(uint(v0), 32) |
||||
pw.WriteBits(uint(v1), nb-32) |
||||
return |
||||
} |
||||
|
||||
func (pw *prefixWriter) WritePrefixCodes(codes []prefix.PrefixCodes, trees []prefix.Encoder) { |
||||
for i, pc := range codes { |
||||
if err := prefix.GeneratePrefixes(pc); err != nil { |
||||
errors.Panic(err) // Using complete trees; should never fail
|
||||
} |
||||
trees[i].Init(pc) |
||||
|
||||
clen := int(pc[0].Len) |
||||
pw.WriteBitsBE64(uint64(clen), 5) |
||||
for _, c := range pc { |
||||
for int(c.Len) < clen { |
||||
pw.WriteBits(3, 2) // 11
|
||||
clen-- |
||||
} |
||||
for int(c.Len) > clen { |
||||
pw.WriteBits(1, 2) // 10
|
||||
clen++ |
||||
} |
||||
pw.WriteBits(0, 1) |
||||
} |
||||
} |
||||
} |
||||
|
||||
// handleDegenerateCodes converts a degenerate tree into a canonical tree.
|
||||
//
|
||||
// For example, when the input is an under-subscribed tree:
|
||||
// input: []PrefixCode{
|
||||
// {Sym: 0, Len: 3},
|
||||
// {Sym: 1, Len: 4},
|
||||
// {Sym: 2, Len: 3},
|
||||
// }
|
||||
// output: []PrefixCode{
|
||||
// {Sym: 0, Len: 3, Val: 0}, // 000
|
||||
// {Sym: 1, Len: 4, Val: 2}, // 0010
|
||||
// {Sym: 2, Len: 3, Val: 4}, // 100
|
||||
// {Sym: 258, Len: 4, Val: 10}, // 1010
|
||||
// {Sym: 259, Len: 3, Val: 6}, // 110
|
||||
// {Sym: 260, Len: 1, Val: 1}, // 1
|
||||
// }
|
||||
//
|
||||
// For example, when the input is an over-subscribed tree:
|
||||
// input: []PrefixCode{
|
||||
// {Sym: 0, Len: 1},
|
||||
// {Sym: 1, Len: 3},
|
||||
// {Sym: 2, Len: 4},
|
||||
// {Sym: 3, Len: 3},
|
||||
// {Sym: 4, Len: 2},
|
||||
// }
|
||||
// output: []PrefixCode{
|
||||
// {Sym: 0, Len: 1, Val: 0}, // 0
|
||||
// {Sym: 1, Len: 3, Val: 3}, // 011
|
||||
// {Sym: 3, Len: 3, Val: 7}, // 111
|
||||
// {Sym: 4, Len: 2, Val: 1}, // 01
|
||||
// }
|
||||
func handleDegenerateCodes(codes prefix.PrefixCodes) prefix.PrefixCodes { |
||||
// Since there is no formal definition for the BZip2 format, there is no
|
||||
// specification that says that the code lengths must form a complete
|
||||
// prefix tree (IE: it is neither over-subscribed nor under-subscribed).
|
||||
// Thus, the original C implementation becomes the reference for how prefix
|
||||
// decoding is done in these edge cases. Unfortunately, the C version does
|
||||
// not error when an invalid tree is used, but rather allows decoding to
|
||||
// continue and only errors if some bit pattern happens to cause an error.
|
||||
// Thus, it is possible for an invalid tree to end up decoding an input
|
||||
// "properly" so long as invalid bit patterns are not present. In order to
|
||||
// replicate this non-specified behavior, we use a ported version of the
|
||||
// C code to generate the codes as a valid canonical tree by substituting
|
||||
// invalid nodes with invalid symbols.
|
||||
//
|
||||
// ====================================================
|
||||
// This program, "bzip2", the associated library "libbzip2", and all
|
||||
// documentation, are copyright (C) 1996-2010 Julian R Seward. All
|
||||
// rights reserved.
|
||||
//
|
||||
// Redistribution and use in source and binary forms, with or without
|
||||
// modification, are permitted provided that the following conditions
|
||||
// are met:
|
||||
//
|
||||
// 1. Redistributions of source code must retain the above copyright
|
||||
// notice, this list of conditions and the following disclaimer.
|
||||
//
|
||||
// 2. The origin of this software must not be misrepresented; you must
|
||||
// not claim that you wrote the original software. If you use this
|
||||
// software in a product, an acknowledgment in the product
|
||||
// documentation would be appreciated but is not required.
|
||||
//
|
||||
// 3. Altered source versions must be plainly marked as such, and must
|
||||
// not be misrepresented as being the original software.
|
||||
//
|
||||
// 4. The name of the author may not be used to endorse or promote
|
||||
// products derived from this software without specific prior written
|
||||
// permission.
|
||||
//
|
||||
// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
|
||||
// OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
// ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
|
||||
// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
|
||||
// GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
|
||||
// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
|
||||
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
//
|
||||
// Julian Seward, jseward@bzip.org
|
||||
// bzip2/libbzip2 version 1.0.6 of 6 September 2010
|
||||
// ====================================================
|
||||
var ( |
||||
limits [maxPrefixBits + 2]int32 |
||||
bases [maxPrefixBits + 2]int32 |
||||
perms [maxNumSyms]int32 |
||||
|
||||
minLen = uint32(maxPrefixBits) |
||||
maxLen = uint32(0) |
||||
) |
||||
|
||||
const ( |
||||
statusOkay = iota |
||||
statusInvalid |
||||
statusNeedBits |
||||
statusMaxBits |
||||
) |
||||
|
||||
// createTables is the BZ2_hbCreateDecodeTables function from the C code.
|
||||
createTables := func(codes []prefix.PrefixCode) { |
||||
for _, c := range codes { |
||||
if c.Len > maxLen { |
||||
maxLen = c.Len |
||||
} |
||||
if c.Len < minLen { |
||||
minLen = c.Len |
||||
} |
||||
} |
||||
|
||||
var pp int |
||||
for i := minLen; i <= maxLen; i++ { |
||||
for j, c := range codes { |
||||
if c.Len == i { |
||||
perms[pp] = int32(j) |
||||
pp++ |
||||
} |
||||
} |
||||
} |
||||
|
||||
var vec int32 |
||||
for _, c := range codes { |
||||
bases[c.Len+1]++ |
||||
} |
||||
for i := 1; i < len(bases); i++ { |
||||
bases[i] += bases[i-1] |
||||
} |
||||
for i := minLen; i <= maxLen; i++ { |
||||
vec += bases[i+1] - bases[i] |
||||
limits[i] = vec - 1 |
||||
vec <<= 1 |
||||
} |
||||
for i := minLen + 1; i <= maxLen; i++ { |
||||
bases[i] = ((limits[i-1] + 1) << 1) - bases[i] |
||||
} |
||||
} |
||||
|
||||
// getSymbol is the GET_MTF_VAL macro from the C code.
|
||||
getSymbol := func(c prefix.PrefixCode) (uint32, int) { |
||||
v := internal.ReverseUint32(c.Val) |
||||
n := c.Len |
||||
|
||||
zn := minLen |
||||
if zn > n { |
||||
return 0, statusNeedBits |
||||
} |
||||
zvec := int32(v >> (32 - zn)) |
||||
v <<= zn |
||||
for { |
||||
if zn > maxLen { |
||||
return 0, statusMaxBits |
||||
} |
||||
if zvec <= limits[zn] { |
||||
break |
||||
} |
||||
zn++ |
||||
if zn > n { |
||||
return 0, statusNeedBits |
||||
} |
||||
zvec = (zvec << 1) | int32(v>>31) |
||||
v <<= 1 |
||||
} |
||||
if zvec-bases[zn] < 0 || zvec-bases[zn] >= maxNumSyms { |
||||
return 0, statusInvalid |
||||
} |
||||
return uint32(perms[zvec-bases[zn]]), statusOkay |
||||
} |
||||
|
||||
// Step 1: Create the prefix trees using the C algorithm.
|
||||
createTables(codes) |
||||
|
||||
// Step 2: Starting with the shortest bit pattern, explore the whole tree.
|
||||
// If tree is under-subscribed, the worst-case runtime is O(1<<maxLen).
|
||||
// If tree is over-subscribed, the worst-case runtime is O(maxNumSyms).
|
||||
var pcodesArr [2 * maxNumSyms]prefix.PrefixCode |
||||
pcodes := pcodesArr[:maxNumSyms] |
||||
var exploreCode func(prefix.PrefixCode) bool |
||||
exploreCode = func(c prefix.PrefixCode) (term bool) { |
||||
sym, status := getSymbol(c) |
||||
switch status { |
||||
case statusOkay: |
||||
// This code is valid, so insert it.
|
||||
c.Sym = sym |
||||
pcodes[sym] = c |
||||
term = true |
||||
case statusInvalid: |
||||
// This code is invalid, so insert an invalid symbol.
|
||||
c.Sym = uint32(len(pcodes)) |
||||
pcodes = append(pcodes, c) |
||||
term = true |
||||
case statusNeedBits: |
||||
// This code is too short, so explore both children.
|
||||
c.Len++ |
||||
c0, c1 := c, c |
||||
c1.Val |= 1 << (c.Len - 1) |
||||
|
||||
b0 := exploreCode(c0) |
||||
b1 := exploreCode(c1) |
||||
switch { |
||||
case !b0 && b1: |
||||
c0.Sym = uint32(len(pcodes)) |
||||
pcodes = append(pcodes, c0) |
||||
case !b1 && b0: |
||||
c1.Sym = uint32(len(pcodes)) |
||||
pcodes = append(pcodes, c1) |
||||
} |
||||
term = b0 || b1 |
||||
case statusMaxBits: |
||||
// This code is too long, so report it upstream.
|
||||
term = false |
||||
} |
||||
return term // Did this code terminate?
|
||||
} |
||||
exploreCode(prefix.PrefixCode{}) |
||||
|
||||
// Step 3: Copy new sparse codes to old output codes.
|
||||
codes = codes[:0] |
||||
for _, c := range pcodes { |
||||
if c.Len > 0 { |
||||
codes = append(codes, c) |
||||
} |
||||
} |
||||
return codes |
||||
} |
||||
@ -0,0 +1,274 @@ |
||||
// Copyright 2015, Joe Tsai. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE.md file.
|
||||
|
||||
package bzip2 |
||||
|
||||
import ( |
||||
"io" |
||||
|
||||
"github.com/dsnet/compress/internal" |
||||
"github.com/dsnet/compress/internal/errors" |
||||
"github.com/dsnet/compress/internal/prefix" |
||||
) |
||||
|
||||
type Reader struct { |
||||
InputOffset int64 // Total number of bytes read from underlying io.Reader
|
||||
OutputOffset int64 // Total number of bytes emitted from Read
|
||||
|
||||
rd prefixReader |
||||
err error |
||||
level int // The current compression level
|
||||
rdHdrFtr int // Number of times we read the stream header and footer
|
||||
blkCRC uint32 // CRC-32 IEEE of each block (as stored)
|
||||
endCRC uint32 // Checksum of all blocks using bzip2's custom method
|
||||
|
||||
crc crc |
||||
mtf moveToFront |
||||
bwt burrowsWheelerTransform |
||||
rle runLengthEncoding |
||||
|
||||
// These fields are allocated with Reader and re-used later.
|
||||
treeSels []uint8 |
||||
codes2D [maxNumTrees][maxNumSyms]prefix.PrefixCode |
||||
codes1D [maxNumTrees]prefix.PrefixCodes |
||||
trees1D [maxNumTrees]prefix.Decoder |
||||
syms []uint16 |
||||
|
||||
fuzzReader // Exported functionality when fuzz testing
|
||||
} |
||||
|
||||
type ReaderConfig struct { |
||||
_ struct{} // Blank field to prevent unkeyed struct literals
|
||||
} |
||||
|
||||
func NewReader(r io.Reader, conf *ReaderConfig) (*Reader, error) { |
||||
zr := new(Reader) |
||||
zr.Reset(r) |
||||
return zr, nil |
||||
} |
||||
|
||||
func (zr *Reader) Reset(r io.Reader) error { |
||||
*zr = Reader{ |
||||
rd: zr.rd, |
||||
|
||||
mtf: zr.mtf, |
||||
bwt: zr.bwt, |
||||
rle: zr.rle, |
||||
|
||||
treeSels: zr.treeSels, |
||||
trees1D: zr.trees1D, |
||||
syms: zr.syms, |
||||
} |
||||
zr.rd.Init(r) |
||||
return nil |
||||
} |
||||
|
||||
func (zr *Reader) Read(buf []byte) (int, error) { |
||||
for { |
||||
cnt, err := zr.rle.Read(buf) |
||||
if err != rleDone && zr.err == nil { |
||||
zr.err = err |
||||
} |
||||
if cnt > 0 { |
||||
zr.crc.update(buf[:cnt]) |
||||
zr.OutputOffset += int64(cnt) |
||||
return cnt, nil |
||||
} |
||||
if zr.err != nil || len(buf) == 0 { |
||||
return 0, zr.err |
||||
} |
||||
|
||||
// Read the next chunk.
|
||||
zr.rd.Offset = zr.InputOffset |
||||
func() { |
||||
defer errors.Recover(&zr.err) |
||||
if zr.rdHdrFtr%2 == 0 { |
||||
// Check if we are already at EOF.
|
||||
if err := zr.rd.PullBits(1); err != nil { |
||||
if err == io.ErrUnexpectedEOF && zr.rdHdrFtr > 0 { |
||||
err = io.EOF // EOF is okay if we read at least one stream
|
||||
} |
||||
errors.Panic(err) |
||||
} |
||||
|
||||
// Read stream header.
|
||||
if zr.rd.ReadBitsBE64(16) != hdrMagic { |
||||
panicf(errors.Corrupted, "invalid stream magic") |
||||
} |
||||
if ver := zr.rd.ReadBitsBE64(8); ver != 'h' { |
||||
if ver == '0' { |
||||
panicf(errors.Deprecated, "bzip1 format is not supported") |
||||
} |
||||
panicf(errors.Corrupted, "invalid version: %q", ver) |
||||
} |
||||
lvl := int(zr.rd.ReadBitsBE64(8)) - '0' |
||||
if lvl < BestSpeed || lvl > BestCompression { |
||||
panicf(errors.Corrupted, "invalid block size: %d", lvl*blockSize) |
||||
} |
||||
zr.level = lvl |
||||
zr.rdHdrFtr++ |
||||
} else { |
||||
// Check and update the CRC.
|
||||
if internal.GoFuzz { |
||||
zr.updateChecksum(-1, zr.crc.val) // Update with value
|
||||
zr.blkCRC = zr.crc.val // Suppress CRC failures
|
||||
} |
||||
if zr.blkCRC != zr.crc.val { |
||||
panicf(errors.Corrupted, "mismatching block checksum") |
||||
} |
||||
zr.endCRC = (zr.endCRC<<1 | zr.endCRC>>31) ^ zr.blkCRC |
||||
} |
||||
buf := zr.decodeBlock() |
||||
zr.rle.Init(buf) |
||||
}() |
||||
if zr.InputOffset, err = zr.rd.Flush(); zr.err == nil { |
||||
zr.err = err |
||||
} |
||||
if zr.err != nil { |
||||
zr.err = errWrap(zr.err, errors.Corrupted) |
||||
return 0, zr.err |
||||
} |
||||
} |
||||
} |
||||
|
||||
func (zr *Reader) Close() error { |
||||
if zr.err == io.EOF || zr.err == errClosed { |
||||
zr.rle.Init(nil) // Make sure future reads fail
|
||||
zr.err = errClosed |
||||
return nil |
||||
} |
||||
return zr.err // Return the persistent error
|
||||
} |
||||
|
||||
func (zr *Reader) decodeBlock() []byte { |
||||
if magic := zr.rd.ReadBitsBE64(48); magic != blkMagic { |
||||
if magic == endMagic { |
||||
endCRC := uint32(zr.rd.ReadBitsBE64(32)) |
||||
if internal.GoFuzz { |
||||
zr.updateChecksum(zr.rd.BitsRead()-32, zr.endCRC) |
||||
endCRC = zr.endCRC // Suppress CRC failures
|
||||
} |
||||
if zr.endCRC != endCRC { |
||||
panicf(errors.Corrupted, "mismatching stream checksum") |
||||
} |
||||
zr.endCRC = 0 |
||||
zr.rd.ReadPads() |
||||
zr.rdHdrFtr++ |
||||
return nil |
||||
} |
||||
panicf(errors.Corrupted, "invalid block or footer magic") |
||||
} |
||||
|
||||
zr.crc.val = 0 |
||||
zr.blkCRC = uint32(zr.rd.ReadBitsBE64(32)) |
||||
if internal.GoFuzz { |
||||
zr.updateChecksum(zr.rd.BitsRead()-32, 0) // Record offset only
|
||||
} |
||||
if zr.rd.ReadBitsBE64(1) != 0 { |
||||
panicf(errors.Deprecated, "block randomization is not supported") |
||||
} |
||||
|
||||
// Read BWT related fields.
|
||||
ptr := int(zr.rd.ReadBitsBE64(24)) // BWT origin pointer
|
||||
|
||||
// Read MTF related fields.
|
||||
var dictArr [256]uint8 |
||||
dict := dictArr[:0] |
||||
bmapHi := uint16(zr.rd.ReadBits(16)) |
||||
for i := 0; i < 256; i, bmapHi = i+16, bmapHi>>1 { |
||||
if bmapHi&1 > 0 { |
||||
bmapLo := uint16(zr.rd.ReadBits(16)) |
||||
for j := 0; j < 16; j, bmapLo = j+1, bmapLo>>1 { |
||||
if bmapLo&1 > 0 { |
||||
dict = append(dict, uint8(i+j)) |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
// Step 1: Prefix encoding.
|
||||
syms := zr.decodePrefix(len(dict)) |
||||
|
||||
// Step 2: Move-to-front transform and run-length encoding.
|
||||
zr.mtf.Init(dict, zr.level*blockSize) |
||||
buf := zr.mtf.Decode(syms) |
||||
|
||||
// Step 3: Burrows-Wheeler transformation.
|
||||
if ptr >= len(buf) { |
||||
panicf(errors.Corrupted, "origin pointer (0x%06x) exceeds block size: %d", ptr, len(buf)) |
||||
} |
||||
zr.bwt.Decode(buf, ptr) |
||||
|
||||
return buf |
||||
} |
||||
|
||||
func (zr *Reader) decodePrefix(numSyms int) (syms []uint16) { |
||||
numSyms += 2 // Remove 0 symbol, add RUNA, RUNB, and EOF symbols
|
||||
if numSyms < 3 { |
||||
panicf(errors.Corrupted, "not enough prefix symbols: %d", numSyms) |
||||
} |
||||
|
||||
// Read information about the trees and tree selectors.
|
||||
var mtf internal.MoveToFront |
||||
numTrees := int(zr.rd.ReadBitsBE64(3)) |
||||
if numTrees < minNumTrees || numTrees > maxNumTrees { |
||||
panicf(errors.Corrupted, "invalid number of prefix trees: %d", numTrees) |
||||
} |
||||
numSels := int(zr.rd.ReadBitsBE64(15)) |
||||
if cap(zr.treeSels) < numSels { |
||||
zr.treeSels = make([]uint8, numSels) |
||||
} |
||||
treeSels := zr.treeSels[:numSels] |
||||
for i := range treeSels { |
||||
sym, ok := zr.rd.TryReadSymbol(&decSel) |
||||
if !ok { |
||||
sym = zr.rd.ReadSymbol(&decSel) |
||||
} |
||||
if int(sym) >= numTrees { |
||||
panicf(errors.Corrupted, "invalid prefix tree selector: %d", sym) |
||||
} |
||||
treeSels[i] = uint8(sym) |
||||
} |
||||
mtf.Decode(treeSels) |
||||
zr.treeSels = treeSels |
||||
|
||||
// Initialize prefix codes.
|
||||
for i := range zr.codes2D[:numTrees] { |
||||
zr.codes1D[i] = zr.codes2D[i][:numSyms] |
||||
} |
||||
zr.rd.ReadPrefixCodes(zr.codes1D[:numTrees], zr.trees1D[:numTrees]) |
||||
|
||||
// Read prefix encoded symbols of compressed data.
|
||||
var tree *prefix.Decoder |
||||
var blkLen, selIdx int |
||||
syms = zr.syms[:0] |
||||
for { |
||||
if blkLen == 0 { |
||||
blkLen = numBlockSyms |
||||
if selIdx >= len(treeSels) { |
||||
panicf(errors.Corrupted, "not enough prefix tree selectors") |
||||
} |
||||
tree = &zr.trees1D[treeSels[selIdx]] |
||||
selIdx++ |
||||
} |
||||
blkLen-- |
||||
sym, ok := zr.rd.TryReadSymbol(tree) |
||||
if !ok { |
||||
sym = zr.rd.ReadSymbol(tree) |
||||
} |
||||
|
||||
if int(sym) == numSyms-1 { |
||||
break // EOF marker
|
||||
} |
||||
if int(sym) >= numSyms { |
||||
panicf(errors.Corrupted, "invalid prefix symbol: %d", sym) |
||||
} |
||||
if len(syms) >= zr.level*blockSize { |
||||
panicf(errors.Corrupted, "number of prefix symbols exceeds block size") |
||||
} |
||||
syms = append(syms, uint16(sym)) |
||||
} |
||||
zr.syms = syms |
||||
return syms |
||||
} |
||||
@ -0,0 +1,101 @@ |
||||
// Copyright 2015, Joe Tsai. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE.md file.
|
||||
|
||||
package bzip2 |
||||
|
||||
import "github.com/dsnet/compress/internal/errors" |
||||
|
||||
// rleDone is a special "error" to indicate that the RLE stage is done.
|
||||
var rleDone = errorf(errors.Unknown, "RLE1 stage is completed") |
||||
|
||||
// runLengthEncoding implements the first RLE stage of bzip2. Every sequence
|
||||
// of 4..255 duplicated bytes is replaced by only the first 4 bytes, and a
|
||||
// single byte representing the repeat length. Similar to the C bzip2
|
||||
// implementation, the encoder will always terminate repeat sequences with a
|
||||
// count (even if it is the end of the buffer), and it will also never produce
|
||||
// run lengths of 256..259. The decoder can handle the latter case.
|
||||
//
|
||||
// For example, if the input was:
|
||||
// input: "AAAAAAABBBBCCCD"
|
||||
//
|
||||
// Then the output will be:
|
||||
// output: "AAAA\x03BBBB\x00CCCD"
|
||||
type runLengthEncoding struct { |
||||
buf []byte |
||||
idx int |
||||
lastVal byte |
||||
lastCnt int |
||||
} |
||||
|
||||
func (rle *runLengthEncoding) Init(buf []byte) { |
||||
*rle = runLengthEncoding{buf: buf} |
||||
} |
||||
|
||||
func (rle *runLengthEncoding) Write(buf []byte) (int, error) { |
||||
for i, b := range buf { |
||||
if rle.lastVal != b { |
||||
rle.lastCnt = 0 |
||||
} |
||||
rle.lastCnt++ |
||||
switch { |
||||
case rle.lastCnt < 4: |
||||
if rle.idx >= len(rle.buf) { |
||||
return i, rleDone |
||||
} |
||||
rle.buf[rle.idx] = b |
||||
rle.idx++ |
||||
case rle.lastCnt == 4: |
||||
if rle.idx+1 >= len(rle.buf) { |
||||
return i, rleDone |
||||
} |
||||
rle.buf[rle.idx] = b |
||||
rle.idx++ |
||||
rle.buf[rle.idx] = 0 |
||||
rle.idx++ |
||||
case rle.lastCnt < 256: |
||||
rle.buf[rle.idx-1]++ |
||||
default: |
||||
if rle.idx >= len(rle.buf) { |
||||
return i, rleDone |
||||
} |
||||
rle.lastCnt = 1 |
||||
rle.buf[rle.idx] = b |
||||
rle.idx++ |
||||
} |
||||
rle.lastVal = b |
||||
} |
||||
return len(buf), nil |
||||
} |
||||
|
||||
func (rle *runLengthEncoding) Read(buf []byte) (int, error) { |
||||
for i := range buf { |
||||
switch { |
||||
case rle.lastCnt == -4: |
||||
if rle.idx >= len(rle.buf) { |
||||
return i, errorf(errors.Corrupted, "missing terminating run-length repeater") |
||||
} |
||||
rle.lastCnt = int(rle.buf[rle.idx]) |
||||
rle.idx++ |
||||
if rle.lastCnt > 0 { |
||||
break // Break the switch
|
||||
} |
||||
fallthrough // Count was zero, continue the work
|
||||
case rle.lastCnt <= 0: |
||||
if rle.idx >= len(rle.buf) { |
||||
return i, rleDone |
||||
} |
||||
b := rle.buf[rle.idx] |
||||
rle.idx++ |
||||
if b != rle.lastVal { |
||||
rle.lastCnt = 0 |
||||
rle.lastVal = b |
||||
} |
||||
} |
||||
buf[i] = rle.lastVal |
||||
rle.lastCnt-- |
||||
} |
||||
return len(buf), nil |
||||
} |
||||
|
||||
func (rle *runLengthEncoding) Bytes() []byte { return rle.buf[:rle.idx] } |
||||
@ -0,0 +1,307 @@ |
||||
// Copyright 2015, Joe Tsai. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE.md file.
|
||||
|
||||
package bzip2 |
||||
|
||||
import ( |
||||
"io" |
||||
|
||||
"github.com/dsnet/compress/internal" |
||||
"github.com/dsnet/compress/internal/errors" |
||||
"github.com/dsnet/compress/internal/prefix" |
||||
) |
||||
|
||||
type Writer struct { |
||||
InputOffset int64 // Total number of bytes issued to Write
|
||||
OutputOffset int64 // Total number of bytes written to underlying io.Writer
|
||||
|
||||
wr prefixWriter |
||||
err error |
||||
level int // The current compression level
|
||||
wrHdr bool // Have we written the stream header?
|
||||
blkCRC uint32 // CRC-32 IEEE of each block
|
||||
endCRC uint32 // Checksum of all blocks using bzip2's custom method
|
||||
|
||||
crc crc |
||||
rle runLengthEncoding |
||||
bwt burrowsWheelerTransform |
||||
mtf moveToFront |
||||
|
||||
// These fields are allocated with Writer and re-used later.
|
||||
buf []byte |
||||
treeSels []uint8 |
||||
treeSelsMTF []uint8 |
||||
codes2D [maxNumTrees][maxNumSyms]prefix.PrefixCode |
||||
codes1D [maxNumTrees]prefix.PrefixCodes |
||||
trees1D [maxNumTrees]prefix.Encoder |
||||
} |
||||
|
||||
type WriterConfig struct { |
||||
Level int |
||||
|
||||
_ struct{} // Blank field to prevent unkeyed struct literals
|
||||
} |
||||
|
||||
func NewWriter(w io.Writer, conf *WriterConfig) (*Writer, error) { |
||||
var lvl int |
||||
if conf != nil { |
||||
lvl = conf.Level |
||||
} |
||||
if lvl == 0 { |
||||
lvl = DefaultCompression |
||||
} |
||||
if lvl < BestSpeed || lvl > BestCompression { |
||||
return nil, errorf(errors.Invalid, "compression level: %d", lvl) |
||||
} |
||||
zw := new(Writer) |
||||
zw.level = lvl |
||||
zw.Reset(w) |
||||
return zw, nil |
||||
} |
||||
|
||||
func (zw *Writer) Reset(w io.Writer) error { |
||||
*zw = Writer{ |
||||
wr: zw.wr, |
||||
level: zw.level, |
||||
|
||||
rle: zw.rle, |
||||
bwt: zw.bwt, |
||||
mtf: zw.mtf, |
||||
|
||||
buf: zw.buf, |
||||
treeSels: zw.treeSels, |
||||
treeSelsMTF: zw.treeSelsMTF, |
||||
trees1D: zw.trees1D, |
||||
} |
||||
zw.wr.Init(w) |
||||
if len(zw.buf) != zw.level*blockSize { |
||||
zw.buf = make([]byte, zw.level*blockSize) |
||||
} |
||||
zw.rle.Init(zw.buf) |
||||
return nil |
||||
} |
||||
|
||||
func (zw *Writer) Write(buf []byte) (int, error) { |
||||
if zw.err != nil { |
||||
return 0, zw.err |
||||
} |
||||
|
||||
cnt := len(buf) |
||||
for { |
||||
wrCnt, err := zw.rle.Write(buf) |
||||
if err != rleDone && zw.err == nil { |
||||
zw.err = err |
||||
} |
||||
zw.crc.update(buf[:wrCnt]) |
||||
buf = buf[wrCnt:] |
||||
if len(buf) == 0 { |
||||
zw.InputOffset += int64(cnt) |
||||
return cnt, nil |
||||
} |
||||
if zw.err = zw.flush(); zw.err != nil { |
||||
return 0, zw.err |
||||
} |
||||
} |
||||
} |
||||
|
||||
func (zw *Writer) flush() error { |
||||
vals := zw.rle.Bytes() |
||||
if len(vals) == 0 { |
||||
return nil |
||||
} |
||||
zw.wr.Offset = zw.OutputOffset |
||||
func() { |
||||
defer errors.Recover(&zw.err) |
||||
if !zw.wrHdr { |
||||
// Write stream header.
|
||||
zw.wr.WriteBitsBE64(hdrMagic, 16) |
||||
zw.wr.WriteBitsBE64('h', 8) |
||||
zw.wr.WriteBitsBE64(uint64('0'+zw.level), 8) |
||||
zw.wrHdr = true |
||||
} |
||||
zw.encodeBlock(vals) |
||||
}() |
||||
var err error |
||||
if zw.OutputOffset, err = zw.wr.Flush(); zw.err == nil { |
||||
zw.err = err |
||||
} |
||||
if zw.err != nil { |
||||
zw.err = errWrap(zw.err, errors.Internal) |
||||
return zw.err |
||||
} |
||||
zw.endCRC = (zw.endCRC<<1 | zw.endCRC>>31) ^ zw.blkCRC |
||||
zw.blkCRC = 0 |
||||
zw.rle.Init(zw.buf) |
||||
return nil |
||||
} |
||||
|
||||
func (zw *Writer) Close() error { |
||||
if zw.err == errClosed { |
||||
return nil |
||||
} |
||||
|
||||
// Flush RLE buffer if there is left-over data.
|
||||
if zw.err = zw.flush(); zw.err != nil { |
||||
return zw.err |
||||
} |
||||
|
||||
// Write stream footer.
|
||||
zw.wr.Offset = zw.OutputOffset |
||||
func() { |
||||
defer errors.Recover(&zw.err) |
||||
if !zw.wrHdr { |
||||
// Write stream header.
|
||||
zw.wr.WriteBitsBE64(hdrMagic, 16) |
||||
zw.wr.WriteBitsBE64('h', 8) |
||||
zw.wr.WriteBitsBE64(uint64('0'+zw.level), 8) |
||||
zw.wrHdr = true |
||||
} |
||||
zw.wr.WriteBitsBE64(endMagic, 48) |
||||
zw.wr.WriteBitsBE64(uint64(zw.endCRC), 32) |
||||
zw.wr.WritePads(0) |
||||
}() |
||||
var err error |
||||
if zw.OutputOffset, err = zw.wr.Flush(); zw.err == nil { |
||||
zw.err = err |
||||
} |
||||
if zw.err != nil { |
||||
zw.err = errWrap(zw.err, errors.Internal) |
||||
return zw.err |
||||
} |
||||
|
||||
zw.err = errClosed |
||||
return nil |
||||
} |
||||
|
||||
func (zw *Writer) encodeBlock(buf []byte) { |
||||
zw.blkCRC = zw.crc.val |
||||
zw.wr.WriteBitsBE64(blkMagic, 48) |
||||
zw.wr.WriteBitsBE64(uint64(zw.blkCRC), 32) |
||||
zw.wr.WriteBitsBE64(0, 1) |
||||
zw.crc.val = 0 |
||||
|
||||
// Step 1: Burrows-Wheeler transformation.
|
||||
ptr := zw.bwt.Encode(buf) |
||||
zw.wr.WriteBitsBE64(uint64(ptr), 24) |
||||
|
||||
// Step 2: Move-to-front transform and run-length encoding.
|
||||
var dictMap [256]bool |
||||
for _, c := range buf { |
||||
dictMap[c] = true |
||||
} |
||||
|
||||
var dictArr [256]uint8 |
||||
var bmapLo [16]uint16 |
||||
dict := dictArr[:0] |
||||
bmapHi := uint16(0) |
||||
for i, b := range dictMap { |
||||
if b { |
||||
c := uint8(i) |
||||
dict = append(dict, c) |
||||
bmapHi |= 1 << (c >> 4) |
||||
bmapLo[c>>4] |= 1 << (c & 0xf) |
||||
} |
||||
} |
||||
|
||||
zw.wr.WriteBits(uint(bmapHi), 16) |
||||
for _, m := range bmapLo { |
||||
if m > 0 { |
||||
zw.wr.WriteBits(uint(m), 16) |
||||
} |
||||
} |
||||
|
||||
zw.mtf.Init(dict, len(buf)) |
||||
syms := zw.mtf.Encode(buf) |
||||
|
||||
// Step 3: Prefix encoding.
|
||||
zw.encodePrefix(syms, len(dict)) |
||||
} |
||||
|
||||
func (zw *Writer) encodePrefix(syms []uint16, numSyms int) { |
||||
numSyms += 2 // Remove 0 symbol, add RUNA, RUNB, and EOB symbols
|
||||
if numSyms < 3 { |
||||
panicf(errors.Internal, "unable to encode EOB marker") |
||||
} |
||||
syms = append(syms, uint16(numSyms-1)) // EOB marker
|
||||
|
||||
// Compute number of prefix trees needed.
|
||||
numTrees := maxNumTrees |
||||
for i, lim := range []int{200, 600, 1200, 2400} { |
||||
if len(syms) < lim { |
||||
numTrees = minNumTrees + i |
||||
break |
||||
} |
||||
} |
||||
|
||||
// Compute number of block selectors.
|
||||
numSels := (len(syms) + numBlockSyms - 1) / numBlockSyms |
||||
if cap(zw.treeSels) < numSels { |
||||
zw.treeSels = make([]uint8, numSels) |
||||
} |
||||
treeSels := zw.treeSels[:numSels] |
||||
for i := range treeSels { |
||||
treeSels[i] = uint8(i % numTrees) |
||||
} |
||||
|
||||
// Initialize prefix codes.
|
||||
for i := range zw.codes2D[:numTrees] { |
||||
pc := zw.codes2D[i][:numSyms] |
||||
for j := range pc { |
||||
pc[j] = prefix.PrefixCode{Sym: uint32(j)} |
||||
} |
||||
zw.codes1D[i] = pc |
||||
} |
||||
|
||||
// First cut at assigning prefix trees to each group.
|
||||
var codes prefix.PrefixCodes |
||||
var blkLen, selIdx int |
||||
for _, sym := range syms { |
||||
if blkLen == 0 { |
||||
blkLen = numBlockSyms |
||||
codes = zw.codes2D[treeSels[selIdx]][:numSyms] |
||||
selIdx++ |
||||
} |
||||
blkLen-- |
||||
codes[sym].Cnt++ |
||||
} |
||||
|
||||
// TODO(dsnet): Use K-means to cluster groups to each prefix tree.
|
||||
|
||||
// Generate lengths and prefixes based on symbol frequencies.
|
||||
for i := range zw.trees1D[:numTrees] { |
||||
pc := prefix.PrefixCodes(zw.codes2D[i][:numSyms]) |
||||
pc.SortByCount() |
||||
if err := prefix.GenerateLengths(pc, maxPrefixBits); err != nil { |
||||
errors.Panic(err) |
||||
} |
||||
pc.SortBySymbol() |
||||
} |
||||
|
||||
// Write out information about the trees and tree selectors.
|
||||
var mtf internal.MoveToFront |
||||
zw.wr.WriteBitsBE64(uint64(numTrees), 3) |
||||
zw.wr.WriteBitsBE64(uint64(numSels), 15) |
||||
zw.treeSelsMTF = append(zw.treeSelsMTF[:0], treeSels...) |
||||
mtf.Encode(zw.treeSelsMTF) |
||||
for _, sym := range zw.treeSelsMTF { |
||||
zw.wr.WriteSymbol(uint(sym), &encSel) |
||||
} |
||||
zw.wr.WritePrefixCodes(zw.codes1D[:numTrees], zw.trees1D[:numTrees]) |
||||
|
||||
// Write out prefix encoded symbols of compressed data.
|
||||
var tree *prefix.Encoder |
||||
blkLen, selIdx = 0, 0 |
||||
for _, sym := range syms { |
||||
if blkLen == 0 { |
||||
blkLen = numBlockSyms |
||||
tree = &zw.trees1D[treeSels[selIdx]] |
||||
selIdx++ |
||||
} |
||||
blkLen-- |
||||
ok := zw.wr.TryWriteSymbol(uint(sym), tree) |
||||
if !ok { |
||||
zw.wr.WriteSymbol(uint(sym), tree) |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,10 @@ |
||||
module github.com/dsnet/compress |
||||
|
||||
go 1.9 |
||||
|
||||
require ( |
||||
github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780 |
||||
github.com/klauspost/compress v1.4.1 |
||||
github.com/klauspost/cpuid v1.2.0 // indirect |
||||
github.com/ulikunitz/xz v0.5.6 |
||||
) |
||||
@ -0,0 +1,8 @@ |
||||
github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780 h1:tFh1tRc4CA31yP6qDcu+Trax5wW5GuMxvkIba07qVLY= |
||||
github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= |
||||
github.com/klauspost/compress v1.4.1 h1:8VMb5+0wMgdBykOV96DwNwKFQ+WTI4pzYURP99CcB9E= |
||||
github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= |
||||
github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE= |
||||
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= |
||||
github.com/ulikunitz/xz v0.5.6 h1:jGHAfXawEGZQ3blwU5wnWKQJvAraT7Ftq9EXjnXYgt8= |
||||
github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= |
||||
@ -0,0 +1,107 @@ |
||||
// Copyright 2015, Joe Tsai. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE.md file.
|
||||
|
||||
// Package internal is a collection of common compression algorithms.
|
||||
//
|
||||
// For performance reasons, these packages lack strong error checking and
|
||||
// require that the caller to ensure that strict invariants are kept.
|
||||
package internal |
||||
|
||||
var ( |
||||
// IdentityLUT returns the input key itself.
|
||||
IdentityLUT = func() (lut [256]byte) { |
||||
for i := range lut { |
||||
lut[i] = uint8(i) |
||||
} |
||||
return lut |
||||
}() |
||||
|
||||
// ReverseLUT returns the input key with its bits reversed.
|
||||
ReverseLUT = func() (lut [256]byte) { |
||||
for i := range lut { |
||||
b := uint8(i) |
||||
b = (b&0xaa)>>1 | (b&0x55)<<1 |
||||
b = (b&0xcc)>>2 | (b&0x33)<<2 |
||||
b = (b&0xf0)>>4 | (b&0x0f)<<4 |
||||
lut[i] = b |
||||
} |
||||
return lut |
||||
}() |
||||
) |
||||
|
||||
// ReverseUint32 reverses all bits of v.
|
||||
func ReverseUint32(v uint32) (x uint32) { |
||||
x |= uint32(ReverseLUT[byte(v>>0)]) << 24 |
||||
x |= uint32(ReverseLUT[byte(v>>8)]) << 16 |
||||
x |= uint32(ReverseLUT[byte(v>>16)]) << 8 |
||||
x |= uint32(ReverseLUT[byte(v>>24)]) << 0 |
||||
return x |
||||
} |
||||
|
||||
// ReverseUint32N reverses the lower n bits of v.
|
||||
func ReverseUint32N(v uint32, n uint) (x uint32) { |
||||
return ReverseUint32(v << (32 - n)) |
||||
} |
||||
|
||||
// ReverseUint64 reverses all bits of v.
|
||||
func ReverseUint64(v uint64) (x uint64) { |
||||
x |= uint64(ReverseLUT[byte(v>>0)]) << 56 |
||||
x |= uint64(ReverseLUT[byte(v>>8)]) << 48 |
||||
x |= uint64(ReverseLUT[byte(v>>16)]) << 40 |
||||
x |= uint64(ReverseLUT[byte(v>>24)]) << 32 |
||||
x |= uint64(ReverseLUT[byte(v>>32)]) << 24 |
||||
x |= uint64(ReverseLUT[byte(v>>40)]) << 16 |
||||
x |= uint64(ReverseLUT[byte(v>>48)]) << 8 |
||||
x |= uint64(ReverseLUT[byte(v>>56)]) << 0 |
||||
return x |
||||
} |
||||
|
||||
// ReverseUint64N reverses the lower n bits of v.
|
||||
func ReverseUint64N(v uint64, n uint) (x uint64) { |
||||
return ReverseUint64(v << (64 - n)) |
||||
} |
||||
|
||||
// MoveToFront is a data structure that allows for more efficient move-to-front
|
||||
// transformations. This specific implementation assumes that the alphabet is
|
||||
// densely packed within 0..255.
|
||||
type MoveToFront struct { |
||||
dict [256]uint8 // Mapping from indexes to values
|
||||
tail int // Number of tail bytes that are already ordered
|
||||
} |
||||
|
||||
func (m *MoveToFront) Encode(vals []uint8) { |
||||
copy(m.dict[:], IdentityLUT[:256-m.tail]) // Reset dict to be identity
|
||||
|
||||
var max int |
||||
for i, val := range vals { |
||||
var idx uint8 // Reverse lookup idx in dict
|
||||
for di, dv := range m.dict { |
||||
if dv == val { |
||||
idx = uint8(di) |
||||
break |
||||
} |
||||
} |
||||
vals[i] = idx |
||||
|
||||
max |= int(idx) |
||||
copy(m.dict[1:], m.dict[:idx]) |
||||
m.dict[0] = val |
||||
} |
||||
m.tail = 256 - max - 1 |
||||
} |
||||
|
||||
func (m *MoveToFront) Decode(idxs []uint8) { |
||||
copy(m.dict[:], IdentityLUT[:256-m.tail]) // Reset dict to be identity
|
||||
|
||||
var max int |
||||
for i, idx := range idxs { |
||||
val := m.dict[idx] // Forward lookup val in dict
|
||||
idxs[i] = val |
||||
|
||||
max |= int(idx) |
||||
copy(m.dict[1:], m.dict[:idx]) |
||||
m.dict[0] = val |
||||
} |
||||
m.tail = 256 - max - 1 |
||||
} |
||||
@ -0,0 +1,12 @@ |
||||
// Copyright 2015, Joe Tsai. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE.md file.
|
||||
|
||||
// +build debug,!gofuzz
|
||||
|
||||
package internal |
||||
|
||||
const ( |
||||
Debug = true |
||||
GoFuzz = false |
||||
) |
||||
@ -0,0 +1,120 @@ |
||||
// Copyright 2016, Joe Tsai. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE.md file.
|
||||
|
||||
// Package errors implements functions to manipulate compression errors.
|
||||
//
|
||||
// In idiomatic Go, it is an anti-pattern to use panics as a form of error
|
||||
// reporting in the API. Instead, the expected way to transmit errors is by
|
||||
// returning an error value. Unfortunately, the checking of "err != nil" in
|
||||
// tight loops commonly found in compression causes non-negligible performance
|
||||
// degradation. While this may not be idiomatic, the internal packages of this
|
||||
// repository rely on panics as a normal means to convey errors. In order to
|
||||
// ensure that these panics do not leak across the public API, the public
|
||||
// packages must recover from these panics and present an error value.
|
||||
//
|
||||
// The Panic and Recover functions in this package provide a safe way to
|
||||
// recover from errors only generated from within this repository.
|
||||
//
|
||||
// Example usage:
|
||||
// func Foo() (err error) {
|
||||
// defer errors.Recover(&err)
|
||||
//
|
||||
// if rand.Intn(2) == 0 {
|
||||
// // Unexpected panics will not be caught by Recover.
|
||||
// io.Closer(nil).Close()
|
||||
// } else {
|
||||
// // Errors thrown by Panic will be caught by Recover.
|
||||
// errors.Panic(errors.New("whoopsie"))
|
||||
// }
|
||||
// }
|
||||
//
|
||||
package errors |
||||
|
||||
import "strings" |
||||
|
||||
const ( |
||||
// Unknown indicates that there is no classification for this error.
|
||||
Unknown = iota |
||||
|
||||
// Internal indicates that this error is due to an internal bug.
|
||||
// Users should file a issue report if this type of error is encountered.
|
||||
Internal |
||||
|
||||
// Invalid indicates that this error is due to the user misusing the API
|
||||
// and is indicative of a bug on the user's part.
|
||||
Invalid |
||||
|
||||
// Deprecated indicates the use of a deprecated and unsupported feature.
|
||||
Deprecated |
||||
|
||||
// Corrupted indicates that the input stream is corrupted.
|
||||
Corrupted |
||||
|
||||
// Closed indicates that the handlers are closed.
|
||||
Closed |
||||
) |
||||
|
||||
var codeMap = map[int]string{ |
||||
Unknown: "unknown error", |
||||
Internal: "internal error", |
||||
Invalid: "invalid argument", |
||||
Deprecated: "deprecated format", |
||||
Corrupted: "corrupted input", |
||||
Closed: "closed handler", |
||||
} |
||||
|
||||
type Error struct { |
||||
Code int // The error type
|
||||
Pkg string // Name of the package where the error originated
|
||||
Msg string // Descriptive message about the error (optional)
|
||||
} |
||||
|
||||
func (e Error) Error() string { |
||||
var ss []string |
||||
for _, s := range []string{e.Pkg, codeMap[e.Code], e.Msg} { |
||||
if s != "" { |
||||
ss = append(ss, s) |
||||
} |
||||
} |
||||
return strings.Join(ss, ": ") |
||||
} |
||||
|
||||
func (e Error) CompressError() {} |
||||
func (e Error) IsInternal() bool { return e.Code == Internal } |
||||
func (e Error) IsInvalid() bool { return e.Code == Invalid } |
||||
func (e Error) IsDeprecated() bool { return e.Code == Deprecated } |
||||
func (e Error) IsCorrupted() bool { return e.Code == Corrupted } |
||||
func (e Error) IsClosed() bool { return e.Code == Closed } |
||||
|
||||
func IsInternal(err error) bool { return isCode(err, Internal) } |
||||
func IsInvalid(err error) bool { return isCode(err, Invalid) } |
||||
func IsDeprecated(err error) bool { return isCode(err, Deprecated) } |
||||
func IsCorrupted(err error) bool { return isCode(err, Corrupted) } |
||||
func IsClosed(err error) bool { return isCode(err, Closed) } |
||||
|
||||
func isCode(err error, code int) bool { |
||||
if cerr, ok := err.(Error); ok && cerr.Code == code { |
||||
return true |
||||
} |
||||
return false |
||||
} |
||||
|
||||
// errWrap is used by Panic and Recover to ensure that only errors raised by
|
||||
// Panic are recovered by Recover.
|
||||
type errWrap struct{ e *error } |
||||
|
||||
func Recover(err *error) { |
||||
switch ex := recover().(type) { |
||||
case nil: |
||||
// Do nothing.
|
||||
case errWrap: |
||||
*err = *ex.e |
||||
default: |
||||
panic(ex) |
||||
} |
||||
} |
||||
|
||||
func Panic(err error) { |
||||
panic(errWrap{&err}) |
||||
} |
||||
@ -0,0 +1,12 @@ |
||||
// Copyright 2016, Joe Tsai. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE.md file.
|
||||
|
||||
// +build gofuzz
|
||||
|
||||
package internal |
||||
|
||||
const ( |
||||
Debug = true |
||||
GoFuzz = true |
||||
) |
||||
@ -0,0 +1,159 @@ |
||||
// Copyright 2015, Joe Tsai. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE.md file.
|
||||
|
||||
// +build debug
|
||||
|
||||
package prefix |
||||
|
||||
import ( |
||||
"fmt" |
||||
"math" |
||||
"strings" |
||||
) |
||||
|
||||
func max(a, b int) int { |
||||
if a > b { |
||||
return a |
||||
} |
||||
return b |
||||
} |
||||
|
||||
func lenBase2(n uint) int { |
||||
return int(math.Ceil(math.Log2(float64(n + 1)))) |
||||
} |
||||
func padBase2(v, n uint, m int) string { |
||||
s := fmt.Sprintf("%b", 1<<n|v)[1:] |
||||
if pad := m - len(s); pad > 0 { |
||||
return strings.Repeat(" ", pad) + s |
||||
} |
||||
return s |
||||
} |
||||
|
||||
func lenBase10(n int) int { |
||||
return int(math.Ceil(math.Log10(float64(n + 1)))) |
||||
} |
||||
func padBase10(n, m int) string { |
||||
s := fmt.Sprintf("%d", n) |
||||
if pad := m - len(s); pad > 0 { |
||||
return strings.Repeat(" ", pad) + s |
||||
} |
||||
return s |
||||
} |
||||
|
||||
func (rc RangeCodes) String() string { |
||||
var maxLen, maxBase int |
||||
for _, c := range rc { |
||||
maxLen = max(maxLen, int(c.Len)) |
||||
maxBase = max(maxBase, int(c.Base)) |
||||
} |
||||
|
||||
var ss []string |
||||
ss = append(ss, "{") |
||||
for i, c := range rc { |
||||
base := padBase10(int(c.Base), lenBase10(maxBase)) |
||||
if c.Len > 0 { |
||||
base += fmt.Sprintf("-%d", c.End()-1) |
||||
} |
||||
ss = append(ss, fmt.Sprintf("\t%s: {len: %s, range: %s},", |
||||
padBase10(int(i), lenBase10(len(rc)-1)), |
||||
padBase10(int(c.Len), lenBase10(maxLen)), |
||||
base, |
||||
)) |
||||
} |
||||
ss = append(ss, "}") |
||||
return strings.Join(ss, "\n") |
||||
} |
||||
|
||||
func (pc PrefixCodes) String() string { |
||||
var maxSym, maxLen, maxCnt int |
||||
for _, c := range pc { |
||||
maxSym = max(maxSym, int(c.Sym)) |
||||
maxLen = max(maxLen, int(c.Len)) |
||||
maxCnt = max(maxCnt, int(c.Cnt)) |
||||
} |
||||
|
||||
var ss []string |
||||
ss = append(ss, "{") |
||||
for _, c := range pc { |
||||
var cntStr string |
||||
if maxCnt > 0 { |
||||
cnt := int(32*float32(c.Cnt)/float32(maxCnt) + 0.5) |
||||
cntStr = fmt.Sprintf("%s |%s", |
||||
padBase10(int(c.Cnt), lenBase10(maxCnt)), |
||||
strings.Repeat("#", cnt), |
||||
) |
||||
} |
||||
ss = append(ss, fmt.Sprintf("\t%s: %s, %s", |
||||
padBase10(int(c.Sym), lenBase10(maxSym)), |
||||
padBase2(uint(c.Val), uint(c.Len), maxLen), |
||||
cntStr, |
||||
)) |
||||
} |
||||
ss = append(ss, "}") |
||||
return strings.Join(ss, "\n") |
||||
} |
||||
|
||||
func (pd Decoder) String() string { |
||||
var ss []string |
||||
ss = append(ss, "{") |
||||
if len(pd.chunks) > 0 { |
||||
ss = append(ss, "\tchunks: {") |
||||
for i, c := range pd.chunks { |
||||
label := "sym" |
||||
if uint(c&countMask) > uint(pd.chunkBits) { |
||||
label = "idx" |
||||
} |
||||
ss = append(ss, fmt.Sprintf("\t\t%s: {%s: %s, len: %s}", |
||||
padBase2(uint(i), uint(pd.chunkBits), int(pd.chunkBits)), |
||||
label, padBase10(int(c>>countBits), 3), |
||||
padBase10(int(c&countMask), 2), |
||||
)) |
||||
} |
||||
ss = append(ss, "\t},") |
||||
|
||||
for j, links := range pd.links { |
||||
ss = append(ss, fmt.Sprintf("\tlinks[%d]: {", j)) |
||||
linkBits := lenBase2(uint(pd.linkMask)) |
||||
for i, c := range links { |
||||
ss = append(ss, fmt.Sprintf("\t\t%s: {sym: %s, len: %s},", |
||||
padBase2(uint(i), uint(linkBits), int(linkBits)), |
||||
padBase10(int(c>>countBits), 3), |
||||
padBase10(int(c&countMask), 2), |
||||
)) |
||||
} |
||||
ss = append(ss, "\t},") |
||||
} |
||||
} |
||||
ss = append(ss, fmt.Sprintf("\tchunkMask: %b,", pd.chunkMask)) |
||||
ss = append(ss, fmt.Sprintf("\tlinkMask: %b,", pd.linkMask)) |
||||
ss = append(ss, fmt.Sprintf("\tchunkBits: %d,", pd.chunkBits)) |
||||
ss = append(ss, fmt.Sprintf("\tMinBits: %d,", pd.MinBits)) |
||||
ss = append(ss, fmt.Sprintf("\tNumSyms: %d,", pd.NumSyms)) |
||||
ss = append(ss, "}") |
||||
return strings.Join(ss, "\n") |
||||
} |
||||
|
||||
func (pe Encoder) String() string { |
||||
var maxLen int |
||||
for _, c := range pe.chunks { |
||||
maxLen = max(maxLen, int(c&countMask)) |
||||
} |
||||
|
||||
var ss []string |
||||
ss = append(ss, "{") |
||||
if len(pe.chunks) > 0 { |
||||
ss = append(ss, "\tchunks: {") |
||||
for i, c := range pe.chunks { |
||||
ss = append(ss, fmt.Sprintf("\t\t%s: %s,", |
||||
padBase10(i, 3), |
||||
padBase2(uint(c>>countBits), uint(c&countMask), maxLen), |
||||
)) |
||||
} |
||||
ss = append(ss, "\t},") |
||||
} |
||||
ss = append(ss, fmt.Sprintf("\tchunkMask: %b,", pe.chunkMask)) |
||||
ss = append(ss, fmt.Sprintf("\tNumSyms: %d,", pe.NumSyms)) |
||||
ss = append(ss, "}") |
||||
return strings.Join(ss, "\n") |
||||
} |
||||
@ -0,0 +1,136 @@ |
||||
// Copyright 2015, Joe Tsai. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE.md file.
|
||||
|
||||
package prefix |
||||
|
||||
import ( |
||||
"sort" |
||||
|
||||
"github.com/dsnet/compress/internal" |
||||
) |
||||
|
||||
// The algorithm used to decode variable length codes is based on the lookup
|
||||
// method in zlib. If the code is less-than-or-equal to maxChunkBits,
|
||||
// then the symbol can be decoded using a single lookup into the chunks table.
|
||||
// Otherwise, the links table will be used for a second level lookup.
|
||||
//
|
||||
// The chunks slice is keyed by the contents of the bit buffer ANDed with
|
||||
// the chunkMask to avoid a out-of-bounds lookup. The value of chunks is a tuple
|
||||
// that is decoded as follow:
|
||||
//
|
||||
// var length = chunks[bitBuffer&chunkMask] & countMask
|
||||
// var symbol = chunks[bitBuffer&chunkMask] >> countBits
|
||||
//
|
||||
// If the decoded length is larger than chunkBits, then an overflow link table
|
||||
// must be used for further decoding. In this case, the symbol is actually the
|
||||
// index into the links tables. The second-level links table returned is
|
||||
// processed in the same way as the chunks table.
|
||||
//
|
||||
// if length > chunkBits {
|
||||
// var index = symbol // Previous symbol is index into links tables
|
||||
// length = links[index][bitBuffer>>chunkBits & linkMask] & countMask
|
||||
// symbol = links[index][bitBuffer>>chunkBits & linkMask] >> countBits
|
||||
// }
|
||||
//
|
||||
// See the following:
|
||||
// http://www.gzip.org/algorithm.txt
|
||||
|
||||
type Decoder struct { |
||||
chunks []uint32 // First-level lookup map
|
||||
links [][]uint32 // Second-level lookup map
|
||||
chunkMask uint32 // Mask the length of the chunks table
|
||||
linkMask uint32 // Mask the length of the link table
|
||||
chunkBits uint32 // Bit-length of the chunks table
|
||||
|
||||
MinBits uint32 // The minimum number of bits to safely make progress
|
||||
NumSyms uint32 // Number of symbols
|
||||
} |
||||
|
||||
// Init initializes Decoder according to the codes provided.
|
||||
func (pd *Decoder) Init(codes PrefixCodes) { |
||||
// Handle special case trees.
|
||||
if len(codes) <= 1 { |
||||
switch { |
||||
case len(codes) == 0: // Empty tree (should error if used later)
|
||||
*pd = Decoder{chunks: pd.chunks[:0], links: pd.links[:0], NumSyms: 0} |
||||
case len(codes) == 1 && codes[0].Len == 0: // Single code tree (bit-length of zero)
|
||||
pd.chunks = append(pd.chunks[:0], codes[0].Sym<<countBits|0) |
||||
*pd = Decoder{chunks: pd.chunks[:1], links: pd.links[:0], NumSyms: 1} |
||||
default: |
||||
panic("invalid codes") |
||||
} |
||||
return |
||||
} |
||||
if internal.Debug && !sort.IsSorted(prefixCodesBySymbol(codes)) { |
||||
panic("input codes is not sorted") |
||||
} |
||||
if internal.Debug && !(codes.checkLengths() && codes.checkPrefixes()) { |
||||
panic("detected incomplete or overlapping codes") |
||||
} |
||||
|
||||
var minBits, maxBits uint32 = valueBits, 0 |
||||
for _, c := range codes { |
||||
if minBits > c.Len { |
||||
minBits = c.Len |
||||
} |
||||
if maxBits < c.Len { |
||||
maxBits = c.Len |
||||
} |
||||
} |
||||
|
||||
// Allocate chunks table as needed.
|
||||
const maxChunkBits = 9 // This can be tuned for better performance
|
||||
pd.NumSyms = uint32(len(codes)) |
||||
pd.MinBits = minBits |
||||
pd.chunkBits = maxBits |
||||
if pd.chunkBits > maxChunkBits { |
||||
pd.chunkBits = maxChunkBits |
||||
} |
||||
numChunks := 1 << pd.chunkBits |
||||
pd.chunks = allocUint32s(pd.chunks, numChunks) |
||||
pd.chunkMask = uint32(numChunks - 1) |
||||
|
||||
// Allocate links tables as needed.
|
||||
pd.links = pd.links[:0] |
||||
pd.linkMask = 0 |
||||
if pd.chunkBits < maxBits { |
||||
numLinks := 1 << (maxBits - pd.chunkBits) |
||||
pd.linkMask = uint32(numLinks - 1) |
||||
|
||||
var linkIdx uint32 |
||||
for i := range pd.chunks { |
||||
pd.chunks[i] = 0 // Logic below relies on zero value as uninitialized
|
||||
} |
||||
for _, c := range codes { |
||||
if c.Len > pd.chunkBits && pd.chunks[c.Val&pd.chunkMask] == 0 { |
||||
pd.chunks[c.Val&pd.chunkMask] = (linkIdx << countBits) | (pd.chunkBits + 1) |
||||
linkIdx++ |
||||
} |
||||
} |
||||
|
||||
pd.links = extendSliceUint32s(pd.links, int(linkIdx)) |
||||
linksFlat := allocUint32s(pd.links[0], numLinks*int(linkIdx)) |
||||
for i, j := 0, 0; i < len(pd.links); i, j = i+1, j+numLinks { |
||||
pd.links[i] = linksFlat[j : j+numLinks] |
||||
} |
||||
} |
||||
|
||||
// Fill out chunks and links tables with values.
|
||||
for _, c := range codes { |
||||
chunk := c.Sym<<countBits | c.Len |
||||
if c.Len <= pd.chunkBits { |
||||
skip := 1 << uint(c.Len) |
||||
for j := int(c.Val); j < len(pd.chunks); j += skip { |
||||
pd.chunks[j] = chunk |
||||
} |
||||
} else { |
||||
linkIdx := pd.chunks[c.Val&pd.chunkMask] >> countBits |
||||
links := pd.links[linkIdx] |
||||
skip := 1 << uint(c.Len-pd.chunkBits) |
||||
for j := int(c.Val >> pd.chunkBits); j < len(links); j += skip { |
||||
links[j] = chunk |
||||
} |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,66 @@ |
||||
// Copyright 2015, Joe Tsai. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE.md file.
|
||||
|
||||
package prefix |
||||
|
||||
import ( |
||||
"sort" |
||||
|
||||
"github.com/dsnet/compress/internal" |
||||
) |
||||
|
||||
type Encoder struct { |
||||
chunks []uint32 // First-level lookup map
|
||||
chunkMask uint32 // Mask the length of the chunks table
|
||||
|
||||
NumSyms uint32 // Number of symbols
|
||||
} |
||||
|
||||
// Init initializes Encoder according to the codes provided.
|
||||
func (pe *Encoder) Init(codes PrefixCodes) { |
||||
// Handle special case trees.
|
||||
if len(codes) <= 1 { |
||||
switch { |
||||
case len(codes) == 0: // Empty tree (should error if used later)
|
||||
*pe = Encoder{chunks: pe.chunks[:0], NumSyms: 0} |
||||
case len(codes) == 1 && codes[0].Len == 0: // Single code tree (bit-length of zero)
|
||||
pe.chunks = append(pe.chunks[:0], codes[0].Val<<countBits|0) |
||||
*pe = Encoder{chunks: pe.chunks[:1], NumSyms: 1} |
||||
default: |
||||
panic("invalid codes") |
||||
} |
||||
return |
||||
} |
||||
if internal.Debug && !sort.IsSorted(prefixCodesBySymbol(codes)) { |
||||
panic("input codes is not sorted") |
||||
} |
||||
if internal.Debug && !(codes.checkLengths() && codes.checkPrefixes()) { |
||||
panic("detected incomplete or overlapping codes") |
||||
} |
||||
|
||||
// Enough chunks to contain all the symbols.
|
||||
numChunks := 1 |
||||
for n := len(codes) - 1; n > 0; n >>= 1 { |
||||
numChunks <<= 1 |
||||
} |
||||
pe.NumSyms = uint32(len(codes)) |
||||
|
||||
retry: |
||||
// Allocate and reset chunks.
|
||||
pe.chunks = allocUint32s(pe.chunks, numChunks) |
||||
pe.chunkMask = uint32(numChunks - 1) |
||||
for i := range pe.chunks { |
||||
pe.chunks[i] = 0 // Logic below relies on zero value as uninitialized
|
||||
} |
||||
|
||||
// Insert each symbol, checking that there are no conflicts.
|
||||
for _, c := range codes { |
||||
if pe.chunks[c.Sym&pe.chunkMask] > 0 { |
||||
// Collision found our "hash" table, so grow and try again.
|
||||
numChunks <<= 1 |
||||
goto retry |
||||
} |
||||
pe.chunks[c.Sym&pe.chunkMask] = c.Val<<countBits | c.Len |
||||
} |
||||
} |
||||
@ -0,0 +1,400 @@ |
||||
// Copyright 2015, Joe Tsai. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE.md file.
|
||||
|
||||
// Package prefix implements bit readers and writers that use prefix encoding.
|
||||
package prefix |
||||
|
||||
import ( |
||||
"fmt" |
||||
"sort" |
||||
|
||||
"github.com/dsnet/compress/internal" |
||||
"github.com/dsnet/compress/internal/errors" |
||||
) |
||||
|
||||
func errorf(c int, f string, a ...interface{}) error { |
||||
return errors.Error{Code: c, Pkg: "prefix", Msg: fmt.Sprintf(f, a...)} |
||||
} |
||||
|
||||
func panicf(c int, f string, a ...interface{}) { |
||||
errors.Panic(errorf(c, f, a...)) |
||||
} |
||||
|
||||
const ( |
||||
countBits = 5 // Number of bits to store the bit-length of the code
|
||||
valueBits = 27 // Number of bits to store the code value
|
||||
|
||||
countMask = (1 << countBits) - 1 |
||||
) |
||||
|
||||
// PrefixCode is a representation of a prefix code, which is conceptually a
|
||||
// mapping from some arbitrary symbol to some bit-string.
|
||||
//
|
||||
// The Sym and Cnt fields are typically provided by the user,
|
||||
// while the Len and Val fields are generated by this package.
|
||||
type PrefixCode struct { |
||||
Sym uint32 // The symbol being mapped
|
||||
Cnt uint32 // The number times this symbol is used
|
||||
Len uint32 // Bit-length of the prefix code
|
||||
Val uint32 // Value of the prefix code (must be in 0..(1<<Len)-1)
|
||||
} |
||||
type PrefixCodes []PrefixCode |
||||
|
||||
type prefixCodesBySymbol []PrefixCode |
||||
|
||||
func (c prefixCodesBySymbol) Len() int { return len(c) } |
||||
func (c prefixCodesBySymbol) Less(i, j int) bool { return c[i].Sym < c[j].Sym } |
||||
func (c prefixCodesBySymbol) Swap(i, j int) { c[i], c[j] = c[j], c[i] } |
||||
|
||||
type prefixCodesByCount []PrefixCode |
||||
|
||||
func (c prefixCodesByCount) Len() int { return len(c) } |
||||
func (c prefixCodesByCount) Less(i, j int) bool { |
||||
return c[i].Cnt < c[j].Cnt || (c[i].Cnt == c[j].Cnt && c[i].Sym < c[j].Sym) |
||||
} |
||||
func (c prefixCodesByCount) Swap(i, j int) { c[i], c[j] = c[j], c[i] } |
||||
|
||||
func (pc PrefixCodes) SortBySymbol() { sort.Sort(prefixCodesBySymbol(pc)) } |
||||
func (pc PrefixCodes) SortByCount() { sort.Sort(prefixCodesByCount(pc)) } |
||||
|
||||
// Length computes the total bit-length using the Len and Cnt fields.
|
||||
func (pc PrefixCodes) Length() (nb uint) { |
||||
for _, c := range pc { |
||||
nb += uint(c.Len * c.Cnt) |
||||
} |
||||
return nb |
||||
} |
||||
|
||||
// checkLengths reports whether the codes form a complete prefix tree.
|
||||
func (pc PrefixCodes) checkLengths() bool { |
||||
sum := 1 << valueBits |
||||
for _, c := range pc { |
||||
sum -= (1 << valueBits) >> uint(c.Len) |
||||
} |
||||
return sum == 0 || len(pc) == 0 |
||||
} |
||||
|
||||
// checkPrefixes reports whether all codes have non-overlapping prefixes.
|
||||
func (pc PrefixCodes) checkPrefixes() bool { |
||||
for i, c1 := range pc { |
||||
for j, c2 := range pc { |
||||
mask := uint32(1)<<c1.Len - 1 |
||||
if i != j && c1.Len <= c2.Len && c1.Val&mask == c2.Val&mask { |
||||
return false |
||||
} |
||||
} |
||||
} |
||||
return true |
||||
} |
||||
|
||||
// checkCanonical reports whether all codes are canonical.
|
||||
// That is, they have the following properties:
|
||||
//
|
||||
// 1. All codes of a given bit-length are consecutive values.
|
||||
// 2. Shorter codes lexicographically precede longer codes.
|
||||
//
|
||||
// The codes must have unique symbols and be sorted by the symbol
|
||||
// The Len and Val fields in each code must be populated.
|
||||
func (pc PrefixCodes) checkCanonical() bool { |
||||
// Rule 1.
|
||||
var vals [valueBits + 1]PrefixCode |
||||
for _, c := range pc { |
||||
if c.Len > 0 { |
||||
c.Val = internal.ReverseUint32N(c.Val, uint(c.Len)) |
||||
if vals[c.Len].Cnt > 0 && vals[c.Len].Val+1 != c.Val { |
||||
return false |
||||
} |
||||
vals[c.Len].Val = c.Val |
||||
vals[c.Len].Cnt++ |
||||
} |
||||
} |
||||
|
||||
// Rule 2.
|
||||
var last PrefixCode |
||||
for _, v := range vals { |
||||
if v.Cnt > 0 { |
||||
curVal := v.Val - v.Cnt + 1 |
||||
if last.Cnt != 0 && last.Val >= curVal { |
||||
return false |
||||
} |
||||
last = v |
||||
} |
||||
} |
||||
return true |
||||
} |
||||
|
||||
// GenerateLengths assigns non-zero bit-lengths to all codes. Codes with high
|
||||
// frequency counts will be assigned shorter codes to reduce bit entropy.
|
||||
// This function is used primarily by compressors.
|
||||
//
|
||||
// The input codes must have the Cnt field populated, be sorted by count.
|
||||
// Even if a code has a count of 0, a non-zero bit-length will be assigned.
|
||||
//
|
||||
// The result will have the Len field populated. The algorithm used guarantees
|
||||
// that Len <= maxBits and that it is a complete prefix tree. The resulting
|
||||
// codes will remain sorted by count.
|
||||
func GenerateLengths(codes PrefixCodes, maxBits uint) error { |
||||
if len(codes) <= 1 { |
||||
if len(codes) == 1 { |
||||
codes[0].Len = 0 |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
// Verify that the codes are in ascending order by count.
|
||||
cntLast := codes[0].Cnt |
||||
for _, c := range codes[1:] { |
||||
if c.Cnt < cntLast { |
||||
return errorf(errors.Invalid, "non-monotonically increasing symbol counts") |
||||
} |
||||
cntLast = c.Cnt |
||||
} |
||||
|
||||
// Construct a Huffman tree used to generate the bit-lengths.
|
||||
//
|
||||
// The Huffman tree is a binary tree where each symbol lies as a leaf node
|
||||
// on this tree. The length of the prefix code to assign is the depth of
|
||||
// that leaf from the root. The Huffman algorithm, which runs in O(n),
|
||||
// is used to generate the tree. It assumes that codes are sorted in
|
||||
// increasing order of frequency.
|
||||
//
|
||||
// The algorithm is as follows:
|
||||
// 1. Start with two queues, F and Q, where F contains all of the starting
|
||||
// symbols sorted such that symbols with lowest counts come first.
|
||||
// 2. While len(F)+len(Q) > 1:
|
||||
// 2a. Dequeue the node from F or Q that has the lowest weight as N0.
|
||||
// 2b. Dequeue the node from F or Q that has the lowest weight as N1.
|
||||
// 2c. Create a new node N that has N0 and N1 as its children.
|
||||
// 2d. Enqueue N into the back of Q.
|
||||
// 3. The tree's root node is Q[0].
|
||||
type node struct { |
||||
cnt uint32 |
||||
|
||||
// n0 or c0 represent the left child of this node.
|
||||
// Since Go does not have unions, only one of these will be set.
|
||||
// Similarly, n1 or c1 represent the right child of this node.
|
||||
//
|
||||
// If n0 or n1 is set, then it represents a "pointer" to another
|
||||
// node in the Huffman tree. Since Go's pointer analysis cannot reason
|
||||
// that these node pointers do not escape (golang.org/issue/13493),
|
||||
// we use an index to a node in the nodes slice as a pseudo-pointer.
|
||||
//
|
||||
// If c0 or c1 is set, then it represents a leaf "node" in the
|
||||
// Huffman tree. The leaves are the PrefixCode values themselves.
|
||||
n0, n1 int // Index to child nodes
|
||||
c0, c1 *PrefixCode |
||||
} |
||||
var nodeIdx int |
||||
var nodeArr [1024]node // Large enough to handle most cases on the stack
|
||||
nodes := nodeArr[:] |
||||
if len(nodes) < len(codes) { |
||||
nodes = make([]node, len(codes)) // Number of internal nodes < number of leaves
|
||||
} |
||||
freqs, queue := codes, nodes[:0] |
||||
for len(freqs)+len(queue) > 1 { |
||||
// These are the two smallest nodes at the front of freqs and queue.
|
||||
var n node |
||||
if len(queue) == 0 || (len(freqs) > 0 && freqs[0].Cnt <= queue[0].cnt) { |
||||
n.c0, freqs = &freqs[0], freqs[1:] |
||||
n.cnt += n.c0.Cnt |
||||
} else { |
||||
n.cnt += queue[0].cnt |
||||
n.n0 = nodeIdx // nodeIdx is same as &queue[0] - &nodes[0]
|
||||
nodeIdx++ |
||||
queue = queue[1:] |
||||
} |
||||
if len(queue) == 0 || (len(freqs) > 0 && freqs[0].Cnt <= queue[0].cnt) { |
||||
n.c1, freqs = &freqs[0], freqs[1:] |
||||
n.cnt += n.c1.Cnt |
||||
} else { |
||||
n.cnt += queue[0].cnt |
||||
n.n1 = nodeIdx // nodeIdx is same as &queue[0] - &nodes[0]
|
||||
nodeIdx++ |
||||
queue = queue[1:] |
||||
} |
||||
queue = append(queue, n) |
||||
} |
||||
rootIdx := nodeIdx |
||||
|
||||
// Search the whole binary tree, noting when we hit each leaf node.
|
||||
// We do not care about the exact Huffman tree structure, but rather we only
|
||||
// care about depth of each of the leaf nodes. That is, the depth determines
|
||||
// how long each symbol is in bits.
|
||||
//
|
||||
// Since the number of leaves is n, there is at most n internal nodes.
|
||||
// Thus, this algorithm runs in O(n).
|
||||
var fixBits bool |
||||
var explore func(int, uint) |
||||
explore = func(rootIdx int, level uint) { |
||||
root := &nodes[rootIdx] |
||||
|
||||
// Explore left branch.
|
||||
if root.c0 == nil { |
||||
explore(root.n0, level+1) |
||||
} else { |
||||
fixBits = fixBits || (level > maxBits) |
||||
root.c0.Len = uint32(level) |
||||
} |
||||
|
||||
// Explore right branch.
|
||||
if root.c1 == nil { |
||||
explore(root.n1, level+1) |
||||
} else { |
||||
fixBits = fixBits || (level > maxBits) |
||||
root.c1.Len = uint32(level) |
||||
} |
||||
} |
||||
explore(rootIdx, 1) |
||||
|
||||
// Fix the bit-lengths if we violate the maxBits requirement.
|
||||
if fixBits { |
||||
// Create histogram for number of symbols with each bit-length.
|
||||
var symBitsArr [valueBits + 1]uint32 |
||||
symBits := symBitsArr[:] // symBits[nb] indicates number of symbols using nb bits
|
||||
for _, c := range codes { |
||||
for int(c.Len) >= len(symBits) { |
||||
symBits = append(symBits, 0) |
||||
} |
||||
symBits[c.Len]++ |
||||
} |
||||
|
||||
// Fudge the tree such that the largest bit-length is <= maxBits.
|
||||
// This is accomplish by effectively doing a tree rotation. That is, we
|
||||
// increase the bit-length of some higher frequency code, so that the
|
||||
// bit-lengths of lower frequency codes can be decreased.
|
||||
//
|
||||
// Visually, this looks like the following transform:
|
||||
//
|
||||
// Level Before After
|
||||
// __ ___
|
||||
// / \ / \
|
||||
// n-1 X / \ /\ /\
|
||||
// n X /\ X X X X
|
||||
// n+1 X X
|
||||
//
|
||||
var treeRotate func(uint) |
||||
treeRotate = func(nb uint) { |
||||
if symBits[nb-1] == 0 { |
||||
treeRotate(nb - 1) |
||||
} |
||||
symBits[nb-1] -= 1 // Push this node to the level below
|
||||
symBits[nb] += 3 // This level gets one node from above, two from below
|
||||
symBits[nb+1] -= 2 // Push two nodes to the level above
|
||||
} |
||||
for i := uint(len(symBits)) - 1; i > maxBits; i-- { |
||||
for symBits[i] > 0 { |
||||
treeRotate(i - 1) |
||||
} |
||||
} |
||||
|
||||
// Assign bit-lengths to each code. Since codes is sorted in increasing
|
||||
// order of frequency, that means that the most frequently used symbols
|
||||
// should have the shortest bit-lengths. Thus, we copy symbols to codes
|
||||
// from the back of codes first.
|
||||
cs := codes |
||||
for nb, cnt := range symBits { |
||||
if cnt > 0 { |
||||
pos := len(cs) - int(cnt) |
||||
cs2 := cs[pos:] |
||||
for i := range cs2 { |
||||
cs2[i].Len = uint32(nb) |
||||
} |
||||
cs = cs[:pos] |
||||
} |
||||
} |
||||
if len(cs) != 0 { |
||||
panic("not all codes were used up") |
||||
} |
||||
} |
||||
|
||||
if internal.Debug && !codes.checkLengths() { |
||||
panic("incomplete prefix tree detected") |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
// GeneratePrefixes assigns a prefix value to all codes according to the
|
||||
// bit-lengths. This function is used by both compressors and decompressors.
|
||||
//
|
||||
// The input codes must have the Sym and Len fields populated and be
|
||||
// sorted by symbol. The bit-lengths of each code must be properly allocated,
|
||||
// such that it forms a complete tree.
|
||||
//
|
||||
// The result will have the Val field populated and will produce a canonical
|
||||
// prefix tree. The resulting codes will remain sorted by symbol.
|
||||
func GeneratePrefixes(codes PrefixCodes) error { |
||||
if len(codes) <= 1 { |
||||
if len(codes) == 1 { |
||||
if codes[0].Len != 0 { |
||||
return errorf(errors.Invalid, "degenerate prefix tree with one node") |
||||
} |
||||
codes[0].Val = 0 |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
// Compute basic statistics on the symbols.
|
||||
var bitCnts [valueBits + 1]uint |
||||
c0 := codes[0] |
||||
bitCnts[c0.Len]++ |
||||
minBits, maxBits, symLast := c0.Len, c0.Len, c0.Sym |
||||
for _, c := range codes[1:] { |
||||
if c.Sym <= symLast { |
||||
return errorf(errors.Invalid, "non-unique or non-monotonically increasing symbols") |
||||
} |
||||
if minBits > c.Len { |
||||
minBits = c.Len |
||||
} |
||||
if maxBits < c.Len { |
||||
maxBits = c.Len |
||||
} |
||||
bitCnts[c.Len]++ // Histogram of bit counts
|
||||
symLast = c.Sym // Keep track of last symbol
|
||||
} |
||||
if minBits == 0 { |
||||
return errorf(errors.Invalid, "invalid prefix bit-length") |
||||
} |
||||
|
||||
// Compute the next code for a symbol of a given bit length.
|
||||
var nextCodes [valueBits + 1]uint |
||||
var code uint |
||||
for i := minBits; i <= maxBits; i++ { |
||||
code <<= 1 |
||||
nextCodes[i] = code |
||||
code += bitCnts[i] |
||||
} |
||||
if code != 1<<maxBits { |
||||
return errorf(errors.Invalid, "degenerate prefix tree") |
||||
} |
||||
|
||||
// Assign the code to each symbol.
|
||||
for i, c := range codes { |
||||
codes[i].Val = internal.ReverseUint32N(uint32(nextCodes[c.Len]), uint(c.Len)) |
||||
nextCodes[c.Len]++ |
||||
} |
||||
|
||||
if internal.Debug && !codes.checkPrefixes() { |
||||
panic("overlapping prefixes detected") |
||||
} |
||||
if internal.Debug && !codes.checkCanonical() { |
||||
panic("non-canonical prefixes detected") |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
func allocUint32s(s []uint32, n int) []uint32 { |
||||
if cap(s) >= n { |
||||
return s[:n] |
||||
} |
||||
return make([]uint32, n, n*3/2) |
||||
} |
||||
|
||||
func extendSliceUint32s(s [][]uint32, n int) [][]uint32 { |
||||
if cap(s) >= n { |
||||
return s[:n] |
||||
} |
||||
ss := make([][]uint32, n, n*3/2) |
||||
copy(ss, s[:cap(s)]) |
||||
return ss |
||||
} |
||||
@ -0,0 +1,93 @@ |
||||
// Copyright 2015, Joe Tsai. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE.md file.
|
||||
|
||||
package prefix |
||||
|
||||
type RangeCode struct { |
||||
Base uint32 // Starting base offset of the range
|
||||
Len uint32 // Bit-length of a subsequent integer to add to base offset
|
||||
} |
||||
type RangeCodes []RangeCode |
||||
|
||||
type RangeEncoder struct { |
||||
rcs RangeCodes |
||||
lut [1024]uint32 |
||||
minBase uint |
||||
} |
||||
|
||||
// End reports the non-inclusive ending range.
|
||||
func (rc RangeCode) End() uint32 { return rc.Base + (1 << rc.Len) } |
||||
|
||||
// MakeRangeCodes creates a RangeCodes, where each region is assumed to be
|
||||
// contiguously stacked, without any gaps, with bit-lengths taken from bits.
|
||||
func MakeRangeCodes(minBase uint, bits []uint) (rc RangeCodes) { |
||||
for _, nb := range bits { |
||||
rc = append(rc, RangeCode{Base: uint32(minBase), Len: uint32(nb)}) |
||||
minBase += 1 << nb |
||||
} |
||||
return rc |
||||
} |
||||
|
||||
// Base reports the inclusive starting range for all ranges.
|
||||
func (rcs RangeCodes) Base() uint32 { return rcs[0].Base } |
||||
|
||||
// End reports the non-inclusive ending range for all ranges.
|
||||
func (rcs RangeCodes) End() uint32 { return rcs[len(rcs)-1].End() } |
||||
|
||||
// checkValid reports whether the RangeCodes is valid. In order to be valid,
|
||||
// the following must hold true:
|
||||
// rcs[i-1].Base <= rcs[i].Base
|
||||
// rcs[i-1].End <= rcs[i].End
|
||||
// rcs[i-1].End >= rcs[i].Base
|
||||
//
|
||||
// Practically speaking, each range must be increasing and must not have any
|
||||
// gaps in between. It is okay for ranges to overlap.
|
||||
func (rcs RangeCodes) checkValid() bool { |
||||
if len(rcs) == 0 { |
||||
return false |
||||
} |
||||
pre := rcs[0] |
||||
for _, cur := range rcs[1:] { |
||||
preBase, preEnd := pre.Base, pre.End() |
||||
curBase, curEnd := cur.Base, cur.End() |
||||
if preBase > curBase || preEnd > curEnd || preEnd < curBase { |
||||
return false |
||||
} |
||||
pre = cur |
||||
} |
||||
return true |
||||
} |
||||
|
||||
func (re *RangeEncoder) Init(rcs RangeCodes) { |
||||
if !rcs.checkValid() { |
||||
panic("invalid range codes") |
||||
} |
||||
*re = RangeEncoder{rcs: rcs, minBase: uint(rcs.Base())} |
||||
for sym, rc := range rcs { |
||||
base := int(rc.Base) - int(re.minBase) |
||||
end := int(rc.End()) - int(re.minBase) |
||||
if base >= len(re.lut) { |
||||
break |
||||
} |
||||
if end > len(re.lut) { |
||||
end = len(re.lut) |
||||
} |
||||
for i := base; i < end; i++ { |
||||
re.lut[i] = uint32(sym) |
||||
} |
||||
} |
||||
} |
||||
|
||||
func (re *RangeEncoder) Encode(offset uint) (sym uint) { |
||||
if idx := int(offset - re.minBase); idx < len(re.lut) { |
||||
return uint(re.lut[idx]) |
||||
} |
||||
sym = uint(re.lut[len(re.lut)-1]) |
||||
retry: |
||||
if int(sym) >= len(re.rcs) || re.rcs[sym].Base > uint32(offset) { |
||||
return sym - 1 |
||||
} |
||||
sym++ |
||||
goto retry // Avoid for-loop so that this function can be inlined
|
||||
} |
||||
@ -0,0 +1,335 @@ |
||||
// Copyright 2015, Joe Tsai. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE.md file.
|
||||
|
||||
package prefix |
||||
|
||||
import ( |
||||
"bufio" |
||||
"bytes" |
||||
"encoding/binary" |
||||
"io" |
||||
"strings" |
||||
|
||||
"github.com/dsnet/compress" |
||||
"github.com/dsnet/compress/internal" |
||||
"github.com/dsnet/compress/internal/errors" |
||||
) |
||||
|
||||
// Reader implements a prefix decoder. If the input io.Reader satisfies the
|
||||
// compress.ByteReader or compress.BufferedReader interface, then it also
|
||||
// guarantees that it will never read more bytes than is necessary.
|
||||
//
|
||||
// For high performance, provide an io.Reader that satisfies the
|
||||
// compress.BufferedReader interface. If the input does not satisfy either
|
||||
// compress.ByteReader or compress.BufferedReader, then it will be internally
|
||||
// wrapped with a bufio.Reader.
|
||||
type Reader struct { |
||||
Offset int64 // Number of bytes read from the underlying io.Reader
|
||||
|
||||
rd io.Reader |
||||
byteRd compress.ByteReader // Set if rd is a ByteReader
|
||||
bufRd compress.BufferedReader // Set if rd is a BufferedReader
|
||||
|
||||
bufBits uint64 // Buffer to hold some bits
|
||||
numBits uint // Number of valid bits in bufBits
|
||||
bigEndian bool // Do we treat input bytes as big endian?
|
||||
|
||||
// These fields are only used if rd is a compress.BufferedReader.
|
||||
bufPeek []byte // Buffer for the Peek data
|
||||
discardBits int // Number of bits to discard from reader
|
||||
fedBits uint // Number of bits fed in last call to PullBits
|
||||
|
||||
// These fields are used to reduce allocations.
|
||||
bb *buffer |
||||
br *bytesReader |
||||
sr *stringReader |
||||
bu *bufio.Reader |
||||
} |
||||
|
||||
// Init initializes the bit Reader to read from r. If bigEndian is true, then
|
||||
// bits will be read starting from the most-significant bits of a byte
|
||||
// (as done in bzip2), otherwise it will read starting from the
|
||||
// least-significant bits of a byte (such as for deflate and brotli).
|
||||
func (pr *Reader) Init(r io.Reader, bigEndian bool) { |
||||
*pr = Reader{ |
||||
rd: r, |
||||
bigEndian: bigEndian, |
||||
|
||||
bb: pr.bb, |
||||
br: pr.br, |
||||
sr: pr.sr, |
||||
bu: pr.bu, |
||||
} |
||||
switch rr := r.(type) { |
||||
case *bytes.Buffer: |
||||
if pr.bb == nil { |
||||
pr.bb = new(buffer) |
||||
} |
||||
*pr.bb = buffer{Buffer: rr} |
||||
pr.bufRd = pr.bb |
||||
case *bytes.Reader: |
||||
if pr.br == nil { |
||||
pr.br = new(bytesReader) |
||||
} |
||||
*pr.br = bytesReader{Reader: rr} |
||||
pr.bufRd = pr.br |
||||
case *strings.Reader: |
||||
if pr.sr == nil { |
||||
pr.sr = new(stringReader) |
||||
} |
||||
*pr.sr = stringReader{Reader: rr} |
||||
pr.bufRd = pr.sr |
||||
case compress.BufferedReader: |
||||
pr.bufRd = rr |
||||
case compress.ByteReader: |
||||
pr.byteRd = rr |
||||
default: |
||||
if pr.bu == nil { |
||||
pr.bu = bufio.NewReader(nil) |
||||
} |
||||
pr.bu.Reset(r) |
||||
pr.rd, pr.bufRd = pr.bu, pr.bu |
||||
} |
||||
} |
||||
|
||||
// BitsRead reports the total number of bits emitted from any Read method.
|
||||
func (pr *Reader) BitsRead() int64 { |
||||
offset := 8*pr.Offset - int64(pr.numBits) |
||||
if pr.bufRd != nil { |
||||
discardBits := pr.discardBits + int(pr.fedBits-pr.numBits) |
||||
offset = 8*pr.Offset + int64(discardBits) |
||||
} |
||||
return offset |
||||
} |
||||
|
||||
// IsBufferedReader reports whether the underlying io.Reader is also a
|
||||
// compress.BufferedReader.
|
||||
func (pr *Reader) IsBufferedReader() bool { |
||||
return pr.bufRd != nil |
||||
} |
||||
|
||||
// ReadPads reads 0-7 bits from the bit buffer to achieve byte-alignment.
|
||||
func (pr *Reader) ReadPads() uint { |
||||
nb := pr.numBits % 8 |
||||
val := uint(pr.bufBits & uint64(1<<nb-1)) |
||||
pr.bufBits >>= nb |
||||
pr.numBits -= nb |
||||
return val |
||||
} |
||||
|
||||
// Read reads bytes into buf.
|
||||
// The bit-ordering mode does not affect this method.
|
||||
func (pr *Reader) Read(buf []byte) (cnt int, err error) { |
||||
if pr.numBits > 0 { |
||||
if pr.numBits%8 != 0 { |
||||
return 0, errorf(errors.Invalid, "non-aligned bit buffer") |
||||
} |
||||
for cnt = 0; len(buf) > cnt && pr.numBits > 0; cnt++ { |
||||
if pr.bigEndian { |
||||
buf[cnt] = internal.ReverseLUT[byte(pr.bufBits)] |
||||
} else { |
||||
buf[cnt] = byte(pr.bufBits) |
||||
} |
||||
pr.bufBits >>= 8 |
||||
pr.numBits -= 8 |
||||
} |
||||
return cnt, nil |
||||
} |
||||
if _, err := pr.Flush(); err != nil { |
||||
return 0, err |
||||
} |
||||
cnt, err = pr.rd.Read(buf) |
||||
pr.Offset += int64(cnt) |
||||
return cnt, err |
||||
} |
||||
|
||||
// ReadOffset reads an offset value using the provided RangeCodes indexed by
|
||||
// the symbol read.
|
||||
func (pr *Reader) ReadOffset(pd *Decoder, rcs RangeCodes) uint { |
||||
rc := rcs[pr.ReadSymbol(pd)] |
||||
return uint(rc.Base) + pr.ReadBits(uint(rc.Len)) |
||||
} |
||||
|
||||
// TryReadBits attempts to read nb bits using the contents of the bit buffer
|
||||
// alone. It returns the value and whether it succeeded.
|
||||
//
|
||||
// This method is designed to be inlined for performance reasons.
|
||||
func (pr *Reader) TryReadBits(nb uint) (uint, bool) { |
||||
if pr.numBits < nb { |
||||
return 0, false |
||||
} |
||||
val := uint(pr.bufBits & uint64(1<<nb-1)) |
||||
pr.bufBits >>= nb |
||||
pr.numBits -= nb |
||||
return val, true |
||||
} |
||||
|
||||
// ReadBits reads nb bits in from the underlying reader.
|
||||
func (pr *Reader) ReadBits(nb uint) uint { |
||||
if err := pr.PullBits(nb); err != nil { |
||||
errors.Panic(err) |
||||
} |
||||
val := uint(pr.bufBits & uint64(1<<nb-1)) |
||||
pr.bufBits >>= nb |
||||
pr.numBits -= nb |
||||
return val |
||||
} |
||||
|
||||
// TryReadSymbol attempts to decode the next symbol using the contents of the
|
||||
// bit buffer alone. It returns the decoded symbol and whether it succeeded.
|
||||
//
|
||||
// This method is designed to be inlined for performance reasons.
|
||||
func (pr *Reader) TryReadSymbol(pd *Decoder) (uint, bool) { |
||||
if pr.numBits < uint(pd.MinBits) || len(pd.chunks) == 0 { |
||||
return 0, false |
||||
} |
||||
chunk := pd.chunks[uint32(pr.bufBits)&pd.chunkMask] |
||||
nb := uint(chunk & countMask) |
||||
if nb > pr.numBits || nb > uint(pd.chunkBits) { |
||||
return 0, false |
||||
} |
||||
pr.bufBits >>= nb |
||||
pr.numBits -= nb |
||||
return uint(chunk >> countBits), true |
||||
} |
||||
|
||||
// ReadSymbol reads the next symbol using the provided prefix Decoder.
|
||||
func (pr *Reader) ReadSymbol(pd *Decoder) uint { |
||||
if len(pd.chunks) == 0 { |
||||
panicf(errors.Invalid, "decode with empty prefix tree") |
||||
} |
||||
|
||||
nb := uint(pd.MinBits) |
||||
for { |
||||
if err := pr.PullBits(nb); err != nil { |
||||
errors.Panic(err) |
||||
} |
||||
chunk := pd.chunks[uint32(pr.bufBits)&pd.chunkMask] |
||||
nb = uint(chunk & countMask) |
||||
if nb > uint(pd.chunkBits) { |
||||
linkIdx := chunk >> countBits |
||||
chunk = pd.links[linkIdx][uint32(pr.bufBits>>pd.chunkBits)&pd.linkMask] |
||||
nb = uint(chunk & countMask) |
||||
} |
||||
if nb <= pr.numBits { |
||||
pr.bufBits >>= nb |
||||
pr.numBits -= nb |
||||
return uint(chunk >> countBits) |
||||
} |
||||
} |
||||
} |
||||
|
||||
// Flush updates the read offset of the underlying ByteReader.
|
||||
// If reader is a compress.BufferedReader, then this calls Discard to update
|
||||
// the read offset.
|
||||
func (pr *Reader) Flush() (int64, error) { |
||||
if pr.bufRd == nil { |
||||
return pr.Offset, nil |
||||
} |
||||
|
||||
// Update the number of total bits to discard.
|
||||
pr.discardBits += int(pr.fedBits - pr.numBits) |
||||
pr.fedBits = pr.numBits |
||||
|
||||
// Discard some bytes to update read offset.
|
||||
var err error |
||||
nd := (pr.discardBits + 7) / 8 // Round up to nearest byte
|
||||
nd, err = pr.bufRd.Discard(nd) |
||||
pr.discardBits -= nd * 8 // -7..0
|
||||
pr.Offset += int64(nd) |
||||
|
||||
// These are invalid after Discard.
|
||||
pr.bufPeek = nil |
||||
return pr.Offset, err |
||||
} |
||||
|
||||
// PullBits ensures that at least nb bits exist in the bit buffer.
|
||||
// If the underlying reader is a compress.BufferedReader, then this will fill
|
||||
// the bit buffer with as many bits as possible, relying on Peek and Discard to
|
||||
// properly advance the read offset. Otherwise, it will use ReadByte to fill the
|
||||
// buffer with just the right number of bits.
|
||||
func (pr *Reader) PullBits(nb uint) error { |
||||
if pr.bufRd != nil { |
||||
pr.discardBits += int(pr.fedBits - pr.numBits) |
||||
for { |
||||
if len(pr.bufPeek) == 0 { |
||||
pr.fedBits = pr.numBits // Don't discard bits just added
|
||||
if _, err := pr.Flush(); err != nil { |
||||
return err |
||||
} |
||||
|
||||
// Peek no more bytes than necessary.
|
||||
// The computation for cntPeek computes the minimum number of
|
||||
// bytes to Peek to fill nb bits.
|
||||
var err error |
||||
cntPeek := int(nb+(-nb&7)) / 8 |
||||
if cntPeek < pr.bufRd.Buffered() { |
||||
cntPeek = pr.bufRd.Buffered() |
||||
} |
||||
pr.bufPeek, err = pr.bufRd.Peek(cntPeek) |
||||
pr.bufPeek = pr.bufPeek[int(pr.numBits/8):] // Skip buffered bits
|
||||
if len(pr.bufPeek) == 0 { |
||||
if pr.numBits >= nb { |
||||
break |
||||
} |
||||
if err == io.EOF { |
||||
err = io.ErrUnexpectedEOF |
||||
} |
||||
return err |
||||
} |
||||
} |
||||
|
||||
n := int(64-pr.numBits) / 8 // Number of bytes to copy to bit buffer
|
||||
if len(pr.bufPeek) >= 8 { |
||||
// Starting with Go 1.7, the compiler should use a wide integer
|
||||
// load here if the architecture supports it.
|
||||
u := binary.LittleEndian.Uint64(pr.bufPeek) |
||||
if pr.bigEndian { |
||||
// Swap all the bits within each byte.
|
||||
u = (u&0xaaaaaaaaaaaaaaaa)>>1 | (u&0x5555555555555555)<<1 |
||||
u = (u&0xcccccccccccccccc)>>2 | (u&0x3333333333333333)<<2 |
||||
u = (u&0xf0f0f0f0f0f0f0f0)>>4 | (u&0x0f0f0f0f0f0f0f0f)<<4 |
||||
} |
||||
|
||||
pr.bufBits |= u << pr.numBits |
||||
pr.numBits += uint(n * 8) |
||||
pr.bufPeek = pr.bufPeek[n:] |
||||
break |
||||
} else { |
||||
if n > len(pr.bufPeek) { |
||||
n = len(pr.bufPeek) |
||||
} |
||||
for _, c := range pr.bufPeek[:n] { |
||||
if pr.bigEndian { |
||||
c = internal.ReverseLUT[c] |
||||
} |
||||
pr.bufBits |= uint64(c) << pr.numBits |
||||
pr.numBits += 8 |
||||
} |
||||
pr.bufPeek = pr.bufPeek[n:] |
||||
if pr.numBits > 56 { |
||||
break |
||||
} |
||||
} |
||||
} |
||||
pr.fedBits = pr.numBits |
||||
} else { |
||||
for pr.numBits < nb { |
||||
c, err := pr.byteRd.ReadByte() |
||||
if err != nil { |
||||
if err == io.EOF { |
||||
err = io.ErrUnexpectedEOF |
||||
} |
||||
return err |
||||
} |
||||
if pr.bigEndian { |
||||
c = internal.ReverseLUT[c] |
||||
} |
||||
pr.bufBits |= uint64(c) << pr.numBits |
||||
pr.numBits += 8 |
||||
pr.Offset++ |
||||
} |
||||
} |
||||
return nil |
||||
} |
||||
@ -0,0 +1,146 @@ |
||||
// Copyright 2015, Joe Tsai. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE.md file.
|
||||
|
||||
package prefix |
||||
|
||||
import ( |
||||
"bytes" |
||||
"io" |
||||
"strings" |
||||
) |
||||
|
||||
// For some of the common Readers, we wrap and extend them to satisfy the
|
||||
// compress.BufferedReader interface to improve performance.
|
||||
|
||||
type buffer struct { |
||||
*bytes.Buffer |
||||
} |
||||
|
||||
type bytesReader struct { |
||||
*bytes.Reader |
||||
pos int64 |
||||
buf []byte |
||||
arr [512]byte |
||||
} |
||||
|
||||
type stringReader struct { |
||||
*strings.Reader |
||||
pos int64 |
||||
buf []byte |
||||
arr [512]byte |
||||
} |
||||
|
||||
func (r *buffer) Buffered() int { |
||||
return r.Len() |
||||
} |
||||
|
||||
func (r *buffer) Peek(n int) ([]byte, error) { |
||||
b := r.Bytes() |
||||
if len(b) < n { |
||||
return b, io.EOF |
||||
} |
||||
return b[:n], nil |
||||
} |
||||
|
||||
func (r *buffer) Discard(n int) (int, error) { |
||||
b := r.Next(n) |
||||
if len(b) < n { |
||||
return len(b), io.EOF |
||||
} |
||||
return n, nil |
||||
} |
||||
|
||||
func (r *bytesReader) Buffered() int { |
||||
r.update() |
||||
if r.Len() > len(r.buf) { |
||||
return len(r.buf) |
||||
} |
||||
return r.Len() |
||||
} |
||||
|
||||
func (r *bytesReader) Peek(n int) ([]byte, error) { |
||||
if n > len(r.arr) { |
||||
return nil, io.ErrShortBuffer |
||||
} |
||||
|
||||
// Return sub-slice of local buffer if possible.
|
||||
r.update() |
||||
if len(r.buf) >= n { |
||||
return r.buf[:n], nil |
||||
} |
||||
|
||||
// Fill entire local buffer, and return appropriate sub-slice.
|
||||
cnt, err := r.ReadAt(r.arr[:], r.pos) |
||||
r.buf = r.arr[:cnt] |
||||
if cnt < n { |
||||
return r.arr[:cnt], err |
||||
} |
||||
return r.arr[:n], nil |
||||
} |
||||
|
||||
func (r *bytesReader) Discard(n int) (int, error) { |
||||
var err error |
||||
if n > r.Len() { |
||||
n, err = r.Len(), io.EOF |
||||
} |
||||
r.Seek(int64(n), io.SeekCurrent) |
||||
return n, err |
||||
} |
||||
|
||||
// update reslices the internal buffer to be consistent with the read offset.
|
||||
func (r *bytesReader) update() { |
||||
pos, _ := r.Seek(0, io.SeekCurrent) |
||||
if off := pos - r.pos; off >= 0 && off < int64(len(r.buf)) { |
||||
r.buf, r.pos = r.buf[off:], pos |
||||
} else { |
||||
r.buf, r.pos = nil, pos |
||||
} |
||||
} |
||||
|
||||
func (r *stringReader) Buffered() int { |
||||
r.update() |
||||
if r.Len() > len(r.buf) { |
||||
return len(r.buf) |
||||
} |
||||
return r.Len() |
||||
} |
||||
|
||||
func (r *stringReader) Peek(n int) ([]byte, error) { |
||||
if n > len(r.arr) { |
||||
return nil, io.ErrShortBuffer |
||||
} |
||||
|
||||
// Return sub-slice of local buffer if possible.
|
||||
r.update() |
||||
if len(r.buf) >= n { |
||||
return r.buf[:n], nil |
||||
} |
||||
|
||||
// Fill entire local buffer, and return appropriate sub-slice.
|
||||
cnt, err := r.ReadAt(r.arr[:], r.pos) |
||||
r.buf = r.arr[:cnt] |
||||
if cnt < n { |
||||
return r.arr[:cnt], err |
||||
} |
||||
return r.arr[:n], nil |
||||
} |
||||
|
||||
func (r *stringReader) Discard(n int) (int, error) { |
||||
var err error |
||||
if n > r.Len() { |
||||
n, err = r.Len(), io.EOF |
||||
} |
||||
r.Seek(int64(n), io.SeekCurrent) |
||||
return n, err |
||||
} |
||||
|
||||
// update reslices the internal buffer to be consistent with the read offset.
|
||||
func (r *stringReader) update() { |
||||
pos, _ := r.Seek(0, io.SeekCurrent) |
||||
if off := pos - r.pos; off >= 0 && off < int64(len(r.buf)) { |
||||
r.buf, r.pos = r.buf[off:], pos |
||||
} else { |
||||
r.buf, r.pos = nil, pos |
||||
} |
||||
} |
||||
@ -0,0 +1,166 @@ |
||||
// Copyright 2015, Joe Tsai. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE.md file.
|
||||
|
||||
package prefix |
||||
|
||||
import ( |
||||
"encoding/binary" |
||||
"io" |
||||
|
||||
"github.com/dsnet/compress/internal/errors" |
||||
) |
||||
|
||||
// Writer implements a prefix encoder. For performance reasons, Writer will not
|
||||
// write bytes immediately to the underlying stream.
|
||||
type Writer struct { |
||||
Offset int64 // Number of bytes written to the underlying io.Writer
|
||||
|
||||
wr io.Writer |
||||
bufBits uint64 // Buffer to hold some bits
|
||||
numBits uint // Number of valid bits in bufBits
|
||||
bigEndian bool // Are bits written in big-endian order?
|
||||
|
||||
buf [512]byte |
||||
cntBuf int |
||||
} |
||||
|
||||
// Init initializes the bit Writer to write to w. If bigEndian is true, then
|
||||
// bits will be written starting from the most-significant bits of a byte
|
||||
// (as done in bzip2), otherwise it will write starting from the
|
||||
// least-significant bits of a byte (such as for deflate and brotli).
|
||||
func (pw *Writer) Init(w io.Writer, bigEndian bool) { |
||||
*pw = Writer{wr: w, bigEndian: bigEndian} |
||||
return |
||||
} |
||||
|
||||
// BitsWritten reports the total number of bits issued to any Write method.
|
||||
func (pw *Writer) BitsWritten() int64 { |
||||
return 8*pw.Offset + 8*int64(pw.cntBuf) + int64(pw.numBits) |
||||
} |
||||
|
||||
// WritePads writes 0-7 bits to the bit buffer to achieve byte-alignment.
|
||||
func (pw *Writer) WritePads(v uint) { |
||||
nb := -pw.numBits & 7 |
||||
pw.bufBits |= uint64(v) << pw.numBits |
||||
pw.numBits += nb |
||||
} |
||||
|
||||
// Write writes bytes from buf.
|
||||
// The bit-ordering mode does not affect this method.
|
||||
func (pw *Writer) Write(buf []byte) (cnt int, err error) { |
||||
if pw.numBits > 0 || pw.cntBuf > 0 { |
||||
if pw.numBits%8 != 0 { |
||||
return 0, errorf(errors.Invalid, "non-aligned bit buffer") |
||||
} |
||||
if _, err := pw.Flush(); err != nil { |
||||
return 0, err |
||||
} |
||||
} |
||||
cnt, err = pw.wr.Write(buf) |
||||
pw.Offset += int64(cnt) |
||||
return cnt, err |
||||
} |
||||
|
||||
// WriteOffset writes ofs in a (sym, extra) fashion using the provided prefix
|
||||
// Encoder and RangeEncoder.
|
||||
func (pw *Writer) WriteOffset(ofs uint, pe *Encoder, re *RangeEncoder) { |
||||
sym := re.Encode(ofs) |
||||
pw.WriteSymbol(sym, pe) |
||||
rc := re.rcs[sym] |
||||
pw.WriteBits(ofs-uint(rc.Base), uint(rc.Len)) |
||||
} |
||||
|
||||
// TryWriteBits attempts to write nb bits using the contents of the bit buffer
|
||||
// alone. It reports whether it succeeded.
|
||||
//
|
||||
// This method is designed to be inlined for performance reasons.
|
||||
func (pw *Writer) TryWriteBits(v, nb uint) bool { |
||||
if 64-pw.numBits < nb { |
||||
return false |
||||
} |
||||
pw.bufBits |= uint64(v) << pw.numBits |
||||
pw.numBits += nb |
||||
return true |
||||
} |
||||
|
||||
// WriteBits writes nb bits of v to the underlying writer.
|
||||
func (pw *Writer) WriteBits(v, nb uint) { |
||||
if _, err := pw.PushBits(); err != nil { |
||||
errors.Panic(err) |
||||
} |
||||
pw.bufBits |= uint64(v) << pw.numBits |
||||
pw.numBits += nb |
||||
} |
||||
|
||||
// TryWriteSymbol attempts to encode the next symbol using the contents of the
|
||||
// bit buffer alone. It reports whether it succeeded.
|
||||
//
|
||||
// This method is designed to be inlined for performance reasons.
|
||||
func (pw *Writer) TryWriteSymbol(sym uint, pe *Encoder) bool { |
||||
chunk := pe.chunks[uint32(sym)&pe.chunkMask] |
||||
nb := uint(chunk & countMask) |
||||
if 64-pw.numBits < nb { |
||||
return false |
||||
} |
||||
pw.bufBits |= uint64(chunk>>countBits) << pw.numBits |
||||
pw.numBits += nb |
||||
return true |
||||
} |
||||
|
||||
// WriteSymbol writes the symbol using the provided prefix Encoder.
|
||||
func (pw *Writer) WriteSymbol(sym uint, pe *Encoder) { |
||||
if _, err := pw.PushBits(); err != nil { |
||||
errors.Panic(err) |
||||
} |
||||
chunk := pe.chunks[uint32(sym)&pe.chunkMask] |
||||
nb := uint(chunk & countMask) |
||||
pw.bufBits |= uint64(chunk>>countBits) << pw.numBits |
||||
pw.numBits += nb |
||||
} |
||||
|
||||
// Flush flushes all complete bytes from the bit buffer to the byte buffer, and
|
||||
// then flushes all bytes in the byte buffer to the underlying writer.
|
||||
// After this call, the bit Writer is will only withhold 7 bits at most.
|
||||
func (pw *Writer) Flush() (int64, error) { |
||||
if pw.numBits < 8 && pw.cntBuf == 0 { |
||||
return pw.Offset, nil |
||||
} |
||||
if _, err := pw.PushBits(); err != nil { |
||||
return pw.Offset, err |
||||
} |
||||
cnt, err := pw.wr.Write(pw.buf[:pw.cntBuf]) |
||||
pw.cntBuf -= cnt |
||||
pw.Offset += int64(cnt) |
||||
return pw.Offset, err |
||||
} |
||||
|
||||
// PushBits pushes as many bytes as possible from the bit buffer to the byte
|
||||
// buffer, reporting the number of bits pushed.
|
||||
func (pw *Writer) PushBits() (uint, error) { |
||||
if pw.cntBuf >= len(pw.buf)-8 { |
||||
cnt, err := pw.wr.Write(pw.buf[:pw.cntBuf]) |
||||
pw.cntBuf -= cnt |
||||
pw.Offset += int64(cnt) |
||||
if err != nil { |
||||
return 0, err |
||||
} |
||||
} |
||||
|
||||
u := pw.bufBits |
||||
if pw.bigEndian { |
||||
// Swap all the bits within each byte.
|
||||
u = (u&0xaaaaaaaaaaaaaaaa)>>1 | (u&0x5555555555555555)<<1 |
||||
u = (u&0xcccccccccccccccc)>>2 | (u&0x3333333333333333)<<2 |
||||
u = (u&0xf0f0f0f0f0f0f0f0)>>4 | (u&0x0f0f0f0f0f0f0f0f)<<4 |
||||
} |
||||
// Starting with Go 1.7, the compiler should use a wide integer
|
||||
// store here if the architecture supports it.
|
||||
binary.LittleEndian.PutUint64(pw.buf[pw.cntBuf:], u) |
||||
|
||||
nb := pw.numBits / 8 // Number of bytes to copy from bit buffer
|
||||
pw.cntBuf += int(nb) |
||||
pw.bufBits >>= 8 * nb |
||||
pw.numBits -= 8 * nb |
||||
return 8 * nb, nil |
||||
} |
||||
@ -0,0 +1,21 @@ |
||||
// Copyright 2015, Joe Tsai. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style
|
||||
// license that can be found in the LICENSE.md file.
|
||||
|
||||
// +build !debug,!gofuzz
|
||||
|
||||
package internal |
||||
|
||||
// Debug indicates whether the debug build tag was set.
|
||||
//
|
||||
// If set, programs may choose to print with more human-readable
|
||||
// debug information and also perform sanity checks that would otherwise be too
|
||||
// expensive to run in a release build.
|
||||
const Debug = false |
||||
|
||||
// GoFuzz indicates whether the gofuzz build tag was set.
|
||||
//
|
||||
// If set, programs may choose to disable certain checks (like checksums) that
|
||||
// would be nearly impossible for gofuzz to properly get right.
|
||||
// If GoFuzz is set, it implies that Debug is set as well.
|
||||
const GoFuzz = false |
||||
@ -0,0 +1,12 @@ |
||||
#!/bin/bash |
||||
# |
||||
# Copyright 2017, Joe Tsai. All rights reserved. |
||||
# Use of this source code is governed by a BSD-style |
||||
# license that can be found in the LICENSE.md file. |
||||
|
||||
# zbench wraps internal/tool/bench and is useful for comparing benchmarks from |
||||
# the implementations in this repository relative to other implementations. |
||||
# |
||||
# See internal/tool/bench/main.go for more details. |
||||
cd $(dirname "${BASH_SOURCE[0]}")/internal/tool/bench |
||||
go run $(go list -f '{{ join .GoFiles "\n" }}') "$@" |
||||
@ -0,0 +1,10 @@ |
||||
#!/bin/bash |
||||
# |
||||
# Copyright 2017, Joe Tsai. All rights reserved. |
||||
# Use of this source code is governed by a BSD-style |
||||
# license that can be found in the LICENSE.md file. |
||||
|
||||
# zfuzz wraps internal/tool/fuzz and is useful for fuzz testing each of |
||||
# the implementations in this repository. |
||||
cd $(dirname "${BASH_SOURCE[0]}")/internal/tool/fuzz |
||||
./fuzz.sh "$@" |
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue