Skip to content

Commit

Permalink
Add CompressOption.preset.
Browse files Browse the repository at this point in the history
this can be used to "train" the model for shared contexts.
can be useful if the model is directly used for compression.

it is currently unclear that this can improve actual packing;
the observed improvement was "only" around 30B for samples
so the eventual decoder cost may outweigh that.
  • Loading branch information
lifthrasiir committed Aug 23, 2021
1 parent 08c3b51 commit 3bf7845
Show file tree
Hide file tree
Showing 2 changed files with 21 additions and 2 deletions.
1 change: 1 addition & 0 deletions index.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@ export class DefaultModel implements LogisticMixModel {

export interface CompressOptions extends AnsOptions {
inBits: number;
preset?: number[];
calculateByteEntropy?: boolean;
}

Expand Down
22 changes: 20 additions & 2 deletions index.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -394,9 +394,18 @@ export class DefaultModel extends LogisticMixModel {
//------------------------------------------------------------------------------

export const compressWithModel = (input, model, options) => {
const { inBits, outBits, precision, calculateByteEntropy } = options;
const { inBits, outBits, precision, preset = [], calculateByteEntropy } = options;
const encoder = new AnsEncoder(options);

for (let offset = 0; offset < preset.length; ++offset) {
const code = preset[offset];
for (let i = inBits - 1; i >= 0; --i) {
model.predict();
model.update((code >> i) & 1);
}
model.flushByte(code, inBits);
}

const byteProbs = [];
for (let offset = 0; offset < input.length; ++offset) {
const code = input[offset];
Expand Down Expand Up @@ -429,9 +438,18 @@ export const compressWithModel = (input, model, options) => {
};

export const decompressWithModel = ({ state, buf, inputLength }, model, options) => {
const { inBits } = options;
const { inBits, preset = [] } = options;
const decoder = new AnsDecoder({ state, buf }, options);

for (let offset = 0; offset < preset.length; ++offset) {
const code = preset[offset];
for (let i = inBits - 1; i >= 0; --i) {
model.predict();
model.update((code >> i) & 1);
}
model.flushByte(code, inBits);
}

const reconstructed = [];
for (let offset = 0; offset < inputLength; ++offset) {
let current = 0;
Expand Down

0 comments on commit 3bf7845

Please sign in to comment.