Class: Cabriolet::Decompressors::LZX

Inherits:
Base
  • Object
show all
Defined in:
lib/cabriolet/decompressors/lzx.rb

Overview

LZX handles LZX compressed data Based on libmspack lzxd.c implementation

The LZX method was created by Jonathan Forbes and Tomi Poutanen, adapted by Microsoft Corporation.

Constant Summary collapse

FRAME_SIZE =

Frame size (32KB per frame)

32_768
BLOCKTYPE_INVALID =

Block types

0
BLOCKTYPE_VERBATIM = 
1
BLOCKTYPE_ALIGNED = 
2
BLOCKTYPE_UNCOMPRESSED = 
3
MIN_MATCH =

Match constants

2
MAX_MATCH = 
257
NUM_CHARS = 
256
PRETREE_NUM_ELEMENTS =

Tree constants

20
PRETREE_MAXSYMBOLS = 
20
PRETREE_TABLEBITS = 
6
ALIGNED_NUM_ELEMENTS = 
8
ALIGNED_MAXSYMBOLS = 
8
ALIGNED_TABLEBITS = 
7
NUM_PRIMARY_LENGTHS = 
7
NUM_SECONDARY_LENGTHS = 
249
LENGTH_MAXSYMBOLS = 
250
LENGTH_TABLEBITS = 
12
POSITION_SLOTS =

Position slots for different window sizes

[30, 32, 34, 36, 38, 42, 50, 66, 98, 162, 290].freeze
EXTRA_BITS =

Extra bits for position slots

[
  0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8,
  9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15, 16, 16
].freeze
POSITION_BASE =

Position base offsets

[
  0, 1, 2, 3, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 192, 256, 384, 512,
  768, 1024, 1536, 2048, 3072, 4096, 6144, 8192, 12_288, 16_384, 24_576, 32_768,
  49_152, 65_536, 98_304, 131_072, 196_608, 262_144, 393_216, 524_288, 655_360,
  786_432, 917_504, 1_048_576, 1_179_648, 1_310_720, 1_441_792, 1_572_864, 1_703_936,
  1_835_008, 1_966_080, 2_097_152, 2_228_224, 2_359_296, 2_490_368, 2_621_440, 2_752_512,
  2_883_584, 3_014_656, 3_145_728, 3_276_800, 3_407_872, 3_538_944, 3_670_016, 3_801_088,
  3_932_160, 4_063_232, 4_194_304, 4_325_376, 4_456_448, 4_587_520, 4_718_592, 4_849_664,
  4_980_736, 5_111_808, 5_242_880, 5_373_952, 5_505_024, 5_636_096, 5_767_168, 5_898_240,
  6_029_312, 6_160_384, 6_291_456, 6_422_528, 6_553_600, 6_684_672, 6_815_744, 6_946_816,
  7_077_888, 7_208_960, 7_340_032, 7_471_104, 7_602_176, 7_733_248, 7_864_320, 7_995_392,
  8_126_464, 8_257_536, 8_388_608, 8_519_680, 8_650_752, 8_781_824, 8_912_896, 9_043_968,
  9_175_040, 9_306_112, 9_437_184, 9_568_256, 9_699_328, 9_830_400, 9_961_472, 10_092_544,
  10_223_616, 10_354_688, 10_485_760, 10_616_832, 10_747_904, 10_878_976, 11_010_048,
  11_141_120, 11_272_192, 11_403_264, 11_534_336, 11_665_408, 11_796_480, 11_927_552,
  12_058_624, 12_189_696, 12_320_768, 12_451_840, 12_582_912, 12_713_984, 12_845_056,
  12_976_128, 13_107_200, 13_238_272, 13_369_344, 13_500_416, 13_631_488, 13_762_560,
  13_893_632, 14_024_704, 14_155_776, 14_286_848, 14_417_920, 14_548_992, 14_680_064,
  14_811_136, 14_942_208, 15_073_280, 15_204_352, 15_335_424, 15_466_496, 15_597_568,
  15_728_640, 15_859_712, 15_990_784, 16_121_856, 16_252_928, 16_384_000, 16_515_072,
  16_646_144, 16_777_216, 16_908_288, 17_039_360, 17_170_432, 17_301_504, 17_432_576,
  17_563_648, 17_694_720, 17_825_792, 17_956_864, 18_087_936, 18_219_008, 18_350_080,
  18_481_152, 18_612_224, 18_743_296, 18_874_368, 19_005_440, 19_136_512, 19_267_584,
  19_398_656, 19_529_728, 19_660_800, 19_791_872, 19_922_944, 20_054_016, 20_185_088,
  20_316_160, 20_447_232, 20_578_304, 20_709_376, 20_840_448, 20_971_520, 21_102_592,
  21_233_664, 21_364_736, 21_495_808, 21_626_880, 21_757_952, 21_889_024, 22_020_096,
  22_151_168, 22_282_240, 22_413_312, 22_544_384, 22_675_456, 22_806_528, 22_937_600,
  23_068_672, 23_199_744, 23_330_816, 23_461_888, 23_592_960, 23_724_032, 23_855_104,
  23_986_176, 24_117_248, 24_248_320, 24_379_392, 24_510_464, 24_641_536, 24_772_608,
  24_903_680, 25_034_752, 25_165_824, 25_296_896, 25_427_968, 25_559_040, 25_690_112,
  25_821_184, 25_952_256, 26_083_328, 26_214_400, 26_345_472, 26_476_544, 26_607_616,
  26_738_688, 26_869_760, 27_000_832, 27_131_904, 27_262_976, 27_394_048, 27_525_120,
  27_656_192, 27_787_264, 27_918_336, 28_049_408, 28_180_480, 28_311_552, 28_442_624,
  28_573_696, 28_704_768, 28_835_840, 28_966_912, 29_097_984, 29_229_056, 29_360_128,
  29_491_200, 29_622_272, 29_753_344, 29_884_416, 30_015_488, 30_146_560, 30_277_632,
  30_408_704, 30_539_776, 30_670_848, 30_801_920, 30_932_992, 31_064_064, 31_195_136,
  31_326_208, 31_457_280, 31_588_352, 31_719_424, 31_850_496, 31_981_568, 32_112_640,
  32_243_712, 32_374_784, 32_505_856, 32_636_928, 32_768_000, 32_899_072, 33_030_144,
  33_161_216, 33_292_288, 33_423_360
].freeze

Instance Attribute Summary collapse

Attributes inherited from Base

#buffer_size, #input, #io_system, #output

Instance Method Summary collapse

Constructor Details

#initialize(io_system, input, output, buffer_size, window_bits:, reset_interval: 0, output_length: 0, is_delta: false, salvage: false, **_kwargs) ⇒ LZX

Initialize LZX decompressor

Parameters:

  • io_system (System::IOSystem) —

    I/O system for reading/writing

  • input (System::FileHandle, System::MemoryHandle) —

    Input handle

  • output (System::FileHandle, System::MemoryHandle) —

    Output handle

  • buffer_size (Integer) —

    Buffer size for I/O operations

  • window_bits (Integer) —

    Window size (15-21 for regular, 17-25 for DELTA)

  • reset_interval (Integer) (defaults to: 0) —

    Frame count between resets (0 = never)

  • output_length (Integer) (defaults to: 0) —

    Expected output length for E8 processing

  • is_delta (Boolean) (defaults to: false) —

    Whether this is LZX DELTA format



104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
 
# File 'lib/cabriolet/decompressors/lzx.rb', line 104

def initialize(io_system, input, output, buffer_size, window_bits:,
               reset_interval: 0, output_length: 0, is_delta: false, salvage: false, **_kwargs)
  super(io_system, input, output, buffer_size)

  # Store salvage flag for error handling
  @salvage = salvage

  # Validate window_bits
  if is_delta
    unless (17..25).cover?(window_bits)
      raise ArgumentError,
            "LZX DELTA window_bits must be 17-25, got #{window_bits}"
    end
  elsif !(15..21).cover?(window_bits)
    raise ArgumentError,
          "LZX window_bits must be 15-21, got #{window_bits}"
  end

  @window_bits = window_bits
  @window_size = 1 << window_bits
  @reset_interval = reset_interval
  @output_length = output_length
  @is_delta = is_delta

  # Calculate number of position slots
  @num_offsets = POSITION_SLOTS[window_bits - 15] << 3
  @maintree_maxsymbols = NUM_CHARS + @num_offsets

  # Initialize window (must be binary to avoid UTF-8 character vs byte mismatch)
  @window = ("\0" * @window_size).b
  @window_posn = 0
  @frame_posn = 0
  @frame = 0

  # Initialize R0, R1, R2 (LRU offset registers)
  @r0 = 1
  @r1 = 1
  @r2 = 1

  # Initialize block state
  @block_type = BLOCKTYPE_INVALID
  @block_length = 0
  @block_remaining = 0
  @header_read = false

  # Intel E8 transformation state
  @intel_filesize = 0
  @intel_started = false
  @e8_buf = ("\0" * FRAME_SIZE).b

  # Initialize bitstream (LZX uses MSB-first bit ordering per libmspack lzxd.c)
  @bitstream = Binary::Bitstream.new(io_system, input, buffer_size,
                                     bit_order: :msb, salvage: salvage)

  # Initialize Huffman trees
  initialize_trees

  # Output tracking
  @offset = 0
  @output_ptr = 0
  @output_end = 0

  # Per libmspack: pending frame data for multi-file extraction.
  # When a decompress call ends mid-frame, the unwritten portion
  # of the frame is stored here for the next call to output.
  @pending_frame_data = nil
  @pending_frame_offset = 0
end

Instance Attribute Details

#is_delta ⇒ Object (readonly)

Returns the value of attribute is_delta.



92
93
94
 
# File 'lib/cabriolet/decompressors/lzx.rb', line 92

def is_delta
  @is_delta
end

#output_length ⇒ Object (readonly)

Returns the value of attribute output_length.



92
93
94
 
# File 'lib/cabriolet/decompressors/lzx.rb', line 92

def output_length
  @output_length
end

#reset_interval ⇒ Object (readonly)

Returns the value of attribute reset_interval.



92
93
94
 
# File 'lib/cabriolet/decompressors/lzx.rb', line 92

def reset_interval
  @reset_interval
end

#window_bits ⇒ Object (readonly)

Returns the value of attribute window_bits.



92
93
94
 
# File 'lib/cabriolet/decompressors/lzx.rb', line 92

def window_bits
  @window_bits
end

Instance Method Details

#decompress(bytes) ⇒ Integer

Decompress LZX data

Per libmspack lzxd.c: the decompressor always decodes full frames (32KB) into the window, but may output fewer bytes if the caller requests less. When multiple files share a folder, decompress is called per file, so partial-frame data must carry over between calls.

Parameters:

  • bytes (Integer) —

    Number of bytes to decompress

Returns:

  • (Integer) —

    Number of bytes decompressed



207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
 
# File 'lib/cabriolet/decompressors/lzx.rb', line 207

def decompress(bytes)
  return 0 if bytes <= 0

  # Read Intel filesize header if not already read (once per stream)
  read_intel_header unless @header_read

  total_written = 0

  # Output any pending frame data from the previous partial-frame write.
  # This handles multi-file extraction where the previous call ended
  # mid-frame and the next file's data starts in the same frame.
  if @pending_frame_data
    avail = @pending_frame_data.bytesize - @pending_frame_offset
    write_amount = [bytes, avail].min
    io_system.write(output,
                    @pending_frame_data[@pending_frame_offset,
                                        write_amount])
    total_written += write_amount
    @offset += write_amount
    @pending_frame_offset += write_amount

    if @pending_frame_offset >= @pending_frame_data.bytesize
      @pending_frame_data = nil
      @pending_frame_offset = 0
    end
  end

  remaining = bytes - total_written
  return total_written if remaining <= 0

  end_frame = ((@offset + remaining) / FRAME_SIZE) + 1

  while @frame < end_frame
    # Check reset interval - reset offset registers at frame boundaries
    if @reset_interval.positive? && (@frame % @reset_interval).zero? && @frame.positive?
      @r0 = @r1 = @r2 = 1
    end

    # Read DELTA chunk size if needed
    @bitstream.read_bits(16) if @is_delta

    # Calculate frame size
    frame_size = calculate_frame_size

    # Decode blocks until frame is complete
    begin
      decode_frame(frame_size)
    rescue DecompressionError => e
      # In salvage mode, if decompression fails, return what we have so far
      if @salvage
        warn "Salvage: LZX decompression failed at frame #{@frame}: #{e.message}"
        return total_written
      else
        raise
      end
    end

    # Apply Intel E8 transformation if needed
    frame_data = if should_apply_e8_transform?(frame_size)
                   apply_e8_transform(frame_size)
                 else
                   @window[@frame_posn, frame_size]
                 end

    # Defensive guard: frame_data should never be nil if the >= window
    # wrap checks below are correct. If it is, the stream is corrupt
    # or a regression has been introduced.
    if frame_data.nil?
      if @salvage
        warn "Salvage: nil frame data at frame_posn=#{@frame_posn}, frame=#{@frame}"
        break
      end

      raise DecompressionError,
            "LZX: nil frame data at position #{@frame_posn}, frame_size=#{frame_size}"
    end

    # Write frame - per libmspack: offset tracks actual output bytes,
    # not full frame bytes. Save unwritten remainder for next call.
    write_amount = [bytes - total_written, frame_size].min
    io_system.write(output, frame_data[0, write_amount])
    total_written += write_amount
    @offset += write_amount

    # Store pending data if partial frame write
    if write_amount < frame_size
      @pending_frame_data = frame_data
      @pending_frame_offset = write_amount
    end

    # Advance frame (always by full frame, matching decode position)
    @frame += 1
    @frame_posn += frame_size
    @frame_posn = 0 if @frame_posn >= @window_size
    @window_posn = 0 if @window_posn >= @window_size

    # Re-align bitstream to 16-bit word boundary between frames.
    # Per libmspack lzxd.c: LZX frames are padded to 16-bit word
    # boundaries (not 8-bit byte boundaries) because the bitstream
    # reads data in 16-bit little-endian words.
    if @bitstream.bits_left.positive?
      @bitstream.ensure_bits(16)
    end
    remove = @bitstream.bits_left & 15
    @bitstream.skip_bits(remove) if remove.positive?
  end

  total_written
end

#free ⇒ void

This method returns an undefined value.

Free resources used by the decompressor

Releases large memory buffers to prevent memory leaks when the decompressor is no longer needed.



187
188
189
190
191
192
193
194
195
196
 
# File 'lib/cabriolet/decompressors/lzx.rb', line 187

def free
  @window = nil
  @e8_buf = nil
  @pending_frame_data = nil
  @bitstream = nil
  @maintree_lengths = nil
  @length_lengths = nil
  @pretree_lengths = nil
  @aligned_lengths = nil
end

#set_output_length(length) ⇒ void

This method returns an undefined value.

Set output length (for Intel E8 processing)

Parameters:

  • length (Integer) —

    Expected output length



177
178
179
 
# File 'lib/cabriolet/decompressors/lzx.rb', line 177

def set_output_length(length)
  @output_length = length if length.positive?
end