Module: Python::Pickle

Defined in:
lib/python/pickle.rb,
lib/python/pickle/tuple.rb,
lib/python/pickle/version.rb,
lib/python/pickle/protocol.rb,
lib/python/pickle/py_class.rb,
lib/python/pickle/protocol0.rb,
lib/python/pickle/protocol1.rb,
lib/python/pickle/protocol2.rb,
lib/python/pickle/protocol3.rb,
lib/python/pickle/protocol4.rb,
lib/python/pickle/protocol5.rb,
lib/python/pickle/py_object.rb,
lib/python/pickle/byte_array.rb,
lib/python/pickle/exceptions.rb,
lib/python/pickle/instruction.rb,
lib/python/pickle/deserializer.rb,
lib/python/pickle/instructions/dup.rb,
lib/python/pickle/instructions/get.rb,
lib/python/pickle/instructions/int.rb,
lib/python/pickle/instructions/obj.rb,
lib/python/pickle/instructions/pop.rb,
lib/python/pickle/instructions/put.rb,
lib/python/pickle/instructions/dict.rb,
lib/python/pickle/instructions/ext1.rb,
lib/python/pickle/instructions/ext2.rb,
lib/python/pickle/instructions/ext4.rb,
lib/python/pickle/instructions/inst.rb,
lib/python/pickle/instructions/list.rb,
lib/python/pickle/instructions/long.rb,
lib/python/pickle/instructions/mark.rb,
lib/python/pickle/instructions/none.rb,
lib/python/pickle/instructions/stop.rb,
lib/python/pickle/instructions/build.rb,
lib/python/pickle/instructions/float.rb,
lib/python/pickle/instructions/frame.rb,
lib/python/pickle/instructions/long1.rb,
lib/python/pickle/instructions/long4.rb,
lib/python/pickle/instructions/proto.rb,
lib/python/pickle/instructions/tuple.rb,
lib/python/pickle/instructions/append.rb,
lib/python/pickle/instructions/global.rb,
lib/python/pickle/instructions/persid.rb,
lib/python/pickle/instructions/reduce.rb,
lib/python/pickle/instructions/string.rb,
lib/python/pickle/instructions/tuple1.rb,
lib/python/pickle/instructions/tuple2.rb,
lib/python/pickle/instructions/tuple3.rb,
lib/python/pickle/instructions/appends.rb,
lib/python/pickle/instructions/bin_get.rb,
lib/python/pickle/instructions/bin_put.rb,
lib/python/pickle/instructions/memoize.rb,
lib/python/pickle/instructions/new_obj.rb,
lib/python/pickle/instructions/bin_int1.rb,
lib/python/pickle/instructions/new_true.rb,
lib/python/pickle/instructions/pop_mark.rb,
lib/python/pickle/instructions/set_item.rb,
lib/python/pickle/instructions/add_items.rb,
lib/python/pickle/instructions/bin_bytes.rb,
lib/python/pickle/instructions/bin_float.rb,
lib/python/pickle/instructions/empty_set.rb,
lib/python/pickle/instructions/has_value.rb,
lib/python/pickle/instructions/new_false.rb,
lib/python/pickle/instructions/set_items.rb,
lib/python/pickle/instructions/bin_bytes8.rb,
lib/python/pickle/instructions/bin_persid.rb,
lib/python/pickle/instructions/bin_string.rb,
lib/python/pickle/instructions/empty_dict.rb,
lib/python/pickle/instructions/empty_list.rb,
lib/python/pickle/instructions/frozen_set.rb,
lib/python/pickle/instructions/new_obj_ex.rb,
lib/python/pickle/instructions/bin_unicode.rb,
lib/python/pickle/instructions/byte_array8.rb,
lib/python/pickle/instructions/empty_tuple.rb,
lib/python/pickle/instructions/next_buffer.rb,
lib/python/pickle/instructions/bin_unicode8.rb,
lib/python/pickle/instructions/long_bin_get.rb,
lib/python/pickle/instructions/stack_global.rb,
lib/python/pickle/instructions/readonly_buffer.rb,
lib/python/pickle/instructions/short_bin_bytes.rb,
lib/python/pickle/instructions/short_bin_string.rb,
lib/python/pickle/instructions/short_bin_unicode.rb,
lib/python/pickle/instructions/has_length_and_value.rb,
lib/python/pickle/instructions/has_namespace_and_name.rb

Overview

A modern Ruby implementation of the Python Pickle serialization format.

Defined Under Namespace

Modules: Instructions Classes: ByteArray, DeserializationError, Deserializer, Error, Instruction, InvalidFormat, Protocol, Protocol0, Protocol1, Protocol2, Protocol3, Protocol4, Protocol5, PyClass, PyObject, Tuple

Constant Summary collapse

PROTOCOL_VERSIONS =

This constant is part of a private API. You should avoid using this constant if possible, as it may be removed or be changed in the future.

Mapping of protocol versions to protocol parsers.

{
  0 => Protocol0,
  1 => Protocol1,
  2 => Protocol2,
  3 => Protocol3,
  4 => Protocol4,
  5 => Protocol5
}
DEFAULT_PROTOCOL =

The default protocol version to use.

4
HIGHEST_PROTOCOL =

The highest protocol version supported.

5
VERSION =

python-pickle version

'0.2.0'

Class Method Summary collapse

Class Method Details

.dump(object, output = nil, protocol: DEFAULT_PROTOCOL) ⇒ Object

Note:

serializing is currently not supported.

Serializes the Ruby object into Python Pickle data.

Parameters:

  • object (Object) —

    The Ruby object to serialize.

  • output (IO) (defaults to: nil) —

    The option output to write the Pickle data to.

  • protocol (Integer) (defaults to: DEFAULT_PROTOCOL) —

    The desired Python Pickle protocol to use.

Raises:

  • (NotImplementedError) 


168
169
170
 
# File 'lib/python/pickle.rb', line 168

def self.dump(object,output=nil, protocol: DEFAULT_PROTOCOL)
  raise(NotImplementedError,"pickle serializing is currently not supported")
end

.infer_protocol_version(io) ⇒ Integer

This method is part of a private API. You should avoid using this method if possible, as it may be removed or be changed in the future.

Infers the protocol version from the IO stream.

Parameters:

  • io (IO) —

    The IO stream to inspect.

Returns:

  • (Integer) —

    The inferred Python Pickle protocol version.

Raises:

  • (InvalidFormat) —

    Could not determine the Pickle version from the first two bytes of the IO stream.



187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
 
# File 'lib/python/pickle.rb', line 187

def self.infer_protocol_version(io)
  opcode = io.getbyte

  begin
    case opcode
    when Protocol2::PROTO
      version = io.getbyte
      io.ungetbyte(version)
      return version
    when Protocol0::POP,
         Protocol0::DUP,
         Protocol0::FLOAT,
         Protocol0::STRING,
         Protocol0::UNICODE,
         Protocol0::DICT,
         Protocol0::GET,
         Protocol0::LIST,
         Protocol0::PUT
      0
    when Protocol1::EMPTY_TUPLE,
         Protocol1::BINFLOAT,
         Protocol1::BININT1,
         Protocol1::BINSTRING,
         Protocol1::SHORT_BINSTRING,
         Protocol1::BINUNICODE,
         Protocol1::EMPTY_LIST,
         Protocol1::APPENDS,
         Protocol1::BINPUT,
         Protocol1::SETITEMS,
         Protocol1::EMPTY_DICT
      1
    when Protocol0::STOP
      # if we've read all the way to the end of the stream and still cannot
      # find any protocol 0 or protocol 1 specific opcodes, assume protocol 0
      0
    when Protocol0::INT, # identical in both protocol 0 and 1
         Protocol0::LONG # identical in both protocol 0 and 1
      chars = io.gets

      begin
        infer_protocol_version(io)
      ensure
        chars.each_byte.reverse_each { |b| io.ungetbyte(b) }
      end
    when Protocol0::MARK,    # identical in both protocol 0 and 1
         Protocol0::NONE,    # identical in both protocol 0 and 1
         Protocol0::REDUCE,  # identical in both protocol 0 and 1
         Protocol0::APPEND,  # identical in both protocol 0 and 1
         Protocol0::BUILD,   # identical in both protocol 0 and 1
         Protocol0::SETITEM, # identical in both protocol 0 and 1
         Protocol0::TUPLE    # identical in both protocol 0 and 1
      infer_protocol_version(io)
    when Protocol0::GLOBAL
      first_nl_string  = io.gets
      second_nl_string = io.gets

      begin
        infer_protocol_version(io)
      ensure
        # push the read bytes back into the IO stream
        second_nl_string.each_byte.reverse_each { |b| io.ungetbyte(b) }
        first_nl_string.each_byte.reverse_each  { |b| io.ungetbyte(b) }
      end
    else
      raise(InvalidFormat,"cannot infer protocol version from opcode (#{opcode.inspect}) at position #{io.pos}")
    end
  ensure
    io.ungetbyte(opcode)
  end
end

.load(data, protocol: nil, **kwargs) ⇒ Object

Deserializes the Python Pickle stream into a Ruby object.

Parameters:

  • data (String, IO) —

    The Python pickle stream to parse.

  • protocol (Integer, nil) (defaults to: nil) —

    The explicit protocol version to use. If nil the protocol version will be inferred by inspecting the first two bytes of the stream.

  • kwargs (Hash{Symbol => Object}) —

    Additional keyword arguments.

Options Hash (**kwargs):

  • :extensions (Hash{Integer => Object}) —

    A Hash of registered extension IDs and their Objects.

  • :constants (Hash{String => Hash{String => Class,Method}}) —

    An optional mapping of custom Python constant names to Ruby classes or methods.

  • :buffers (Enumerable, nil) —

    An enumerable list of out-of-band buffers.

Raises:



106
107
108
109
110
111
112
113
114
115
116
117
118
 
# File 'lib/python/pickle.rb', line 106

def self.load(data, protocol: nil, **kwargs)
  deserializer = Deserializer.new(**kwargs)

  parse(data, protocol: protocol) do |instruction|
    status, object = deserializer.execute(instruction)

    if status == :halt
      return object
    end
  end

  raise(DeserializationError,"failed to deserialize any object data from stream")
end

.load_file(path, **kwargs) ⇒ Object

Deserializes a Python Pickle file.

Parameters:

  • path (String) —

    The path of the file.

  • kwargs (Hash{Symbol => Object}) —

    Additional keyword arguments.

Options Hash (**kwargs):

  • :extensions (Hash{Integer => Object}) —

    A Hash of registered extension IDs and their Objects.

  • :constants (Hash{String => Hash{String => Class,Method}}) —

    An optional mapping of custom Python constant names to Ruby classes or methods.

  • :buffers (Enumerable, nil) —

    An enumerable list of out-of-band buffers.

Returns:

  • (Object) —

    The deserialized object.



142
143
144
145
146
147
148
149
150
 
# File 'lib/python/pickle.rb', line 142

def self.load_file(path,**kwargs)
  result = nil

  File.open(path,'rb') do |file|
    result = load(file,**kwargs)
  end

  return result
end

.parse(data, protocol: nil) {|instruction| ... } ⇒ Array<Instruction>

Parses a Python pickle stream.

Parameters:

  • data (String, IO) —

    The Python pickle stream to parse.

  • protocol (Integer, nil) (defaults to: nil) —

    The explicit protocol version to use. If nil the protocol version will be inferred by inspecting the first two bytes of the stream.

Yields:

  • (instruction) —

    If a block is given, it will be passed each parsed Pickle instruction.

Yield Parameters:

  • instruction (Instruction) —

    A parsed Pickle instruction from the Pickle stream.

Returns:

  • (Array<Instruction>) —

    All parsed Pickle instructions from the Pickle stream.



59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
 
# File 'lib/python/pickle.rb', line 59

def self.parse(data, protocol: nil, &block)
  io = case data 
       when String then StringIO.new(data)
       when IO     then data
       else
         raise(ArgumentError,"argument must be either an IO object or a String: #{io.inspect}")
       end

  if protocol
    if (protocol < 0) || (protocol > HIGHEST_PROTOCOL)
      raise(ArgumentError,"protocol must be between 0 or #{HIGHEST_PROTOCOL}, but was #{protocol.inspect}")
    end
  else
    protocol = infer_protocol_version(io)
  end

  protocol_class = PROTOCOL_VERSIONS.fetch(protocol)
  protocol       = protocol_class.new(io)

  return protocol.read(&block)
end