Class: Mu::Pcap::TCP

Inherits:
Packet show all
Defined in:
lib/diy/parser/mu/pcap/tcp.rb

Defined Under Namespace

Classes: MergeError, ReorderError, ReorderState

Constant Summary collapse

TH_FIN = 
0x01
TH_SYN = 
0x02
TH_RST = 
0x04
TH_PUSH = 
0x08
TH_ACK = 
0x10
TH_URG = 
0x20
TH_ECE = 
0x40
TH_CWR = 
0x80
MSS = 
2
MAX_SEGMENT_PAYLOAD =

Split-up TCP packets that are too large to serialize. (I.e., total length including all headers greater than 65535 - 20 - 20 - 14.)

65535 - 20 - 20 - 14

Constants inherited from Packet

Packet::IGNORE_UDP_PORTS

Instance Attribute Summary collapse

Attributes inherited from Packet

#payload, #payload_raw

Class Method Summary collapse

Instance Method Summary collapse

Methods inherited from Packet

#deepdup, isolate_l7, normalize, #payload_bytes, #to_bytes

Constructor Details

#initialize ⇒ TCP

Returns a new instance of TCP.



25
26
27
28
29
30
31
32
33
34
35
36
 
# File 'lib/diy/parser/mu/pcap/tcp.rb', line 25

def initialize
    super
    @src_port = 0
    @dst_port = 0
    @seq = 0
    @ack = 0
    @flags = 0
    @window = 0
    @urgent = 0
    @mss = 0
    @proto_family = nil
end

Instance Attribute Details

#ack ⇒ Object

Returns the value of attribute ack.



12
13
14
 
# File 'lib/diy/parser/mu/pcap/tcp.rb', line 12

def ack
  @ack
end

#dst_port ⇒ Object

Returns the value of attribute dst_port.



12
13
14
 
# File 'lib/diy/parser/mu/pcap/tcp.rb', line 12

def dst_port
  @dst_port
end

#flags ⇒ Object

Returns the value of attribute flags.



12
13
14
 
# File 'lib/diy/parser/mu/pcap/tcp.rb', line 12

def flags
  @flags
end

#mss ⇒ Object

Returns the value of attribute mss.



12
13
14
 
# File 'lib/diy/parser/mu/pcap/tcp.rb', line 12

def mss
  @mss
end

#proto_family ⇒ Object

Returns the value of attribute proto_family.



12
13
14
 
# File 'lib/diy/parser/mu/pcap/tcp.rb', line 12

def proto_family
  @proto_family
end

#seq ⇒ Object

Returns the value of attribute seq.



12
13
14
 
# File 'lib/diy/parser/mu/pcap/tcp.rb', line 12

def seq
  @seq
end

#src_port ⇒ Object

Returns the value of attribute src_port.



12
13
14
 
# File 'lib/diy/parser/mu/pcap/tcp.rb', line 12

def src_port
  @src_port
end

#urgent ⇒ Object

Returns the value of attribute urgent.



12
13
14
 
# File 'lib/diy/parser/mu/pcap/tcp.rb', line 12

def urgent
  @urgent
end

#window ⇒ Object

Returns the value of attribute window.



12
13
14
 
# File 'lib/diy/parser/mu/pcap/tcp.rb', line 12

def window
  @window
end

Class Method Details

.create_message_boundaries(packets) ⇒ Object 



277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
 
# File 'lib/diy/parser/mu/pcap/tcp.rb', line 277

def self.create_message_boundaries packets
    # Get complete bytes for each tcp flow before trying to 
    # identify the protocol.
    flow_to_bytes = {}
    packets.each do |packet|
        if tcp? packet
            tcp = packet.payload.payload
            flow = packet.flow_id
            bytes = flow_to_bytes[flow] ||= ""
            bytes << tcp.payload.to_s
        end
    end

    # If any proto plugin can parse a message off of the stream we will
    # use that plugin to detect message boundaries and guide message
    # reassembly.
    flow_to_packetizer = {}
    flow_to_bytes.each_pair do |flow, bytes|
        [ Reader::HttpFamily ].each do |klass|
            reader = klass.new
            reader.pcap2scenario = true
            if reader.read_message bytes
                tx_key = flow.flatten.sort_by {|o| o.to_s}

                tx = flow_to_packetizer[tx_key] ||= StreamPacketizer.new(klass.new)
                break
            end
        end
    end

    # Merge/split packets along message boundaries. This is done as an 
    # atomic transaction per tcp connection. The loop below adds merged
    # packets alongside the original unmerged packets. If the stream
    # is completely merged (no fragments left at end) we remove the
    # original packets otherwise we rollback by removing the newly
    # created packets.
    changes = Hash.new do |hash,key|
        # tuple of original/replacement packets per flow.
        hash[key] = [[], []]
    end
    rollback_list = []

    merged = []
    partial_messages = Hash.new {|hash,key| hash[key] = []}
    packets.each do |packet|
        merged << packet

        next if not tcp? packet
        tcp = packet.payload.payload

        flow = packet.flow_id

        # Check if we have message boundaries for this flow
        tx_key = flow.flatten.sort_by {|o| o.to_s}
        if not tx = flow_to_packetizer[tx_key]
            next
        end

        # Keep track of new vs orig packets so we can delete one set at the end.
        orig_packets, new_packets = changes[flow]
        orig_packets << packet

        if tcp.payload.empty?
            p = packet.deepdup
            new_packets << p
            p.payload.payload.proto_family = tx.parser.family
            next
        end

        # Does the current packet result in any completed messages?
        tx.push(flow, tcp.payload)
        fragments = partial_messages[flow]
        if tx.msg_count(flow) == 0
            # No, record packet as a fragment and move to next packet.
            fragments << packet
            next
        end

        # Yes, packet did result in completed messages. Create a new
        # tcp packet for each higher level protocol message.
        first_inc_packet = (fragments.empty? ? packet : fragments[0])
        next_seq = first_inc_packet.payload.payload.seq
        while tcp_payload = tx.next_msg(flow)
            if tcp_payload.size > MAX_SEGMENT_PAYLOAD
                # Abort merging for this flow because this packet
                # will be split and result in a scenario where
                # we send one logical message but try and receive
                # two.
                rollback_list << tx_key 
                $stderr.puts "Warning: Message too big, cannot enforce " \
                             "message boundaries."
            end
            next_packet = packet.deepdup
            new_packets << next_packet
            next_tcp = next_packet.payload.payload
            next_tcp.seq = next_seq
            next_tcp.payload = tcp_payload
            next_tcp.proto_family = tx.parser.family
            next_seq += tcp_payload.size
            merged << next_packet
        end
        fragments.clear

        # If there are unconsumed bytes then add a fragment to the
        # incomplete list.
        if extra_bytes = tx.extra_bytes(flow)
            frag = packet.deepdup
            new_packets << frag
            fragments << frag
            tcp = frag.payload.payload
            tcp.payload = extra_bytes
            tcp.seq     = next_seq
            tcp.proto_family = tx.parser.family
        end
    end

    # Figure out which connections have incompletely merged flows.
    # Rollback for those and commit the rest.
    partial_messages.each_pair do |flow, list|
        if not list.empty?
            tx_key = flow.flatten.sort_by {|o| o.to_s}
            $stderr.puts "Warning: Left over fragments, cannot force message boundaries."
            rollback_list << tx_key
        end
    end
    changes.each_pair do |flow, orig_new|
        orig, new = orig_new
        tx_key = flow.flatten.sort_by {|o| o.to_s}
        if rollback_list.include?(tx_key)
            new.each {|p| p.payload = :remove}
        else
            orig.each {|p| p.payload = :remove}
        end
    end
    merged.reject! {|p| p.payload == :remove}

    merged
end

.from_bytes(bytes) ⇒ Object 



42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
 
# File 'lib/diy/parser/mu/pcap/tcp.rb', line 42

def self.from_bytes bytes
    Pcap.assert bytes.length >= 20, 'Truncated TCP header: ' +
        "expected 20 bytes, got #{bytes.length} bytes"
    sport, dport, seq, ack, offset, flags, win, sum, urp = 
        bytes.unpack('nnNNCCnnn')
    offset = (offset >> 4) * 4
    Pcap.assert offset >= 20, 'Truncated TCP header: ' +
           "expected at least 20 bytes, got #{offset} bytes"
    Pcap.assert bytes.length >= offset, 'Truncated TCP header: ' +
        "expected at least #{offset} bytes, got #{bytes.length} bytes"

    if TH_SYN == flags
        ss = TCP.get_option bytes[20, offset-20], MSS
    else
        ss = 0
    end

    IPv4.check_options bytes[20, offset-20], 'TCP'

    tcp = TCP.new
    tcp.src_port = sport
    tcp.dst_port = dport
    tcp.seq      = seq
    tcp.ack      = ack
    tcp.flags    = flags
    tcp.window   = win
    tcp.urgent   = urp
    tcp.mss      = ss
    tcp.payload = tcp.payload_raw = bytes[offset..-1]
    return tcp
end

.get_option(options, option_type) ⇒ Object 



74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
 
# File 'lib/diy/parser/mu/pcap/tcp.rb', line 74

def self.get_option options, option_type
    while not options.empty?
        type = options.slice!(0, 1)[0].ord
        if type == 0 or type == 1
            next
        end
        length = options.slice!(0, 1)[0].ord
        if 2 < length
            case length
                when 3
                    format = "C"
                when 4
                    format = "n"
                when 6
                    format = "N"
                when 10
                    format = "Q"
                else
                    Pcap.warning "Bad TCP option length: #{length}"
            end
            option = options.slice!(0, length - 2).unpack(format)[0]
        end
        if option_type == type
            return option
        end
    end
    return 0
end

.merge(packets) ⇒ Object

Merge adjacent TCP packets. Non-data TCP packets are also removed. reorder() should be run first. This can create packets that are larger than the maximum possible IPv4 packet - use split() to make them smaller.



213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
 
# File 'lib/diy/parser/mu/pcap/tcp.rb', line 213

def self.merge packets
    merged_packets = []
    merged_packet = nil
    next_seq = nil
    packets.each do |packet|
        if not tcp? packet
            # Skip non-TCP packets.
            if merged_packet
                merged_packets << merged_packet
                merged_packet = nil
            end
            merged_packets << packet
        elsif packet.payload.v4? and packet.payload.fragment?
            # Sanity check: must not be a fragment
            raise MergeError, 'TCP stream contains IP fragments'
        else
            tcp = packet.payload.payload
            if tcp.flags & TCP::TH_SYN == 0 and tcp.payload == ''
                # Ignore non-data packets.  SYNs are kept so the TCP
                # transport is created at the correct spot.
            elsif not merged_packet or 
                merged_packet.flow_id != packet.flow_id
                # New TCP stream
                if merged_packet
                    merged_packets << merged_packet
                end
                merged_packet = packet.deepdup
                next_seq = tcp.seq + tcp.payload.length
            elsif seq_eq tcp.seq, next_seq
                # Next expected sequence number
                merged_packet.payload.payload.payload << tcp.payload
                next_seq += tcp.payload.length
            elsif seq_lte(tcp.seq + tcp.payload.length, next_seq)
                # Old data: ignore
            elsif seq_lt tcp.seq, next_seq
                # Overlapping segment: merge newest part
                length = seq_sub(tcp.seq + tcp.payload.length, next_seq)
                bytes = tcp.payload[-length..-1]
                merged_packet.payload.payload.payload << bytes
                next_seq += length
            else
                # Error (sanify check, reorder_tcp will raise an error)
                raise MergeError, 'TCP stream is missing segments'
            end
            if next_seq
                if tcp.flags & TCP::TH_SYN != 0
                    next_seq += 1
                end
                if tcp.flags & TCP::TH_FIN != 0
                    next_seq += 1
                end
                next_seq %= 2**32
            end
        end
    end
    if merged_packet
        merged_packets << merged_packet
    end

    merged_packets = create_message_boundaries(merged_packets)

    return merged_packets
end

.pretty_flow_name(packet) ⇒ Object

Generate a pretty name for a TCP flow



482
483
484
485
486
 
# File 'lib/diy/parser/mu/pcap/tcp.rb', line 482

def self.pretty_flow_name packet
    ip = packet.payload
    return "#{ip.src}:#{ip.payload.src_port} <-> " +
        "#{ip.dst}:#{ip.payload.dst_port}"
end

.reorder(packets) ⇒ Object

Reorder packets by TCP sequence number. TCP packets are assumed to be over IP over Ethernet.



123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
 
# File 'lib/diy/parser/mu/pcap/tcp.rb', line 123

def self.reorder packets
    packets = packets.dup
    reordered_packets = []
    flow_to_state = {}
    while not packets.empty?
        packet = packets.shift
        # Don't reorder non-TCP packets
        if not tcp? packet
            reordered_packets << packet
            next
        end
        # Sanity check: must not be a fragment
        if packet.payload.v4? and packet.payload.fragment?
            raise ReorderError, "TCP stream contains IP fragments"
        end
        tcp = packet.payload.payload
        # Must not contain urgent data
        if tcp.flags & TH_URG != 0
            raise ReorderError, "TCP stream contains urgent data: "+
                pretty_flow_name(packet)
        end
        # Get/create state
        if flow_to_state.member? packet.flow_id
            state = flow_to_state[packet.flow_id]
        else
            state = ReorderState.new nil, []
            flow_to_state[packet.flow_id] = state
        end
        if not state.next_seq
            # First packet in TCP stream
            reordered_packets << packet
            state.next_seq = tcp.seq + tcp.payload.length
            if tcp.flags & TCP::TH_SYN != 0
                state.next_seq += 1
            end
            if tcp.flags & TCP::TH_FIN != 0
                state.next_seq += 1
            end
            state.next_seq %= 2**32
        elsif seq_eq(tcp.seq, state.next_seq)
            # Next expected sequence number in TCP stream

            # SYN must not appear in middle of stream
            if tcp.flags & TCP::TH_SYN != 0
                raise ReorderError, "SYN in middle of TCP stream " +
                    pretty_flow_name(packet)
            end

            reordered_packets << packet
            state.next_seq += tcp.payload.length
            if tcp.flags & TCP::TH_FIN != 0
                state.next_seq += 1
            end
            state.next_seq %= 2**32

            # Reinject any packets in the queue into the packet stream
            if not state.queued.empty?
                packets.unshift(*state.queued)
                state.queued.clear
            end
        elsif seq_lt(tcp.seq, state.next_seq)
            # Old sequence number
            if seq_lte(tcp.seq + tcp.payload.length, state.next_seq)
                # No overlap: retransmitted packet, ignore
            else
                # Overlap: reassembler must slice in overlapping data
                reordered_packets << packet
            end
        else
            # Future sequence number - queue
            state.queued << packet
        end
    end

    flow_to_state.each do |flow_id, state|
        if not state.queued.empty?
            raise ReorderError, "Data missing from TCP stream "+
                pretty_flow_name(state.queued[0]) + ': ' +
                "expecting sequence number #{state.next_seq}"
        end
    end

    return reordered_packets
end

.seq_eq(a, b) ⇒ Object

Compare TCP sequence numbers modulo 2**32.



469
470
471
 
# File 'lib/diy/parser/mu/pcap/tcp.rb', line 469

def self.seq_eq a, b
    return seq_sub(a, b) == 0
end

.seq_lt(a, b) ⇒ Object 



473
474
475
 
# File 'lib/diy/parser/mu/pcap/tcp.rb', line 473

def self.seq_lt a, b
    return seq_sub(a, b) < 0
end

.seq_lte(a, b) ⇒ Object 



477
478
479
 
# File 'lib/diy/parser/mu/pcap/tcp.rb', line 477

def self.seq_lte a, b
    return seq_sub(a, b) <= 0
end

.seq_sub(a, b) ⇒ Object

Subtract two sequence numbers module 2**32.



458
459
460
461
462
463
464
465
466
 
# File 'lib/diy/parser/mu/pcap/tcp.rb', line 458

def self.seq_sub a, b
    if a - b > 2**31
        return -((b - a) % 2**32)
    elsif a - b < -2**31
        return (a - b) % 2**32
    else
        return a - b
    end
end

.split(packets) ⇒ Object 



420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
 
# File 'lib/diy/parser/mu/pcap/tcp.rb', line 420

def self.split packets
    split_packets = []
    packets.each do |packet|
        if not tcp? packet
            # Skip non-TCP packets.
            split_packets << packet
            next
        elsif packet.payload.v4? and packet.payload.fragment?
            # Sanity check: must not be a fragment
            raise MergeError, 'TCP stream contains IP fragments'
        elsif packet.payload.payload.payload.length <= MAX_SEGMENT_PAYLOAD
            split_packets << packet
        else
            tcp = packet.payload.payload
            payload = tcp.payload
            tcp.payload = payload.slice! 0, MAX_SEGMENT_PAYLOAD
            next_seq = tcp.seq + tcp.payload.length
            split_packets << packet
            while payload != ''
                next_packet = packet.deepdup
                next_tcp = next_packet.payload.payload
                next_tcp.seq = next_seq
                next_tcp.payload = payload.slice! 0, MAX_SEGMENT_PAYLOAD
                next_seq += next_tcp.payload.length
                split_packets << next_packet
            end
        end
    end
    return split_packets
end

.tcp?(packet) ⇒ Boolean

Returns:

  • (Boolean) 


451
452
453
454
455
 
# File 'lib/diy/parser/mu/pcap/tcp.rb', line 451

def self.tcp? packet
    return packet.is_a?(Ethernet) &&
        packet.payload.is_a?(IP) &&
        packet.payload.payload.is_a?(TCP)
end

Instance Method Details

#==(other) ⇒ Object 



492
493
494
495
496
497
498
499
500
501
 
# File 'lib/diy/parser/mu/pcap/tcp.rb', line 492

def == other
    return super &&
        self.src_port == other.src_port &&
        self.dst_port == other.dst_port &&
        self.seq      == other.seq &&
        self.ack      == other.ack &&
        self.flags    == other.flags &&
        self.window   == other.window &&
        self.urgent   == other.urgent
end

#flow_id ⇒ Object 



38
39
40
 
# File 'lib/diy/parser/mu/pcap/tcp.rb', line 38

def flow_id
    return [:tcp, @src_port, @dst_port]
end

#to_s ⇒ Object 



488
489
490
 
# File 'lib/diy/parser/mu/pcap/tcp.rb', line 488

def to_s
    return "tcp(%d, %d, %s)" % [@src_port, @dst_port, @payload.inspect]
end

#write(io, ip) ⇒ Object 



103
104
105
106
107
108
109
110
111
112
113
114
115
 
# File 'lib/diy/parser/mu/pcap/tcp.rb', line 103

def write io, ip
    if @payload.length + 40 > 65535
        raise NotImplementedError, "TCP segment too large"
    end
    pseudo_header = ip.pseudo_header 20 + @payload.length
    header = [@src_port, @dst_port, @seq, @ack, 5 << 4, @flags, @window, 
              0, @urgent].pack('nnNNCCnnn')
    checksum = IP.checksum pseudo_header + header + @payload
    header = [@src_port, @dst_port, @seq, @ack, 5 << 4, @flags, @window, 
              checksum, @urgent].pack('nnNNCCnnn')
    io.write header
    io.write @payload
end