Module: Pocolog

Defined in:
lib/pocolog.rb,
lib/pocolog/format.rb,
lib/pocolog/logger.rb,
lib/pocolog/repair.rb,
lib/pocolog/convert.rb,
lib/pocolog/upgrade.rb,
lib/pocolog/version.rb,
lib/pocolog/logfiles.rb,
lib/pocolog/format/v2.rb,
lib/pocolog/data_stream.rb,
lib/pocolog/io_sequence.rb,
lib/pocolog/stream_info.rb,
lib/pocolog/upgrade/dsl.rb,
lib/pocolog/block_stream.rb,
lib/pocolog/stream_index.rb,
lib/pocolog/test_helpers.rb,
lib/pocolog/file_upgrader.rb,
lib/pocolog/stream_aligner.rb,
lib/pocolog/warn_deprecated.rb,
lib/pocolog/cli/tty_reporter.rb,
lib/pocolog/upgrade/ops/base.rb,
lib/pocolog/cli/null_reporter.rb,
lib/pocolog/sample_enumerator.rb,
lib/pocolog/file_index_builder.rb,
lib/pocolog/upgrade/exceptions.rb,
lib/pocolog/upgrade/ops/custom.rb,
lib/pocolog/upgrade/ops/identity.rb,
lib/pocolog/upgrade/ops/sequence.rb,
lib/pocolog/upgrade/ops/enum_cast.rb,
lib/pocolog/upgrade/ops/array_cast.rb,
lib/pocolog/io_sequence_stream_info.rb,
lib/pocolog/upgrade/ops/numeric_cast.rb,
lib/pocolog/upgrade/ops/compound_cast.rb,
lib/pocolog/upgrade/converter_registry.rb,
lib/pocolog/upgrade/ops/container_cast.rb

Defined Under Namespace

Modules: CLI, Format, TestHelpers, Upgrade Classes: BlockStream, DataStream, FileUpgrader, IOSequence, IndexBuilderStreamInfo, InternalError, InvalidBlockFound, InvalidFile, InvalidIndex, Logfiles, Logger, MissingIndexPrologue, MissingPrologue, NotEnoughData, ObsoleteIndexVersion, ObsoleteVersion, SampleEnumerator, StreamAligner, StreamIndex, StreamInfo

Constant Summary collapse

STREAM_BLOCK = 
1
DATA_BLOCK = 
2
CONTROL_BLOCK = 
3
BLOCK_TYPES = 
[STREAM_BLOCK, DATA_BLOCK, CONTROL_BLOCK]
DATA_STREAM = 
1
CONTROL_SET_TIMEBASE = 
0
CONTROL_SET_TIMEOFFSET = 
1
VERSION = 
"0.4"

Class Method Summary collapse

Instance Method Summary collapse

Class Method Details

.big_endian? ⇒ Boolean

true if this machine is big endian

Returns:

  • (Boolean) 


15
16
17
 
# File 'lib/pocolog/format.rb', line 15

def self.big_endian?
    "LAAS".unpack('L').pack('N') == "LAAS"
end

.create_index_from_raw_info(block_stream, raw_info, interval_rt: []) ⇒ Array<StreamInfo,nil>

Create an list of StreamInfo object based on basic information about the file

Parameters:

Returns:



64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
 
# File 'lib/pocolog/file_index_builder.rb', line 64

def self.create_index_from_raw_info(block_stream, raw_info, interval_rt: [])
    raw_info.each_with_index.map do |stream_info, stream_i|
        next unless stream_info

        index_map = stream_info.index_map
        base_time = nil
        stream_interval_rt = interval_rt[stream_i]

        unless index_map.empty?
            # Read the realtime of the first and last samples if they are
            # not provided
            stream_interval_rt ||= create_index_read_interval_rt(block_stream, index_map)

            base_time = index_map[1]
            index_map = StreamIndex.map_entries_internal(index_map) do |pos, time|
                [pos, time - base_time]
            end
        end

        StreamInfo.from_raw_data(
            stream_info.stream_block_pos,
            stream_interval_rt || [], base_time, index_map
        )
    end
end

.create_index_read_interval_rt(block_stream, index_map) ⇒ Object 



90
91
92
93
94
95
96
97
98
99
 
# File 'lib/pocolog/file_index_builder.rb', line 90

def self.create_index_read_interval_rt(block_stream, index_map)
    block_stream.seek(index_map.first)
    block_stream.read_next_block_header
    first_block = block_stream.read_data_block_header

    block_stream.seek(index_map[-2])
    block_stream.read_next_block_header
    last_block = block_stream.read_data_block_header
    [first_block.rt_time, last_block.rt_time]
end

.file_index_builder(block_stream, skip_payload: true) ⇒ Array<StreamInfo,nil>

Build the index of a block stream

Parameters:

  • block_stream (BlockStream) —

    the block stream that represents the log file

Returns:



21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
 
# File 'lib/pocolog/file_index_builder.rb', line 21

def self.file_index_builder(block_stream, skip_payload: true)
    # We build the information expected by StreamInfo.from_raw_data
    # That is (declaration_block, interval_rt, base_time, index_map)
    #
    # The first pass extracts information from the file itself, but
    # avoiding as much tests-in-the-loop as possible. This array
    # therefore only stores the declaration block and index map
    raw_stream_info = []
    block_pos = block_stream.tell
    interval_rt = []
    while (block = block_stream.read_next_block_header)
        stream_index = block.stream_index
        interval_rt[stream_index] ||= []

        if block.kind == STREAM_BLOCK
            raw_stream_info[stream_index] =
                IndexBuilderStreamInfo.new(block_pos, [])
        elsif block.kind == DATA_BLOCK
            data_block = block_stream.read_data_block_header
            interval_rt[stream_index][0] ||= data_block.rt_time
            interval_rt[stream_index][1] = data_block.rt_time
            index_map = raw_stream_info[stream_index].index_map
            index_map << block_pos << data_block.lg_time
        end

        if skip_payload
            block_stream.skip_payload
        else
            block_stream.read_payload
        end
        block_pos = block_stream.tell
    end
    create_index_from_raw_info(
        block_stream, raw_stream_info, interval_rt: interval_rt
    )
end

.file_stream(file_name, stream_name) ⇒ Object

Returns the stream called stream_name from file



658
659
660
661
 
# File 'lib/pocolog/logfiles.rb', line 658

def self.file_stream(file_name, stream_name)
    file = Logfiles.open(file_name)
    file.stream(stream_name)
end

.find_valid_range(block_stream) ⇒ Object

Find the byte position just after the last valid block in the stream

It starts the search at the stream’s current position



69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
 
# File 'lib/pocolog/repair.rb', line 69

def self.find_valid_range(block_stream)
    stream_sample = {}
    current_pos = block_stream.tell

    while (block = block_stream.read_next_block_header)
        if block.kind == STREAM_BLOCK
            stream = block_stream.read_stream_block
            stream_sample[block.stream_index] = stream.type.new
        elsif block.kind == DATA_BLOCK
            _, marshalled_data = block_stream.read_data_block
            stream_sample[block.stream_index]
                .from_buffer_direct(marshalled_data)
            current_pos = block_stream.tell
        end
    end

    current_pos
rescue InvalidFile
    current_pos
end

.repair_file(path, keep_invalid_files: true, backup: true, reporter: CLI::NullReporter.new) ⇒ Boolean

Attempt to repair a broken file

Returns:

  • (Boolean) —

    true if the file was valid, false otherwise



9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
 
# File 'lib/pocolog/repair.rb', line 9

def self.repair_file(
    path, keep_invalid_files: true, backup: true, reporter: CLI::NullReporter.new
)
    size = File.stat(path).size
    reporter.reset_progressbar('[:bar]', total: size)
    block_stream = BlockStream.open(path)
    broken_path = "#{path}.broken" if backup

    begin
        block_stream.read_prologue
    rescue InvalidFile
        reporter.error "#{path}: missing or invalid prologue, nothing to salvage"
        return false if keep_invalid_files

        if broken_path
            FileUtils.mv path, broken_path
            reporter.error "moved #{path} to #{broken_path}"
        else
            FileUtils.rm_f path
            reporter.error "deleted #{path}"
        end
        return false
    end

    stream_info = {}
    stream_sample = {}
    current_pos = block_stream.tell

    begin
        while (block = block_stream.read_next_block_header)
            if block.kind == STREAM_BLOCK
                stream = block_stream.read_stream_block
                stream_info[block.stream_index] = stream
                stream_sample[block.stream_index] = stream.type.new
            else
                _, marshalled_data = block_stream.read_data_block
                stream_sample[block.stream_index]
                    .from_buffer_direct(marshalled_data)
            end
            current_pos = block_stream.tell
            reporter.current = current_pos
        end
        reporter.finish
        true
    rescue InvalidFile
        reporter.finish
        remaining = File.stat(path).size - current_pos
        reporter.error "#{path}: broken at position #{current_pos} "\
                       "(#{remaining} bytes thrown away)"
        reporter.error "  copying the current file as #{path}.broken"
        FileUtils.cp path, "#{path}.broken"
        reporter.error '  truncating the existing file'
        File.open(path, 'a') { |io| io.truncate(current_pos) }
        false
    end
end

.show_deprecations=(flag) ⇒ Object 



6
7
8
 
# File 'lib/pocolog/warn_deprecated.rb', line 6

def self.show_deprecations=(flag)
    @show_deprecations = flag
end

.show_deprecations? ⇒ Boolean

Returns:

  • (Boolean) 


2
3
4
 
# File 'lib/pocolog/warn_deprecated.rb', line 2

def self.show_deprecations?
    @show_deprecations
end

.warn_deprecated(message) ⇒ Object

Warns about a deprecation, showing the first backtrace lines



13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
 
# File 'lib/pocolog/warn_deprecated.rb', line 13

def self.warn_deprecated(message)
    return if !show_deprecations?

    # Show the message, regardless of the actual logger setup
    current_logger_level = logger.level
    logger.level = Logger::WARN

    warn message
    caller(1)[0, 4].each do |line|
        warn "  #{line}"
    end

ensure
    if current_logger_level
        logger.level = current_logger_level
    end
end

Instance Method Details

#index_map ⇒ Array<(Integer,Integer,Integer)>

(block_pos,lg_time,sample_index) tuples. The logical time is in absolute microseconds.

Returns:

  • (Array<(Integer,Integer,Integer)>) —

    at list of



14
 
# File 'lib/pocolog/file_index_builder.rb', line 14

IndexBuilderStreamInfo = Struct.new :stream_block_pos, :index_map

#stream_block_pos ⇒ Integer

Returns the position of the stream definition in the IO.

Returns:

  • (Integer) —

    the position of the stream definition in the IO



14
 
# File 'lib/pocolog/file_index_builder.rb', line 14

IndexBuilderStreamInfo = Struct.new :stream_block_pos, :index_map