Module: Persist

Defined in:
lib/scout/persist/tsv.rb,
lib/scout/tsv/annotation/repo.rb,
lib/scout/persist/tsv/adapter/sharder.rb,
lib/scout/persist/tsv/adapter/packed_index.rb,
lib/scout/persist/tsv/adapter/tokyocabinet.rb,
lib/scout/persist/tsv/adapter/fix_width_table.rb

Constant Summary collapse

REPO_CACHE = 
{}

Class Method Summary collapse

Class Method Details

.annotation_repo_persist(repo, name, &block) ⇒ Object 



3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
 
# File 'lib/scout/tsv/annotation/repo.rb', line 3

def self.annotation_repo_persist(repo, name, &block)

  if String === repo
    repo = REPO_CACHE[repo] ||= begin
                                  repo = repo.find if Path === repo
                                  repo = Persist.open_tokyocabinet(repo, false, :list, :BDB)
                                  repo_fields = ["literal", "annotation_types", "JSON"]
                                  TSV.setup(repo, :fields => repo_fields, :key_field => "Annotation ID")
                                  repo.save_annotation_hash
                                  repo
                                end
    repo.close
  end

  repo_fields = repo.fields

  subkey = name + ":"

  keys = repo.read_and_close do
    repo.range subkey + 0.chr, true, subkey + 254.chr, true
  end

  case
  when (keys.length == 1 and keys.first == subkey + 'NIL')
    nil
  when (keys.length == 1 and keys.first == subkey + 'EMPTY')
    []
  when (keys.length == 1 && keys.first =~ /:SINGLE$/)
    key = keys.first
    values = repo.read_and_close do
      repo[key]
    end
    Annotation.load_tsv_values(key, values, *repo_fields)
  when (keys.any? and not keys.first =~ /ANNOTATED_DOUBLE_ARRAY/)
    repo.read_and_close do
      keys.sort_by{|k| k.split(":").last.to_i}.collect{|key|
        v = repo[key]
        Annotation.load_tsv_values(key, v, *repo_fields)
      }
    end
  when (keys.any? and keys.first =~ /ANNOTATED_DOUBLE_ARRAY/)
    repo.read_and_close do

      res = keys.sort_by{|k| k.split(":").last.to_i}.collect{|key|
        v = repo[key]
        Annotation.load_tsv_values(key, v, *repo_fields)
      }

      res.first.annotate res
      res.extend AnnotatedArray

      res
    end
  else
    annotations = yield

    repo.write_and_close do 
      case
      when annotations.nil?
        repo[subkey + "NIL"] = nil
      when annotations.empty?
        repo[subkey + "EMPTY"] = nil
      when (not Array === annotations or (AnnotatedArray === annotations and not Array === annotations.first))
        tsv_values = Annotation.obj_tsv_values(annotations, repo_fields) 
        repo[subkey + annotations.id << ":" << "SINGLE"] = tsv_values
      when (not Array === annotations or (AnnotatedArray === annotations and AnnotatedArray === annotations.first))
        annotations.each_with_index do |e,i|
          next if e.nil?
          tsv_values = Annotation.obj_tsv_values(e, repo_fields) 
          repo[subkey + "ANNOTATED_DOUBLE_ARRAY:" << i.to_s] = tsv_values
        end
      else
        annotations.each_with_index do |e,i|
          next if e.nil?
          tsv_values = Annotation.obj_tsv_values(e, repo_fields) 
          repo[subkey + i.to_s] = tsv_values
        end
      end
    end

    annotations
  end

end

.open_database(path, write, serializer = nil, type = "HDB", options = {}) ⇒ Object 



27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
 
# File 'lib/scout/persist/tsv.rb', line 27

def self.open_database(path, write, serializer = nil, type = "HDB", options = {})
  db = case type
       when 'fwt'
         value_size, range, update, in_memory, pos_function = IndiferentHash.process_options options.dup, :value_size, :range, :update, :in_memory, :pos_function
         if pos_function
           Persist.open_fwt(path, value_size, range, serializer, update, in_memory, &pos_function)
         else
           Persist.open_fwt(path, value_size, range, serializer, update, in_memory)
         end
       when 'pki'
         pattern, pos_function = IndiferentHash.process_options options.dup, :pattern, :pos_function
         if pos_function
           Persist.open_pki(path, write, pattern, &pos_function)
         else
           Persist.open_pki(path, write, pattern)
         end
       else
         Persist.open_tokyocabinet(path, write, serializer, type)
       end
  db
end

.open_fwt(path, value_size, range = false, serializer = nil, update = false, in_memory = false, &pos_function) ⇒ Object 



81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
 
# File 'lib/scout/persist/tsv/adapter/fix_width_table.rb', line 81

def self.open_fwt(path, value_size, range = false, serializer = nil, update = false, in_memory = false, &pos_function)
  FileUtils.mkdir_p File.dirname(path) unless File.exist?(File.dirname(path))

  database = FixWidthTable.new(path, value_size, range, update, in_memory, &pos_function)

  database.extend FWTAdapter

  database.pos_function = pos_function

  unless serializer == :clean
    TSV.setup database
    database.serializer ||= TSVAdapter.serializer_module(serializer)
  end

  database
end

.open_pki(path, write, pattern, &pos_function) ⇒ Object 



85
86
87
88
89
90
91
92
93
94
 
# File 'lib/scout/persist/tsv/adapter/packed_index.rb', line 85

def self.open_pki(path, write, pattern, &pos_function)
  FileUtils.mkdir_p File.dirname(path) unless File.exist?(File.dirname(path))

  database = PackedIndex.new(path, write, pattern, &pos_function)
  database.extend PKIAdapter

  database.pos_function = pos_function

  database
end

.open_sharder(persistence_path, write = false, db_type = nil, persist_options = {}, &block) ⇒ Object 



47
48
49
50
51
52
53
 
# File 'lib/scout/persist/tsv/adapter/sharder.rb', line 47

def self.open_sharder(persistence_path, write=false, db_type=nil, persist_options={}, &block)
  database = Sharder.new(persistence_path, write, db_type, persist_options, &block)
  database.extend ShardAdapter
  database.serializer = TSVAdapter.serializer_module(persist_options[:serializer]) if persist_options[:serializer]
  database.save_annotation_hash
  database
end

.open_tokyocabinet(path, write, serializer = nil, tokyocabinet_class = TokyoCabinet::HDB) ⇒ Object 



13
14
15
16
17
18
19
20
21
22
23
24
 
# File 'lib/scout/persist/tsv/adapter/tokyocabinet.rb', line 13

def self.open_tokyocabinet(path, write, serializer = nil, tokyocabinet_class = TokyoCabinet::HDB)
  write = true unless File.exist? path

  FileUtils.mkdir_p File.dirname(path) unless File.exist?(File.dirname(path))

  database = ScoutCabinet.open(path, write, tokyocabinet_class)

  database.extend TKAdapter
  database.serializer ||= TSVAdapter.serializer_module(serializer)

  database
end

.persist_tsv(file, filename = nil, options = {}, persist_options = {}, &block) ⇒ Object 



70
71
72
73
74
75
76
77
78
79
80
 
# File 'lib/scout/persist/tsv.rb', line 70

def self.persist_tsv(file, filename = nil, options = {}, persist_options = {}, &block)
  persist_options = IndiferentHash.add_defaults persist_options,
    IndiferentHash.pull_keys(options, :persist)

  persist_options[:data] ||= options.delete(:data)
  engine = IndiferentHash.process_options persist_options, :engine, engine: "HDB"
  other_options = IndiferentHash.pull_keys persist_options, :other
  other_options[:original_options] = options

  Persist.tsv(file, engine: engine, persist_options: persist_options.merge(other: other_options), &block)
end

.tsv(id, options = {}, engine: nil, persist_options: {}) ⇒ Object 



49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
 
# File 'lib/scout/persist/tsv.rb', line 49

def self.tsv(id, options = {}, engine: nil, persist_options: {})
  engine ||= persist_options[:engine] || :HDB
  Persist.persist(id, engine, persist_options.merge(:other_options => options)) do |filename|
    if filename.nil?
      yield(persist_options[:data] || {})
    else
      if persist_options.include?(:shard_function)
        data = persist_options[:data] ||= Persist.open_sharder(filename, true, engine, options.merge(persist_options), &persist_options[:shard_function])
      else
        data = persist_options[:data] ||= Persist.open_database(filename, true, persist_options[:serializer], engine, options)
      end

      data.serializer = TSVAdapter.serializer_module(persist_options[:serializer]) if persist_options[:serializer]

      yield(data)
      data.save_annotation_hash if Annotation.is_annotated?(data)
      data
    end
  end
end