Class: Sonar::Connector::FileStore

Inherits:
Object
  • Object
show all
Defined in:
lib/sonar_connector_filestore.rb

Overview

a FileStore has an on-disk directory structure :

  • root, effectively a parent directory

  • name : the filestore directory name

  • areas : names of acceptable sub-directories in the FileStore directory

so a filestore with (@root==“/foo”, @name==:bar, @areas=[:area51, :area52]) would have directories :

/foo
/foo/bar
/foo/bar/area51
/foo/bar/area52

Defined Under Namespace

Classes: LeaveInSourceArea

Class Attribute Summary collapse

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(root, name, areas, opts = {}) ⇒ FileStore

Returns a new instance of FileStore.



51
52
53
54
55
56
57
58
59
60
61
62
63
64
 
# File 'lib/sonar_connector_filestore.rb', line 51

def initialize(root, name, areas, opts={})
  raise "directory '#{root}' does not exist or is not a directory" if !File.directory?(root)
  @root = root

  raise "#{name} is not a valid filestore name" if !FileStore.valid_filestore_name?(name)
  @name = name
  FileUtils.mkdir_p(filestore_path)

  @areas = Set.new([*areas])
  @areas.each{|area| raise "#{area} is not a valid area name" if !FileStore.valid_area_name?(area)}
  @areas.each{|area| FileUtils.mkdir_p(area_path(area))}

  @logger = opts[:logger]
end

Class Attribute Details

.logger ⇒ Object

the default logger…



24
25
26
 
# File 'lib/sonar_connector_filestore.rb', line 24

def logger
  @logger
end

Instance Attribute Details

#areas ⇒ Object (readonly)

Returns the value of attribute areas.



31
32
33
 
# File 'lib/sonar_connector_filestore.rb', line 31

def areas
  @areas
end

#logger ⇒ Object 



66
67
68
 
# File 'lib/sonar_connector_filestore.rb', line 66

def logger
  @logger || FileStore.logger
end

#name ⇒ Object (readonly)

Returns the value of attribute name.



30
31
32
 
# File 'lib/sonar_connector_filestore.rb', line 30

def name
  @name
end

#root ⇒ Object (readonly)

Returns the value of attribute root.



29
30
31
 
# File 'lib/sonar_connector_filestore.rb', line 29

def root
  @root
end

Class Method Details

.ordinary_directory?(f) ⇒ Boolean

Returns:

  • (Boolean) 


47
48
49
 
# File 'lib/sonar_connector_filestore.rb', line 47

def self.ordinary_directory?(f)
  ordinary_directory_name?(f.to_s) && File.directory?(f.to_s)
end

.ordinary_directory_name?(f) ⇒ Boolean

Returns:

  • (Boolean) 


43
44
45
 
# File 'lib/sonar_connector_filestore.rb', line 43

def self.ordinary_directory_name?(f)
  File.basename(f.to_s) !~ /^\./
end

.valid_area_name?(a) ⇒ Boolean

Returns:

  • (Boolean) 


39
40
41
 
# File 'lib/sonar_connector_filestore.rb', line 39

def self.valid_area_name?(a)
  a.to_s != "tmp"
end

.valid_filestore_name?(f) ⇒ Boolean

Returns:

  • (Boolean) 


34
35
36
37
 
# File 'lib/sonar_connector_filestore.rb', line 34

def self.valid_filestore_name?(f)
  (f.to_s == File.basename(f.to_s)) && 
    ordinary_directory_name?(f)
end

Instance Method Details

#area_count ⇒ Object

hash of counts keyed by area



170
171
172
 
# File 'lib/sonar_connector_filestore.rb', line 170

def area_count
  @areas.reduce({}){|h,area| h[area]=count(area) ; h}
end

#area_files(area, max = nil) ⇒ Object

fetch at most max regular file paths from an area



159
160
161
 
# File 'lib/sonar_connector_filestore.rb', line 159

def area_files(area, max=nil)
  relative_file_paths(area_path(area), max)
end

#area_path(area) ⇒ Object 



83
84
85
86
 
# File 'lib/sonar_connector_filestore.rb', line 83

def area_path(area)
  check_area(area)
  File.join(filestore_path, area.to_s)
end

#area_size ⇒ Object

hash of sizes keyed by area



181
182
183
 
# File 'lib/sonar_connector_filestore.rb', line 181

def area_size
  @areas.reduce({}){|h,area| h[area]=size(area) ; h}
end

#check_area(area) ⇒ Object 



79
80
81
 
# File 'lib/sonar_connector_filestore.rb', line 79

def check_area(area)
  raise "no such area: #{area}" if !@areas.include?(area) && area!=:tmp
end

#count(area) ⇒ Object

number of items in an area



164
165
166
167
 
# File 'lib/sonar_connector_filestore.rb', line 164

def count(area)
  ap = area_path(area)
  Dir[File.join(ap, "*")].length
end

#delete(area, filename) ⇒ Object

remove a file from an area



206
207
208
 
# File 'lib/sonar_connector_filestore.rb', line 206

def delete(area, filename)
  FileUtils.rm_r(file_path(area, filename))
end

#destroy! ⇒ Object 



70
71
72
73
 
# File 'lib/sonar_connector_filestore.rb', line 70

def destroy!
  logger.info("destroying: #{filestore_path}")
  FileUtils.rm_rf(filestore_path)
end

#file_path(area, filename) ⇒ Object 



88
89
90
 
# File 'lib/sonar_connector_filestore.rb', line 88

def file_path(area, filename)
  File.join(area_path(area), filename)
end

#filestore_path ⇒ Object 



75
76
77
 
# File 'lib/sonar_connector_filestore.rb', line 75

def filestore_path
  File.join(root, name.to_s)
end

#flip(area, filestore, to_area, unique_names = true) ⇒ Object

flip files from an area into a sub-directory of an area in another filestore, named by the name of this filestore thus fs1.flip(:complete, fs2, :working ) moves fs1/complete/* => fs2/working/fs1/* if unique_names is false, then unique directories are constructued in the targetfs to flip to, otherwise identical names are assumed to be identical files and will overwrite already present files



230
231
232
233
234
235
236
237
238
239
240
241
 
# File 'lib/sonar_connector_filestore.rb', line 230

def flip(area, filestore, to_area, unique_names=true)
  ap = area_path(area)
  paths = []

  scrub!(area) # only move what we need to

  # collect all moveable paths
  for_each(area) do |f|
    paths << File.join(ap, f)
  end
  filestore.receive_flip(name, to_area, paths, unique_names) if paths.length>0
end

#for_each(area) ⇒ Object

iterate over all files in top level of an area, calling a block on each



186
187
188
189
190
191
192
 
# File 'lib/sonar_connector_filestore.rb', line 186

def for_each(area)
  ap = area_path(area)
  Dir.foreach(area_path(area)) do |f|
    fp = File.join(ap,f)
    yield f if File.file?(fp) || FileStore.ordinary_directory?(fp)
  end
end

#move(from_area, filename, to_area) ⇒ Object

move a file from one area to another



211
212
213
 
# File 'lib/sonar_connector_filestore.rb', line 211

def move(from_area, filename, to_area)
  move_file(area_path(from_area), filename, area_path(to_area))
end

#process(source_area, error_area = nil, success_area = nil) ⇒ Object

process files from source_area. move it to error_area if the block raises an exception and to success_area if the block completes. if LeaveInSourceArea is raised, don’t do anything with the files



100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
 
# File 'lib/sonar_connector_filestore.rb', line 100

def process(source_area, error_area=nil, success_area=nil)
  raise "i need a block" if !block_given?
  
  files = area_files(source_area)
  files.each do |f|
    begin
      yield f
      if success_area
        move(source_area, f, success_area)
      else
        delete(source_area, f)
      end
    rescue LeaveInSourceArea=>e
      logger.info("leaving files in #{source_area}")
      raise
    rescue Exception=>e
      logger.warn(FileStore.to_s){[e.class.to_s, e.message, *e.backtrace].join("\n")}
      if error_area
        move(source_area, f, error_area)
      else
        delete(source_area, f)
      end
      raise
    end
  end
end

#process_batch(batch_size, source_area, error_area = nil, success_area = nil) ⇒ Object

process a batch of files from source_area. move them to error_area if the block raises and exception, and to success_area if the block completes, and leave where they are if LeaveInSourceArea is raised. returns the number of items processed, 0 if all work is done.



131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
 
# File 'lib/sonar_connector_filestore.rb', line 131

def process_batch(batch_size, source_area, error_area=nil, success_area=nil)
  raise "i need a block" if !block_given?

  batch = area_files(source_area, batch_size)
  return 0 if batch.size==0
  begin
    yield batch
    if success_area
      batch.each{|p| move(source_area, p, success_area)}
    else
      batch.each{|p| delete(source_area, p)}
    end
  rescue LeaveInSourceArea=>e
    logger.info("leaving files in #{source_area}")
    raise
  rescue Exception=>e
    logger.warn(FileStore.to_s){[e.class.to_s, e.message, *e.backtrace].join("\n")}
    if error_area
      batch.each{|p| move(source_area, p, error_area)}
    else
      batch.each{|p| delete(source_area, p)}
    end
    raise
  end
  return batch.size
end

#read(area, filename) ⇒ Object

read a file from an area



201
202
203
 
# File 'lib/sonar_connector_filestore.rb', line 201

def read(area, filename)
  File.read(file_path(area, filename))
end

#receive_flip(from_filestore_name, to_area, paths, unique_names) ⇒ Object

receive a flip… move all paths to be flipped into a temporary directory, and then move that directory into place in one atomic move operation



246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
 
# File 'lib/sonar_connector_filestore.rb', line 246

def receive_flip(from_filestore_name, to_area, paths, unique_names)
#        $stderr << "receive_flip(#{from_filestore_name}, #{to_area}, #{paths.inspect}, #{unique_names})\n"
  tmp_area_path = area_path(:tmp)

  # tmp_uuid
  tmp_uuid = unique_name

  # first move all moveable paths to a unique named tmp area within the receive area
  tmp_path = File.join(tmp_area_path, tmp_uuid)
  if paths.length>0
    FileUtils.mkdir_p(tmp_path)
    paths.each do |path|
      FileUtils.mv(path, tmp_path)
    end
  end

  # move everything from the receive area... recovers interrupted receive_flips too
  to_path = area_path(to_area)
  Dir.foreach(tmp_area_path) do |path|
    path_1 = File.join(tmp_area_path, path)
    if unique_names

      if FileStore.ordinary_directory?(path_1)
        # names are unique, so don't move the uuid folders
        Dir.foreach(path_1) do |file_path|
          path_2 = File.join(path_1, file_path)
          FileUtils.mv(path_2, to_path, :force=>true) if File.file?(path_2) || FileStore.ordinary_directory?(path_2)              
        end
      elsif File.file?(path_1) # names are unique, so ok to move plain files too
        FileUtils.mv(path_1, to_path, :force=>true)
      end

    else
      # move uuid named dirs
      FileUtils.mv(path_1, to_path, :force=>true) if File.file?(path_1) || FileStore.ordinary_directory?(path_1)
    end
  end

  # finally remove any empty tmp dirs
  scrub!(:tmp)
end

#scrub!(area) ⇒ Object

remove any empty directories from an area



216
217
218
 
# File 'lib/sonar_connector_filestore.rb', line 216

def scrub!(area)
  scrub_path(area_path(area), false)
end

#size(area) ⇒ Object

disk usage of an area in kb



175
176
177
178
 
# File 'lib/sonar_connector_filestore.rb', line 175

def size(area)
  ap = area_path(area)
  `du -k #{ap}`.gsub(/\W+tmp\W*$/m,'').to_i
end

#write(area, filename, content) ⇒ Object

write a file to an area



195
196
197
198
 
# File 'lib/sonar_connector_filestore.rb', line 195

def write(area, filename, content)
  ensure_area_directory(area, filename)
  File.open(file_path(area, filename), "w"){ |io| io << content }
end