Class: MiGA::Project

Inherits:
MiGA
  • Object
show all
Includes:
ProjectResult
Defined in:
lib/miga/project.rb

Overview

MiGA representation of a project.

Constant Summary collapse

@@FOLDERS =

Top-level folders inside a project.

%w[data metadata daemon]
@@DATA_FOLDERS =

Folders for results.

%w[
  01.raw_reads 02.trimmed_reads 03.read_quality 04.trimmed_fasta
  05.assembly 06.cds
  07.annotation 07.annotation/01.function 07.annotation/02.taxonomy
  07.annotation/01.function/01.essential
  07.annotation/01.function/02.ssu
  07.annotation/02.taxonomy/01.mytaxa
  07.annotation/03.qa 07.annotation/03.qa/01.checkm
  07.annotation/03.qa/02.mytaxa_scan
  08.mapping 08.mapping/01.read-ctg 08.mapping/02.read-gene
  09.distances 09.distances/01.haai 09.distances/02.aai
  09.distances/03.ani 09.distances/04.ssu
  10.clades 10.clades/01.find 10.clades/02.ani 10.clades/03.ogs
  10.clades/04.phylogeny 10.clades/04.phylogeny/01.essential
  10.clades/04.phylogeny/02.core 10.clades/05.metadata
  90.stats
]
@@RESULT_DIRS = 
{
  # Distances
  haai_distances: "09.distances/01.haai",
  aai_distances: "09.distances/02.aai",
  ani_distances: "09.distances/03.ani",
  #ssu_distances: "09.distances/04.ssu",
  # Clade identification
  clade_finding: "10.clades/01.find",
  # Clade analysis
  subclades: "10.clades/02.ani",
  ogs: "10.clades/03.ogs",
  #ess_phylogeny: "10.clades/04.phylogeny/01.essential",
  #core_phylogeny: "10.clades/04.phylogeny/02.core",
  #clade_metadata: "10.clades/05.metadata"
  project_stats: "90.stats"
}
@@KNOWN_TYPES = 
{
  mixed: {
    description: "Mixed collection of genomes, metagenomes, and viromes.",
    single: true, multi: true},
  genomes: {description: "Collection of genomes.",
    single: true, multi: false},
  clade: {description: "Collection of closely-related genomes (ANI >= 90%).",
    single: true, multi: false},
  metagenomes: {description: "Collection of metagenomes and/or viromes.",
    single: false, multi: true}
}
@@DISTANCE_TASKS = 
[:project_stats,
:haai_distances, :aai_distances, :ani_distances, :clade_finding]
@@INCLADE_TASKS = 
[:subclades, :ogs]

Constants included from MiGA

CITATION, VERSION, VERSION_DATE, VERSION_NAME

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Methods inherited from MiGA

CITATION, DEBUG, DEBUG_OFF, DEBUG_ON, DEBUG_TRACE_OFF, DEBUG_TRACE_ON, FULL_VERSION, LONG_VERSION, VERSION, VERSION_DATE, clean_fasta_file, initialized?, #result_files_exist?, root_path, tabulate

Constructor Details

#initialize(path, update = false) ⇒ Project

Create a new MiGA::Project at path, if it doesn’t exist and update is false, or load an existing one.



112
113
114
115
116
117
118
119
120
 
# File 'lib/miga/project.rb', line 112

def initialize(path, update=false)
  @datasets = {}
  @path = File.absolute_path(path)
  self.create if not update and not Project.exist? self.path
  self.load if self.metadata.nil?
  self.load_plugins
  self.metadata[:type] = :mixed if type.nil?
  raise "Unrecognized project type: #{type}." if @@KNOWN_TYPES[type].nil?
end

Instance Attribute Details

#metadata ⇒ Object (readonly)

Information about the project as MiGA::Metadata.



107
108
109
 
# File 'lib/miga/project.rb', line 107

def metadata
  @metadata
end

#path ⇒ Object (readonly)

Absolute path to the project folder.



103
104
105
 
# File 'lib/miga/project.rb', line 103

def path
  @path
end

Class Method Details

.DISTANCE_TASKS ⇒ Object

Project-wide distance estimations.



76
 
# File 'lib/miga/project.rb', line 76

def self.DISTANCE_TASKS ; @@DISTANCE_TASKS ; end

.exist?(path) ⇒ Boolean

Does the project at path exist?



87
88
89
 
# File 'lib/miga/project.rb', line 87

def self.exist?(path)
  Dir.exist?(path) and File.exist?("#{path}/miga.project.json")
end

.INCLADE_TASKS ⇒ Object

Project-wide tasks for :clade projects.



82
 
# File 'lib/miga/project.rb', line 82

def self.INCLADE_TASKS ; @@INCLADE_TASKS ; end

.KNOWN_TYPES ⇒ Object

Supported types of projects.



61
 
# File 'lib/miga/project.rb', line 61

def self.KNOWN_TYPES ; @@KNOWN_TYPES ; end

.load(path) ⇒ Object

Load the project at path. Returns MiGA::Project if project exists, nil otherwise.



94
95
96
97
 
# File 'lib/miga/project.rb', line 94

def self.load(path)
  return nil unless Project.exist? path
  Project.new path
end

.RESULT_DIRS ⇒ Object

Directories containing the results from project-wide tasks.



41
 
# File 'lib/miga/project.rb', line 41

def self.RESULT_DIRS ; @@RESULT_DIRS ; end

Instance Method Details

#add_dataset(name) ⇒ Object

Add dataset identified by name and return MiGA::Dataset.



204
205
206
207
208
209
210
211
 
# File 'lib/miga/project.rb', line 204

def add_dataset(name)
  unless metadata[:datasets].include? name
    MiGA::Dataset.new(self, name)
    @metadata[:datasets] << name
    save
  end
  dataset(name)
end

#add_result(name, save = true) ⇒ Object

Add the result identified by Symbol name, and return MiGA::Result. Save the result if save.



269
270
271
272
273
274
275
276
277
 
# File 'lib/miga/project.rb', line 269

def add_result(name, save=true)
  return nil if @@RESULT_DIRS[name].nil?
  base = "#{path}/data/#{@@RESULT_DIRS[name]}/miga-project"
  return MiGA::Result.load("#{base}.json") unless save
  return nil unless result_files_exist?(base, ".done")
  r = send("add_result_#{name}", base)
  r.save unless r.nil?
  r
end

#create ⇒ Object

Create an empty project.



124
125
126
127
128
129
130
131
132
133
134
135
136
137
 
# File 'lib/miga/project.rb', line 124

def create
  unless MiGA::MiGA.initialized?
    raise "Impossible to create project in uninitialized MiGA."
  end
  dirs = [path] + @@FOLDERS.map{|d| "#{path}/#{d}" } +
    @@DATA_FOLDERS.map{ |d| "#{path}/data/#{d}"}
  dirs.each{ |d| Dir.mkdir(d) unless Dir.exist? d }
  @metadata = MiGA::Metadata.new(self.path + "/miga.project.json",
    {datasets: [], name: File.basename(path)})
  FileUtils.cp("#{ENV["MIGA_HOME"]}/.miga_daemon.json",
    "#{path}/daemon/daemon.json") unless
      File.exist? "#{path}/daemon/daemon.json"
  self.load
end

#dataset(name) ⇒ Object

Returns MiGA::Dataset.



180
181
182
183
184
185
186
 
# File 'lib/miga/project.rb', line 180

def dataset(name)
  name = name.miga_name
  return nil unless MiGA::Dataset.exist?(self, name)
  @datasets ||= {}
  @datasets[name] ||= MiGA::Dataset.new(self, name)
  @datasets[name]
end

#dataset_names ⇒ Object

Returns Array of String (without evaluating dataset objects).



174
175
176
 
# File 'lib/miga/project.rb', line 174

def dataset_names
  metadata[:datasets]
end

#datasets ⇒ Object

Returns Array of MiGA::Dataset.



168
169
170
 
# File 'lib/miga/project.rb', line 168

def datasets
  metadata[:datasets].map{ |name| dataset(name) }
end

#done_preprocessing?(save = true) ⇒ Boolean

Are all the datasets in the project preprocessed? Save intermediate results if save.



316
317
318
 
# File 'lib/miga/project.rb', line 316

def done_preprocessing?(save=true)
  datasets.map{|ds| (not ds.is_ref?) or ds.done_preprocessing?(save) }.all?
end

#each_dataset(&blk) ⇒ Object

Iterate through datasets, with one or two variables passed to blk. If one, the dataset MiGA::Dataset object is passed. If two, the name and the dataset object are passed.



192
193
194
195
196
197
198
199
200
 
# File 'lib/miga/project.rb', line 192

def each_dataset(&blk)
  metadata[:datasets].each do |name|
    if blk.arity == 1
      blk.call(dataset(name))
    else
      blk.call(name, dataset(name))
    end
  end
end

#each_dataset_profile_advance(&blk) ⇒ Object

Call blk passing the result of MiGA::Dataset#profile_advance for each registered dataset.



338
339
340
 
# File 'lib/miga/project.rb', line 338

def each_dataset_profile_advance(&blk)
  each_dataset { |ds| blk.call(ds.profile_advance) }
end

#import_dataset(ds, method = :hardlink) ⇒ Object

Import the dataset ds, a MiGA::Dataset, using method which is any method supported by File#generic_transfer.



226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
 
# File 'lib/miga/project.rb', line 226

def import_dataset(ds, method=:hardlink)
  raise "Impossible to import dataset, it already exists: #{ds.name}." if
    MiGA::Dataset.exist?(self, ds.name)
  # Import dataset results
  ds.each_result do |task, result|
    # import result files
    result.each_file do |file|
      File.generic_transfer("#{result.dir}/#{file}",
        "#{path}/data/#{MiGA::Dataset.RESULT_DIRS[task]}/#{file}", method)
    end
    # import result metadata
    %w(json start done).each do |suffix|
      if File.exist? "#{result.dir}/#{ds.name}.#{suffix}"
        File.generic_transfer("#{result.dir}/#{ds.name}.#{suffix}",
          "#{path}/data/#{MiGA::Dataset.RESULT_DIRS[task]}/" +
                     "#{ds.name}.#{suffix}", method)
      end
    end
  end
  # Import dataset metadata
  File.generic_transfer("#{ds.project.path}/metadata/#{ds.name}.json",
    "#{self.path}/metadata/#{ds.name}.json", method)
  # Save dataset
  self.add_dataset(ds.name)
end

#install_plugin(path) ⇒ Object

Installs the plugin in the specified path.



344
345
346
347
348
349
350
351
352
353
 
# File 'lib/miga/project.rb', line 344

def install_plugin(path)
  abs_path = File.absolute_path(path)
  raise "Plugin already installed in project: #{abs_path}." unless
    metadata[:plugins].nil? or not metadata[:plugins].include?(abs_path)
  raise "Malformed MiGA plugin: #{abs_path}." unless
    File.exist?(File.expand_path("miga-plugin.json", abs_path))
  self.metadata[:plugins] ||= []
  self.metadata[:plugins] << abs_path
  save
end

#is_clade? ⇒ Boolean

Is this a clade project?



164
 
# File 'lib/miga/project.rb', line 164

def is_clade? ; type==:clade ; end

#load ⇒ Object

(Re-)load project data and metadata.



148
149
150
151
152
 
# File 'lib/miga/project.rb', line 148

def load
  @datasets = {}
  @metadata = MiGA::Metadata.load "#{path}/miga.project.json"
  raise "Couldn't find project metadata at #{path}" if metadata.nil?
end

#load_plugins ⇒ Object 



369
370
371
 
# File 'lib/miga/project.rb', line 369

def load_plugins
  plugins.each { |pl| require File.expand_path("lib-plugin.rb", pl) }
end

#name ⇒ Object

Name of the project.



156
 
# File 'lib/miga/project.rb', line 156

def name ; metadata[:name] ; end

#next_distances(save = true) ⇒ Object

Get the next distances task, saving intermediate results if save. Returns a Symbol.



282
283
284
 
# File 'lib/miga/project.rb', line 282

def next_distances(save=true)
  @@DISTANCE_TASKS.find{ |t| add_result(t, save).nil? }
end

#next_inclade(save = true) ⇒ Object

Get the next inclade task, saving intermediate results if save. Returns a Symbol.



289
290
291
292
 
# File 'lib/miga/project.rb', line 289

def next_inclade(save=true)
  return nil unless metadata[:type]==:clade
  @@INCLADE_TASKS.find{ |t| add_result(t, save).nil? }
end

#plugins ⇒ Object

List plugins installed in the project.



367
 
# File 'lib/miga/project.rb', line 367

def plugins ; metadata[:plugins] ||= [] ; end

#profile_datasets_advance ⇒ Object

Returns a two-dimensional matrix (Array of Array) where the first index corresponds to the dataset, the second index corresponds to the dataset task, and the value corresponds to:

  • 0: Before execution.

  • 1: Done (or not required).

  • 2: To do.



327
328
329
330
331
332
333
 
# File 'lib/miga/project.rb', line 327

def profile_datasets_advance
  advance = []
  self.each_dataset_profile_advance do |ds_adv|
    advance << ds_adv
  end
  advance
end

#result(name) ⇒ Object

Get result identified by Symbol name, returns MiGA::Result.



254
255
256
257
258
 
# File 'lib/miga/project.rb', line 254

def result(name)
  dir = @@RESULT_DIRS[name.to_sym]
  return nil if dir.nil?
  MiGA::Result.load("#{path}/data/#{dir}/miga-project.json")
end

#results ⇒ Object

Get all results, an Array of MiGA::Result.



262
263
264
 
# File 'lib/miga/project.rb', line 262

def results
  @@RESULT_DIRS.keys.map{ |k| result(k) }.reject{ |r| r.nil? }
end

#save ⇒ Object

Save any changes persistently.



141
142
143
144
 
# File 'lib/miga/project.rb', line 141

def save
  metadata.save
  self.load
end

#type ⇒ Object

Type of project.



160
 
# File 'lib/miga/project.rb', line 160

def type ; metadata[:type] ; end

#uninstall_plugin(path) ⇒ Object

Uninstall the plugin in the specified path.



357
358
359
360
361
362
363
 
# File 'lib/miga/project.rb', line 357

def uninstall_plugin(path)
  abs_path = File.absolute_path(path)
  raise "Plugin not currently installed: #{abs_path}." if
    metadata[:plugins].nil? or not metadata[:plugins].include?(abs_path)
  self.metadata[:plugins].delete(abs_path)
  save
end

#unlink_dataset(name) ⇒ Object

Unlink dataset identified by name and return MiGA::Dataset.



215
216
217
218
219
220
221
 
# File 'lib/miga/project.rb', line 215

def unlink_dataset(name)
  d = dataset(name)
  return nil if d.nil?
  self.metadata[:datasets].delete(name)
  save
  d
end

#unregistered_datasets ⇒ Object

Find all datasets with (potential) result files but are yet unregistered.



296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
 
# File 'lib/miga/project.rb', line 296

def unregistered_datasets
  datasets = []
  MiGA::Dataset.RESULT_DIRS.values.each do |dir|
    dir_p = "#{path}/data/#{dir}"
    next unless Dir.exist? dir_p
    Dir.entries(dir_p).each do |file|
      next unless
        file =~ %r{
          \.(fa(a|sta|stqc?)?|fna|solexaqa|gff[23]?|done|ess)(\.gz)?$
        }x
      m = /([^\.]+)/.match(file)
      datasets << m[1] unless m.nil? or m[1] == "miga-project"
    end
  end
  datasets.uniq - metadata[:datasets]
end