Class: Pho::FileManagement::FileSplitter

Inherits:
Object
  • Object
show all
Defined in:
lib/pho/upload.rb

Overview

Supports splitting RDF data files into smaller chunks of ntriples

Constant Summary collapse

DEFAULT_CHUNK_SIZE = 
10000

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(dir = "/tmp", triples = DEFAULT_CHUNK_SIZE, handler = Pho::FileManagement::StatementHandler.new) ⇒ FileSplitter

Create a file splitter instance

dir

temporary directory into which split files should be written

triples

number of triples per split file

handler

statement handler to allow pre-processing of statements



58
59
60
61
62
63
 
# File 'lib/pho/upload.rb', line 58

def initialize(dir="/tmp", triples=DEFAULT_CHUNK_SIZE, 
    handler=Pho::FileManagement::StatementHandler.new)
  @dir = dir
  @triples = triples
  @handler = handler
end

Instance Attribute Details

#dir ⇒ Object (readonly)

Returns the value of attribute dir.



49
50
51
 
# File 'lib/pho/upload.rb', line 49

def dir
  @dir
end

#handler ⇒ Object (readonly)

Returns the value of attribute handler.



49
50
51
 
# File 'lib/pho/upload.rb', line 49

def handler
  @handler
end

#triples ⇒ Object (readonly)

Returns the value of attribute triples.



49
50
51
 
# File 'lib/pho/upload.rb', line 49

def triples
  @triples
end

Instance Method Details

#split_file(filename, format = :ntriples) ⇒ Object

Split a single file, in any parseable RDF format into smaller chunks of ntriples. Chunked files are stored in default temporary directory for this instance

filename

name of the file to split

format

input format, default is :ntriples



71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
 
# File 'lib/pho/upload.rb', line 71

def split_file(filename, format=:ntriples)
  
  basename = File.basename(filename, ".#{filename.split(".").last}")
  count = 0
  stmts = []
  RDF::Reader.for(format).new(File.new(filename)) do |reader|
    reader.each_statement do |statement|            
      count += 1
      stmts << @handler.handle( statement ) 
      if count % @triples == 0
        RDF::Writer.open( File.join(@dir, "#{basename}_#{count}.nt") ) do |writer|
          stmts.each do |s|
            writer << s
          end
        end              
        stmts = []              
      end
    end
  end
  if !stmts.empty?
    RDF::Writer.open( File.join(@dir, "#{basename}_#{count}.nt") ) do |writer|
      stmts.each do |s|
        writer << s
      end
    end
  end
end

#split_files(list_of_filenames, format = :ntriples) ⇒ Object

Split a list of files into smaller chunks

list_of_filenames

array of filenames

format

format of the files, default is :ntriples



103
104
105
106
107
 
# File 'lib/pho/upload.rb', line 103

def split_files(list_of_filenames, format=:ntriples)
  list_of_filenames.each do |name|
    split_file(name, format)
  end
end