Class: Traject::CommandLine

Inherits:
Object
  • Object
show all
Defined in:
lib/traject/command_line.rb

Overview

The class that executes for the Traject command line utility.

Warning, does do things like exit entire program on error at present. You probably don't want to use this class for anything but an actual shell command line, if you want to execute indexing directly, just use the Traject::Indexer directly.

A CommandLine object has a single persistent Indexer object it uses

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(argv = ARGV) ⇒ CommandLine

Returns a new instance of CommandLine.



22
23
24
25
26
27
28
29
30
 
# File 'lib/traject/command_line.rb', line 22

def initialize(argv=ARGV)
  self.console = $stderr

  self.orig_argv      = argv.dup
  self.remaining_argv = argv

  self.slop    = create_slop!
  self.options = parse_options(self.remaining_argv)
end

Instance Attribute Details

#console ⇒ Object

Returns the value of attribute console.



20
21
22
 
# File 'lib/traject/command_line.rb', line 20

def console
  @console
end

#indexer ⇒ Object

Returns the value of attribute indexer.



19
20
21
 
# File 'lib/traject/command_line.rb', line 19

def indexer
  @indexer
end

#options ⇒ Object

Returns the value of attribute options.



18
19
20
 
# File 'lib/traject/command_line.rb', line 18

def options
  @options
end

#orig_argv ⇒ Object

orig_argv is origina one passed in, remaining_argv is after destructive processing by slop, still has file args in it etc.



17
18
19
 
# File 'lib/traject/command_line.rb', line 17

def orig_argv
  @orig_argv
end

#remaining_argv ⇒ Object

orig_argv is origina one passed in, remaining_argv is after destructive processing by slop, still has file args in it etc.



17
18
19
 
# File 'lib/traject/command_line.rb', line 17

def remaining_argv
  @remaining_argv
end

#slop ⇒ Object

Returns the value of attribute slop.



18
19
20
 
# File 'lib/traject/command_line.rb', line 18

def slop
  @slop
end

Instance Method Details

#arg_check! ⇒ Object 



198
199
200
201
202
203
204
205
206
 
# File 'lib/traject/command_line.rb', line 198

def arg_check!
  if options[:command] == "process" && (options[:conf].nil? || options[:conf].length == 0)
    self.console.puts "Error: Missing required configuration file"
    self.console.puts "Exiting..."
    self.console.puts
    self.console.puts self.slop.help
    exit 2
  end
end

#assemble_settings_hash(options) ⇒ Object 



209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
 
# File 'lib/traject/command_line.rb', line 209

def assemble_settings_hash(options)
  settings = {}

  # `-s key=value` command line
  (options[:setting] || []).each do |setting_pair|
    if setting_pair =~ /\A([^=]+)\=(.*)\Z/
      key, value = $1, $2
      settings[key] = value
    else
      self.console.puts "Unrecognized setting argument '#{setting_pair}':"
      self.console.puts "Should be of format -s key=value"
      exit 3
    end
  end

  # other command line shortcuts for settings
  if options[:debug]
    settings["log.level"] = "debug"
  end
  if options[:'debug-mode']
    require 'traject/debug_writer'
    settings["writer_class_name"] = "Traject::DebugWriter"
    settings["log.level"] = "debug"
    settings["processing_thread_pool"] = 0
  end
  if options[:writer]
    settings["writer_class_name"] = options[:writer]
  end
  if options[:reader]
    settings["reader_class_name"] = options[:reader]
  end
  if options[:solr]
    settings["solr.url"] = options[:solr]
  end
  if options[:marc_type]
    settings["marc_source.type"] = options[:marc_type]
  end
  if options[:output_file]
    settings["output_file"] = options[:output_file]
  end

  return settings
end

#command_commit! ⇒ Object

Raises:

  • (ArgumentError) 


85
86
87
88
89
90
91
92
93
94
 
# File 'lib/traject/command_line.rb', line 85

def command_commit!
  require 'open-uri'
  raise ArgumentError.new("No solr.url setting provided") if indexer.settings['solr.url'].to_s.empty?

  url = "#{indexer.settings['solr.url']}/update?commit=true"
  indexer.logger.info("Sending commit to: #{url}")
  indexer.logger.info(  open(url).read )

  return true
end

#command_marcout!(io) ⇒ Object 



96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
 
# File 'lib/traject/command_line.rb', line 96

def command_marcout!(io)
  require 'marc'

  output_type = indexer.settings["marcout.type"].to_s
  output_type = "binary" if output_type.empty?

  output_arg      = unless indexer.settings["output_file"].to_s.empty?
    indexer.settings["output_file"]
  else
    $stdout
  end

  indexer.logger.info("   marcout writing type:#{output_type} to file:#{output_arg}")

  case output_type
  when "binary"
    writer = MARC::Writer.new(output_arg)

    allow_oversized = indexer.settings["marcout.allow_oversized"]
    if allow_oversized
      allow_oversized = (allow_oversized.to_s == "true")
      writer.allow_oversized = allow_oversized
    end
  when "xml"
    writer = MARC::XMLWriter.new(output_arg)
  when "human"
    writer = output_arg.kind_of?(String) ? File.open(output_arg, "w:binary") : output_arg
  else
    raise ArgumentError.new("traject marcout unrecognized marcout.type: #{output_type}")
  end

  reader      = indexer.reader!(io)

  reader.each do |record|
    writer.write record
  end

  writer.close

  return true
end

#create_slop! ⇒ Object 



254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
 
# File 'lib/traject/command_line.rb', line 254

def create_slop!
  return Slop.new(:strict => true) do
    banner "traject [options] -c configuration.rb [-c config2.rb] file.mrc"

    on 'v', 'version', "print version information to stderr"
    on 'd', 'debug', "Include debug log, -s log.level=debug"
    on 'h', 'help', "print usage information to stderr"
    on 'c', 'conf', 'configuration file path (repeatable)', :argument => true, :as => Array
    on :s, :setting, "settings: `-s key=value` (repeatable)", :argument => true, :as => Array
    on :r, :reader, "Set reader class, shortcut for -s reader_class_name=", :argument => true
    on :o, "output_file", "output file for Writer classes that write to files", :argument => true
    on :w, :writer, "Set writer class, shortcut for -s writer_class_name=", :argument => true
    on :u, :solr, "Set solr url, shortcut for -s solr.url=", :argument => true
    on :t, :marc_type, "xml, json or binary. shortcut for -s marc_source.type=", :argument => true
    on :I, "load_path", "append paths to ruby $LOAD_PATH", :argument => true, :as => Array, :delimiter => ":"

    on :x, "command", "alternate traject command: process (default); marcout; commit", :argument => true, :default => "process"

    on "stdin", "read input from stdin"
    on "debug-mode", "debug logging, single threaded, output human readable hashes"
  end
end

#execute ⇒ Object

Returns true on success or false on failure; may also raise exceptions; may also exit program directly itself (yeah, could use some normalization)



34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
 
# File 'lib/traject/command_line.rb', line 34

def execute
  if options[:version]
    self.console.puts "traject version #{Traject::VERSION}"
    return
  end
  if options[:help]
    self.console.puts slop.help
    return
  end


  (options[:load_path] || []).each do |path|
    $LOAD_PATH << path unless $LOAD_PATH.include? path
  end

  arg_check!

  self.indexer = initialize_indexer!

  ######
  # SAFE TO LOG to indexer.logger starting here, after indexer is set up from conf files
  # with logging config.
  #####

  indexer.logger.info("traject executing with: `#{orig_argv.join(' ')}`")

  # Okay, actual command process! All command_ methods should return true
  # on success, or false on failure.
  result =
    case options[:command]
    when "process"
      (io, filename) = get_input_io(self.remaining_argv)
      indexer.settings['command_line.filename'] = filename if filename
      indexer.process(io)
    when "marcout"
       (io, filename) = get_input_io(self.remaining_argv)
      indexer.settings['command_line.filename'] = filename if filename
      command_marcout!(io)
    when "commit"
      command_commit!
    else
      raise ArgumentError.new("Unrecognized traject command: #{options[:command]}")
    end

  return result
rescue Exception => e
  # Try to log unexpected exceptions if possible
  indexer && indexer.logger && indexer.logger.fatal("Traject::CommandLine: Unexpected exception, terminating execution: #{e.inspect}") rescue nil
  raise e
end

#get_input_io(argv) ⇒ Object 



138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
 
# File 'lib/traject/command_line.rb', line 138

def get_input_io(argv)
  # ARGF might be perfect for this, but problems with it include:
  # * jruby is broken, no way to set it's encoding, leads to encoding errors reading non-ascii
  #   https://github.com/jruby/jruby/issues/891
  # * It's apparently not enough like an IO object for at least one of the ruby-marc XML
  #   readers:
  #   NoMethodError: undefined method `to_inputstream' for ARGF:Object
  #      init at /Users/jrochkind/.gem/jruby/1.9.3/gems/marc-0.5.1/lib/marc/xml_parsers.rb:369
  #
  # * It INSISTS on reading from ARGFV, making it hard to test, or use when you want to give
  #   it a list of files on something other than ARGV.
  #
  # So for now we do just one file, or stdin if specified. Sorry!

  filename = nil
  if options[:stdin]
    indexer.logger.info("Reading from standard input")
    io = $stdin
  elsif argv.length > 1
    self.console.puts "Sorry, traject can only handle one input file at a time right now. `#{argv}` Exiting..."
    exit 1
  elsif argv.length == 0
    io = File.open(File::NULL, 'r')
    indexer.logger.info("Warning, no file input given. Use command-line argument '--stdin' to use standard input ")
  else
    io = File.open(argv.first, 'r')
    filename = argv.first
    indexer.logger.info "Reading from #{filename}"
  end

  return io, filename
end

#initialize_indexer! ⇒ Object 



277
278
279
280
281
282
 
# File 'lib/traject/command_line.rb', line 277

def initialize_indexer!
  indexer = Traject::Indexer.new self.assemble_settings_hash(self.options)
  load_configuration_files!(indexer, options[:conf])

  return indexer
end

#load_configuration_files!(my_indexer, conf_files) ⇒ Object 



171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
 
# File 'lib/traject/command_line.rb', line 171

def load_configuration_files!(my_indexer, conf_files)
  conf_files.each do |conf_path|
    begin
      file_io = File.open(conf_path)
    rescue Errno::ENOENT => e
      self.console.puts "Could not find configuration file '#{conf_path}', exiting..."
      exit 2
    end

    begin
      my_indexer.instance_eval(file_io.read, conf_path)
    rescue Exception => e
      if (conf_trace = e.backtrace.find {|l| l.start_with? conf_path}) &&
         (conf_trace =~ /\A.*\:(\d+)\:in/)
        line_number = $1
      end

      self.console.puts "Error processing configuration file '#{conf_path}' at line #{line_number}"
      self.console.puts "  #{e.class}: #{e.message}"
      if e.backtrace.first =~ /\A(.*)\:in/
        self.console.puts "  from #{$1}"
      end
      exit 3
    end
  end
end

#parse_options(argv) ⇒ Object 



284
285
286
287
288
289
290
291
292
293
294
295
296
297
 
# File 'lib/traject/command_line.rb', line 284

def parse_options(argv)

  begin
    self.slop.parse!(argv)
  rescue Slop::Error => e
    self.console.puts "Error: #{e.message}"
    self.console.puts "Exiting..."
    self.console.puts
    self.console.puts slop.help
    exit 1
  end

  return self.slop.to_hash
end