Module: Spark

Includes:
Helper::System
Defined in:
lib/spark/sampler.rb,
lib/spark.rb,
lib/spark/cli.rb,
lib/spark/rdd.rb,
lib/spark/sort.rb,
lib/spark/sort.rb,
lib/spark/build.rb,
lib/spark/error.rb,
lib/spark/mllib.rb,
lib/spark/config.rb,
lib/spark/ext/io.rb,
lib/spark/helper.rb,
lib/spark/logger.rb,
lib/spark/command.rb,
lib/spark/context.rb,
lib/spark/sampler.rb,
lib/spark/version.rb,
lib/spark/constant.rb,
lib/spark/ext/hash.rb,
lib/spark/broadcast.rb,
lib/spark/ext/module.rb,
lib/spark/ext/object.rb,
lib/spark/ext/string.rb,
lib/spark/serializer.rb,
lib/spark/accumulator.rb,
lib/spark/accumulator.rb,
lib/spark/ext/integer.rb,
lib/spark/java_bridge.rb,
lib/spark/mllib/matrix.rb,
lib/spark/mllib/matrix.rb,
lib/spark/mllib/matrix.rb,
lib/spark/mllib/matrix.rb,
lib/spark/mllib/vector.rb,
lib/spark/mllib/vector.rb,
lib/spark/mllib/vector.rb,
lib/spark/mllib/vector.rb,
lib/spark/stat_counter.rb,
lib/spark/ext/ip_socket.rb,
lib/spark/helper/logger.rb,
lib/spark/helper/parser.rb,
lib/spark/helper/system.rb,
lib/spark/serializer/oj.rb,
lib/spark/storage_level.rb,
lib/spark/command_builder.rb,
lib/spark/java_bridge/rjb.rb,
lib/spark/serializer/base.rb,
lib/spark/serializer/pair.rb,
lib/spark/serializer/text.rb,
lib/spark/helper/serialize.rb,
lib/spark/helper/statistic.rb,
lib/spark/java_bridge/base.rb,
lib/spark/command_validator.rb,
lib/spark/java_bridge/jruby.rb,
lib/spark/serializer/batched.rb,
lib/spark/serializer/marshal.rb,
lib/spark/serializer/cartesian.rb,
lib/spark/serializer/compressed.rb,
lib/spark/mllib/regression/lasso.rb,
lib/spark/mllib/regression/ridge.rb,
lib/spark/mllib/clustering/kmeans.rb,
lib/spark/mllib/clustering/kmeans.rb,
lib/spark/mllib/regression/common.rb,
lib/spark/mllib/regression/common.rb,
lib/spark/mllib/regression/linear.rb,
lib/spark/serializer/auto_batched.rb,
lib/spark/serializer/message_pack.rb,
lib/spark/mllib/classification/svm.rb,
lib/spark/mllib/classification/svm.rb,
lib/spark/mllib/classification/common.rb,
lib/spark/mllib/classification/common.rb,
lib/spark/mllib/regression/labeled_point.rb,
lib/spark/mllib/classification/naive_bayes.rb,
lib/spark/mllib/classification/naive_bayes.rb,
lib/spark/mllib/ruby_matrix/matrix_adapter.rb,
lib/spark/mllib/ruby_matrix/vector_adapter.rb,
lib/spark/mllib/clustering/gaussian_mixture.rb,
lib/spark/mllib/clustering/gaussian_mixture.rb,
lib/spark/mllib/classification/logistic_regression.rb,
lib/spark/mllib/classification/logistic_regression.rb,
lib/spark/mllib/classification/logistic_regression.rb,
ext/ruby_c/ruby-spark.c

Overview

Spark::JavaBridge::Base

Parent for all adapter (ruby - java)

Defined Under Namespace

Modules: Build, CommandValidator, Constant, CoreExtension, Digest, Helper, InternalSorter, JavaBridge, Mllib, RandomGenerator, Sampler, Serializer Classes: Accumulator, AccumulatorError, Broadcast, BroadcastError, BuildError, CLI, Command, CommandBuilder, CommandError, Config, ConfigurationError, Context, ContextError, ExternalSorter, JavaBridgeError, Logger, MllibError, NotImplemented, ParseError, PipelinedRDD, RDD, RDDError, SerializeError, StatCounter, StorageLevel

Constant Summary collapse

DEFAULT_CONFIG_FILE = 
File.join(Dir.home, '.ruby-spark.conf')
VERSION = 
'1.2.1'

Class Method Summary collapse

Methods included from Helper::System

included

Class Method Details

.clear_config ⇒ Object

Destroy current configuration. This can be useful for restarting config to set new. It has no effect if context is already started.



75
76
77
 
# File 'lib/spark.rb', line 75

def self.clear_config
  @config = nil
end

.config(&block) ⇒ Object

Returns current configuration. Configurations can be changed until context is initialized. In this case config is locked only for reading.

Configuration can be changed:

Spark.config.set('spark.app.name', 'RubySpark')

Spark.config['spark.app.name'] = 'RubySpark'

Spark.config do
  set 'spark.app.name', 'RubySpark'
end



63
64
65
66
67
68
69
70
71
 
# File 'lib/spark.rb', line 63

def self.config(&block)
  @config ||= Spark::Config.new

  if block_given?
    @config.instance_eval(&block)
  else
    @config
  end
end

.context ⇒ Object Also known as: sc

Return a current active context or nil.

TODO: Run ‘start` if context is nil?



83
84
85
 
# File 'lib/spark.rb', line 83

def self.context
  @context
end

.java_bridge ⇒ Object Also known as: jb 



218
219
220
 
# File 'lib/spark.rb', line 218

def self.java_bridge
  @java_bridge
end

.load_defaults ⇒ Object

Load default configuration for Spark and RubySpark By default are values stored at ~/.ruby-spark.conf File is automatically created



120
121
122
123
124
125
126
 
# File 'lib/spark.rb', line 120

def self.load_defaults
  unless File.exists?(DEFAULT_CONFIG_FILE)
    save_defaults_to(DEFAULT_CONFIG_FILE)
  end

  load_defaults_from(DEFAULT_CONFIG_FILE)
end

.load_defaults_from(file_path) ⇒ Object

Clear prev setting and load new from file



129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
 
# File 'lib/spark.rb', line 129

def self.load_defaults_from(file_path)
  # Parse values
  values = File.readlines(file_path)
  values.map!(&:strip)
  values.select!{|value| value.start_with?('gem.')}
  values.map!{|value| value.split(nil, 2)}
  values = Hash[values]

  # Clear prev values
  @target_dir = nil
  @ruby_spark_jar = nil
  @spark_home = nil

  # Load new
  @target_dir = values['gem.target']
end

.load_lib(target = nil) ⇒ Object

Load dependent libraries, can be use once Cannot load before CLI::install

Parameters:

target

path to directory where are located sparks .jar files or single Spark jar



208
209
210
211
212
213
214
215
216
 
# File 'lib/spark.rb', line 208

def self.load_lib(target=nil)
  return if @java_bridge

  target ||= Spark.target_dir

  @java_bridge = JavaBridge.init(target)
  @java_bridge.import_all
  nil
end

.logger ⇒ Object

Global settings and variables



170
171
172
 
# File 'lib/spark.rb', line 170

def self.logger
  @logger ||= Spark::Logger.new
end

.print_logo(message = nil) ⇒ Object 



35
36
37
38
39
40
41
42
43
44
45
46
47
48
 
# File 'lib/spark.rb', line 35

def self.(message=nil)
  puts <<-STRING

  Welcome to
                __           ____              __
      ______ __/ /  __ __   / __/__  ___ _____/ /__
     / __/ // / _ \\/ // /  _\\ \\/ _ \\/ _ `/ __/  '_/
    /_/  \\_,_/_.__/\\_, /  /___/ .__/\\_,_/_/ /_/\\_\\   version #{Spark::VERSION}
                  /___/      /_/

  #{message}

  STRING
end

.root ⇒ Object Also known as: home

Root of the gem



175
176
177
 
# File 'lib/spark.rb', line 175

def self.root
  @root ||= File.expand_path('..', File.dirname(__FILE__))
end

.ruby_spark_jar ⇒ Object 



189
190
191
 
# File 'lib/spark.rb', line 189

def self.ruby_spark_jar
  @ruby_spark_jar ||= File.join(target_dir, 'ruby-spark.jar')
end

.save_defaults_to(file_path) ⇒ Object

Create target dir and new config file



147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
 
# File 'lib/spark.rb', line 147

def self.save_defaults_to(file_path)
  dir = File.join(Dir.home, ".ruby-spark.#{SecureRandom.uuid}")

  if Dir.exist?(dir)
    save_defaults_to(file_path)
  else
    Dir.mkdir(dir, 0700)
    file = File.open(file_path, 'w')
    file.puts "# Directory where will be Spark saved"
    file.puts "gem.target   #{dir}"
    file.puts ""
    file.puts "# You can also defined spark properties"
    file.puts "# spark.master                       spark://master:7077"
    file.puts "# spark.ruby.serializer              marshal"
    file.puts "# spark.ruby.serializer.batch_size   2048"
    file.close
  end
end

.spark_ext_dir ⇒ Object 



193
194
195
 
# File 'lib/spark.rb', line 193

def self.spark_ext_dir
  @spark_ext_dir ||= File.join(root, 'ext', 'spark')
end

.start ⇒ Object

Initialize spark context if not already. Config will be automatically loaded on constructor. From that point ‘config` will use configuration from running Spark and will be locked only for reading.



90
91
92
93
94
95
96
 
# File 'lib/spark.rb', line 90

def self.start
  if started?
    # Already started
  else
    @context ||= Spark::Context.new
  end
end

.started? ⇒ Boolean

Returns:

  • (Boolean) 


109
110
111
 
# File 'lib/spark.rb', line 109

def self.started?
  !!@context
end

.stop ⇒ Object 



98
99
100
101
102
103
104
105
106
107
 
# File 'lib/spark.rb', line 98

def self.stop
  @context.stop
  RubyWorker.stopServer
  logger.info('Workers were stopped')
rescue
  nil
ensure
  @context = nil
  clear_config
end

.target_dir ⇒ Object

Default directory for java extensions



180
181
182
 
# File 'lib/spark.rb', line 180

def self.target_dir
  @target_dir ||= File.join(root, 'target')
end

.worker_dir ⇒ Object

Directory where is worker.rb



185
186
187
 
# File 'lib/spark.rb', line 185

def self.worker_dir
  @worker_dir ||= File.join(root, 'lib', 'spark', 'worker')
end