Module: Pipeline

Defined in:
lib/pipeline.rb,
lib/pipeline/version.rb

Defined Under Namespace

Modules: Options, Util Classes: AV, BaseFilter, BaseMounter, BaseReporter, BaseTask, Brakeman, BundleAudit, CSVReporter, Checkmarx, DawnScanner, DepCheckListener, DependencyError, DockerMounter, ESLint, Event, FIM, FileSystemMounter, Filters, FindSecurityBugs, Finding, GitMounter, ISOMounter, JSONReporter, JiraConfigError, JiraOneTimeFilter, JiraReporter, Mounters, NoPipelineError, NoTargetError, NodeSecurityProject, Npm, OWASPDependencyCheck, PMD, RemoveAllFilter, Reporters, RetireJS, SFL, ScanJS, Scanner, Snyk, Tasks, Test, TextReporter, Tracker, URLMounter, ZAPCondensingFilter, Zap

Constant Summary collapse

Warnings_Found_Exit_Code =

This exit code is used when warnings are found and the –exit-on-warn option is set

3
CONFIG_FILES = 
[
  File.expand_path("./config/pipeline.yml"),
  File.expand_path("~/.pipeline/config.yml"),
  File.expand_path("/etc/pipeline/config.yml")
]
Version = 
"0.8.7"

Class Method Summary collapse

Class Method Details

.add_external_tasks(options) ⇒ Object 



307
308
309
310
311
 
# File 'lib/pipeline.rb', line 307

def self.add_external_tasks options
  options[:additional_tasks_path].each do |path|
    Pipeline::Tasks.initialize_tasks path
  end if options[:additional_tasks_path]
end

.config_file(custom_location = nil) ⇒ Object 



105
106
107
108
 
# File 'lib/pipeline.rb', line 105

def self.config_file custom_location = nil
  supported_locations = [File.expand_path(custom_location || "")] + CONFIG_FILES
  supported_locations.detect {|f| File.file?(f) }
end

.debug(message) ⇒ Object 



290
291
292
293
 
# File 'lib/pipeline.rb', line 290

def self.debug message
  $stderr.puts message if @debug
  $logfile.puts "[#{Time.now}] #{message}" if $logfile
end

.default_options ⇒ Object

Default set of options



111
112
113
114
115
116
117
118
119
120
121
122
 
# File 'lib/pipeline.rb', line 111

def self.default_options
  {
    :parallel_tasks => true,
    :skip_tasks => Set.new(),
    :exit_on_warn => true,
    :output_format => :text,
    :working_dir => "~/line/tmp/",
    :zap_host => "http://localhost",
    :zap_port => "9999",
    :labels => Set.new() << "filesystem" << "code"     # Defaults to run.
  }
end

.dump_config(options) ⇒ Object

Output configuration to YAML



206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
 
# File 'lib/pipeline.rb', line 206

def self.dump_config options
  if options[:create_config].is_a? String
    file = options[:create_config]
  else
    file = nil
  end

  options.delete :create_config

  options.each do |k,v|
    if v.is_a? Set
      options[k] = v.to_a
    end
  end

  if file
    File.open file, "w" do |f|
      YAML.dump options, f
    end
    puts "Output configuration to #{file}"
  else
    puts YAML.dump(options)
  end
  exit
end

.error(message) ⇒ Object 



275
276
277
278
 
# File 'lib/pipeline.rb', line 275

def self.error message
  $stderr.puts message
  $logfile.puts "[#{Time.now}] #{message}" if $logfile
end

.get_output_format(options) ⇒ Object

Determine output formats based on options or options



126
127
128
129
130
131
132
133
134
135
136
137
138
139
 
# File 'lib/pipeline.rb', line 126

def self.get_output_format options
  if options[:output_file]
    get_format_from_output_file options[:output_file]
  elsif options[:output_format]
    get_format_from_output_format options[:output_format]
  else
    begin
      require 'terminal-table'
      return [:to_s]
    rescue LoadError
      return [:to_json]
    end
  end
end

.list_checks(options) ⇒ Object

Output list of tasks (for ‘-k` option)



184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
 
# File 'lib/pipeline.rb', line 184

def self.list_checks options
  require 'pipeline/scanner'

  add_external_tasks options

  if options[:list_optional_tasks]
    $stderr.puts "Optional Tasks:"
    tasks = Tasks.optional_tasks
  else
    $stderr.puts "Available tasks:"
    tasks = Tasks.tasks
  end

  format_length = 30

  $stderr.puts "-" * format_length
  tasks.each do |task|
    $stderr.printf("%-#{format_length}s\n", task.name)
  end
end

.load_options(custom_location, quiet) ⇒ Object

Load options from YAML file



85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
 
# File 'lib/pipeline.rb', line 85

def self.load_options custom_location, quiet
  #Load configuration file
  if config = config_file(custom_location)
    options = YAML.load_file config

    if options
      options.each { |k, v| options[k] = Set.new v if v.is_a? Array }

      # notify if options[:quiet] and quiet is nil||false
      notify "[Notice] Using configuration in #{config}" unless (options[:quiet] || quiet)
      options
    else
      notify "[Notice] Empty configuration file: #{config}" unless quiet
      {}
    end
  else
    {}
  end
end

.load_pipeline_dependency(name) ⇒ Object 



295
296
297
298
299
300
301
302
303
304
305
 
# File 'lib/pipeline.rb', line 295

def self.load_pipeline_dependency name
  return if @loaded_dependencies.include? name

  begin
    require name
  rescue LoadError => e
    $stderr.puts e.message
    $stderr.puts "Please install the appropriate dependency."
    exit! -1
  end
end

.notify(message) ⇒ Object 



285
286
287
288
 
# File 'lib/pipeline.rb', line 285

def self.notify message
  $stderr.puts message #unless @debug
  $logfile.puts "[#{Time.now}] #{message}" if $logfile
end

.run(options) ⇒ Object

Run Pipeline.

Options:

* :config_file - configuration file
* :exit_on_warn - return false if warnings found, true otherwise. Not recommended for library use (default: false)
* :output_files - files for output
* :output_formats - formats for output (:to_s, :to_tabs, :to_csv, :to_html)
* :parallel_checks - run checks in parallel (default: true)
* :print_report - if no output file specified, print to stdout (default: false)
* :quiet - suppress most messages (default: true)



27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
 
# File 'lib/pipeline.rb', line 27

def self.run options
  options = set_options options

  @quiet = !!options[:quiet]
  @debug = !!options[:debug]

  if @quiet
    options[:report_progress] = false
  end

  unless options[:logfile].nil?
    if options[:logfile].is_a? File
      $logfile = options[:logfile]
    else
      $logfile = File.open(options[:logfile], 'a')
    end

    begin
      scan options
    ensure
      $logfile.close unless options[:logfile].is_a? File
    end
  end
end

.scan(options) ⇒ Object

Run a scan. Generally called from Pipeline.run instead of directly.



233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
 
# File 'lib/pipeline.rb', line 233

def self.scan options
  #Load scanner
  notify "Loading scanner..."

  begin
    require 'pipeline/scanner'
    require 'pipeline/tracker'
    require 'pipeline/mounters'
    require 'pipeline/filters'
    require 'pipeline/reporters'

  rescue LoadError => e
    $stderr.puts e.message
    raise NoPipelineError, "Cannot find lib/ directory or load the key pipeline."
  end

#    debug "API: #{options[:jira_api_url.to_s]}"
#    debug "Project: #{options[:jira_project.to_s]}"
#    debug "Cookie: #{options[:jira_cookie.to_s]}"

  add_external_tasks options

  tracker = Tracker.new options
  debug "Mounting ... #{options[:target]}"
  # Make the target accessible.
  target = Pipeline::Mounters.mount tracker

  #Start scanning
  scanner = Scanner.new
  notify "Processing target...#{options[:target]}"
  scanner.process target, tracker

  # Filter the results (Don't report anything that has been reported before)
  Pipeline::Filters.filter tracker

  # Generate Report
  notify "Generating report...#{options[:output_format]}"
  Pipeline::Reporters.run_report tracker

  tracker
end

.set_options(options) ⇒ Object

Sets up options for run, checks given application path



53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
 
# File 'lib/pipeline.rb', line 53

def self.set_options options
  if options.is_a? String
    options = { :target => options }
  end

  if options[:quiet] == :command_line
    command_line = true
    options.delete :quiet
  end

  options = default_options.merge(load_options(options[:config_file], options[:quiet])).merge(options)

  if options[:quiet].nil? and not command_line
    options[:quiet] = true
  end

  options[:output_format] = get_output_format options

  if options[:appname].nil?
    path = options[:target]
    options[:appname] = File.split(path).last
  end
  options
end

.warn(message) ⇒ Object 



280
281
282
283
 
# File 'lib/pipeline.rb', line 280

def self.warn message
  $stderr.puts message unless @quiet
  $logfile.puts "[#{Time.now}] #{message}" if $logfile
end