Module: GoodData::LCM2

Defined in:
lib/gooddata/lcm/lcm2.rb,
lib/gooddata/lcm/dsl/dsl.rb,
lib/gooddata/lcm/types/param.rb,
lib/gooddata/lcm/actions/help.rb,
lib/gooddata/lcm/dsl/type_dsl.rb,
lib/gooddata/lcm/dsl/params_dsl.rb,
lib/gooddata/lcm/types/base_type.rb,
lib/gooddata/lcm/types/class/class.rb,
lib/gooddata/lcm/types/scalar/bool.rb,
lib/gooddata/lcm/types/scalar/hash.rb,
lib/gooddata/lcm/types/special/any.rb,
lib/gooddata/lcm/types/special/enum.rb,
lib/gooddata/lcm/user_bricks_helper.rb,
lib/gooddata/lcm/actions/base_action.rb,
lib/gooddata/lcm/actions/hello_world.rb,
lib/gooddata/lcm/helpers/tags_helper.rb,
lib/gooddata/lcm/types/scalar/object.rb,
lib/gooddata/lcm/types/scalar/string.rb,
lib/gooddata/lcm/types/special/array.rb,
lib/gooddata/lcm/actions/collect_meta.rb,
lib/gooddata/lcm/helpers/check_helper.rb,
lib/gooddata/lcm/types/complex/tokens.rb,
lib/gooddata/lcm/types/scalar/integer.rb,
lib/gooddata/lcm/actions/purge_clients.rb,
lib/gooddata/lcm/types/class/gd_client.rb,
lib/gooddata/lcm/types/class/gd_logger.rb,
lib/gooddata/lcm/types/complex/complex.rb,
lib/gooddata/lcm/types/complex/segment.rb,
lib/gooddata/lcm/types/class/ads_client.rb,
lib/gooddata/lcm/types/class/gd_product.rb,
lib/gooddata/lcm/types/class/gd_project.rb,
lib/gooddata/lcm/types/class/smart_hash.rb,
lib/gooddata/lcm/actions/collect_clients.rb,
lib/gooddata/lcm/actions/segments_filter.rb,
lib/gooddata/lcm/actions/synchronize_cas.rb,
lib/gooddata/lcm/actions/synchronize_ldm.rb,
lib/gooddata/lcm/actions/collect_segments.rb,
lib/gooddata/lcm/actions/synchronize_meta.rb,
lib/gooddata/lcm/actions/apply_custom_maql.rb,
lib/gooddata/lcm/actions/associate_clients.rb,
lib/gooddata/lcm/actions/execute_schedules.rb,
lib/gooddata/lcm/actions/provision_clients.rb,
lib/gooddata/lcm/actions/synchronize_users.rb,
lib/gooddata/lcm/actions/collect_ca_metrics.rb,
lib/gooddata/lcm/actions/set_master_project.rb,
lib/gooddata/lcm/actions/collect_ldm_objects.rb,
lib/gooddata/lcm/actions/synchronize_clients.rb,
lib/gooddata/lcm/helpers/safe_failure_helper.rb,
lib/gooddata/lcm/types/complex/release_query.rb,
lib/gooddata/lcm/actions/collect_data_product.rb,
lib/gooddata/lcm/actions/ensure_release_table.rb,
lib/gooddata/lcm/actions/update_release_table.rb,
lib/gooddata/lcm/helpers/release_table_helper.rb,
lib/gooddata/lcm/actions/synchronize_processes.rb,
lib/gooddata/lcm/actions/synchronize_schedules.rb,
lib/gooddata/lcm/actions/update_metric_formats.rb,
lib/gooddata/lcm/actions/collect_tagged_objects.rb,
lib/gooddata/lcm/actions/create_segment_masters.rb,
lib/gooddata/lcm/actions/synchronize_ldm_layout.rb,
lib/gooddata/lcm/actions/collect_client_projects.rb,
lib/gooddata/lcm/actions/collect_segment_clients.rb,
lib/gooddata/lcm/actions/synchronize_label_types.rb,
lib/gooddata/lcm/actions/synchronize_tag_objects.rb,
lib/gooddata/lcm/actions/synchronize_user_groups.rb,
lib/gooddata/lcm/types/complex/update_preference.rb,
lib/gooddata/lcm/actions/synchronize_new_segments.rb,
lib/gooddata/lcm/actions/synchronize_user_filters.rb,
lib/gooddata/lcm/types/complex/users_brick_config.rb,
lib/gooddata/lcm/actions/collect_users_brick_users.rb,
lib/gooddata/lcm/actions/import_object_collections.rb,
lib/gooddata/lcm/actions/synchronize_color_palette.rb,
lib/gooddata/lcm/actions/migrate_gdc_date_dimension.rb,
lib/gooddata/lcm/types/complex/synchronization_info.rb,
lib/gooddata/lcm/types/special/enum/synchronize_ldm.rb,
lib/gooddata/lcm/actions/synchronize_etls_in_segment.rb,
lib/gooddata/lcm/actions/synchronize_dataset_mappings.rb,
lib/gooddata/lcm/actions/ensure_technical_users_domain.rb,
lib/gooddata/lcm/actions/ensure_technical_users_project.rb,
lib/gooddata/lcm/actions/collect_dynamic_schedule_params.rb,
lib/gooddata/lcm/actions/collect_projects_warning_status.rb,
lib/gooddata/lcm/actions/rename_existing_client_projects.rb,
lib/gooddata/lcm/actions/collect_multiple_projects_column.rb,
lib/gooddata/lcm/actions/synchronize_attribute_drillpaths.rb,
lib/gooddata/lcm/actions/initialize_continue_on_error_option.rb,
lib/gooddata/lcm/actions/synchronize_kd_dashboard_permission.rb,
lib/gooddata/lcm/actions/synchronize_pp_dashboard_permission.rb

Defined Under Namespace

Modules: Dsl, Type Classes: ApplyCustomMaql, AssociateClients, BaseAction, CollectClientProjects, CollectClients, CollectComputedAttributeMetrics, CollectDataProduct, CollectDymanicScheduleParams, CollectLdmObjects, CollectMeta, CollectMultipleProjectsColumn, CollectProjectsWarningStatus, CollectSegmentClients, CollectSegments, CollectTaggedObjects, CollectUsersBrickUsers, CreateSegmentMasters, EnsureReleaseTable, EnsureTechnicalUsersDomain, EnsureTechnicalUsersProject, ExecuteSchedules, HelloWorld, Help, Helpers, ImportObjectCollections, InitializeContinueOnErrorOption, MigrateGdcDateDimension, ProvisionClients, PurgeClients, RenameExistingClientProjects, SegmentsFilter, SetMasterProject, SmartHash, SynchronizeAttributeDrillpath, SynchronizeClients, SynchronizeColorPalette, SynchronizeComputedAttributes, SynchronizeDataSetMapping, SynchronizeETLsInSegment, SynchronizeKDDashboardPermissions, SynchronizeLabelTypes, SynchronizeLdm, SynchronizeLdmLayout, SynchronizeMeta, SynchronizeNewSegments, SynchronizePPDashboardPermissions, SynchronizeProcesses, SynchronizeSchedules, SynchronizeTagObjects, SynchronizeUserFilters, SynchronizeUserGroups, SynchronizeUsers, UpdateMetricFormats, UpdateReleaseTable, UserBricksHelper

Constant Summary collapse

MODES = 
{

  hello: [
    HelloWorld
  ],

  help: [
    Help
  ],

  release: [
    EnsureReleaseTable,
    CollectDataProduct,
    SegmentsFilter,
    CreateSegmentMasters,
    EnsureTechnicalUsersDomain,
    EnsureTechnicalUsersProject,
    SynchronizeLdm,
    CollectLdmObjects,
    CollectMeta,
    CollectTaggedObjects,
    CollectComputedAttributeMetrics,
    ImportObjectCollections,
    SynchronizeComputedAttributes,
    SynchronizeDataSetMapping,
    SynchronizeLdmLayout,
    SynchronizeProcesses,
    SynchronizeSchedules,
    SynchronizeColorPalette,
    SynchronizeUserGroups,
    SynchronizePPDashboardPermissions,
    SynchronizeKDDashboardPermissions,
    SynchronizeNewSegments,
    UpdateReleaseTable
  ],

  release_set_master_project: [
    EnsureReleaseTable,
    CollectDataProduct,
    SegmentsFilter,
    SetMasterProject,
    UpdateReleaseTable
  ],

  provision: [
    EnsureReleaseTable,
    CollectDataProduct,
    CollectSegments,
    CollectClientProjects,
    PurgeClients,
    CollectClients,
    AssociateClients,
    RenameExistingClientProjects,
    InitializeContinueOnErrorOption,
    ProvisionClients,
    UpdateMetricFormats,
    EnsureTechnicalUsersDomain,
    EnsureTechnicalUsersProject,
    CollectDymanicScheduleParams,
    SynchronizeDataSetMapping,
    SynchronizeLdmLayout,
    SynchronizeUserGroups,
    SynchronizePPDashboardPermissions,
    SynchronizeKDDashboardPermissions,
    SynchronizeETLsInSegment,
    CollectProjectsWarningStatus
  ],

  rollout: [
    EnsureReleaseTable,
    CollectDataProduct,
    CollectSegments,
    CollectSegmentClients,
    EnsureTechnicalUsersDomain,
    EnsureTechnicalUsersProject,
    InitializeContinueOnErrorOption,
    SynchronizeLdm,
    SynchronizeDataSetMapping,
    SynchronizeLdmLayout,
    MigrateGdcDateDimension,
    SynchronizeClients,
    SynchronizeUserGroups,
    SynchronizePPDashboardPermissions,
    SynchronizeKDDashboardPermissions,
    UpdateMetricFormats,
    SynchronizeComputedAttributes,
    CollectDymanicScheduleParams,
    SynchronizeETLsInSegment,
    CollectProjectsWarningStatus
  ],

  users: [
    CollectDataProduct,
    CollectMultipleProjectsColumn,
    CollectSegments,
    SynchronizeUsers
  ],

  user_filters: [
    CollectDataProduct,
    CollectMultipleProjectsColumn,
    CollectUsersBrickUsers,
    CollectSegments,
    SynchronizeUserFilters
  ],

  schedules_execution: [
    ExecuteSchedules
  ]

}
MODE_NAMES = 
MODES.keys

Class Method Summary collapse

Class Method Details

.check_unused_params(actions, params) ⇒ Object 



456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
 
# File 'lib/gooddata/lcm/lcm2.rb', line 456

def check_unused_params(actions, params)
  default_params = [
    :client_gdc_hostname,
    :client_gdc_protocol,
    :fail_early,
    :gdc_logger,
    :gdc_password,
    :gdc_username,
    :strict
  ]

  action_params = actions.map do |action|
    action.const_get(:PARAMS).keys.map(&:downcase)
  end

  action_params.flatten!.uniq!

  param_names = params.keys.map(&:downcase)

  unused_params = param_names - (action_params + default_params)

  if unused_params.any?
    GoodData.logger.warn("Following params are not used by any action: #{JSON.pretty_generate(unused_params)}")

    rows = []
    actions.each do |action|
      action_params = action.const_get(:PARAMS)
      action_params.each do |_k, v|
        rows << [action.short_name, v[:name], v[:description], v[:type].class.short_name]
      end
    end

    table = Terminal::Table.new :headings => ['Action', 'Parameter', 'Description', 'Parameter Type'], :rows => rows
    GoodData.logger.info("\n#{table}")
  end
end

.convert_params(params) ⇒ Object 



191
192
193
194
195
196
197
198
199
200
201
 
# File 'lib/gooddata/lcm/lcm2.rb', line 191

def convert_params(params)
  # Symbolize all keys
  GoodData::Helpers.symbolize_keys!(params)
  params.keys.each do |k|
    params[k.downcase] = params[k]
  end
  params.reject! do |k, _|
    k.downcase != k
  end
  convert_to_smart_hash(params)
end

.convert_to_smart_hash(params) ⇒ Object 



203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
 
# File 'lib/gooddata/lcm/lcm2.rb', line 203

def convert_to_smart_hash(params)
  if params.is_a?(Hash)
    res = SmartHash.new
    params.each_pair do |k, v|
      if v.is_a?(Hash) || v.is_a?(Array)
        res[k] = convert_to_smart_hash(v)
      else
        res[k] = v
      end
    end
    res
  elsif params.is_a?(Array)
    params.map do |item|
      convert_to_smart_hash(item)
    end
  else
    params
  end
end

.get_mode_actions(mode) ⇒ Object 



223
224
225
226
227
228
229
230
231
 
# File 'lib/gooddata/lcm/lcm2.rb', line 223

def get_mode_actions(mode)
  mode = mode.to_sym
  actions = MODES[mode]
  if mode == :generic_lifecycle
    []
  else
    actions || fail("Invalid mode specified '#{mode}', supported modes are: '#{MODE_NAMES.join(', ')}'")
  end
end

.perform(mode, params = {}) ⇒ Object 



297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
 
# File 'lib/gooddata/lcm/lcm2.rb', line 297

def perform(mode, params = {})
  # Default setting for $pmap_default_thread_count = 20 in gooddata.rb file. We are going to decrease default
  # number of threads count to 10 for LCM bricks only
  $pmap_default_thread_count = 10 # rubocop:disable GlobalVars

  params = convert_params(params)

  GoodData.gd_logger.brick = mode

  final_mode = if params.set_master_project && mode == 'release'
                 'release_set_master_project'
               else
                 mode
               end

  # Get actions for mode specified
  actions = get_mode_actions(final_mode)

  if params.actions
    actions = params.actions.map do |action|
      "GoodData::LCM2::#{action}".split('::').inject(Object) do |o, c|
        begin
          o.const_get(c)
        rescue NameError
          fail NameError, "Cannot find action 'GoodData::LCM2::#{action}'"
        end
      end
    end
  end

  # TODO: Check all action params first

  fail_early = if params.key?(:fail_early)
                 params.fail_early.to_b
               else
                 true
               end

  strict_mode = if params.key?(:strict)
                  params.strict.to_b
                else
                  true
                end

  skip_actions = (params.skip_actions || [])
  actions = actions.reject do |action|
    skip_actions.include?(action.name.split('::').last)
  end

  sync_mode = params.fetch(:sync_mode, nil)
  if mode == 'users' && %w[add_to_organization remove_from_organization].include?(sync_mode)
    actions = actions.reject do |action|
      %w[CollectDataProduct CollectSegments].include?(action.name.split('::').last)
    end
  end
  check_unused_params(actions, params)
  print_action_names(mode, actions)

  # Run actions
  errors = []
  results = []
  actions.each do |action|
    GoodData.logger.info("\n")
    # Invoke action
    begin
      out = run_action action, params
    rescue Exception => e # rubocop:disable Style/RescueException
      errors << {
        action: action,
        err: e,
        backtrace: e.backtrace
      }
      break if fail_early
      results << nil
    end

    # in case fail_early = false, we need to execute another action
    next unless out

    # Handle output results and params
    res = out.is_a?(Array) ? out : out[:results]
    out_params = out.is_a?(Hash) ? out[:params] || {} : {}
    new_params = convert_to_smart_hash(out_params)

    # Merge with new params
    params.merge!(new_params)

    # Print action result
    print_action_result(action, res) if action.send(:print_result, params)

    # Store result for final summary
    results << res
  end

  brick_results = {}
  actions.each_with_index do |action, index|
    brick_results[action.short_name] = results[index]
  end

  result = {
    actions: actions.map(&:short_name),
    results: brick_results,
    params: params,
    success: errors.empty?
  }

  if errors.any?
    error_message = JSON.pretty_generate(errors)
    if strict_mode
      raise GoodData::LcmExecutionError.new(errors[0][:err], error_message)
    else
      GoodData.logger.error(error_message)
    end
  end

  process_sync_failed_projects(params) if GoodData::LCM2::Helpers.collect_synced_status(params) && strict_mode

  result
end

.print_action_names(mode, actions) ⇒ Object 



233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
 
# File 'lib/gooddata/lcm/lcm2.rb', line 233

def print_action_names(mode, actions)
  title = "Actions to be performed for mode '#{mode}'"

  headings = %w(# NAME DESCRIPTION)

  rows = []
  actions.each_with_index do |action, index|
    rows << [index, action.short_name, action.const_defined?(:DESCRIPTION) && action.const_get(:DESCRIPTION)]
  end

  table = Terminal::Table.new :title => title, :headings => headings do |t|
    rows.each_with_index do |row, index|
      t << row
      t.add_separator if index < rows.length - 1
    end
  end
  GoodData.logger.info("\n#{table}")
end

.print_action_result(action, messages) ⇒ Object 



252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
 
# File 'lib/gooddata/lcm/lcm2.rb', line 252

def print_action_result(action, messages)
  title = "Result of #{action.short_name}"

  keys = if action.const_defined?('RESULT_HEADER')
           action.const_get('RESULT_HEADER')
         else
           GoodData.logger.warn("Action #{action.name} does not have RESULT_HEADERS, inferring headers from results.")
           (messages.first && messages.first.keys) || []
         end

  headings = keys.map(&:upcase)

  rows = messages && messages.map do |message|
    unless message
      GoodData.logger.warn("Found an empty message in the results of the #{action.name} action")
      next
    end
    row = []
    keys.each do |heading|
      row << message[heading]
    end
    row
  end

  rows ||= []
  rows.compact!

  table = Terminal::Table.new :title => title, :headings => headings do |t|
    rows.each_with_index do |row, index|
      t << (row || [])
      t.add_separator if index < rows.length - 1
    end
  end

  GoodData.logger.info("\n#{table}")
end

.print_actions_result(actions, results) ⇒ Object 



289
290
291
292
293
294
295
 
# File 'lib/gooddata/lcm/lcm2.rb', line 289

def print_actions_result(actions, results)
  actions.each_with_index do |action, index|
    print_action_result(action, results[index])
    GoodData.logger.info
  end
  nil
end

.process_sync_failed_projects(params) ⇒ Object 



417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
 
# File 'lib/gooddata/lcm/lcm2.rb', line 417

def process_sync_failed_projects(params)
  sync_failed_list = params[:sync_failed_list]
  sync_project_list = sync_failed_list[:project_client_mappings]
  sync_failed_project_list = sync_failed_list[:failed_detailed_projects]

  if sync_project_list && sync_failed_project_list && sync_project_list.size.positive? && sync_failed_project_list.size.positive?
    failed_project = sync_failed_project_list[0]
    summary_message = "Existing errors during execution. See log for details"
    error_message = failed_project[:message]
    if sync_project_list.size == sync_failed_project_list.size
      raise GoodData::LcmExecutionError.new(summary_message, error_message)
    else
      raise GoodData::LcmExecutionWarning.new(summary_message, error_message)
    end
  end
end

.run_action(action, params) ⇒ Object 



434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
 
# File 'lib/gooddata/lcm/lcm2.rb', line 434

def run_action(action, params)
  begin
    GoodData.gd_logger.start_action action, GoodData.gd_logger
    GoodData.gd_logger.info("Running #{action.name} action ...")
    params.clear_filters
    # Check if all required parameters were passed
    BaseAction.check_params(action.const_get('PARAMS'), params)
    params.setup_filters(action.const_get('PARAMS'))
    out = action.send(:call, params)
  rescue Exception => e # rubocop:disable Style/RescueException
    # Log to splunk
    GoodData.gd_logger.error("action=#{action} status=failed message=#{e} exception=#{e.backtrace}")
    # Log to execution log
    GoodData.logger.error("Execution #{action} failed. Error: #{e}. Detail:#{e.backtrace}")
    raise e
  ensure
    params.clear_filters
    GoodData.gd_logger.end_action GoodData.gd_logger
  end
  out
end