Class: GoodData::LCM2::SynchronizeUserFilters

Inherits:

BaseAction

• Object
• BaseAction
• GoodData::LCM2::SynchronizeUserFilters

Defined in:

lib/gooddata/lcm/actions/synchronize_user_filters.rb

Constant Summary

DESCRIPTION =

'Synchronizes User Permissions Between Projects'

PARAMS =

define_params(self) do
  description 'Client Used For Connecting To GD'
  param :gdc_gd_client, instance_of(Type::GdClientType), required: true

  description 'Input Source'
  param :input_source, instance_of(Type::HashType), required: true

  description 'Synchronization Mode (e.g. sync_one_project_based_on_pid)'
  param :sync_mode, instance_of(Type::StringType), required: false, default: 'sync_project'

  description 'Column That Contains Target Project IDs'
  param :multiple_projects_column, instance_of(Type::StringType), required: false

  description 'Filters Config'
  param :filters_config, instance_of(Type::HashType), required: true

  description 'Input Source Contains CSV Headers?'
  param :csv_headers, instance_of(Type::StringType), required: false

  description 'Restrict If Missing Values In Input Source'
  param :restrict_if_missing_all_values, instance_of(Type::StringType), required: false

  description 'Ignore Missing Values In Input Source'
  param :ignore_missing_values, instance_of(Type::StringType), required: false

  description 'Do Not Touch Filters That Are Not Mentioned'
  param :do_not_touch_filters_that_are_not_mentioned, instance_of(Type::StringType), required: false

  description 'Restricts synchronization to specified segments'
  param :segments_filter, array_of(instance_of(Type::StringType)), required: false

  description 'Organization Name'
  param :organization, instance_of(Type::StringType), required: false

  description 'Domain'
  param :domain, instance_of(Type::StringType), required: false

  description 'DataProduct to manage'
  param :data_product, instance_of(Type::GDDataProductType), required: true

  description 'Segments to manage'
  param :segments, array_of(instance_of(Type::SegmentType)), required: false

  description 'Logger'
  param :gdc_logger, instance_of(Type::GdLogger), required: true

  description 'GDC Project'
  param :gdc_project, instance_of(Type::GdProjectType), required: false

  description 'GDC Project Id'
  param :gdc_project_id, instance_of(Type::StringType), required: false

  description 'User brick users'
  param :users_brick_users, instance_of(Type::ObjectType), required: false, default: []

  description 'Makes the brick run without altering user filters'
  param :dry_run, instance_of(Type::StringType), required: false, default: false
end

MODES =

%w(
  sync_project
  sync_multiple_projects_based_on_pid
  sync_one_project_based_on_pid
  sync_one_project_based_on_custom_id
  sync_multiple_projects_based_on_custom_id
  sync_domain_client_workspaces
)

Constants inherited from BaseAction

BaseAction::FAILED_CLIENTS, BaseAction::FAILED_PROJECTS, BaseAction::FAILED_SEGMENTS, BaseAction::SYNC_FAILED_LIST

Constants included from Dsl::Dsl

Dsl::Dsl::DEFAULT_OPTS, Dsl::Dsl::TYPES

Class Method Summary

• .call(params) ⇒ Object
• .load_data(params, symbolized_config) ⇒ Object
• .sync_user_filters(project, filters, params, filters_config) ⇒ Object

Methods inherited from BaseAction

add_failed_client, add_failed_project, add_failed_segment, add_new_clients_to_project_client_mapping, check_params, collect_synced_status, continue_on_error, print_result, process_failed_project, process_failed_projects, sync_failed_client, sync_failed_project, sync_failed_segment, without_check

Methods included from Dsl::Dsl

#define_params, #define_type, #process

Class Method Details

.call(params) ⇒ Object

# File 'lib/gooddata/lcm/actions/synchronize_user_filters.rb', line 89

def call(params)
  client = params.gdc_gd_client
  domain_name = params.organization || params.domain
  fail "Either organisation or domain has to be specified in params" unless domain_name

  domain = client.domain(domain_name) if domain_name
  project = client.projects(params.gdc_project) || client.projects(params.gdc_project_id)
  fail "Either project or project_id has to be specified in params" unless project

  data_product = params.data_product

  config = params.filters_config
  fail 'User filters brick requires configuration how the filter should be setup. For this use the param "filters_config"' if config.blank?

  symbolized_config = GoodData::Helpers.deep_dup(config)
  symbolized_config = GoodData::Helpers.symbolize_keys(symbolized_config)
  symbolized_config[:labels] = symbolized_config[:labels].map { |l| GoodData::Helpers.symbolize_keys(l) }
  multiple_projects_column = params.multiple_projects_column

  mode = params.sync_mode
  unless MODES.include?(mode)
    fail "The parameter \"sync_mode\" has to have one of the values #{MODES.map(&:to_s).join(', ')} or has to be empty."
  end

  user_filters = load_data(params, symbolized_config)

  run_params = {
    restrict_if_missing_all_values: params.restrict_if_missing_all_values == 'true',
    ignore_missing_values: params.ignore_missing_values == 'true',
    do_not_touch_filters_that_are_not_mentioned: params.do_not_touch_filters_that_are_not_mentioned == 'true',
    domain: domain,
    dry_run: params[:dry_run].to_b,
    users_brick_input: params.users_brick_users
  }
  all_clients = domain.clients(:all, data_product).to_a
  GoodData.gd_logger.info("Synchronizing in mode=#{mode}, number_of_clients=#{all_clients.size}, data_rows=#{user_filters.size} ,")

  GoodData.logger.info("Synchronizing in mode \"#{mode}\"")
  results = []
  case mode
  when 'sync_project', 'sync_one_project_based_on_pid', 'sync_one_project_based_on_custom_id'
    if mode == 'sync_one_project_based_on_pid'
      filter = project.pid
    elsif mode == 'sync_one_project_based_on_custom_id'
      filter = UserBricksHelper.resolve_client_id(domain, project, params.data_product)
    end
    user_filters = user_filters.select { |f| f[:pid] == filter } if filter

    GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{project.pid}, data_rows=#{user_filters.size} ,")
    current_results = sync_user_filters(project, user_filters, run_params, symbolized_config)

    results.concat(current_results[:results]) unless current_results.nil? || current_results[:results].empty?
  when 'sync_multiple_projects_based_on_pid', 'sync_multiple_projects_based_on_custom_id'
    users_by_project = run_params[:users_brick_input].group_by { |u| u[:pid] }
    user_filters.group_by { |u| u[:pid] }.flat_map.pmap do |id, new_filters|
      users = users_by_project[id]
      fail "The #{multiple_projects_column} cannot be empty" if id.blank?

      if mode == 'sync_multiple_projects_based_on_custom_id'
        c = all_clients.detect { |specific_client| specific_client.id == id }
        current_project = c.project
      elsif mode == 'sync_multiple_projects_based_on_pid'
        current_project = client.projects(id)
      end

      GoodData.gd_logger.info("Synchronizing in mode=#{mode}, project_id=#{id}, data_rows=#{new_filters.size} ,")
      current_results = sync_user_filters(current_project, new_filters, run_params.merge(users_brick_input: users), symbolized_config)

      results.concat(current_results[:results]) unless current_results.nil? || current_results[:results].empty?
    end
  when 'sync_domain_client_workspaces'
    domain_clients = all_clients
    if params.segments
      segment_uris = params.segments.map(&:uri)
      domain_clients = domain_clients.select { |c| segment_uris.include?(c.segment_uri) }
    end

    working_client_ids = []
    semaphore = Mutex.new

    users_by_project = run_params[:users_brick_input].group_by { |u| u[:pid] }
    user_filters.group_by { |u| u[multiple_projects_column] }.flat_map.pmap do |client_id, new_filters|
      users = users_by_project[client_id]
      fail "Client id cannot be empty" if client_id.blank?

      c = all_clients.detect { |specific_client| specific_client.id == client_id }
      if c.nil?
        params.gdc_logger.warn "Client #{client_id} is not found"
        next
      end
      if params.segments && !segment_uris.include?(c.segment_uri)
        params.gdc_logger.warn "Client #{client_id} is outside segments_filter #{params.segments}"
        next
      end
      current_project = c.project
      fail "Client #{client_id} does not have project." unless current_project

      semaphore.synchronize do
        working_client_ids << client_id.to_s
      end

      GoodData.gd_logger.info("Synchronizing in mode=#{mode}, client_id=#{client_id}, data_rows=#{new_filters.size} ,")
      partial_results = sync_user_filters(current_project, new_filters, run_params.merge(users_brick_input: users), symbolized_config)
      results.concat(partial_results[:results]) unless partial_results.nil? || partial_results[:results].empty?
    end

    GoodData.gd_logger.info("Synchronizing in mode=#{mode}, working_client_ids=#{working_client_ids.join(', ')} ,") if working_client_ids.size < 50

    unless run_params[:do_not_touch_filters_that_are_not_mentioned]
      to_be_deleted_clients = UserBricksHelper.non_working_clients(domain_clients, working_client_ids)
      to_be_deleted_clients.peach do |c|
        begin
          current_project = c.project
          users = users_by_project[c.client_id]
          params.gdc_logger.info "Delete all filters in project #{current_project.pid} of client #{c.client_id}"

          GoodData.gd_logger.info("Delete all filters in project_id=#{current_project.pid}, client_id=#{c.client_id} ,")
          current_results = sync_user_filters(current_project, [], run_params.merge(users_brick_input: users), symbolized_config)

          results.concat(current_results[:results]) unless current_results.nil? || current_results[:results].empty?
        rescue StandardError => e
          params.gdc_logger.error "Failed to clear filters of  #{c.client_id} due to: #{e.inspect}"
        end
      end
    end
  end

  {
    results: results
  }
end

.load_data(params, symbolized_config) ⇒ Object

# File 'lib/gooddata/lcm/actions/synchronize_user_filters.rb', line 226

def load_data(params, symbolized_config)
  filters = []
  headers_in_options = params.csv_headers == 'false' || true
  csv_with_headers = if GoodData::UserFilterBuilder.row_based?(symbolized_config)
                       false
                     else
                       headers_in_options
                     end

  multiple_projects_column = params.multiple_projects_column
  data_source = GoodData::Helpers::DataSource.new(params.input_source)

  tmp = without_check(PARAMS, params) do
    File.open(data_source.realize(params), 'r:UTF-8')
  end

  begin
    GoodData.logger.info('Start reading data')
    row_count = 0
    CSV.foreach(tmp, :headers => csv_with_headers, :return_headers => false, :header_converters => :downcase, :encoding => 'utf-8') do |row|
      filters << row.to_hash.merge(pid: row[multiple_projects_column.downcase])
      row_count += 1
      GoodData.logger.info("Read #{row_count} rows") if (row_count % 50_000).zero?
    end
    GoodData.logger.info("Done reading data, total #{row_count} rows")
  rescue Exception => e # rubocop:disable RescueException
    fail "There was an error during loading data. Message: #{e.message}. Error: #{e}"
  end

  if filters.empty? && %w(sync_multiple_projects_based_on_pid sync_multiple_projects_based_on_custom_id).include?(params.sync_mode)
    fail 'The filter set can not be empty when using sync_multiple_projects_* mode as the filters contain \
          the project ids in which the permissions should be changed'
  end

  filters
end

.sync_user_filters(project, filters, params, filters_config) ⇒ Object

# File 'lib/gooddata/lcm/actions/synchronize_user_filters.rb', line 221

def sync_user_filters(project, filters, params, filters_config)
  # Do not change this line -> can cause paralelisation errors in jRuby viz TMA-963
  project.add_data_permissions(GoodData::UserFilterBuilder.get_filters(filters, filters_config), params)
end