Top Level Namespace

Defined Under Namespace

Modules: FC Classes: Autosync, AutosyncThread, BaseThread, CheckThread, CopyTaskThread, DeleteTaskThread, GlobalDaemonThread, Iostat, RunTasksThread, UpdateTasksThread

Instance Method Summary collapse

Instance Method Details

#apply_connection(options) ⇒ Object 



81
82
83
84
85
86
87
 
# File 'lib/manage/schema.rb', line 81

def apply_connection(options)
  return unless options
  FC::DB.close
  FC::DB.connect_by_config(options)
  FC::DB.connect.ping
  File.write(FC::DB.options_yml_path, Psych.dump(options))
end

#apply_entity(klass:, items:, update_only: false, key_field: :name) ⇒ Object 



102
103
104
105
106
107
108
109
110
111
112
113
114
 
# File 'lib/manage/schema.rb', line 102

def apply_entity(klass:, items:, update_only: false, key_field: :name)
  errors = []
  return errors unless items
  items.each do |item|
    begin
      result = klass.apply!(data: item, update_only: update_only, key_field: key_field)
      errors << result if result.is_a? String
    rescue => save_error
      errors << "Error while saving #{klass} \"#{item[key_field]}\": #{save_error}"
    end
  end
  errors
end

#apply_vars(vars:, update_only: false) ⇒ Object 



89
90
91
92
93
94
95
96
97
98
99
100
 
# File 'lib/manage/schema.rb', line 89

def apply_vars(vars:, update_only: false)
  errors = []
  existing_vars = FC::Var.get_all
  vars.each do |k, v|
    if !existing_vars.keys.include?(k) && update_only
      errors << "FC::Var \"#{k}\" not found"
      next
    end
    FC::Var.set(k, v)
  end
  errors
end

#autosync ⇒ Object 



78
79
80
81
82
83
84
85
86
87
88
89
90
 
# File 'lib/daemon.rb', line 78

def autosync
  if !$autosync_thread || !$autosync_thread.alive?
    intervals = FC::Var.get_autosync
    storage_interval = intervals[FC::Storage.curr_host] || intervals['all']
    return if storage_interval.zero? # do not run aytosync
    storages = $storages.select do |s|
      s.autosync_at.to_i + storage_interval < Time.now.to_i
    end
    return unless storages.any?
    $log.debug("spawn AutosyncThread for storages #{storages.map(&:name).join(', ')}")
    $autosync_thread = AutosyncThread.new(storages)
  end
end

#autosync_add ⇒ Object 



10
11
12
13
14
15
16
17
18
19
20
 
# File 'lib/manage/autosync.rb', line 10

def autosync_add
  hosts = ['all'] + all_hosts
  puts 'Set autosync interval'
  begin
    host = stdin_read_val("Host (default #{FC::Storage.curr_host})", true).strip
    host = FC::Storage.curr_host if host.empty?
    puts "Host can be one of: #{hosts.join(', ')}" unless hosts.index(host)
  end until hosts.index(host)
  interval = stdin_read_val('Autosync interval, seconds (0 - never, empty = all)', true)
  confirm_autosync_set(host, interval)
end

#autosync_change ⇒ Object 



22
23
24
25
26
27
28
29
30
 
# File 'lib/manage/autosync.rb', line 22

def autosync_change
  return if (host = find_host).to_s.empty?
  puts "Change autosync interval for host #{host}"
  interval = FC::Var.get_autosync[host]
  txt = interval.to_s.empty? ? 'default (=all)' : nil
  txt = interval.to_i.zero? ? 'never' : "#{interval}" unless txt
  interval = stdin_read_val("Autosync interval, seconds (now #{txt}, 0 - never, empty = all)", true)
  confirm_autosync_set(host, interval)
end

#autosync_list ⇒ Object 



4
5
6
7
8
 
# File 'lib/manage/autosync.rb', line 4

def autosync_list
  FC::Var.get_autosync.each do |name, val|
    puts name.to_s+(val.to_i > 0 ? " - every: #{val} seconds" : " - never")
  end
end

#colorize_string(str, color) ⇒ Object 



69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
 
# File 'lib/utils.rb', line 69

def colorize_string(str, color)
  return str unless color
  case color.to_s
  when 'red'
    color_code = 31
  when 'green'
    color_code = 32
  when 'yellow'
    color_code = 33
  when 'blue'
    color_code = 34
  when 'pink'
    color_code = 35
  else
    color_code = color.to_i
  end 
  "\e[#{color_code}m#{str}\e[0m"
end

#copy_rules_add ⇒ Object 



25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
 
# File 'lib/manage/copy_rules.rb', line 25

def copy_rules_add
  puts "Add copy rule"
  copy_storages = stdin_read_val('Copy storages')
  storages = FC::Storage.where.map(&:name)
  copy_storages = copy_storages.split(',').select{|s| storages.member?(s.strip)}.join(',').strip
  rule_str = stdin_read_val('Rule')
  
  begin
    rule = FC::CopyRule.new(:rule => rule_str, :copy_storages => copy_storages)
    rule.test
  rescue Exception => e
    puts "Error: #{e.message}"
    exit
  end
  puts %Q{\nRule
  Copy storages:  #{rule.copy_storages}
  Rule:           #{rule.rule}}
  s = Readline.readline("Continue? (y/n) ", false).strip.downcase
  puts ""
  if s == "y" || s == "yes"
    begin
      rule.save
    rescue Exception => e
      puts "Error: #{e.message}"
      exit
    end
    puts "ok"
  else
    puts "Canceled."
  end
end

#copy_rules_change ⇒ Object 



70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
 
# File 'lib/manage/copy_rules.rb', line 70

def copy_rules_change
  if rule = find_rule
    puts "Change rule #{rule.id}"
    copy_storages = stdin_read_val("Copy storages (now #{rule.copy_storages})", true)
    storages = FC::Storage.where.map(&:name)
    rule.copy_storages = copy_storages.split(',').select{|s| storages.member?(s.strip)}.join(',').strip unless copy_storages.empty?
    rule_str = stdin_read_val("Rule (now #{rule.rule})", true)
    rule.rule = rule_str unless rule_str.empty?
    
    puts %Q{\nRule
    Id:             #{rule.id}
    Copy storages:  #{rule.copy_storages}
    Rule:           #{rule.rule}}
    s = Readline.readline("Continue? (y/n) ", false).strip.downcase
    puts ""
    if s == "y" || s == "yes"
      begin
        rule.test
        rule.save
      rescue Exception => e
        puts "Error: #{e.message}"
        exit
      end
      puts "ok"
    else
      puts "Canceled."
    end
  end
end

#copy_rules_list ⇒ Object 



4
5
6
7
8
9
10
11
12
13
 
# File 'lib/manage/copy_rules.rb', line 4

def copy_rules_list
  rules = FC::CopyRule.where("1 ORDER BY id")
  if rules.size == 0
    puts "No rules."
  else
    rules.each do |rule|
      puts "##{rule.id}, copy storages: #{rule.copy_storages}, rule: #{rule.rule}"
    end
  end
end

#copy_rules_rm ⇒ Object 



57
58
59
60
61
62
63
64
65
66
67
68
 
# File 'lib/manage/copy_rules.rb', line 57

def copy_rules_rm
  if rule = find_rule
    s = Readline.readline("Continue? (y/n) ", false).strip.downcase
    puts ""
    if s == "y" || s == "yes"
      rule.delete
      puts "ok"
    else
      puts "Canceled."
    end
  end
end

#copy_rules_show ⇒ Object 



15
16
17
18
19
20
21
22
23
 
# File 'lib/manage/copy_rules.rb', line 15

def copy_rules_show
  if rule = find_rule
    #count = FC::DB.query("SELECT count(*) as cnt FROM #{FC::ItemStorage.table_name} WHERE storage_name='#{Mysql2::Client.escape(storage.name)}'").first['cnt']
    puts %Q{Rule
  Id:             #{rule.id}
  Copy storages:  #{rule.copy_storages}
  Rule:           #{rule.rule}}
  end
end

#copy_speed_add ⇒ Object 



10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
 
# File 'lib/manage/copy_speed.rb', line 10

def copy_speed_add
  hosts = ['all'] + all_hosts
  puts "Add copy speed limit"
  begin
    host = stdin_read_val("Host (default #{FC::Storage.curr_host})", true).strip
    host = FC::Storage.curr_host if host.empty?
    puts "Host can be one of: #{hosts.join(', ')}" unless hosts.index(host)
  end until hosts.index(host)
  limit = stdin_read_val("Speed limit, Mbit/s (default 0 - unlimit)", true).to_f
  puts %Q{\nCopy speed limit
  Host:         #{host}
  Speed limit:  #{limit > 0 ? limit : 'unlimit'}}
  s = Readline.readline("Continue? (y/n) ", false).strip.downcase
  puts ""
  if s == "y" || s == "yes"
    begin
      FC::Var.set_speed_limit(host, limit)
    rescue Exception => e
      puts "Error: #{e.message}"
      exit
    end
    puts "ok"
  else
    puts "Canceled."
  end
end

#copy_speed_change ⇒ Object 



37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
 
# File 'lib/manage/copy_speed.rb', line 37

def copy_speed_change
  if host = find_host
    puts "Change copy speed limit for host #{host}"
    curr_limit = FC::Var.get_speed_limits[host]
    limit = stdin_read_val("Speed limit, Mbit/s (now #{curr_limit ? curr_limit.to_s+', 0 to unlimit' : 'unlimit'})", true)
    puts limit.to_f
    puts limit == ''
    limit = limit == '' ? curr_limit : limit.to_f
    puts %Q{\nCopy speed limit
    Host:         #{host}
    Speed limit:  #{limit > 0 ? limit : 'unlimit'}}
    s = Readline.readline("Continue? (y/n) ", false).strip.downcase
    puts ""
    if s == "y" || s == "yes"
      begin
        FC::Var.set_speed_limit(host, limit)
      rescue Exception => e
        puts "Error: #{e.message}"
        exit
      end
      puts "ok"
    else
      puts "Canceled."
    end
  end
end

#copy_speed_list ⇒ Object 



4
5
6
7
8
 
# File 'lib/manage/copy_speed.rb', line 4

def copy_speed_list
  FC::Var.get_speed_limits.each do |name, val|
    puts name.to_s+(val ? " - limit: #{val}Mbit" : " - unlimit")
  end
end

#create_dump ⇒ Object 



62
63
64
65
66
67
68
69
70
 
# File 'lib/manage/schema.rb', line 62

def create_dump
  schema = { }
  schema[:storages] = FC::Storage.where.map(&:dump)
  schema[:policies] = FC::Policy.where.map(&:dump)
  schema[:copy_rules] = FC::CopyRule.where.map(&:dump)
  schema[:vars] = FC::Var.get_all.select { |k, _| k.is_a? Symbol }
  schema[:connection] = FC::DB.instance_variable_get('@options')
  schema
end

#create_or_update_schema(schema:, update:) ⇒ Object 



53
54
55
56
57
58
59
60
 
# File 'lib/manage/schema.rb', line 53

def create_or_update_schema(schema:, update:)
  errors = []
  errors << apply_entity(klass: FC::Storage, items: schema[:storages], update_only: update)
  errors << apply_entity(klass: FC::Policy, items: schema[:policies], update_only: update)
  errors << apply_entity(klass: FC::CopyRule, items: schema[:copy_rules], key_field: :rule, update_only: update)
  errors << apply_vars(vars: schema[:vars], update_only: update)
  errors.flatten
end

#error(msg, options = {}) ⇒ Object 



11
12
13
14
 
# File 'lib/daemon.rb', line 11

def error(msg, options = {})
  $log.error(msg)
  FC::Error.new(options.merge(:host => FC::Storage.curr_host, :message => msg)).save
end

#human_to_size(size) ⇒ Object 



40
41
42
43
44
45
46
47
48
 
# File 'lib/utils.rb', line 40

def human_to_size(size)
  r = /^(\d+(\.\d+)?)\s*(.*)/
  units = {'k' => 1024, 'm' => 1024*1024, 'g' => 1024*1024*1024, 't' => 1024*1024*1024*1024}
  return nil unless matches = size.to_s.match(r)
  unit = units[matches[3].to_s.strip.downcase[0]]
  result = matches[1].to_f
  result *= unit if unit
  result.to_i
end

#item_add ⇒ Object 



41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
 
# File 'lib/manage/item.rb', line 41

def item_add
  path = ARGV[2] || ''
  name = ARGV[3] || ''
  policy = FC::Policy.where('id = ?', ARGV[4]).first
  policy = FC::Policy.where('name = ?', ARGV[4]).first unless policy
  puts "Policy #{ARGV[4]} not found." unless policy
  
  if policy
    begin
      item = FC::Item.create_from_local(path, name, policy, :tag => 'fc-manage-add', :replace => true, :not_local => true)
      item_storage = item.get_item_storages.first
      storage = FC::Storage.where('name = ?', item_storage.storage_name).first
      puts "Saved as #{storage.name+':'+storage.path+item.name}"
    rescue Exception => e
      puts e.message
    end
  end
end

#item_add_local ⇒ Object 



60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
 
# File 'lib/manage/item.rb', line 60

def item_add_local
  storage = FC::Storage.where('name = ?', ARGV[2]).first
  puts "Storage #{ARGV[2]} not found." unless storage
  name = ARGV[3] || ''
  policy = FC::Policy.where('id = ?', ARGV[4]).first
  policy = FC::Policy.where('name = ?', ARGV[4]).first unless policy
  puts "Policy #{ARGV[4]} not found." unless policy
  tag = ARGV[5] || 'fc-manage-add-local'
  outer_id = ARGV[6]
  
  if policy && storage
    if name.index(storage.path) == 0
      path = name
      name = name.sub(storage.path, '/').gsub('//', '/')
    else
      path = (storage.path+name).gsub('//', '/')
    end
    begin
      item = FC::Item.create_from_local(path, name, policy, :tag => tag, :outer_id => outer_id, :replace => true)
      item_storage = item.get_item_storages.first
      storage = FC::Storage.where('name = ?', item_storage.storage_name).first
      puts "Saved as #{storage.name+':'+storage.path+item.name}"
    rescue Exception => e
      puts e.message
    end
  end
end

#item_info ⇒ Object 



1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
 
# File 'lib/manage/item.rb', line 1

def item_info
  name = ARGV[2] || ''
  name = name.gsub('_', '\\_').gsub('%', '\\%').gsub('?', '_').gsub('*', '%')
  count = FC::DB.query("SELECT count(*) as cnt FROM #{FC::Item.table_name} WHERE name like '#{name}'").first['cnt']
  puts "Find #{count} items:"
  if (count > 1)
    items = FC::DB.query("SELECT name FROM #{FC::Item.table_name} WHERE name like '#{name}' ORDER BY id DESC LIMIT 100")
    items.each{|r| puts r["name"]}
    puts "Last item:"
  end
  item = FC::DB.query("SELECT i.id, i.name, tag, outer_id, p.name as policy, size, status, time, i.copies FROM #{FC::Item.table_name} as i, #{FC::Policy.table_name} as p WHERE i.name like '#{name}' AND p.id=policy_id ORDER BY i.id DESC LIMIT 1").first
  if item
    item_storages = FC::DB.query("SELECT storage_name, status, time FROM #{FC::ItemStorage.table_name} WHERE item_id=#{item["id"]}")
    puts %Q{
    ID:               #{item["id"]}
    Outer id:         #{item["outer_id"]}
    Name:             #{item["name"]}
    Status:           #{item["status"]}
    Tag:              #{item["tag"]}
    Policy:           #{item["policy"]}
    Size:             #{size_to_human(item["size"])}
    Time:             #{Time.at(item["time"])}
    Copies:           #{item["copies"]}}
    if item_storages.size > 0
      puts "Item on storages:"
      item_storages.each do |r|
        s = "    #{r["storage_name"]}:"
        s << " "*[(22-s.length), 1].max
        s << case r["status"]
          when "ready" then colorize_string("ready", :green)
          when "error" then colorize_string("ready", :red)
          else r["status"]
        end
        s << " - #{Time.at(r["time"])}"
        puts s
      end
    end
  end
end

#item_rm ⇒ Object 



88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
 
# File 'lib/manage/item.rb', line 88

def item_rm
  name = ARGV[2] || ''
  item = FC::Item.where('name = ?', name).first
  if !item
    puts "Item #{name} not found."
  else
    s = Readline.readline('Immediate delete? (y/n) ', false).strip.downcase
    puts ''
    immediate_delete = s == 'y' || s == 'yes'
    s = Readline.readline('Delete? (y/n) ', false).strip.downcase
    puts ''
    if s == 'y' || s == 'yes'
      immediate_delete ? item.immediate_delete : item.mark_deleted
      puts 'ok'
    else
      puts 'Canceled.'
    end
  end
end

#load_schema_file ⇒ Object 



72
73
74
75
76
77
78
79
 
# File 'lib/manage/schema.rb', line 72

def load_schema_file
  schema = Psych.load(File.read(ARGV[2]))
  schema[:storages] ||= []
  schema[:policies] ||= []
  schema[:copy_rules] ||= []
  schema[:vars] ||= {}
  schema
end

#manual_sync(storage, dry_run) ⇒ Object 



232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
 
# File 'lib/manage/storages.rb', line 232

def manual_sync(storage, dry_run)
  syncer = Autosync.new(storage, dry_run)
  syncer.run
  puts "Deleted #{syncer.files_to_delete.size} files"
  puts "Deleted #{syncer.items_to_delete.size} items_storages"
  if (ARGV[3])
    File.open(ARGV[3], 'w') do |file|
      syncer.files_to_delete.each { |f| file.puts f }
    end
    puts "Save deleted files to #{ARGV[3]}"
  end

  if (ARGV[4])
    File.open(ARGV[4], 'w') do |file|
      syncer.items_to_delete.each { |item_storage_id| file.puts item_storage_id }
    end
    puts "Save deleted items_storages to #{ARGV[4]}"
  end
end

#option_parser_init(descriptions, text) ⇒ Object 



1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
 
# File 'lib/utils.rb', line 1

def option_parser_init(descriptions, text)
  options = {
    :__keys => {}
  }
  optparse = OptionParser.new do |opts|
    opts.banner = text
    opts.separator "Options:"
    
    descriptions.each_entry do |key, desc|
      options[key] = desc[:default]
      opts.on("-#{desc[:short]}", "--#{desc[:full]}#{desc[:no_val] ? '' : '='+desc[:full].upcase}", desc[:text]) do |s|
        options[:__keys][key] = s
        options[key] = s
      end
    end
    opts.on_tail("-?", "--help", "Show this message") do
      puts opts
      exit
    end
    opts.on_tail("-v", "--version", "Show version") do
      puts FC::VERSION
      exit
    end
  end
  optparse.parse!
  options['optparse'] = optparse
  options
end

#policies_add ⇒ Object 



25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
 
# File 'lib/manage/policies.rb', line 25

def policies_add
  puts "Add Policy"
  name = stdin_read_val('Name')
  create_storages = stdin_read_val('Create storages')
  copies = stdin_read_val('Copies').to_i
  delete_deferred_time = stdin_read_val('Delete deferred time').to_i
  
  storages = FC::Storage.where.map(&:name)
  create_storages = create_storages.split(',').select{|s| storages.member?(s.strip)}.join(',').strip
  
  begin
    policy = FC::Policy.new(
      :name => name,
      :create_storages => create_storages,
      :copies => copies,
      :delete_deferred_time => delete_deferred_time
    )
  rescue Exception => e
    puts "Error: #{e.message}"
    exit
  end
  puts %Q{\nPolicy
  Name:                 #{name}
  Create storages:      #{create_storages}
  Copies:               #{copies}
  Delete deferred time: #{delete_deferred_time}}
  s = Readline.readline("Continue? (y/n) ", false).strip.downcase
  puts ""
  if s == "y" || s == "yes"
    begin
      policy.save
    rescue Exception => e
      puts "Error: #{e.message}"
      exit
    end
    puts "ok"
  else
    puts "Canceled."
  end
end

#policies_change ⇒ Object 



79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
 
# File 'lib/manage/policies.rb', line 79

def policies_change
  if policy = find_policy
    puts "Change policy ##{policy.id} #{policy.name}"
    name = stdin_read_val("Name (now #{policy.name})", true)
    create_storages = stdin_read_val("Create storages (now #{policy.create_storages})", true)
    copies = stdin_read_val("Copies (now #{policy.copies})", true)
    delete_deferred_time = stdin_read_val("Delete deferred time (now #{policy.delete_deferred_time})", true)
    
    storages = FC::Storage.where.map(&:name)
    create_storages = create_storages.split(',').select{|s| storages.member?(s.strip)}.join(',').strip unless create_storages.empty?
        
    policy.name = name unless name.empty?
    policy.create_storages = create_storages unless create_storages.empty?
    policy.copies = copies.to_i unless copies.empty?
    policy.delete_deferred_time = delete_deferred_time.to_i unless delete_deferred_time.empty?
    
    puts %Q{\nStorage
    Name:                 #{policy.name}
    Create storages:      #{policy.create_storages}
    Copies:               #{policy.copies}
    Delete deferred time: #{policy.delete_deferred_time}}
    s = Readline.readline("Continue? (y/n) ", false).strip.downcase
    puts ""
    if s == "y" || s == "yes"
      begin
        policy.save
      rescue Exception => e
        puts "Error: #{e.message}"
        exit
      end
      puts "ok"
    else
      puts "Canceled."
    end
  end
end

#policies_list ⇒ Object 



1
2
3
4
5
6
7
8
9
10
 
# File 'lib/manage/policies.rb', line 1

def policies_list
  policies = FC::Policy.where
  if policies.size == 0
    puts "No storages."
  else
    policies.each do |policy|
      puts "##{policy.id} #{policy.name}, create storages: #{policy.create_storages}, copies: #{policy.copies}"
    end
  end
end

#policies_rm ⇒ Object 



66
67
68
69
70
71
72
73
74
75
76
77
 
# File 'lib/manage/policies.rb', line 66

def policies_rm
  if policy = find_policy
    s = Readline.readline("Continue? (y/n) ", false).strip.downcase
    puts ""
    if s == "y" || s == "yes"
      policy.delete
      puts "ok"
    else
      puts "Canceled."
    end
  end
end

#policies_show ⇒ Object 



12
13
14
15
16
17
18
19
20
21
22
23
 
# File 'lib/manage/policies.rb', line 12

def policies_show
  if policy = find_policy
    count = FC::DB.query("SELECT count(*) as cnt FROM #{FC::Item.table_name} WHERE policy_id = #{policy.id}").first['cnt']
    puts %Q{Policy
  ID:                   #{policy.id}
  Name:                 #{policy.name}
  Create storages:      #{policy.create_storages}
  Copies:               #{policy.copies}
  Delete deferred time: #{policy.delete_deferred_time}
  Items:                #{count}}
  end
end

#run_global_daemon ⇒ Object 



22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
 
# File 'lib/daemon.rb', line 22

def run_global_daemon
  $log.debug('Run global daemon check')
  timeout = FC::Var.get('daemon_global_wait_time', 120).to_f
  timeout = 0.3 if timeout < 0.3
  r = FC::DB.query("SELECT #{FC::DB.prefix}vars.*, UNIX_TIMESTAMP() as curr_time FROM #{FC::DB.prefix}vars WHERE name='global_daemon_host'").first
  if !r || r['curr_time'].to_i - r['time'].to_i > timeout
    $log.debug('Set global daemon host to current')
    FC::Var.set('global_daemon_host', FC::Storage.curr_host)
    sleep 1
    r = FC::DB.query("SELECT #{FC::DB.prefix}vars.*, UNIX_TIMESTAMP() as curr_time FROM #{FC::DB.prefix}vars WHERE name='global_daemon_host'").first
  end
  if r['val'] == FC::Storage.curr_host
    if !$global_daemon_thread || !$global_daemon_thread.alive?
      $log.debug("spawn GlobalDaemonThread")
      $global_daemon_thread = GlobalDaemonThread.new(timeout)
    end 
  elsif $global_daemon_thread && $global_daemon_thread.alive?
    $log.warn("Kill global daemon thread (new host = #{r['val']})")
    $global_daemon_thread.exit
  end
end

#run_tasks ⇒ Object 



71
72
73
74
75
76
 
# File 'lib/daemon.rb', line 71

def run_tasks
  if !$run_tasks_thread || !$run_tasks_thread.alive?
    $log.debug("spawn RunTasksThread")
    $run_tasks_thread = RunTasksThread.new
  end
end

#schema_apply ⇒ Object 



37
38
39
40
41
42
 
# File 'lib/manage/schema.rb', line 37

def schema_apply
  schema = load_schema_file
  apply_connection(schema[:connection])
  errors = create_or_update_schema(schema: schema, update: true)
  puts errors.join("\n") if errors && errors.any?
end

#schema_create ⇒ Object 



2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
 
# File 'lib/manage/schema.rb', line 2

def schema_create
  unless ARGV[2]
    puts 'config file not set'
    exit(1)
  end
  force = ARGV[3].to_s == 'force'
  schema = load_schema_file

  apply_connection(schema[:connection])
  # check existing objects
  unless force
    errors = ''
    schema[:storages].each do |cs|
      errors << "Storage \"#{cs[:name]}\" already exists\n" if FC::Storage.where('name = ?', cs[:name]).any?
    end
    schema[:policies].each do |pl|
      errors << "Policy \"#{pl[:name]}\" already exists\n" if FC::Policy.where('name = ?', pl[:name]).any?
    end
    schema[:copy_rules].each do |pl|
      errors << "CopyRule \"#{pl[:rule]}\" already exists\n" if FC::CopyRule.where('rule = ?', pl[:rule]).any?
    end
    existing_vars = FC::Var.get_all
    schema[:vars].keys.each do |v|
      errors << "Var \"#{v}\" already exists\n" if existing_vars.include?(v)
    end

    unless errors.empty?
      puts errors
      exit(1)
    end
  end
  errors = create_or_update_schema(schema: schema, update: false)
  puts errors.join("\n") if errors && errors.any?
end

#schema_dump ⇒ Object 



44
45
46
47
48
49
50
51
 
# File 'lib/manage/schema.rb', line 44

def schema_dump
  dump = create_dump
  if ARGV[2]
    File.write ARGV[2], Psych.dump(dump)
  else
    puts Psych.dump(dump)
  end
end

#show_current_host ⇒ Object 



1
2
3
 
# File 'lib/manage/show.rb', line 1

def show_current_host
 puts "Current host: #{FC::Storage.curr_host}"
end

#show_errors ⇒ Object 



14
15
16
17
18
19
20
21
22
23
24
 
# File 'lib/manage/show.rb', line 14

def show_errors
  count = ARGV[2] || 10
  errors = FC::Error.where("1 ORDER BY id desc LIMIT #{count.to_i}")
  if errors.size == 0
    puts "No errors."
  else
    errors.each do |error|
      puts "#{Time.at(error.time)} item_id: #{error.item_id}, item_storage_id: #{error.item_storage_id}, host: #{error.host}, message: #{error.message}"
    end
  end
end

#show_global_daemon ⇒ Object 



5
6
7
8
9
10
11
12
 
# File 'lib/manage/show.rb', line 5

def show_global_daemon
  r = FC::DB.query("SELECT #{FC::DB.prefix}vars.*, UNIX_TIMESTAMP() as curr_time FROM #{FC::DB.prefix}vars WHERE name='global_daemon_host'").first
  if r['val']
    puts "Global daemon run on #{r['val']}\nLast run #{r['curr_time']-r['time']} seconds ago."
  else
    puts "Global daemon is not runnning."
  end
end

#show_host_info ⇒ Object 



26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
 
# File 'lib/manage/show.rb', line 26

def show_host_info
  host = ARGV[2] || FC::Storage.curr_host
  storages = FC::Storage.where("host = ?", host)
  if storages.size == 0
    puts "No storages."
  else
    puts "Info for host #{host}"
    storages.each do |storage|
      counts = FC::DB.query("SELECT status, count(*) as cnt FROM #{FC::ItemStorage.table_name} WHERE storage_name='#{Mysql2::Client.escape(storage.name)}' GROUP BY status")
      str = "#{storage.name} #{size_to_human(storage.size)}/#{size_to_human(storage.size_limit)} "
      str += "#{storage.up? ? colorize_string('UP', :green) : colorize_string('DOWN', :red)}"
      str += " #{storage.check_time_delay} seconds ago" if storage.check_time
      str += "\n"
      counts.each do |r|
        str += "   Items storages #{r['status']}: #{r['cnt']}\n"
      end
      puts str
    end
  end
end

#show_items_info ⇒ Object 



47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
 
# File 'lib/manage/show.rb', line 47

def show_items_info
  puts "Items by status:"
  counts = FC::DB.query("SELECT status, count(*) as cnt FROM #{FC::Item.table_name} WHERE 1 GROUP BY status")
  counts.each do |r|
    puts "   #{r['status']}: #{r['cnt']}"
  end
  puts "Items storages by status:"
  counts = FC::DB.query("SELECT status, count(*) as cnt FROM #{FC::ItemStorage.table_name} WHERE 1 GROUP BY status")
  counts.each do |r|
    puts "   #{r['status']}: #{r['cnt']}"
  end
  count = FC::DB.query("SELECT count(*) as cnt FROM #{FC::Item.table_name} as i, #{FC::Policy.table_name} as p WHERE i.policy_id = p.id AND i.copies > 0 AND i.copies < p.copies AND i.status = 'ready'").first['cnt']
  puts "Items to copy: #{count}"
  count = FC::DB.query("SELECT count(*) as cnt FROM #{FC::Item.table_name} as i, #{FC::Policy.table_name} as p WHERE i.policy_id = p.id AND i.copies > p.copies AND i.status = 'ready'").first['cnt']
  puts "Items to delete: #{count}"
end

#size_to_human(size) ⇒ Object 



30
31
32
33
34
35
36
37
38
 
# File 'lib/utils.rb', line 30

def size_to_human(size)
  return "0" if size == 0
  units = %w{B KB MB GB TB}
  minus = size < 0
  size = -1 * size if minus
  e = (Math.log(size)/Math.log(1024)).floor
  s = "%.2f" % (size.to_f / 1024**e)
  (minus ? '-' : '')+s.sub(/\.?0*$/, units[e])
end

#stdin_read_val(name, can_empty = false, default_value = nil) ⇒ Object 



50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
 
# File 'lib/utils.rb', line 50

def stdin_read_val(name, can_empty = false, default_value = nil)
  while val = Readline.readline("#{name}: ", false).strip.downcase
    if val.empty? && !can_empty
      puts "Input non empty #{name}."
    else
      val = default_value if default_value && val.empty?
      if block_given?
        if err = yield(val) 
          puts err
        else 
          return val
        end
      else 
        return val
      end
    end
  end
end

#storages_add ⇒ Object 



41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
 
# File 'lib/manage/storages.rb', line 41

def storages_add
  host = FC::Storage.curr_host
  puts "Add storage to host #{host}"
  name = stdin_read_val('Name')
  dc = stdin_read_val('DC')
  path = stdin_read_val('Path')
  url = stdin_read_val('Url')
  url_weight = stdin_read_val('URL weight', true).to_i
  write_weight = stdin_read_val('Write weight', true).to_i
  is_auto_size = %(y yes).include?(stdin_read_val('Auto size (y/n)?').downcase)
  if is_auto_size
    auto_size = human_to_size stdin_read_val('Minimal free disk space') {|val| "Minimal free disk space not is valid size." unless human_to_size(val) || human_to_size(val) < 1 }
    size_limit = 0
  else
    auto_size = 0
    size_limit = human_to_size stdin_read_val('Size limit') {|val| "Size limit not is valid size." unless human_to_size(val)}
  end

  copy_storages = stdin_read_val('Copy storages', true)
  storages = FC::Storage.where.map(&:name)
  copy_storages = copy_storages.split(',').select{|s| storages.member?(s.strip)}.join(',').strip
  begin
    path = path +'/' unless path[-1] == '/'
    path = '/' + path unless path[0] == '/'
    storage = FC::Storage.new(:name => name, :dc => dc, :host => host, :path => path, :url => url, :size_limit => size_limit, :copy_storages => copy_storages, :url_weight => url_weight, :write_weight => write_weight, :auto_size => auto_size)
    print 'Calc current size.. '
    size = storage.file_size('', true)
    puts "ok"
  rescue Exception => e
    puts "Error: #{e.message}"
    exit
  end

  if storage.auto_size?
    storage.size = size
    size_limit = storage.get_real_size
  end
  free = size_limit - size
  puts %Q{\nStorage
  Name:         #{name}
  DC:           #{dc}
  Host:         #{host} 
  Path:         #{path} 
  Url:          #{url}
  URL weight:   #{url_weight}
  Write weight: #{write_weight}
  Size:         #{size_to_human size} (#{(size.to_f*100 / size_limit).to_i}%)
  Free:         #{size_to_human free} (#{(free.to_f*100 / size_limit).to_i}%)
  Size type:    #{storage.auto_size? ? "Auto (min #{ size_to_human(auto_size) })" : 'Static' }
  Size limit:   #{size_to_human size_limit}
  Copy storages #{copy_storages}}
  s = Readline.readline("Continue? (y/n) ", false).strip.downcase
  puts ""
  if s == "y" || s == "yes"
    storage.size = size
    begin
      storage.save
    rescue Exception => e
      puts "Error: #{e.message}"
      exit
    end
    puts "ok"
  else
    puts "Canceled."
  end
end

#storages_change ⇒ Object 



138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
 
# File 'lib/manage/storages.rb', line 138

def storages_change
  if storage = find_storage
    puts "Change storage #{storage.name}"
    dc = stdin_read_val("DC (now #{storage.dc})", true)
    host = stdin_read_val("Host (now #{storage.host})", true)
    path = stdin_read_val("Path (now #{storage.path})", true)
    url = stdin_read_val("Url (now #{storage.url})", true)
    url_weight = stdin_read_val("URL weight (now #{storage.url_weight})", true)
    write_weight = stdin_read_val("Write weight (now #{storage.write_weight})", true)
    is_auto_size = %(y yes).include?(stdin_read_val("Auto size (now #{storage.auto_size? ? 'yes' : 'no'})", true, storage.auto_size? ? 'yes' : 'no').downcase)
    if is_auto_size
      auto_size = human_to_size stdin_read_val("Minimal free disk space (now #{size_to_human(storage.auto_size)})", true, size_to_human(storage.auto_size)) {|val| "Minimal free disk space not is valid size." if !human_to_size(val) || human_to_size(val) < 1}
      size_limit = 0
    else
      auto_size = 0
      size_limit = stdin_read_val("Size (now #{size_to_human(storage.size_limit)})", true) {|val| "Size limit not is valid size." if !val.empty? && !human_to_size(val)}
    end
    copy_storages = stdin_read_val("Copy storages (now #{storage.copy_storages})", true)
    
    storage.dc = dc unless dc.empty?
    storage.host = host unless host.empty?
    if !path.empty? && path != storage.path
      path = path +'/' unless path[-1] == '/'
      path = '/' + path unless path[0] == '/'
      storage.path = path 
      print "Calc current size.. "
      storage.size = storage.file_size('', true)
      puts "ok"
    end
    storage.auto_size = auto_size
    size_limit = size_to_human(storage.get_real_size) if storage.auto_size?
    storage.url = url unless url.empty?
    storage.url_weight = url_weight.to_i unless url_weight.empty?
    storage.write_weight = write_weight.to_i unless write_weight.empty?
    storage.size_limit = human_to_size(size_limit) unless size_limit.empty?
    storages = FC::Storage.where.map(&:name)
    storage.copy_storages = copy_storages.split(',').select{|s| storages.member?(s.strip)}.join(',').strip unless copy_storages.empty?
    
    puts %Q{\nStorage
    Name:          #{storage.name}
    DC:            #{storage.dc}
    Host:          #{storage.host} 
    Path:          #{storage.path} 
    Url:           #{storage.url}
    URL weight:    #{storage.url_weight}
    Write weight:  #{storage.write_weight}
    Size:          #{size_to_human storage.size} (#{(storage.size_rate*100).to_i}%)
    Free:          #{size_to_human storage.free} (#{(storage.free_rate*100).to_i}%)
    Size type:     #{storage.auto_size? ? "Auto (Min #{size_to_human auto_size})" : 'Static' }
    Size limit:    #{size_to_human storage.size_limit }
    Copy storages: #{storage.copy_storages}}
    s = Readline.readline("Continue? (y/n) ", false).strip.downcase
    puts ""
    if s == "y" || s == "yes"
      begin
        storage.save
      rescue Exception => e
        puts "Error: #{e.message}"
        exit
      end
      puts "ok"
    else
      puts "Canceled."
    end
  end
end

#storages_check ⇒ Object 



50
51
52
53
54
55
56
57
58
59
60
61
62
 
# File 'lib/daemon.rb', line 50

def storages_check
  $log.debug('Run storages check')
  $check_threads.each do |storage_name, thread|
    if thread.alive?
      error "Storage #{storage_name} check timeout"
      thread.exit
    end
  end
  $storages.each do|storage| 
    $log.debug("spawn CheckThread for #{storage.name}")
    $check_threads[storage.name] = CheckThread.new(storage.name)
  end
end

#storages_list ⇒ Object 



4
5
6
7
8
9
10
11
12
13
14
15
16
17
 
# File 'lib/manage/storages.rb', line 4

def storages_list
  storages = FC::Storage.where("1 ORDER BY host")
  if storages.size == 0
    puts "No storages."
  else
    storages.each do |storage|
      str = " #{colorize_string(storage.dc, :blue)}\t#{colorize_string(storage.host, :yellow)} #{storage.name} #{size_to_human(storage.size)}/#{size_to_human(storage.size_limit)} "
      str += "#{(storage.free_rate*100).to_i}% free "
      str += "#{storage.up? ? colorize_string('UP', :green) : colorize_string('DOWN', :red)}"
      str += " #{storage.check_time_delay} seconds ago" if storage.check_time
      puts str
    end
  end
end

#storages_rm ⇒ Object 



108
109
110
111
112
113
114
115
116
117
118
119
 
# File 'lib/manage/storages.rb', line 108

def storages_rm
  if storage = find_storage
    s = Readline.readline("Continue? (y/n) ", false).strip.downcase
    puts ""
    if s == "y" || s == "yes"
      storage.delete
      puts "ok"
    else
      puts "Canceled."
    end
  end
end

#storages_show ⇒ Object 



19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
 
# File 'lib/manage/storages.rb', line 19

def storages_show
  if storage = find_storage
    count = FC::DB.query("SELECT count(*) as cnt FROM #{FC::ItemStorage.table_name} WHERE storage_name='#{Mysql2::Client.escape(storage.name)}'").first['cnt']
    puts %Q{Storage
  Name:           #{storage.name}
  Host:           #{storage.host} 
  DC:             #{storage.dc}
  Path:           #{storage.path} 
  Url:            #{storage.url}
  Url weight:     #{storage.url_weight}
  Write weight    #{storage.write_weight}
  Size:           #{size_to_human storage.size} (#{(storage.size_rate*100).to_i}%)
  Free:           #{size_to_human storage.free} (#{(storage.free_rate*100).to_i}%) 
  Size limit:     #{size_to_human storage.size_limit}
  Size type:      #{storage.auto_size? ? "Auto (min #{ size_to_human storage.auto_size })" : 'Static'}
  Copy storages:  #{storage.copy_storages}
  Check time:     #{storage.check_time ? "#{Time.at(storage.check_time)} (#{storage.check_time_delay} seconds ago)" : ''}
  Status:         #{storage.up? ? colorize_string('UP', :green) : colorize_string('DOWN', :red)}
  Items storages: #{count}}
  end
end

#storages_sync ⇒ Object 



215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
 
# File 'lib/manage/storages.rb', line 215

def storages_sync
  if storage = find_storage
    return puts "Storage #{storage.name} is not local." if storage.host != FC::Storage.curr_host
    puts "Synchronize (#{storage.name}) storage and file system (#{storage.path}).."
    s = Readline.readline('Continue? (y/n) ', false).strip.downcase
    puts ''
    if s == 'y' || s == 'yes'
      init_console_logger
      manual_sync(storage, false)
      s = Readline.readline('Update storage size? (y/n) ', false).strip.downcase
      storages_update_size if s == 'y' || s == 'yes'
    else
      puts "Canceled."
    end
  end
end

#storages_sync_info ⇒ Object 



205
206
207
208
209
210
211
212
213
 
# File 'lib/manage/storages.rb', line 205

def storages_sync_info
  if storage = find_storage
    return puts "Storage #{storage.name} is not local." if storage.host != FC::Storage.curr_host
    puts "Get synchronization info for (#{storage.name}) storage and file system (#{storage.path}).."
    init_console_logger
    manual_sync(storage, true)
    puts 'Done.'
  end
end

#storages_sync_info_old ⇒ Object 



252
253
254
255
256
257
258
259
 
# File 'lib/manage/storages.rb', line 252

def storages_sync_info_old
  if storage = find_storage
    return puts "Storage #{storage.name} is not local." if storage.host != FC::Storage.curr_host
    puts "Get synchronization info for (#{storage.name}) storage and file system (#{storage.path}).."
    make_storages_sync(storage, false)
    puts "Done."
  end
end

#storages_sync_old ⇒ Object 



261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
 
# File 'lib/manage/storages.rb', line 261

def storages_sync_old
  if storage = find_storage
    return puts "Storage #{storage.name} is not local." if storage.host != FC::Storage.curr_host
    puts "Synchronize (#{storage.name}) storage and file system (#{storage.path}).."
    s = Readline.readline("Continue? (y/n) ", false).strip.downcase
    puts ""
    if s == "y" || s == "yes"
      make_storages_sync(storage, true)
      puts "Synchronize done."
      FC::DB.connect
      storages_update_size
    else
      puts "Canceled."
    end
  end
end

#storages_update_size ⇒ Object 



121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
 
# File 'lib/manage/storages.rb', line 121

def storages_update_size
  if storage = find_storage
    FC::DB.close
    print "Calc current size.. "
    size = storage.file_size('', true)
    storage.size = size
    FC::DB.reconnect
    begin
      storage.save
    rescue Exception => e
      puts "Error: #{e.message}"
      exit
    end
    puts "ok"
  end
end

#update_storages ⇒ Object 



44
45
46
47
48
 
# File 'lib/daemon.rb', line 44

def update_storages
  $log.debug('Update storages')
  $all_storages = FC::Storage.where
  $storages = $all_storages.select{|s| s.host == FC::Storage.curr_host}
end

#update_tasks ⇒ Object 



64
65
66
67
68
69
 
# File 'lib/daemon.rb', line 64

def update_tasks
  if !$update_tasks_thread || !$update_tasks_thread.alive?
    $log.debug("spawn UpdateTasksThread")
    $update_tasks_thread = UpdateTasksThread.new
  end
end

#var_change ⇒ Object 



21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
 
# File 'lib/manage/var.rb', line 21

def var_change
  if var = find_var
    puts "Change var #{var['name']}"
    val = stdin_read_val("Value (now #{var['val']})")
        
    puts %Q{\nVar
    Name:         #{var['name']}
    Value:        #{val}}
    s = Readline.readline("Continue? (y/n) ", false).strip.downcase
    puts ""
    if s == "y" || s == "yes"
      begin
        FC::Var.set(var['name'], val)
      rescue Exception => e
        puts "Error: #{e.message}"
        exit
      end
      puts "ok"
    else
      puts "Canceled."
    end
  end
end

#var_list ⇒ Object 



1
2
3
4
5
6
7
8
9
10
 
# File 'lib/manage/var.rb', line 1

def var_list
  vars = FC::DB.query("SELECT * FROM #{FC::DB.prefix}vars WHERE descr IS NOT NULL")
  if vars.size == 0
    puts "No vars."
  else
    vars.each do |var|
      puts var['name']
    end
  end
end

#var_show ⇒ Object 



12
13
14
15
16
17
18
19
 
# File 'lib/manage/var.rb', line 12

def var_show
  if var = find_var
    puts %Q{Var
  Name:         #{var['name']}
  Value:        #{var['val']}
  Description:  #{var['descr']}}
  end
end