Module: TreasureData::Command

Defined in:
lib/td/command/db.rb,
lib/td/command/job.rb,
lib/td/command/aggr.rb,
lib/td/command/help.rb,
lib/td/command/list.rb,
lib/td/command/query.rb,
lib/td/command/sched.rb,
lib/td/command/table.rb,
lib/td/command/apikey.rb,
lib/td/command/common.rb,
lib/td/command/import.rb,
lib/td/command/result.rb,
lib/td/command/runner.rb,
lib/td/command/schema.rb,
lib/td/command/server.rb,
lib/td/command/status.rb,
lib/td/command/account.rb

Defined Under Namespace

Modules: List Classes: JsonParser, MessagePackParser, Runner, TextParser

Constant Summary collapse

IMPORT_TEMPLATES = 
{
  'apache' => [
                /^([^ ]*) [^ ]* ([^ ]*) \[([^\]]*)\] "(\S+)(?: +([^ ]*) +\S*)?" ([^ ]*) ([^ ]*)(?: "([^\"]*)" "([^\"]*)")?$/,
                ['host', 'user', 'time', 'method', 'path', 'code', 'size', 'referer', 'agent'],
                "%d/%b/%Y:%H:%M:%S %z"],
  'syslog' => [
                /^([^ ]* [^ ]* [^ ]*) ([^ ]*) ([a-zA-Z0-9_\/\.\-]*)(?:\[([0-9]+)\])?[^\:]*\: *(.*)$/,
                ['time', 'host', 'ident', 'pid', 'message'],
                "%b %d %H:%M:%S"],
}

Instance Method Summary collapse

Instance Method Details

#account(op) ⇒ Object 



5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
 
# File 'lib/td/command/account.rb', line 5

def (op)
  op.banner << "\noptions:\n"

  force = false
  op.on('-f', '--force', 'overwrite current account setting', TrueClass) {|b|
    force = true
  }

  user_name = op.cmd_parse

  conf = nil
  begin
    conf = Config.read
  rescue ConfigError
  end
  if conf && conf['account.apikey']
    unless force
      if conf['account.user']
        $stderr.puts "Account is already configured with '#{conf['account.user']}' account."
      else
        $stderr.puts "Account is already configured."
      end
      $stderr.puts "Add '-f' option to overwrite."
      exit 0
    end
  end

  unless user_name
    begin
      print "User name: "
      line = STDIN.gets || ""
      user_name = line.strip
    rescue Interrupt
      $stderr.puts "\ncanceled."
      exit 1
    end
  end

  if user_name.empty?
    $stderr.puts "canceled."
    exit 0
  end

  client = nil

  3.times do
    begin
      system "stty -echo"  # TODO termios
      print "Password: "
      password = STDIN.gets || ""
      password = password[0..-2]  # strip \n
    rescue Interrupt
      $stderr.print "\ncanceled."
      exit 1
    ensure
      system "stty echo"   # TODO termios
      print "\n"
    end

    if password.empty?
      $stderr.puts "canceled."
      exit 0
    end

    begin
      # enalbe SSL for the authentication
      client = Client.authenticate(user_name, password, :ssl=>true)
    rescue TreasureData::AuthError
      $stderr.puts "User name or password mismatched."
    end

    break if client
  end
  return unless client

  $stderr.puts "Authenticated successfully."

  conf ||= Config.new
  conf["account.user"] = user_name
  conf["account.apikey"] = client.apikey
  conf.save

  $stderr.puts "Use '#{$prog} db:create <db_name>' to create a database."
end

#aggr_add_attr(op) ⇒ Object 



137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
 
# File 'lib/td/command/aggr.rb', line 137

def aggr_add_attr(op)
  comment = nil

  op.on('-m', '--comment COMMENT', 'comment of this entry') {|s|
    comment = s
  }

  name, db_name, table_name, entry_name, method_name, *parameters = op.cmd_parse

  params = {}
  parameters.each {|pa|
    k, v = pa.split('=')
    params[k] = v
  }

  client = get_client

  get_table(client, db_name, table_name)

  begin
    client.create_aggregation_attr_entry(name, entry_name, comment, db_name, table_name, method_name, params)
  rescue NotFoundError
    cmd_debug_error $!
    $stderr.puts "Aggregation schema '#{name}' does not exist."
    $stderr.puts "Use '#{$prog} aggr:create #{name}' to create the aggregation schema."
    exit 1
  rescue AlreadyExistsError
    $stderr.puts "Aggregation attribute entry '#{entry_name}' already exists."
    exit 1
  end

  $stderr.puts "Aggregation attribute entry '#{entry_name}' is created."
end

#aggr_add_log(op) ⇒ Object 



99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
 
# File 'lib/td/command/aggr.rb', line 99

def aggr_add_log(op)
  comment = nil
  value_key = nil
  count_key = nil

  op.on('-m', '--comment COMMENT', 'comment of this entry') {|s|
    comment = s
  }
  op.on('-v', '--value KEY_NAME', 'key name of value field') {|s|
    value_key = s
  }
  op.on('-c', '--count KEY_NAME', 'key name of count field') {|s|
    count_key = s
  }

  name, db_name, table_name, entry_name, o1_key, o2_key, o3_key = op.cmd_parse

  okeys = [o1_key, o2_key, o3_key].compact

  client = get_client

  get_table(client, db_name, table_name)

  begin
    client.create_aggregation_log_entry(name, entry_name, comment, db_name, table_name, okeys, value_key, count_key)
  rescue NotFoundError
    cmd_debug_error $!
    $stderr.puts "Aggregation schema '#{name}' does not exist."
    $stderr.puts "Use '#{$prog} aggr:create #{name}' to create the aggregation schema."
    exit 1
  rescue AlreadyExistsError
    $stderr.puts "Aggregation log entry '#{entry_name}' already exists."
    exit 1
  end

  $stderr.puts "Aggregation log entry '#{entry_name}' is created."
end

#aggr_create(op) ⇒ Object 



73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
 
# File 'lib/td/command/aggr.rb', line 73

def aggr_create(op)
  name, relation_key = op.cmd_parse

  client = get_client

  begin
    client.create_aggregation_schema(name, relation_key)
  rescue AlreadyExistsError
    cmd_debug_error $!
    $stderr.puts "Aggregation '#{name}' already exists."
    exit 1
  end

  $stderr.puts "Aggregation schema '#{name}' is created."
end

#aggr_del_attr(op) ⇒ Object 



186
187
188
189
190
191
192
193
194
195
196
197
198
199
 
# File 'lib/td/command/aggr.rb', line 186

def aggr_del_attr(op)
  name, entry_name = op.cmd_parse

  client = get_client

  begin
    client.delete_aggregation_attr_entry(name, entry_name)
  rescue NotFoundError
    $stderr.puts "Aggregation log entry '#{entry_name}' does not exist."
    exit 1
  end

  $stderr.puts "Aggregation log entry '#{entry_name}' is deleted."
end

#aggr_del_log(op) ⇒ Object 



171
172
173
174
175
176
177
178
179
180
181
182
183
184
 
# File 'lib/td/command/aggr.rb', line 171

def aggr_del_log(op)
  name, entry_name = op.cmd_parse

  client = get_client

  begin
    client.delete_aggregation_log_entry(name, entry_name)
  rescue NotFoundError
    $stderr.puts "Aggregation log entry '#{entry_name}' does not exist."
    exit 1
  end

  $stderr.puts "Aggregation log entry '#{entry_name}' is deleted."
end

#aggr_delete(op) ⇒ Object 



89
90
91
92
93
94
95
96
97
 
# File 'lib/td/command/aggr.rb', line 89

def aggr_delete(op)
  name = op.cmd_parse

  client = get_client

  client.delete_aggregation_schema(name)

  $stderr.puts "Aggregation schema '#{name}' is deleted."
end

#aggr_list(op) ⇒ Object 



5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
 
# File 'lib/td/command/aggr.rb', line 5

def aggr_list(op)
  op.cmd_parse

  client = get_client

  ass = client.aggregation_schemas

  rows = []
  ass.each {|as|
    rows << {:Name=>as.name, :Relation=>as.relation_key}
  }

  puts cmd_render_table(rows, :fields => [:Name, :Relation])

  if rows.empty?
    $stderr.puts "There are no aggregation schemas."
    $stderr.puts "Use '#{$prog} aggr:create <name>' to create a aggregation schema."
  end
end

#aggr_show(op) ⇒ Object 



25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
 
# File 'lib/td/command/aggr.rb', line 25

def aggr_show(op)
  name = op.cmd_parse

  client = get_client

  begin
    as = client.aggregation_schema(name)
  rescue
    cmd_debug_error $!
    $stderr.puts "Aggregation '#{name}' does not exist."
    exit 1
  end

  log_rows = []
  as.logs.each {|las|
    log_rows << {
      :Table=>las.table.identifier,
      :Name=>las.name,
      :o1_key=>las.okeys[0].to_s,
      :o2_key=>las.okeys[1].to_s,
      :o3_key=>las.okeys[2].to_s,
      :value_key=>las.value_key.to_s,
      :count_key=>las.count_key.to_s,
      :Comment=>las.comment.to_s
    }
  }

  attr_rows = []
  as.attributes.each {|aas|
    params = aas.parameters.to_a.map {|k,v| "#{k}=#{v}" }.join(' ')
    attr_rows << {
      :Table=>aas.table.identifier,
      :Name=>aas.name,
      :Method=>aas.method_name,
      :Parameters=>params,
      :Comment=>aas.comment.to_s,
    }
  }

  puts "Log entries:"
  puts cmd_render_table(log_rows, :fields => [:Table, :Name, :o1_key, :o2_key, :o3_key, :value_key, :count_key, :Comment], :max_width=>400)

  puts ''

  puts "Attribute entries:"
  puts cmd_render_table(attr_rows, :fields => [:Table, :Name, :Method, :Parameters, :Comment], :max_width=>400)
end

#apikey_set(op) ⇒ Object 



21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
 
# File 'lib/td/command/apikey.rb', line 21

def apikey_set(op)
  op.banner << "\noptions:\n"

  force = false
  op.on('-f', '--force', 'overwrite current account setting', TrueClass) {|b|
    force = true
  }

  apikey = op.cmd_parse

  conf = nil
  begin
    conf = Config.read
  rescue ConfigError
  end
  if conf && conf['account.apikey']
    unless force
      if conf['account.user']
        $stderr.puts "Account is already configured with '#{conf['account.user']}' account."
      else
        $stderr.puts "Account is already configured."
      end
      $stderr.puts "Add '-f' option to overwrite."
      exit 0
    end
  end

  conf ||= Config.new
  conf.delete("account.user")
  conf["account.apikey"] = apikey
  conf.save

  $stderr.puts "API key is set."

  $stderr.puts "Use '#{$prog} db:create <db_name>' to create a database."
end

#apikey_show(op) ⇒ Object 



5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
 
# File 'lib/td/command/apikey.rb', line 5

def apikey_show(op)
  conf = nil
  begin
    conf = Config.read
  rescue ConfigError
  end

  if !conf || !conf['account.apikey']
    $stderr.puts "Account is not configured yet."
    $stderr.puts "Use '#{$prog} apikey:set' or '#{$prog} account' first."
    exit 1
  end

  puts conf['account.apikey']
end

#db_create(op) ⇒ Object 



44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
 
# File 'lib/td/command/db.rb', line 44

def db_create(op)
  db_name = op.cmd_parse

  API.validate_database_name(db_name)

  client = get_client

  begin
    client.create_database(db_name)
  rescue AlreadyExistsError
    $stderr.puts "Database '#{db_name}' already exists."
    exit 1
  end

  $stderr.puts "Database '#{db_name}' is created."
  $stderr.puts "Use '#{$prog} table:create #{db_name} <table_name>' to create a table."
end

#db_delete(op) ⇒ Object 



62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
 
# File 'lib/td/command/db.rb', line 62

def db_delete(op)
  force = false
  op.on('-f', '--force', 'clear tables and delete the database', TrueClass) {|b|
    force = true
  }

  db_name = op.cmd_parse

  client = get_client

  begin
    db = client.database(db_name)

    if !force && !db.tables.empty?
      $stderr.puts "Database '#{db_name}' is not empty. Use '-f' option or drop tables first."
      exit 1
    end

    db.delete
  rescue NotFoundError
    $stderr.puts "Database '#{db_name}' does not exist."
    exit 1
  end

  $stderr.puts "Database '#{db_name}' is deleted."
end

#db_list(op) ⇒ Object 



26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
 
# File 'lib/td/command/db.rb', line 26

def db_list(op)
  op.cmd_parse

  client = get_client
  dbs = client.databases

  rows = []
  dbs.each {|db|
    rows << {:Name => db.name}
  }
  puts cmd_render_table(rows, :fields => [:Name])

  if dbs.empty?
    $stderr.puts "There are no databases."
    $stderr.puts "Use '#{$prog} db:create <db_name>' to create a database."
  end
end

#db_show(op) ⇒ Object 



5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
 
# File 'lib/td/command/db.rb', line 5

def db_show(op)
  db_name = op.cmd_parse

  client = get_client

  db = get_database(client, db_name)

  rows = []
  db.tables.each {|table|
    pschema = table.schema.fields.map {|f|
      "#{f.name}:#{f.type}"
    }.join(', ')
    rows << {:Table => table.name, :Type => table.type.to_s, :Count => table.count.to_s, :Schema=>pschema.to_s}
  }
  rows = rows.sort_by {|map|
    [map[:Type].size, map[:Table]]
  }

  puts cmd_render_table(rows, :fields => [:Table, :Type, :Count, :Schema])
end

#help(op) ⇒ Object 



5
6
7
8
9
10
11
12
13
14
15
16
 
# File 'lib/td/command/help.rb', line 5

def help(op)
  cmd = op.cmd_parse

  usage = List.cmd_usage(cmd)
  unless usage
    $stderr.puts "'#{cmd}' is not a td command. Run '#{$prog}' to show the list."
    List.show_guess(cmd)
    exit 1
  end

  puts usage
end

#help_all(op) ⇒ Object 



18
19
20
21
22
23
24
 
# File 'lib/td/command/help.rb', line 18

def help_all(op)
  cmd = op.cmd_parse

  TreasureData::Command::List.show_help(op.summary_indent)
  puts ""
  puts "Type '#{$prog} help COMMAND' for more information on a specific command."
end

#job_kill(op) ⇒ Object 



111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
 
# File 'lib/td/command/job.rb', line 111

def job_kill(op)
  job_id = op.cmd_parse

  client = get_client

  former_status = client.kill(job_id)
  if TreasureData::Job::FINISHED_STATUS.include?(former_status)
    $stderr.puts "Job #{job_id} is already finished (#{former_status})"
    exit 0
  end

  if former_status == TreasureData::Job::STATUS_RUNNING
    $stderr.puts "Job #{job_id} is killed."
  else
    $stderr.puts "Job #{job_id} is canceled."
  end
end

#job_list(op) ⇒ Object 



5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
 
# File 'lib/td/command/job.rb', line 5

def job_list(op)
  page = 0
  skip = 0
  running = false
  error = false

  op.on('-p', '--page PAGE', 'skip N pages', Integer) {|i|
    page = i
  }
  op.on('-s', '--skip N', 'skip N jobs', Integer) {|i|
    skip = i
  }
  op.on('-R', '--running', 'show only running jobs', TrueClass) {|b|
    running = b
  }
  op.on('-E', '--error', 'show only error jobs', TrueClass) {|b|
    error = b
  }

  max = op.cmd_parse

  max = (max || 20).to_i

  client = get_client

  if page
    skip += max * page
  end
  jobs = client.jobs(skip, skip+max-1)

  rows = []
  jobs.each {|job|
    next if running && !job.running?
    next if error && !job.error?
    start = job.start_at
    elapsed = cmd_format_elapsed(start, job.end_at)
    rows << {:JobID => job.job_id, :Status => job.status, :Query => job.query.to_s, :Start => (start ? start.localtime : ''), :Elapsed => elapsed, :Result => job.rset_name}
  }

  puts cmd_render_table(rows, :fields => [:JobID, :Status, :Start, :Elapsed, :Result, :Query])
end

#job_show(op) ⇒ Object 



47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
 
# File 'lib/td/command/job.rb', line 47

def job_show(op)
  verbose = nil
  wait = false
  output = nil
  format = 'tsv'

  op.on('-v', '--verbose', 'show logs', TrueClass) {|b|
    verbose = b
  }
  op.on('-w', '--wait', 'wait for finishing the job', TrueClass) {|b|
    wait = b
  }
  op.on('-o', '--output PATH', 'write result to the file') {|s|
    output = s
  }
  op.on('-f', '--format FORMAT', 'format of the result to write to the file (tsv, csv, json or msgpack)') {|s|
    unless ['tsv', 'csv', 'json', 'msgpack'].include?(s)
      raise "Unknown format #{s.dump}. Supported format: tsv, csv, json, msgpack"
    end
    format = s
  }

  job_id = op.cmd_parse

  client = get_client

  job = client.job(job_id)

  puts "JobID        : #{job.job_id}"
  puts "URL          : #{job.url}"
  puts "Status       : #{job.status}"
  puts "Query        : #{job.query}"
  puts "Result table : #{job.rset_name}"

  if wait && !job.finished?
    wait_job(job)
    if job.success?
      puts "Result       :"
      show_result(job, output, format)
    end

  else
    if job.success?
      puts "Result       :"
      show_result(job, output, format)
    end

    if verbose
      puts ""
      puts "cmdout:"
      job.debug['cmdout'].to_s.split("\n").each {|line|
        puts "  "+line
      }
      puts ""
      puts "stderr:"
      job.debug['stderr'].to_s.split("\n").each {|line|
        puts "  "+line
      }
    end
  end

  $stderr.puts "Use '-v' option to show detailed messages." unless verbose
end

#query(op) ⇒ Object 



5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
 
# File 'lib/td/command/query.rb', line 5

def query(op)
  db_name = nil
  wait = false
  output = nil
  format = 'tsv'
  result = nil

  op.on('-d', '--database DB_NAME', 'use the database (required)') {|s|
    db_name = s
  }
  op.on('-w', '--wait', 'wait for finishing the job', TrueClass) {|b|
    wait = b
  }
  op.on('-r', '--result RESULT_TABLE', 'write result to the result table (use result:create command)') {|s|
    result = s
  }
  op.on('-o', '--output PATH', 'write result to the file') {|s|
    output = s
  }
  op.on('-f', '--format FORMAT', 'format of the result to write to the file (tsv, csv, json or msgpack)') {|s|
    unless ['tsv', 'csv', 'json', 'msgpack'].include?(s)
      raise "Unknown format #{s.dump}. Supported format: tsv, csv, json, msgpack"
    end
    format = s
  }

  sql = op.cmd_parse

  unless db_name
    $stderr.puts "-d, --database DB_NAME option is required."
    exit 1
  end

  client = get_client

  # local existance check
  get_database(client, db_name)

  job = client.query(db_name, sql, result)

  $stderr.puts "Job #{job.job_id} is queued."
  $stderr.puts "Use '#{$prog} job:show #{job.job_id}' to show the status."
  $stderr.puts "See #{job.url} to see the progress."

  if wait && !job.finished?
    wait_job(job)
    puts "Status     : #{job.status}"
    if job.success?
      puts "Result     :"
      show_result(job, output, format)
    end
  end
end

#result_connect(op) ⇒ Object 



75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
 
# File 'lib/td/command/result.rb', line 75

def result_connect(op)
  mysql = 'mysql'

  op.on('-e', '--execute MYSQL', 'mysql command') {|s|
    mysql = s
  }

  sql = op.cmd_parse

  client = get_client

  info = client.result_set_info

  cmd = [mysql, '-h', info.host, '-P', info.port.to_s, '-u', info.user, "--password=#{info.password}", info.database]

  cmd_start = Time.now

  if sql
    IO.popen(cmd, "w") {|io|
      io.write sql
      io.close
    }
  else
    STDERR.puts "> #{cmd.join(' ')}"
    system(*cmd)
  end

  cmd_alive = Time.now - cmd_start
  if $?.to_i != 0 && cmd_alive < 1.0
    STDERR.puts "Command died within 1 second with exit code #{$?.to_i}."
    STDERR.puts "Please confirm mysql command is installed."
  end
end

#result_create(op) ⇒ Object 



43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
 
# File 'lib/td/command/result.rb', line 43

def result_create(op)
  name = op.cmd_parse

  API.validate_database_name(name)

  client = get_client

  begin
    client.create_result_set(name)
  rescue AlreadyExistsError
    $stderr.puts "Result table '#{name}' already exists."
    exit 1
  end

  $stderr.puts "Result table '#{name}' is created."
end

#result_delete(op) ⇒ Object 



60
61
62
63
64
65
66
67
68
69
70
71
72
73
 
# File 'lib/td/command/result.rb', line 60

def result_delete(op)
  name = op.cmd_parse

  client = get_client

  begin
    client.delete_result_set(name)
  rescue NotFoundError
    $stderr.puts "Result table '#{name}' does not exist."
    exit 1
  end

  $stderr.puts "Result table '#{name}' is deleted."
end

#result_info(op) ⇒ Object 



5
6
7
8
9
10
11
12
13
14
15
16
17
18
 
# File 'lib/td/command/result.rb', line 5

def result_info(op)
  op.cmd_parse

  client = get_client

  info = client.result_set_info

  puts "Type       : #{info.type}"
  puts "Host       : #{info.host}"
  puts "Port       : #{info.port}"
  puts "User       : #{info.user}"
  puts "Password   : #{info.password}"
  puts "Database   : #{info.database}"
end

#result_list(op) ⇒ Object 



20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
 
# File 'lib/td/command/result.rb', line 20

def result_list(op)
  op.cmd_parse

  client = get_client

  rsets = client.result_sets

  rows = []
  rsets.each {|rset|
    rows << {:Name => rset.name}
  }
  rows = rows.sort_by {|map|
    map[:Name]
  }

  puts cmd_render_table(rows, :fields => [:Name])

  if rsets.empty?
    $stderr.puts "There are result tables."
    $stderr.puts "Use '#{$prog} result:create <name>' to create a result table."
  end
end

#sched_create(op) ⇒ Object 



23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
 
# File 'lib/td/command/sched.rb', line 23

def sched_create(op)
  db_name = nil
  result = nil

  op.on('-d', '--database DB_NAME', 'use the database (required)') {|s|
    db_name = s
  }
  op.on('-r', '--result RESULT_TABLE', 'write result to the result table (use result:create command)') {|s|
    result = s
  }

  name, cron, sql = op.cmd_parse

  unless db_name
    $stderr.puts "-d, --database DB_NAME option is required."
    exit 1
  end

  client = get_client

  # local existance check
  get_database(client, db_name)

  begin
    first_time = client.create_schedule(name, :cron=>cron, :query=>sql, :database=>db_name, :result=>result)
  rescue AlreadyExistsError
    cmd_debug_error $!
    $stderr.puts "Schedule '#{name}' already exists."
    exit 1
  end

  $stderr.puts "Schedule '#{name}' is created. It starts at #{first_time}."
end

#sched_delete(op) ⇒ Object 



57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
 
# File 'lib/td/command/sched.rb', line 57

def sched_delete(op)
  name = op.cmd_parse

  client = get_client

  begin
    client.delete_schedule(name)
  rescue NotFoundError
    cmd_debug_error $!
    $stderr.puts "Schedule '#{name}' does not exist."
    $stderr.puts "Use '#{$prog} sched:list' to show list of the schedules."
    exit 1
  end

  $stderr.puts "Schedule '#{name}' is deleted."
end

#sched_history(op) ⇒ Object 



74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
 
# File 'lib/td/command/sched.rb', line 74

def sched_history(op)
  page = 0
  skip = 0

  op.on('-p', '--page PAGE', 'skip N pages', Integer) {|i|
    page = i
  }
  op.on('-s', '--skip N', 'skip N schedules', Integer) {|i|
    skip = i
  }

  name, max = op.cmd_parse

  max = (max || 20).to_i

  if page
    skip += max * page
  end

  client = get_client

  begin
    history = client.history(name, skip, skip+max-1)
  rescue NotFoundError
    cmd_debug_error $!
    $stderr.puts "Schedule '#{name}' does not exist."
    $stderr.puts "Use '#{$prog} sched:list' to show list of the schedules."
    exit 1
  end

  rows = []
  history.each {|j|
    rows << {:Time => j.scheduled_at.localtime, :JobID => j.job_id, :Status => j.status, :Result=>j.rset_name}
  }

  puts cmd_render_table(rows, :fields => [:JobID, :Time, :Status, :Result])
end

#sched_list(op) ⇒ Object 



5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
 
# File 'lib/td/command/sched.rb', line 5

def sched_list(op)
  op.cmd_parse

  client = get_client

  scheds = client.schedules

  rows = []
  scheds.each {|sched|
    rows << {:Name => sched.name, :Cron => sched.cron, :Result => sched.rset_name, :Query => sched.query}
  }
  rows = rows.sort_by {|map|
    map[:Name]
  }

  puts cmd_render_table(rows, :fields => [:Name, :Cron, :Result, :Query])
end

#schema_add(op) ⇒ Object 



30
31
32
33
34
35
36
37
38
39
40
41
42
 
# File 'lib/td/command/schema.rb', line 30

def schema_add(op)
  db_name, table_name, *columns = op.cmd_parse
  schema = parse_columns(columns)

  client = get_client
  table = get_table(client, db_name, table_name)

  schema = table.schema.merge(schema)

  client.update_schema(table.db_name, table.name, schema)

  $stderr.puts "Schema is updated on #{db_name}.#{table_name} table."
end

#schema_remove(op) ⇒ Object 



44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
 
# File 'lib/td/command/schema.rb', line 44

def schema_remove(op)
  db_name, table_name, *columns = op.cmd_parse

  client = get_client
  table = get_table(client, db_name, table_name)

  schema = table.schema

  columns.each {|col|
    deleted = false
    schema.fields.delete_if {|f|
      f.name == col && deleted = true
    }
    unless deleted
      $stderr.puts "Column name '#{col}' does not exist."
      exit 1
    end
  }

  client.update_schema(table.db_name, table.name, schema)

  $stderr.puts "Schema is updated on #{db_name}.#{table_name} table."
end

#schema_set(op) ⇒ Object 



18
19
20
21
22
23
24
25
26
27
28
 
# File 'lib/td/command/schema.rb', line 18

def schema_set(op)
  db_name, table_name, *columns = op.cmd_parse
  schema = parse_columns(columns)

  client = get_client
  table = get_table(client, db_name, table_name)

  client.update_schema(table.db_name, table.name, schema)

  $stderr.puts "Schema is updated on #{db_name}.#{table_name} table."
end

#schema_show(op) ⇒ Object 



5
6
7
8
9
10
11
12
13
14
15
16
 
# File 'lib/td/command/schema.rb', line 5

def schema_show(op)
  db_name, table_name = op.cmd_parse

  client = get_client
  table = get_table(client, db_name, table_name)

  puts "#{db_name}.#{table_name} ("
  table.schema.fields.each {|f|
    puts "  #{f.name}:#{f.type}"
  }
  puts ")"
end

#server_status(op) ⇒ Object 



5
6
7
8
9
 
# File 'lib/td/command/server.rb', line 5

def server_status(op)
  op.cmd_parse

  puts Client.server_status
end

#status(op) ⇒ Object 



5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
 
# File 'lib/td/command/status.rb', line 5

def status(op)
  op.cmd_parse

  client = get_client

  # +----------------+
  # |     scheds     |
  # +----------------+
  # +----------------+
  # |      jobs      |
  # +----------------+
  # +------+ +-------+
  # |tables| |results|
  # +------+ +-------+

  scheds = []
  jobs = []
  tables = []
  results = []

  s = client.schedules
  s.each {|sched|
    scheds << {:Name => sched.name, :Cron => sched.cron, :Result => sched.rset_name, :Query => sched.query}
  }
  scheds = scheds.sort_by {|map|
    map[:Name]
  }
  x1, y1 = status_render(0, 0, "[Schedules]", scheds, :fields => [:Name, :Cron, :Result, :Query])

  j = client.jobs(0, 4)
  j.each {|job|
    start = job.start_at
    elapsed = cmd_format_elapsed(start, job.end_at)
    jobs << {:JobID => job.job_id, :Status => job.status, :Query => job.query.to_s, :Start => (start ? start.localtime : ''), :Elapsed => elapsed, :Result => job.rset_name}
  }
  x2, y2 = status_render(0, 0, "[Jobs]", jobs, :fields => [:JobID, :Status, :Start, :Elapsed, :Result, :Query])

  dbs = client.databases
  dbs.map {|db|
    db.tables.each {|table|
      tables << {:Database => db.name, :Table => table.name, :Count => table.count.to_s}
    }
  }
  x3, y3 = status_render(0, 0, "[Tables]", tables, :fields => [:Database, :Table, :Count])

  r = client.result_sets
  r.each {|rset|
    results << {:Name => rset.name}
  }
  results = results.sort_by {|map|
    map[:Name]
  }
  x4, y4 = status_render(x3+2, y3, "[Results]", results, :fields => [:Name])

  (y3-y4-1).times do
    print "\eD"
  end
  print "\eE"
end

#table_create(op) ⇒ Object 



5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
 
# File 'lib/td/command/table.rb', line 5

def table_create(op)
  db_name, table_name = op.cmd_parse

  #API.validate_database_name(db_name)
  API.validate_table_name(table_name)

  client = get_client

  begin
    client.create_log_table(db_name, table_name)
  rescue NotFoundError
    cmd_debug_error $!
    $stderr.puts "Database '#{db_name}' does not exist."
    $stderr.puts "Use '#{$prog} db:create #{db_name}' to create the database."
    exit 1
  rescue AlreadyExistsError
    cmd_debug_error $!
    $stderr.puts "Table '#{db_name}.#{table_name}' already exists."
    exit 1
  end

  $stderr.puts "Table '#{db_name}.#{table_name}' is created."
end

#table_delete(op) ⇒ Object 



29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
 
# File 'lib/td/command/table.rb', line 29

def table_delete(op)
  db_name, table_name = op.cmd_parse

  client = get_client

  begin
    client.delete_table(db_name, table_name)
  rescue NotFoundError
    cmd_debug_error $!
    $stderr.puts "Table '#{db_name}.#{table_name}' does not exist."
    $stderr.puts "Use '#{$prog} table:list #{db_name}' to show list of the tables."
    exit 1
  end

  $stderr.puts "Table '#{db_name}.#{table_name}' is deleted."
end

#table_import(op) ⇒ Object

TODO import-item TODO tail



19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
 
# File 'lib/td/command/import.rb', line 19

def table_import(op)
  op.banner << "\nsupported formats:\n"
  op.banner << "  apache\n"
  op.banner << "  syslog\n"
  op.banner << "  msgpack\n"
  op.banner << "  json\n"

  format = 'apache'
  time_key = nil

  op.on('--format FORMAT', "file format (default: #{format})") {|s|
    format = s
  }

  op.on('--apache', "same as --format apache; apache common log format") {
    format = 'apache'
  }

  op.on('--syslog', "same as --format syslog; syslog") {
    format = 'syslog'
  }

  op.on('--msgpack', "same as --format msgpack; msgpack stream format") {
    format = 'msgpack'
  }

  op.on('--json', "same as --format json; LF-separated json format") {
    format = 'json'
  }

  op.on('-t', '--time-key COL_NAME', "time key name for json and msgpack format (e.g. 'created_at')") {|s|
    time_key = s
  }

  db_name, table_name, *paths = op.cmd_parse

  client = get_client

  case format
  when 'json', 'msgpack'
    unless time_key
      $stderr.puts "-t, --time-key COL_NAME (e.g. '-t created_at') parameter is required for #{format} format"
      exit 1
    end
    if format == 'json'
      require 'json'
      require 'time'
      parser = JsonParser.new(time_key)
    else
      parser = MessagePackParser.new(time_key)
    end

  else
    regexp, names, time_format = IMPORT_TEMPLATES[format]
    if !regexp || !names || !time_format
      $stderr.puts "Unknown format '#{format}'"
      exit 1
    end
    parser = TextParser.new(names, regexp, time_format)
  end

  get_table(client, db_name, table_name)

  require 'zlib'

  files = paths.map {|path|
    if path == '-'
      $stdin
    elsif path =~ /\.gz$/
      Zlib::GzipReader.open(path)
    else
      File.open(path)
    end
  }

  require 'msgpack'
  require 'tempfile'
  #require 'thread'

  files.zip(paths).each {|file,path|
    import_log_file(file, path, client, db_name, table_name, parser)
  }

  puts "done."
end

#table_list(op) ⇒ Object 



46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
 
# File 'lib/td/command/table.rb', line 46

def table_list(op)
  db_name = op.cmd_parse

  client = get_client

  if db_name
    db = get_database(client, db_name)
    dbs = [db]
  else
    dbs = client.databases
  end

  rows = []
  dbs.each {|db|
    db.tables.each {|table|
      pschema = table.schema.fields.map {|f|
        "#{f.name}:#{f.type}"
      }.join(', ')
      rows << {:Database => db.name, :Table => table.name, :Type => table.type.to_s, :Count => table.count.to_s, :Schema=>pschema.to_s}
    }
  }
  rows = rows.sort_by {|map|
    [map[:Database], map[:Type].size, map[:Table]]
  }

  puts cmd_render_table(rows, :fields => [:Database, :Table, :Type, :Count, :Schema])

  if rows.empty?
    if db_name
      $stderr.puts "Database '#{db_name}' has no tables."
      $stderr.puts "Use '#{$prog} table:create <db.table>' to create a table."
    elsif dbs.empty?
      $stderr.puts "There are no databases."
      $stderr.puts "Use '#{$prog} db:create <db>' to create a database."
    else
      $stderr.puts "There are no tables."
      $stderr.puts "Use '#{$prog} table:create <db.table>' to create a table."
    end
  end
end

#table_show(op) ⇒ Object 



87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
 
# File 'lib/td/command/table.rb', line 87

def table_show(op)
  db_name, table_name = op.cmd_parse

  client = get_client

  table = get_table(client, db_name, table_name)

  puts "Name      : #{table.db_name}.#{table.name}"
  puts "Type      : #{table.type}"
  puts "Count     : #{table.count}"
  puts "Schema    : ("
  table.schema.fields.each {|f|
    puts "    #{f.name}:#{f.type}"
  }
  puts ")"
end

#table_tail(op) ⇒ Object 



104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
 
# File 'lib/td/command/table.rb', line 104

def table_tail(op)
  from = nil
  to = nil
  count = nil

  op.on('-t', '--to TIME', 'end time of logs to get') {|s|
    if s.to_i.to_s == s
      to = s
    else
      to = Time.parse(s).to_i
    end
  }
  op.on('-f', '--from TIME', 'start time of logs to get') {|s|
    if s.to_i.to_s == s
      from = s
    else
      from = Time.parse(s).to_i
    end
  }
  op.on('-n', '--count N', 'number of logs to get', Integer) {|i|
    count = i
  }

  if count == nil
    # smart count calculation
    begin
      require "curses"
     if Curses.stdscr.maxy - 1 <= 40
        count = 5
      else
        count = 10
      end
     Curses.close_screen
    rescue Exception
      count = 5
    end
  end

  db_name, table_name = op.cmd_parse

  client = get_client

  table = get_table(client, db_name, table_name)

  rows = table.tail(count, to, from)

  require 'json'
  rows.each {|row|
    puts row.to_json
  }
end