Module: TreasureData::Command

Defined in:
lib/td/command/db.rb,
lib/td/command/acl.rb,
lib/td/command/job.rb,
lib/td/command/org.rb,
lib/td/command/aggr.rb,
lib/td/command/help.rb,
lib/td/command/list.rb,
lib/td/command/role.rb,
lib/td/command/user.rb,
lib/td/command/query.rb,
lib/td/command/sched.rb,
lib/td/command/table.rb,
lib/td/command/apikey.rb,
lib/td/command/common.rb,
lib/td/command/export.rb,
lib/td/command/import.rb,
lib/td/command/result.rb,
lib/td/command/runner.rb,
lib/td/command/sample.rb,
lib/td/command/schema.rb,
lib/td/command/server.rb,
lib/td/command/status.rb,
lib/td/command/account.rb,
lib/td/command/ip_limit.rb,
lib/td/command/password.rb,
lib/td/command/bulk_import.rb

Defined Under Namespace

Modules: List Classes: JsonParser, MessagePackParser, Runner, TextParser

Constant Summary collapse

JOB_WAIT_MAX_RETRY_COUNT_ON_NETWORK_ERROR =

TODO

10
PRIORITY_FORMAT_MAP = 
{
  -2 => 'VERY LOW',
  -1 => 'LOW',
  0 => 'NORMAL',
  1 => 'HIGH',
  2 => 'VERY HIGH',
}
PRIORITY_PARSE_MAP = 
{
  /\Avery[ _\-]?low\z/i => -2,
  /\A-2\z/ => -2,
  /\Alow\z/i => -1,
  /\A-1\z/ => -1,
  /\Anorm(?:al)?\z/i => 0,
  /\A[\-\+]?0\z/ => 0,
  /\Ahigh\z/i => 1,
  /\A[\+]?1\z/ => 1,
  /\Avery[ _\-]?high\z/i => 2,
  /\A[\+]?2\z/ => 2,
}
HIVE_RESERVED_KEYWORDS = 
%W[
  TRUE FALSE ALL AND OR NOT LIKE ASC DESC ORDER BY GROUP WHERE FROM AS SELECT DISTINCT INSERT OVERWRITE
  OUTER JOIN LEFT RIGHT FULL ON PARTITION PARTITIONS TABLE TABLES TBLPROPERTIES SHOW MSCK DIRECTORY LOCAL
  TRANSFORM USING CLUSTER DISTRIBUTE SORT UNION LOAD DATA INPATH IS NULL CREATE EXTERNAL ALTER DESCRIBE
  DROP REANME TO COMMENT BOOLEAN TINYINT SMALLINT INT BIGINT FLOAT DOUBLE DATE DATETIME TIMESTAMP STRING
  BINARY ARRAY MAP REDUCE PARTITIONED CLUSTERED SORTED INTO BUCKETS ROW FORMAT DELIMITED FIELDS TERMINATED
  COLLECTION ITEMS KEYS LINES STORED SEQUENCEFILE TEXTFILE INPUTFORMAT OUTPUTFORMAT LOCATION TABLESAMPLE BUCKET OUT
  OF CAST ADD REPLACE COLUMNS RLIKE REGEXP TEMPORARY FUNCTION EXPLAIN EXTENDED SERDE WITH SERDEPROPERTIES LIMIT SET TBLPROPERTIES
]
IMPORT_TEMPLATES = 
{
  'apache' => [
                /^([^ ]*) [^ ]* ([^ ]*) \[([^\]]*)\] "(\S+)(?: +([^ ]*) +\S*)?" ([^ ]*) ([^ ]*)(?: "([^\"]*)" "([^\"]*)")?$/,
                ['host', 'user', 'time', 'method', 'path', 'code', 'size', 'referer', 'agent'],
                "%d/%b/%Y:%H:%M:%S %z"],
  'syslog' => [
                /^([^ ]* [^ ]* [^ ]*) ([^ ]*) ([a-zA-Z0-9_\/\.\-]*)(?:\[([0-9]+)\])?[^\:]*\: *(.*)$/,
                ['time', 'host', 'ident', 'pid', 'message'],
                "%b %d %H:%M:%S"],
}
PART_SPLIT_SIZE = 
16*1024*1024

Instance Method Summary collapse

Instance Method Details

#account(op) ⇒ Object 



6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
 
# File 'lib/td/command/account.rb', line 6

def (op)
  op.banner << "\noptions:\n"

  force = false
  op.on('-f', '--force', 'overwrite current account setting', TrueClass) {|b|
    force = true
  }

  user_name = op.cmd_parse

  conf = nil
  begin
    conf = Config.read
  rescue ConfigError
  end
  if conf && conf['account.apikey']
    unless force
      if conf['account.user']
        $stderr.puts "Account is already configured with '#{conf['account.user']}' account."
      else
        $stderr.puts "Account is already configured."
      end
      $stderr.puts "Add '-f' option to overwrite."
      exit 0
    end
  end

  puts "Enter your Treasure Data credentials."
  unless user_name
    begin
      print "Email: "
      line = STDIN.gets || ""
      user_name = line.strip
    rescue Interrupt
      $stderr.puts "\ncanceled."
      exit 1
    end
  end

  if user_name.empty?
    $stderr.puts "canceled."
    exit 0
  end

  client = nil

  3.times do
    begin
      print "Password (typing will be hidden): "
      password = get_password
    rescue Interrupt
      $stderr.print "\ncanceled."
      exit 1
    ensure
      system "stty echo"   # TODO termios
      print "\n"
    end

    if password.empty?
      $stderr.puts "canceled."
      exit 0
    end

    begin
      # enalbe SSL for the authentication
      client = Client.authenticate(user_name, password, :ssl=>true)
    rescue TreasureData::AuthError
      $stderr.puts "User name or password mismatched."
    end

    break if client
  end
  return unless client

  $stderr.puts "Authenticated successfully."

  conf ||= Config.new
  conf["account.user"] = user_name
  conf["account.apikey"] = client.apikey
  conf.save

  $stderr.puts "Use '#{$prog} db:create <db_name>' to create a database."
end

#account_usage(op) ⇒ Object 



90
91
92
93
94
95
96
97
98
 
# File 'lib/td/command/account.rb', line 90

def (op)
  op.cmd_parse

  client = get_client

  a = client.

  $stderr.puts "Storage:  #{a.storage_size_string}"
end

#acl_grant(op) ⇒ Object 



25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
 
# File 'lib/td/command/acl.rb', line 25

def acl_grant(op)
  grant_option = true

  op.on('--no-grant-option', '-N', 'Grant without grant option', TrueClass) {|b|
    grant_option = !b
  }

  subject, action, scope = op.cmd_parse

  client = get_client

  client.grant_access_control(subject, action, scope, grant_option)

  $stderr.puts "Access control [#{subject} #{action} #{scope}] is created #{grant_option ? 'with' : 'without'} grant option."
end

#acl_list(op) ⇒ Object 



5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
 
# File 'lib/td/command/acl.rb', line 5

def acl_list(op)
  op.cmd_parse

  client = get_client

  acl = client.access_controls

  rows = []
  acl.each {|ac|
    rows << {:Subject => ac.subject, :Action => ac.action, :Scope => ac.scope, :"Grant option" => ac.grant_option}
  }

  puts cmd_render_table(rows, :fields => [:Subject, :Action, :Scope, :"Grant option"])

  if rows.empty?
    $stderr.puts "There are no access controls."
    $stderr.puts "Use '#{$prog} acl:grant <subject> <action> <scope>' to grant permissions."
  end
end

#acl_revoke(op) ⇒ Object 



41
42
43
44
45
46
47
48
49
 
# File 'lib/td/command/acl.rb', line 41

def acl_revoke(op)
  subject, action, scope = op.cmd_parse

  client = get_client

  client.revoke_access_control(subject, action, scope)

  $stderr.puts "Access control [#{subject} #{action} #{scope}] is removed."
end

#aggr_add_attr(op) ⇒ Object 



143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
 
# File 'lib/td/command/aggr.rb', line 143

def aggr_add_attr(op)
  comment = nil

  op.on('-m', '--comment COMMENT', 'comment of this entry') {|s|
    comment = s
  }

  name, db_name, table_name, entry_name, method_name, *parameters = op.cmd_parse

  params = {}
  parameters.each {|pa|
    k, v = pa.split('=')
    params[k] = v
  }

  client = get_client

  get_table(client, db_name, table_name)

  begin
    client.create_aggregation_attr_entry(name, entry_name, comment, db_name, table_name, method_name, params)
  rescue NotFoundError
    cmd_debug_error $!
    $stderr.puts "Aggregation schema '#{name}' does not exist."
    $stderr.puts "Use '#{$prog} aggr:create #{name}' to create the aggregation schema."
    exit 1
  rescue AlreadyExistsError
    $stderr.puts "Aggregation attribute entry '#{entry_name}' already exists."
    exit 1
  end

  $stderr.puts "Aggregation attribute entry '#{entry_name}' is created."
end

#aggr_add_log(op) ⇒ Object 



105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
 
# File 'lib/td/command/aggr.rb', line 105

def aggr_add_log(op)
  comment = nil
  value_key = nil
  count_key = nil

  op.on('-m', '--comment COMMENT', 'comment of this entry') {|s|
    comment = s
  }
  op.on('-v', '--value KEY_NAME', 'key name of value field') {|s|
    value_key = s
  }
  op.on('-c', '--count KEY_NAME', 'key name of count field') {|s|
    count_key = s
  }

  name, db_name, table_name, entry_name, o1_key, o2_key, o3_key = op.cmd_parse

  okeys = [o1_key, o2_key, o3_key].compact

  client = get_client

  get_table(client, db_name, table_name)

  begin
    client.create_aggregation_log_entry(name, entry_name, comment, db_name, table_name, okeys, value_key, count_key)
  rescue NotFoundError
    cmd_debug_error $!
    $stderr.puts "Aggregation schema '#{name}' does not exist."
    $stderr.puts "Use '#{$prog} aggr:create #{name}' to create the aggregation schema."
    exit 1
  rescue AlreadyExistsError
    $stderr.puts "Aggregation log entry '#{entry_name}' already exists."
    exit 1
  end

  $stderr.puts "Aggregation log entry '#{entry_name}' is created."
end

#aggr_create(op) ⇒ Object 



73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
 
# File 'lib/td/command/aggr.rb', line 73

def aggr_create(op)
  timezone = nil

  op.on('-t', '--timezone TZ', 'name of the timezone (like Asia/Tokyo)') {|s|
    timezone = s
  }

  name, relation_key = op.cmd_parse

  client = get_client

  begin
    client.create_aggregation_schema(name, relation_key, {'timezone'=>timezone})
  rescue AlreadyExistsError
    cmd_debug_error $!
    $stderr.puts "Aggregation '#{name}' already exists."
    exit 1
  end

  $stderr.puts "Aggregation schema '#{name}' is created."
end

#aggr_del_attr(op) ⇒ Object 



192
193
194
195
196
197
198
199
200
201
202
203
204
205
 
# File 'lib/td/command/aggr.rb', line 192

def aggr_del_attr(op)
  name, entry_name = op.cmd_parse

  client = get_client

  begin
    client.delete_aggregation_attr_entry(name, entry_name)
  rescue NotFoundError
    $stderr.puts "Aggregation log entry '#{entry_name}' does not exist."
    exit 1
  end

  $stderr.puts "Aggregation log entry '#{entry_name}' is deleted."
end

#aggr_del_log(op) ⇒ Object 



177
178
179
180
181
182
183
184
185
186
187
188
189
190
 
# File 'lib/td/command/aggr.rb', line 177

def aggr_del_log(op)
  name, entry_name = op.cmd_parse

  client = get_client

  begin
    client.delete_aggregation_log_entry(name, entry_name)
  rescue NotFoundError
    $stderr.puts "Aggregation log entry '#{entry_name}' does not exist."
    exit 1
  end

  $stderr.puts "Aggregation log entry '#{entry_name}' is deleted."
end

#aggr_delete(op) ⇒ Object 



95
96
97
98
99
100
101
102
103
 
# File 'lib/td/command/aggr.rb', line 95

def aggr_delete(op)
  name = op.cmd_parse

  client = get_client

  client.delete_aggregation_schema(name)

  $stderr.puts "Aggregation schema '#{name}' is deleted."
end

#aggr_list(op) ⇒ Object 



5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
 
# File 'lib/td/command/aggr.rb', line 5

def aggr_list(op)
  op.cmd_parse

  client = get_client

  ass = client.aggregation_schemas

  rows = []
  ass.each {|as|
    rows << {:Name=>as.name, :Relation=>as.relation_key, :Timezone=>as.timezone}
  }

  puts cmd_render_table(rows, :fields => [:Name, :Relation, :Timezone])

  if rows.empty?
    $stderr.puts "There are no aggregation schemas."
    $stderr.puts "Use '#{$prog} aggr:create <name>' to create a aggregation schema."
  end
end

#aggr_show(op) ⇒ Object 



25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
 
# File 'lib/td/command/aggr.rb', line 25

def aggr_show(op)
  name = op.cmd_parse

  client = get_client

  begin
    as = client.aggregation_schema(name)
  rescue
    cmd_debug_error $!
    $stderr.puts "Aggregation '#{name}' does not exist."
    exit 1
  end

  log_rows = []
  as.logs.each {|las|
    log_rows << {
      :Table=>las.table.identifier,
      :Name=>las.name,
      :o1_key=>las.okeys[0].to_s,
      :o2_key=>las.okeys[1].to_s,
      :o3_key=>las.okeys[2].to_s,
      :value_key=>las.value_key.to_s,
      :count_key=>las.count_key.to_s,
      :Comment=>las.comment.to_s
    }
  }

  attr_rows = []
  as.attributes.each {|aas|
    params = aas.parameters.to_a.map {|k,v| "#{k}=#{v}" }.join(' ')
    attr_rows << {
      :Table=>aas.table.identifier,
      :Name=>aas.name,
      :Method=>aas.method_name,
      :Parameters=>params,
      :Comment=>aas.comment.to_s,
    }
  }

  puts "Log entries:"
  puts cmd_render_table(log_rows, :fields => [:Table, :Name, :o1_key, :o2_key, :o3_key, :value_key, :count_key, :Comment], :max_width=>400)

  puts ''

  puts "Attribute entries:"
  puts cmd_render_table(attr_rows, :fields => [:Table, :Name, :Method, :Parameters, :Comment], :max_width=>400)
end

#apikey_set(op) ⇒ Object 



26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
 
# File 'lib/td/command/apikey.rb', line 26

def apikey_set(op)
  op.banner << "\noptions:\n"

  force = false
  op.on('-f', '--force', 'overwrite current account setting', TrueClass) {|b|
    force = true
  }

  apikey = op.cmd_parse

  conf = nil
  begin
    conf = Config.read
  rescue ConfigError
  end
  if conf && conf['account.apikey']
    unless force
      if conf['account.user']
        $stderr.puts "Account is already configured with '#{conf['account.user']}' account."
      else
        $stderr.puts "Account is already configured."
      end
      $stderr.puts "Add '-f' option to overwrite."
      exit 0
    end
  end

  conf ||= Config.new
  conf.delete("account.user")
  conf["account.apikey"] = apikey
  conf.save

  $stderr.puts "API key is set."

  $stderr.puts "Use '#{$prog} db:create <db_name>' to create a database."
end

#apikey_show(op) ⇒ Object 



5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
 
# File 'lib/td/command/apikey.rb', line 5

def apikey_show(op)
  if Config.apikey
    puts Config.apikey
    return
  end

  conf = nil
  begin
    conf = Config.read
  rescue ConfigError
  end

  if !conf || !conf['account.apikey']
    $stderr.puts "Account is not configured yet."
    $stderr.puts "Use '#{$prog} apikey:set' or '#{$prog} account' first."
    exit 1
  end

  puts conf['account.apikey']
end

#bulk_import_commit(op) ⇒ Object 



268
269
270
271
272
273
274
275
276
 
# File 'lib/td/command/bulk_import.rb', line 268

def bulk_import_commit(op)
  name = op.cmd_parse

  client = get_client

  job = client.commit_bulk_import(name)

  $stderr.puts "Bulk import session '#{name}' started to commit."
end

#bulk_import_create(op) ⇒ Object 



27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
 
# File 'lib/td/command/bulk_import.rb', line 27

def bulk_import_create(op)
  org = nil

  op.on('-g', '--org ORGANIZATION', "create the bulk import session under this organization") {|s|
    org = s
  }

  name, db_name, table_name = op.cmd_parse

  client = get_client

  table = get_table(client, db_name, table_name)

  opts = {}
  opts['organization'] = org if org
  client.create_bulk_import(name, db_name, table_name, opts)

  $stderr.puts "Bulk import session '#{name}' is created."
end

#bulk_import_delete(op) ⇒ Object 



47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
 
# File 'lib/td/command/bulk_import.rb', line 47

def bulk_import_delete(op)
  name = op.cmd_parse

  client = get_client

  begin
    client.delete_bulk_import(name)
  rescue NotFoundError
    cmd_debug_error $!
    $stderr.puts "Bulk import session '#{name}' does not exist."
    exit 1
  end

  $stderr.puts "Bulk import session '#{name}' is deleted."
end

#bulk_import_delete_part(op) ⇒ Object

obsoleted



194
195
196
197
198
199
200
201
202
 
# File 'lib/td/command/bulk_import.rb', line 194

def bulk_import_delete_part(op)
  name, part_name = op.cmd_parse

  client = get_client

  client.bulk_import_delete_part(name, part_name)

  $stderr.puts "Part '#{part_name}' is deleted."
end

#bulk_import_delete_parts(op) ⇒ Object 



204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
 
# File 'lib/td/command/bulk_import.rb', line 204

def bulk_import_delete_parts(op)
  part_prefix = ""

  op.on('-P', '--prefix NAME', 'add prefix to parts name') {|s|
    part_prefix = s
  }

  name, *part_names = op.cmd_parse

  client = get_client

  part_names.each {|part_name|
    part_name = part_prefix + part_name

    $stderr.puts "Deleting '#{part_name}'..."
    client.bulk_import_delete_part(name, part_name)
  }

  $stderr.puts "done."
end

#bulk_import_error_records(op) ⇒ Object 



278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
 
# File 'lib/td/command/bulk_import.rb', line 278

def bulk_import_error_records(op)
  name = op.cmd_parse

  client = get_client

  bis = client.bulk_imports
  bi = bis.find {|bi| name == bi.name }
  unless bi
    $stderr.puts "Bulk import session '#{name}' does not exist."
    $stderr.puts "Use '#{$prog} bulk_import:create <name> <db> <table>' to create a session."
    exit 1
  end

  if bi.status == "uploading" || bi.status == "performing"
    $stderr.puts "Bulk import session '#{name}' is not performed."
    $stderr.puts "Use '#{$prog} bulk_import:perform <name>' to run."
    exit 1
  end

  require 'yajl'
  client.bulk_import_error_records(name) {|r|
    puts Yajl.dump(r)
  }
end

#bulk_import_freeze(op) ⇒ Object 



303
304
305
306
307
308
309
310
311
 
# File 'lib/td/command/bulk_import.rb', line 303

def bulk_import_freeze(op)
  name = op.cmd_parse

  client = get_client

  client.freeze_bulk_import(name)

  $stderr.puts "Bulk import session '#{name}' is frozen."
end

#bulk_import_list(op) ⇒ Object 



5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
 
# File 'lib/td/command/bulk_import.rb', line 5

def bulk_import_list(op)
  op.cmd_parse

  client = get_client

  bis = client.bulk_imports

  rows = []
  has_org = false
  bis.each {|bi|
    rows << {:Name=>bi.name, :Table=>"#{bi.database}.#{bi.table}", :Status=>bi.status.to_s.capitalize, :Frozen=>bi.upload_frozen? ? 'Frozen' : '', :JobID=>bi.job_id, :"Valid Parts"=>bi.valid_parts, :"Error Parts"=>bi.error_parts, :"Valid Records"=>bi.valid_records, :"Error Records"=>bi.error_records, :Organization=>bi.org_name}
    has_org = true if bi.org_name
  }

  puts cmd_render_table(rows, :fields => gen_table_fields(has_org, [:Name, :Table, :Status, :Frozen, :JobID, :"Valid Parts", :"Error Parts", :"Valid Records", :"Error Records"]), :max_width=>200)

  if rows.empty?
    $stderr.puts "There are no bulk import sessions."
    $stderr.puts "Use '#{$prog} bulk_import:create <name> <db> <table>' to create a session."
  end
end

#bulk_import_perform(op) ⇒ Object 



225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
 
# File 'lib/td/command/bulk_import.rb', line 225

def bulk_import_perform(op)
  wait = false
  force = false

  op.on('-w', '--wait', 'wait for finishing the job', TrueClass) {|b|
    wait = b
  }
  op.on('-f', '--force', 'force start performing', TrueClass) {|b|
    force = b
  }

  name = op.cmd_parse

  client = get_client

  unless force
    bis = client.bulk_imports
    bi = bis.find {|bi| name == bi.name }
    if bi
      if bi.status == 'performing'
        $stderr.puts "Bulk import session '#{name}' is already performing."
        $stderr.puts "Add '-f' option to force start."
        $stderr.puts "Use '#{$prog} job:kill #{bi.job_id}' to cancel the last trial."
        exit 1
      elsif bi.status == 'ready'
        $stderr.puts "Bulk import session '#{name}' is already ready to commit."
        $stderr.puts "Add '-f' option to force start."
        exit 1
      end
    end
  end

  job = client.perform_bulk_import(name)

  $stderr.puts "Job #{job.job_id} is queued."
  $stderr.puts "Use '#{$prog} job:show [-w] #{job.job_id}' to show the status."

  if wait
    require 'td/command/job'  # wait_job
    wait_job(job)
  end
end

#bulk_import_prepare_parts(op) ⇒ Object 



326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
 
# File 'lib/td/command/bulk_import.rb', line 326

def bulk_import_prepare_parts(op)
  outdir = nil
  split_size_kb = PART_SPLIT_SIZE / 1024  # kb

  require 'td/file_reader'
  reader = FileReader.new
  reader.init_optparse(op)

  op.on('-s', '--split-size SIZE_IN_KB', "size of each parts (default: #{split_size_kb})", Integer) {|i|
    split_size_kb = i
  }
  op.on('-o', '--output DIR', 'output directory') {|s|
    outdir = s
  }

  files = op.cmd_parse

  # TODO ruby 1.9
  files = [files] unless files.is_a?(Array)

  unless outdir
    $stderr.puts "-o, --output DIR option is required."
    exit 1
  end

  split_size = split_size_kb * 1024

  require 'fileutils'
  FileUtils.mkdir_p(outdir)

  require 'yajl'
  require 'msgpack'
  require 'zlib'

  error = Proc.new {|reason,data|
    begin
      $stderr.puts "#{reason}: #{Yajl.dump(data)}"
    rescue
      $stderr.puts "#{reason}"
    end
  }

  # TODO multi process
  files.each {|ifname|
    $stderr.puts "Processing #{ifname}..."
    record_num = 0

    basename = File.basename(ifname).sub(/\.(?:csv|tsv|json|msgpack)(?:\.gz)?$/i,'').split('.').join('_')
    File.open(ifname) {|io|
      of_index = 0
      out = nil
      zout = nil
      begin
        reader.parse(io, error) {|record|
          if zout == nil
            ofname = "#{basename}_#{of_index}.msgpack.gz"
            $stderr.puts "  Preparing part \"#{basename}_#{of_index}\"..."
            out = File.open("#{outdir}/#{ofname}", 'wb')
            zout = Zlib::GzipWriter.new(out)

            t = record['time']
            $stderr.puts "  sample: #{Time.at(t).utc} #{Yajl.dump(record)}"
          end

          zout.write(record.to_msgpack)
          record_num += 1

          if out.size > split_size
            zout.close
            of_index += 1
            out = nil
            zout = nil
          end
        }
      ensure
        if zout
          zout.close
          zout = nil
        end
      end
      $stderr.puts "  #{ifname}: #{record_num} entries."
    }
  }
end

#bulk_import_prepare_parts2(op) ⇒ Object 



447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
 
# File 'lib/td/command/bulk_import.rb', line 447

def bulk_import_prepare_parts2(op)
  opts = prepare_parts2_config(op)

  # java command
  javacmd = 'java'

  # make jvm options
  jvm_opts = []
  jvm_opts << "-Xmx1024m" # TODO

  # find java/*.jar and td.jar
  base_path = File.expand_path('../../..', File.dirname(__FILE__)) # TODO
  libjars = Dir.glob("#{base_path}/java/**/*.jar")
  found = libjars.find { |path| File.basename(path) =~ /^td-bulk-import/ }
  td_command_jar = libjars.delete(found)

  # make application options
  app_opts = []
  app_opts << "-cp \"#{td_command_jar}\""

  # make system properties
  sysprops = []
  sysprops.concat(prepare_parts2_sysprops(opts))

  # make application arguments
  app_args = []
  app_args << 'com.treasure_data.tools.BulkImportTool'
  app_args << 'prepare_parts'
  app_args << opts[18]

  # TODO consider parameters including spaces; don't use join(' ')
  command = "#{javacmd} #{jvm_opts.join(' ')} #{app_opts.join(' ')} #{sysprops.join(' ')} #{app_args.join(' ')}"

  exec command
end

#bulk_import_show(op) ⇒ Object 



63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
 
# File 'lib/td/command/bulk_import.rb', line 63

def bulk_import_show(op)
  name = op.cmd_parse

  client = get_client

  bis = client.bulk_imports
  bi = bis.find {|bi| name == bi.name }
  unless bi
    $stderr.puts "Bulk import session '#{name}' does not exist."
    $stderr.puts "Use '#{$prog} bulk_import:create <name> <db> <table>' to create a session."
    exit 1
  end

  $stderr.puts "Organization : #{bi.org_name}"
  $stderr.puts "Name         : #{bi.name}"
  $stderr.puts "Database     : #{bi.database}"
  $stderr.puts "Table        : #{bi.table}"
  $stderr.puts "Status       : #{bi.status.to_s.capitalize}"
  $stderr.puts "Frozen       : #{bi.upload_frozen?}"
  $stderr.puts "JobID        : #{bi.job_id}"
  $stderr.puts "Valid Records: #{bi.valid_records}"
  $stderr.puts "Error Records: #{bi.error_records}"
  $stderr.puts "Valid Parts  : #{bi.valid_parts}"
  $stderr.puts "Error Parts  : #{bi.error_parts}"
  $stderr.puts "Uploaded Parts :"

  list = client.list_bulk_import_parts(name)
  list.each {|name|
    puts "  #{name}"
  }
end

#bulk_import_unfreeze(op) ⇒ Object 



313
314
315
316
317
318
319
320
321
 
# File 'lib/td/command/bulk_import.rb', line 313

def bulk_import_unfreeze(op)
  name = op.cmd_parse

  client = get_client

  client.unfreeze_bulk_import(name)

  $stderr.puts "Bulk import session '#{name}' is unfrozen."
end

#bulk_import_upload_part(op) ⇒ Object

obsoleted



96
97
98
99
100
101
102
103
104
105
106
107
 
# File 'lib/td/command/bulk_import.rb', line 96

def bulk_import_upload_part(op)
  retry_limit = 10
  retry_wait = 1

  name, part_name, path = op.cmd_parse

  File.open(path, "rb") {|io|
    bulk_import_upload_impl(name, part_name, io, io.size, retry_limit, retry_wait)
  }

  $stderr.puts "Part '#{part_name}' is uploaded."
end

#bulk_import_upload_parts(op) ⇒ Object 



109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
 
# File 'lib/td/command/bulk_import.rb', line 109

def bulk_import_upload_parts(op)
  retry_limit = 10
  retry_wait = 1
  suffix_count = 0
  part_prefix = ""
  auto_perform = false
  parallel = 2

  op.on('-P', '--prefix NAME', 'add prefix to parts name') {|s|
    part_prefix = s
  }
  op.on('-s', '--use-suffix COUNT', 'use COUNT number of . (dots) in the source file name to the parts name', Integer) {|i|
    suffix_count = i
  }
  op.on('--auto-perform', 'perform bulk import job automatically', TrueClass) {|b|
    auto_perform = b
  }
  op.on('--parallel NUM', 'perform uploading in parallel (default: 2; max 8)', Integer) {|i|
    parallel = i
  }

  name, *files = op.cmd_parse

  parallel = 1 if parallel <= 1
  parallel = 8 if parallel >= 8

  threads = (1..parallel).map {|i|
    Thread.new do
      errors = []
      until files.empty?
        ifname = files.shift
        basename = File.basename(ifname)
        begin
          part_name = part_prefix + basename.split('.')[0..suffix_count].join('.')

          File.open(ifname, "rb") {|io|
            size = io.size
            $stderr.write "Uploading '#{ifname}' -> '#{part_name}'... (#{size} bytes)\n"

            bulk_import_upload_impl(name, part_name, io, size, retry_limit, retry_wait)
          }
        rescue
          errors << [ifname, $!]
        end
      end
      errors
    end
  }

  errors = []
  threads.each {|t|
    errors.concat t.value
  }

  unless errors.empty?
    $stderr.puts "failed to upload #{errors.size} files."
    $stderr.puts "backtraces:"
    errors.each {|(ifname,ex)|
      $stderr.puts "  #{ifname}: #{ex}"
      ex.backtrace.each {|bt|
        $stderr.puts "      #{ifname}: #{bt}"
      }
    }
    $stderr.puts "files:"
    ifnames = errors.map {|(ifname,ex)| ifname }
    ifnames.each {|ifname|
      $stderr.puts "  #{ifname}"
    }
    $stderr.puts "You can retry uploading by following command:"
    $stderr.puts "td bulk_import:upload_parts #{name} #{ifnames.map {|ifname| "'#{ifname}'" }.join(' ')}"
    exit 1
  end

  $stderr.puts "done."

  if auto_perform
    client = get_client
    job = client.perform_bulk_import(name)

    $stderr.puts "Job #{job.job_id} is queued."
    $stderr.puts "Use '#{$prog} job:show [-w] #{job.job_id}' to show the status."
  end
end

#bulk_import_upload_parts2(op) ⇒ Object 



411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
 
# File 'lib/td/command/bulk_import.rb', line 411

def bulk_import_upload_parts2(op)
  opts = upload_parts2_config(op)

  # java command
  javacmd = 'java'

  # make jvm options
  jvm_opts = []
  jvm_opts << "-Xmx1024m" # TODO

  # find java/*.jar and td.jar
  base_path = File.expand_path('../../..', File.dirname(__FILE__)) # TODO
  libjars = Dir.glob("#{base_path}/java/**/*.jar")
  found = libjars.find { |path| File.basename(path) =~ /^td-bulk-import/ }
  td_command_jar = libjars.delete(found)

  # make application options
  app_opts = []
  app_opts << "-cp \"#{td_command_jar}\""

  # make system properties
  sysprops = []
  sysprops.concat(upload_parts2_sysprops(opts))

  # make application arguments
  app_args = []
  app_args << 'com.treasure_data.tools.BulkImportTool'
  app_args << 'upload_parts'
  app_args << opts[21]

  # TODO consider parameters including spaces; don't use join(' ')
  command = "#{javacmd} #{jvm_opts.join(' ')} #{app_opts.join(' ')} #{sysprops.join(' ')} #{app_args.join(' ')}"

  exec command
end

#db_create(op) ⇒ Object 



46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
 
# File 'lib/td/command/db.rb', line 46

def db_create(op)
  org = nil

  op.on('-g', '--org ORGANIZATION', "create the database under this organization") {|s|
    org = s
  }

  db_name = op.cmd_parse

  API.validate_database_name(db_name)

  client = get_client

  opts = {}
  opts['organization'] = org if org
  begin
    client.create_database(db_name, opts)
  rescue AlreadyExistsError
    $stderr.puts "Database '#{db_name}' already exists."
    exit 1
  end

  $stderr.puts "Database '#{db_name}' is created."
  $stderr.puts "Use '#{$prog} table:create #{db_name} <table_name>' to create a table."
end

#db_delete(op) ⇒ Object 



72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
 
# File 'lib/td/command/db.rb', line 72

def db_delete(op)
  force = false
  op.on('-f', '--force', 'clear tables and delete the database', TrueClass) {|b|
    force = true
  }

  db_name = op.cmd_parse

  client = get_client

  begin
    db = client.database(db_name)

    if !force && !db.tables.empty?
      $stderr.puts "Database '#{db_name}' is not empty. Use '-f' option or drop tables first."
      exit 1
    end

    db.delete
  rescue NotFoundError
    $stderr.puts "Database '#{db_name}' does not exist."
    exit 1
  end

  $stderr.puts "Database '#{db_name}' is deleted."
end

#db_list(op) ⇒ Object 



26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
 
# File 'lib/td/command/db.rb', line 26

def db_list(op)
  op.cmd_parse

  client = get_client
  dbs = client.databases

  rows = []
  has_org = false
  dbs.each {|db|
    rows << {:Name=>db.name, :Count=>db.count, :Organization=>db.org_name}
    has_org = true if db.org_name
  }
  puts cmd_render_table(rows, :fields => gen_table_fields(has_org, [:Name, :Count]))

  if dbs.empty?
    $stderr.puts "There are no databases."
    $stderr.puts "Use '#{$prog} db:create <db_name>' to create a database."
  end
end

#db_show(op) ⇒ Object 



5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
 
# File 'lib/td/command/db.rb', line 5

def db_show(op)
  db_name = op.cmd_parse

  client = get_client

  db = get_database(client, db_name)

  rows = []
  db.tables.each {|table|
    pschema = table.schema.fields.map {|f|
      "#{f.name}:#{f.type}"
    }.join(', ')
    rows << {:Table => table.name, :Type => table.type.to_s, :Count => table.count.to_s, :Schema=>pschema.to_s}
  }
  rows = rows.sort_by {|map|
    [map[:Type].size, map[:Table]]
  }

  puts cmd_render_table(rows, :fields => [:Table, :Type, :Count, :Schema])
end

#help(op) ⇒ Object 



5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
 
# File 'lib/td/command/help.rb', line 5

def help(op)
  cmd = op.cmd_parse

  c = List.get_option(cmd)
  if c == nil
     $stderr.puts "'#{cmd}' is not a td command. Run '#{$prog}' to show the list."
     List.show_guess(cmd)
     exit 1

  elsif c.name != cmd && c.group == cmd
    # group command
    puts List.cmd_usage(cmd)
    exit 1

  else
    method = List.get_method(cmd)
    method.call(['--help'])
  end
end

#help_all(op) ⇒ Object 



25
26
27
28
29
30
31
 
# File 'lib/td/command/help.rb', line 25

def help_all(op)
  cmd = op.cmd_parse

  TreasureData::Command::List.show_help(op.summary_indent)
  puts ""
  puts "Type '#{$prog} help COMMAND' for more information on a specific command."
end

#ip_limit_delete(op) ⇒ Object 



44
45
46
47
48
49
50
51
 
# File 'lib/td/command/ip_limit.rb', line 44

def ip_limit_delete(op)
  organization = op.cmd_parse

  client = get_client
  client.delete_ip_limit(organization)

  $stderr.puts "All IP range limitations are deleted from #{organization}"
end

#ip_limit_list(op) ⇒ Object 



4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
 
# File 'lib/td/command/ip_limit.rb', line 4

def ip_limit_list(op)
  op.cmd_parse

  client = get_client

  ip_limits = client.ip_limits
  rows = ip_limits.map { |ip_limit|
    {:Organization => ip_limit.org, 'IP Range' => ip_limit.ip_range}
  }

  puts cmd_render_table(rows, :fields => [:Organization, 'IP Range'])

  if rows.empty?
    $stderr.puts "There are no IP range limitations."
    $stderr.puts "Use '#{$prog} ip_limit:set <organization> <ip_range>' to create IP range limitation."
  end
end

#ip_limit_set(op) ⇒ Object 



35
36
37
38
39
40
41
42
 
# File 'lib/td/command/ip_limit.rb', line 35

def ip_limit_set(op)
  organization, *ip_ranges = op.cmd_parse

  client = get_client
  client.set_ip_limit(organization, ip_ranges)

  $stderr.puts "IP range limitations [#{ip_ranges.join(' ')}] are set to #{organization}"
end

#ip_limit_show(op) ⇒ Object 



22
23
24
25
26
27
28
29
30
31
32
33
 
# File 'lib/td/command/ip_limit.rb', line 22

def ip_limit_show(op)
  organization = op.cmd_parse

  client = get_client

  ip_limits = client.ip_limits
  rows = ip_limits.select { |ip_limit|
    ip_limit.org == organization
  }.map { |ip_limit| {'IP Range' => ip_limit.ip_range} }

  puts cmd_render_table(rows, :fields => ['IP Range'])
end

#job_kill(op) ⇒ Object 



164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
 
# File 'lib/td/command/job.rb', line 164

def job_kill(op)
  job_id = op.cmd_parse

  client = get_client

  former_status = client.kill(job_id)
  if TreasureData::Job::FINISHED_STATUS.include?(former_status)
    $stderr.puts "Job #{job_id} is already finished (#{former_status})"
    exit 0
  end

  if former_status == TreasureData::Job::STATUS_RUNNING
    $stderr.puts "Job #{job_id} is killed."
  else
    $stderr.puts "Job #{job_id} is canceled."
  end
end

#job_list(op) ⇒ Object 



29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
 
# File 'lib/td/command/job.rb', line 29

def job_list(op)
  page = 0
  skip = 0
  status = nil
  slower_than = nil

  op.on('-p', '--page PAGE', 'skip N pages', Integer) {|i|
    page = i
  }
  op.on('-s', '--skip N', 'skip N jobs', Integer) {|i|
    skip = i
  }
  op.on('-R', '--running', 'show only running jobs', TrueClass) {|b|
    status = 'running'
  }
  op.on('-S', '--success', 'show only succeeded jobs', TrueClass) {|b|
    status = 'success'
  }
  op.on('-E', '--error', 'show only failed jobs', TrueClass) {|b|
    status = 'error'
  }
  op.on('--slow [SECONDS]', 'show slow queries (default threshold: 3600 seconds)', Integer) { |i|
    slower_than = i || 3600
  }

  max = op.cmd_parse

  max = (max || 20).to_i

  client = get_client

  if page
    skip += max * page
  end

  conditions = nil
  if slower_than
    conditions = {:slower_than => slower_than}
  end

  jobs = client.jobs(skip, skip+max-1, status, conditions)

  rows = []
  has_org = false
  jobs.each {|job|
    start = job.start_at
    elapsed = cmd_format_elapsed(start, job.end_at)
    priority = job_priority_name_of(job.priority)
    rows << {:JobID => job.job_id, :Database => job.db_name, :Status => job.status, :Type => job.type, :Query => job.query.to_s, :Start => (start ? start.localtime : ''), :Elapsed => elapsed, :Priority => priority, :Result => job.result_url, :Organization => job.org_name}
    has_org = true if job.org_name
  }

  puts cmd_render_table(rows, :fields => gen_table_fields(has_org, [:JobID, :Status, :Start, :Elapsed, :Priority, :Result, :Type, :Database, :Query]), :max_width => 140)
end

#job_show(op) ⇒ Object 



84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
 
# File 'lib/td/command/job.rb', line 84

def job_show(op)
  verbose = nil
  wait = false
  output = nil
  format = 'tsv'
  render_opts = {}

  op.on('-v', '--verbose', 'show logs', TrueClass) {|b|
    verbose = b
  }
  op.on('-w', '--wait', 'wait for finishing the job', TrueClass) {|b|
    wait = b
  }
  op.on('-G', '--vertical', 'use vertical table to show results', TrueClass) {|b|
    render_opts[:vertical] = b
  }
  op.on('-o', '--output PATH', 'write result to the file') {|s|
    output = s
  }
  op.on('-f', '--format FORMAT', 'format of the result to write to the file (tsv, csv, json or msgpack)') {|s|
    unless ['tsv', 'csv', 'json', 'msgpack', 'msgpack.gz'].include?(s)
      raise "Unknown format #{s.dump}. Supported format: tsv, csv, json, msgpack, msgpack.gz"
    end
    format = s
  }

  job_id = op.cmd_parse

  client = get_client

  job = client.job(job_id)

  puts "Organization : #{job.org_name}"
  puts "JobID        : #{job.job_id}"
  #puts "URL          : #{job.url}"
  puts "Status       : #{job.status}"
  puts "Type         : #{job.type}"
  puts "Priority     : #{job_priority_name_of(job.priority)}"
  puts "Retry limit  : #{job.retry_limit}"
  puts "Result       : #{job.result_url}"
  puts "Database     : #{job.db_name}"
  puts "Query        : #{job.query}"

  if wait && !job.finished?
    wait_job(job)
    if job.success? && job.type == :hive
      puts "Result       :"
      show_result(job, output, format, render_opts)
    end

  else
    if job.success? && job.type == :hive
      puts "Result       :"
      show_result(job, output, format, render_opts)
    end

    if verbose
      puts ""
      puts "cmdout:"
      job.debug['cmdout'].to_s.split("\n").each {|line|
        puts "  "+line
      }
      puts ""
      puts "stderr:"
      job.debug['stderr'].to_s.split("\n").each {|line|
        puts "  "+line
      }
    end
  end

  $stderr.puts "Use '-v' option to show detailed messages." unless verbose
end

#job_status(op) ⇒ Object 



157
158
159
160
161
162
 
# File 'lib/td/command/job.rb', line 157

def job_status(op)
  job_id = op.cmd_parse
  client = get_client

  puts client.job_status(job_id)
end

#org_create(op) ⇒ Object 



41
42
43
44
45
46
47
48
49
 
# File 'lib/td/command/org.rb', line 41

def org_create(op)
  name = op.cmd_parse

  client = get_client

  client.create_organization(name)

  $stderr.puts "Organization '#{name}' is created."
end

#org_delete(op) ⇒ Object 



51
52
53
54
55
56
57
58
59
 
# File 'lib/td/command/org.rb', line 51

def org_delete(op)
  name = op.cmd_parse

  client = get_client

  client.delete_organization(name)

  $stderr.puts "Organization '#{name}' is deleted."
end

#org_list(op) ⇒ Object 



21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
 
# File 'lib/td/command/org.rb', line 21

def org_list(op)
  op.cmd_parse

  client = get_client

  orgs = client.organizations

  rows = []
  orgs.each {|org|
    rows << {:Name => org.name}
  }

  puts cmd_render_table(rows, :fields => [:Name])

  if rows.empty?
    $stderr.puts "There are no organizations."
    $stderr.puts "Use '#{$prog} org:create <name>' to create an organization."
  end
end

#org_show(op) ⇒ Object 



5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
 
# File 'lib/td/command/org.rb', line 5

def org_show(op)
  name = op.cmd_parse

  client = get_client

  orgs = client.organizations
  org = orgs.find {|org| name == org.name }
  unless org
    $stderr.puts "Organization '#{name}' does not exist."
    $stderr.puts "Use '#{$prog} org:create <name>' to create an organization."
    exit 1
  end

  $stderr.puts "Name         : #{org.name}"
end

#password_change(op) ⇒ Object 



5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
 
# File 'lib/td/command/password.rb', line 5

def password_change(op)
  op.cmd_parse

  old_password = nil
  password = nil

  begin
    system "stty -echo"  # TODO termios
    print "Old password (typing will be hidden): "
    old_password = STDIN.gets || ""
    old_password = old_password[0..-2]  # strip \n
  rescue Interrupt
    $stderr.print "\ncanceled."
    exit 1
  ensure
    system "stty echo"   # TODO termios
    print "\n"
  end

  if old_password.empty?
    $stderr.puts "canceled."
    exit 0
  end

  3.times do
    begin
      system "stty -echo"  # TODO termios
      print "New password (typing will be hidden): "
      password = STDIN.gets || ""
      password = password[0..-2]  # strip \n
    rescue Interrupt
      $stderr.print "\ncanceled."
      exit 1
    ensure
      system "stty echo"   # TODO termios
      print "\n"
    end

    if password.empty?
      $stderr.puts "canceled."
      exit 0
    end

    begin
      system "stty -echo"  # TODO termios
      print "Retype new password: "
      password2 = STDIN.gets || ""
      password2 = password2[0..-2]  # strip \n
    rescue Interrupt
      $stderr.print "\ncanceled."
      exit 1
    ensure
      system "stty echo"   # TODO termios
      print "\n"
    end

    if password == password2
      break
    end

    puts "Doesn't match."
  end

  client = get_client(:ssl => true)

  client.change_my_password(old_password, password)

  $stderr.puts "Password changed."
end

#query(op) ⇒ Object 



5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
 
# File 'lib/td/command/query.rb', line 5

def query(op)
  org = nil
  db_name = nil
  wait = false
  output = nil
  format = 'tsv'
  render_opts = {}
  result_url = nil
  result_user = nil
  result_ask_password = false
  priority = nil
  retry_limit = nil
  query = nil
  sampling_all = nil

  op.on('-g', '--org ORGANIZATION', "issue the query under this organization") {|s|
    org = s
  }
  op.on('-d', '--database DB_NAME', 'use the database (required)') {|s|
    db_name = s
  }
  op.on('-w', '--wait', 'wait for finishing the job', TrueClass) {|b|
    wait = b
  }
  op.on('-G', '--vertical', 'use vertical table to show results', TrueClass) {|b|
    render_opts[:vertical] = b
  }
  op.on('-o', '--output PATH', 'write result to the file') {|s|
    output = s
  }
  op.on('-f', '--format FORMAT', 'format of the result to write to the file (tsv, csv, json or msgpack)') {|s|
    unless ['tsv', 'csv', 'json', 'msgpack'].include?(s)
      raise "Unknown format #{s.dump}. Supported format: tsv, csv, json, msgpack"
    end
    format = s
  }
  op.on('-r', '--result RESULT_URL', 'write result to the URL (see also result:create subcommand)') {|s|
    result_url = s
  }
  op.on('-u', '--user NAME', 'set user name for the result URL') {|s|
    result_user = s
  }
  op.on('-p', '--password', 'ask password for the result URL') {|s|
    result_ask_password = true
  }
  op.on('-P', '--priority PRIORITY', 'set priority') {|s|
    priority = job_priority_id_of(s)
    unless priority
      raise "unknown priority #{s.inspect} should be -2 (very-low), -1 (low), 0 (normal), 1 (high) or 2 (very-high)"
    end
  }
  op.on('-R', '--retry COUNT', 'automatic retrying count', Integer) {|i|
    retry_limit = i
  }
  op.on('-q', '--query PATH', 'use file instead of inline query') {|s|
    query = File.open(s) { |f| f.read.strip }
  }
  op.on('--sampling DENOMINATOR', 'enable random sampling to reduce records 1/DENOMINATOR', Integer) {|i|
    sampling_all = i
  }

  sql = op.cmd_parse

  unless db_name
    $stderr.puts "-d, --database DB_NAME option is required."
    exit 1
  end

  if sql == '-'
    sql = STDIN.read
  elsif sql.nil?
    sql = query
  end

  unless sql
    $stderr.puts "<sql> argument or -q,--query PATH option is required."
    exit 1
  end

  if result_url
    require 'td/command/result'
    result_url = build_result_url(result_url, result_user, result_ask_password)
  end

  client = get_client

  # local existance check
  get_database(client, db_name)

  opts = {}
  opts['organization'] = org if org
  opts['sampling_all'] = sampling_all if sampling_all
  job = client.query(db_name, sql, result_url, priority, retry_limit, opts)

  $stderr.puts "Job #{job.job_id} is queued."
  $stderr.puts "Use '#{$prog} job:show #{job.job_id}' to show the status."
  #$stderr.puts "See #{job.url} to see the progress."

  if wait && !job.finished?
    wait_job(job)
    puts "Status     : #{job.status}"
    if job.success?
      puts "Result     :"
      show_result(job, output, format, render_opts)
    end
  end
end

#result_create(op) ⇒ Object 



49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
 
# File 'lib/td/command/result.rb', line 49

def result_create(op)
  org = nil
  result_user = nil
  result_ask_password = false

  op.on('-g', '--org ORGANIZATION', "create the result under this organization") {|s|
    org = s
  }
  op.on('-u', '--user NAME', 'set user name for authentication') {|s|
    result_user = s
  }
  op.on('-p', '--password', 'ask password for authentication') {|s|
    result_ask_password = true
  }

  name, url = op.cmd_parse

  API.validate_database_name(name)

  client = get_client

  url = build_result_url(url, result_user, result_ask_password)

  opts = {}
  opts['organization'] = org if org
  begin
    client.create_result(name, url, opts)
  rescue AlreadyExistsError
    $stderr.puts "Result URL '#{name}' already exists."
    exit 1
  end

  $stderr.puts "Result URL '#{name}' is created."
end

#result_delete(op) ⇒ Object 



84
85
86
87
88
89
90
91
92
93
94
95
96
97
 
# File 'lib/td/command/result.rb', line 84

def result_delete(op)
  name = op.cmd_parse

  client = get_client

  begin
    client.delete_result(name)
  rescue NotFoundError
    $stderr.puts "Result URL '#{name}' does not exist."
    exit 1
  end

  $stderr.puts "Result URL '#{name}' is deleted."
end

#result_list(op) ⇒ Object 



24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
 
# File 'lib/td/command/result.rb', line 24

def result_list(op)
  op.cmd_parse

  client = get_client

  rs = client.results

  rows = []
  has_org = false
  rs.each {|r|
    rows << {:Name => r.name, :URL => r.url, :Organization => r.org_name}
    has_org = true if r.org_name
  }
  rows = rows.sort_by {|map|
    map[:Name]
  }

  puts cmd_render_table(rows, :fields => gen_table_fields(has_org, [:Name, :URL]))

  if rs.empty?
    $stderr.puts "There are no result URLs."
    $stderr.puts "Use '#{$prog} result:create <name> <url>' to create a result URL."
  end
end

#result_show(op) ⇒ Object 



5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
 
# File 'lib/td/command/result.rb', line 5

def result_show(op)
  name = op.cmd_parse

  client = get_client

  rs = client.results
  r = rs.find {|r| name == r.name }

  unless r
    $stderr.puts "Result URL '#{name}' does not exist."
    $stderr.puts "Use '#{$prog} result:create #{name} <URL>' to create the URL."
    exit 1
  end

  puts "Organization : #{r.org_name}"
  puts "Name         : #{r.name}"
  puts "URL          : #{r.url}"
end

#role_create(op) ⇒ Object 



43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
 
# File 'lib/td/command/role.rb', line 43

def role_create(op)
  org = nil

  op.on('-g', '--org ORGANIZATION', "create the role under this organization") {|s|
    org = s
  }

  name = op.cmd_parse

  client = get_client

  client.create_role(name, org)

  $stderr.puts "Role '#{name}' is created."
end

#role_delete(op) ⇒ Object 



59
60
61
62
63
64
65
66
67
 
# File 'lib/td/command/role.rb', line 59

def role_delete(op)
  name = op.cmd_parse

  client = get_client

  client.delete_role(name)

  $stderr.puts "Role '#{name}' is deleted."
end

#role_grant(op) ⇒ Object 



69
70
71
72
73
74
75
76
77
 
# File 'lib/td/command/role.rb', line 69

def role_grant(op)
  name, user = op.cmd_parse

  client = get_client

  client.grant_role(name, user)

  $stderr.puts "Role '#{name}' is granted to user '#{user}'."
end

#role_list(op) ⇒ Object 



23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
 
# File 'lib/td/command/role.rb', line 23

def role_list(op)
  op.cmd_parse

  client = get_client

  roles = client.roles

  rows = []
  roles.each {|role|
    rows << {:Name => role.name, :Organization => role.org_name, :Users => role.user_names.join(',')}
  }

  puts cmd_render_table(rows, :fields => [:Name, :Organization, :Users])

  if rows.empty?
    $stderr.puts "There are no roles."
    $stderr.puts "Use '#{$prog} org:create <name>' to create a role."
  end
end

#role_revoke(op) ⇒ Object 



79
80
81
82
83
84
85
86
87
 
# File 'lib/td/command/role.rb', line 79

def role_revoke(op)
  name, user = op.cmd_parse

  client = get_client

  client.revoke_role(name, user)

  $stderr.puts "Role '#{name}' is revoked from user '#{user}'."
end

#role_show(op) ⇒ Object 



5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
 
# File 'lib/td/command/role.rb', line 5

def role_show(op)
  name = op.cmd_parse

  client = get_client

  roles = client.roles
  role = roles.find {|role| name == role.name }
  unless role
    $stderr.puts "Role '#{name}' does not exist."
    $stderr.puts "Use '#{$prog} role:create <name>' to create a role."
    exit 1
  end

  $stderr.puts "Organization : #{role.org_name}"
  $stderr.puts "Name         : #{role.name}"
  $stderr.puts "Users        : #{role.user_names.join(', ')}"
end

#sample_apache(op) ⇒ Object 



5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
 
# File 'lib/td/command/sample.rb', line 5

def sample_apache(op)
  fname = op.cmd_parse

  require 'json'

  t = Time.now.to_i
  i = 0
  last_time = nil

  data = File.join(File.dirname(__FILE__), '../../../data/sample_apache.json')
  File.open(data) {|df|
    File.open(fname, 'w') {|of|
      df.each_line {|line|
        record = JSON.parse(line)
        record['time'] = last_time = (t - (i**1.3)).to_i
        of.puts record.to_json
        i += 1
      }
    }
  }

  $stderr.puts "Create #{fname} with #{i} records whose time is"
  $stderr.puts "from #{Time.at(last_time)} to #{Time.at(t)}."
  $stderr.puts "Use '#{$prog} table:import <db> <table> --json #{fname}' to import this file."
end

#sched_create(op) ⇒ Object 



27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
 
# File 'lib/td/command/sched.rb', line 27

def sched_create(op)
  org = nil
  db_name = nil
  timezone = nil
  delay = 0
  result_url = nil
  result_user = nil
  result_ask_password = false
  priority = nil
  retry_limit = nil

  op.on('-g', '--org ORGANIZATION', "create the schedule under this organization") {|s|
    org = s
  }
  op.on('-d', '--database DB_NAME', 'use the database (required)') {|s|
    db_name = s
  }
  op.on('-t', '--timezone TZ', 'name of the timezone (like Asia/Tokyo)') {|s|
    timezone = s
  }
  op.on('-D', '--delay SECONDS', 'delay time of the schedule', Integer) {|i|
    delay = i
  }
  op.on('-r', '--result RESULT_URL', 'write result to the URL (see also result:create subcommand)') {|s|
    result_url = s
  }
  op.on('-u', '--user NAME', 'set user name for the result URL') {|s|
    result_user = s
  }
  op.on('-p', '--password', 'ask password for the result URL') {|s|
    result_ask_password = true
  }
  op.on('-P', '--priority PRIORITY', 'set priority') {|s|
    priority = job_priority_id_of(s)
    unless priority
      raise "unknown priority #{s.inspect} should be -2 (very-low), -1 (low), 0 (normal), 1 (high) or 2 (very-high)"
    end
  }
  op.on('-R', '--retry COUNT', 'automatic retrying count', Integer) {|i|
    retry_limit = i
  }

  name, cron, sql = op.cmd_parse

  unless db_name
    $stderr.puts "-d, --database DB_NAME option is required."
    exit 1
  end

  if result_url
    require 'td/command/result'
    result_url = build_result_url(result_url, result_user, result_ask_password)
  end

  client = get_client

  # local existance check
  get_database(client, db_name)

  begin
    first_time = client.create_schedule(name, :cron=>cron, :query=>sql, :database=>db_name, :result=>result_url, :timezone=>timezone, :delay=>delay, :priority=>priority, :retry_limit=>retry_limit, :organization=>org)
  rescue AlreadyExistsError
    cmd_debug_error $!
    $stderr.puts "Schedule '#{name}' already exists."
    exit 1
  end

  $stderr.puts "Schedule '#{name}' is created. It starts at #{first_time.localtime}."
end

#sched_delete(op) ⇒ Object 



97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
 
# File 'lib/td/command/sched.rb', line 97

def sched_delete(op)
  name = op.cmd_parse

  client = get_client

  begin
    client.delete_schedule(name)
  rescue NotFoundError
    cmd_debug_error $!
    $stderr.puts "Schedule '#{name}' does not exist."
    $stderr.puts "Use '#{$prog} sched:list' to show list of the schedules."
    exit 1
  end

  $stderr.puts "Schedule '#{name}' is deleted."
end

#sched_history(op) ⇒ Object 



184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
 
# File 'lib/td/command/sched.rb', line 184

def sched_history(op)
  require 'td/command/job'  # job_priority_name_of

  page = 0
  skip = 0

  op.on('-p', '--page PAGE', 'skip N pages', Integer) {|i|
    page = i
  }
  op.on('-s', '--skip N', 'skip N schedules', Integer) {|i|
    skip = i
  }

  name, max = op.cmd_parse

  max = (max || 20).to_i

  if page
    skip += max * page
  end

  client = get_client

  begin
    history = client.history(name, skip, skip+max-1)
  rescue NotFoundError
    cmd_debug_error $!
    $stderr.puts "Schedule '#{name}' does not exist."
    $stderr.puts "Use '#{$prog} sched:list' to show list of the schedules."
    exit 1
  end

  scheds = client.schedules
  if s = scheds.find {|s| s.name == name }
    puts "Organization : #{s.org_name}"
    puts "Name         : #{s.name}"
    puts "Cron         : #{s.cron}"
    puts "Timezone     : #{s.timezone}"
    puts "Delay        : #{s.delay} sec"
    puts "Next         : #{s.next_time}"
    puts "Result       : #{s.result_url}"
    puts "Priority     : #{job_priority_name_of(s.priority)}"
    puts "Retry limit  : #{s.retry_limit}"
    puts "Database     : #{s.database}"
    puts "Query        : #{s.query}"
  end

  rows = []
  history.each {|j|
    rows << {:Time => j.scheduled_at.localtime, :JobID => j.job_id, :Status => j.status, :Priority => job_priority_name_of(j.priority), :Result=>j.result_url}
  }

  puts cmd_render_table(rows, :fields => [:JobID, :Time, :Status, :Priority, :Result])
end

#sched_list(op) ⇒ Object 



5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
 
# File 'lib/td/command/sched.rb', line 5

def sched_list(op)
  require 'td/command/job'  # job_priority_name_of

  op.cmd_parse

  client = get_client

  scheds = client.schedules

  rows = []
  has_org = false
  scheds.each {|sched|
    rows << {:Name => sched.name, :Cron => sched.cron, :Timezone => sched.timezone, :Delay => sched.delay, :Priority => job_priority_name_of(sched.priority), :Result => sched.result_url, :Database => sched.database, :Query => sched.query, :"Next schedule" => sched.next_time ? sched.next_time.localtime : nil, :Organization => sched.org_name }
    has_org = true if sched.org_name
  }
  rows = rows.sort_by {|map|
    map[:Name]
  }

  puts cmd_render_table(rows, :fields => gen_table_fields(has_org, [:Name, :Cron, :Timezone, :"Next schedule", :Delay, :Priority, :Result, :Database, :Query]), :max_width=>500)
end

#sched_run(op) ⇒ Object 



239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
 
# File 'lib/td/command/sched.rb', line 239

def sched_run(op)
  num = 1

  op.on('-n', '--num N', 'number of jobs to run', Integer) {|i|
    num = i
  }

  name, time = op.cmd_parse

  if time.to_i.to_s == time.to_s
    # UNIX time
    t = Time.at(time.to_i)
  else
    require 'time'
    begin
      t = Time.parse(time)
    rescue
      $stderr.puts "invalid time format: #{time}"
      exit 1
    end
  end

  client = get_client

  begin
    jobs = client.run_schedule(name, t.to_i, num)
  rescue NotFoundError
    cmd_debug_error $!
    $stderr.puts "Schedule '#{name}' does not exist."
    $stderr.puts "Use '#{$prog} sched:list' to show list of the schedules."
    exit 1
  end

  rows = []
  jobs.each_with_index {|job,i|
    rows << {:JobID => job.job_id, :Time => job.scheduled_at ? job.scheduled_at.localtime : nil}
  }

  $stderr.puts "Scheduled #{num} jobs from #{t}."
  puts cmd_render_table(rows, :fields => [:JobID, :Time], :max_width=>500)
end

#sched_update(op) ⇒ Object 



114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
 
# File 'lib/td/command/sched.rb', line 114

def sched_update(op)
  cron = nil
  sql = nil
  db_name = nil
  result = nil
  timezone = nil
  delay = nil
  priority = nil
  retry_limit = nil

  op.on('-s', '--schedule CRON', 'change the schedule') {|s|
    cron = s
  }
  op.on('-q', '--query SQL', 'change the query') {|s|
    sql = s
  }
  op.on('-d', '--database DB_NAME', 'change the database') {|s|
    db_name = s
  }
  op.on('-r', '--result RESULT_TABLE', 'change the result table') {|s|
    result = s
  }
  op.on('-t', '--timezone TZ', 'change the name of the timezone (like Asia/Tokyo)') {|s|
    timezone = s
  }
  op.on('-D', '--delay SECONDS', 'change the delay time of the schedule', Integer) {|i|
    delay = i
  }
  op.on('-P', '--priority PRIORITY', 'set priority') {|s|
    priority = job_priority_id_of(s)
    unless priority
      raise "unknown priority #{s.inspect} should be -2 (very-low), -1 (low), 0 (normal), 1 (high) or 2 (very-high)"
    end
  }
  op.on('-R', '--retry COUNT', 'automatic retrying count', Integer) {|i|
    retry_limit = i
  }


  name = op.cmd_parse

  params = {}
  params['cron'] = cron if cron
  params['query'] = sql if sql
  params['database'] = db_name if db_name
  params['result'] = result if result
  params['timezone'] = timezone if timezone
  params['delay'] = delay.to_s if delay
  params['priority'] = priority.to_s if priority
  params['retry_limit'] = retry_limit.to_s if retry_limit

  if params.empty?
    $stderr.puts op.to_s
    exit 1
  end

  client = get_client

  begin
    client.update_schedule(name, params)
  rescue NotFoundError
    cmd_debug_error $!
    $stderr.puts "Schedule '#{name}' does not exist."
    $stderr.puts "Use '#{$prog} sched:list' to show list of the schedules."
    exit 1
  end

  $stderr.puts "Schedule '#{name}' is updated."
end

#schema_add(op) ⇒ Object 



30
31
32
33
34
35
36
37
38
39
40
41
42
 
# File 'lib/td/command/schema.rb', line 30

def schema_add(op)
  db_name, table_name, *columns = op.cmd_parse
  schema = parse_columns(columns)

  client = get_client
  table = get_table(client, db_name, table_name)

  schema = table.schema.merge(schema)

  client.update_schema(table.db_name, table.name, schema)

  $stderr.puts "Schema is updated on #{db_name}.#{table_name} table."
end

#schema_remove(op) ⇒ Object 



44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
 
# File 'lib/td/command/schema.rb', line 44

def schema_remove(op)
  db_name, table_name, *columns = op.cmd_parse

  client = get_client
  table = get_table(client, db_name, table_name)

  schema = table.schema

  columns.each {|col|
    deleted = false
    schema.fields.delete_if {|f|
      f.name == col && deleted = true
    }
    unless deleted
      $stderr.puts "Column name '#{col}' does not exist."
      exit 1
    end
  }

  client.update_schema(table.db_name, table.name, schema)

  $stderr.puts "Schema is updated on #{db_name}.#{table_name} table."
end

#schema_set(op) ⇒ Object 



18
19
20
21
22
23
24
25
26
27
28
 
# File 'lib/td/command/schema.rb', line 18

def schema_set(op)
  db_name, table_name, *columns = op.cmd_parse
  schema = parse_columns(columns)

  client = get_client
  table = get_table(client, db_name, table_name)

  client.update_schema(table.db_name, table.name, schema)

  $stderr.puts "Schema is updated on #{db_name}.#{table_name} table."
end

#schema_show(op) ⇒ Object 



5
6
7
8
9
10
11
12
13
14
15
16
 
# File 'lib/td/command/schema.rb', line 5

def schema_show(op)
  db_name, table_name = op.cmd_parse

  client = get_client
  table = get_table(client, db_name, table_name)

  puts "#{db_name}.#{table_name} ("
  table.schema.fields.each {|f|
    puts "  #{f.name}:#{f.type}"
  }
  puts ")"
end

#server_status(op) ⇒ Object 



5
6
7
8
9
 
# File 'lib/td/command/server.rb', line 5

def server_status(op)
  op.cmd_parse

  puts Client.server_status
end

#status(op) ⇒ Object 



5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
 
# File 'lib/td/command/status.rb', line 5

def status(op)
  op.cmd_parse

  client = get_client

  # +----------------+
  # |     scheds     |
  # +----------------+
  # +----------------+
  # |      jobs      |
  # +----------------+
  # +------+ +-------+
  # |tables| |results|
  # +------+ +-------+

  scheds = []
  jobs = []
  tables = []
  results = []

  s = client.schedules
  s.each {|sched|
    scheds << {:Name => sched.name, :Cron => sched.cron, :Result => sched.result_url, :Query => sched.query}
  }
  scheds = scheds.sort_by {|map|
    map[:Name]
  }
  x1, y1 = status_render(0, 0, "[Schedules]", scheds, :fields => [:Name, :Cron, :Result, :Query])

  j = client.jobs(0, 4)
  j.each {|job|
    start = job.start_at
    elapsed = cmd_format_elapsed(start, job.end_at)
    jobs << {:JobID => job.job_id, :Status => job.status, :Query => job.query.to_s, :Start => (start ? start.localtime : ''), :Elapsed => elapsed, :Result => job.result_url}
  }
  x2, y2 = status_render(0, 0, "[Jobs]", jobs, :fields => [:JobID, :Status, :Start, :Elapsed, :Result, :Query])

  dbs = client.databases
  dbs.map {|db|
    db.tables.each {|table|
      tables << {:Database => db.name, :Table => table.name, :Count => table.count.to_s, :Size => table.estimated_storage_size_string}
    }
  }
  x3, y3 = status_render(0, 0, "[Tables]", tables, :fields => [:Database, :Table, :Count, :Size])

  rs = client.results
  rs.each {|r|
    results << {:Name => r.name, :URL => r.url}
  }
  results = results.sort_by {|map|
    map[:Name]
  }
  x4, y4 = status_render(x3+2, y3, "[Results]", results, :fields => [:Name, :URL])

  (y3-y4-1).times do
    print "\eD"
  end
  print "\eE"
end

#table_create(op) ⇒ Object 



14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
 
# File 'lib/td/command/table.rb', line 14

def table_create(op)
  db_name, table_name = op.cmd_parse

  #API.validate_database_name(db_name)
  API.validate_table_name(table_name)

  if HIVE_RESERVED_KEYWORDS.include?(table_name.upcase)
    $stderr.puts "* WARNING *"
    $stderr.puts "  '#{table_name}' is a reserved keyword in Hive. We recommend renaming the table."
    $stderr.puts "  For a list of all reserved keywords, see our FAQ: http://docs.treasure-data.com/articles/faq"
  end

  client = get_client

  begin
    client.create_log_table(db_name, table_name)
  rescue NotFoundError
    cmd_debug_error $!
    $stderr.puts "Database '#{db_name}' does not exist."
    $stderr.puts "Use '#{$prog} db:create #{db_name}' to create the database."
    exit 1
  rescue AlreadyExistsError
    cmd_debug_error $!
    $stderr.puts "Table '#{db_name}.#{table_name}' already exists."
    exit 1
  end

  $stderr.puts "Table '#{db_name}.#{table_name}' is created."
end

#table_delete(op) ⇒ Object 



44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
 
# File 'lib/td/command/table.rb', line 44

def table_delete(op)
  force = false
  op.on('-f', '--force', 'never prompt', TrueClass) {|b|
    force = true
  }

  db_name, table_name = op.cmd_parse

  client = get_client

  begin
    unless force
      table = get_table(client, db_name, table_name)
      $stderr.print "Do you really delete '#{table_name}' in '#{db_name}'? [y/N]: "
      ok = nil
      while line = $stdin.gets
        line.strip!
        if line =~ /^y(?:es)?$/i
          ok = true
          break
        elsif line.empty? || line =~ /^n(?:o)?$/i
          break
        else
          $stderr.print "Type 'Y' or 'N': "
        end
      end
      unless ok
        $stderr.puts "canceled."
        exit 1
      end
    end
    client.delete_table(db_name, table_name)
  rescue NotFoundError
    cmd_debug_error $!
    $stderr.puts "Table '#{db_name}.#{table_name}' does not exist."
    $stderr.puts "Use '#{$prog} table:list #{db_name}' to show list of the tables."
    exit 1
  end

  $stderr.puts "Table '#{db_name}.#{table_name}' is deleted."
end

#table_export(op) ⇒ Object 



243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
 
# File 'lib/td/command/table.rb', line 243

def table_export(op)
  from = nil
  to = nil
  s3_bucket = nil
  wait = false

  ## TODO
  #op.on('-t', '--to TIME', 'end time of logs to get') {|s|
  #  if s.to_i.to_s == s
  #    to = s
  #  else
  #    require 'time'
  #    to = Time.parse(s).to_i
  #  end
  #}
  #op.on('-f', '--from TIME', 'start time of logs to get') {|s|
  #  if s.to_i.to_s == s
  #    from = s
  #  else
  #    require 'time'
  #    from = Time.parse(s).to_i
  #  end
  #}
  op.on('-w', '--wait', 'wait for finishing the job', TrueClass) {|b|
    wait = b
  }
  op.on('--s3-bucket NAME', 'name of the s3 bucket to output') {|s|
    s3_bucket = s
  }

  db_name, table_name = op.cmd_parse

  unless s3_bucket
    $stderr.puts "--s3-bucket NAME option is required"
    exit 1
  end

  client = get_client

  table = get_table(client, db_name, table_name)

  opts = {}
  opts['s3_bucket'] = s3_bucket
  opts['s3_file_format'] ='json.gz'
  opts['from'] = from.to_s if from
  opts['to']   = to.to_s if to

  job = table.export('s3', opts)

  $stderr.puts "Export job #{job.job_id} is queued."
  $stderr.puts "Use '#{$prog} job:show #{job.job_id}' to show the status."

  if wait && !job.finished?
    wait_job(job)
    puts "Status     : #{job.status}"
  end
end

#table_import(op) ⇒ Object

TODO import-item TODO tail



19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
 
# File 'lib/td/command/import.rb', line 19

def table_import(op)
  op.banner << "\nsupported formats:\n"
  op.banner << "  apache\n"
  op.banner << "  syslog\n"
  op.banner << "  msgpack\n"
  op.banner << "  json\n"

  format = 'apache'
  time_key = 'time'
  auto_create = false

  op.on('--format FORMAT', "file format (default: #{format})") {|s|
    format = s
  }

  op.on('--apache', "same as --format apache; apache common log format") {
    format = 'apache'
  }

  op.on('--syslog', "same as --format syslog; syslog") {
    format = 'syslog'
  }

  op.on('--msgpack', "same as --format msgpack; msgpack stream format") {
    format = 'msgpack'
  }

  op.on('--json', "same as --format json; LF-separated json format") {
    format = 'json'
  }

  op.on('-t', '--time-key COL_NAME', "time key name for json and msgpack format (e.g. 'created_at')") {|s|
    time_key = s
  }

  op.on('--auto-create-table', "Create table and database if doesn't exist", TrueClass) { |b|
    auto_create = b
  }

  db_name, table_name, *paths = op.cmd_parse

  client = get_client

  if auto_create
    # Merge with db_create and table_create after refactoring
    API.validate_database_name(db_name)
    begin
      client.create_database(db_name)
      $stderr.puts "Database '#{db_name}' is created."
    rescue AlreadyExistsError
    end

    API.validate_table_name(table_name)
    begin
      client.create_log_table(db_name, table_name)
      $stderr.puts "Table '#{db_name}.#{table_name}' is created."
    rescue AlreadyExistsError
    end
  end

  case format
  when 'json', 'msgpack'
    #unless time_key
    #  $stderr.puts "-t, --time-key COL_NAME (e.g. '-t created_at') parameter is required for #{format} format"
    #  exit 1
    #end
    if format == 'json'
      require 'json'
      require 'time'
      parser = JsonParser.new(time_key)
    else
      parser = MessagePackParser.new(time_key)
    end

  else
    regexp, names, time_format = IMPORT_TEMPLATES[format]
    if !regexp || !names || !time_format
      $stderr.puts "Unknown format '#{format}'"
      exit 1
    end
    parser = TextParser.new(names, regexp, time_format)
  end

  get_table(client, db_name, table_name)

  require 'zlib'

  files = paths.map {|path|
    if path == '-'
      $stdin
    elsif path =~ /\.gz$/
      require 'td/compat_gzip_reader'
      Zlib::GzipReader.open(path)
    else
      File.open(path)
    end
  }

  require 'msgpack'
  require 'tempfile'
  #require 'thread'

  files.zip(paths).each {|file,path|
    import_log_file(file, path, client, db_name, table_name, parser)
  }

  puts "done."
end

#table_list(op) ⇒ Object 



86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
 
# File 'lib/td/command/table.rb', line 86

def table_list(op)
  require 'parallel'

  num_threads = 4
  show_size_in_bytes = false

  op.on('-n', '--num_threads VAL', 'number of threads to get list in parallel') { |i|
    num_threads = Integer(i)
  }
  op.on('--show-bytes', 'show estimated table size in bytes') {
    show_size_in_bytes = true
  }

  db_name = op.cmd_parse

  client = get_client

  if db_name
    database = get_database(client, db_name)
    databases = [database]
  else
    databases = client.databases
  end

  rows = []
  ::Parallel.each(databases, :in_threads => num_threads) {|db|
    db.tables.each {|table|
      pschema = table.schema.fields.map {|f|
        "#{f.name}:#{f.type}"
      }.join(', ')
      rows << {
        :Database => db.name, :Table => table.name, :Type => table.type.to_s, :Count => table.count.to_s.gsub(/(?<=\d)(?=(?:\d{3})+(?!\d))/, ','),
        :Size => show_size_in_bytes ? table.estimated_storage_size.to_s.gsub(/(?<=\d)(?=(?:\d{3})+(?!\d))/, ',') : table.estimated_storage_size_string,
        'Last import' => table.last_import ? table.last_import.localtime : nil,
        :Schema => pschema
      }
    }
  }
  rows = rows.sort_by {|map|
    [map[:Database], map[:Type].size, map[:Table]]
  }

  puts cmd_render_table(rows, :fields => [:Database, :Table, :Type, :Count, :Size, 'Last import', :Schema], :max_width=>500)

  if rows.empty?
    if db_name
      $stderr.puts "Database '#{db_name}' has no tables."
      $stderr.puts "Use '#{$prog} table:create <db> <table>' to create a table."
    elsif databases.empty?
      $stderr.puts "There are no databases."
      $stderr.puts "Use '#{$prog} db:create <db>' to create a database."
    else
      $stderr.puts "There are no tables."
      $stderr.puts "Use '#{$prog} table:create <db> <table>' to create a table."
    end
  end
end

#table_partial_delete(op) ⇒ Object 



301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
 
# File 'lib/td/command/table.rb', line 301

def table_partial_delete(op)
  org = nil
  from = nil
  to = nil
  wait = false

  op.on('-g', '--org ORGANIZATION', "delete data partially under this organization") {|s|
    org = s
  }
  op.on('-t', '--to TIME', 'end time of logs to delete') {|s|
    if s.to_i.to_s == s
      # UNIX time
      to = s.to_i
    else
      require 'time'
      to = Time.parse(s).to_i
    end
  }
  op.on('-f', '--from TIME', 'start time of logs to delete') {|s|
    if s.to_i.to_s == s
      from = s.to_i
    else
      require 'time'
      from = Time.parse(s).to_i
    end
  }
  op.on('-w', '--wait', 'wait for finishing the job', TrueClass) {|b|
    wait = b
  }

  db_name, table_name = op.cmd_parse

  unless from
    $stderr.puts "-f, --from TIME option is required"
    exit 1
  end

  unless to
    $stderr.puts "-t, --to TIME option is required"
    exit 1
  end

  if from % 3600 != 0 || to % 3600 != 0
    $stderr.puts "time must be a multiple of 3600 (1 hour)"
    exit 1
  end

  client = get_client

  table = get_table(client, db_name, table_name)

  opts = {}
  opts['organization'] = org if org
  job = client.partial_delete(db_name, table_name, to, from, opts)

  $stderr.puts "Partial delete job #{job.job_id} is queued."
  $stderr.puts "Use '#{$prog} job:show #{job.job_id}' to show the status."

  if wait && !job.finished?
    wait_job(job)
    puts "Status     : #{job.status}"
  end
end

#table_show(op) ⇒ Object 



157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
 
# File 'lib/td/command/table.rb', line 157

def table_show(op)
  db_name, table_name = op.cmd_parse

  client = get_client

  table = get_table(client, db_name, table_name)

  puts "Name      : #{table.db_name}.#{table.name}"
  puts "Type      : #{table.type}"
  puts "Count     : #{table.count}"
  puts "Schema    : ("
  table.schema.fields.each {|f|
    puts "    #{f.name}:#{f.type}"
  }
  puts ")"
end

#table_swap(op) ⇒ Object 



144
145
146
147
148
149
150
151
152
153
154
155
 
# File 'lib/td/command/table.rb', line 144

def table_swap(op)
  db_name, table_name1, table_name2 = op.cmd_parse

  client = get_client

  table1 = get_table(client, db_name, table_name1)
  table2 = get_table(client, db_name, table_name2)

  client.swap_table(db_name, table_name1, table_name2)

  $stderr.puts "'#{db_name}.#{table_name1}' and '#{db_name}.#{table_name2}' are swapped."
end

#table_tail(op) ⇒ Object 



174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
 
# File 'lib/td/command/table.rb', line 174

def table_tail(op)
  from = nil
  to = nil
  count = nil
  pretty = nil

  op.on('-t', '--to TIME', 'end time of logs to get') {|s|
    if s.to_i.to_s == s
      to = s.to_i
    else
      require 'time'
      to = Time.parse(s).to_i
    end
  }
  op.on('-f', '--from TIME', 'start time of logs to get') {|s|
    if s.to_i.to_s == s
      from = s.to_i
    else
      require 'time'
      from = Time.parse(s).to_i
    end
  }
  op.on('-n', '--count N', 'number of logs to get', Integer) {|i|
    count = i
  }
  op.on('-P', '--pretty', 'pretty print', TrueClass) {|b|
    pretty = b
  }

  if count == nil
    # smart count calculation
    begin
      require "curses"
     if Curses.stdscr.maxy - 1 <= 40
        count = 5
      else
        count = 10
      end
     Curses.close_screen
    rescue Exception
      count = 5
    end
  end

  db_name, table_name = op.cmd_parse

  client = get_client

  table = get_table(client, db_name, table_name)

  rows = table.tail(count, to, from)

  require 'json'
  if pretty
    opts = {
      :indent => ' '*2,
      :object_nl => "\n",
      :space => ' '
    }
    rows.each {|row|
      puts row.to_json(opts)
    }
  else
    rows.each {|row|
      puts row.to_json
    }
  end
end

#user_apikey_list(op) ⇒ Object

TODO user:apikey:remove <name> <apikey> def user_apikey_remove(op) end



188
189
190
191
192
193
194
195
196
197
198
199
200
201
 
# File 'lib/td/command/user.rb', line 188

def user_apikey_list(op)
  name = op.cmd_parse

  client = get_client

  keys = client.list_apikeys(name)

  rows = []
  keys.each {|key|
    rows << {:Key => key}
  }

  puts cmd_render_table(rows, :fields => [:Key])
end

#user_create(op) ⇒ Object 



44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
 
# File 'lib/td/command/user.rb', line 44

def user_create(op)
  org = nil
  email = nil
  random_password = false
  create_org = nil

  op.on('-g', '--org ORGANIZATION', "create the user under this organization") {|s|
    org = s
  }

  op.on('-G', "create the user under the a new organization", TrueClass) {|b|
    create_org = b
  }

  op.on('-e', '--email EMAIL', "Use this email address to identify the user") {|s|
    email = s
  }

  op.on('-R', '--random-password', "Generate random password", TrueClass) {|b|
    random_password = b
  }

  name = op.cmd_parse

  unless email
    $stderr.puts "-e, --email EMAIL option is required."
    exit 1
  end

  if random_password
    lower = ('a'..'z').to_a
    upper = ('A'..'Z').to_a
    digit = ('0'..'9').to_a
    symbol = %w[_ @ - + ;]

    r = []
    3.times { r << lower.sort_by{rand}.first }
    3.times { r << upper.sort_by{rand}.first }
    2.times { r << digit.sort_by{rand}.first }
    1.times { r << symbol.sort_by{rand}.first }
    password = r.sort_by{rand}.join

    puts "Password: #{password}"

  else
    3.times do
      begin
        system "stty -echo"  # TODO termios
        print "Password (typing will be hidden): "
        password = STDIN.gets || ""
        password = password[0..-2]  # strip \n
      rescue Interrupt
        $stderr.print "\ncanceled."
        exit 1
      ensure
        system "stty echo"   # TODO termios
        print "\n"
      end

      if password.empty?
        $stderr.puts "canceled."
        exit 0
      end

      begin
        system "stty -echo"  # TODO termios
        print "Retype password: "
        password2 = STDIN.gets || ""
        password2 = password2[0..-2]  # strip \n
      rescue Interrupt
        $stderr.print "\ncanceled."
        exit 1
      ensure
        system "stty echo"   # TODO termios
        print "\n"
      end

      if password == password2
        break
      end

      puts "Doesn't match."
    end
  end

  client = get_client(:ssl => true)

  if create_org
    org ||= name
    client.create_organization(org)
  end

  ok = false
  begin
    client.add_user(name, org)

    begin
      client.change_email(name, email)
      client.change_password(name, password)
      client.add_apikey(name)
      ok = true

    ensure
      if !ok
        client.remove_user(name)
      end
    end

  ensure
    if create_org && !ok
      client.delete_organization(org)
    end
  end

  if create_org
    $stderr.puts "Organization '#{org}' and user '#{name}' are created."
  else
    $stderr.puts "User '#{name}' is created."
  end
  $stderr.puts "Use '#{$prog} user:apikeys #{name}' to show the API key."
end

#user_delete(op) ⇒ Object 



166
167
168
169
170
171
172
173
174
 
# File 'lib/td/command/user.rb', line 166

def user_delete(op)
  name = op.cmd_parse

  client = get_client

  client.remove_user(name)

  $stderr.puts "User '#{name}' is deleted."
end

#user_list(op) ⇒ Object 



24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
 
# File 'lib/td/command/user.rb', line 24

def user_list(op)
  op.cmd_parse

  client = get_client

  users = client.users

  rows = []
  users.each {|user|
    rows << {:Name => user.name, :Organization => user.org_name, :Email => user.email, :Roles => user.role_names.join(',')}
  }

  puts cmd_render_table(rows, :fields => [:Name, :Organization, :Email, :Roles])

  if rows.empty?
    $stderr.puts "There are no users."
    $stderr.puts "Use '#{$prog} user:create <name>' to create an users."
  end
end

#user_password_change(op) ⇒ Object 



203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
 
# File 'lib/td/command/user.rb', line 203

def user_password_change(op)
  name = op.cmd_parse

  password = nil

  3.times do
    begin
      system "stty -echo"  # TODO termios
      print "New password (typing will be hidden): "
      password = STDIN.gets || ""
      password = password[0..-2]  # strip \n
    rescue Interrupt
      $stderr.print "\ncanceled."
      exit 1
    ensure
      system "stty echo"   # TODO termios
      print "\n"
    end

    if password.empty?
      $stderr.puts "canceled."
      exit 0
    end

    begin
      system "stty -echo"  # TODO termios
      print "Retype new password: "
      password2 = STDIN.gets || ""
      password2 = password2[0..-2]  # strip \n
    rescue Interrupt
      $stderr.print "\ncanceled."
      exit 1
    ensure
      system "stty echo"   # TODO termios
      print "\n"
    end

    if password == password2
      break
    end

    puts "Doesn't match."
  end

  client = get_client(:ssl => true)

  client.change_password(name, password)

  $stderr.puts "Password of user '#{name}' changed."
end

#user_show(op) ⇒ Object 



5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
 
# File 'lib/td/command/user.rb', line 5

def user_show(op)
  name = op.cmd_parse

  client = get_client

  users = client.users
  user = users.find {|user| name == user.name }
  unless user
    $stderr.puts "User '#{name}' does not exist."
    $stderr.puts "Use '#{$prog} user:create <name>' to create an user."
    exit 1
  end

  $stderr.puts "Name         : #{user.name}"
  $stderr.puts "Organization : #{user.org_name}"
  $stderr.puts "Email        : #{user.email}"
  $stderr.puts "Roles        : #{user.role_names.join(', ')}"
end