Class: Bitcoin::Storage::Backends::UtxoStore

Inherits:
StoreBase
  • Object
show all
Defined in:
lib/bitcoin/storage/utxo/utxo_store.rb

Overview

Storage backend using Sequel to connect to arbitrary SQL databases. Inherits from StoreBase and implements its interface.

Constant Summary collapse

SCRIPT_TYPES =

possible script types

[:unknown, :pubkey, :hash160, :multisig, :p2sh]
DEFAULT_CONFIG = 
{
  # cache head block; it is only updated when new block comes in,
  # so this should only be used by the store receiving new blocks.
  cache_head: false,
  # cache this many utxo records before syncing to disk.
  # this should only be enabled during initial sync, because
  # with it the store cannot reorg properly.
  utxo_cache: 250,
  # cache this many blocks.
  # NOTE: this is also the maximum number of blocks the store can reorg.
  block_cache: 120,
  # keep an index of utxos for all addresses, not just the ones
  # we are explicitly told about.
  index_all_addrs: false
}

Constants inherited from StoreBase

StoreBase::MAIN, StoreBase::ORPHAN, StoreBase::SEQUEL_ADAPTERS, StoreBase::SIDE

Instance Attribute Summary collapse

Attributes inherited from StoreBase

#config, #log

Instance Method Summary collapse

Methods inherited from StoreBase

#backend_name, #check_metadata, #get_idx_from_tx_hash, #get_locator, #get_txin_for_txout, #get_txouts_for_address, #import, #in_sync?, #init_sequel_store, #migrate, #new_block, #new_tx, #parse_script, #sqlite_pragmas, #store_addr, #store_block, #store_tx, #update_block

Constructor Details

#initialize(config, *args) ⇒ UtxoStore

create sequel store with given config



37
38
39
40
41
 
# File 'lib/bitcoin/storage/utxo/utxo_store.rb', line 37

def initialize config, *args
  super config, *args
  @spent_outs, @new_outs, @watched_addrs = [], [], []
  @deleted_utxos, @tx_cache, @block_cache = {}, {}, {}
end

Instance Attribute Details

#db ⇒ Object

sequel database connection



18
19
20
 
# File 'lib/bitcoin/storage/utxo/utxo_store.rb', line 18

def db
  @db
end

Instance Method Details

#add_watched_address(address) ⇒ Object 



194
195
196
197
198
 
# File 'lib/bitcoin/storage/utxo/utxo_store.rb', line 194

def add_watched_address address
  hash160 = Bitcoin.hash160_from_address(address)
  @db[:addr].insert(hash160: hash160)  unless @db[:addr][hash160: hash160]
  @watched_addrs << hash160  unless @watched_addrs.include?(hash160)
end

#check_consistency(*args) ⇒ Object 



355
356
357
 
# File 'lib/bitcoin/storage/utxo/utxo_store.rb', line 355

def check_consistency(*args)
  log.warn { "Utxo store doesn't support consistency check" }
end

#connect ⇒ Object

connect to database



44
45
46
47
48
49
 
# File 'lib/bitcoin/storage/utxo/utxo_store.rb', line 44

def connect
  super
  load_watched_addrs
#      rescan

end

#flush_new_outs(depth) ⇒ Object 



174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
 
# File 'lib/bitcoin/storage/utxo/utxo_store.rb', line 174

def flush_new_outs depth
  log.time "flushed #{@new_outs.size} new txouts in %.4fs" do
    new_utxo_ids = @db[:utxo].insert_multiple(@new_outs.map{|o|o[0]})
    @new_outs.each.with_index do |d, idx|
      d[1].each do |i, hash160|
        next  unless i && hash160
        store_addr(new_utxo_ids[idx], hash160)
      end
    end

    @new_outs.each.with_index do |d, idx|
      d[2].each do |i, script|
        next  unless i && script
        store_name(script, new_utxo_ids[idx])
      end
    end
    @new_outs = []
  end
end

#flush_spent_outs(depth) ⇒ Object 



154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
 
# File 'lib/bitcoin/storage/utxo/utxo_store.rb', line 154

def flush_spent_outs depth
  log.time "flushed #{@spent_outs.size} spent txouts in %.4fs" do
    if @spent_outs.any?
      @spent_outs.each_slice(250) do |slice|
        if @db.adapter_scheme == :postgres
          condition = slice.map {|o| "(tx_hash = '#{o[:tx_hash]}' AND tx_idx = #{o[:tx_idx]})" }.join(" OR ")
        else
          condition = slice.map {|o| "(tx_hash = X'#{o[:tx_hash].hth}' AND tx_idx = #{o[:tx_idx]})" }.join(" OR ")
        end
        @db["DELETE FROM addr_txout WHERE EXISTS
               (SELECT 1 FROM utxo WHERE
                 utxo.id = addr_txout.txout_id AND (#{condition}));"].all
        @db["DELETE FROM utxo WHERE #{condition};"].first

      end
    end
    @spent_outs = []
  end
end

#get_balance(hash160) ⇒ Object 



307
308
309
 
# File 'lib/bitcoin/storage/utxo/utxo_store.rb', line 307

def get_balance hash160
  get_txouts_for_hash160(hash160).map(&:value).inject(:+) || 0
end

#get_block(blk_hash) ⇒ Object

get block for given blk_hash



250
251
252
 
# File 'lib/bitcoin/storage/utxo/utxo_store.rb', line 250

def get_block(blk_hash)
  wrap_block(@db[:blk][:hash => blk_hash.htb.blob])
end

#get_block_by_depth(depth) ⇒ Object

get block by given depth



255
256
257
 
# File 'lib/bitcoin/storage/utxo/utxo_store.rb', line 255

def get_block_by_depth(depth)
  wrap_block(@db[:blk][:depth => depth, :chain => MAIN])
end

#get_block_by_id(block_id) ⇒ Object

get block by given id



271
272
273
 
# File 'lib/bitcoin/storage/utxo/utxo_store.rb', line 271

def get_block_by_id(block_id)
  wrap_block(@db[:blk][:id => block_id])
end

#get_block_by_prev_hash(prev_hash) ⇒ Object

get block by given prev_hash



260
261
262
 
# File 'lib/bitcoin/storage/utxo/utxo_store.rb', line 260

def get_block_by_prev_hash(prev_hash)
  wrap_block(@db[:blk][:prev_hash => prev_hash.htb.blob, :chain => MAIN])
end

#get_block_by_tx(tx_hash) ⇒ Object

get block by given tx_hash



265
266
267
268
 
# File 'lib/bitcoin/storage/utxo/utxo_store.rb', line 265

def get_block_by_tx(tx_hash)
  block_id = @db[:utxo][tx_hash: tx_hash.blob][:blk_id]
  get_block_by_id(block_id)
end

#get_depth ⇒ Object

get depth of MAIN chain



244
245
246
247
 
# File 'lib/bitcoin/storage/utxo/utxo_store.rb', line 244

def get_depth
  return -1  unless get_head
  get_head.depth
end

#get_head ⇒ Object

get head block (highest block from the MAIN chain)



238
239
240
241
 
# File 'lib/bitcoin/storage/utxo/utxo_store.rb', line 238

def get_head
  (@config[:cache_head] && @head) ? @head :
    @head = wrap_block(@db[:blk].filter(:chain => MAIN).order(:depth).last)
end

#get_tx(tx_hash) ⇒ Object

get transaction for given tx_hash



276
277
278
 
# File 'lib/bitcoin/storage/utxo/utxo_store.rb', line 276

def get_tx(tx_hash)
  @tx_cache[tx_hash] ||= wrap_tx(tx_hash)
end

#get_tx_by_id(tx_id) ⇒ Object

get transaction by given tx_id



281
282
283
 
# File 'lib/bitcoin/storage/utxo/utxo_store.rb', line 281

def get_tx_by_id(tx_id)
  get_tx(tx_id)
end

#get_txout_by_id(id) ⇒ Object 



285
286
287
 
# File 'lib/bitcoin/storage/utxo/utxo_store.rb', line 285

def get_txout_by_id(id)
  wrap_txout(@db[:utxo][id: id])
end

#get_txout_for_txin(txin) ⇒ Object

get corresponding Models::TxOut for txin



290
291
292
 
# File 'lib/bitcoin/storage/utxo/utxo_store.rb', line 290

def get_txout_for_txin(txin)
  wrap_txout(@db[:utxo][tx_hash: txin.prev_out.reverse.hth.blob, tx_idx: txin.prev_out_index])
end

#get_txouts_for_hash160(hash160, unconfirmed = false) ⇒ Object

get all Models::TxOut matching given hash160



301
302
303
304
305
 
# File 'lib/bitcoin/storage/utxo/utxo_store.rb', line 301

def get_txouts_for_hash160(hash160, unconfirmed = false)
  addr = @db[:addr][hash160: hash160]
  return []  unless addr
  @db[:addr_txout].where(addr_id: addr[:id]).map {|ao| wrap_txout(@db[:utxo][id: ao[:txout_id]]) }.compact
end

#get_txouts_for_pk_script(script) ⇒ Object

get all Models::TxOut matching given script



295
296
297
298
 
# File 'lib/bitcoin/storage/utxo/utxo_store.rb', line 295

def get_txouts_for_pk_script(script)
  utxos = @db[:utxo].filter(pk_script: script.blob).order(:blk_id)
  utxos.map {|utxo| wrap_txout(utxo) }
end

#has_block(blk_hash) ⇒ Object

check if block blk_hash exists



228
229
230
 
# File 'lib/bitcoin/storage/utxo/utxo_store.rb', line 228

def has_block(blk_hash)
  !!@db[:blk].where(:hash => blk_hash.htb.blob).get(1)
end

#has_tx(tx_hash) ⇒ Object

check if transaction tx_hash exists



233
234
235
 
# File 'lib/bitcoin/storage/utxo/utxo_store.rb', line 233

def has_tx(tx_hash)
  !!@db[:utxo].where(:tx_hash => tx_hash.blob).get(1)
end

#load_watched_addrs ⇒ Object 



200
201
202
 
# File 'lib/bitcoin/storage/utxo/utxo_store.rb', line 200

def load_watched_addrs
  @watched_addrs = @db[:addr].all.map{|a| a[:hash160] }  unless @config[:index_all_addrs]
end

#persist_block(blk, chain, depth, prev_work = 0) ⇒ Object

persist given block blk to storage.



58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
 
# File 'lib/bitcoin/storage/utxo/utxo_store.rb', line 58

def persist_block blk, chain, depth, prev_work = 0
  load_watched_addrs
  @db.transaction do
    attrs = {
      :hash => blk.hash.htb.blob,
      :depth => depth,
      :chain => chain,
      :version => blk.ver,
      :prev_hash => blk.prev_block.reverse.blob,
      :mrkl_root => blk.mrkl_root.reverse.blob,
      :time => blk.time,
      :bits => blk.bits,
      :nonce => blk.nonce,
      :blk_size => blk.payload.bytesize,
      :work => (prev_work + blk.block_work).to_s
    }
    existing = @db[:blk].filter(:hash => blk.hash.htb.blob)
    if existing.any?
      existing.update attrs
      block_id = existing.first[:id]
    else
      block_id = @db[:blk].insert(attrs)
    end

    if @config[:block_cache] > 0
      @block_cache.shift  if @block_cache.size > @config[:block_cache]
      @deleted_utxos.shift  if @deleted_utxos.size > @config[:block_cache]
      @block_cache[blk.hash] = blk
    end

    if chain == MAIN
      persist_transactions(blk.tx, block_id, depth)
      @tx_cache = {}
      @head = wrap_block(attrs.merge(id: block_id))  if chain == MAIN
    end
    return depth, chain
  end
end

#persist_transactions(txs, block_id, depth) ⇒ Object 



97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
 
# File 'lib/bitcoin/storage/utxo/utxo_store.rb', line 97

def persist_transactions txs, block_id, depth
  txs.each.with_index do |tx, tx_blk_idx|
    tx.in.each.with_index do |txin, txin_tx_idx|
      next  if txin.coinbase?
      size = @new_outs.size
      @new_outs.delete_if {|o| o[0][:tx_hash] == txin.prev_out.reverse.hth &&
        o[0][:tx_idx] == txin.prev_out_index }
      @spent_outs << {
        tx_hash: txin.prev_out.reverse.hth.to_sequel_blob,
        tx_idx: txin.prev_out_index  }  if @new_outs.size == size
    end
    tx.out.each.with_index do |txout, txout_tx_idx|
      _, a, n = *parse_script(txout, txout_tx_idx)
      @new_outs << [{
          :tx_hash => tx.hash.blob,
          :tx_idx => txout_tx_idx,
          :blk_id => block_id,
          :pk_script => txout.pk_script.blob,
          :value => txout.value },
        @config[:index_all_addrs] ? a : a.select {|a| @watched_addrs.include?(a[1]) },
        Bitcoin.namecoin? ? n : [] ]
    end
  end
  flush_spent_outs(depth)  if @spent_outs.size > @config[:utxo_cache]
  flush_new_outs(depth)  if @new_outs.size > @config[:utxo_cache]
end

#reorg(new_side, new_main) ⇒ Object 



124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
 
# File 'lib/bitcoin/storage/utxo/utxo_store.rb', line 124

def reorg new_side, new_main
  new_side.each do |block_hash|
    raise "trying to remove non-head block!"  unless get_head.hash == block_hash
    depth = get_depth
    blk = @db[:blk][hash: block_hash.htb.blob]
    delete_utxos = @db[:utxo].where(blk_id: blk[:id])
    @db[:addr_txout].where("txout_id IN ?", delete_utxos.map{|o|o[:id]}).delete

    delete_utxos.delete
    (@deleted_utxos[depth] || []).each do |utxo|
      utxo[:pk_script] = utxo[:pk_script].to_sequel_blob
      utxo_id = @db[:utxo].insert(utxo)
      addrs = Bitcoin::Script.new(utxo[:pk_script]).get_addresses
      addrs.each do |addr|
        hash160 = Bitcoin.hash160_from_address(addr)
        store_addr(utxo_id, hash160)
      end
    end

    @db[:blk].where(id: blk[:id]).update(chain: SIDE)
  end

  new_main.each do |block_hash|
    block = @db[:blk][hash: block_hash.htb.blob]
    blk = @block_cache[block_hash]
    persist_transactions(blk.tx, block[:id], block[:depth])
    @db[:blk].where(id: block[:id]).update(chain: MAIN)
  end
end

#rescan ⇒ Object 



204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
 
# File 'lib/bitcoin/storage/utxo/utxo_store.rb', line 204

def rescan
  load_watched_addrs
  @rescan_lock ||= Monitor.new
  @rescan_lock.synchronize do
    log.info { "Rescanning #{@db[:utxo].count} utxos for #{@watched_addrs.size} addrs" }
    count = @db[:utxo].count; n = 100_000
    @db[:utxo].order(:id).each_slice(n).with_index do |slice, index|
      log.debug { "rescan progress: %.2f%" % (100.0 / count * (index*n)) }
      slice.each do |utxo|
        next  if utxo[:pk_script].bytesize >= 10_000
        hash160 = Bitcoin::Script.new(utxo[:pk_script]).get_hash160
        if @config[:index_all_addrs] || @watched_addrs.include?(hash160)
          log.info { "Found utxo for address #{Bitcoin.hash160_to_address(hash160)}: " +
            "#{utxo[:tx_hash][0..8]}:#{utxo[:tx_idx]} (#{utxo[:value]})" }
          addr = @db[:addr][hash160: hash160]
          addr_utxo = {addr_id: addr[:id], txout_id: utxo[:id]}
          @db[:addr_txout].insert(addr_utxo)  unless @db[:addr_txout][addr_utxo]
        end
      end
    end
  end
end

#reset ⇒ Object

reset database; delete all data



52
53
54
55
 
# File 'lib/bitcoin/storage/utxo/utxo_store.rb', line 52

def reset
  [:blk, :utxo, :addr, :addr_txout].each {|table| @db[table].delete }
  @head = nil
end

#wrap_block(block) ⇒ Object

wrap given block into Models::Block



312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
 
# File 'lib/bitcoin/storage/utxo/utxo_store.rb', line 312

def wrap_block(block)
  return nil  unless block

  data = {:id => block[:id], :depth => block[:depth], :chain => block[:chain],
    :work => block[:work].to_i, :hash => block[:hash].hth}
  blk = Bitcoin::Storage::Models::Block.new(self, data)

  blk.ver = block[:version]
  blk.prev_block = block[:prev_hash].reverse
  blk.mrkl_root = block[:mrkl_root].reverse
  blk.time = block[:time].to_i
  blk.bits = block[:bits]
  blk.nonce = block[:nonce]

  if cached = @block_cache[block[:hash].hth]
    blk.tx = cached.tx
  end

  blk.recalc_block_hash
  blk
end

#wrap_tx(tx_hash) ⇒ Object

wrap given transaction into Models::Transaction



335
336
337
338
339
340
341
342
343
 
# File 'lib/bitcoin/storage/utxo/utxo_store.rb', line 335

def wrap_tx(tx_hash)
  utxos = @db[:utxo].where(tx_hash: tx_hash.blob)
  return nil  unless utxos.any?
  data = { blk_id: utxos.first[:blk_id] }
  tx = Bitcoin::Storage::Models::Tx.new(self, data)
  tx.hash = tx_hash # utxos.first[:tx_hash].hth
  utxos.each {|u| tx.out[u[:tx_idx]] = wrap_txout(u) }
  return tx
end

#wrap_txout(utxo) ⇒ Object

wrap given output into Models::TxOut



346
347
348
349
350
351
352
353
 
# File 'lib/bitcoin/storage/utxo/utxo_store.rb', line 346

def wrap_txout(utxo)
  return nil  unless utxo
  data = {id: utxo[:id], tx_id: utxo[:tx_hash], tx_idx: utxo[:tx_idx]}
  txout = Bitcoin::Storage::Models::TxOut.new(self, data)
  txout.value = utxo[:value]
  txout.pk_script = utxo[:pk_script]
  txout
end