17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
|
# File 'lib/scraperwiki/sqlite_save_info.rb', line 17
def SQLiteMagic._do_save_sqlite(unique_keys, data, swdatatblname)
if @db.nil?
@db = SQLite3::Database.new("scraperwiki.sqlite")
end
res = { }
if data.class == Hash
data = [data]
end
if !@sqlitesaveinfo.include?(swdatatblname)
ssinfo = SqliteSaveInfo.new(swdatatblname, @db)
@sqlitesaveinfo[swdatatblname] = ssinfo
if not ssinfo.rebuildinfo() and data.length > 0
ssinfo.buildinitialtable(data[0])
ssinfo.rebuildinfo()
res["tablecreated"] = swdatatblname
end
else
ssinfo = @sqlitesaveinfo[swdatatblname]
end
@db.transaction()
nrecords = 0
data.each do |ldata|
newcols = ssinfo.newcolumns(ldata)
if newcols.length > 0
newcols.each_with_index do |kv, i|
ssinfo.addnewcolumn(kv[0], kv[1])
res["newcolumn %d" % i] = "%s %s" % kv
end
ssinfo.rebuildinfo()
end
if nrecords == 0 && unique_keys.length > 0
idxname, idxkeys = ssinfo.findclosestindex(unique_keys)
if !idxname || idxkeys != unique_keys.to_set
lres = ssinfo.makenewindex(idxname, unique_keys)
if lres.include?('error')
return lres
end
res.merge!(lres)
end
end
lres = ssinfo.insertdata(ldata)
nrecords += 1
end
@db.commit()
return res
end
|