Module: ShoppingBulkAPI

Defined in:
lib/api_helpers/shopping_bulk_api.rb

Defined Under Namespace

Classes: SearchType

Class Method Summary collapse

Class Method Details

.batch_search_v3(search_hash, sandbox = false) ⇒ Object

batch lookup products takes a search hash, which should have a :search_type key (one member from SearchType Enum above) for searching products, pass an array of products in :products (:search_type => ShoppingBulkAPI::SearchType::PRODUCT) for searching from shopping product ids, pass an array of shopping product ids in :shopping_product_ids (:search_type => ShoppingBulkAPI::SearchType::SHOPPING_PRODUCT_ID) for those two above, you can pass :batch_lookup, which is how many we should look up w/ shopping at once for searching a keyword, pass a :keywords array of strings (any you want to be included in results, ordered) (:search_type => ShoppingBulkAPI::SearchType::KEYWORDS)



58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
 
# File 'lib/api_helpers/shopping_bulk_api.rb', line 58

def self.batch_search_v3(search_hash, sandbox=false)  
  search_hash[:batch_lookup] ||= 20
  
  case search_hash[:search_type]
  when SearchType::SHOPPING_PRODUCT_ID
    items = search_hash[:shopping_product_ids]
    search_hash[:get_extra_product_info] ||= false
  when SearchType::PRODUCT
    # list of shopping product ids to their associated product
    # useful to bring back the shopping_product_ids into products
    search_hash[:product_ids_hash] = search_hash[:products].inject({}) do |ha, product|
      shopping_product_source = product.shopping_ids.detect{|product_source| !product_source.questionable?}
      unless shopping_product_source.nil?
        shopping_id = shopping_product_source.source_id
        if ha.has_key?(shopping_id)
          puts "DUPLICATE KEY FOR #{shopping_id} !! #{ha[shopping_id].inspect} VS #{product.id}"
        end
        ha[shopping_id] = product.id
      end
      ha
    end
    # just the shopping product ids
    search_hash[:product_ids] = search_hash[:product_ids_hash].keys
    items = search_hash[:product_ids]
    search_hash[:get_extra_product_info] ||= false
  when SearchType::KEYWORDS
    items = search_hash[:keywords]
    search_hash[:get_extra_product_info] = true # force extra info, how else will we get the name/etc. ?!
  else
    raise ArgumentError, "Invalid :search_type specified: #{search_hash[:search_type].inspect}"
  end
  
  # defaults
  all_offers = self.default_offers
  all_product_infos = self.default_product_infos(search_hash[:get_extra_product_info])
  missed_ids = []
  second_misses = []
  # puts "SEARCH HASH: #{search_hash.inspect}"
  # look 'em up in batches!
  items.each_slice(search_hash[:batch_lookup]) do |batch_items|
    search_hash[:batch_items] = batch_items
    misses, offers, product_infos = self.single_batch_search_v3(search_hash, sandbox)
    all_product_infos.update(product_infos)
    all_offers.update(offers)
    missed_ids += misses unless misses.empty?
  end
  
  # for the ones we missed, we're going to try looking them up one more time
  # before giving up entirely.
  # (only applies to non-category/keyword searches)
  if missed_ids.length > 0
    # now look up the missed IDs in their own batch
    missed_ids.each_slice(search_hash[:batch_lookup]) do |batch_items|
      search_hash[:batch_items] = batch_items
      misses, offers, product_infos = self.single_batch_search_v3(search_hash, sandbox)
      all_product_infos.update(product_infos)
      all_offers.update(offers)
      second_misses += misses unless misses.empty?
      offers = nil
      product_infos = nil
    end
  end
  
  # only care to look up one-by-one if we're going to do something with the data
  # (for product lookups, then, to hide or update shopping ids)
  if !second_misses.empty? && search_hash[:search_type] == SearchType::PRODUCT
    # missed again? gotta look up one-by-one!
    products_to_hide = []
    second_misses.each do |product_id|
      our_product_id = search_hash[:product_ids_hash][product_id]
      search_hash[:batch_items] = [product_id]
      final_miss, offers, product_infos = self.single_batch_search_v3(search_hash, sandbox)
      all_product_infos.update(product_infos)
      all_offers.update(offers)
      if !final_miss.empty?
        puts "****** COULDN'T LOOK UP INFO FOR #{our_product_id} ( SHOPPING ID #{product_id}) !! Adding to hide queue..."
        products_to_hide << product_id
      else
        # shopping gave us a product ID back that doesn't match our shopping product ID! gotta update!
        new_shopping_id = product_infos[our_product_id][:reported_product_id]
        if ps = ProductSource.find_by_source_name_and_source_id(ProductSource::Name::SHOPPING, new_shopping_id)
          puts "SHOPPING PRODUCT ID ALREADY EXISTS AT #{ps.product_id} -- HIDING DUPLICATE #{our_product_id}"
          products_to_hide << product_id
        else
          puts "UPDATING SOURCE ID: FROM #{product_id.inspect} TO #{new_shopping_id.inspect} FOR product id##{our_product_id}"
          ProductSource.update_all("source_id = E'#{new_shopping_id}'", "source_id = E'#{product_id}' AND product_id = #{our_product_id}")
        end
      end
    end
    puts "HIDING PRODUCTS: #{products_to_hide.inspect}"
    if products_to_hide.length > 0
      products_to_hide.each do |shopping_product_id|
        ProductSource.increment_not_found_count(ProductSource::Name::SHOPPING, shopping_product_id)
      end
    end
  end
  # return all that jazz
  [all_offers, all_product_infos]
end

.default_offers ⇒ Object 



183
184
185
 
# File 'lib/api_helpers/shopping_bulk_api.rb', line 183

def self.default_offers
  Hash.new([]).clone
end

.default_product_infos(get_extra_product_info) ⇒ Object 



187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
 
# File 'lib/api_helpers/shopping_bulk_api.rb', line 187

def self.default_product_infos get_extra_product_info
  # smart defaults for error handling
  if get_extra_product_info
    product_infos = Hash.new({
      :avg_secondary_cpcs => nil,
      :primary_cpc => nil,
      :reported_product_id => nil,
      :images => {},
      :manufacturer => nil,
      :name => nil,
      :description => nil,
      :review_url => nil,
      :review_count => nil,
      :rating => nil
    })
  else # even if they don't ask for it! BAM!
    product_infos = Hash.new({
      :avg_secondary_cpcs => nil,
      :primary_cpc => nil,
      :reported_product_id => nil
    })
  end
  
  product_infos.clone
end

.do_search_v3(search_hash, sandbox = false) ⇒ Object 



213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
 
# File 'lib/api_helpers/shopping_bulk_api.rb', line 213

def self.do_search_v3(search_hash, sandbox=false)
  # defaults
  offers = self.default_offers
  product_infos = self.default_product_infos(search_hash[:get_extra_product_info])
  misses = []
  
  case search_hash[:search_type]
  when SearchType::PRODUCT, SearchType::SHOPPING_PRODUCT_ID
    search_hash[:batch_items].compact! # remove nils
    if search_hash[:batch_items].empty?
      # nothing to look for! dummy.
      puts "NO PRODUCT ID PASSED!"
      # return blanks
      return [misses, offers, product_infos]
    end
    params = {
      'productId' => search_hash[:batch_items],
      'showProductOffers' => true,
      'trackingId' => search_hash[:tracking_id],
      'numItems' => 20
    }
  when SearchType::KEYWORDS
    params = {
      #'categoryId' => search_hash[:category],
      'keyword' => Array(search_hash[:keywords].collect{|x| CGI::escape(x) }), # can be an array, thass coo' wit me.
      'showProductOffers' => true,
      'trackingId' => search_hash[:tracking_id],
      'numOffersPerProduct' => 20,
      'pageNumber' => 1,
      'numItems' => (search_hash[:num_items].nil? || search_hash[:num_items].to_s.empty?) ? 1 : search_hash[:num_items],
      # 'productSortType' => 'price',
      # 'productSortOrder' => 'asc'
    }
  end
  
  result = make_v3_request :GeneralSearch, params, sandbox
  

  if search_hash[:search_type] == SearchType::PRODUCT || search_hash[:search_type] == SearchType::SHOPPING_PRODUCT_ID
    if search_hash[:batch_items].length == 1 && result.at('product')
      # if we're looking up one ID, it doesn't matter if the ID they returned doesn't match ours
      misses = []
    elsif result.at('product') # if we got ANY products back
      misses = search_hash[:batch_items] - (result / 'product').collect{|x| x.attributes['id']}
    else # probably an error happened
      misses = search_hash[:batch_items]
    end
  end
  
  errors = result.search('exception[@type=error]')
  if errors.length > 0
    # we got an error, or more than one!
    if (search_hash[:search_type] == SearchType::PRODUCT || search_hash[:search_type] == SearchType::SHOPPING_PRODUCT_ID) && search_hash[:batch_items].length == 1 && errors.length == 1 && errors.first.at('message').innerText == "Could not find ProductIDs #{search_hash[:batch_items].first}"
      # happens when we look up one product id and it's not a valid product id according to shopping. we ignore this kind of error.
    else
      puts "*** ERROR *** Could not look up offers by product ids:"
      errors.each do |error|
        puts " - #{error.at('message').innerText}"
      end
      # notify hoptoad of this shit!
      HoptoadNotifier.notify(
        :error_class => "ShoppingOfferError",
        :error_message => %{
          We got error(s) while trying to get the shopping offers! #{search_hash[:batch_items].inspect}
        },
        :request => { :params => Hash[*errors.collect{|x| ["Error ##{errors.index(x)}", x.at('message').innerText] }.flatten] }
      )
    end
    
    # return blanks
    return [misses, offers, product_infos]
  end
  
  (result / 'product').each do |product|
    product_id = product.attributes['id']
    if (search_hash[:search_type] == SearchType::PRODUCT || search_hash[:search_type] == SearchType::SHOPPING_PRODUCT_ID) 
      # this happens when they give us back an ID that we didn't ask for
      # (if we are only looking at one product, we don't care what ID they give us back,
      # we know it's the product we were looking for)
      if search_hash[:batch_items].length == 1 && !search_hash[:batch_items].include?(product_id)
        product_id = search_hash[:batch_items].first # revert back to the ID we asked for, we put the other in product_infos[x][:reported_product_id]
      elsif !search_hash[:batch_items].include?(product_id)
        # skip it, already included in the misses ( hopefully ... )
        next
      end
    end
    
    offers[product_id]={}
    product_infos[product_id] = {
      :reported_product_id => product.attributes['id'] # their reported ID doesn't necessarily match up with our ID
    }
    if search_hash[:get_extra_product_info]
      product_infos[product_id][:name] = product.at('name').innerText
      product_infos[product_id][:review_url] = (product.at('reviewURL').innerText rescue nil)
      product_infos[product_id][:review_count] = (product.at('rating/reviewCount').innerText rescue nil)
      product_infos[product_id][:rating] = (product.at('rating/rating').innerText rescue nil)
      
      try_description = product.at('fullDescription').innerText
      if try_description.nil? || try_description.empty?
        try_description = product.at('shortDescription').innerText
      end
      product_infos[product_id][:description] = (try_description.nil? || try_description.empty?) ? '' : try_description[0...255]
        
      images = (product / 'images' / 'image[@available="true"]').collect{|x|
        {
          :width => x.attributes['width'].to_i,
          :height => x.attributes['height'].to_i,
          :url => x.at('sourceURL').innerText
        }
      }.sort_by{|x| x[:width] * x[:height] }
      
      product_infos[product_id][:images] = {
        :small_image => images[0],
        :medium_image => images[1],
        :large_image => images[2]
      }
        
      # possible_manufacturers = (product / 'offer > manufacturer').collect{|x| x.innerText}.compact.uniq
      # 
      # if possible_manufacturers.length == 1
      #   product_infos[product_id][:manufacturer] = possible_manufacturers.first # easy peasy lemon squezy
      # elsif possible_manufacturers.length > 1
      #   # figure out which manufacturer is the most popular
      #   manufacturers_popularity_index = possible_manufacturers.inject({}) {|ha, manufacturer| ha[manufacturer] ||= 0; ha[manufacturer] += 1; ha }
      #   product_infos[product_id][:manufacturer] = manufacturers_popularity_index.sort_by{|key, val| val }.last.first
      # else
      #   product_infos[product_id][:manufacturer] = nil # zip, zero, doodad :(
      # end
    end
    
    (product.at('offers') / 'offer').each do |offer|
      store = offer.at('store')
      store_hash = {
        :name => store.at('name').innerText,
        :trusted => store.attributes['trusted'] == "true",
        :id => store.attributes['id'].to_i,
        :authorized_reseller => store.attributes['authorizedReseller'] == "true"
      }
       = store.at('logo')
      if .attributes['available'] == "true"
        store_hash[:logo] = {
          :width => .attributes['width'],
          :height => .attributes['height'],
          :url => .at('sourceURL').innerText
        }
      else
        store_hash[:logo] = nil
      end

      # store rating
      store_rating = store.at('ratingInfo')
      store_hash[:rating] = {
        :number => store_rating.at('rating').nil? ? nil : normalize_merchant_rating(store_rating.at('rating').innerText.to_f),
        :count => store_rating.at('reviewCount').innerText.to_i,
        :url =>  store_rating.at('reviewURL').nil? ? nil : store_rating.at('reviewURL').innerText
      }
      shipping_info = offer.at('shippingCost').attributes['checkSite'] == "true" ? nil : to_d_or_nil(offer.at('shippingCost').innerText)
      price_info = to_d_or_nil(offer.at('basePrice').innerText)
      if shipping_info && price_info
        total_price = shipping_info + price_info
      else
        total_price = price_info
      end

      # in-stock
      stock_status = offer.at('stockStatus').innerText
      in_stock = stock_status != 'out-of-stock' && stock_status != 'back-order'

      if in_stock
        offers[product_id][store_hash[:id]] = { :merchant_code => store_hash[:id].to_s,
                                                :merchant_name => store_hash[:name],
                                                :merchant_logo_url => store_hash[:logo].nil? ? nil : store_hash[:logo][:url],
                                                :cpc => offer.at('cpc').nil? ? nil : (offer.at('cpc').innerText.to_f*100).to_i,
                                                :price => to_d_or_nil(offer.at('basePrice').innerText),
                                                :shipping => offer.at('shippingCost').attributes['checkSite'] == "true" ? nil : to_d_or_nil(offer.at('shippingCost').innerText),
                                                :offer_url => offer.at('offerURL').innerText,
                                                :offer_tier => 1,
                                                :merchant_rating => store_hash[:rating][:number],
                                                :num_merchant_reviews => store_hash[:rating][:count] }
      end
    end
    # return an array, don't care about the hash. was used for dup checking.
    offers[product_id] = offers[product_id].values.sort_by{|x| x[:price] + (x[:shipping] || 0) }
  end
  
  [misses, offers, product_infos]
end

.find_related_terms_v3(keyword, sandbox = false) ⇒ Object 



422
423
424
425
 
# File 'lib/api_helpers/shopping_bulk_api.rb', line 422

def self.find_related_terms_v3 keyword, sandbox=false
   result = make_v3_request :GeneralSearch, {'keyword' => keyword}, sandbox
   (result / 'relatedTerms > term').collect{|x| x.innerText}
end

.get_all_categories ⇒ Object 



12
13
14
15
16
17
18
19
 
# File 'lib/api_helpers/shopping_bulk_api.rb', line 12

def self.get_all_categories
  params = {
    'categoryId' => 0,
    'showAllDescendants' => true
  }
  result = make_v3_request :CategoryTree, params
  parse_category(result.at('category[@id="0"]'))
end

.get_attribute_from_shopping_id_v3(shopping_id, attribute) ⇒ Object

get any ol’ random attribute from a shopping id for instance, ‘Screen Size’ is a good’un.



407
408
409
410
411
412
413
 
# File 'lib/api_helpers/shopping_bulk_api.rb', line 407

def self.get_attribute_from_shopping_id_v3 shopping_id, attribute
  product_info = find_by_product_id_v3 shopping_id
  values = product_info[:specifications].values.flatten
  index = values.index(attribute)
  # we +1 here because the flattened values are [name, value] oriented
  index.nil? ? nil : values[index+1]
end

.normalize_merchant_rating(merchant_rating) ⇒ Object 



401
402
403
 
# File 'lib/api_helpers/shopping_bulk_api.rb', line 401

def self.normalize_merchant_rating(merchant_rating)
  merchant_rating.nil? ? nil : (merchant_rating * 20.0).round
end

.parse_category(category, parent_id = nil) ⇒ Object

parse a category, then look for sub-categories and parse those too!



22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
 
# File 'lib/api_helpers/shopping_bulk_api.rb', line 22

def self.parse_category(category, parent_id=nil)
  categories = []
  id = category.attributes['id'].to_i
  # main category, does not count as a parent and should not be added
  if id == 0
    name = nil
    id = nil
  else
    name = category.at('name').innerText
  end
  hash = {
    :banned => banned_categories.include?(name),
    :id => id,
    :name => name,
    :parent_id => parent_id
  }
  if sub_categories = category.at('categories')
    categories << hash.merge({:end_point => false}) unless name == nil || id == nil
    # if there are sub categories, we don't want to add the parent to the list
    # that we'll be searching.
    (sub_categories / '> category').each do |sub_category|
      categories += parse_category(sub_category, id)
    end
  else
    categories << hash.merge({:end_point => true})
  end
  
  categories
end

.parse_images_v3(images_element) ⇒ Object 



415
416
417
418
419
420
 
# File 'lib/api_helpers/shopping_bulk_api.rb', line 415

def self.parse_images_v3 images_element
  images_element.inject({}) do |ha,obj|
    ha["#{obj.attributes['width']}x#{obj.attributes['height']}"] = [obj.attributes['available'] == 'true', obj.at('sourceURL').innerText]
    ha
  end
end

.single_batch_search_v3(search_hash, sandbox = false) ⇒ Object

find a single batch of offers and shove them info all_offers hash this is just a helper for batch_search_v3 and shouldn’t be called directly for looking up a whole lot of product offers, look at batch_search_v3 above



161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
 
# File 'lib/api_helpers/shopping_bulk_api.rb', line 161

def self.single_batch_search_v3(search_hash, sandbox=false)
  misses, offers, product_infos = do_search_v3(search_hash, sandbox)
          
  # turn shopping ids into product ids for the returned results ( both offers and product_infos )
  # (only if they initially gave us a set of products)
  if search_hash[:search_type] == SearchType::PRODUCT
    search_hash[:batch_items].each do |product_id|
      [offers, product_infos].each do |item|
        our_id = search_hash[:product_ids_hash][product_id]
        # if OUR product id is the same as shopping's, we don't delete. obvi.
        next if our_id == product_id
        if our_id.nil?
          puts "******** NIL FOR #{product_id}"
        end
        item[our_id] = item[product_id]
        item.delete(product_id)
      end
    end
  end
  [misses, offers, product_infos]
end