Top Level Namespace

Defined Under Namespace

Modules: Global Classes: Nil, Searcher

Instance Method Summary collapse

Instance Method Details

#get_list_from_bing(keyword, page = 2) ⇒ Object 



18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
 
# File 'lib/searcher/bing.rb', line 18

def get_list_from_bing(keyword,page=2)
  content = search_from_bing(keyword,page)

  super_link = Array.new
  regex = /<d:Url.*?<\/d:Url>/
  #Global.save_to_file(content,'bing.html','/htmls')

  content.scan(regex).each  do  |n|
    regex_http = /http.*?</
    real_url = n.match(regex_http)
    real_url = real_url.to_s.delete('<')
    #super_link.push(real_url)
    #Global.save_link_info(real_url, 'bing')
    super_link <<  [real_url,"bing"]
  end
  super_link
end

#get_list_from_google(keyword, page = 2) ⇒ Object 



25
26
27
28
29
 
# File 'lib/searcher/google.rb', line 25

def get_list_from_google(keyword,page=2)
  #content = search_from_google(keyword,page)
  #Global.save_to_file(content,'google.html','/htmls')
  search_from_google(keyword,page)
end

#search_from_bing(keyword, page = 2) ⇒ Object 



3
4
5
6
7
8
9
10
11
12
13
14
15
16
 
# File 'lib/searcher/bing.rb', line 3

def search_from_bing(keyword,page=2)
   = 'Onm2ZtMfIJsKdKLopx6/VpyADuqrdJPhsacwUuez7Ds='
  bing_keyword = 'https://api.datamarket.azure.com/Bing/Search/Web?Query=%27' + URI.encode(keyword) + '%27' + '&$skip=0'
  uri = URI(bing_keyword)

  req = Net::HTTP::Get.new(uri.request_uri)
  req.basic_auth('', )

  res = Net::HTTP.start(uri.hostname, uri.port, :use_ssl => uri.scheme == 'https') { |http|
    http.request(req)
  }

  res.body
end

#search_from_google(keyword, page = 2) ⇒ Object 



3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
 
# File 'lib/searcher/google.rb', line 3

def search_from_google(keyword,page=2)
  res,links = '',[]
  (1..page).each do |pn|
    url_with_keyword = 'https://www.googleapis.com/customsearch/v1?key=AIzaSyBvybq0NEaMtMkAkPUd7hhC-17AzcOc9x8&cx=013036536707430787589:_pqjad5hr1a&alt=json&fields=items(link)&q=' + URI.encode(keyword) + '&start=' + pn.to_s
    url = URI.parse(url_with_keyword)
    http = Net::HTTP.new(url.host, url.port)
    http.use_ssl = true
    http.verify_mode = OpenSSL::SSL::VERIFY_NONE
    request = Net::HTTP::Get.new(url.request_uri)
    response = http.request(request)
    res += response.body
    links_strings = JSON.parse(response.body) rescue nil
    links_strings['items'].each do |link|
      links << [link['link'],"google"]
      #Global.save_link_info(link['link'], 'google')
    end
    #links
  end
  links
end