Class: SeoPositionTracker::Scraper

Inherits:
Object
  • Object
show all
Defined in:
lib/seo-position-tracker-ruby.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(query, api_key, keywords, websites, language = nil, country = nil, location = nil, domain = nil) ⇒ Scraper

Returns a new instance of Scraper.



102
103
104
105
106
107
108
109
110
111
 
# File 'lib/seo-position-tracker-ruby.rb', line 102

def initialize(query, api_key, keywords, websites, language = nil, country = nil, location = nil, domain = nil)
    @query = query
    @api_key = api_key
    @keywords = keywords
    @websites = websites
    @language = language
    @country = country
    @location = location
    @domain = domain
end

Instance Attribute Details

#api_key ⇒ Object

Returns the value of attribute api_key.



100
101
102
 
# File 'lib/seo-position-tracker-ruby.rb', line 100

def api_key
  @api_key
end

#country ⇒ Object

Returns the value of attribute country.



100
101
102
 
# File 'lib/seo-position-tracker-ruby.rb', line 100

def country
  @country
end

#domain ⇒ Object

Returns the value of attribute domain.



100
101
102
 
# File 'lib/seo-position-tracker-ruby.rb', line 100

def domain
  @domain
end

#keywords ⇒ Object

Returns the value of attribute keywords.



100
101
102
 
# File 'lib/seo-position-tracker-ruby.rb', line 100

def keywords
  @keywords
end

#language ⇒ Object

Returns the value of attribute language.



100
101
102
 
# File 'lib/seo-position-tracker-ruby.rb', line 100

def language
  @language
end

#location ⇒ Object

Returns the value of attribute location.



100
101
102
 
# File 'lib/seo-position-tracker-ruby.rb', line 100

def location
  @location
end

#query ⇒ Object

Returns the value of attribute query.



100
101
102
 
# File 'lib/seo-position-tracker-ruby.rb', line 100

def query
  @query
end

#websites ⇒ Object

Returns the value of attribute websites.



100
101
102
 
# File 'lib/seo-position-tracker-ruby.rb', line 100

def websites
  @websites
end

Instance Method Details



240
241
242
 
# File 'lib/seo-position-tracker-ruby.rb', line 240

def print(data)
    puts JSON.pretty_generate(data)
end

#save_to_csv(data) ⇒ Object 



216
217
218
219
220
221
222
223
224
 
# File 'lib/seo-position-tracker-ruby.rb', line 216

def save_to_csv(data)
    keys = data[0].keys

    File.open("#{@query.gsub(' ', '_')}.csv", 'w', encoding: 'utf-8') do |csv_file|
        writer = CSV.new(csv_file)
        writer << keys
        data.each { |row| writer << row.values }
    end
end

#save_to_json(data) ⇒ Object 



226
227
228
229
230
 
# File 'lib/seo-position-tracker-ruby.rb', line 226

def save_to_json(data)
    File.open("#{@query.gsub(' ', '_')}.json", 'w', encoding: 'utf-8') do |json_file|
        json_file.write(JSON.pretty_generate(data))
    end
end

#save_to_txt(data) ⇒ Object 



232
233
234
235
236
237
238
 
# File 'lib/seo-position-tracker-ruby.rb', line 232

def save_to_txt(data)
    File.open("#{@query.gsub(' ', '_')}.txt", 'w', encoding: 'utf-8') do |txt_file|
        data.each do |element|
            txt_file.puts("#{element[:engine]}, #{element[:position]}, #{element[:title]}, #{element[:link]}")
        end
    end
end

#scrape_bing(country = 'us', location = 'United States') ⇒ Object 



133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
 
# File 'lib/seo-position-tracker-ruby.rb', line 133

def scrape_bing(country = 'us', location = 'United States')
    country, location = check_params(country=country, location=location)
    
    params = {
        api_key: @api_key,                # https://serpapi.com/manage-api-key
        q: @query,                        # search query
        engine: 'bing',                   # search engine
        cc: country,                      # country of the search
        location: location,               # location of the search
        count: 50                         # 50 results from Bing search
    }
    
    search = BingSearch.new(params)       # data extraction on the SerpApi backend
    results = search.get_hash             # JSON -> Ruby hash
    
    find_positions(results, 'bing')
end

#scrape_duckduckgo(location = 'us-en') ⇒ Object 



151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
 
# File 'lib/seo-position-tracker-ruby.rb', line 151

def scrape_duckduckgo(location = 'us-en')
    location = check_params(location=location)
    
    params = {
        api_key: @api_key,                # https://serpapi.com/manage-api-key
        q: @query,                        # search query
        engine: 'duckduckgo',             # search engine
        kl: location                      # location of the search
    }
    
    search = DuckduckgoSearch.new(params) # data extraction on the SerpApi backend
    results = search.get_hash             # JSON -> Ruby hash
    
    find_positions(results, 'duckduckgo')
end

#scrape_google(language = 'en', country = 'us', location = 'United States', domain = 'google.com') ⇒ Object 



113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
 
# File 'lib/seo-position-tracker-ruby.rb', line 113

def scrape_google(language = 'en', country = 'us', location = 'United States', domain = 'google.com')
    language, country, location, domain = check_params(language=language, country=country, location=location, domain=domain)
    
    params = {
        api_key: @api_key,                # https://serpapi.com/manage-api-key
        q: @query,                        # search query
        engine: 'google',                 # search engine
        google_domain: domain,            # Google domain to use
        hl: language,                     # language of the search
        gl: country,                      # country of the search
        location: location,               # location of the search
        num: 100                          # 100 results from Google search
    }
    
    search = GoogleSearch.new(params)     # data extraction on the SerpApi backend
    results = search.get_hash             # JSON -> Ruby hash
    
    find_positions(results, 'google')
end

#scrape_naver ⇒ Object 



202
203
204
205
206
207
208
209
210
211
212
213
214
 
# File 'lib/seo-position-tracker-ruby.rb', line 202

def scrape_naver
    params = {
        api_key: @api_key,                # https://serpapi.com/manage-api-key
        query: @query,                    # search query
        engine: 'naver',                  # search engine
        where: 'web'                      # web organic results
    }
    
    search = NaverSearch.new(params)      # data extraction on the SerpApi backend
    results = search.get_hash             # JSON -> Ruby hash
    
    find_positions(results, 'naver')
end

#scrape_yahoo(language = 'language_en', country = 'us', domain = 'uk') ⇒ Object 



167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
 
# File 'lib/seo-position-tracker-ruby.rb', line 167

def scrape_yahoo(language = 'language_en', country = 'us', domain = 'uk')
    language, country, domain = check_params(language=language, country=country, domain=domain)
    
    params = {
        api_key: @api_key,                # https://serpapi.com/manage-api-key
        p: @query,                        # search query
        engine: 'yahoo',                  # search engine
        yahoo_domain: domain,             # Yahoo! domain to use
        vl: language,                     # language of the search
        vc: country                       # country of the search
    }
    
    search = YahooSearch.new(params)      # data extraction on the SerpApi backend
    results = search.get_hash             # JSON -> Ruby hash
    
    find_positions(results, 'yahoo')
end

#scrape_yandex(language = 'en', domain = 'yandex.com') ⇒ Object 



185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
 
# File 'lib/seo-position-tracker-ruby.rb', line 185

def scrape_yandex(language = 'en', domain = 'yandex.com')
    language, domain = check_params(language=language, domain=domain)
    
    params = {
        api_key: @api_key,                # https://serpapi.com/manage-api-key
        text: @query,                     # search query
        engine: 'yandex',                 # search engine
        yandex_domain: domain,            # Yandex domain to use
        language: language                # language of the search
    }
    
    search = YandexSearch.new(params)     # data extraction on the SerpApi backend
    results = search.get_hash             # JSON -> Ruby hash
    
    find_positions(results, 'yandex')
end