Class: Gnip::GnipFullArchive::FullArchive

Inherits:
Object
  • Object
show all
Includes:
HTTParty
Defined in:
lib/gnip/gnip-full-archive/full_archive.rb

Defined Under Namespace

Classes: InvalidRequestException

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(client) ⇒ FullArchive

alias :total :total_entries



14
15
16
17
18
 
# File 'lib/gnip/gnip-full-archive/full_archive.rb', line 14

def initialize(client)
  @search_url = "https://data-api.twitter.com/search/fullarchive/accounts/#{client.}/#{client.label}.json"
  @counts_url = "https://data-api.twitter.com/search/fullarchive/accounts/#{client.}/#{client.label}/counts.json"
  @auth = { username: client.username, password: client.password }
end

Instance Attribute Details

#counts_url ⇒ Object (readonly)

Returns the value of attribute counts_url.



10
11
12
 
# File 'lib/gnip/gnip-full-archive/full_archive.rb', line 10

def counts_url
  @counts_url
end

#search_url ⇒ Object (readonly)

Returns the value of attribute search_url.



10
11
12
 
# File 'lib/gnip/gnip-full-archive/full_archive.rb', line 10

def search_url
  @search_url
end

Instance Method Details

#search(options = {}) ⇒ Object

Search using the full-archive search endpoint return an hash containing up to 500 results and the cursor to the next page options query to twitter options default is 500 options as datetime options as datetime options cursor to the next page



26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
 
# File 'lib/gnip/gnip-full-archive/full_archive.rb', line 26

def search(options = {})
  search_options = {}
  search_options[:query] = options[:query] || ''
  search_options[:maxResults]  = options[:per_page] || 500
  search_options[:fromDate]    = Gnip.format_date(options[:date_from]) if options[:date_from]
  search_options[:toDate]      = Gnip.format_date(options[:date_to]) if options[:date_to]
  search_options[:next] = options[:next_cursor] if options[:next_cursor]
  url = [search_url, search_options.to_query].join('?')
  begin
    gnip_call = self.class.get(url, basic_auth: @auth)
    response = gnip_call.response
    parsed_response = gnip_call.parsed_response
    parsed_response = (parsed_response || {}).with_indifferent_access
    raise response.message unless parsed_response.present?

    if parsed_response[:error].present?
      response = { results: [],
                   next: nil,
                   url: url,
                   error: parsed_response[:error][:message],
                   code: response.code.to_i }
    else
      response = { results: parsed_response[:results],
                   url: url,
                   next: parsed_response[:next],
                   code: response.code.to_i }
    end
  rescue StandardError => e
    response = { results: [],
                 url: url,
                 next: nil,
                 error: e.message,
                 code: 500 }
  end
  response
end

#total(options = {}) ⇒ Object

return total contents in a specific date interval with a passed query



111
112
113
114
115
116
 
# File 'lib/gnip/gnip-full-archive/full_archive.rb', line 111

def total(options = {})
  extra = {}
  response = total_by_time_period(options)
  extra = { error: response[:error] } if response[:error].present?
  { query: options[:query], total: response[:results].map { |item| item[:count] }.reduce(:+), calls: response[:calls] }.merge!(extra)
end

#total_by_time_period(options = {}) ⇒ Object

full aarchive search endpoints return total contents by day, minute, hour paginated so to get totals across time period passed may need to run more than one call, the stop condition is cursor nil bucket: must be one of [minute, hour, day]



66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
 
# File 'lib/gnip/gnip-full-archive/full_archive.rb', line 66

def total_by_time_period(options = {})
  response = options[:response] || {}
  search_options = {}
  search_options[:query]    = options[:query] || ''
  search_options[:bucket]   = options[:bucket] || 'day'
  search_options[:fromDate] = Gnip.format_date(options[:date_from]) if options[:date_from]
  search_options[:toDate]   = Gnip.format_date(options[:date_to]) if options[:date_to]
  search_options[:next]     = options[:next_cursor] if options[:next_cursor]

  url = [counts_url, search_options.to_query].join('?')
  call_done = 0

  begin
    gnip_call = self.class.get(url, basic_auth: @auth)

    parsed_response = gnip_call.parsed_response
    parsed_response = (parsed_response || {}).with_indifferent_access

    raise gnip_call.response.message unless parsed_response.present?

    if parsed_response[:error].present?
      response = { results: [], next: nil, error: parsed_response[:error][:message], code: gnip_call.response.code.to_i, calls: (response[:calls] || 0) + 1 }
    else
      call_done = 1 # we have received a valid response
      parsed_response[:results].each_with_index do |item, i|
        parsed_response[:results][i] = item.merge(timePeriod: DateTime.parse(item[:timePeriod]).to_s)
      end
      response = { results: (response[:results] || []) + parsed_response[:results], next: parsed_response[:next], code: gnip_call.response.code.to_i, calls: (response[:calls] || 0) + 1 }
    end
  rescue StandardError => e
    response = { results: [], next: nil, error: e.message, code: 500, calls: (response[:calls] || 0) + call_done }
  end
  # If the next cursor is not present we fetched all the data
  # It happens that twitter returns the same cursor, in that case we stop
  return response if !parsed_response[:next].to_s.present? || (parsed_response[:next].to_s.present? && parsed_response[:next] == search_options[:next])

  total_by_time_period(query: search_options[:query],
                       date_from: search_options[:fromDate],
                       date_to: search_options[:toDate],
                       bucket: search_options[:bucket],
                       next_cursor: parsed_response[:next],
                       response: response)
end