Class: Gnip::GnipFullArchive::FullArchive

Inherits:
Object
  • Object
show all
Includes:
HTTParty
Defined in:
lib/gnip/gnip-full-archive/full_archive.rb

Direct Known Subclasses

ThirtyDay

Defined Under Namespace

Classes: InvalidRequestException

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(client) ⇒ FullArchive

alias :total :total_entries



15
16
17
18
19
 
# File 'lib/gnip/gnip-full-archive/full_archive.rb', line 15

def initialize(client)
  @search_url = "https://data-api.twitter.com/search/fullarchive/accounts/#{client.}/#{client.label}.json"
  @counts_url = "https://data-api.twitter.com/search/fullarchive/accounts/#{client.}/#{client.label}/counts.json"
  @auth = { username: client.username, password: client.password }
end

Instance Attribute Details

#counts_url ⇒ Object (readonly)

Returns the value of attribute counts_url.



11
12
13
 
# File 'lib/gnip/gnip-full-archive/full_archive.rb', line 11

def counts_url
  @counts_url
end

#search_url ⇒ Object (readonly)

Returns the value of attribute search_url.



11
12
13
 
# File 'lib/gnip/gnip-full-archive/full_archive.rb', line 11

def search_url
  @search_url
end

Instance Method Details

#search(options = {}) ⇒ Object

Search using the full-archive search endpoint return an hash containing up to 500 results and the cursor to the next page options query to twitter options default is 500 options as datetime options as datetime options cursor to the next page



27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
 
# File 'lib/gnip/gnip-full-archive/full_archive.rb', line 27

def search(options = {})
  search_options = {}
  search_options[:query] = options[:query] || ''
  search_options[:maxResults] = options[:per_page] || 500
  search_options[:fromDate] = Gnip.format_date(options[:date_from]) if options[:date_from]
  search_options[:toDate] = Gnip.format_date(options[:date_to]) if options[:date_to]
  search_options[:next] = options[:next_cursor] if options[:next_cursor]
  url = [search_url, search_options.to_query].join('?')
  begin
    gnip_call = self.class.get(url, basic_auth: @auth)
    response = gnip_call.response
    parsed_response = gnip_call.parsed_response
    parsed_response = (parsed_response || {}).with_indifferent_access
    raise response.message unless parsed_response.present?

    if parsed_response[:error].present?
      response = { results: [],
                  next: nil,
                  url: url,
                  error: parsed_response[:error][:message],
                  code: response.code.to_i }
    else
      response = { results: parsed_response[:results],
                  url: url,
                  next: parsed_response[:next],
                  code: response.code.to_i }
    end
  rescue StandardError => e
    response = { results: [],
                url: url,
                next: nil,
                error: e.message,
                code: 500 }
  end
  response
end

#total(options = {}) ⇒ Object

return total contents in a specific date interval with a passed query



112
113
114
115
116
117
 
# File 'lib/gnip/gnip-full-archive/full_archive.rb', line 112

def total(options = {})
  extra = {}
  response = total_by_time_period(options)
  extra = { error: response[:error] } if response[:error].present?
  { query: options[:query], total: response[:results].map { |item| item[:count] }.reduce(:+), calls: response[:calls] }.merge!(extra)
end

#total_by_time_period(options = {}) ⇒ Object

full aarchive search endpoints return total contents by day, minute, hour paginated so to get totals across time period passed may need to run more than one call, the stop condition is cursor nil bucket: must be one of [minute, hour, day]



67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
 
# File 'lib/gnip/gnip-full-archive/full_archive.rb', line 67

def total_by_time_period(options = {})
  response = options[:response] || {}
  search_options = {}
  search_options[:query] = options[:query] || ''
  search_options[:bucket] = options[:bucket] || 'day'
  search_options[:fromDate] = Gnip.format_date(options[:date_from]) if options[:date_from]
  search_options[:toDate] = Gnip.format_date(options[:date_to]) if options[:date_to]
  search_options[:next] = options[:next_cursor] if options[:next_cursor]

  url = [counts_url, search_options.to_query].join('?')
  call_done = 0

  begin
    gnip_call = self.class.get(url, basic_auth: @auth)

    parsed_response = gnip_call.parsed_response
    parsed_response = (parsed_response || {}).with_indifferent_access

    raise gnip_call.response.message unless parsed_response.present?

    if parsed_response[:error].present?
      response = { results: [], next: nil, error: parsed_response[:error][:message], code: gnip_call.response.code.to_i, calls: (response[:calls] || 0) + 1 }
    else
      call_done = 1 # we have received a valid response
      parsed_response[:results].each_with_index do |item, i|
        parsed_response[:results][i] = item.merge(timePeriod: DateTime.parse(item[:timePeriod]).to_s)
      end
      response = { results: (response[:results] || []) + parsed_response[:results], next: parsed_response[:next], code: gnip_call.response.code.to_i, calls: (response[:calls] || 0) + 1 }
    end
  rescue StandardError => e
    response = { results: [], next: nil, error: e.message, code: 500, calls: (response[:calls] || 0) + call_done }
  end
  # If the next cursor is not present we fetched all the data
  # It happens that twitter returns the same cursor, in that case we stop
  return response if !parsed_response[:next].to_s.present? || (parsed_response[:next].to_s.present? && parsed_response[:next] == search_options[:next])

  total_by_time_period(query: search_options[:query],
                       date_from: search_options[:fromDate],
                       date_to: search_options[:toDate],
                       bucket: search_options[:bucket],
                       next_cursor: parsed_response[:next],
                       response: response)
end