Class: GoodDataMarketo::RESTAdapter

Inherits:
Object
  • Object
show all
Defined in:
lib/gooddata_marketo/adapters/rest.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(config = {}) ⇒ RESTAdapter

Returns a new instance of RESTAdapter.



12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
 
# File 'lib/gooddata_marketo/adapters/rest.rb', line 12

def initialize config = {}

  @api_limit = MARKETO_API_LIMIT
  @lead_list_dump = config[:file] || LEAD_LIST_DUMP_CSV || 'marketo_leads_dump.csv'
  @marketo_subdomain = config[:subdomain] || MARKETO_SUBDOMAIN
  @marketo_rest_secret = config[:secret] || config[:user] || MARKETO_REST_SECRET
  @marketo_rest_id = config[:id] || config[:user] || MARKETO_REST_ID
  @marketo_domain = "https://#{@marketo_subdomain}.mktorest.com"
  @webdav = config[:webdav]
  @lead_list_ids = []
  @token = self.get_token unless config[:token]
  @token_uri = "?access_token=#{@token}"

  # Example endpoint = "/rest/v1/lists.json"

end

Instance Attribute Details

#token ⇒ Object

Returns the value of attribute token.



9
10
11
 
# File 'lib/gooddata_marketo/adapters/rest.rb', line 9

def token
  @token
end

#token_timer ⇒ Object

Returns the value of attribute token_timer.



10
11
12
 
# File 'lib/gooddata_marketo/adapters/rest.rb', line 10

def token_timer
  @token_timer
end

Instance Method Details

#clean ⇒ Object 



232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
 
# File 'lib/gooddata_marketo/adapters/rest.rb', line 232

def clean

  lli = 'lead_list_ids.json'
  llnpt = 'lead_list_next_page_token.json'

  # Remove list ids from webdav and local
  if @webdav.exists?(lli)
    @webdav.delete(lli)
  end

  if @webdav.exists?(llnpt)
    @webdav.delete(llnpt)
  end

  File.delete(lli) if File.exists? lli
  File.delete(llnpt) if File.exists? llnpt

  # Upload marketo_dump to webdav for backup.
  @webdav.upload(@lead_list_dump)

end

#get(url) ⇒ Object 



39
40
41
42
43
44
45
46
47
48
 
# File 'lib/gooddata_marketo/adapters/rest.rb', line 39

def get url
  @api_limit -= 1
  begin
    puts "#{Time.now} => REST:GET (#{url})" if GoodDataMarketo.logging
    response = RestClient.get url
  rescue Exception => exc
    puts exc if GoodDataMarketo.logging
    puts "#{Time.now} => Possible API Limit reached, last request was  ##{@api_limit} of #{MARKETO_API_LIMIT}"
  end
end

#get_all_lead_emails(config = {}) ⇒ Object 



186
187
188
189
 
# File 'lib/gooddata_marketo/adapters/rest.rb', line 186

def get_all_lead_emails config = {}
  config[:fields] = 'email'
  self.get_all_leads config
end

#get_all_lead_ids(config = {}) ⇒ Object 



181
182
183
184
 
# File 'lib/gooddata_marketo/adapters/rest.rb', line 181

def get_all_lead_ids config = {}
  config[:fields] = 'id'
  self.get_all_leads config
end

#get_all_leads(config = {}) ⇒ Object 



90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
 
# File 'lib/gooddata_marketo/adapters/rest.rb', line 90

def get_all_leads config = {}

  self.refresh_token?

  # Check if we already have a leads list, if so, download it.
  self.load_lists

  # Check if there was a stream broken while downloading a specific list.
  self.load_last_page

  # Check if there was a partial ID dump available on WebDAV, if so download it..
  self.load_id_dump

  @fields = config[:fields] || 'id'

  # For each of the lead list ids, download their respective leads by id.
  @csv = CSV.open(@lead_list_dump,'a+')

  lists = @lead_list_ids

  lists.each do |list|

    id = @lead_list_ids.shift

    domain = @marketo_domain
    parameters = "&fields=#{@fields}"
    endpoint= "/rest/v1/list/#{id}/leads.json"
    url = domain + endpoint + @token_uri + parameters

    # Conditional recursive function ends if nextPageToken is not available in response. Writes results to CSV.
    def rest_lead_stream url, list_id

      response = self.get url
      json = JSON.parse(response.body, :symbolize_names => true)
      results = json[:result]

      if results
        results.each { |result| @csv << [result[:id]] }
        @csv.flush
        puts "#{Time.now} => REST:leadList:#{list_id}:Results:#{results.length}" if GoodDataMarketo.logging
      end

      next_page_token = json[:nextPageToken]

      # If there is another page, remember it and then attempt the next load.
      if next_page_token

        self.remember_next_page :token => token, :list => list_id
        domain = @marketo_domain
        parameters = "&fields=#{@fields}"
        endpoint= "/rest/v1/list/#{list_id}/leads.json"
        url = domain + endpoint + @token_uri + parameters + "&nextPageToken=" + next_page_token
        rest_lead_stream url, list_id

      else

        # Update the local and remote lead lists
        File.open('lead_list_ids.json','w'){ |f| JSON.dump(@lead_list_ids, f) }
        @webdav.upload('lead_list_ids.json')

      end

    end

    # While on this list, check to see if it failed working on it before and resume where it left off.
    if @next_page_token_list == list

      puts "#{Time.now} => REST(Resumed):leadList:#{list}:NextPageToken:#{@next_page_token}" if GoodDataMarketo.logging
      domain = @marketo_domain
      parameters = "" #"&fields=#{fields}"
      endpoint= "/rest/v1/list/#{list}/leads.json"
      url = domain + endpoint + @token_uri + parameters + "&nextPageToken=" + @next_page_token

      rest_lead_stream url, list

    else

      rest_lead_stream url, list

    end

  end

  # Remove lists from WebDav & Local, upload Marketo Ids Dump to webdav.
  self.clean

  # Update the etl controller that an update as occured.
  self.resolve_with_etl_controller

end

#get_lead_lists(next_page_token = nil) ⇒ Object 



50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
 
# File 'lib/gooddata_marketo/adapters/rest.rb', line 50

def get_lead_lists next_page_token = nil

  endpoint = "/rest/v1/lists.json"
  url = @marketo_domain + endpoint + @token_uri

  if next_page_token
    endpoint = "/rest/v1/lists.json"
    url = @marketo_domain + endpoint + @token_uri + "&nextPageToken="+next_page_token
  end

  response = self.get url

  json = JSON.parse(response.body, :symbolize_names => true)

  json[:result].each { |m|  @lead_list_ids << m[:id] }

  next_page_token = json[:nextPageToken]

  if next_page_token
    self.get_lead_lists next_page_token
  else
    @lead_list_ids
  end

end

#get_multiple_leads(config = {}) ⇒ Object 



76
77
78
79
80
81
82
83
84
85
86
87
88
 
# File 'lib/gooddata_marketo/adapters/rest.rb', line 76

def get_multiple_leads config = {}

  self.refresh_token?

  ids = config[:ids] || config[:values] || [28567,30885,32240,37161,40832]
  domain = @marketo_domain
  parameters = "&filterType=id"+"&filterValues="+ids.join(',')
  endpoint= "/rest/v1/leads.json"
  url = domain + endpoint + @token_uri + parameters

  self.get url

end

#get_token ⇒ Object 



254
255
256
257
258
259
260
261
262
263
264
265
266
267
 
# File 'lib/gooddata_marketo/adapters/rest.rb', line 254

def get_token


  domain = @marketo_domain
  endpoint = "/identity/oauth/token?grant_type=client_credentials"
  url = domain + endpoint + "&client_id=" + @marketo_rest_id + "&client_secret=" + @marketo_rest_secret
  response = self.get url
  results = JSON.parse(response.body)
  access_token = results["access_token"]
  @token_timer = Time.now
  @token = access_token
  access_token

end

#load_id_dump ⇒ Object 



207
208
209
210
211
212
213
214
215
 
# File 'lib/gooddata_marketo/adapters/rest.rb', line 207

def load_id_dump
  # Check if the lists leads have already been dumped. Append to it.
  if @webdav.exists? @lead_list_dump
    file = @webdav.download @lead_list_dump
    puts "#{Time.now} => WEBDAV:ID_DUMP:Load" if GoodDataMarketo.logging
    f = File.open(@lead_list_dump,'w')
    f.write(file)
  end
end

#load_last_page ⇒ Object 



191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
 
# File 'lib/gooddata_marketo/adapters/rest.rb', line 191

def load_last_page

  if @webdav.include? 'lead_list_next_page_token.json'
    next_page_token_json = @webdav.download 'lead_list_next_page_token.json'
    json = JSON.parse(next_page_token_json)
  elsif File.exists? 'lead_list_next_page_token.json'
    json = JSON.parse( IO.read('lead_list_next_page_token.json'))
  else
    json = {}
  end

  @next_page_token = json[:token]
  @next_page_token_list = json[:list]

end

#load_lists ⇒ Object 



217
218
219
220
221
222
223
224
225
 
# File 'lib/gooddata_marketo/adapters/rest.rb', line 217

def load_lists
  if @webdav.include? 'lead_list_ids.json'
    lists = @webdav.download 'lead_list_ids.json'
    @lead_list_ids = JSON.parse(lists)
    puts "#{Time.now} => WEBDAV:ListsLoaded:#{@lead_list_ids.length}:lead_list_ids.json" if GoodDataMarketo.logging
  else
    self.get_lead_lists
  end
end

#refresh_token? ⇒ Boolean

If token has existed for more than 6000 seconds, refresh it.

Returns:

  • (Boolean) 


30
31
32
33
34
35
36
37
 
# File 'lib/gooddata_marketo/adapters/rest.rb', line 30

def refresh_token?
  if (@token_timer + 6000) < Time.now
    self.get_token
    true
  else
    false
  end
end

#remember_next_page(config = {}) ⇒ Object 



227
228
229
230
 
# File 'lib/gooddata_marketo/adapters/rest.rb', line 227

def remember_next_page config = {}
  File.open('lead_list_next_page_token.json','w'){ |f| JSON.dump(config, f) }
  @webdav.upload('lead_list_next_page_token.json')
end

#resolve_with_etl_controller ⇒ Object 



281
282
283
284
285
 
# File 'lib/gooddata_marketo/adapters/rest.rb', line 281

def resolve_with_etl_controller

  puts 'Complete'
  # Here it would update the log that a full upload as occured.
end

#usage ⇒ Object 



269
270
271
272
273
274
275
276
277
278
279
 
# File 'lib/gooddata_marketo/adapters/rest.rb', line 269

def usage
  domain = @marketo_domain
  endpoint = "/rest/v1/stats/usage.json"
  url = domain + endpoint + @token_uri

  # Note this is the only rest call which does not use the log so as not to print the keys to log.
  response = RestClient.get url

  #Parse reponse and return only access token
  results = JSON.parse(response.body)['result']
end