Module: Webhookdb::Replicator::GithubRepoV1Mixin

Included in:
GithubIssueCommentV1, GithubIssueV1, GithubPullV1, GithubReleaseV1, GithubRepositoryEventV1
Defined in:
lib/webhookdb/replicator/github_repo_v1_mixin.rb

Overview

Mixin for repo-specific resources like issues and pull requests.

Constant Summary collapse

API_VERSION = 
"2022-11-28"
JSON_CONTENT_TYPE = 
"application/vnd.github+json"

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Instance Attribute Details

#service_integration ⇒ Webhookdb::ServiceIntegration

Returns:



16
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 16

def _mixin_backfill_url = raise NotImplementedError("/issues, /pulls, etc")

Class Method Details

._api_docs_url(tail) ⇒ Object 



9
10
11
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 9

def self._api_docs_url(tail)
  return "https://docs.github.com/en/rest#{tail}?apiVersion=#{API_VERSION}"
end

Instance Method Details

#_fetch_backfill_page(pagination_token, last_backfilled:) ⇒ Object 



182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 182

def _fetch_backfill_page(pagination_token, last_backfilled:)
  if pagination_token.present?
    url = pagination_token
    query = {}
  else
    url = "https://api.github.com/repos/#{self.service_integration.api_url}#{self._mixin_backfill_url}"
    query = {per_page: 100}
    query.merge!(self._mixin_query_params(last_backfilled:))
  end
  response, data = self._http_get(url, query)
  next_link = nil
  if response.headers.key?("link")
    links = Webhookdb::Github.parse_link_header(response.headers["link"])
    next_link = links[:next] if links.key?(:next)
  end
  return data, next_link
end

#_fetch_enrichment(resource, _event, _request) ⇒ Object 



218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 218

def _fetch_enrichment(resource, _event, _request)
  # If we're not set up to backfill, we cannot make an API call.
  return nil if self.service_integration.backfill_secret.nil?
  # We should fetch the full resource if the replicator needs it,
  # and the resource does not have the key we require.
  sentinel_key = self._mixin_fetch_resource_if_field_missing
  return nil if sentinel_key.nil? || resource.key?(sentinel_key)
  resource_url = resource.fetch("url")
  begin
    _response, data = self._http_get(resource_url, {})
  rescue Webhookdb::Http::Error => e
    # If the HTTP call fails due to an auth issue (or a deleted item),
    # we should still upsert what we have.
    # Tokens expire or can be revoked, but we don't want the webhook to stop inserting.
    ignore_error = [401, 403, 404].include?(e.response.code)
    return nil if ignore_error
    raise e
  end
  return data
end

#_fullreponame ⇒ Object 



29
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 29

def _fullreponame = self.service_integration.api_url

#_handle_repo_name_state_machine(step, tfield) ⇒ Object

If api_url isn’t set, prompt for it (via repo_name or api_url field).



109
110
111
112
113
114
115
116
117
118
119
120
121
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 109

def _handle_repo_name_state_machine(step, tfield)
  if self.service_integration.api_url.blank?
    step.output = %(You are about to start replicating #{self.resource_name_plural} for a repository into WebhookDB.

First we need the full repository name, like 'webhookdb/webhookdb-cli'.)
    step.set_prompt("Repository name:").transition_field(self.service_integration, tfield)
    return true
  end
  return false if self._valid_repo_name?(self.service_integration.api_url)
  step.output = %(That repository is not valid. Include both the owner and name, like 'webhookdb/webhookdb-cli'.)
  step.set_prompt("Repository name:").transition_field(self.service_integration, tfield)
  return true
end

#_http_get(url, query) ⇒ Object 



200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 200

def _http_get(url, query)
  response = Webhookdb::Http.get(
    url,
    query,
    headers: {
      "Accept" => JSON_CONTENT_TYPE,
      "Authorization" => "Bearer #{self.service_integration.backfill_secret}",
      "X-GitHub-Api-Version" => API_VERSION,
    },
    logger: self.logger,
    timeout: Webhookdb::Github.http_timeout,
  )
  # Handle the GH-specific vnd JSON or general application/json
  parsed = response.parsed_response
  (parsed = Oj.load(parsed)) if response.headers["Content-Type"] == JSON_CONTENT_TYPE
  return response, parsed
end

#_is_repo_public? ⇒ Boolean

If we can make an unauthed request and find the repo, it is public.

Returns:

  • (Boolean) 


124
125
126
127
128
129
130
131
132
133
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 124

def _is_repo_public?
  resp = Webhookdb::Http.post(
    "https://github.com/#{self.service_integration.api_url}",
    method: :head,
    check: false,
    timeout: 5,
    logger: nil,
  )
  return resp.code == 200
end

#_mixin_backfill_url ⇒ Webhookdb::ServiceIntegration

Returns:



16
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 16

def _mixin_backfill_url = raise NotImplementedError("/issues, /pulls, etc")

#_mixin_fetch_resource_if_field_missing ⇒ Object

Some resources, like issues and pull requests, have a ‘simple’ representation in the list, and a full representation when fetched individually. Return the field that can be used to determine if the full resource needs to be fetched.



27
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 27

def _mixin_fetch_resource_if_field_missing = nil

#_mixin_fine_grained_permission ⇒ Object

github.com/settings/personal-access-tokens/new



21
22
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 21

def _mixin_fine_grained_permission = raise NotImplementedError("Issues", etc)
# Query params to use in the list call. Should include sorting when available.

#_mixin_query_params(last_backfilled:) ⇒ Object

Query params to use in the list call. Should include sorting when available.

Raises:

  • (NotImplementedError) 


23
24
25
26
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 23

def _mixin_query_params(last_backfilled:) = raise NotImplementedError
# Some resources, like issues and pull requests, have a 'simple' representation
# in the list, and a full representation when fetched individually.
# Return the field that can be used to determine if the full resource needs to be fetched.

#_mixin_webhook_events ⇒ Object 



17
18
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 17

def _mixin_webhook_events = raise NotImplementedError("Issues, Pulls, Issue comments, etc")
# https://docs.github.com/en/webhooks/webhook-events-and-payloads?actionType=demilestoned#issues

#_mixin_webhook_key ⇒ Object

docs.github.com/en/webhooks/webhook-events-and-payloads?actionType=demilestoned#issues



19
20
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 19

def _mixin_webhook_key = raise NotImplementedError("issue, etc")
# https://github.com/settings/personal-access-tokens/new

#_prepare_for_insert(resource, event, request, enrichment) ⇒ Object 



239
240
241
242
243
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 239

def _prepare_for_insert(resource, event, request, enrichment)
  # if enrichment is not nil, it's the detailed resource.
  # See _mixin_fetch_resource_if_field_missing
  return super(enrichment || resource, event, request, nil)
end

#_reponame ⇒ Object 



31
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 31

def _reponame = self._fullreponame.split("/").last

#_repoowner ⇒ Object 



30
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 30

def _repoowner = self._fullreponame.split("/").first

#_resource_and_event(request) ⇒ Object

Extract the resource from the request. The resource can be a normal resource, or a webhook, with X-GitHub-Hook-ID key as per docs.github.com/en/webhooks/webhook-events-and-payloads The headers are the only things that identify a webhook payload consistently.

Note that webhooks to a given integration can be for events we do not expect, such as someone sending events we aren’t handling (ie, if they don’t uncheck Pushes, we may get push events sent to the github_issue_v1 integration), and also for automated events like ‘ping’.



43
44
45
46
47
48
49
50
51
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 43

def _resource_and_event(request)
  # Note the canonical casing on the header name. GitHub sends X-GitHub-Hook-ID
  # but it's normalized here.
  is_webhook = (request.headers || {})["X-Github-Hook-Id"]
  return request.body, nil unless is_webhook
  resource = request.body.fetch(self._mixin_webhook_key, nil)
  return nil, nil if resource.nil?
  return resource, request.body
end

#_resource_to_data(resource, _event, _request, enrichment) ⇒ Object 



245
246
247
248
249
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 245

def _resource_to_data(resource, _event, _request, enrichment)
  # if enrichment is not nil, it's the detailed resource.
  # See _mixin_fetch_resource_if_field_missing
  return enrichment || resource
end

#_update_where_expr ⇒ Object 



53
54
55
56
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 53

def _update_where_expr
  ts = self._timestamp_column_name
  return self.qualified_table_sequel_identifier[ts] < Sequel[:excluded][ts]
end

#_valid_repo_name?(s) ⇒ Boolean

Returns:

  • (Boolean) 


32
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 32

def _valid_repo_name?(s) = %r{^[\w\-.]+/[\w\-.]+$} =~ s

#_verify_backfill_err_msg ⇒ Object 



176
177
178
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 176

def _verify_backfill_err_msg
  return "That access token didn't seem to work. Please look over the instructions and try again."
end

#_webhook_response(request) ⇒ Object 



58
59
60
61
62
63
64
65
66
67
68
69
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 58

def _webhook_response(request)
  hash = request.env["HTTP_X_HUB_SIGNATURE_256"]
  return Webhookdb::WebhookResponse.error("missing sha256") if hash.nil?
  secret = self.service_integration.webhook_secret
  return Webhookdb::WebhookResponse.error("no secret set, run `webhookdb integration setup`", status: 409) if
    secret.nil?
  request.body.rewind
  request_data = request.body.read
  verified = Webhookdb::Github.verify_webhook(request_data, hash, secret)
  return Webhookdb::WebhookResponse.ok if verified
  return Webhookdb::WebhookResponse.error("invalid sha256")
end

#_webhook_state_change_fields ⇒ Object 



71
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 71

def _webhook_state_change_fields = super + ["repo_name"]

#calculate_backfill_state_machine ⇒ Object 



135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 135

def calculate_backfill_state_machine
  step = Webhookdb::Replicator::StateMachineStep.new
  return step if self._handle_repo_name_state_machine(step, "api_url")
  unless self.service_integration.backfill_secret.present?
    repo_public = self._is_repo_public?
    step.output = %(In order to backfill #{self.resource_name_plural},
WebhookDB requires an access token to authenticate.

You should go to https://github.com/settings/personal-access-tokens/new and create a new Personal Access Token.

For 'Expiration', give a custom date far in the future.

For 'Resource owner', choose the '#{self._repoowner}' organization.
**If it does not appear**, Fine-grained tokens are not enabled.
See instructions below.

For 'Repository access', choose 'Only select repositories', and the '#{self._fullreponame}' repository.

For 'Repository permissions', go to '#{self._mixin_fine_grained_permission}' and choose 'Read-only access'.

If you didn't see the needed owner under 'Resource owner,' it's because fine-grained tokens are not enabled.
Instead, create a new Classic personal access token from https://github.com/settings/tokens/new.
In the 'Note', mention this token is for WebhookDB,
give it an expiration, and under 'Scopes', ensure #{repo_public ? 'repo->public_repo' : 'repo'} is checked,
since #{self._fullreponame} is #{repo_public ? 'public' : 'private'}.

Then click 'Generate token'.)
    return step.secret_prompt("Personal access token").backfill_secret(self.service_integration)
  end

  unless (result = self.verify_backfill_credentials).verified
    self.service_integration.replicator.clear_backfill_information
    step.output = result.message
    return step.secret_prompt("Personal access token").backfill_secret(self.service_integration)
  end

  step.output = %(Great! We are going to start backfilling your #{self.resource_name_plural}.
#{self._query_help_output})
  return step.completed
end

#calculate_webhook_state_machine ⇒ Object 



78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 78

def calculate_webhook_state_machine
  step = Webhookdb::Replicator::StateMachineStep.new
  return step if self._handle_repo_name_state_machine(step, "repo_name")
  if self.service_integration.webhook_secret.blank?
    step.output = %(Now, head to this route to create a webhook:

https://github.com/#{self.service_integration.api_url}/settings/hooks/new

For 'Payload URL', use this endpoint that is now available:

#{self._webhook_endpoint}

For 'Content type', choose 'application/json'. Form encoding works but loses some detail in events.

For 'Secret', choose your own secure secret, or use this one: '#{Webhookdb::Id.rand_enc(16)}'

For 'Which events would you like to trigger this webhook',
choose 'Let me select individual events',
uncheck 'Pushes', and select the following:

#{self._mixin_webhook_events.join("\n  ")}

Make sure 'Active' is checked, and press 'Add webhook'.)
    return step.secret_prompt("Webhook Secret").webhook_secret(self.service_integration)
  end
  step.output = %(Great! WebhookDB is now listening for #{self.resource_name_singular} webhooks.
#{self._query_help_output})
  return step.completed
end

#process_state_change(field, value) ⇒ Object 



73
74
75
76
 
# File 'lib/webhookdb/replicator/github_repo_v1_mixin.rb', line 73

def process_state_change(field, value)
  attr = field == "repo_name" ? "api_url" : field
  return super(field, value, attr:)
end