Module: Dapp::Dimg::Dapp::Command::CleanupRepo
- Included in:
- Dapp
- Defined in:
- lib/dapp/dimg/dapp/command/cleanup_repo.rb
Constant Summary collapse
- GIT_TAGS_LIMIT_POLICY =
10
- EXPIRY_DATE_PERIOD_POLICY =
60 * 60 * 24 * 30
- GIT_COMMITS_LIMIT_POLICY =
50
- GIT_COMMITS_EXPIRY_DATE_PERIOD_POLICY =
60 * 60 * 24 * 30
Instance Method Summary collapse
- #actual_detailed_dimgs_images_by_scheme(registry) ⇒ Object
- #cleanup_repo ⇒ Object
- #cleanup_repo_by_nonexistent_git_base(registry, repo_dimgs_images_by_scheme, dapp_tag_scheme) ⇒ Object
- #cleanup_repo_by_nonexistent_git_primitive(registry, detailed_dimgs_images_by_scheme) ⇒ Object
- #cleanup_repo_by_policies(registry, detailed_dimgs_images_by_scheme) ⇒ Object
- #cleanup_repo_by_policies_base(registry, detailed_dimgs_images, expiry_date_policy:, limit_policy:, log_primitive:) ⇒ Object
- #cleanup_repo_commits_by_policies(registry, detailed_dimgs_images_by_scheme) ⇒ Object
- #cleanup_repo_tags_by_policies(registry, detailed_dimgs_images_by_scheme) ⇒ Object
- #consistent_git_remote_branches ⇒ Object
- #consistent_git_tags ⇒ Object
-
#cronjob_images(client) ⇒ Object
cronjob items[] spec jobTemplate spec template spec containers[] image.
-
#daemonset_images(client) ⇒ Object
daemonsets items[] spec template spec containers[] image.
- #deployed_docker_images ⇒ Object
-
#deployment_images(client) ⇒ Object
deployment items[] spec template spec containers[] image.
- #expiry_date_policy_value(env_key, default:) ⇒ Object
- #git_commits_expiry_date_policy ⇒ Object
- #git_commits_limit_policy ⇒ Object
- #git_tag_by_consistent_git_tag(consistent_git_tag) ⇒ Object
- #git_tag_by_consistent_tag_name ⇒ Object
- #git_tags_expiry_date_policy ⇒ Object
- #git_tags_limit_policy ⇒ Object
-
#job_images(client) ⇒ Object
job items[] spec template spec containers[] image.
-
#pod_images(client) ⇒ Object
pod items[] spec containers[] image.
- #policy_value(env_key, default:) ⇒ Object
-
#replicaset_images(client) ⇒ Object
replicasets items[] spec template spec containers[] image.
-
#replicationcontroller_images(client) ⇒ Object
replicationcontroller items[] spec template spec containers[] image.
- #search_deployed_docker_images ⇒ Object
- #search_deployed_docker_images_from_kube(client) ⇒ Object
-
#statefulset_images(client) ⇒ Object
replicasets items[] spec template spec containers[] image.
- #without_kube? ⇒ Boolean
Instance Method Details
#actual_detailed_dimgs_images_by_scheme(registry) ⇒ Object
32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 32 def actual_detailed_dimgs_images_by_scheme(registry) {}.tap do |detailed_dimgs_images_by_scheme| tagging_schemes.each { |scheme| detailed_dimgs_images_by_scheme[scheme] = [] } repo_detailed_dimgs_images(registry).each do |image| image_repository = [option_repo, image[:dimg]].compact.join('/') image_name = [image_repository, image[:tag]].join(':') should_be_ignored = deployed_docker_images.include?(image_name) if should_be_ignored log "Keep in repo image that is being used in kubernetes: #{image_name}" next end (detailed_dimgs_images_by_scheme[image[:labels]['dapp-tag-scheme']] ||= []) << image end end end |
#cleanup_repo ⇒ Object
11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 11 def cleanup_repo lock_repo(repo = option_repo) do log_step_with_indent(repo) do log_step_with_indent('Searching for images being used in kubernetes clusters in one of the kube-config contexts') do deployed_docker_images.each do |deployed_img| log(deployed_img) end end registry = dimg_registry(repo) if git_own_repo_exist? cleanup_repo_by_nonexistent_git_primitive(registry, actual_detailed_dimgs_images_by_scheme(registry)) cleanup_repo_by_policies(registry, actual_detailed_dimgs_images_by_scheme(registry)) end repo_dimgstages_cleanup(registry, repo_dimgs, repo_dimgstages) if with_stages? end end end |
#cleanup_repo_by_nonexistent_git_base(registry, repo_dimgs_images_by_scheme, dapp_tag_scheme) ⇒ Object
73 74 75 76 77 78 79 80 81 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 73 def cleanup_repo_by_nonexistent_git_base(registry, repo_dimgs_images_by_scheme, dapp_tag_scheme) nonexist_repo_images = repo_dimgs_images_by_scheme[dapp_tag_scheme] .select { |dimg_image| dimg_image[:labels]['dapp-tag-scheme'] == dapp_tag_scheme } .select { |dimg_image| !(yield dimg_image) } log_step_with_indent(:"#{dapp_tag_scheme.split('_').join(' ')} nonexistent") do nonexist_repo_images.each { |repo_image| delete_repo_image(registry, repo_image) } end unless nonexist_repo_images.empty? end |
#cleanup_repo_by_nonexistent_git_primitive(registry, detailed_dimgs_images_by_scheme) ⇒ Object
51 52 53 54 55 56 57 58 59 60 61 62 63 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 51 def cleanup_repo_by_nonexistent_git_primitive(registry, detailed_dimgs_images_by_scheme) %w(git_tag git_branch git_commit).each do |scheme| cleanup_repo_by_nonexistent_git_base(registry, detailed_dimgs_images_by_scheme, scheme) do |detailed_dimg_image| case scheme when 'git_tag' then .include?(detailed_dimg_image[:tag]) when 'git_branch' then consistent_git_remote_branches.include?(detailed_dimg_image[:tag]) when 'git_commit' then git_own_repo.commit_exists?(detailed_dimg_image[:tag]) else raise end end unless detailed_dimgs_images_by_scheme[scheme].empty? end end |
#cleanup_repo_by_policies(registry, detailed_dimgs_images_by_scheme) ⇒ Object
83 84 85 86 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 83 def cleanup_repo_by_policies(registry, detailed_dimgs_images_by_scheme) (registry, detailed_dimgs_images_by_scheme) cleanup_repo_commits_by_policies(registry, detailed_dimgs_images_by_scheme) end |
#cleanup_repo_by_policies_base(registry, detailed_dimgs_images, expiry_date_policy:, limit_policy:, log_primitive:) ⇒ Object
108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 108 def cleanup_repo_by_policies_base(registry, detailed_dimgs_images, expiry_date_policy:, limit_policy:, log_primitive:) sorted_detailed_dimgs_images = detailed_dimgs_images.sort_by { |dimg| dimg[:created_at] }.reverse expired_dimgs_images, not_expired_dimgs_images = sorted_detailed_dimgs_images.partition do |dimg_image| dimg_image[:created_at] < expiry_date_policy end log_step_with_indent(:"git #{log_primitive} date policy (before #{DateTime.strptime(expiry_date_policy.to_s, '%s')})") do expired_dimgs_images.each { |dimg| delete_repo_image(registry, dimg) } end unless expired_dimgs_images.empty? not_expired_dimgs_images .each_with_object({}) { |dimg, images_by_dimg| (images_by_dimg[dimg[:dimg]] ||= []) << dimg } .each do |dimg_name, images| next if images[limit_policy..-1].nil? log_step_with_indent(:"git #{log_primitive} limit policy (> #{limit_policy}) (`#{dimg_name || 'nameless'}` dimg)") do images[limit_policy..-1].each { |dimg| delete_repo_image(registry, dimg) } end end end |
#cleanup_repo_commits_by_policies(registry, detailed_dimgs_images_by_scheme) ⇒ Object
98 99 100 101 102 103 104 105 106 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 98 def cleanup_repo_commits_by_policies(registry, detailed_dimgs_images_by_scheme) cleanup_repo_by_policies_base( registry, detailed_dimgs_images_by_scheme['git_commit'].select { |dimg| git_own_repo.commit_exists?(dimg[:tag]) }, expiry_date_policy: git_commits_expiry_date_policy, limit_policy: git_commits_limit_policy, log_primitive: 'commit' ) end |
#cleanup_repo_tags_by_policies(registry, detailed_dimgs_images_by_scheme) ⇒ Object
88 89 90 91 92 93 94 95 96 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 88 def (registry, detailed_dimgs_images_by_scheme) cleanup_repo_by_policies_base( registry, detailed_dimgs_images_by_scheme['git_tag'].select { |dimg| .include?(dimg[:tag]) }, expiry_date_policy: , limit_policy: , log_primitive: 'tag' ) end |
#consistent_git_remote_branches ⇒ Object
69 70 71 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 69 def consistent_git_remote_branches @consistent_git_remote_branches ||= git_own_repo.remote_branches.map(&method(:consistent_uniq_slugify)) end |
#consistent_git_tags ⇒ Object
65 66 67 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 65 def git_tag_by_consistent_tag_name.keys end |
#cronjob_images(client) ⇒ Object
cronjob items[] spec jobTemplate spec template spec containers[] image
229 230 231 232 233 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 229 def cronjob_images(client) client.cronjob_list['items'].map do |item| item['spec']['jobTemplate']['spec']['template']['spec']['containers'].map{ |cont| cont['image'] } end end |
#daemonset_images(client) ⇒ Object
daemonsets items[] spec template spec containers[] image
236 237 238 239 240 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 236 def daemonset_images(client) client.daemonset_list['items'].map do |item| item['spec']['template']['spec']['containers'].map{ |cont| cont['image'] } end end |
#deployed_docker_images ⇒ Object
169 170 171 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 169 def deployed_docker_images @deployed_docker_images ||= search_deployed_docker_images end |
#deployment_images(client) ⇒ Object
deployment items[] spec template spec containers[] image
243 244 245 246 247 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 243 def deployment_images(client) client.deployment_list['items'].map do |item| item['spec']['template']['spec']['containers'].map{ |cont| cont['image'] } end end |
#expiry_date_policy_value(env_key, default:) ⇒ Object
145 146 147 148 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 145 def expiry_date_policy_value(env_key, default:) expiry_date_period_policy = policy_value(env_key, default: default) Time.now.to_i - expiry_date_period_policy end |
#git_commits_expiry_date_policy ⇒ Object
137 138 139 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 137 def git_commits_expiry_date_policy @git_commits_expiry_date_policy ||= expiry_date_policy_value('GIT_COMMITS_EXPIRY_DATE_PERIOD_POLICY', default: GIT_COMMITS_EXPIRY_DATE_PERIOD_POLICY) end |
#git_commits_limit_policy ⇒ Object
141 142 143 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 141 def git_commits_limit_policy @git_commits_limit_policy ||= policy_value('GIT_COMMITS_LIMIT_POLICY', default: GIT_COMMITS_LIMIT_POLICY) end |
#git_tag_by_consistent_git_tag(consistent_git_tag) ⇒ Object
161 162 163 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 161 def git_tag_by_consistent_git_tag(consistent_git_tag) git_tag_by_consistent_tag_name[consistent_git_tag] end |
#git_tag_by_consistent_tag_name ⇒ Object
165 166 167 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 165 def git_tag_by_consistent_tag_name @git_consistent_tags ||= git_own_repo..map { |t| [consistent_uniq_slugify(t), t] }.to_h end |
#git_tags_expiry_date_policy ⇒ Object
129 130 131 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 129 def @git_tags_expiry_date_policy ||= expiry_date_policy_value('EXPIRY_DATE_PERIOD_POLICY', default: EXPIRY_DATE_PERIOD_POLICY) end |
#git_tags_limit_policy ⇒ Object
133 134 135 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 133 def @git_tags_limit_policy ||= policy_value('GIT_TAGS_LIMIT_POLICY', default: GIT_TAGS_LIMIT_POLICY) end |
#job_images(client) ⇒ Object
job items[] spec template spec containers[] image
250 251 252 253 254 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 250 def job_images(client) client.job_list['items'].map do |item| item['spec']['template']['spec']['containers'].map{ |cont| cont['image'] } end end |
#pod_images(client) ⇒ Object
pod items[] spec containers[] image
222 223 224 225 226 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 222 def pod_images(client) client.pod_list['items'].map do |item| item['spec']['containers'].map{ |cont| cont['image'] } end end |
#policy_value(env_key, default:) ⇒ Object
150 151 152 153 154 155 156 157 158 159 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 150 def policy_value(env_key, default:) return default if (val = ENV[env_key]).nil? if val.to_i.to_s == val val.to_i else log_warning("WARNING: `#{env_key}` value `#{val}` is ignored (using default value `#{default}`)!") default end end |
#replicaset_images(client) ⇒ Object
replicasets items[] spec template spec containers[] image
257 258 259 260 261 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 257 def replicaset_images(client) client.replicaset_list['items'].map do |item| item['spec']['template']['spec']['containers'].map{ |cont| cont['image'] } end end |
#replicationcontroller_images(client) ⇒ Object
replicationcontroller items[] spec template spec containers[] image
271 272 273 274 275 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 271 def replicationcontroller_images(client) client.replicationcontroller_list['items'].map do |item| item['spec']['template']['spec']['containers'].map{ |cont| cont['image'] } end end |
#search_deployed_docker_images ⇒ Object
173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 173 def search_deployed_docker_images return [] if without_kube? config = ::Dapp::Kube::Kubernetes::Config.new_auto_if_available return [] if config.nil? config.context_names. map {|context_name| client = ::Dapp::Kube::Kubernetes::Client.new( config, context_name, "default", timeout: [:kubernetes_timeout], ) search_deployed_docker_images_from_kube(client) }.flatten.sort.uniq end |
#search_deployed_docker_images_from_kube(client) ⇒ Object
191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 191 def search_deployed_docker_images_from_kube(client) namespaces = [] # check connectivity for 2 seconds begin namespaces = client.namespace_list(excon_parameters: {:connect_timeout => 30}) rescue Excon::Error::Timeout raise ::Dapp::Error::Default, code: :kube_connect_timeout end # get images from containers from pods from all namespaces. namespaces['items'].map do |item| item['metadata']['name'] end.map do |ns| [].tap do |arr| client.with_namespace(ns) do arr << pod_images(client) arr << deployment_images(client) arr << replicaset_images(client) arr << statefulset_images(client) arr << daemonset_images(client) arr << job_images(client) arr << cronjob_images(client) arr << replicationcontroller_images(client) end end end.flatten.select do |img| img.start_with? option_repo end.sort.uniq end |
#statefulset_images(client) ⇒ Object
replicasets items[] spec template spec containers[] image
264 265 266 267 268 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 264 def statefulset_images(client) client.statefulset_list['items'].map do |item| item['spec']['template']['spec']['containers'].map{ |cont| cont['image'] } end end |
#without_kube? ⇒ Boolean
277 278 279 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 277 def without_kube? !![:without_kube] end |