Module: Dapp::Dimg::Dapp::Command::CleanupRepo
- Included in:
- Dapp
- Defined in:
- lib/dapp/dimg/dapp/command/cleanup_repo.rb
Constant Summary collapse
- GIT_TAGS_LIMIT_POLICY =
10
- EXPIRY_DATE_PERIOD_POLICY =
60 * 60 * 24 * 30
- GIT_COMMITS_LIMIT_POLICY =
50
- GIT_COMMITS_EXPIRY_DATE_PERIOD_POLICY =
60 * 60 * 24 * 30
Instance Method Summary collapse
- #actual_detailed_dimgs_images_by_scheme(registry) ⇒ Object
- #cleanup_repo ⇒ Object
- #cleanup_repo_by_nonexistent_git_base(registry, repo_dimgs_images_by_scheme, dapp_tag_scheme) ⇒ Object
- #cleanup_repo_by_nonexistent_git_primitive(registry, detailed_dimgs_images_by_scheme) ⇒ Object
- #cleanup_repo_by_policies(registry, detailed_dimgs_images_by_scheme) ⇒ Object
- #cleanup_repo_by_policies_base(registry, detailed_dimgs_images, expiry_date_policy:, limit_policy:, log_primitive:) ⇒ Object
- #cleanup_repo_commits_by_policies(registry, detailed_dimgs_images_by_scheme) ⇒ Object
- #cleanup_repo_tags_by_policies(registry, detailed_dimgs_images_by_scheme) ⇒ Object
- #consistent_git_remote_branches ⇒ Object
- #consistent_git_tags ⇒ Object
-
#cronjob_images(client) ⇒ Object
cronjob items[] spec jobTemplate spec template spec containers[] image.
-
#daemonset_images(client) ⇒ Object
daemonsets items[] spec template spec containers[] image.
- #deployed_docker_images ⇒ Object
-
#deployment_images(client) ⇒ Object
deployment items[] spec template spec containers[] image.
- #expiry_date_policy_value(env_key, default:) ⇒ Object
- #git_commits_expiry_date_policy ⇒ Object
- #git_commits_limit_policy ⇒ Object
- #git_tag_by_consistent_git_tag(consistent_git_tag) ⇒ Object
- #git_tag_by_consistent_tag_name ⇒ Object
- #git_tags_expiry_date_policy ⇒ Object
- #git_tags_limit_policy ⇒ Object
-
#job_images(client) ⇒ Object
job items[] spec template spec containers[] image.
-
#pod_images(client) ⇒ Object
pod items[] spec containers[] image.
- #policy_value(env_key, default:) ⇒ Object
-
#replicaset_images(client) ⇒ Object
replicasets items[] spec template spec containers[] image.
-
#replicationcontroller_images(client) ⇒ Object
replicationcontroller items[] spec template spec containers[] image.
- #search_deployed_docker_images ⇒ Object
- #search_deployed_docker_images_from_kube(client) ⇒ Object
-
#statefulset_images(client) ⇒ Object
replicasets items[] spec template spec containers[] image.
- #without_kube? ⇒ Boolean
Instance Method Details
#actual_detailed_dimgs_images_by_scheme(registry) ⇒ Object
36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 36 def actual_detailed_dimgs_images_by_scheme(registry) {}.tap do |detailed_dimgs_images_by_scheme| tagging_schemes.each { |scheme| detailed_dimgs_images_by_scheme[scheme] = [] } repo_detailed_dimgs_images(registry).each do |image| image_repository = [option_repo, image[:dimg]].compact.join('/') image_name = [image_repository, image[:tag]].join(':') should_be_ignored = deployed_docker_images.include?(image_name) if should_be_ignored log "Keep in repo image that is being used in kubernetes: #{image_name}" next end (detailed_dimgs_images_by_scheme[image[:labels]['dapp-tag-scheme']] ||= []) << image end end end |
#cleanup_repo ⇒ Object
11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 11 def cleanup_repo lock_repo(repo = option_repo) do log_step_with_indent(repo) do log_step_with_indent('Searching for images being used in kubernetes clusters in one of the kube-config contexts') do deployed_docker_images.each do |deployed_img| log(deployed_img) end end registry = dimg_registry(repo) if git_own_repo_exist? cleanup_repo_by_nonexistent_git_primitive(registry, actual_detailed_dimgs_images_by_scheme(registry)) cleanup_repo_by_policies(registry, actual_detailed_dimgs_images_by_scheme(registry)) end begin repo_dimgs = repo_dimgs_images(registry) repo_dimgstages = repo_dimgstages_images(registry) repo_dimgstages_cleanup(registry, repo_dimgs, repo_dimgstages) end if with_stages? end end end |
#cleanup_repo_by_nonexistent_git_base(registry, repo_dimgs_images_by_scheme, dapp_tag_scheme) ⇒ Object
77 78 79 80 81 82 83 84 85 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 77 def cleanup_repo_by_nonexistent_git_base(registry, repo_dimgs_images_by_scheme, dapp_tag_scheme) nonexist_repo_images = repo_dimgs_images_by_scheme[dapp_tag_scheme] .select { |dimg_image| dimg_image[:labels]['dapp-tag-scheme'] == dapp_tag_scheme } .select { |dimg_image| !(yield dimg_image) } log_step_with_indent(:"#{dapp_tag_scheme.split('_').join(' ')} nonexistent") do nonexist_repo_images.each { |repo_image| delete_repo_image(registry, repo_image) } end unless nonexist_repo_images.empty? end |
#cleanup_repo_by_nonexistent_git_primitive(registry, detailed_dimgs_images_by_scheme) ⇒ Object
55 56 57 58 59 60 61 62 63 64 65 66 67 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 55 def cleanup_repo_by_nonexistent_git_primitive(registry, detailed_dimgs_images_by_scheme) %w(git_tag git_branch git_commit).each do |scheme| cleanup_repo_by_nonexistent_git_base(registry, detailed_dimgs_images_by_scheme, scheme) do |detailed_dimg_image| case scheme when 'git_tag' then .include?(detailed_dimg_image[:tag]) when 'git_branch' then consistent_git_remote_branches.include?(detailed_dimg_image[:tag]) when 'git_commit' then git_own_repo.commit_exists?(detailed_dimg_image[:tag]) else raise end end unless detailed_dimgs_images_by_scheme[scheme].empty? end end |
#cleanup_repo_by_policies(registry, detailed_dimgs_images_by_scheme) ⇒ Object
87 88 89 90 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 87 def cleanup_repo_by_policies(registry, detailed_dimgs_images_by_scheme) (registry, detailed_dimgs_images_by_scheme) cleanup_repo_commits_by_policies(registry, detailed_dimgs_images_by_scheme) end |
#cleanup_repo_by_policies_base(registry, detailed_dimgs_images, expiry_date_policy:, limit_policy:, log_primitive:) ⇒ Object
112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 112 def cleanup_repo_by_policies_base(registry, detailed_dimgs_images, expiry_date_policy:, limit_policy:, log_primitive:) sorted_detailed_dimgs_images = detailed_dimgs_images.sort_by { |dimg| dimg[:created_at] }.reverse expired_dimgs_images, not_expired_dimgs_images = sorted_detailed_dimgs_images.partition do |dimg_image| dimg_image[:created_at] < expiry_date_policy end log_step_with_indent(:"git #{log_primitive} date policy (before #{DateTime.strptime(expiry_date_policy.to_s, '%s')})") do expired_dimgs_images.each { |dimg| delete_repo_image(registry, dimg) } end unless expired_dimgs_images.empty? not_expired_dimgs_images .each_with_object({}) { |dimg, images_by_dimg| (images_by_dimg[dimg[:dimg]] ||= []) << dimg } .each do |dimg_name, images| next if images[limit_policy..-1].nil? log_step_with_indent(:"git #{log_primitive} limit policy (> #{limit_policy}) (`#{dimg_name || 'nameless'}` dimg)") do images[limit_policy..-1].each { |dimg| delete_repo_image(registry, dimg) } end end end |
#cleanup_repo_commits_by_policies(registry, detailed_dimgs_images_by_scheme) ⇒ Object
102 103 104 105 106 107 108 109 110 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 102 def cleanup_repo_commits_by_policies(registry, detailed_dimgs_images_by_scheme) cleanup_repo_by_policies_base( registry, detailed_dimgs_images_by_scheme['git_commit'].select { |dimg| git_own_repo.commit_exists?(dimg[:tag]) }, expiry_date_policy: git_commits_expiry_date_policy, limit_policy: git_commits_limit_policy, log_primitive: 'commit' ) end |
#cleanup_repo_tags_by_policies(registry, detailed_dimgs_images_by_scheme) ⇒ Object
92 93 94 95 96 97 98 99 100 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 92 def (registry, detailed_dimgs_images_by_scheme) cleanup_repo_by_policies_base( registry, detailed_dimgs_images_by_scheme['git_tag'].select { |dimg| .include?(dimg[:tag]) }, expiry_date_policy: , limit_policy: , log_primitive: 'tag' ) end |
#consistent_git_remote_branches ⇒ Object
73 74 75 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 73 def consistent_git_remote_branches @consistent_git_remote_branches ||= git_own_repo.remote_branches.map(&method(:consistent_uniq_slugify)) end |
#consistent_git_tags ⇒ Object
69 70 71 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 69 def git_tag_by_consistent_tag_name.keys end |
#cronjob_images(client) ⇒ Object
cronjob items[] spec jobTemplate spec template spec containers[] image
233 234 235 236 237 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 233 def cronjob_images(client) client.cronjob_list['items'].map do |item| item['spec']['jobTemplate']['spec']['template']['spec']['containers'].map{ |cont| cont['image'] } end end |
#daemonset_images(client) ⇒ Object
daemonsets items[] spec template spec containers[] image
240 241 242 243 244 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 240 def daemonset_images(client) client.daemonset_list['items'].map do |item| item['spec']['template']['spec']['containers'].map{ |cont| cont['image'] } end end |
#deployed_docker_images ⇒ Object
173 174 175 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 173 def deployed_docker_images @deployed_docker_images ||= search_deployed_docker_images end |
#deployment_images(client) ⇒ Object
deployment items[] spec template spec containers[] image
247 248 249 250 251 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 247 def deployment_images(client) client.deployment_list['items'].map do |item| item['spec']['template']['spec']['containers'].map{ |cont| cont['image'] } end end |
#expiry_date_policy_value(env_key, default:) ⇒ Object
149 150 151 152 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 149 def expiry_date_policy_value(env_key, default:) expiry_date_period_policy = policy_value(env_key, default: default) Time.now.to_i - expiry_date_period_policy end |
#git_commits_expiry_date_policy ⇒ Object
141 142 143 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 141 def git_commits_expiry_date_policy @git_commits_expiry_date_policy ||= expiry_date_policy_value('GIT_COMMITS_EXPIRY_DATE_PERIOD_POLICY', default: GIT_COMMITS_EXPIRY_DATE_PERIOD_POLICY) end |
#git_commits_limit_policy ⇒ Object
145 146 147 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 145 def git_commits_limit_policy @git_commits_limit_policy ||= policy_value('GIT_COMMITS_LIMIT_POLICY', default: GIT_COMMITS_LIMIT_POLICY) end |
#git_tag_by_consistent_git_tag(consistent_git_tag) ⇒ Object
165 166 167 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 165 def git_tag_by_consistent_git_tag(consistent_git_tag) git_tag_by_consistent_tag_name[consistent_git_tag] end |
#git_tag_by_consistent_tag_name ⇒ Object
169 170 171 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 169 def git_tag_by_consistent_tag_name @git_consistent_tags ||= git_own_repo..map { |t| [consistent_uniq_slugify(t), t] }.to_h end |
#git_tags_expiry_date_policy ⇒ Object
133 134 135 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 133 def @git_tags_expiry_date_policy ||= expiry_date_policy_value('EXPIRY_DATE_PERIOD_POLICY', default: EXPIRY_DATE_PERIOD_POLICY) end |
#git_tags_limit_policy ⇒ Object
137 138 139 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 137 def @git_tags_limit_policy ||= policy_value('GIT_TAGS_LIMIT_POLICY', default: GIT_TAGS_LIMIT_POLICY) end |
#job_images(client) ⇒ Object
job items[] spec template spec containers[] image
254 255 256 257 258 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 254 def job_images(client) client.job_list['items'].map do |item| item['spec']['template']['spec']['containers'].map{ |cont| cont['image'] } end end |
#pod_images(client) ⇒ Object
pod items[] spec containers[] image
226 227 228 229 230 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 226 def pod_images(client) client.pod_list['items'].map do |item| item['spec']['containers'].map{ |cont| cont['image'] } end end |
#policy_value(env_key, default:) ⇒ Object
154 155 156 157 158 159 160 161 162 163 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 154 def policy_value(env_key, default:) return default if (val = ENV[env_key]).nil? if val.to_i.to_s == val val.to_i else log_warning("WARNING: `#{env_key}` value `#{val}` is ignored (using default value `#{default}`)!") default end end |
#replicaset_images(client) ⇒ Object
replicasets items[] spec template spec containers[] image
261 262 263 264 265 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 261 def replicaset_images(client) client.replicaset_list['items'].map do |item| item['spec']['template']['spec']['containers'].map{ |cont| cont['image'] } end end |
#replicationcontroller_images(client) ⇒ Object
replicationcontroller items[] spec template spec containers[] image
275 276 277 278 279 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 275 def replicationcontroller_images(client) client.replicationcontroller_list['items'].map do |item| item['spec']['template']['spec']['containers'].map{ |cont| cont['image'] } end end |
#search_deployed_docker_images ⇒ Object
177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 177 def search_deployed_docker_images return [] if without_kube? config = ::Dapp::Kube::Kubernetes::Config.new_auto_if_available return [] if config.nil? config.context_names. map {|context_name| client = ::Dapp::Kube::Kubernetes::Client.new( config, context_name, "default", timeout: [:kubernetes_timeout], ) search_deployed_docker_images_from_kube(client) }.flatten.sort.uniq end |
#search_deployed_docker_images_from_kube(client) ⇒ Object
195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 195 def search_deployed_docker_images_from_kube(client) namespaces = [] # check connectivity for 2 seconds begin namespaces = client.namespace_list(excon_parameters: {:connect_timeout => 30}) rescue Excon::Error::Timeout raise ::Dapp::Error::Default, code: :kube_connect_timeout end # get images from containers from pods from all namespaces. namespaces['items'].map do |item| item['metadata']['name'] end.map do |ns| [].tap do |arr| client.with_namespace(ns) do arr << pod_images(client) arr << deployment_images(client) arr << replicaset_images(client) arr << statefulset_images(client) arr << daemonset_images(client) arr << job_images(client) arr << cronjob_images(client) arr << replicationcontroller_images(client) end end end.flatten.select do |img| img.start_with? option_repo end.sort.uniq end |
#statefulset_images(client) ⇒ Object
replicasets items[] spec template spec containers[] image
268 269 270 271 272 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 268 def statefulset_images(client) client.statefulset_list['items'].map do |item| item['spec']['template']['spec']['containers'].map{ |cont| cont['image'] } end end |
#without_kube? ⇒ Boolean
281 282 283 |
# File 'lib/dapp/dimg/dapp/command/cleanup_repo.rb', line 281 def without_kube? !![:without_kube] end |