Class: Gemirro::Indexer

Inherits:
Gem::Indexer
  • Object
show all
Defined in:
lib/gemirro/indexer.rb

Overview

The Indexer class is responsible for downloading useful file directly on the source host, such as specs-..gz, marshal information, etc…

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(directory, options = {}) ⇒ Array

Create an indexer that will index the gems in directory.

Parameters:

  • directory (String) —

    Destination directory

  • options (Hash) (defaults to: {}) —

    Indexer options



34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
 
# File 'lib/gemirro/indexer.rb', line 34

def initialize(directory, options = {})
  require 'fileutils'
  require 'tmpdir'
  require 'zlib'

  unless defined?(Builder::XChar)
    raise 'Gem::Indexer requires that the XML Builder ' \
    'library be installed:' \
    "\n\tgem install builder"
  end

  options = { build_modern: true }.merge options

  @build_modern = options[:build_modern]

  @dest_directory = directory
  @directory = File.join(Dir.tmpdir,
                         "gem_generate_index_#{rand(1_000_000_000)}")

  marshal_name = "Marshal.#{::Gem.marshal_version}"

  @master_index = File.join @directory, 'yaml'
  @marshal_index = File.join @directory, marshal_name

  @quick_dir = File.join @directory, 'quick'
  @quick_marshal_dir = File.join @quick_dir, marshal_name
  @quick_marshal_dir_base = File.join 'quick', marshal_name # FIX: UGH

  @quick_index = File.join @quick_dir, 'index'
  @latest_index = File.join @quick_dir, 'latest_index'

  @specs_index = File.join @directory, "specs.#{::Gem.marshal_version}"
  @latest_specs_index =
    File.join(@directory, "latest_specs.#{::Gem.marshal_version}")
  @prerelease_specs_index =
    File.join(@directory, "prerelease_specs.#{::Gem.marshal_version}")
  @dest_specs_index =
    File.join(@dest_directory, "specs.#{::Gem.marshal_version}")
  @dest_latest_specs_index =
    File.join(@dest_directory, "latest_specs.#{::Gem.marshal_version}")
  @dest_prerelease_specs_index =
    File.join(@dest_directory, "prerelease_specs.#{::Gem.marshal_version}")

  @files = []
end

Instance Attribute Details

#dest_directory ⇒ String

Returns:

  • (String) 


19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
 
# File 'lib/gemirro/indexer.rb', line 19

class Indexer < ::Gem::Indexer
  attr_accessor(:files,
                :quick_marshal_dir,
                :directory,
                :dest_directory,
                :only_origin,
                :updated_gems)

  ##
  # Create an indexer that will index the gems in +directory+.
  #
  # @param [String] directory Destination directory
  # @param [Hash] options Indexer options
  # @return [Array]
  ##
  def initialize(directory, options = {})
    require 'fileutils'
    require 'tmpdir'
    require 'zlib'

    unless defined?(Builder::XChar)
      raise 'Gem::Indexer requires that the XML Builder ' \
      'library be installed:' \
      "\n\tgem install builder"
    end

    options = { build_modern: true }.merge options

    @build_modern = options[:build_modern]

    @dest_directory = directory
    @directory = File.join(Dir.tmpdir,
                           "gem_generate_index_#{rand(1_000_000_000)}")

    marshal_name = "Marshal.#{::Gem.marshal_version}"

    @master_index = File.join @directory, 'yaml'
    @marshal_index = File.join @directory, marshal_name

    @quick_dir = File.join @directory, 'quick'
    @quick_marshal_dir = File.join @quick_dir, marshal_name
    @quick_marshal_dir_base = File.join 'quick', marshal_name # FIX: UGH

    @quick_index = File.join @quick_dir, 'index'
    @latest_index = File.join @quick_dir, 'latest_index'

    @specs_index = File.join @directory, "specs.#{::Gem.marshal_version}"
    @latest_specs_index =
      File.join(@directory, "latest_specs.#{::Gem.marshal_version}")
    @prerelease_specs_index =
      File.join(@directory, "prerelease_specs.#{::Gem.marshal_version}")
    @dest_specs_index =
      File.join(@dest_directory, "specs.#{::Gem.marshal_version}")
    @dest_latest_specs_index =
      File.join(@dest_directory, "latest_specs.#{::Gem.marshal_version}")
    @dest_prerelease_specs_index =
      File.join(@dest_directory, "prerelease_specs.#{::Gem.marshal_version}")

    @files = []
  end

  ##
  # Generate indices on the destination directory
  #
  def install_indices
    install_indicies
  end

  ##
  # Generate indicies on the destination directory
  #
  # @return [Array]
  #
  def install_indicies
    Utils.logger
         .debug("Downloading index into production dir #{@dest_directory}")

    files = @files
    files.delete @quick_marshal_dir if files.include? @quick_dir

    if files.include?(@quick_marshal_dir) && !files.include?(@quick_dir)
      files.delete @quick_marshal_dir
      dst_name = File.join(@dest_directory, @quick_marshal_dir_base)
      FileUtils.mkdir_p(File.dirname(dst_name), verbose: verbose)
      FileUtils.rm_rf(dst_name, verbose: verbose)
      FileUtils.mv(@quick_marshal_dir, dst_name,
                   verbose: verbose, force: true)
    end

    files.each do |path|
      file = path.sub(%r{^#{Regexp.escape @directory}/?}, '')
      src_name = File.join(@directory, file)
      dst_name = File.join(@dest_directory, file)

      if ["#{@specs_index}.gz",
          "#{@latest_specs_index}.gz",
          "#{@prerelease_specs_index}.gz"].include?(path)
        res = build_zlib_file(file, src_name, dst_name, true)
        next unless res
      else
        source_content = download_from_source(file)
        next if source_content.nil?
        MirrorFile.new(dst_name).write(source_content)
      end

      FileUtils.rm_rf(path)
    end
  end

  ##
  # Download file from source
  #
  # @param [String] file File path
  # @return [String]
  #
  def download_from_source(file)
    source_host = Gemirro.configuration.source.host
    Utils.logger.info("Download from source: #{file}")
    resp = Http.get("#{source_host}/#{File.basename(file)}")
    return unless resp.code == 200
    resp.body
  end

  ##
  # Build indices
  #
  # @return [Array]
  #
  def build_indices
    build_indicies
  end

  ##
  # Build indicies
  #
  # @return [Array]
  #
  def build_indicies
    specs = *map_gems_to_specs(gem_file_list)
    specs.select! { |s| s.class == ::Gem::Specification }
    ::Gem::Specification.dirs = []
    ::Gem::Specification.all = specs

    if ::Gem::VERSION >= '2.5.0'
      build_marshal_gemspecs specs
      build_modern_indices specs if @build_modern
      compress_indices
    else
      build_marshal_gemspecs
      build_modern_indicies if @build_modern
      compress_indicies
    end
  end

  ##
  # Map gems file to specs
  #
  # @param [Array] gems Gems list
  # @return [Array]
  #
  def map_gems_to_specs(gems)
    gems.map.with_index do |gemfile, index|
      # rubocop:disable Metrics/LineLength
      Utils.logger.info("[#{index + 1}/#{gems.size}]: Processing #{gemfile.split('/')[-1]}")
      # rubocop:enable Metrics/LineLength

      if File.size(gemfile).zero?
        Utils.logger.warn("Skipping zero-length gem: #{gemfile}")
        next
      end

      begin
        spec = if ::Gem::Package.respond_to? :open
                 ::Gem::Package
                   .open(File.open(gemfile, 'rb'), 'r', &:metadata)
               else
                 ::Gem::Package.new(gemfile).spec
               end

        spec.loaded_from = gemfile

        # HACK: fuck this shit - borks all tests that use pl1
        if File.basename(gemfile, '.gem') != spec.original_name
          exp = spec.full_name
          exp << " (#{spec.original_name})" if
            spec.original_name != spec.full_name
          msg = "Skipping misnamed gem: #{gemfile} should be named #{exp}"
          Utils.logger.warn(msg)
          next
        end

        version = spec.version.version
        unless version =~ /^\d+\.\d+\.\d+.*/
          msg = "Skipping gem #{spec.full_name} - invalid version #{version}"
          Utils.logger.warn(msg)
          next
        end

        if ::Gem::VERSION >= '2.5.0'
          spec.abbreviate
          spec.sanitize
        else
          abbreviate spec
          sanitize spec
        end

        spec
      rescue SignalException
        msg = 'Received signal, exiting'
        Utils.logger.error(msg)
        raise
      rescue StandardError => e
        msg = ["Unable to process #{gemfile}",
               "#{e.message} (#{e.class})",
               "\t#{e.backtrace.join "\n\t"}"].join("\n")
        Utils.logger.debug(msg)
      end
    end.compact
  end

  def update_index
    make_temp_directories

    specs_mtime = File.stat(@dest_specs_index).mtime
    newest_mtime = Time.at(0)

    @updated_gems = gem_file_list.select do |gem|
      gem_mtime = File.stat(gem).mtime
      newest_mtime = gem_mtime if gem_mtime > newest_mtime
      gem_mtime > specs_mtime
    end

    if @updated_gems.empty?
      Utils.logger.info('No new gems')
      terminate_interaction(0)
    end

    specs = map_gems_to_specs(@updated_gems)
    prerelease, released = specs.partition { |s| s.version.prerelease? }

    ::Gem::Specification.dirs = []
    ::Gem::Specification.all = *specs
    files = if ::Gem::VERSION >= '2.5.0'
              build_marshal_gemspecs specs
            else
              build_marshal_gemspecs
            end

    ::Gem.time('Updated indexes') do
      update_specs_index(released, @dest_specs_index, @specs_index)
      update_specs_index(released,
                         @dest_latest_specs_index,
                         @latest_specs_index)
      update_specs_index(prerelease,
                         @dest_prerelease_specs_index,
                         @prerelease_specs_index)
    end

    if ::Gem::VERSION >= '2.5.0'
      compress_indices
    else
      compress_indicies
    end

    Utils.logger.info("Updating production dir #{@dest_directory}") if verbose
    files << @specs_index
    files << "#{@specs_index}.gz"
    files << @latest_specs_index
    files << "#{@latest_specs_index}.gz"
    files << @prerelease_specs_index
    files << "#{@prerelease_specs_index}.gz"

    files.each do |path|
      file = path.sub(%r{^#{Regexp.escape @directory}/?}, '')
      src_name = File.join(@directory, file)
      dst_name = File.join(@dest_directory, file)

      if ["#{@specs_index}.gz",
          "#{@latest_specs_index}.gz",
          "#{@prerelease_specs_index}.gz"].include?(path)
        res = build_zlib_file(file, src_name, dst_name)
        next unless res
      else
        FileUtils.mv(src_name,
                     dst_name,
                     verbose: verbose,
                     force: true)
      end

      File.utime(newest_mtime, newest_mtime, dst_name)
    end
  end

  def build_zlib_file(file, src_name, dst_name, from_source = false)
    content = Marshal.load(Zlib::GzipReader.open(src_name).read)
    create_zlib_file("#{dst_name}.orig", content)

    return false if @only_origin

    if from_source
      source_content = download_from_source(file)
      source_content = Marshal.load(Zlib::GzipReader
                                      .new(StringIO
                                             .new(source_content)).read)
    else
      source_content = Marshal.load(Zlib::GzipReader.open(dst_name).read)
    end

    return false if source_content.nil?
    new_content = source_content.concat(content).uniq
    create_zlib_file(dst_name, new_content)
  end

  def create_zlib_file(dst_name, content)
    temp_file = Tempfile.new('gemirro')

    Zlib::GzipWriter.open(temp_file.path) do |io|
      io.write(Marshal.dump(content))
    end

    FileUtils.mv(temp_file.path,
                 dst_name,
                 verbose: verbose,
                 force: true)
    Utils.cache.flush_key(File.basename(dst_name))
  end

  def verbose
    @verbose ||= ::Gem.configuration.really_verbose
  end
end

#directory ⇒ String

Returns:

  • (String) 


19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
 
# File 'lib/gemirro/indexer.rb', line 19

class Indexer < ::Gem::Indexer
  attr_accessor(:files,
                :quick_marshal_dir,
                :directory,
                :dest_directory,
                :only_origin,
                :updated_gems)

  ##
  # Create an indexer that will index the gems in +directory+.
  #
  # @param [String] directory Destination directory
  # @param [Hash] options Indexer options
  # @return [Array]
  ##
  def initialize(directory, options = {})
    require 'fileutils'
    require 'tmpdir'
    require 'zlib'

    unless defined?(Builder::XChar)
      raise 'Gem::Indexer requires that the XML Builder ' \
      'library be installed:' \
      "\n\tgem install builder"
    end

    options = { build_modern: true }.merge options

    @build_modern = options[:build_modern]

    @dest_directory = directory
    @directory = File.join(Dir.tmpdir,
                           "gem_generate_index_#{rand(1_000_000_000)}")

    marshal_name = "Marshal.#{::Gem.marshal_version}"

    @master_index = File.join @directory, 'yaml'
    @marshal_index = File.join @directory, marshal_name

    @quick_dir = File.join @directory, 'quick'
    @quick_marshal_dir = File.join @quick_dir, marshal_name
    @quick_marshal_dir_base = File.join 'quick', marshal_name # FIX: UGH

    @quick_index = File.join @quick_dir, 'index'
    @latest_index = File.join @quick_dir, 'latest_index'

    @specs_index = File.join @directory, "specs.#{::Gem.marshal_version}"
    @latest_specs_index =
      File.join(@directory, "latest_specs.#{::Gem.marshal_version}")
    @prerelease_specs_index =
      File.join(@directory, "prerelease_specs.#{::Gem.marshal_version}")
    @dest_specs_index =
      File.join(@dest_directory, "specs.#{::Gem.marshal_version}")
    @dest_latest_specs_index =
      File.join(@dest_directory, "latest_specs.#{::Gem.marshal_version}")
    @dest_prerelease_specs_index =
      File.join(@dest_directory, "prerelease_specs.#{::Gem.marshal_version}")

    @files = []
  end

  ##
  # Generate indices on the destination directory
  #
  def install_indices
    install_indicies
  end

  ##
  # Generate indicies on the destination directory
  #
  # @return [Array]
  #
  def install_indicies
    Utils.logger
         .debug("Downloading index into production dir #{@dest_directory}")

    files = @files
    files.delete @quick_marshal_dir if files.include? @quick_dir

    if files.include?(@quick_marshal_dir) && !files.include?(@quick_dir)
      files.delete @quick_marshal_dir
      dst_name = File.join(@dest_directory, @quick_marshal_dir_base)
      FileUtils.mkdir_p(File.dirname(dst_name), verbose: verbose)
      FileUtils.rm_rf(dst_name, verbose: verbose)
      FileUtils.mv(@quick_marshal_dir, dst_name,
                   verbose: verbose, force: true)
    end

    files.each do |path|
      file = path.sub(%r{^#{Regexp.escape @directory}/?}, '')
      src_name = File.join(@directory, file)
      dst_name = File.join(@dest_directory, file)

      if ["#{@specs_index}.gz",
          "#{@latest_specs_index}.gz",
          "#{@prerelease_specs_index}.gz"].include?(path)
        res = build_zlib_file(file, src_name, dst_name, true)
        next unless res
      else
        source_content = download_from_source(file)
        next if source_content.nil?
        MirrorFile.new(dst_name).write(source_content)
      end

      FileUtils.rm_rf(path)
    end
  end

  ##
  # Download file from source
  #
  # @param [String] file File path
  # @return [String]
  #
  def download_from_source(file)
    source_host = Gemirro.configuration.source.host
    Utils.logger.info("Download from source: #{file}")
    resp = Http.get("#{source_host}/#{File.basename(file)}")
    return unless resp.code == 200
    resp.body
  end

  ##
  # Build indices
  #
  # @return [Array]
  #
  def build_indices
    build_indicies
  end

  ##
  # Build indicies
  #
  # @return [Array]
  #
  def build_indicies
    specs = *map_gems_to_specs(gem_file_list)
    specs.select! { |s| s.class == ::Gem::Specification }
    ::Gem::Specification.dirs = []
    ::Gem::Specification.all = specs

    if ::Gem::VERSION >= '2.5.0'
      build_marshal_gemspecs specs
      build_modern_indices specs if @build_modern
      compress_indices
    else
      build_marshal_gemspecs
      build_modern_indicies if @build_modern
      compress_indicies
    end
  end

  ##
  # Map gems file to specs
  #
  # @param [Array] gems Gems list
  # @return [Array]
  #
  def map_gems_to_specs(gems)
    gems.map.with_index do |gemfile, index|
      # rubocop:disable Metrics/LineLength
      Utils.logger.info("[#{index + 1}/#{gems.size}]: Processing #{gemfile.split('/')[-1]}")
      # rubocop:enable Metrics/LineLength

      if File.size(gemfile).zero?
        Utils.logger.warn("Skipping zero-length gem: #{gemfile}")
        next
      end

      begin
        spec = if ::Gem::Package.respond_to? :open
                 ::Gem::Package
                   .open(File.open(gemfile, 'rb'), 'r', &:metadata)
               else
                 ::Gem::Package.new(gemfile).spec
               end

        spec.loaded_from = gemfile

        # HACK: fuck this shit - borks all tests that use pl1
        if File.basename(gemfile, '.gem') != spec.original_name
          exp = spec.full_name
          exp << " (#{spec.original_name})" if
            spec.original_name != spec.full_name
          msg = "Skipping misnamed gem: #{gemfile} should be named #{exp}"
          Utils.logger.warn(msg)
          next
        end

        version = spec.version.version
        unless version =~ /^\d+\.\d+\.\d+.*/
          msg = "Skipping gem #{spec.full_name} - invalid version #{version}"
          Utils.logger.warn(msg)
          next
        end

        if ::Gem::VERSION >= '2.5.0'
          spec.abbreviate
          spec.sanitize
        else
          abbreviate spec
          sanitize spec
        end

        spec
      rescue SignalException
        msg = 'Received signal, exiting'
        Utils.logger.error(msg)
        raise
      rescue StandardError => e
        msg = ["Unable to process #{gemfile}",
               "#{e.message} (#{e.class})",
               "\t#{e.backtrace.join "\n\t"}"].join("\n")
        Utils.logger.debug(msg)
      end
    end.compact
  end

  def update_index
    make_temp_directories

    specs_mtime = File.stat(@dest_specs_index).mtime
    newest_mtime = Time.at(0)

    @updated_gems = gem_file_list.select do |gem|
      gem_mtime = File.stat(gem).mtime
      newest_mtime = gem_mtime if gem_mtime > newest_mtime
      gem_mtime > specs_mtime
    end

    if @updated_gems.empty?
      Utils.logger.info('No new gems')
      terminate_interaction(0)
    end

    specs = map_gems_to_specs(@updated_gems)
    prerelease, released = specs.partition { |s| s.version.prerelease? }

    ::Gem::Specification.dirs = []
    ::Gem::Specification.all = *specs
    files = if ::Gem::VERSION >= '2.5.0'
              build_marshal_gemspecs specs
            else
              build_marshal_gemspecs
            end

    ::Gem.time('Updated indexes') do
      update_specs_index(released, @dest_specs_index, @specs_index)
      update_specs_index(released,
                         @dest_latest_specs_index,
                         @latest_specs_index)
      update_specs_index(prerelease,
                         @dest_prerelease_specs_index,
                         @prerelease_specs_index)
    end

    if ::Gem::VERSION >= '2.5.0'
      compress_indices
    else
      compress_indicies
    end

    Utils.logger.info("Updating production dir #{@dest_directory}") if verbose
    files << @specs_index
    files << "#{@specs_index}.gz"
    files << @latest_specs_index
    files << "#{@latest_specs_index}.gz"
    files << @prerelease_specs_index
    files << "#{@prerelease_specs_index}.gz"

    files.each do |path|
      file = path.sub(%r{^#{Regexp.escape @directory}/?}, '')
      src_name = File.join(@directory, file)
      dst_name = File.join(@dest_directory, file)

      if ["#{@specs_index}.gz",
          "#{@latest_specs_index}.gz",
          "#{@prerelease_specs_index}.gz"].include?(path)
        res = build_zlib_file(file, src_name, dst_name)
        next unless res
      else
        FileUtils.mv(src_name,
                     dst_name,
                     verbose: verbose,
                     force: true)
      end

      File.utime(newest_mtime, newest_mtime, dst_name)
    end
  end

  def build_zlib_file(file, src_name, dst_name, from_source = false)
    content = Marshal.load(Zlib::GzipReader.open(src_name).read)
    create_zlib_file("#{dst_name}.orig", content)

    return false if @only_origin

    if from_source
      source_content = download_from_source(file)
      source_content = Marshal.load(Zlib::GzipReader
                                      .new(StringIO
                                             .new(source_content)).read)
    else
      source_content = Marshal.load(Zlib::GzipReader.open(dst_name).read)
    end

    return false if source_content.nil?
    new_content = source_content.concat(content).uniq
    create_zlib_file(dst_name, new_content)
  end

  def create_zlib_file(dst_name, content)
    temp_file = Tempfile.new('gemirro')

    Zlib::GzipWriter.open(temp_file.path) do |io|
      io.write(Marshal.dump(content))
    end

    FileUtils.mv(temp_file.path,
                 dst_name,
                 verbose: verbose,
                 force: true)
    Utils.cache.flush_key(File.basename(dst_name))
  end

  def verbose
    @verbose ||= ::Gem.configuration.really_verbose
  end
end

#files ⇒ Array

Returns:

  • (Array) 


19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
 
# File 'lib/gemirro/indexer.rb', line 19

class Indexer < ::Gem::Indexer
  attr_accessor(:files,
                :quick_marshal_dir,
                :directory,
                :dest_directory,
                :only_origin,
                :updated_gems)

  ##
  # Create an indexer that will index the gems in +directory+.
  #
  # @param [String] directory Destination directory
  # @param [Hash] options Indexer options
  # @return [Array]
  ##
  def initialize(directory, options = {})
    require 'fileutils'
    require 'tmpdir'
    require 'zlib'

    unless defined?(Builder::XChar)
      raise 'Gem::Indexer requires that the XML Builder ' \
      'library be installed:' \
      "\n\tgem install builder"
    end

    options = { build_modern: true }.merge options

    @build_modern = options[:build_modern]

    @dest_directory = directory
    @directory = File.join(Dir.tmpdir,
                           "gem_generate_index_#{rand(1_000_000_000)}")

    marshal_name = "Marshal.#{::Gem.marshal_version}"

    @master_index = File.join @directory, 'yaml'
    @marshal_index = File.join @directory, marshal_name

    @quick_dir = File.join @directory, 'quick'
    @quick_marshal_dir = File.join @quick_dir, marshal_name
    @quick_marshal_dir_base = File.join 'quick', marshal_name # FIX: UGH

    @quick_index = File.join @quick_dir, 'index'
    @latest_index = File.join @quick_dir, 'latest_index'

    @specs_index = File.join @directory, "specs.#{::Gem.marshal_version}"
    @latest_specs_index =
      File.join(@directory, "latest_specs.#{::Gem.marshal_version}")
    @prerelease_specs_index =
      File.join(@directory, "prerelease_specs.#{::Gem.marshal_version}")
    @dest_specs_index =
      File.join(@dest_directory, "specs.#{::Gem.marshal_version}")
    @dest_latest_specs_index =
      File.join(@dest_directory, "latest_specs.#{::Gem.marshal_version}")
    @dest_prerelease_specs_index =
      File.join(@dest_directory, "prerelease_specs.#{::Gem.marshal_version}")

    @files = []
  end

  ##
  # Generate indices on the destination directory
  #
  def install_indices
    install_indicies
  end

  ##
  # Generate indicies on the destination directory
  #
  # @return [Array]
  #
  def install_indicies
    Utils.logger
         .debug("Downloading index into production dir #{@dest_directory}")

    files = @files
    files.delete @quick_marshal_dir if files.include? @quick_dir

    if files.include?(@quick_marshal_dir) && !files.include?(@quick_dir)
      files.delete @quick_marshal_dir
      dst_name = File.join(@dest_directory, @quick_marshal_dir_base)
      FileUtils.mkdir_p(File.dirname(dst_name), verbose: verbose)
      FileUtils.rm_rf(dst_name, verbose: verbose)
      FileUtils.mv(@quick_marshal_dir, dst_name,
                   verbose: verbose, force: true)
    end

    files.each do |path|
      file = path.sub(%r{^#{Regexp.escape @directory}/?}, '')
      src_name = File.join(@directory, file)
      dst_name = File.join(@dest_directory, file)

      if ["#{@specs_index}.gz",
          "#{@latest_specs_index}.gz",
          "#{@prerelease_specs_index}.gz"].include?(path)
        res = build_zlib_file(file, src_name, dst_name, true)
        next unless res
      else
        source_content = download_from_source(file)
        next if source_content.nil?
        MirrorFile.new(dst_name).write(source_content)
      end

      FileUtils.rm_rf(path)
    end
  end

  ##
  # Download file from source
  #
  # @param [String] file File path
  # @return [String]
  #
  def download_from_source(file)
    source_host = Gemirro.configuration.source.host
    Utils.logger.info("Download from source: #{file}")
    resp = Http.get("#{source_host}/#{File.basename(file)}")
    return unless resp.code == 200
    resp.body
  end

  ##
  # Build indices
  #
  # @return [Array]
  #
  def build_indices
    build_indicies
  end

  ##
  # Build indicies
  #
  # @return [Array]
  #
  def build_indicies
    specs = *map_gems_to_specs(gem_file_list)
    specs.select! { |s| s.class == ::Gem::Specification }
    ::Gem::Specification.dirs = []
    ::Gem::Specification.all = specs

    if ::Gem::VERSION >= '2.5.0'
      build_marshal_gemspecs specs
      build_modern_indices specs if @build_modern
      compress_indices
    else
      build_marshal_gemspecs
      build_modern_indicies if @build_modern
      compress_indicies
    end
  end

  ##
  # Map gems file to specs
  #
  # @param [Array] gems Gems list
  # @return [Array]
  #
  def map_gems_to_specs(gems)
    gems.map.with_index do |gemfile, index|
      # rubocop:disable Metrics/LineLength
      Utils.logger.info("[#{index + 1}/#{gems.size}]: Processing #{gemfile.split('/')[-1]}")
      # rubocop:enable Metrics/LineLength

      if File.size(gemfile).zero?
        Utils.logger.warn("Skipping zero-length gem: #{gemfile}")
        next
      end

      begin
        spec = if ::Gem::Package.respond_to? :open
                 ::Gem::Package
                   .open(File.open(gemfile, 'rb'), 'r', &:metadata)
               else
                 ::Gem::Package.new(gemfile).spec
               end

        spec.loaded_from = gemfile

        # HACK: fuck this shit - borks all tests that use pl1
        if File.basename(gemfile, '.gem') != spec.original_name
          exp = spec.full_name
          exp << " (#{spec.original_name})" if
            spec.original_name != spec.full_name
          msg = "Skipping misnamed gem: #{gemfile} should be named #{exp}"
          Utils.logger.warn(msg)
          next
        end

        version = spec.version.version
        unless version =~ /^\d+\.\d+\.\d+.*/
          msg = "Skipping gem #{spec.full_name} - invalid version #{version}"
          Utils.logger.warn(msg)
          next
        end

        if ::Gem::VERSION >= '2.5.0'
          spec.abbreviate
          spec.sanitize
        else
          abbreviate spec
          sanitize spec
        end

        spec
      rescue SignalException
        msg = 'Received signal, exiting'
        Utils.logger.error(msg)
        raise
      rescue StandardError => e
        msg = ["Unable to process #{gemfile}",
               "#{e.message} (#{e.class})",
               "\t#{e.backtrace.join "\n\t"}"].join("\n")
        Utils.logger.debug(msg)
      end
    end.compact
  end

  def update_index
    make_temp_directories

    specs_mtime = File.stat(@dest_specs_index).mtime
    newest_mtime = Time.at(0)

    @updated_gems = gem_file_list.select do |gem|
      gem_mtime = File.stat(gem).mtime
      newest_mtime = gem_mtime if gem_mtime > newest_mtime
      gem_mtime > specs_mtime
    end

    if @updated_gems.empty?
      Utils.logger.info('No new gems')
      terminate_interaction(0)
    end

    specs = map_gems_to_specs(@updated_gems)
    prerelease, released = specs.partition { |s| s.version.prerelease? }

    ::Gem::Specification.dirs = []
    ::Gem::Specification.all = *specs
    files = if ::Gem::VERSION >= '2.5.0'
              build_marshal_gemspecs specs
            else
              build_marshal_gemspecs
            end

    ::Gem.time('Updated indexes') do
      update_specs_index(released, @dest_specs_index, @specs_index)
      update_specs_index(released,
                         @dest_latest_specs_index,
                         @latest_specs_index)
      update_specs_index(prerelease,
                         @dest_prerelease_specs_index,
                         @prerelease_specs_index)
    end

    if ::Gem::VERSION >= '2.5.0'
      compress_indices
    else
      compress_indicies
    end

    Utils.logger.info("Updating production dir #{@dest_directory}") if verbose
    files << @specs_index
    files << "#{@specs_index}.gz"
    files << @latest_specs_index
    files << "#{@latest_specs_index}.gz"
    files << @prerelease_specs_index
    files << "#{@prerelease_specs_index}.gz"

    files.each do |path|
      file = path.sub(%r{^#{Regexp.escape @directory}/?}, '')
      src_name = File.join(@directory, file)
      dst_name = File.join(@dest_directory, file)

      if ["#{@specs_index}.gz",
          "#{@latest_specs_index}.gz",
          "#{@prerelease_specs_index}.gz"].include?(path)
        res = build_zlib_file(file, src_name, dst_name)
        next unless res
      else
        FileUtils.mv(src_name,
                     dst_name,
                     verbose: verbose,
                     force: true)
      end

      File.utime(newest_mtime, newest_mtime, dst_name)
    end
  end

  def build_zlib_file(file, src_name, dst_name, from_source = false)
    content = Marshal.load(Zlib::GzipReader.open(src_name).read)
    create_zlib_file("#{dst_name}.orig", content)

    return false if @only_origin

    if from_source
      source_content = download_from_source(file)
      source_content = Marshal.load(Zlib::GzipReader
                                      .new(StringIO
                                             .new(source_content)).read)
    else
      source_content = Marshal.load(Zlib::GzipReader.open(dst_name).read)
    end

    return false if source_content.nil?
    new_content = source_content.concat(content).uniq
    create_zlib_file(dst_name, new_content)
  end

  def create_zlib_file(dst_name, content)
    temp_file = Tempfile.new('gemirro')

    Zlib::GzipWriter.open(temp_file.path) do |io|
      io.write(Marshal.dump(content))
    end

    FileUtils.mv(temp_file.path,
                 dst_name,
                 verbose: verbose,
                 force: true)
    Utils.cache.flush_key(File.basename(dst_name))
  end

  def verbose
    @verbose ||= ::Gem.configuration.really_verbose
  end
end

#only_origin ⇒ Boolean

Returns:

  • (Boolean) 


19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
 
# File 'lib/gemirro/indexer.rb', line 19

class Indexer < ::Gem::Indexer
  attr_accessor(:files,
                :quick_marshal_dir,
                :directory,
                :dest_directory,
                :only_origin,
                :updated_gems)

  ##
  # Create an indexer that will index the gems in +directory+.
  #
  # @param [String] directory Destination directory
  # @param [Hash] options Indexer options
  # @return [Array]
  ##
  def initialize(directory, options = {})
    require 'fileutils'
    require 'tmpdir'
    require 'zlib'

    unless defined?(Builder::XChar)
      raise 'Gem::Indexer requires that the XML Builder ' \
      'library be installed:' \
      "\n\tgem install builder"
    end

    options = { build_modern: true }.merge options

    @build_modern = options[:build_modern]

    @dest_directory = directory
    @directory = File.join(Dir.tmpdir,
                           "gem_generate_index_#{rand(1_000_000_000)}")

    marshal_name = "Marshal.#{::Gem.marshal_version}"

    @master_index = File.join @directory, 'yaml'
    @marshal_index = File.join @directory, marshal_name

    @quick_dir = File.join @directory, 'quick'
    @quick_marshal_dir = File.join @quick_dir, marshal_name
    @quick_marshal_dir_base = File.join 'quick', marshal_name # FIX: UGH

    @quick_index = File.join @quick_dir, 'index'
    @latest_index = File.join @quick_dir, 'latest_index'

    @specs_index = File.join @directory, "specs.#{::Gem.marshal_version}"
    @latest_specs_index =
      File.join(@directory, "latest_specs.#{::Gem.marshal_version}")
    @prerelease_specs_index =
      File.join(@directory, "prerelease_specs.#{::Gem.marshal_version}")
    @dest_specs_index =
      File.join(@dest_directory, "specs.#{::Gem.marshal_version}")
    @dest_latest_specs_index =
      File.join(@dest_directory, "latest_specs.#{::Gem.marshal_version}")
    @dest_prerelease_specs_index =
      File.join(@dest_directory, "prerelease_specs.#{::Gem.marshal_version}")

    @files = []
  end

  ##
  # Generate indices on the destination directory
  #
  def install_indices
    install_indicies
  end

  ##
  # Generate indicies on the destination directory
  #
  # @return [Array]
  #
  def install_indicies
    Utils.logger
         .debug("Downloading index into production dir #{@dest_directory}")

    files = @files
    files.delete @quick_marshal_dir if files.include? @quick_dir

    if files.include?(@quick_marshal_dir) && !files.include?(@quick_dir)
      files.delete @quick_marshal_dir
      dst_name = File.join(@dest_directory, @quick_marshal_dir_base)
      FileUtils.mkdir_p(File.dirname(dst_name), verbose: verbose)
      FileUtils.rm_rf(dst_name, verbose: verbose)
      FileUtils.mv(@quick_marshal_dir, dst_name,
                   verbose: verbose, force: true)
    end

    files.each do |path|
      file = path.sub(%r{^#{Regexp.escape @directory}/?}, '')
      src_name = File.join(@directory, file)
      dst_name = File.join(@dest_directory, file)

      if ["#{@specs_index}.gz",
          "#{@latest_specs_index}.gz",
          "#{@prerelease_specs_index}.gz"].include?(path)
        res = build_zlib_file(file, src_name, dst_name, true)
        next unless res
      else
        source_content = download_from_source(file)
        next if source_content.nil?
        MirrorFile.new(dst_name).write(source_content)
      end

      FileUtils.rm_rf(path)
    end
  end

  ##
  # Download file from source
  #
  # @param [String] file File path
  # @return [String]
  #
  def download_from_source(file)
    source_host = Gemirro.configuration.source.host
    Utils.logger.info("Download from source: #{file}")
    resp = Http.get("#{source_host}/#{File.basename(file)}")
    return unless resp.code == 200
    resp.body
  end

  ##
  # Build indices
  #
  # @return [Array]
  #
  def build_indices
    build_indicies
  end

  ##
  # Build indicies
  #
  # @return [Array]
  #
  def build_indicies
    specs = *map_gems_to_specs(gem_file_list)
    specs.select! { |s| s.class == ::Gem::Specification }
    ::Gem::Specification.dirs = []
    ::Gem::Specification.all = specs

    if ::Gem::VERSION >= '2.5.0'
      build_marshal_gemspecs specs
      build_modern_indices specs if @build_modern
      compress_indices
    else
      build_marshal_gemspecs
      build_modern_indicies if @build_modern
      compress_indicies
    end
  end

  ##
  # Map gems file to specs
  #
  # @param [Array] gems Gems list
  # @return [Array]
  #
  def map_gems_to_specs(gems)
    gems.map.with_index do |gemfile, index|
      # rubocop:disable Metrics/LineLength
      Utils.logger.info("[#{index + 1}/#{gems.size}]: Processing #{gemfile.split('/')[-1]}")
      # rubocop:enable Metrics/LineLength

      if File.size(gemfile).zero?
        Utils.logger.warn("Skipping zero-length gem: #{gemfile}")
        next
      end

      begin
        spec = if ::Gem::Package.respond_to? :open
                 ::Gem::Package
                   .open(File.open(gemfile, 'rb'), 'r', &:metadata)
               else
                 ::Gem::Package.new(gemfile).spec
               end

        spec.loaded_from = gemfile

        # HACK: fuck this shit - borks all tests that use pl1
        if File.basename(gemfile, '.gem') != spec.original_name
          exp = spec.full_name
          exp << " (#{spec.original_name})" if
            spec.original_name != spec.full_name
          msg = "Skipping misnamed gem: #{gemfile} should be named #{exp}"
          Utils.logger.warn(msg)
          next
        end

        version = spec.version.version
        unless version =~ /^\d+\.\d+\.\d+.*/
          msg = "Skipping gem #{spec.full_name} - invalid version #{version}"
          Utils.logger.warn(msg)
          next
        end

        if ::Gem::VERSION >= '2.5.0'
          spec.abbreviate
          spec.sanitize
        else
          abbreviate spec
          sanitize spec
        end

        spec
      rescue SignalException
        msg = 'Received signal, exiting'
        Utils.logger.error(msg)
        raise
      rescue StandardError => e
        msg = ["Unable to process #{gemfile}",
               "#{e.message} (#{e.class})",
               "\t#{e.backtrace.join "\n\t"}"].join("\n")
        Utils.logger.debug(msg)
      end
    end.compact
  end

  def update_index
    make_temp_directories

    specs_mtime = File.stat(@dest_specs_index).mtime
    newest_mtime = Time.at(0)

    @updated_gems = gem_file_list.select do |gem|
      gem_mtime = File.stat(gem).mtime
      newest_mtime = gem_mtime if gem_mtime > newest_mtime
      gem_mtime > specs_mtime
    end

    if @updated_gems.empty?
      Utils.logger.info('No new gems')
      terminate_interaction(0)
    end

    specs = map_gems_to_specs(@updated_gems)
    prerelease, released = specs.partition { |s| s.version.prerelease? }

    ::Gem::Specification.dirs = []
    ::Gem::Specification.all = *specs
    files = if ::Gem::VERSION >= '2.5.0'
              build_marshal_gemspecs specs
            else
              build_marshal_gemspecs
            end

    ::Gem.time('Updated indexes') do
      update_specs_index(released, @dest_specs_index, @specs_index)
      update_specs_index(released,
                         @dest_latest_specs_index,
                         @latest_specs_index)
      update_specs_index(prerelease,
                         @dest_prerelease_specs_index,
                         @prerelease_specs_index)
    end

    if ::Gem::VERSION >= '2.5.0'
      compress_indices
    else
      compress_indicies
    end

    Utils.logger.info("Updating production dir #{@dest_directory}") if verbose
    files << @specs_index
    files << "#{@specs_index}.gz"
    files << @latest_specs_index
    files << "#{@latest_specs_index}.gz"
    files << @prerelease_specs_index
    files << "#{@prerelease_specs_index}.gz"

    files.each do |path|
      file = path.sub(%r{^#{Regexp.escape @directory}/?}, '')
      src_name = File.join(@directory, file)
      dst_name = File.join(@dest_directory, file)

      if ["#{@specs_index}.gz",
          "#{@latest_specs_index}.gz",
          "#{@prerelease_specs_index}.gz"].include?(path)
        res = build_zlib_file(file, src_name, dst_name)
        next unless res
      else
        FileUtils.mv(src_name,
                     dst_name,
                     verbose: verbose,
                     force: true)
      end

      File.utime(newest_mtime, newest_mtime, dst_name)
    end
  end

  def build_zlib_file(file, src_name, dst_name, from_source = false)
    content = Marshal.load(Zlib::GzipReader.open(src_name).read)
    create_zlib_file("#{dst_name}.orig", content)

    return false if @only_origin

    if from_source
      source_content = download_from_source(file)
      source_content = Marshal.load(Zlib::GzipReader
                                      .new(StringIO
                                             .new(source_content)).read)
    else
      source_content = Marshal.load(Zlib::GzipReader.open(dst_name).read)
    end

    return false if source_content.nil?
    new_content = source_content.concat(content).uniq
    create_zlib_file(dst_name, new_content)
  end

  def create_zlib_file(dst_name, content)
    temp_file = Tempfile.new('gemirro')

    Zlib::GzipWriter.open(temp_file.path) do |io|
      io.write(Marshal.dump(content))
    end

    FileUtils.mv(temp_file.path,
                 dst_name,
                 verbose: verbose,
                 force: true)
    Utils.cache.flush_key(File.basename(dst_name))
  end

  def verbose
    @verbose ||= ::Gem.configuration.really_verbose
  end
end

#quick_marshal_dir ⇒ String

Returns:

  • (String) 


19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
 
# File 'lib/gemirro/indexer.rb', line 19

class Indexer < ::Gem::Indexer
  attr_accessor(:files,
                :quick_marshal_dir,
                :directory,
                :dest_directory,
                :only_origin,
                :updated_gems)

  ##
  # Create an indexer that will index the gems in +directory+.
  #
  # @param [String] directory Destination directory
  # @param [Hash] options Indexer options
  # @return [Array]
  ##
  def initialize(directory, options = {})
    require 'fileutils'
    require 'tmpdir'
    require 'zlib'

    unless defined?(Builder::XChar)
      raise 'Gem::Indexer requires that the XML Builder ' \
      'library be installed:' \
      "\n\tgem install builder"
    end

    options = { build_modern: true }.merge options

    @build_modern = options[:build_modern]

    @dest_directory = directory
    @directory = File.join(Dir.tmpdir,
                           "gem_generate_index_#{rand(1_000_000_000)}")

    marshal_name = "Marshal.#{::Gem.marshal_version}"

    @master_index = File.join @directory, 'yaml'
    @marshal_index = File.join @directory, marshal_name

    @quick_dir = File.join @directory, 'quick'
    @quick_marshal_dir = File.join @quick_dir, marshal_name
    @quick_marshal_dir_base = File.join 'quick', marshal_name # FIX: UGH

    @quick_index = File.join @quick_dir, 'index'
    @latest_index = File.join @quick_dir, 'latest_index'

    @specs_index = File.join @directory, "specs.#{::Gem.marshal_version}"
    @latest_specs_index =
      File.join(@directory, "latest_specs.#{::Gem.marshal_version}")
    @prerelease_specs_index =
      File.join(@directory, "prerelease_specs.#{::Gem.marshal_version}")
    @dest_specs_index =
      File.join(@dest_directory, "specs.#{::Gem.marshal_version}")
    @dest_latest_specs_index =
      File.join(@dest_directory, "latest_specs.#{::Gem.marshal_version}")
    @dest_prerelease_specs_index =
      File.join(@dest_directory, "prerelease_specs.#{::Gem.marshal_version}")

    @files = []
  end

  ##
  # Generate indices on the destination directory
  #
  def install_indices
    install_indicies
  end

  ##
  # Generate indicies on the destination directory
  #
  # @return [Array]
  #
  def install_indicies
    Utils.logger
         .debug("Downloading index into production dir #{@dest_directory}")

    files = @files
    files.delete @quick_marshal_dir if files.include? @quick_dir

    if files.include?(@quick_marshal_dir) && !files.include?(@quick_dir)
      files.delete @quick_marshal_dir
      dst_name = File.join(@dest_directory, @quick_marshal_dir_base)
      FileUtils.mkdir_p(File.dirname(dst_name), verbose: verbose)
      FileUtils.rm_rf(dst_name, verbose: verbose)
      FileUtils.mv(@quick_marshal_dir, dst_name,
                   verbose: verbose, force: true)
    end

    files.each do |path|
      file = path.sub(%r{^#{Regexp.escape @directory}/?}, '')
      src_name = File.join(@directory, file)
      dst_name = File.join(@dest_directory, file)

      if ["#{@specs_index}.gz",
          "#{@latest_specs_index}.gz",
          "#{@prerelease_specs_index}.gz"].include?(path)
        res = build_zlib_file(file, src_name, dst_name, true)
        next unless res
      else
        source_content = download_from_source(file)
        next if source_content.nil?
        MirrorFile.new(dst_name).write(source_content)
      end

      FileUtils.rm_rf(path)
    end
  end

  ##
  # Download file from source
  #
  # @param [String] file File path
  # @return [String]
  #
  def download_from_source(file)
    source_host = Gemirro.configuration.source.host
    Utils.logger.info("Download from source: #{file}")
    resp = Http.get("#{source_host}/#{File.basename(file)}")
    return unless resp.code == 200
    resp.body
  end

  ##
  # Build indices
  #
  # @return [Array]
  #
  def build_indices
    build_indicies
  end

  ##
  # Build indicies
  #
  # @return [Array]
  #
  def build_indicies
    specs = *map_gems_to_specs(gem_file_list)
    specs.select! { |s| s.class == ::Gem::Specification }
    ::Gem::Specification.dirs = []
    ::Gem::Specification.all = specs

    if ::Gem::VERSION >= '2.5.0'
      build_marshal_gemspecs specs
      build_modern_indices specs if @build_modern
      compress_indices
    else
      build_marshal_gemspecs
      build_modern_indicies if @build_modern
      compress_indicies
    end
  end

  ##
  # Map gems file to specs
  #
  # @param [Array] gems Gems list
  # @return [Array]
  #
  def map_gems_to_specs(gems)
    gems.map.with_index do |gemfile, index|
      # rubocop:disable Metrics/LineLength
      Utils.logger.info("[#{index + 1}/#{gems.size}]: Processing #{gemfile.split('/')[-1]}")
      # rubocop:enable Metrics/LineLength

      if File.size(gemfile).zero?
        Utils.logger.warn("Skipping zero-length gem: #{gemfile}")
        next
      end

      begin
        spec = if ::Gem::Package.respond_to? :open
                 ::Gem::Package
                   .open(File.open(gemfile, 'rb'), 'r', &:metadata)
               else
                 ::Gem::Package.new(gemfile).spec
               end

        spec.loaded_from = gemfile

        # HACK: fuck this shit - borks all tests that use pl1
        if File.basename(gemfile, '.gem') != spec.original_name
          exp = spec.full_name
          exp << " (#{spec.original_name})" if
            spec.original_name != spec.full_name
          msg = "Skipping misnamed gem: #{gemfile} should be named #{exp}"
          Utils.logger.warn(msg)
          next
        end

        version = spec.version.version
        unless version =~ /^\d+\.\d+\.\d+.*/
          msg = "Skipping gem #{spec.full_name} - invalid version #{version}"
          Utils.logger.warn(msg)
          next
        end

        if ::Gem::VERSION >= '2.5.0'
          spec.abbreviate
          spec.sanitize
        else
          abbreviate spec
          sanitize spec
        end

        spec
      rescue SignalException
        msg = 'Received signal, exiting'
        Utils.logger.error(msg)
        raise
      rescue StandardError => e
        msg = ["Unable to process #{gemfile}",
               "#{e.message} (#{e.class})",
               "\t#{e.backtrace.join "\n\t"}"].join("\n")
        Utils.logger.debug(msg)
      end
    end.compact
  end

  def update_index
    make_temp_directories

    specs_mtime = File.stat(@dest_specs_index).mtime
    newest_mtime = Time.at(0)

    @updated_gems = gem_file_list.select do |gem|
      gem_mtime = File.stat(gem).mtime
      newest_mtime = gem_mtime if gem_mtime > newest_mtime
      gem_mtime > specs_mtime
    end

    if @updated_gems.empty?
      Utils.logger.info('No new gems')
      terminate_interaction(0)
    end

    specs = map_gems_to_specs(@updated_gems)
    prerelease, released = specs.partition { |s| s.version.prerelease? }

    ::Gem::Specification.dirs = []
    ::Gem::Specification.all = *specs
    files = if ::Gem::VERSION >= '2.5.0'
              build_marshal_gemspecs specs
            else
              build_marshal_gemspecs
            end

    ::Gem.time('Updated indexes') do
      update_specs_index(released, @dest_specs_index, @specs_index)
      update_specs_index(released,
                         @dest_latest_specs_index,
                         @latest_specs_index)
      update_specs_index(prerelease,
                         @dest_prerelease_specs_index,
                         @prerelease_specs_index)
    end

    if ::Gem::VERSION >= '2.5.0'
      compress_indices
    else
      compress_indicies
    end

    Utils.logger.info("Updating production dir #{@dest_directory}") if verbose
    files << @specs_index
    files << "#{@specs_index}.gz"
    files << @latest_specs_index
    files << "#{@latest_specs_index}.gz"
    files << @prerelease_specs_index
    files << "#{@prerelease_specs_index}.gz"

    files.each do |path|
      file = path.sub(%r{^#{Regexp.escape @directory}/?}, '')
      src_name = File.join(@directory, file)
      dst_name = File.join(@dest_directory, file)

      if ["#{@specs_index}.gz",
          "#{@latest_specs_index}.gz",
          "#{@prerelease_specs_index}.gz"].include?(path)
        res = build_zlib_file(file, src_name, dst_name)
        next unless res
      else
        FileUtils.mv(src_name,
                     dst_name,
                     verbose: verbose,
                     force: true)
      end

      File.utime(newest_mtime, newest_mtime, dst_name)
    end
  end

  def build_zlib_file(file, src_name, dst_name, from_source = false)
    content = Marshal.load(Zlib::GzipReader.open(src_name).read)
    create_zlib_file("#{dst_name}.orig", content)

    return false if @only_origin

    if from_source
      source_content = download_from_source(file)
      source_content = Marshal.load(Zlib::GzipReader
                                      .new(StringIO
                                             .new(source_content)).read)
    else
      source_content = Marshal.load(Zlib::GzipReader.open(dst_name).read)
    end

    return false if source_content.nil?
    new_content = source_content.concat(content).uniq
    create_zlib_file(dst_name, new_content)
  end

  def create_zlib_file(dst_name, content)
    temp_file = Tempfile.new('gemirro')

    Zlib::GzipWriter.open(temp_file.path) do |io|
      io.write(Marshal.dump(content))
    end

    FileUtils.mv(temp_file.path,
                 dst_name,
                 verbose: verbose,
                 force: true)
    Utils.cache.flush_key(File.basename(dst_name))
  end

  def verbose
    @verbose ||= ::Gem.configuration.really_verbose
  end
end

#updated_gems ⇒ Array

Returns:

  • (Array) 


19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
 
# File 'lib/gemirro/indexer.rb', line 19

class Indexer < ::Gem::Indexer
  attr_accessor(:files,
                :quick_marshal_dir,
                :directory,
                :dest_directory,
                :only_origin,
                :updated_gems)

  ##
  # Create an indexer that will index the gems in +directory+.
  #
  # @param [String] directory Destination directory
  # @param [Hash] options Indexer options
  # @return [Array]
  ##
  def initialize(directory, options = {})
    require 'fileutils'
    require 'tmpdir'
    require 'zlib'

    unless defined?(Builder::XChar)
      raise 'Gem::Indexer requires that the XML Builder ' \
      'library be installed:' \
      "\n\tgem install builder"
    end

    options = { build_modern: true }.merge options

    @build_modern = options[:build_modern]

    @dest_directory = directory
    @directory = File.join(Dir.tmpdir,
                           "gem_generate_index_#{rand(1_000_000_000)}")

    marshal_name = "Marshal.#{::Gem.marshal_version}"

    @master_index = File.join @directory, 'yaml'
    @marshal_index = File.join @directory, marshal_name

    @quick_dir = File.join @directory, 'quick'
    @quick_marshal_dir = File.join @quick_dir, marshal_name
    @quick_marshal_dir_base = File.join 'quick', marshal_name # FIX: UGH

    @quick_index = File.join @quick_dir, 'index'
    @latest_index = File.join @quick_dir, 'latest_index'

    @specs_index = File.join @directory, "specs.#{::Gem.marshal_version}"
    @latest_specs_index =
      File.join(@directory, "latest_specs.#{::Gem.marshal_version}")
    @prerelease_specs_index =
      File.join(@directory, "prerelease_specs.#{::Gem.marshal_version}")
    @dest_specs_index =
      File.join(@dest_directory, "specs.#{::Gem.marshal_version}")
    @dest_latest_specs_index =
      File.join(@dest_directory, "latest_specs.#{::Gem.marshal_version}")
    @dest_prerelease_specs_index =
      File.join(@dest_directory, "prerelease_specs.#{::Gem.marshal_version}")

    @files = []
  end

  ##
  # Generate indices on the destination directory
  #
  def install_indices
    install_indicies
  end

  ##
  # Generate indicies on the destination directory
  #
  # @return [Array]
  #
  def install_indicies
    Utils.logger
         .debug("Downloading index into production dir #{@dest_directory}")

    files = @files
    files.delete @quick_marshal_dir if files.include? @quick_dir

    if files.include?(@quick_marshal_dir) && !files.include?(@quick_dir)
      files.delete @quick_marshal_dir
      dst_name = File.join(@dest_directory, @quick_marshal_dir_base)
      FileUtils.mkdir_p(File.dirname(dst_name), verbose: verbose)
      FileUtils.rm_rf(dst_name, verbose: verbose)
      FileUtils.mv(@quick_marshal_dir, dst_name,
                   verbose: verbose, force: true)
    end

    files.each do |path|
      file = path.sub(%r{^#{Regexp.escape @directory}/?}, '')
      src_name = File.join(@directory, file)
      dst_name = File.join(@dest_directory, file)

      if ["#{@specs_index}.gz",
          "#{@latest_specs_index}.gz",
          "#{@prerelease_specs_index}.gz"].include?(path)
        res = build_zlib_file(file, src_name, dst_name, true)
        next unless res
      else
        source_content = download_from_source(file)
        next if source_content.nil?
        MirrorFile.new(dst_name).write(source_content)
      end

      FileUtils.rm_rf(path)
    end
  end

  ##
  # Download file from source
  #
  # @param [String] file File path
  # @return [String]
  #
  def download_from_source(file)
    source_host = Gemirro.configuration.source.host
    Utils.logger.info("Download from source: #{file}")
    resp = Http.get("#{source_host}/#{File.basename(file)}")
    return unless resp.code == 200
    resp.body
  end

  ##
  # Build indices
  #
  # @return [Array]
  #
  def build_indices
    build_indicies
  end

  ##
  # Build indicies
  #
  # @return [Array]
  #
  def build_indicies
    specs = *map_gems_to_specs(gem_file_list)
    specs.select! { |s| s.class == ::Gem::Specification }
    ::Gem::Specification.dirs = []
    ::Gem::Specification.all = specs

    if ::Gem::VERSION >= '2.5.0'
      build_marshal_gemspecs specs
      build_modern_indices specs if @build_modern
      compress_indices
    else
      build_marshal_gemspecs
      build_modern_indicies if @build_modern
      compress_indicies
    end
  end

  ##
  # Map gems file to specs
  #
  # @param [Array] gems Gems list
  # @return [Array]
  #
  def map_gems_to_specs(gems)
    gems.map.with_index do |gemfile, index|
      # rubocop:disable Metrics/LineLength
      Utils.logger.info("[#{index + 1}/#{gems.size}]: Processing #{gemfile.split('/')[-1]}")
      # rubocop:enable Metrics/LineLength

      if File.size(gemfile).zero?
        Utils.logger.warn("Skipping zero-length gem: #{gemfile}")
        next
      end

      begin
        spec = if ::Gem::Package.respond_to? :open
                 ::Gem::Package
                   .open(File.open(gemfile, 'rb'), 'r', &:metadata)
               else
                 ::Gem::Package.new(gemfile).spec
               end

        spec.loaded_from = gemfile

        # HACK: fuck this shit - borks all tests that use pl1
        if File.basename(gemfile, '.gem') != spec.original_name
          exp = spec.full_name
          exp << " (#{spec.original_name})" if
            spec.original_name != spec.full_name
          msg = "Skipping misnamed gem: #{gemfile} should be named #{exp}"
          Utils.logger.warn(msg)
          next
        end

        version = spec.version.version
        unless version =~ /^\d+\.\d+\.\d+.*/
          msg = "Skipping gem #{spec.full_name} - invalid version #{version}"
          Utils.logger.warn(msg)
          next
        end

        if ::Gem::VERSION >= '2.5.0'
          spec.abbreviate
          spec.sanitize
        else
          abbreviate spec
          sanitize spec
        end

        spec
      rescue SignalException
        msg = 'Received signal, exiting'
        Utils.logger.error(msg)
        raise
      rescue StandardError => e
        msg = ["Unable to process #{gemfile}",
               "#{e.message} (#{e.class})",
               "\t#{e.backtrace.join "\n\t"}"].join("\n")
        Utils.logger.debug(msg)
      end
    end.compact
  end

  def update_index
    make_temp_directories

    specs_mtime = File.stat(@dest_specs_index).mtime
    newest_mtime = Time.at(0)

    @updated_gems = gem_file_list.select do |gem|
      gem_mtime = File.stat(gem).mtime
      newest_mtime = gem_mtime if gem_mtime > newest_mtime
      gem_mtime > specs_mtime
    end

    if @updated_gems.empty?
      Utils.logger.info('No new gems')
      terminate_interaction(0)
    end

    specs = map_gems_to_specs(@updated_gems)
    prerelease, released = specs.partition { |s| s.version.prerelease? }

    ::Gem::Specification.dirs = []
    ::Gem::Specification.all = *specs
    files = if ::Gem::VERSION >= '2.5.0'
              build_marshal_gemspecs specs
            else
              build_marshal_gemspecs
            end

    ::Gem.time('Updated indexes') do
      update_specs_index(released, @dest_specs_index, @specs_index)
      update_specs_index(released,
                         @dest_latest_specs_index,
                         @latest_specs_index)
      update_specs_index(prerelease,
                         @dest_prerelease_specs_index,
                         @prerelease_specs_index)
    end

    if ::Gem::VERSION >= '2.5.0'
      compress_indices
    else
      compress_indicies
    end

    Utils.logger.info("Updating production dir #{@dest_directory}") if verbose
    files << @specs_index
    files << "#{@specs_index}.gz"
    files << @latest_specs_index
    files << "#{@latest_specs_index}.gz"
    files << @prerelease_specs_index
    files << "#{@prerelease_specs_index}.gz"

    files.each do |path|
      file = path.sub(%r{^#{Regexp.escape @directory}/?}, '')
      src_name = File.join(@directory, file)
      dst_name = File.join(@dest_directory, file)

      if ["#{@specs_index}.gz",
          "#{@latest_specs_index}.gz",
          "#{@prerelease_specs_index}.gz"].include?(path)
        res = build_zlib_file(file, src_name, dst_name)
        next unless res
      else
        FileUtils.mv(src_name,
                     dst_name,
                     verbose: verbose,
                     force: true)
      end

      File.utime(newest_mtime, newest_mtime, dst_name)
    end
  end

  def build_zlib_file(file, src_name, dst_name, from_source = false)
    content = Marshal.load(Zlib::GzipReader.open(src_name).read)
    create_zlib_file("#{dst_name}.orig", content)

    return false if @only_origin

    if from_source
      source_content = download_from_source(file)
      source_content = Marshal.load(Zlib::GzipReader
                                      .new(StringIO
                                             .new(source_content)).read)
    else
      source_content = Marshal.load(Zlib::GzipReader.open(dst_name).read)
    end

    return false if source_content.nil?
    new_content = source_content.concat(content).uniq
    create_zlib_file(dst_name, new_content)
  end

  def create_zlib_file(dst_name, content)
    temp_file = Tempfile.new('gemirro')

    Zlib::GzipWriter.open(temp_file.path) do |io|
      io.write(Marshal.dump(content))
    end

    FileUtils.mv(temp_file.path,
                 dst_name,
                 verbose: verbose,
                 force: true)
    Utils.cache.flush_key(File.basename(dst_name))
  end

  def verbose
    @verbose ||= ::Gem.configuration.really_verbose
  end
end

Instance Method Details

#build_indices ⇒ Array

Build indices

Returns:

  • (Array) 


147
148
149
 
# File 'lib/gemirro/indexer.rb', line 147

def build_indices
  build_indicies
end

#build_indicies ⇒ Array

Build indicies

Returns:

  • (Array) 


156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
 
# File 'lib/gemirro/indexer.rb', line 156

def build_indicies
  specs = *map_gems_to_specs(gem_file_list)
  specs.select! { |s| s.class == ::Gem::Specification }
  ::Gem::Specification.dirs = []
  ::Gem::Specification.all = specs

  if ::Gem::VERSION >= '2.5.0'
    build_marshal_gemspecs specs
    build_modern_indices specs if @build_modern
    compress_indices
  else
    build_marshal_gemspecs
    build_modern_indicies if @build_modern
    compress_indicies
  end
end

#build_zlib_file(file, src_name, dst_name, from_source = false) ⇒ Object 



312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
 
# File 'lib/gemirro/indexer.rb', line 312

def build_zlib_file(file, src_name, dst_name, from_source = false)
  content = Marshal.load(Zlib::GzipReader.open(src_name).read)
  create_zlib_file("#{dst_name}.orig", content)

  return false if @only_origin

  if from_source
    source_content = download_from_source(file)
    source_content = Marshal.load(Zlib::GzipReader
                                    .new(StringIO
                                           .new(source_content)).read)
  else
    source_content = Marshal.load(Zlib::GzipReader.open(dst_name).read)
  end

  return false if source_content.nil?
  new_content = source_content.concat(content).uniq
  create_zlib_file(dst_name, new_content)
end

#create_zlib_file(dst_name, content) ⇒ Object 



332
333
334
335
336
337
338
339
340
341
342
343
344
 
# File 'lib/gemirro/indexer.rb', line 332

def create_zlib_file(dst_name, content)
  temp_file = Tempfile.new('gemirro')

  Zlib::GzipWriter.open(temp_file.path) do |io|
    io.write(Marshal.dump(content))
  end

  FileUtils.mv(temp_file.path,
               dst_name,
               verbose: verbose,
               force: true)
  Utils.cache.flush_key(File.basename(dst_name))
end

#download_from_source(file) ⇒ String

Download file from source

Parameters:

  • file (String) —

    File path

Returns:

  • (String) 


134
135
136
137
138
139
140
 
# File 'lib/gemirro/indexer.rb', line 134

def download_from_source(file)
  source_host = Gemirro.configuration.source.host
  Utils.logger.info("Download from source: #{file}")
  resp = Http.get("#{source_host}/#{File.basename(file)}")
  return unless resp.code == 200
  resp.body
end

#install_indices ⇒ Object

Generate indices on the destination directory



83
84
85
 
# File 'lib/gemirro/indexer.rb', line 83

def install_indices
  install_indicies
end

#install_indicies ⇒ Array

Generate indicies on the destination directory

Returns:

  • (Array) 


92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
 
# File 'lib/gemirro/indexer.rb', line 92

def install_indicies
  Utils.logger
       .debug("Downloading index into production dir #{@dest_directory}")

  files = @files
  files.delete @quick_marshal_dir if files.include? @quick_dir

  if files.include?(@quick_marshal_dir) && !files.include?(@quick_dir)
    files.delete @quick_marshal_dir
    dst_name = File.join(@dest_directory, @quick_marshal_dir_base)
    FileUtils.mkdir_p(File.dirname(dst_name), verbose: verbose)
    FileUtils.rm_rf(dst_name, verbose: verbose)
    FileUtils.mv(@quick_marshal_dir, dst_name,
                 verbose: verbose, force: true)
  end

  files.each do |path|
    file = path.sub(%r{^#{Regexp.escape @directory}/?}, '')
    src_name = File.join(@directory, file)
    dst_name = File.join(@dest_directory, file)

    if ["#{@specs_index}.gz",
        "#{@latest_specs_index}.gz",
        "#{@prerelease_specs_index}.gz"].include?(path)
      res = build_zlib_file(file, src_name, dst_name, true)
      next unless res
    else
      source_content = download_from_source(file)
      next if source_content.nil?
      MirrorFile.new(dst_name).write(source_content)
    end

    FileUtils.rm_rf(path)
  end
end

#map_gems_to_specs(gems) ⇒ Array

Map gems file to specs

Parameters:

  • gems (Array) —

    Gems list

Returns:

  • (Array) 


179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
 
# File 'lib/gemirro/indexer.rb', line 179

def map_gems_to_specs(gems)
  gems.map.with_index do |gemfile, index|
    # rubocop:disable Metrics/LineLength
    Utils.logger.info("[#{index + 1}/#{gems.size}]: Processing #{gemfile.split('/')[-1]}")
    # rubocop:enable Metrics/LineLength

    if File.size(gemfile).zero?
      Utils.logger.warn("Skipping zero-length gem: #{gemfile}")
      next
    end

    begin
      spec = if ::Gem::Package.respond_to? :open
               ::Gem::Package
                 .open(File.open(gemfile, 'rb'), 'r', &:metadata)
             else
               ::Gem::Package.new(gemfile).spec
             end

      spec.loaded_from = gemfile

      # HACK: fuck this shit - borks all tests that use pl1
      if File.basename(gemfile, '.gem') != spec.original_name
        exp = spec.full_name
        exp << " (#{spec.original_name})" if
          spec.original_name != spec.full_name
        msg = "Skipping misnamed gem: #{gemfile} should be named #{exp}"
        Utils.logger.warn(msg)
        next
      end

      version = spec.version.version
      unless version =~ /^\d+\.\d+\.\d+.*/
        msg = "Skipping gem #{spec.full_name} - invalid version #{version}"
        Utils.logger.warn(msg)
        next
      end

      if ::Gem::VERSION >= '2.5.0'
        spec.abbreviate
        spec.sanitize
      else
        abbreviate spec
        sanitize spec
      end

      spec
    rescue SignalException
      msg = 'Received signal, exiting'
      Utils.logger.error(msg)
      raise
    rescue StandardError => e
      msg = ["Unable to process #{gemfile}",
             "#{e.message} (#{e.class})",
             "\t#{e.backtrace.join "\n\t"}"].join("\n")
      Utils.logger.debug(msg)
    end
  end.compact
end

#update_index ⇒ Object 



239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
 
# File 'lib/gemirro/indexer.rb', line 239

def update_index
  make_temp_directories

  specs_mtime = File.stat(@dest_specs_index).mtime
  newest_mtime = Time.at(0)

  @updated_gems = gem_file_list.select do |gem|
    gem_mtime = File.stat(gem).mtime
    newest_mtime = gem_mtime if gem_mtime > newest_mtime
    gem_mtime > specs_mtime
  end

  if @updated_gems.empty?
    Utils.logger.info('No new gems')
    terminate_interaction(0)
  end

  specs = map_gems_to_specs(@updated_gems)
  prerelease, released = specs.partition { |s| s.version.prerelease? }

  ::Gem::Specification.dirs = []
  ::Gem::Specification.all = *specs
  files = if ::Gem::VERSION >= '2.5.0'
            build_marshal_gemspecs specs
          else
            build_marshal_gemspecs
          end

  ::Gem.time('Updated indexes') do
    update_specs_index(released, @dest_specs_index, @specs_index)
    update_specs_index(released,
                       @dest_latest_specs_index,
                       @latest_specs_index)
    update_specs_index(prerelease,
                       @dest_prerelease_specs_index,
                       @prerelease_specs_index)
  end

  if ::Gem::VERSION >= '2.5.0'
    compress_indices
  else
    compress_indicies
  end

  Utils.logger.info("Updating production dir #{@dest_directory}") if verbose
  files << @specs_index
  files << "#{@specs_index}.gz"
  files << @latest_specs_index
  files << "#{@latest_specs_index}.gz"
  files << @prerelease_specs_index
  files << "#{@prerelease_specs_index}.gz"

  files.each do |path|
    file = path.sub(%r{^#{Regexp.escape @directory}/?}, '')
    src_name = File.join(@directory, file)
    dst_name = File.join(@dest_directory, file)

    if ["#{@specs_index}.gz",
        "#{@latest_specs_index}.gz",
        "#{@prerelease_specs_index}.gz"].include?(path)
      res = build_zlib_file(file, src_name, dst_name)
      next unless res
    else
      FileUtils.mv(src_name,
                   dst_name,
                   verbose: verbose,
                   force: true)
    end

    File.utime(newest_mtime, newest_mtime, dst_name)
  end
end

#verbose ⇒ Object 



346
347
348
 
# File 'lib/gemirro/indexer.rb', line 346

def verbose
  @verbose ||= ::Gem.configuration.really_verbose
end