Class: Gemirro::Indexer

Inherits:
Gem::Indexer
  • Object
show all
Defined in:
lib/gemirro/indexer.rb

Overview

The Indexer class is responsible for downloading useful file directly on the source host, such as specs-..gz, marshal information, etc…

Instance Attribute Summary collapse

Instance Method Summary collapse

Instance Attribute Details

#dest_directory ⇒ String

Returns:

  • (String) 


19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
 
# File 'lib/gemirro/indexer.rb', line 19

class Indexer < ::Gem::Indexer
  attr_accessor(:files,
                :quick_marshal_dir,
                :directory,
                :dest_directory,
                :only_origin)

  ##
  # Generate indicies on the destination directory
  #
  # @return [Array]
  #
  def install_indicies
    verbose = ::Gem.configuration.really_verbose
    say "Downloading index into production dir #{@dest_directory}" if verbose

    files = @files
    files.delete @quick_marshal_dir if files.include? @quick_dir

    if files.include?(@quick_marshal_dir) && !files.include?(@quick_dir)
      files.delete @quick_marshal_dir
      dst_name = File.join(@dest_directory, @quick_marshal_dir_base)
      FileUtils.mkdir_p(File.dirname(dst_name), verbose: verbose)
      FileUtils.rm_rf(dst_name, verbose: verbose)
      FileUtils.mv(@quick_marshal_dir, dst_name,
                   verbose: verbose, force: true)
    end

    files.each do |path|
      file = path.sub(%r{^#{Regexp.escape @directory}/?}, '')
      dst_name = File.join @dest_directory, file

      if ["#{@specs_index}.gz",
          "#{@latest_specs_index}.gz",
          "#{@prerelease_specs_index}.gz"].include?(path)

        content = Marshal.load(Zlib::GzipReader.open(path).read)
        Zlib::GzipWriter.open("#{dst_name}.orig") do |io|
          io.write(Marshal.dump(content))
        end

        unless @only_origin
          source_content = download_from_source(file)
          next if source_content.nil?
          source_content = Marshal.load(Zlib::GzipReader
                                          .new(StringIO
                                                 .new(source_content)).read)
          new_content = source_content.concat(content).uniq

          Zlib::GzipWriter.open(dst_name) do |io|
            io.write(Marshal.dump(new_content))
          end
        end
      else
        source_content = download_from_source(file)
        next if source_content.nil?
        MirrorFile.new(dst_name).write(source_content)
      end

      FileUtils.rm_rf(path)
    end
  end

  def download_from_source(file)
    source_host = Gemirro.configuration.source.host
    resp = Http.get("#{source_host}/#{File.basename(file)}")
    return unless resp.code == 200
    resp.body
  end

  def update_gemspecs
    make_temp_directories

    specs_mtime = File.stat(@dest_specs_index).mtime
    newest_mtime = Time.at 0

    updated_gems = gem_file_list.select do |gem|
      gem_mtime = File.stat(gem).mtime
      newest_mtime = gem_mtime if gem_mtime > newest_mtime
      gem_mtime >= specs_mtime
    end

    terminate_interaction(0) if updated_gems.empty?

    specs = map_gems_to_specs updated_gems
    ::Gem::Specification.dirs = []
    ::Gem::Specification.add_specs(*specs)
    files = build_marshal_gemspecs

    files.each do |path|
      file = path.sub(%r{^#{Regexp.escape @directory}/?}, '')
      src_name = File.join @directory, file
      dst_name = File.join @dest_directory, file
      FileUtils.mv(src_name, dst_name)
      File.utime newest_mtime, newest_mtime, dst_name
    end
  end
end

#directory ⇒ String

Returns:

  • (String) 


19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
 
# File 'lib/gemirro/indexer.rb', line 19

class Indexer < ::Gem::Indexer
  attr_accessor(:files,
                :quick_marshal_dir,
                :directory,
                :dest_directory,
                :only_origin)

  ##
  # Generate indicies on the destination directory
  #
  # @return [Array]
  #
  def install_indicies
    verbose = ::Gem.configuration.really_verbose
    say "Downloading index into production dir #{@dest_directory}" if verbose

    files = @files
    files.delete @quick_marshal_dir if files.include? @quick_dir

    if files.include?(@quick_marshal_dir) && !files.include?(@quick_dir)
      files.delete @quick_marshal_dir
      dst_name = File.join(@dest_directory, @quick_marshal_dir_base)
      FileUtils.mkdir_p(File.dirname(dst_name), verbose: verbose)
      FileUtils.rm_rf(dst_name, verbose: verbose)
      FileUtils.mv(@quick_marshal_dir, dst_name,
                   verbose: verbose, force: true)
    end

    files.each do |path|
      file = path.sub(%r{^#{Regexp.escape @directory}/?}, '')
      dst_name = File.join @dest_directory, file

      if ["#{@specs_index}.gz",
          "#{@latest_specs_index}.gz",
          "#{@prerelease_specs_index}.gz"].include?(path)

        content = Marshal.load(Zlib::GzipReader.open(path).read)
        Zlib::GzipWriter.open("#{dst_name}.orig") do |io|
          io.write(Marshal.dump(content))
        end

        unless @only_origin
          source_content = download_from_source(file)
          next if source_content.nil?
          source_content = Marshal.load(Zlib::GzipReader
                                          .new(StringIO
                                                 .new(source_content)).read)
          new_content = source_content.concat(content).uniq

          Zlib::GzipWriter.open(dst_name) do |io|
            io.write(Marshal.dump(new_content))
          end
        end
      else
        source_content = download_from_source(file)
        next if source_content.nil?
        MirrorFile.new(dst_name).write(source_content)
      end

      FileUtils.rm_rf(path)
    end
  end

  def download_from_source(file)
    source_host = Gemirro.configuration.source.host
    resp = Http.get("#{source_host}/#{File.basename(file)}")
    return unless resp.code == 200
    resp.body
  end

  def update_gemspecs
    make_temp_directories

    specs_mtime = File.stat(@dest_specs_index).mtime
    newest_mtime = Time.at 0

    updated_gems = gem_file_list.select do |gem|
      gem_mtime = File.stat(gem).mtime
      newest_mtime = gem_mtime if gem_mtime > newest_mtime
      gem_mtime >= specs_mtime
    end

    terminate_interaction(0) if updated_gems.empty?

    specs = map_gems_to_specs updated_gems
    ::Gem::Specification.dirs = []
    ::Gem::Specification.add_specs(*specs)
    files = build_marshal_gemspecs

    files.each do |path|
      file = path.sub(%r{^#{Regexp.escape @directory}/?}, '')
      src_name = File.join @directory, file
      dst_name = File.join @dest_directory, file
      FileUtils.mv(src_name, dst_name)
      File.utime newest_mtime, newest_mtime, dst_name
    end
  end
end

#files ⇒ Array

Returns:

  • (Array) 


19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
 
# File 'lib/gemirro/indexer.rb', line 19

class Indexer < ::Gem::Indexer
  attr_accessor(:files,
                :quick_marshal_dir,
                :directory,
                :dest_directory,
                :only_origin)

  ##
  # Generate indicies on the destination directory
  #
  # @return [Array]
  #
  def install_indicies
    verbose = ::Gem.configuration.really_verbose
    say "Downloading index into production dir #{@dest_directory}" if verbose

    files = @files
    files.delete @quick_marshal_dir if files.include? @quick_dir

    if files.include?(@quick_marshal_dir) && !files.include?(@quick_dir)
      files.delete @quick_marshal_dir
      dst_name = File.join(@dest_directory, @quick_marshal_dir_base)
      FileUtils.mkdir_p(File.dirname(dst_name), verbose: verbose)
      FileUtils.rm_rf(dst_name, verbose: verbose)
      FileUtils.mv(@quick_marshal_dir, dst_name,
                   verbose: verbose, force: true)
    end

    files.each do |path|
      file = path.sub(%r{^#{Regexp.escape @directory}/?}, '')
      dst_name = File.join @dest_directory, file

      if ["#{@specs_index}.gz",
          "#{@latest_specs_index}.gz",
          "#{@prerelease_specs_index}.gz"].include?(path)

        content = Marshal.load(Zlib::GzipReader.open(path).read)
        Zlib::GzipWriter.open("#{dst_name}.orig") do |io|
          io.write(Marshal.dump(content))
        end

        unless @only_origin
          source_content = download_from_source(file)
          next if source_content.nil?
          source_content = Marshal.load(Zlib::GzipReader
                                          .new(StringIO
                                                 .new(source_content)).read)
          new_content = source_content.concat(content).uniq

          Zlib::GzipWriter.open(dst_name) do |io|
            io.write(Marshal.dump(new_content))
          end
        end
      else
        source_content = download_from_source(file)
        next if source_content.nil?
        MirrorFile.new(dst_name).write(source_content)
      end

      FileUtils.rm_rf(path)
    end
  end

  def download_from_source(file)
    source_host = Gemirro.configuration.source.host
    resp = Http.get("#{source_host}/#{File.basename(file)}")
    return unless resp.code == 200
    resp.body
  end

  def update_gemspecs
    make_temp_directories

    specs_mtime = File.stat(@dest_specs_index).mtime
    newest_mtime = Time.at 0

    updated_gems = gem_file_list.select do |gem|
      gem_mtime = File.stat(gem).mtime
      newest_mtime = gem_mtime if gem_mtime > newest_mtime
      gem_mtime >= specs_mtime
    end

    terminate_interaction(0) if updated_gems.empty?

    specs = map_gems_to_specs updated_gems
    ::Gem::Specification.dirs = []
    ::Gem::Specification.add_specs(*specs)
    files = build_marshal_gemspecs

    files.each do |path|
      file = path.sub(%r{^#{Regexp.escape @directory}/?}, '')
      src_name = File.join @directory, file
      dst_name = File.join @dest_directory, file
      FileUtils.mv(src_name, dst_name)
      File.utime newest_mtime, newest_mtime, dst_name
    end
  end
end

#only_origin ⇒ Boolean

Returns:

  • (Boolean) 


19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
 
# File 'lib/gemirro/indexer.rb', line 19

class Indexer < ::Gem::Indexer
  attr_accessor(:files,
                :quick_marshal_dir,
                :directory,
                :dest_directory,
                :only_origin)

  ##
  # Generate indicies on the destination directory
  #
  # @return [Array]
  #
  def install_indicies
    verbose = ::Gem.configuration.really_verbose
    say "Downloading index into production dir #{@dest_directory}" if verbose

    files = @files
    files.delete @quick_marshal_dir if files.include? @quick_dir

    if files.include?(@quick_marshal_dir) && !files.include?(@quick_dir)
      files.delete @quick_marshal_dir
      dst_name = File.join(@dest_directory, @quick_marshal_dir_base)
      FileUtils.mkdir_p(File.dirname(dst_name), verbose: verbose)
      FileUtils.rm_rf(dst_name, verbose: verbose)
      FileUtils.mv(@quick_marshal_dir, dst_name,
                   verbose: verbose, force: true)
    end

    files.each do |path|
      file = path.sub(%r{^#{Regexp.escape @directory}/?}, '')
      dst_name = File.join @dest_directory, file

      if ["#{@specs_index}.gz",
          "#{@latest_specs_index}.gz",
          "#{@prerelease_specs_index}.gz"].include?(path)

        content = Marshal.load(Zlib::GzipReader.open(path).read)
        Zlib::GzipWriter.open("#{dst_name}.orig") do |io|
          io.write(Marshal.dump(content))
        end

        unless @only_origin
          source_content = download_from_source(file)
          next if source_content.nil?
          source_content = Marshal.load(Zlib::GzipReader
                                          .new(StringIO
                                                 .new(source_content)).read)
          new_content = source_content.concat(content).uniq

          Zlib::GzipWriter.open(dst_name) do |io|
            io.write(Marshal.dump(new_content))
          end
        end
      else
        source_content = download_from_source(file)
        next if source_content.nil?
        MirrorFile.new(dst_name).write(source_content)
      end

      FileUtils.rm_rf(path)
    end
  end

  def download_from_source(file)
    source_host = Gemirro.configuration.source.host
    resp = Http.get("#{source_host}/#{File.basename(file)}")
    return unless resp.code == 200
    resp.body
  end

  def update_gemspecs
    make_temp_directories

    specs_mtime = File.stat(@dest_specs_index).mtime
    newest_mtime = Time.at 0

    updated_gems = gem_file_list.select do |gem|
      gem_mtime = File.stat(gem).mtime
      newest_mtime = gem_mtime if gem_mtime > newest_mtime
      gem_mtime >= specs_mtime
    end

    terminate_interaction(0) if updated_gems.empty?

    specs = map_gems_to_specs updated_gems
    ::Gem::Specification.dirs = []
    ::Gem::Specification.add_specs(*specs)
    files = build_marshal_gemspecs

    files.each do |path|
      file = path.sub(%r{^#{Regexp.escape @directory}/?}, '')
      src_name = File.join @directory, file
      dst_name = File.join @dest_directory, file
      FileUtils.mv(src_name, dst_name)
      File.utime newest_mtime, newest_mtime, dst_name
    end
  end
end

#quick_marshal_dir ⇒ String

Returns:

  • (String) 


19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
 
# File 'lib/gemirro/indexer.rb', line 19

class Indexer < ::Gem::Indexer
  attr_accessor(:files,
                :quick_marshal_dir,
                :directory,
                :dest_directory,
                :only_origin)

  ##
  # Generate indicies on the destination directory
  #
  # @return [Array]
  #
  def install_indicies
    verbose = ::Gem.configuration.really_verbose
    say "Downloading index into production dir #{@dest_directory}" if verbose

    files = @files
    files.delete @quick_marshal_dir if files.include? @quick_dir

    if files.include?(@quick_marshal_dir) && !files.include?(@quick_dir)
      files.delete @quick_marshal_dir
      dst_name = File.join(@dest_directory, @quick_marshal_dir_base)
      FileUtils.mkdir_p(File.dirname(dst_name), verbose: verbose)
      FileUtils.rm_rf(dst_name, verbose: verbose)
      FileUtils.mv(@quick_marshal_dir, dst_name,
                   verbose: verbose, force: true)
    end

    files.each do |path|
      file = path.sub(%r{^#{Regexp.escape @directory}/?}, '')
      dst_name = File.join @dest_directory, file

      if ["#{@specs_index}.gz",
          "#{@latest_specs_index}.gz",
          "#{@prerelease_specs_index}.gz"].include?(path)

        content = Marshal.load(Zlib::GzipReader.open(path).read)
        Zlib::GzipWriter.open("#{dst_name}.orig") do |io|
          io.write(Marshal.dump(content))
        end

        unless @only_origin
          source_content = download_from_source(file)
          next if source_content.nil?
          source_content = Marshal.load(Zlib::GzipReader
                                          .new(StringIO
                                                 .new(source_content)).read)
          new_content = source_content.concat(content).uniq

          Zlib::GzipWriter.open(dst_name) do |io|
            io.write(Marshal.dump(new_content))
          end
        end
      else
        source_content = download_from_source(file)
        next if source_content.nil?
        MirrorFile.new(dst_name).write(source_content)
      end

      FileUtils.rm_rf(path)
    end
  end

  def download_from_source(file)
    source_host = Gemirro.configuration.source.host
    resp = Http.get("#{source_host}/#{File.basename(file)}")
    return unless resp.code == 200
    resp.body
  end

  def update_gemspecs
    make_temp_directories

    specs_mtime = File.stat(@dest_specs_index).mtime
    newest_mtime = Time.at 0

    updated_gems = gem_file_list.select do |gem|
      gem_mtime = File.stat(gem).mtime
      newest_mtime = gem_mtime if gem_mtime > newest_mtime
      gem_mtime >= specs_mtime
    end

    terminate_interaction(0) if updated_gems.empty?

    specs = map_gems_to_specs updated_gems
    ::Gem::Specification.dirs = []
    ::Gem::Specification.add_specs(*specs)
    files = build_marshal_gemspecs

    files.each do |path|
      file = path.sub(%r{^#{Regexp.escape @directory}/?}, '')
      src_name = File.join @directory, file
      dst_name = File.join @dest_directory, file
      FileUtils.mv(src_name, dst_name)
      File.utime newest_mtime, newest_mtime, dst_name
    end
  end
end

Instance Method Details

#download_from_source(file) ⇒ Object 



82
83
84
85
86
87
 
# File 'lib/gemirro/indexer.rb', line 82

def download_from_source(file)
  source_host = Gemirro.configuration.source.host
  resp = Http.get("#{source_host}/#{File.basename(file)}")
  return unless resp.code == 200
  resp.body
end

#install_indicies ⇒ Array

Generate indicies on the destination directory

Returns:

  • (Array) 


31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
 
# File 'lib/gemirro/indexer.rb', line 31

def install_indicies
  verbose = ::Gem.configuration.really_verbose
  say "Downloading index into production dir #{@dest_directory}" if verbose

  files = @files
  files.delete @quick_marshal_dir if files.include? @quick_dir

  if files.include?(@quick_marshal_dir) && !files.include?(@quick_dir)
    files.delete @quick_marshal_dir
    dst_name = File.join(@dest_directory, @quick_marshal_dir_base)
    FileUtils.mkdir_p(File.dirname(dst_name), verbose: verbose)
    FileUtils.rm_rf(dst_name, verbose: verbose)
    FileUtils.mv(@quick_marshal_dir, dst_name,
                 verbose: verbose, force: true)
  end

  files.each do |path|
    file = path.sub(%r{^#{Regexp.escape @directory}/?}, '')
    dst_name = File.join @dest_directory, file

    if ["#{@specs_index}.gz",
        "#{@latest_specs_index}.gz",
        "#{@prerelease_specs_index}.gz"].include?(path)

      content = Marshal.load(Zlib::GzipReader.open(path).read)
      Zlib::GzipWriter.open("#{dst_name}.orig") do |io|
        io.write(Marshal.dump(content))
      end

      unless @only_origin
        source_content = download_from_source(file)
        next if source_content.nil?
        source_content = Marshal.load(Zlib::GzipReader
                                        .new(StringIO
                                               .new(source_content)).read)
        new_content = source_content.concat(content).uniq

        Zlib::GzipWriter.open(dst_name) do |io|
          io.write(Marshal.dump(new_content))
        end
      end
    else
      source_content = download_from_source(file)
      next if source_content.nil?
      MirrorFile.new(dst_name).write(source_content)
    end

    FileUtils.rm_rf(path)
  end
end

#update_gemspecs ⇒ Object 



89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
 
# File 'lib/gemirro/indexer.rb', line 89

def update_gemspecs
  make_temp_directories

  specs_mtime = File.stat(@dest_specs_index).mtime
  newest_mtime = Time.at 0

  updated_gems = gem_file_list.select do |gem|
    gem_mtime = File.stat(gem).mtime
    newest_mtime = gem_mtime if gem_mtime > newest_mtime
    gem_mtime >= specs_mtime
  end

  terminate_interaction(0) if updated_gems.empty?

  specs = map_gems_to_specs updated_gems
  ::Gem::Specification.dirs = []
  ::Gem::Specification.add_specs(*specs)
  files = build_marshal_gemspecs

  files.each do |path|
    file = path.sub(%r{^#{Regexp.escape @directory}/?}, '')
    src_name = File.join @directory, file
    dst_name = File.join @dest_directory, file
    FileUtils.mv(src_name, dst_name)
    File.utime newest_mtime, newest_mtime, dst_name
  end
end