Class: GemMirror::GemsFetcher

Inherits:
Object
  • Object
show all
Defined in:
lib/gem_mirror/gems_fetcher.rb

Overview

The GemsFetcher class is responsible for downloading Gems from an external source as well as downloading all the associated dependencies.

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(source, versions_file) ⇒ GemsFetcher

Returns a new instance of GemsFetcher.

Parameters:



20
21
22
23
 
# File 'lib/gem_mirror/gems_fetcher.rb', line 20

def initialize(source, versions_file)
  @source        = source
  @versions_file = versions_file
end

Instance Attribute Details

#source ⇒ Source (readonly)

Returns:



13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
 
# File 'lib/gem_mirror/gems_fetcher.rb', line 13

class GemsFetcher
  attr_reader :source, :versions_file

  ##
  # @param [Source] source
  # @param [GemMirror::VersionsFile] versions_file
  #
  def initialize(source, versions_file)
    @source        = source
    @versions_file = versions_file
  end

  ##
  # Fetches the Gems and all associated dependencies.
  #
  def fetch
    source.gems.each do |gem|
      versions_for(gem).each do |version|
        filename = gem.filename(version)
        begin
          satisfied = gem.requirement.satisfied_by?(version)
        rescue StandardError
          logger.debug("Error determining is requirement satisfied for #{filename}")
        end
        name = gem.name

        if gem_exists?(filename) || ignore_gem?(name, version) || !satisfied
          logger.debug("Skipping #{filename}")
          next
        end

        # Prevent circular dependencies from messing things up.
        configuration.ignore_gem(gem.name, version)

        spec = fetch_specification(gem, version)

        next unless spec

        spec = load_specification(spec)
        deps = dependencies_for(spec)

        unless deps.empty?
          logger.info("Fetching dependencies for #{filename}")

          fetch_dependencies(deps)
        end

        logger.info("Fetching #{filename}")

        gemfile = fetch_gem(gem, version)

        configuration.mirror_directory.add_file(filename, gemfile) if gemfile
      end
    end
  end

  ##
  # Returns an Array containing the versions that should be fetched for a
  # Gem.
  #
  # @param [GemMirror::Gem] gem
  # @return [Array]
  #
  def versions_for(gem)
    available       = versions_file.versions_for(gem.name)
    versions        = gem.version? ? [gem.version] : available
    available_names = available.map(&:to_s)

    # Get rid of invalid versions. Due to Gem::Version having a custom ==
    # method, which treats "3.4" the same as "3.4.0" we'll have to compare
    # the versions as String instances.
    versions = versions.select do |version|
      available_names.include?(version.to_s)
    end

    versions = [available.last] if versions.empty?

    versions
  end

  ##
  # Tries to download the specification for a Gem and version. This method
  # returns the raw inflated data instead of an instance of
  # `Gem::Specification`.
  #
  # @param [GemMirror::Gem] gem
  # @param [Gem::Version] version
  # @return [String]
  #
  def fetch_specification(gem, version)
    specification = nil
    filename      = gem.filename(version)

    begin
      specification = source.fetch_specification(gem.name, version)
    rescue StandardError => e
      logger.error("Failed to retrieve #{filename}: #{e.message}")
      logger.debug("Adding #{filename} to the list of ignored Gems")

      configuration.ignore_gem(gem.name, version)
    end

    specification
  end

  ##
  # Tries to download the Gemfile for the specified Gem and version.
  #
  # @param [GemMirror::Gem] gem
  # @param [Gem::Version] version
  # @return [String]
  #
  def fetch_gem(gem, version)
    gemfile  = nil
    filename = gem.filename(version)

    begin
      gemfile = source.fetch_gem(gem.name, version)
    rescue StandardError => e
      logger.error("Failed to retrieve #{filename}: #{e.message}")
      logger.debug("Adding #{filename} to the list of ignored Gems")

      configuration.ignore_gem(gem.name, version)
    end

    gemfile
  end

  ##
  # Reads the inflated data of a Gemspec and returns the loaded specification
  # instance.
  #
  # @param [String] raw_spec
  # @return [Gem::Specification]
  #
  def load_specification(raw_spec)
    stream  = Zlib::Inflate.new
    content = stream.inflate(raw_spec)

    stream.finish
    stream.close

    Marshal.load(content)
  end

  ##
  # Fetches the Gem files for the specified dependencies.
  #
  # @param [Array] deps
  #
  def fetch_dependencies(deps)
    self.class.new(source.updated(deps), versions_file).fetch
  end

  ##
  # Returns an Array containing all the dependencies of a given Gem
  # specification.
  #
  # @param [Gem::Specification] spec
  # @return [Array]
  #
  def dependencies_for(spec)
    possible_dependencies = if configuration.development
                              spec.dependencies
                            else
                              spec.runtime_dependencies
                            end

    dependencies = filter_dependencies(possible_dependencies)

    assign_gem_versions(dependencies)
  end

  ##
  # Filters a list of dependencies based on whether or not they are ignored.
  #
  # @param [Array] possible_dependencies
  # @return [Array]
  #
  def filter_dependencies(possible_dependencies)
    dependencies = []

    possible_dependencies.each do |dependency|
      gem = Gem.new(dependency.name, dependency.requirement)

      dependencies << gem unless ignore_gem?(gem.name, gem.version)
    end

    dependencies
  end

  ##
  # Processes a list of Gems and sets their versions to the latest one
  # available in case no specific version is given.
  #
  # @param [Array] gems
  # @return [Array]
  #
  def assign_gem_versions(gems)
    gems.map do |gem|
      unless gem.version?
        latest = versions_file.versions_for(gem.name).last
        gem    = Gem.new(gem.name, latest.to_s) if latest
      end

      gem
    end
  end

  ##
  # @see GemMirror::Configuration#logger
  # @return [Logger]
  #
  def logger
    configuration.logger
  end

  ##
  # @see GemMirror.configuration
  #
  def configuration
    GemMirror.configuration
  end

  ##
  # Checks if a given Gem has already been downloaded.
  #
  # @param [String] filename
  # @return [TrueClass|FalseClass]
  #
  def gem_exists?(filename)
    configuration.mirror_directory.file_exists?(filename)
  end

  ##
  # @see GemMirror::Configuration#ignore_gem?
  #
  def ignore_gem?(*args)
    configuration.ignore_gem?(*args)
  end
end

#versions_file ⇒ GemMirror::VersionsFile (readonly)

Returns:



13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
 
# File 'lib/gem_mirror/gems_fetcher.rb', line 13

class GemsFetcher
  attr_reader :source, :versions_file

  ##
  # @param [Source] source
  # @param [GemMirror::VersionsFile] versions_file
  #
  def initialize(source, versions_file)
    @source        = source
    @versions_file = versions_file
  end

  ##
  # Fetches the Gems and all associated dependencies.
  #
  def fetch
    source.gems.each do |gem|
      versions_for(gem).each do |version|
        filename = gem.filename(version)
        begin
          satisfied = gem.requirement.satisfied_by?(version)
        rescue StandardError
          logger.debug("Error determining is requirement satisfied for #{filename}")
        end
        name = gem.name

        if gem_exists?(filename) || ignore_gem?(name, version) || !satisfied
          logger.debug("Skipping #{filename}")
          next
        end

        # Prevent circular dependencies from messing things up.
        configuration.ignore_gem(gem.name, version)

        spec = fetch_specification(gem, version)

        next unless spec

        spec = load_specification(spec)
        deps = dependencies_for(spec)

        unless deps.empty?
          logger.info("Fetching dependencies for #{filename}")

          fetch_dependencies(deps)
        end

        logger.info("Fetching #{filename}")

        gemfile = fetch_gem(gem, version)

        configuration.mirror_directory.add_file(filename, gemfile) if gemfile
      end
    end
  end

  ##
  # Returns an Array containing the versions that should be fetched for a
  # Gem.
  #
  # @param [GemMirror::Gem] gem
  # @return [Array]
  #
  def versions_for(gem)
    available       = versions_file.versions_for(gem.name)
    versions        = gem.version? ? [gem.version] : available
    available_names = available.map(&:to_s)

    # Get rid of invalid versions. Due to Gem::Version having a custom ==
    # method, which treats "3.4" the same as "3.4.0" we'll have to compare
    # the versions as String instances.
    versions = versions.select do |version|
      available_names.include?(version.to_s)
    end

    versions = [available.last] if versions.empty?

    versions
  end

  ##
  # Tries to download the specification for a Gem and version. This method
  # returns the raw inflated data instead of an instance of
  # `Gem::Specification`.
  #
  # @param [GemMirror::Gem] gem
  # @param [Gem::Version] version
  # @return [String]
  #
  def fetch_specification(gem, version)
    specification = nil
    filename      = gem.filename(version)

    begin
      specification = source.fetch_specification(gem.name, version)
    rescue StandardError => e
      logger.error("Failed to retrieve #{filename}: #{e.message}")
      logger.debug("Adding #{filename} to the list of ignored Gems")

      configuration.ignore_gem(gem.name, version)
    end

    specification
  end

  ##
  # Tries to download the Gemfile for the specified Gem and version.
  #
  # @param [GemMirror::Gem] gem
  # @param [Gem::Version] version
  # @return [String]
  #
  def fetch_gem(gem, version)
    gemfile  = nil
    filename = gem.filename(version)

    begin
      gemfile = source.fetch_gem(gem.name, version)
    rescue StandardError => e
      logger.error("Failed to retrieve #{filename}: #{e.message}")
      logger.debug("Adding #{filename} to the list of ignored Gems")

      configuration.ignore_gem(gem.name, version)
    end

    gemfile
  end

  ##
  # Reads the inflated data of a Gemspec and returns the loaded specification
  # instance.
  #
  # @param [String] raw_spec
  # @return [Gem::Specification]
  #
  def load_specification(raw_spec)
    stream  = Zlib::Inflate.new
    content = stream.inflate(raw_spec)

    stream.finish
    stream.close

    Marshal.load(content)
  end

  ##
  # Fetches the Gem files for the specified dependencies.
  #
  # @param [Array] deps
  #
  def fetch_dependencies(deps)
    self.class.new(source.updated(deps), versions_file).fetch
  end

  ##
  # Returns an Array containing all the dependencies of a given Gem
  # specification.
  #
  # @param [Gem::Specification] spec
  # @return [Array]
  #
  def dependencies_for(spec)
    possible_dependencies = if configuration.development
                              spec.dependencies
                            else
                              spec.runtime_dependencies
                            end

    dependencies = filter_dependencies(possible_dependencies)

    assign_gem_versions(dependencies)
  end

  ##
  # Filters a list of dependencies based on whether or not they are ignored.
  #
  # @param [Array] possible_dependencies
  # @return [Array]
  #
  def filter_dependencies(possible_dependencies)
    dependencies = []

    possible_dependencies.each do |dependency|
      gem = Gem.new(dependency.name, dependency.requirement)

      dependencies << gem unless ignore_gem?(gem.name, gem.version)
    end

    dependencies
  end

  ##
  # Processes a list of Gems and sets their versions to the latest one
  # available in case no specific version is given.
  #
  # @param [Array] gems
  # @return [Array]
  #
  def assign_gem_versions(gems)
    gems.map do |gem|
      unless gem.version?
        latest = versions_file.versions_for(gem.name).last
        gem    = Gem.new(gem.name, latest.to_s) if latest
      end

      gem
    end
  end

  ##
  # @see GemMirror::Configuration#logger
  # @return [Logger]
  #
  def logger
    configuration.logger
  end

  ##
  # @see GemMirror.configuration
  #
  def configuration
    GemMirror.configuration
  end

  ##
  # Checks if a given Gem has already been downloaded.
  #
  # @param [String] filename
  # @return [TrueClass|FalseClass]
  #
  def gem_exists?(filename)
    configuration.mirror_directory.file_exists?(filename)
  end

  ##
  # @see GemMirror::Configuration#ignore_gem?
  #
  def ignore_gem?(*args)
    configuration.ignore_gem?(*args)
  end
end

Instance Method Details

#assign_gem_versions(gems) ⇒ Array

Processes a list of Gems and sets their versions to the latest one available in case no specific version is given.

Parameters:

  • gems (Array)

Returns:

  • (Array) 


211
212
213
214
215
216
217
218
219
220
 
# File 'lib/gem_mirror/gems_fetcher.rb', line 211

def assign_gem_versions(gems)
  gems.map do |gem|
    unless gem.version?
      latest = versions_file.versions_for(gem.name).last
      gem    = Gem.new(gem.name, latest.to_s) if latest
    end

    gem
  end
end

#configuration ⇒ Object

See Also:



233
234
235
 
# File 'lib/gem_mirror/gems_fetcher.rb', line 233

def configuration
  GemMirror.configuration
end

#dependencies_for(spec) ⇒ Array

Returns an Array containing all the dependencies of a given Gem specification.

Parameters:

  • spec (Gem::Specification)

Returns:

  • (Array) 


174
175
176
177
178
179
180
181
182
183
184
 
# File 'lib/gem_mirror/gems_fetcher.rb', line 174

def dependencies_for(spec)
  possible_dependencies = if configuration.development
                            spec.dependencies
                          else
                            spec.runtime_dependencies
                          end

  dependencies = filter_dependencies(possible_dependencies)

  assign_gem_versions(dependencies)
end

#fetch ⇒ Object

Fetches the Gems and all associated dependencies.



28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
 
# File 'lib/gem_mirror/gems_fetcher.rb', line 28

def fetch
  source.gems.each do |gem|
    versions_for(gem).each do |version|
      filename = gem.filename(version)
      begin
        satisfied = gem.requirement.satisfied_by?(version)
      rescue StandardError
        logger.debug("Error determining is requirement satisfied for #{filename}")
      end
      name = gem.name

      if gem_exists?(filename) || ignore_gem?(name, version) || !satisfied
        logger.debug("Skipping #{filename}")
        next
      end

      # Prevent circular dependencies from messing things up.
      configuration.ignore_gem(gem.name, version)

      spec = fetch_specification(gem, version)

      next unless spec

      spec = load_specification(spec)
      deps = dependencies_for(spec)

      unless deps.empty?
        logger.info("Fetching dependencies for #{filename}")

        fetch_dependencies(deps)
      end

      logger.info("Fetching #{filename}")

      gemfile = fetch_gem(gem, version)

      configuration.mirror_directory.add_file(filename, gemfile) if gemfile
    end
  end
end

#fetch_dependencies(deps) ⇒ Object

Fetches the Gem files for the specified dependencies.

Parameters:

  • deps (Array) 


163
164
165
 
# File 'lib/gem_mirror/gems_fetcher.rb', line 163

def fetch_dependencies(deps)
  self.class.new(source.updated(deps), versions_file).fetch
end

#fetch_gem(gem, version) ⇒ String

Tries to download the Gemfile for the specified Gem and version.

Parameters:

Returns:

  • (String) 


125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
 
# File 'lib/gem_mirror/gems_fetcher.rb', line 125

def fetch_gem(gem, version)
  gemfile  = nil
  filename = gem.filename(version)

  begin
    gemfile = source.fetch_gem(gem.name, version)
  rescue StandardError => e
    logger.error("Failed to retrieve #{filename}: #{e.message}")
    logger.debug("Adding #{filename} to the list of ignored Gems")

    configuration.ignore_gem(gem.name, version)
  end

  gemfile
end

#fetch_specification(gem, version) ⇒ String

Tries to download the specification for a Gem and version. This method returns the raw inflated data instead of an instance of ‘Gem::Specification`.

Parameters:

Returns:

  • (String) 


102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
 
# File 'lib/gem_mirror/gems_fetcher.rb', line 102

def fetch_specification(gem, version)
  specification = nil
  filename      = gem.filename(version)

  begin
    specification = source.fetch_specification(gem.name, version)
  rescue StandardError => e
    logger.error("Failed to retrieve #{filename}: #{e.message}")
    logger.debug("Adding #{filename} to the list of ignored Gems")

    configuration.ignore_gem(gem.name, version)
  end

  specification
end

#filter_dependencies(possible_dependencies) ⇒ Array

Filters a list of dependencies based on whether or not they are ignored.

Parameters:

  • possible_dependencies (Array)

Returns:

  • (Array) 


192
193
194
195
196
197
198
199
200
201
202
 
# File 'lib/gem_mirror/gems_fetcher.rb', line 192

def filter_dependencies(possible_dependencies)
  dependencies = []

  possible_dependencies.each do |dependency|
    gem = Gem.new(dependency.name, dependency.requirement)

    dependencies << gem unless ignore_gem?(gem.name, gem.version)
  end

  dependencies
end

#gem_exists?(filename) ⇒ TrueClass|FalseClass

Checks if a given Gem has already been downloaded.

Parameters:

  • filename (String)

Returns:

  • (TrueClass|FalseClass) 


243
244
245
 
# File 'lib/gem_mirror/gems_fetcher.rb', line 243

def gem_exists?(filename)
  configuration.mirror_directory.file_exists?(filename)
end

#ignore_gem?(*args) ⇒ Boolean

Returns:

  • (Boolean)

See Also:



250
251
252
 
# File 'lib/gem_mirror/gems_fetcher.rb', line 250

def ignore_gem?(*args)
  configuration.ignore_gem?(*args)
end

#load_specification(raw_spec) ⇒ Gem::Specification

Reads the inflated data of a Gemspec and returns the loaded specification instance.

Parameters:

  • raw_spec (String)

Returns:

  • (Gem::Specification) 


148
149
150
151
152
153
154
155
156
 
# File 'lib/gem_mirror/gems_fetcher.rb', line 148

def load_specification(raw_spec)
  stream  = Zlib::Inflate.new
  content = stream.inflate(raw_spec)

  stream.finish
  stream.close

  Marshal.load(content)
end

#logger ⇒ Logger

Returns:

  • (Logger)

See Also:



226
227
228
 
# File 'lib/gem_mirror/gems_fetcher.rb', line 226

def logger
  configuration.logger
end

#versions_for(gem) ⇒ Array

Returns an Array containing the versions that should be fetched for a Gem.

Parameters:

Returns:

  • (Array) 


76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
 
# File 'lib/gem_mirror/gems_fetcher.rb', line 76

def versions_for(gem)
  available       = versions_file.versions_for(gem.name)
  versions        = gem.version? ? [gem.version] : available
  available_names = available.map(&:to_s)

  # Get rid of invalid versions. Due to Gem::Version having a custom ==
  # method, which treats "3.4" the same as "3.4.0" we'll have to compare
  # the versions as String instances.
  versions = versions.select do |version|
    available_names.include?(version.to_s)
  end

  versions = [available.last] if versions.empty?

  versions
end