Class: Dependabot::Linguist::Repository

Inherits:
Object
  • Object
show all
Defined in:
lib/dependabot/linguist/repository.rb

Overview

Repository wraps a Linguist::Repository, to discover “linguist languages” present in a repository, then maps them to Dependabot Ecosystems, finally verifying that those ecosystems are valid for the places linguist found the languages it thought was relevant to each dependabot ecosystem.

Instance Method Summary collapse

Constructor Details

#initialize(repo_path, repo_name, ignore_linguist: 0, verbose: false) ⇒ Repository

Returns a new instance of Repository.



18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
 
# File 'lib/dependabot/linguist/repository.rb', line 18

def initialize(repo_path, repo_name, ignore_linguist: 0, verbose: false)
  @repo_path = repo_path.chomp.delete_suffix("/")
  @repo_name = repo_name
  begin
    @repo = Rugged::Repository.new(@repo_path)
  rescue Rugged::RepositoryError, Rugged::OSError
    # Either the folder doesn't exist, or it does and doesn't have a `.git/`
    # Try to clone into it, if it's public
    puts "Repository #{@repo_name} not found at #{@repo_path}; falling back to cloning public url"
    @repo = Rugged::Repository.clone_at("https://github.com/#{@repo_name}.git", @repo_path)
  end
  @ignore_linguist = [[0, ignore_linguist].max, 2].min
  @verbose = verbose
  @linguist = ::Linguist::Repository.new(@repo, @repo.head.target_id)
end

Instance Method Details

#all_directories ⇒ Object

Get ALL directories for the repo path.



157
158
159
160
161
 
# File 'lib/dependabot/linguist/repository.rb', line 157

def all_directories
  # /**/*/ rather than /**/ would remove the base path, but delete_prefix
  # will also remove it, so it needs to be specially added.
  @all_directories ||= (["/"] | Dir.glob("#{@repo_path}/**/*/").map { |subpath| subpath.delete_prefix(@repo_path).delete_suffix("/") })
end

#all_ecosystem_classes ⇒ Object 



180
181
182
 
# File 'lib/dependabot/linguist/repository.rb', line 180

def all_ecosystem_classes
  @all_ecosystem_classes ||= PACKAGE_ECOSYSTEM_TO_FILE_FETCHERS_REGISTRY_KEY.transform_values { |k, v| [k, Dependabot::FileFetchers.for_package_manager(v)] }
end

#all_sources ⇒ Object

Get ALL sources from ALL directories for the repo path.



164
165
166
 
# File 'lib/dependabot/linguist/repository.rb', line 164

def all_sources
  @all_sources ||= all_directories.collect { |directory| Dependabot::Source.new(provider: "github", repo: @repo_name, directory: directory) }
end

#directories_per_ecosystem_validated_by_dependabot ⇒ Object

directories_per_ecosystem_validated_by_dependabot maps each identified present ecosystem to a list of the directories that linguist found files for, that were then validated by running the file_fetcher files on them.



187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
 
# File 'lib/dependabot/linguist/repository.rb', line 187

def directories_per_ecosystem_validated_by_dependabot
  @directories_per_ecosystem_validated_by_dependabot ||= nil
  if @directories_per_ecosystem_validated_by_dependabot.nil?
    enable_options = { kubernetes_updates: true }
    @directories_per_ecosystem_validated_by_dependabot = {}
    case @ignore_linguist
    when 1
      # If ignore linguist is 1, we rely on it to block "vendored"
      # files from the sources, but we run all dependabot classes
      sources = linguist_sources.values
      ecosystem_classes = all_ecosystem_classes
    when 2
      # If ignore linguist is 2, we just don't use it at all.
      sources = all_sources
      ecosystem_classes = all_ecosystem_classes
    else # when 0 is part of this.
      # If ignore linguist is 0, we don't ignore it and rely
      # on it to find sources and pick dependabot classes
      sources = nil
      ecosystem_classes = file_fetcher_class_per_package_ecosystem
    end
    ecosystem_classes.each do |package_ecosystem, file_fetcher_class|
      @directories_per_ecosystem_validated_by_dependabot[package_ecosystem] = []
      puts "Spawning class instances for #{package_ecosystem}, in repo #{@repo_path}, class #{file_fetcher_class}" if @verbose
      sources = directories_per_package_ecosystem[package_ecosystem].collect { |directories| linguist_sources[directories] } unless [1, 2].any? @ignore_linguist
      sources.each do |source|
        fetcher = file_fetcher_class.new(source: source, credentials: [], repo_contents_path: @repo_path, options: enable_options)
        begin
          unless fetcher.files.map(&:name).empty?
            @directories_per_ecosystem_validated_by_dependabot[package_ecosystem] |= [source.directory]
            puts "-- Dependency files FOUND for package-ecosystem #{package_ecosystem} at #{source.directory}; #{fetcher.files.map(&:name)}" if @verbose
          end
        rescue Dependabot::DependabotError => e
          # Most of these will be Dependabot::DependencyFileNotFound
          # or Dependabot::PathDependenciesNotReachable
          puts "-- Caught a DependabotError, #{e.class}, for package-ecosystem #{package_ecosystem} at #{source.directory}: #{e.message}" if @verbose
        end
      end
    end
    @directories_per_ecosystem_validated_by_dependabot = @directories_per_ecosystem_validated_by_dependabot.delete_if { |_, v| v.empty? }.sort.to_h
  end
  @directories_per_ecosystem_validated_by_dependabot
end

#directories_per_linguist_language ⇒ Object

directories_per_linguist_language inverts the linguist_cache map to “<Language>” => [“<folder_path>”, …], a list of folders per language!



63
64
65
66
67
68
69
70
71
72
73
74
75
 
# File 'lib/dependabot/linguist/repository.rb', line 63

def directories_per_linguist_language
  @directories_per_linguist_language ||= linguist_cache.keys.to_h { |source_file_path|
    # create the map "<file_path>" => "<folder_path>"
    [source_file_path, "/#{source_file_path.slice(0, source_file_path.rindex("/") || 0)}"]
  }.group_by { |source_file_path, _source_folder_path|
    # create the map "<Language>" => [["<file_path>", "<folder_path>"], ...]
    linguist_cache[source_file_path][0]
  }.to_h { |linguist_language, file_then_folder_arr|
    # create the map "<Language>" => ["<folder_path>", ...] by taking the
    # (&:last) out of each ["<file_path>", "<folder_path>"] pair, uniquely
    [linguist_language, file_then_folder_arr.map(&:last).uniq]
  }
end

#directories_per_package_ecosystem ⇒ Object

directories_per_package_ecosystem squashes the map of directories_per_package_manager according to the map of managers to ecosystems, as some managers share a common ecosystem name.



116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
 
# File 'lib/dependabot/linguist/repository.rb', line 116

def directories_per_package_ecosystem
  @directories_per_package_ecosystem ||= nil
  if @directories_per_package_ecosystem.nil?
    @directories_per_package_ecosystem = {}
    directories_per_package_manager.each do |dependabot_package_manager, source_directories|
      Dependabot::Linguist.package_managers_to_package_ecosystems([dependabot_package_manager]).each do |dependabot_package_ecosystem|
        if @directories_per_package_ecosystem[dependabot_package_ecosystem].nil?
          @directories_per_package_ecosystem[dependabot_package_ecosystem] = []
        end
        @directories_per_package_ecosystem[dependabot_package_ecosystem] |= source_directories
      end
    end
  end
  @directories_per_package_ecosystem
end

#directories_per_package_manager ⇒ Object

directories_per_package_manager splits and merges the results of directories_per_linguist_language; split across each package manager that is relevant to the language, and then merges the list of file paths for that language into the list of file paths for each package manager!



83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
 
# File 'lib/dependabot/linguist/repository.rb', line 83

def directories_per_package_manager
  @directories_per_package_manager ||= {}.tap do |this|
    directories_per_linguist_language.each do |linguist_language, source_directories|
      Dependabot::Linguist.linguist_languages_to_package_managers([linguist_language]).each do |dependabot_package_manager|
        this[dependabot_package_manager] = (this[dependabot_package_manager] || []) | source_directories
      end
    end
    # GitHub Actions must be added seperately..
    # if any yaml exist in the workflows folder, it needs to be added at "/"
    if (directories_per_linguist_language["YAML"] || []).any? "/.github/workflows"
      this[PackageManagers::GITHUB_ACTIONS] = ["/"]
    end
    # Because actions are handled like this we also need to regexp for /\/action\.ya?ml$/
    (files_per_linguist_language["YAML"] || []).each do |source_file_path|
      # File paths aren't cleaned from linguist, so prepend the '/' here.
      # This lets it match the \/ before action.ya?ml if it's in the root dir.
      # /(?<dir>\S*)\/(?<file>action\.ya?ml)$/
      action_match = "/#{source_file_path}".match %r{(?<dir>\S*)/(?<file>action\.ya?ml)$}
      if action_match
        # But that also means we then need to check if dir is empty, if it's the root dir
        if action_match[:dir].empty?
          this[PackageManagers::GITHUB_ACTIONS] = (this[PackageManagers::GITHUB_ACTIONS] || []) | ["/"]
        else
          this[PackageManagers::GITHUB_ACTIONS] = (this[PackageManagers::GITHUB_ACTIONS] || []) | [action_match[:dir]]
        end
      end
    end
  end
end

#file_fetcher_class_per_package_ecosystem ⇒ Object

file_fetcher_class_per_package_ecosystem maps ecosystem names to the class objects for each dependabot file fetcher class that’s relevant based on the list of ecosystems found by linguist languages.



135
136
137
138
139
140
141
142
143
144
145
146
147
 
# File 'lib/dependabot/linguist/repository.rb', line 135

def file_fetcher_class_per_package_ecosystem
  @file_fetcher_class_per_package_ecosystem ||= nil
  if @file_fetcher_class_per_package_ecosystem.nil?
    @file_fetcher_class_per_package_ecosystem = {}
    directories_per_package_ecosystem.each_key do |possible_ecosystem|
      @file_fetcher_class_per_package_ecosystem[possible_ecosystem] =
        Dependabot::FileFetchers.for_package_manager(
          Dependabot::Linguist::PACKAGE_ECOSYSTEM_TO_FILE_FETCHERS_REGISTRY_KEY[possible_ecosystem]
        )
    end
  end
  @file_fetcher_class_per_package_ecosystem
end

#files_per_linguist_language ⇒ Object

files_per_linguist_language inverts the linguist_cache map to “<Language>” => [“<file_path>”, …], a list of files per language! Note that they are not cleaned in the same way the folder paths in each of the “directories per *” are prepended with a ‘/’.



54
55
56
57
58
59
 
# File 'lib/dependabot/linguist/repository.rb', line 54

def files_per_linguist_language
  @files_per_linguist_language ||= linguist_cache.keys.group_by { |source_file_path|
    # create the map "<Language>" => ["<file_path>", ...]
    linguist_cache[source_file_path][0]
  }
end

#linguist_cache ⇒ Object

linguist_cache, linguist.cache, is a map of “<file_path>” => [“<Language>”, <loc>] for any files found for any language looked for.



42
43
44
 
# File 'lib/dependabot/linguist/repository.rb', line 42

def linguist_cache
  @linguist_cache ||= @linguist.cache
end

#linguist_directories ⇒ Object

Get the list of all directories identified by linguist, that had their language mapped to a relevant dependabot ecosystem.



170
171
172
 
# File 'lib/dependabot/linguist/repository.rb', line 170

def linguist_directories
  @linguist_directories ||= directories_per_package_ecosystem.values.flatten.uniq
end

#linguist_languages ⇒ Object

Wraps Linguist::Repository.new(~).languages



35
36
37
 
# File 'lib/dependabot/linguist/repository.rb', line 35

def linguist_languages
  @linguist_languages ||= @linguist.languages
end

#linguist_sources ⇒ Object

Get the list of all sources from all directories identified by linguist, that had their language mapped to a relevant dependabot ecosystem.



176
177
178
 
# File 'lib/dependabot/linguist/repository.rb', line 176

def linguist_sources
  @linguist_sources ||= linguist_directories.to_h { |directory| [directory, Dependabot::Source.new(provider: "github", repo: @repo_name, directory: directory)] }
end

#put_discovery_info ⇒ Object

Print out the lists of languages, managers, and ecosystems found here.



150
151
152
153
154
 
# File 'lib/dependabot/linguist/repository.rb', line 150

def put_discovery_info
  puts "List of languages: #{directories_per_linguist_language.keys}"
  puts "List of package managers: #{directories_per_package_manager.keys}"
  puts "List of package ecosystems: #{directories_per_package_ecosystem.keys}"
end