Class: AllGems::GemWorker

Inherits:
Object
  • Object
show all
Defined in:
lib/allgems/GemWorker.rb

Defined Under Namespace

Classes: DocError, Error, FetchError

Class Attribute Summary collapse

Class Method Summary collapse

Class Attribute Details

.pool ⇒ Object (readonly)

Returns the value of attribute pool.



9
10
11
 
# File 'lib/allgems/GemWorker.rb', line 9

def pool
  @pool
end

Class Method Details

.direct_unpack(path, basedir) ⇒ Object

path

path to the gem file

basedir

directory to unpack in

Last ditch effort to unpack the gem



110
111
112
113
114
 
# File 'lib/allgems/GemWorker.rb', line 110

def direct_unpack(path, basedir)
    AllGems.logger.warn "Attempting forcible unpack on: #{path}"
    unpackdir = path.slice(0, path.rindex('.'))
    self.run_command("cd #{basedir} && gem unpack #{path} && mv #{unpackdir}/* #{basedir}/unpack/ && rm -rf #{unpackdir}")
end

.fetch(spec, uri, save_path) ⇒ Object

spec

Gem::Specification

uri

URI of gem files home

save_path

path to save gem

Fetch the gem file from the server. Returns the path to the gem on the local machine



48
49
50
51
52
53
54
55
56
57
58
59
60
61
 
# File 'lib/allgems/GemWorker.rb', line 48

def fetch(spec, uri, save_path)
    AllGems.logger.info "Fetching gem from: #{uri}/gems/#{spec.full_name}.gem"
    FileUtils.touch(save_path)
    begin
        remote_path = "#{uri}/gems/#{spec.full_name}.gem"
        remote_uri = URI.parse(remote_path)
        file = File.open(save_path, 'wb')
        file.write(self.fetch_remote(remote_uri))
        file.close
        save_path
    rescue StandardError => boom
        raise FetchError.new(spec.name, spec.version, remote_path, boom)
    end
end

.fetch_remote(uri, depth = 0) ⇒ Object

uri

URI of gem

depth

number of times called

Fetch gem from given URI

Raises:

  • (IOError) 


66
67
68
69
70
71
72
73
74
75
76
 
# File 'lib/allgems/GemWorker.rb', line 66

def fetch_remote(uri, depth=0)
    raise IOError.new("Depth too deep") if depth > 9
    response = Net::HTTP.get_response(uri)
    if(response.is_a?(Net::HTTPSuccess))
        response.body
    elsif(response.is_a?(Net::HTTPRedirection))
        self.fetch_remote(URI.parse(response['location']), depth + 1)
    else
        raise IOError.new("Unknown response type: #{response}")
    end
end

.generate_documentation(spec, dir) ⇒ Object

dir

base directory location of gem contents

Generates the documentation of the given directory of ruby code. Appends ‘unpack’ to the given directory for code discovery. Documentation will be output in “#dir/doc”



120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
 
# File 'lib/allgems/GemWorker.rb', line 120

def generate_documentation(spec, dir)
    AllGems.logger.info "Generating documentation for #{spec.full_name}"
    AllGems.doc_format.each do |f|
        command = nil
        args = []
        case f.to_sym
            when :rdoc
                command = 'rdoc'
                args << '--format=darkfish' << '-aFNqH' << "--op=#{dir}/doc/rdoc" << "#{dir}/unpack"
            when :sdoc
                command = 'sdoc'
                args << '-T direct' << "-o #{dir}/doc/sdoc" << "#{dir}/unpack"
            when :hanna
                command = "ruby #{AllGems.hanna_hack}"
                args << "-o #{dir}/doc/hanna" << "#{dir}/unpack"
            else
                next # if we don't know what to do with it, skip it
        end
        action = lambda do |spec, dir, command, args, f|
            FileUtils.rm_r("#{dir}/doc/#{f}", :force => true) # make sure we are clean before we get dirty
            result = self.run_command("#{command} #{args}")
            raise DocError.new(spec.name, spec.version) unless result
            AllGems.logger.info "Completed documentation for #{spec.full_name}"
            FileUtils.chmod_R(0755, "#{dir}/doc/#{f}") # fix any bad permissions
            Indexer.index_gem(spec, "#{dir}/doc/#{f}", f)
        end
        @pool << [action, [spec, dir.dup, command.dup, args.join(' '), f]]
    end
end

.get_spec(name, version) ⇒ Object

name

name of gem

version

version of gem

Fetches the Gem::Specification for the given gem



35
36
37
38
39
40
41
 
# File 'lib/allgems/GemWorker.rb', line 35

def get_spec(name, version)
    AllGems.logger.info "Fetching gemspec for #{name}-#{version}"
    dep = Gem::Dependency.new(name, version)
    spec = nil
    @glock.synchronize{spec = Gem::SpecFetcher.fetcher.fetch dep, true}
    spec[0]
end

.process(args) ⇒ Object

:name

name of the gem

:version

version of the gem

:database

database connection

Raises:

  • (NameError) 


19
20
21
22
23
24
25
26
27
28
29
30
 
# File 'lib/allgems/GemWorker.rb', line 19

def process(args)
    AllGems.logger.info "Processing gem: #{args[:name]}-#{args[:version]}"
    spec,uri = self.get_spec(args[:name], args[:version])
    raise NameError.new("Name not found: #{args[:name]} - #{args[:version]}") if spec.nil?
    basedir = "#{AllGems.data_directory}/#{spec.name}/#{spec.version.version}"
    FileUtils.mkdir_p "#{basedir}/unpack"
    AllGems.logger.info "Created new directory: #{basedir}/unpack"
    gempath = self.fetch(spec, uri, "#{basedir}/#{spec.full_name}.gem")
    self.unpack(gempath, basedir)
    self.generate_documentation(spec, basedir)
    self.save_data(spec, args[:database])
end

.run_command(command, return_output = false) ⇒ Object

command

command to run

return_output

return output

Runs a command. Returns true if status returns 0. If return_output is true, return value is: [status, output]



154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
 
# File 'lib/allgems/GemWorker.rb', line 154

def run_command(command, return_output=false)
    AllGems.logger.debug "Command to be executed: #{command}"
    pro = nil
    output = []
    status = nil
    begin
        pro = IO.popen(command)
        Process.setpriority(Process::PRIO_PROCESS, pro.pid, 19) unless ON_WINDOWS
        until(pro.closed? || pro.eof?)
            output << pro.gets
        end
    ensure
        unless(pro.nil?)
            pid, status = Process.waitpid2(pro.pid)
        else
            status = 1
        end
    end
    return return_output ? [status == 0, output.join] : status == 0
end

.save(gempath, newpath) ⇒ Object

gempath

path to gem file

newname

path to move gem file to

Moves the gem to the given location



81
82
83
84
85
 
# File 'lib/allgems/GemWorker.rb', line 81

def save(gempath, newpath)
    AllGems.logger.info "Moving #{gempath} to #{newpath}"
    FileUtils.mv gempath, newpath
    newpath
end

.save_data(spec, db) ⇒ Object

spec

Gem::Specification

Save data to the database about this gem



177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
 
# File 'lib/allgems/GemWorker.rb', line 177

def save_data(spec, db)
    AllGems.logger.info "Saving meta data for #{spec.full_name}"
    @slock.synchronize do
        gid = db[:gems].filter(:name => spec.name).first
        gid = gid.nil? ? db[:gems].insert(:name => spec.name) : gid[:id]
        pid = db[:platforms].filter(:platform => spec.platform).first
        pid = pid.nil? ? db[:platforms].insert(:platform => spec.platform) : pid[:id]
        vid = db[:versions] << {:version => spec.version.version, :gem_id => gid, :platform_id => pid, :release => spec.date}
        db[:specs] << {:version_id => vid, :spec => [Marshal.dump(spec)].pack('m')}
        db[:gems].filter(:id => gid).update(:summary => spec.summary)
        db[:gems].filter(:id => gid).update(:description => spec.description)
    end
    AllGems.logger.info "Meta data saving complete for #{spec.full_name}"
    true
end

.setup ⇒ Object

Get the worker ready to go



11
12
13
14
15
 
# File 'lib/allgems/GemWorker.rb', line 11

def setup
    @glock = Mutex.new
    @slock = Mutex.new
    @pool = ActionPool::Pool.new(:max_threads => 10, :a_to => 60*5)
end

.unpack(path, basedir, depth = 0) ⇒ Object

path

path to the gem file

basedir

directory to unpack in

depth

number of times called

Unpacks the gem into the basedir under the ‘unpack’ directory



91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
 
# File 'lib/allgems/GemWorker.rb', line 91

def unpack(path, basedir, depth=0)
    AllGems.logger.info "Unpacking gem: #{path}"
    begin
        Gem::Installer.new(path, :unpack => true).unpack "#{basedir}/unpack"
        FileUtils.chmod_R(0755, "#{basedir}/unpack") # fix any bad permissions
        FileUtils.rm(path)
    rescue
        if(File.size(path) < 1 || depth > 10)
            raise IOError.new("Failed to unpack gem: #{path}") unless self.direct_unpack(path, basedir)
        else
            self.unpack(path, basedir, depth+1)
        end
    end
    true
end