Class: SourceForge

Inherits:

Object

• Object
• SourceForge

Defined in:

lib/sourceforge.rb

Constant Summary

CHECKSUMS =

'md5'

DEFAULT_SERVERS =

[
  'easynews', # US
  'umn',      # US
  'ufpr',     # brazil
  'kent',     # UK
  'surfnet',  # NL
  'internap', # US
  'ovh',      # FR
  'switch',   # CH
  'heanet',   # IE
  'superb',   # US
  'nchc',     # TW
  'jaist',    # JP
  'mesh',     # DE
]

Instance Method Summary

• #choose_files ⇒ Object

  Ask user to choose some files.

• #do_download ⇒ Object

  Do the download of files.

• #file_menu(filter) ⇒ Object

  Print out a menu with the list of files to download.

• #filter_downloads(filter) ⇒ Object

  Filter downloads.

• #find_latest_releases ⇒ Object

  Given a project, try to find the latest file releases for it in sourceforge.

• #initialize ⇒ SourceForge constructor

  Constructor.

• #parse_args ⇒ Object

  Parse command-line arguments.

• #start_downloads ⇒ Object

  Start downloading files stored in @downloads array.

• #verify_checksums ⇒ Object

  For each checksum downloaded, verify it.

Constructor Details

#initialize ⇒ SourceForge

Constructor.

# File 'lib/sourceforge.rb', line 479

def initialize
  parse_args
  @project_list.each { |@project|
    @downloads = find_latest_releases
    do_download
    verify_checksums
  }
end

Instance Method Details

#choose_files ⇒ Object

Ask user to choose some files.

# File 'lib/sourceforge.rb', line 301

def choose_files
  puts
  puts  "Choose one or more files to download:"
  puts  "(Use '1 3 4' to toggle file or '3-5' for toggling a range)"
  print ">"
  cmd = $stdin.gets.split(/[,\s]+/)
  answer = []
  cmd.each { |c|
    if c =~ /(\d+)-(\d+)/
	answer += ($1.to_i..$2.to_i).to_a
    else
	answer << c.to_i
    end
  }
  return answer.uniq
end

#do_download ⇒ Object

Do the download of files

# File 'lib/sourceforge.rb', line 321

def do_download
  @checksums = []

  if @args['-src'] and @downloads.size > 1
    @downloads.delete_if { |f| f =~ /\.(exe|rpm|deb)$/i }

    # If files are labelled with src or source, remove
    # all other files from list.
    src_re = /(?:src|source)/i
    unless @downloads.grep(src_re).empty?
	@downloads.delete_if { |f| f !~ src_re }
    end

    # If sources are available in multiple compression
    # formats, prefer smallest linux format.
    filter_downloads( 
       [ 
	 /\.tar\.7za$/i,
	 /\.tar\.bz2$/i,
	 /\.tar\.gz$/i,
	 /\.tgz$/i,
	 /\.zip$/i,
	 /\.tar$/i,
       ]
       )
  else
    # Delete any source file archives...
    @downloads.delete_if { |f| f =~ /(?:src|source)/i }


    if @downloads.size > 1 and not @args['-all']
	# if more than one download, choose based on platform
	filter = [ 
 /\.tar\.bz2$/i, 
 /\.tar\.gz$/i, 
 /\.tgz$/i, 
 /\.sh$/i 
	]
	
	case RUBY_PLATFORM
	when /mswin/
 unless @downloads.grep(/\.(exe|zip)$/).empty?
   filter = [ 
     /\.exe$/i, 
     /\.zip$/i, 
     /\.tar\.bz2$/i, 
     /\.tar\.gz$/i, 
     /\.tgz$/i, 
     /\.sh$/i
   ]
 end
	when /linux/
 unless @downloads.grep(/\.(rpm|deb)$/).empty?
   filter = [ 
     /\.rpm$/i, 
     /\.deb$/i, 
     /\.tar\.bz2$/i, 
     /\.tar\.gz$/i, 
     /\.tgz$/i, 
     /\.sh$/i
   ]
 end
	when /osx/
 unless @downloads.grep(/\.dmg$/).empty?
   filter = [ 
     /\.dmg$/i, 
     /\.tar\.bz2$/i, 
     /\.tar\.gz$/i, 
     /\.tgz$/i, 
     /\.sh$/i
   ]
 end
	end

	filter_downloads(filter)
    elsif @downloads.size > 1
	filter_downloads([ /(?!#{CHECKSUMS})$/i ])
    end
  end

  if @downloads.empty?
    puts "Nothing to download."
    exit(1)
  end


  filter = (1..@downloads.size).to_a

  if @args['-ch']
    filter = @args['-ch']
  end

  if @args['-ch'] and filter.empty?
    answer = [1]
    while not answer.empty?
	file_menu(filter)
	answer  = choose_files
	filter  = filter + answer - ( filter & answer )
    end
  else
    file_menu(filter)
  end

  unless filter.empty?
    i = 0
    @downloads.delete_if { 
	i += 1
	not filter.include?(i)
    }
  end

  if @downloads.empty?
    puts "Nothing to download."
    exit(1)
  end

  @checksums = @downloads.grep(/\.#{CHECKSUMS}$/i)

  start_downloads
end

#file_menu(filter) ⇒ Object

Print out a menu with the list of files to download.

# File 'lib/sourceforge.rb', line 287

def file_menu(filter)
  puts "Files to download:"
  @downloads.each_with_index { |x, i| 
    print "\t%2d) %-40s" % [i+1, x]
    if filter.include?(i+1)
	print "\t*YES*"
    end
    print "\n"
  }
end

#filter_downloads(filter) ⇒ Object

Filter downloads. If any regex of filter matches, all other elements are removed. Also, checksum files are added to downloads (if available)

# File 'lib/sourceforge.rb', line 260

def filter_downloads(filter)
  checksums = @downloads.grep(/\.#{CHECKSUMS}$/i)

  @downloads -= checksums

  filter.each { |re|
    unless @downloads.grep(re).empty?
	@downloads.delete_if { |f| f !~ re }
	break
    end
  }

  @downloads.each { |d|
    root = d.sub(/\.\w+$/, '')
    item = ( checksums.grep(/^#{d}\.#{CHECKSUMS}$/) +
             checksums.grep(/^#{root}\.#{CHECKSUMS}$/) )
    next if item.empty?
    @checksums << item[0]
  }

  @downloads += @checksums
  @downloads.sort!
end

#find_latest_releases ⇒ Object

Given a project, try to find the latest file releases for it in sourceforge.

Return: Array of files

# File 'lib/sourceforge.rb', line 118

def find_latest_releases
  puts "Connecting to sourceforge..."
  url   = URI.parse( 'http://sourceforge.net/projects/' + @project )
  req   = Net::HTTP::Get.new(url.path)

  
  http = @proxy.start(url.host)
  res = http.request(req)
  if res.code.to_i != 200
    $stderr.puts "Could not open connection to sourceforge.  "
    $stderr.puts "Error code: #{res.code}"
    $stderr.puts res.message
    $stderr.puts res.body
    exit(res.code.to_i)
  end
  

  res.body =~ /(\/project\/showfiles.php\?group_id=\d+)/
  path = $1
  unless path
    $stderr.puts "Non-existent project \"#{@project}\"."
    exit(1)
  end

  req = Net::HTTP::Get.new(path)
  res = http.request(req)
  if res.code.to_i != 200
    $stderr.puts "Could not get #{path}.  Error code: #{res.code}"
    $stderr.puts res.message
    $stderr.puts res.body
    exit(200)
  end

  p = HTMLTree::XMLParser.new(false, true)
  p.feed( res.body )
  xml = p.tree

  re = /^http:\/\/(pr)?downloads\.sourceforge\.net\/#{@project}\/([^\?]+)/

  files = []
  xml.elements.each('//tr/td/a') { |x|
    parent = x.parent.parent
    clas = parent.attributes['class']
    next if clas != 'show'
    attr = x.attributes['href']
    next if attr !~ re
    files << $2
  }

  return files
end

#parse_args ⇒ Object

Parse command-line arguments.

# File 'lib/sourceforge.rb', line 44

def parse_args
  @args = Getopt::Declare.new(<<'EOH')

This script allows you to easily automate the downloading of
the latest version of any sourceforge project (as stored up 
to 31/12/2006) from any of sourceforge server by having it
parse the web pages for the project and extract the latest
release.
It supports connecting through a proxy if either the 
environment variable HTTP_PROXY or http_proxy is defined.
Also, downloads can be resumed like wget, in case the 
download is abruptly terminated.
If .md5 checksums are available, they will also be downloaded
and verified using ruby's digest/md5.

Options:
 -np		    	connect to the net without a proxy
 -all		    	Try download all files for all OSes.
    	Default: only current os.
 -o <dir:d>	    	Output directory to place files in.
    	Default: .
 -src		    	Try to download source files only.
 -ls		    	list valid names of servers to download
    	from.
 -s <server:s>	name of the server to download from.
    	Default: #{@server}.
 -ch [<num:+i>...]	Choose to download some files only.
 <project:s>...	name of the project(s) to download.

EOH

  @server = @args['-s'] || DEFAULT_SERVERS[0]

  dir = @args['-o'] || '.'
  Dir.chdir(dir)

  ### This proxy is set to use Privoxy/TOR's default configuration.
  ### If no proxy, make @proxy_host = nil or use -np option.
  unless @args['-np']
    @proxy = ENV['http_proxy'] || ENV['HTTP_PROXY']
    if @proxy.kind_of?(String)
	@proxy =~ /http:\/\/([^:]+):(\d+)/
	@proxy_host = $1
	@proxy_port = $2
	puts "Using proxy #{@proxy_host}:#{@proxy_port}"
    end
    @proxy_user = nil
    @proxy_pass = nil
  end

  if @args['-ls']
    puts DEFAULT_SERVERS
    exit 
  end

  @project_list = @args['<project>']
  unless @project_list
    $stderr.puts "Error: required parameter '<project>' missing."
    $stderr.puts
    $stderr.puts "(try '#{$0} -help' for more information)"
    exit(1)
  end

  @proxy = Net::HTTP::Proxy(@proxy_host, @proxy_port,
	      @proxy_user, @proxy_pass)
end

#start_downloads ⇒ Object

Start downloading files stored in @downloads array

# File 'lib/sourceforge.rb', line 173

def start_downloads
  path = "http://#{@server}.dl.sourceforge.net/sourceforge/#{@project}/"
  url  = URI.parse( path )
  puts
  puts "Downloading from '#{url.host}':"
  
  @downloads.each { |d|
    print "\t#{d}"

    begin
	f = nil
	header = nil
	existSize = 0

	http = @proxy.start(url.host)
	if File.exists?(d) and File.size(d) > 0
 existSize = File.size(d)
 f = open(d, 'ab')
 header = { 'Range' => 'bytes=%d-' % existSize }
	else
 f = open(d, 'wb')
	end
	req  = Net::HTTP::Get.new( url.path + d, header )
	
	http.request( req, header ) { |response|
 webSize = response.content_length.to_i + existSize
 if webSize == existSize
   puts "\tAlready downloaded."
   break
 end
 
 code = response.code.to_i
 if code == 302
   puts "\t** NOT IN MIRROR **"
   break
 elsif code != 200 and code != 206
   puts response.body
   break
 else
   size = webSize - existSize
   if size == webSize
     puts "\t#{webSize} bytes."
   else
     if code != 204
f = open(d, 'wb')
puts "\t#{webSize} bytes."
     else
puts " Continue #{size} of #{webSize} bytes."
     end
   end
   ### 
   line_length = 68
   puts "\t|" + '-'*line_length + '|'
   num = (existSize.to_f / webSize.to_f * line_length).to_i
   $stdout.print "\t " + '#' * num
   $stdout.flush
   numBytes = existSize.to_f
   response.read_body { |data|
     f.print data
     numBytes += data.size
     newnum = (numBytes / webSize.to_f * line_length).to_i - num
     if newnum > 0
$stdout.print '+' * newnum
$stdout.flush
num += newnum
     end
   }
   puts
 end
	}
	# When using TOR, sometimes when a circuit changes, you can
	# end up with a timeout/response error.  We just retry the request.
    rescue Timeout::Error, Net::HTTPBadResponse => e
	puts " #{e} - retry"
	retry
    end

    f.close
  }
end

#verify_checksums ⇒ Object

For each checksum downloaded, verify it.

# File 'lib/sourceforge.rb', line 446

def verify_checksums
  return if @checksums.empty?

  require 'digest/md5'

  puts "Verifying Checksums..."
  
  @checksums.each { |c|
    File.open(c, 'r') { |cm|
	while not cm.eof?
 line = cm.readline
 checksum, file = line.split(/\s+\*?/)
 sum = Digest::MD5.new
 File.open(file, 'rb') { |f|
   while not f.eof?
     sum << f.read(8196)
   end
 }
 print "\t#{file}...\t"
 if sum != checksum
   puts "FAILED!!!"
 else
   puts "OK"
          File.unlink c
 end
	end
    }
  }
end