diff --git a/README b/README index 8a1e849..eca3ab7 100644 --- a/README +++ b/README @@ -6,12 +6,11 @@ Requirements ========== the following rubygems are required: -- pony - rest-client - xml-simple Usage ========== -e.g ruby scraper.rb pics user@email.com +e.g ruby scraper.rb pics robots monsters -This will take all the imgur pics from the /r/pics subreddit and send them as an attachment to user@email.com +This will take all the imgur images (jpg, gif, and png) from the /r/pics, /r/robots, and /r/monsters subreddits, and save them in subdirectories of the current directory with the same names as the subreddits. diff --git a/scraper.rb b/scraper.rb index 901509d..e9dffe7 100644 --- a/scraper.rb +++ b/scraper.rb @@ -1,59 +1,97 @@ require 'rubygems' require 'restclient' require 'xmlsimple' -require 'pony' - -# First argument should be a subreddit -subreddit = ARGV[0] -# destination is the second argument -destination = ARGV[1] - -# get the rss feed -data = RestClient.get "http://www.reddit.com/r/#{subreddit}/.rss" -if data.code != 200 - puts "Invalid subreddit (url)" - exit -end -# get xml from the data -begin - xml = XmlSimple.xml_in(data) -rescue - puts "XML was invalid." - exit + +iregex = /http:\/\/(i\.){0,1}(minus|imgur).com\/([0-9a-zA-Z]){3,}(\.jpg|\.gif|\.png){0,1}/i +newfiles = Hash.new 0 + + +ARGV.each do |subreddit| + destination = subreddit + + # get the rss feed + begin + data = RestClient.get "http://www.reddit.com/r/#{subreddit}/.rss" + if data.code != 200 + puts "Invalid subreddit (url)" + next + end + rescue + puts "Couldn't get #{subreddit}/.rss" + next + end + # get xml from the data + begin + xml = XmlSimple.xml_in(data) + rescue + puts "XML for subreddit #{subreddit} was invalid." + exit + end + + # make the empty directory pics + begin + FileUtils.mkdir destination + rescue + end + + # get all the links + links = xml["channel"][0]["item"].collect { |i| + iregex.match(i["description"][0]) + } + + puts "Checking #{subreddit}" + + # download all the files + links.each { |lnk| + next if lnk.nil? + l = lnk[0] + img = (/(jpg|gif|png)$/i).match l + if !img + [".jpg", ".gif", ".png"].each { |ending| + candidate = l.to_s + ending + i = File.join destination, File.basename(candidate) + gstr = "#{File.join(destination, File.basename(l.to_s))}*" + g = Dir.glob gstr + already_have = g.size > 0 + if already_have + break + else + begin + RestClient.get(candidate) { |response, request, result| + next unless response.code.to_s.start_with? "200" + File.open(i, 'w') { |f| f.write response } + puts " Wrote #{i}" + newfiles[subreddit] = newfiles[subreddit] + 1 + break + } + rescue + next + end + end + } + else + i = File.join destination, File.basename(l) + already_have = File.exists? i + if already_have + next + else + begin + File.open(i, 'w') { |f| f.write(RestClient.get(l)) } + puts " Wrote #{i}" + newfiles[subreddit] = newfiles[subreddit] + 1 + rescue + end + end + end + } end -# make the empty directory pics -FileUtils.rm_rf 'pics' if File.exist? 'pics' -FileUtils.mkdir 'pics' - -# get all the links -links = xml["channel"][0]["item"].collect { |i| - i["description"][0].scan(/http:\/\/i.imgur.com\/[0-9a-zA-Z]+\.jpg/).first -} - -# download all the files -links.each { |l| - File.open('pics/' + File.basename(l), 'w') { |f| f.write(RestClient.get(l)) } unless l.nil? -} - -# put the files into an array to attach to the file -files = {} -Dir.entries('pics').each { |f| - files[File.basename(f)] = File.read('pics/' + f) unless f == "." || f == ".." -} - -# send the email -begin - sent = Pony.mail( - :to => destination, - :from => destination, - :subject => 'Reddit Scraper', - :body => 'Here are your wonderful files...', - :attachments => files - ) -rescue - puts "Email wasn't sent." if !sent +puts "" +total = 0 +newfiles.keys.sort.each do |k| + num = newfiles[k] + total = total + num + puts "#{k}: #{num}" if num > 0 end -# remove the directory -FileUtils.rm_rf 'pics' +puts "\nWrote #{total} new files."