From 0b17f70280339c2e7d4d61721094863c5e48b304 Mon Sep 17 00:00:00 2001 From: Joe Ardent Date: Thu, 19 Jul 2012 21:23:36 -0700 Subject: [PATCH 1/8] saves images instead of mails them will now also grab gifs and pngs --- README | 5 ++- scraper.rb | 99 +++++++++++++++++++++++++----------------------------- 2 files changed, 48 insertions(+), 56 deletions(-) diff --git a/README b/README index 8a1e849..eca3ab7 100644 --- a/README +++ b/README @@ -6,12 +6,11 @@ Requirements ========== the following rubygems are required: -- pony - rest-client - xml-simple Usage ========== -e.g ruby scraper.rb pics user@email.com +e.g ruby scraper.rb pics robots monsters -This will take all the imgur pics from the /r/pics subreddit and send them as an attachment to user@email.com +This will take all the imgur images (jpg, gif, and png) from the /r/pics, /r/robots, and /r/monsters subreddits, and save them in subdirectories of the current directory with the same names as the subreddits. diff --git a/scraper.rb b/scraper.rb index 901509d..c8897dd 100644 --- a/scraper.rb +++ b/scraper.rb @@ -1,59 +1,52 @@ require 'rubygems' require 'restclient' require 'xmlsimple' -require 'pony' -# First argument should be a subreddit -subreddit = ARGV[0] -# destination is the second argument -destination = ARGV[1] - -# get the rss feed -data = RestClient.get "http://www.reddit.com/r/#{subreddit}/.rss" -if data.code != 200 - puts "Invalid subreddit (url)" - exit -end -# get xml from the data -begin - xml = XmlSimple.xml_in(data) -rescue - puts "XML was invalid." - exit -end - -# make the empty directory pics -FileUtils.rm_rf 'pics' if File.exist? 'pics' -FileUtils.mkdir 'pics' - -# get all the links -links = xml["channel"][0]["item"].collect { |i| - i["description"][0].scan(/http:\/\/i.imgur.com\/[0-9a-zA-Z]+\.jpg/).first -} - -# download all the files -links.each { |l| - File.open('pics/' + File.basename(l), 'w') { |f| f.write(RestClient.get(l)) } unless l.nil? -} - -# put the files into an array to attach to the file -files = {} -Dir.entries('pics').each { |f| - files[File.basename(f)] = File.read('pics/' + f) unless f == "." || f == ".." -} - -# send the email -begin - sent = Pony.mail( - :to => destination, - :from => destination, - :subject => 'Reddit Scraper', - :body => 'Here are your wonderful files...', - :attachments => files - ) -rescue - puts "Email wasn't sent." if !sent +iregex = /http:\/\/i.imgur.com\/[0-9a-zA-Z]+\.(jpg|gif|png)/i +newfiles = 0 + +ARGV.each do |subreddit| + destination = subreddit + + # get the rss feed + data = RestClient.get "http://www.reddit.com/r/#{subreddit}/.rss" + if data.code != 200 + puts "Invalid subreddit (url)" + exit + end + # get xml from the data + begin + xml = XmlSimple.xml_in(data) + rescue + puts "XML for subreddit #{subreddit} was invalid." + exit + end + + # make the empty directory pics + begin + FileUtils.mkdir destination + rescue + end + + # get all the links + links = xml["channel"][0]["item"].collect { |i| + iregex.match(i["description"][0]) + } + + # download all the files + links.each { |lnk| + next if lnk.nil? + l = lnk[0] + i = File.join destination, File.basename(l) + already_have = File.exists? i + if already_have + puts "Skipping write of #{i}" + else + File.open(i, 'w') { |f| f.write(RestClient.get(l)) } + puts "Writing #{i}" + newfiles = newfiles + 1 + end + } end -# remove the directory -FileUtils.rm_rf 'pics' +puts "\nWrote #{newfiles} new files." From 23ef1832c95c017ff32d8d9bae5303ba19dc5677 Mon Sep 17 00:00:00 2001 From: Joe Ardent Date: Wed, 25 Jul 2012 00:06:45 -0700 Subject: [PATCH 2/8] also recognize i.minus.com URLs --- scraper.rb | 38 ++++++++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/scraper.rb b/scraper.rb index c8897dd..7e594cd 100644 --- a/scraper.rb +++ b/scraper.rb @@ -2,7 +2,7 @@ require 'restclient' require 'xmlsimple' -iregex = /http:\/\/i.imgur.com\/[0-9a-zA-Z]+\.(jpg|gif|png)/i +iregex = /http:\/\/i.(minus|imgur).com\/[0-9a-zA-Z]+\.(jpg|gif|png){0,1}/i newfiles = 0 ARGV.each do |subreddit| @@ -37,14 +37,36 @@ links.each { |lnk| next if lnk.nil? l = lnk[0] - i = File.join destination, File.basename(l) - already_have = File.exists? i - if already_have - puts "Skipping write of #{i}" + img = (/(jpg|gif|png)$/i).match l + if !img + [".jpg", ".gif", ".png"].each { |ending| + puts " Checking #{l}" + candidate = l.to_s + ending + i = File.join destination, File.basename(candidate) + already_have = File.exists? i + if already_have + puts "Skipping write of #{i}" + break + else + RestClient.get(candidate) { |response, request, result| + next if response.code.to_s.start_with? "404" + File.open(i, 'w') { |f| f.write response } + puts "Writing #{i}" + newfiles = newfiles + 1 + break + } + end + } else - File.open(i, 'w') { |f| f.write(RestClient.get(l)) } - puts "Writing #{i}" - newfiles = newfiles + 1 + i = File.join destination, File.basename(l) + already_have = File.exists? i + if already_have + puts "Skipping write of #{i}" + else + File.open(i, 'w') { |f| f.write(RestClient.get(l)) } + puts "Writing #{i}" + newfiles = newfiles + 1 + end end } end From 9b8862975d1850c508d7d70ac7c06af2f3d7993f Mon Sep 17 00:00:00 2001 From: Joe Ardent Date: Wed, 25 Jul 2012 00:15:58 -0700 Subject: [PATCH 3/8] more robust image URL guessing --- scraper.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scraper.rb b/scraper.rb index 7e594cd..d1fb3b9 100644 --- a/scraper.rb +++ b/scraper.rb @@ -2,7 +2,7 @@ require 'restclient' require 'xmlsimple' -iregex = /http:\/\/i.(minus|imgur).com\/[0-9a-zA-Z]+\.(jpg|gif|png){0,1}/i +iregex = /http:\/\/(i\.){0,1}(minus|imgur).com\/[0-9a-zA-Z]+(\.jpg|\.gif|\.png){0,1}/i newfiles = 0 ARGV.each do |subreddit| From 2f9e365510e1b8c962f13db2f0bc880a82512136 Mon Sep 17 00:00:00 2001 From: Joe Ardent Date: Tue, 31 Jul 2012 20:40:57 -0700 Subject: [PATCH 4/8] just better all around --- scraper.rb | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/scraper.rb b/scraper.rb index d1fb3b9..a56c5ed 100644 --- a/scraper.rb +++ b/scraper.rb @@ -2,7 +2,7 @@ require 'restclient' require 'xmlsimple' -iregex = /http:\/\/(i\.){0,1}(minus|imgur).com\/[0-9a-zA-Z]+(\.jpg|\.gif|\.png){0,1}/i +iregex = /http:\/\/(i\.){0,1}(minus|imgur).com\/([0-9a-zA-Z]){3,}(\.jpg|\.gif|\.png){0,1}/i newfiles = 0 ARGV.each do |subreddit| @@ -43,17 +43,19 @@ puts " Checking #{l}" candidate = l.to_s + ending i = File.join destination, File.basename(candidate) - already_have = File.exists? i + gstr = "#{File.join(destination, File.basename(l.to_s))}*" + puts " gstr: #{gstr}" + g = Dir.glob gstr + already_have = g.size > 0 if already_have puts "Skipping write of #{i}" break else RestClient.get(candidate) { |response, request, result| - next if response.code.to_s.start_with? "404" + next unless response.code.to_s.start_with? "200" File.open(i, 'w') { |f| f.write response } puts "Writing #{i}" newfiles = newfiles + 1 - break } end } From 0c941558cb1b2b568cd191da2907115979f50790 Mon Sep 17 00:00:00 2001 From: Joe Ardent Date: Sat, 4 Aug 2012 11:39:24 -0700 Subject: [PATCH 5/8] guards against 404s --- scraper.rb | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/scraper.rb b/scraper.rb index a56c5ed..c02c346 100644 --- a/scraper.rb +++ b/scraper.rb @@ -65,9 +65,12 @@ if already_have puts "Skipping write of #{i}" else - File.open(i, 'w') { |f| f.write(RestClient.get(l)) } - puts "Writing #{i}" - newfiles = newfiles + 1 + begin + File.open(i, 'w') { |f| f.write(RestClient.get(l)) } + puts "Writing #{i}" + newfiles = newfiles + 1 + rescue + end end end } From 78d1deffc82a8749c7407d3e94729edc05f66443 Mon Sep 17 00:00:00 2001 From: Joe Ardent Date: Tue, 25 Dec 2012 16:35:23 -0800 Subject: [PATCH 6/8] catch timeout exceptions --- scraper.rb | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/scraper.rb b/scraper.rb index c02c346..136cf8c 100644 --- a/scraper.rb +++ b/scraper.rb @@ -9,10 +9,15 @@ destination = subreddit # get the rss feed - data = RestClient.get "http://www.reddit.com/r/#{subreddit}/.rss" - if data.code != 200 - puts "Invalid subreddit (url)" - exit + begin + data = RestClient.get "http://www.reddit.com/r/#{subreddit}/.rss" + if data.code != 200 + puts "Invalid subreddit (url)" + next + end + rescue + puts "Couldn't get #{subreddit}/.rss" + next end # get xml from the data begin @@ -51,12 +56,16 @@ puts "Skipping write of #{i}" break else - RestClient.get(candidate) { |response, request, result| - next unless response.code.to_s.start_with? "200" - File.open(i, 'w') { |f| f.write response } - puts "Writing #{i}" - newfiles = newfiles + 1 - } + begin + RestClient.get(candidate) { |response, request, result| + next unless response.code.to_s.start_with? "200" + File.open(i, 'w') { |f| f.write response } + puts "Writing #{i}" + newfiles = newfiles + 1 + } + rescue + next + end end } else From 156d61c9eca16394a839a02787213249f041f40e Mon Sep 17 00:00:00 2001 From: Joe Ardent Date: Tue, 25 Dec 2012 17:34:48 -0800 Subject: [PATCH 7/8] cleans up status output --- scraper.rb | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/scraper.rb b/scraper.rb index 136cf8c..eae9e44 100644 --- a/scraper.rb +++ b/scraper.rb @@ -3,7 +3,8 @@ require 'xmlsimple' iregex = /http:\/\/(i\.){0,1}(minus|imgur).com\/([0-9a-zA-Z]){3,}(\.jpg|\.gif|\.png){0,1}/i -newfiles = 0 +newfiles = Hash.new 0 + ARGV.each do |subreddit| destination = subreddit @@ -38,6 +39,8 @@ iregex.match(i["description"][0]) } + puts "Checking #{subreddit}" + # download all the files links.each { |lnk| next if lnk.nil? @@ -45,23 +48,20 @@ img = (/(jpg|gif|png)$/i).match l if !img [".jpg", ".gif", ".png"].each { |ending| - puts " Checking #{l}" candidate = l.to_s + ending i = File.join destination, File.basename(candidate) gstr = "#{File.join(destination, File.basename(l.to_s))}*" - puts " gstr: #{gstr}" g = Dir.glob gstr already_have = g.size > 0 if already_have - puts "Skipping write of #{i}" break else begin RestClient.get(candidate) { |response, request, result| next unless response.code.to_s.start_with? "200" File.open(i, 'w') { |f| f.write response } - puts "Writing #{i}" - newfiles = newfiles + 1 + puts " Wrote #{i}" + newfiles[subreddit] = newfiles[subreddit] + 1 } rescue next @@ -72,12 +72,12 @@ i = File.join destination, File.basename(l) already_have = File.exists? i if already_have - puts "Skipping write of #{i}" + next else begin File.open(i, 'w') { |f| f.write(RestClient.get(l)) } - puts "Writing #{i}" - newfiles = newfiles + 1 + puts " Wrote #{i}" + newfiles[subreddit] = newfiles[subreddit] + 1 rescue end end @@ -85,4 +85,12 @@ } end -puts "\nWrote #{newfiles} new files." +puts "" +total = 0 +newfiles.keys.sort.each do |k| + num = newfiles[k] + total = total + num + puts "#{k}: #{num}" if num > 0 +end + +puts "\nWrote #{total} new files." From bcb918d98823828d6486100a8fc476ab5bce1388 Mon Sep 17 00:00:00 2001 From: Joe Ardent Date: Thu, 13 Mar 2014 22:33:20 -0700 Subject: [PATCH 8/8] Stop trying to guess names after success. --- scraper.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/scraper.rb b/scraper.rb index eae9e44..e9dffe7 100644 --- a/scraper.rb +++ b/scraper.rb @@ -62,6 +62,7 @@ File.open(i, 'w') { |f| f.write response } puts " Wrote #{i}" newfiles[subreddit] = newfiles[subreddit] + 1 + break } rescue next