Ruby+Erlang抓取贴吧图片

清华大佬耗费三个月吐血整理的几百G的资源,免费分享!....>>>

#coding utf-8
#!/usr/bin/env ruby -w

require "open-uri"
require "nokogiri"
require "thread"
require 'erlectricity'

class Spider
  def initialize(dir, url)
    @url = url
    @dir = dir
  end

  def create_dir
    `rm -rf #{@dir}` if File.directory? @dir
    Dir.mkdir @dir
  end

  def get_imgs(doc)
    links = doc.css('img.BDE_Image').collect { |img| img['src'] }
  end

  def run(pn, f)
    create_dir()
    threads = []
    (1..pn).each do |i|
      threads << Thread.new do
        puts "going to #@url in page #{i}"
        doc = Nokogiri::HTML(open("#@url?pn=#{i}"))
        links = get_imgs(doc)
        f.send!([:links, links])
      end
    end
    threads.each &:join
  end
end

receive do |f|
  _pn = 1

  f.when([:pn, Fixnum]) do |pn|
    _pn = pn
    f.send!([:pn, _pn])
    f.receive_loop
  end

  f.when([:url, String]) do |url|
    if url =~ /.*\/p\/(\d+)\/?/
      spider = Spider.new($1, url)
      spider.run(_pn, f)
      f.send!(:exit)
    else
      f.send!([:error, "error."])
      f.receive_loop
    end
  end
end