file_scraper.rb 689 B

1234567891011121314151617181920212223242526272829303132333435
  1. module Docs
  2. class FileScraper < Scraper
  3. Response = Struct.new :body, :url
  4. class << self
  5. attr_accessor :dir
  6. end
  7. private
  8. def request_one(url)
  9. Response.new read_file(file_path_for(url)), URL.parse(url)
  10. end
  11. def request_all(start_url)
  12. queue = [start_url]
  13. until queue.empty?
  14. result = yield request_one(queue.shift)
  15. queue.concat(result) if result.is_a? Array
  16. end
  17. end
  18. def process_response?(response)
  19. response.body.present?
  20. end
  21. def file_path_for(url)
  22. File.join self.class.dir, url.sub(base_url.to_s, '')
  23. end
  24. def read_file(path)
  25. File.read(path) rescue nil
  26. end
  27. end
  28. end