Skip to content

Commit

Permalink
Begin implementing cache
Browse files Browse the repository at this point in the history
  • Loading branch information
gjtorikian committed Sep 18, 2015
1 parent 94b6af7 commit 074f8a2
Show file tree
Hide file tree
Showing 4 changed files with 75 additions and 8 deletions.
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -24,3 +24,5 @@ docs/
_yardoc
doc/
.DS_Store

.htmlproofer.log
55 changes: 48 additions & 7 deletions lib/html/proofer/cache.rb
Original file line number Diff line number Diff line change
@@ -1,16 +1,57 @@
require 'json'

module HTML
class Proofer
module Cache
def create_nokogiri(path)
if File.exist? path
content = File.open(path).read
class Cache
attr_reader :status

FILENAME = '.htmlproofer.log'

def initialize(now, options)
@status = {}
@now = now
@status[:time] = @now
@status[:urls] = []

if options.nil? || options.empty?
@store = false
else
@store = true
@timeframe = options[:timeframe]
end

if File.exist?(FILENAME)
@exists = true
@cache_log = JSON.parse(File.read(FILENAME))
else
content = path
@exists = false
end
end

def store?
@store
end

def exists?
@exists && within_timeframe
end

def within_timeframe

end

def load

end

def add(url, status)
return unless store?
@status[:urls] << { :url => url, :status => status }
end

Nokogiri::HTML(content)
def write
File.write(FILENAME, @status.to_json)
end
module_function :create_nokogiri
end
end
end
13 changes: 12 additions & 1 deletion lib/html/proofer/url_validator.rb
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
require 'typhoeus'
require 'uri'
require_relative './utils'
require_relative './cache'

module HTML
class Proofer
Expand All @@ -16,10 +17,17 @@ def initialize(logger, external_urls, options, typhoeus_opts, hydra_opts)
@options = options
@hydra = Typhoeus::Hydra.new(hydra_opts)
@typhoeus_opts = typhoeus_opts
@cache = Cache.new(Time.now, @options[:cache])
end

def run
external_link_checker(external_urls)
if @cache.exists?
@cache.load
else
external_link_checker(external_urls)
@cache.write if @cache.store?
end

@failed_tests
end

Expand Down Expand Up @@ -81,14 +89,17 @@ def response_handler(response, filenames)

if response_code.between?(200, 299)
check_hash_in_2xx_response(href, effective_url, response, filenames)
@cache.add(href, response_code)
elsif response.timed_out?
handle_timeout(href, filenames, response_code)
@cache.add(href, 0)
elsif method == :head
queue_request(:get, href, filenames)
else
return if @options[:only_4xx] && !response_code.between?(400, 499)
# Received a non-successful http response.
add_external_issue(filenames, "External link #{href} failed: #{response_code} #{response.return_message}", response_code)
@cache.add(href, response_code)
end
end

Expand Down
13 changes: 13 additions & 0 deletions spec/html/proofer/cache_spec.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
require 'spec_helper'

describe 'Cache test' do
it 'ignores an invalid tag by default' do
brokenLinkExternalFilepath = "#{FIXTURES_DIR}/links/brokenLinkExternal.html"
proofer = run_proofer(brokenLinkExternalFilepath)
expect(proofer.failed_tests.first).to match(/failed: 0/)

run_proofer(brokenLinkExternalFilepath, { :cache => { :timeframe => '30d' } })
end


end

2 comments on commit 074f8a2

@doktorbro
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Typhoeus includes built in support for caching. It’s documented in the readme. Could we use this interface?

@gjtorikian
Copy link
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Awesome suggestion. Thanks!

Please sign in to comment.