diff --git a/README.md b/README.md index 3c305ce3..4441343b 100644 --- a/README.md +++ b/README.md @@ -179,7 +179,7 @@ In addition, there are a few "namespaced" options. These are: * `:validation` * `:typhoeus` * `:parallel` - +* `:cache` See below for more information. @@ -223,6 +223,33 @@ HTML::Proofer.new("out/", {:ext => ".htm", :parallel => { :in_processes => 3} }) In this example, `:in_processes => 3` is passed into Parallel as a configuration option. +## Configuring caching + +Checking external URLs can slow your tests down. If you'd like to speed that up, you can enable caching for your external links. Caching simply means to skip links that are valid for a certain period of time. + +While running tests, HTML::Proofer will always write to a log file within a directory called *tmp/.htmlproofer*. You should probably ignore this folder in your version control system. You can enable caching for this log file by passing in the option `:cache`, with a hash containing a single key, `:timeframe`. `:timeframe` defines the length of time the cache will be used before the link is checked again. The format of `:timeframe` is a number followed by a letter indicating the length of time. For example: + +* `M` means months +* `w` means weeks +* `d` means days +* `h` means hours + +For example, passing the following options means "recheck links older than thirty days": + +``` ruby +{ :cache => { :timeframe => '30d' } } +``` + +And the following options means "recheck links older than two weeks": + +``` ruby +{ :cache => { :timeframe => '2w' } } +``` + +Links that were failures are kept in the cache and *always* rechecked. If they pass, the cache is updated to note the new timestamp. + +The cache operates on external links only. + ## Logging HTML-Proofer can be as noisy or as quiet as you'd like. There are two ways to log information: diff --git a/html-proofer.gemspec b/html-proofer.gemspec index 4b51b41b..cb0e75e0 100644 --- a/html-proofer.gemspec +++ b/html-proofer.gemspec @@ -25,10 +25,12 @@ Gem::Specification.new do |gem| gem.add_dependency 'yell', '~> 2.0' gem.add_dependency 'parallel', '~> 1.3' gem.add_dependency 'addressable', '~> 2.3' + gem.add_dependency 'activesupport', '~> 4.2' gem.add_development_dependency 'redcarpet' gem.add_development_dependency 'rspec', '~> 3.1' gem.add_development_dependency 'rake' gem.add_development_dependency 'awesome_print' gem.add_development_dependency 'vcr', '~> 2.9' + gem.add_development_dependency 'timecop', '~> 0.8' end diff --git a/lib/html/proofer.rb b/lib/html/proofer.rb index 64feeafe..8d74e976 100644 --- a/lib/html/proofer.rb +++ b/lib/html/proofer.rb @@ -8,17 +8,15 @@ def require_all(path) require_all 'proofer' require_all 'proofer/check_runner' require_all 'proofer/checks' -require_relative './proofer/utils' -require_relative './proofer/xpathfunctions' require 'parallel' +require 'fileutils' begin require 'awesome_print' rescue LoadError; end module HTML - class Proofer include HTML::Proofer::Utils @@ -36,6 +34,8 @@ class Proofer } def initialize(src, opts = {}) + FileUtils.mkdir_p(STORAGE_DIR) unless File.exist?(STORAGE_DIR) + @src = src if opts[:verbose] @@ -89,9 +89,7 @@ def logger end def run - count = checks.length - check_text = "#{checks} " << (count == 1 ? 'check' : 'checks') - logger.log :info, :blue, "Running #{check_text} on #{@src} on *#{@options[:ext]}... \n\n" + logger.log :info, :blue, "Running #{checks} on #{@src} on *#{@options[:ext]}... \n\n" if @src.is_a?(Array) && !@options[:disable_external] check_list_of_links @@ -130,7 +128,9 @@ def check_directory_of_files validate_urls unless @options[:disable_external] - logger.log :info, :blue, "Ran on #{files.length} files!\n\n" + count = files.length + file_text = pluralize(count, 'file', 'files') + logger.log :info, :blue, "Ran on #{file_text}!\n\n" end # Walks over each implemented check and runs them on the files, in parallel. @@ -200,7 +200,7 @@ def print_failed_tests sorted_failures.sort_and_report count = @failed_tests.length - failure_text = "#{count} " << (count == 1 ? 'failure' : 'failures') + failure_text = pluralize(count, 'failure', 'failures') fail logger.colorize :red, "HTML-Proofer found #{failure_text}!" end end diff --git a/lib/html/proofer/cache.rb b/lib/html/proofer/cache.rb index ffec4c27..5f087cec 100644 --- a/lib/html/proofer/cache.rb +++ b/lib/html/proofer/cache.rb @@ -1,16 +1,141 @@ +require_relative 'utils' + +require 'json' +require 'active_support/core_ext/string' +require 'active_support/core_ext/date' +require 'active_support/core_ext/numeric/time' + module HTML class Proofer - module Cache - def create_nokogiri(path) - if File.exist? path - content = File.open(path).read + class Cache + include HTML::Proofer::Utils + + FILENAME = File.join(STORAGE_DIR, 'cache.log') + + attr_accessor :exists, :load, :cache_log, :cache_time + + def initialize(logger, options) + @logger = logger + @cache_log = {} + + if options.nil? || options.empty? + @load = false + else + @load = true + @parsed_timeframe = parsed_timeframe(options[:timeframe] || '30d') + end + @cache_time = Time.now + + if File.exist?(FILENAME) + @exists = true + contents = File.read(FILENAME) + @cache_log = contents.empty? ? {} : JSON.parse(contents) + else + @exists = false + end + end + + def within_timeframe?(time) + (@parsed_timeframe..@cache_time).cover?(time) + end + + def urls + @cache_log['urls'] || [] + end + + def parsed_timeframe(timeframe) + time, date = timeframe.match(/(\d+)(\D)/).captures + time = time.to_f + case date + when 'M' + time.months.ago + when 'w' + time.weeks.ago + when 'd' + time.days.ago + when 'h' + time.hours.ago else - content = path + fail ArgumentError, "#{date} is not a valid timeframe!" end + end + + def add(url, filenames, status, msg = '') + data = { + :time => @cache_time, + :filenames => filenames, + :status => status, + :message => msg + } + + @cache_log[clean_url(url)] = data + end + + def detect_url_changes(found) + existing_urls = @cache_log.keys.map { |url| clean_url(url) } + found_urls = found.keys.map { |url| clean_url(url) } + + # prepare to add new URLs detected + additions = found.reject do |url, _| + url = clean_url(url) + if existing_urls.include?(url) + true + else + @logger.log :debug, :yellow, "Adding #{url} to cache check" + false + end + end + + new_link_count = additions.length + new_link_text = pluralize(new_link_count, 'link', 'links') + @logger.log :info, :blue, "Adding #{new_link_text} to the cache..." + + # remove from cache URLs that no longer exist + del = 0 + @cache_log.delete_if do |url, _| + url = clean_url(url) + if !found_urls.include?(url) + @logger.log :debug, :yellow, "Removing #{url} from cache check" + del += 1 + true + else + false + end + end + + del_link_text = pluralize(del, 'link', 'links') + @logger.log :info, :blue, "Removing #{del_link_text} from the cache..." + + additions + end + + def write + File.write(FILENAME, @cache_log.to_json) + end + + def load? + @load.nil? + end + + + # FIXME: there seems to be some discrepenacy where Typhoeus occasionally adds + # a trailing slash to URL strings, which causes issues with the cache + def slashless_url(url) + url.chomp('/') + end + + # FIXME: it seems that Typhoeus actually acts on escaped URLs, + # but there's no way to get at that information, and the cache + # stores unescaped URLs. Because of this, some links, such as + # github.com/search/issues?q=is:open+is:issue+fig are not matched + # as github.com/search/issues?q=is%3Aopen+is%3Aissue+fig + def unescape_url(url) + Addressable::URI.unescape(url) + end - Nokogiri::HTML(content) + def clean_url(url) + slashless_url(unescape_url(url)) end - module_function :create_nokogiri end end end diff --git a/lib/html/proofer/url_validator.rb b/lib/html/proofer/url_validator.rb index 2232084a..fdc13805 100644 --- a/lib/html/proofer/url_validator.rb +++ b/lib/html/proofer/url_validator.rb @@ -1,6 +1,7 @@ require 'typhoeus' require 'uri' require_relative './utils' +require_relative './cache' module HTML class Proofer @@ -18,16 +19,40 @@ def initialize(logger, external_urls, options, typhoeus_opts, hydra_opts) @hydra = Typhoeus::Hydra.new(hydra_opts) @typhoeus_opts = typhoeus_opts @external_domain_paths_with_queries = {} + @cache = Cache.new(@logger, @options[:cache]) end def run @iterable_external_urls = remove_query_values - external_link_checker(@iterable_external_urls) + + if @cache.exists && @cache.load + cache_count = @cache.cache_log.length + cache_text = pluralize(cache_count, 'link', 'links') + + logger.log :info, :blue, "Found #{cache_text} in the cache..." + + urls_to_check = @cache.detect_url_changes(@iterable_external_urls) + + @cache.cache_log.each_pair do |url, cache| + if @cache.within_timeframe?(cache['time']) + next if cache['message'].empty? # these were successes to skip + urls_to_check[url] = cache['filenames'] # these are failures to retry + else + urls_to_check[url] = cache['filenames'] # pass or fail, recheck expired links + end + end + + external_link_checker(urls_to_check) + else + external_link_checker(@iterable_external_urls) + end + + @cache.write @failed_tests end def remove_query_values - return if @external_urls.nil? + return nil if @external_urls.nil? iterable_external_urls = @external_urls.dup @external_urls.keys.each do |url| uri = begin @@ -75,14 +100,16 @@ def external_link_checker(external_urls) external_urls = Hash[external_urls.sort] count = external_urls.length - check_text = "#{count} " << (count == 1 ? 'external link' : 'external links') + check_text = pluralize(count, 'external link', 'external links') logger.log :info, :blue, "Checking #{check_text}..." Ethon.logger = logger # log from Typhoeus/Ethon url_processor(external_urls) - logger.log :debug, :yellow, "Running requests for all #{hydra.queued_requests.size} external URLs..." + logger.log :debug, :yellow, "Running requests for:" + logger.log :debug, :yellow, "###\n" + external_urls.keys.join("\n") + "\n###" + hydra.run end @@ -125,14 +152,19 @@ def response_handler(response, filenames) if response_code.between?(200, 299) check_hash_in_2xx_response(href, effective_url, response, filenames) + @cache.add(href, filenames, response_code) elsif response.timed_out? handle_timeout(href, filenames, response_code) + elsif response_code == 0 + handle_failure(href, filenames, response_code) elsif method == :head queue_request(:get, href, filenames) else return if @options[:only_4xx] && !response_code.between?(400, 499) # Received a non-successful http response. - add_external_issue(filenames, "External link #{href} failed: #{response_code} #{response.return_message}", response_code) + msg = "External link #{href} failed: #{response_code} #{response.return_message}" + add_external_issue(filenames, msg, response_code) + @cache.add(href, filenames, response_code, msg) end end @@ -153,12 +185,23 @@ def check_hash_in_2xx_response(href, effective_url, response, filenames) return unless body_doc.xpath(xpath).empty? - add_external_issue filenames, "External link #{href} failed: #{effective_url} exists, but the hash '#{hash}' does not", response.code + msg = "External link #{href} failed: #{effective_url} exists, but the hash '#{hash}' does not" + add_external_issue(filenames, msg, response.code) + @cache.add(href, filenames, response.code, msg) end def handle_timeout(href, filenames, response_code) + msg = "External link #{href} failed: got a time out (response code #{response_code})" + @cache.add(href, filenames, 0, msg) + return if @options[:only_4xx] + add_external_issue(filenames, msg, response_code) + end + + def handle_failure(href, filenames, response_code) + msg = "External link #{href} failed: response code #{response_code} means something's wrong" + @cache.add(href, filenames, 0, msg) return if @options[:only_4xx] - add_external_issue filenames, "External link #{href} failed: got a time out", response_code + add_external_issue(filenames, msg, response_code) end def add_external_issue(filenames, desc, status = nil) diff --git a/lib/html/proofer/utils.rb b/lib/html/proofer/utils.rb index 5476c928..de66ba71 100644 --- a/lib/html/proofer/utils.rb +++ b/lib/html/proofer/utils.rb @@ -3,6 +3,12 @@ module HTML class Proofer module Utils + STORAGE_DIR = File.join('tmp', '.htmlproofer') + + def pluralize(count, single, plural) + "#{count} " << (count == 1 ? single : plural) + end + def create_nokogiri(path) if File.exist? path content = File.open(path).read diff --git a/spec/html/proofer/cache_spec.rb b/spec/html/proofer/cache_spec.rb new file mode 100644 index 00000000..923f6d2a --- /dev/null +++ b/spec/html/proofer/cache_spec.rb @@ -0,0 +1,105 @@ +require 'spec_helper' + +describe 'Cache test' do + + it 'knows how to write to cache' do + stub_const('HTML::Proofer::Cache::FILENAME', "#{FIXTURES_DIR}/cache/.htmlproofer.log") + + brokenLinkExternalFilepath = "#{FIXTURES_DIR}/links/brokenLinkExternal.html" + expect_any_instance_of(HTML::Proofer::Cache).to receive(:write) + run_proofer(brokenLinkExternalFilepath) + + log = read_cache + expect(log.keys.length).to eq(2) + statuses = log.values.map { |h| h['status'] } + expect(statuses.count(200)).to eq(1) + expect(statuses.count(0)).to eq(1) + end + + it 'knows how to load a cache' do + stub_const('HTML::Proofer::Cache::FILENAME', "#{FIXTURES_DIR}/cache/.simple_load.log") + + expect_any_instance_of(HTML::Proofer::Cache).to receive(:write) + expect_any_instance_of(HTML::Proofer::Cache).to receive(:load).once + + brokenLinkExternalFilepath = "#{FIXTURES_DIR}/links/bad_external_links.html" + run_proofer(brokenLinkExternalFilepath, { :cache => { :timeframe => '30d' } }) + end + + it 'fails on an invalid date' do + file = "#{FIXTURES_DIR}/links/brokenLinkExternal.html" + expect { + run_proofer(file, { :cache => { :timeframe => '30x' } }) + }.to raise_error ArgumentError + end + + it 'does not write file if timestamp is within date' do + stub_const('HTML::Proofer::Cache::FILENAME', "#{FIXTURES_DIR}/cache/.within_date.log") + + # this is frozen to within 7 days of the log + new_time = Time.local(2015, 10, 20, 12, 0, 0) + Timecop.freeze(new_time) + + log = read_cache + current_time = log.values.first['time'] + + expect_any_instance_of(HTML::Proofer::Cache).to receive(:write) + run_proofer(['www.github.com'], { :cache => { :timeframe => '30d' } }) + + # note that the timestamp did not change + log = read_cache + new_time = log.values.first['time'] + expect(current_time).to eq(new_time) + + Timecop.return + end + + it 'does write file if timestamp is not within date' do + stub_const('HTML::Proofer::Cache::FILENAME', "#{FIXTURES_DIR}/cache/.not_within_date.log") + + # this is frozen to within 20 days after the log + new_time = Time.local(2014, 10, 21, 12, 0, 0) + Timecop.travel(new_time) + + # since the timestamp changed, we expect an add + expect_any_instance_of(HTML::Proofer::Cache).to receive(:add) + run_proofer(['www.github.com'], { :cache => { :timeframe => '4d' } }) + + Timecop.return + end + + it 'does write file if a new URL is added' do + stub_const('HTML::Proofer::Cache::FILENAME', "#{FIXTURES_DIR}/cache/.new_url.log") + + # this is frozen to within 7 days of the log + new_time = Time.local(2015, 10, 20, 12, 0, 0) + Timecop.freeze(new_time) + + expect_any_instance_of(HTML::Proofer::Cache).to receive(:write) + # we expect a new link to be added, but github.com can stay... + expect_any_instance_of(HTML::Proofer::Cache).to receive(:add).with('www.google.com', nil, 200) + + # ...because it's within the 30d time frame + run_proofer(['www.github.com', 'www.google.com'], { :cache => { :timeframe => '30d' } }) + + Timecop.return + end + + it 'does recheck failures, regardless of cache' do + stub_const('HTML::Proofer::Cache::FILENAME', "#{FIXTURES_DIR}/cache/.recheck_failure.log") + + # this is frozen to within 7 days of the log + new_time = Time.local(2015, 10, 20, 12, 0, 0) + Timecop.freeze(new_time) + + expect_any_instance_of(HTML::Proofer::Cache).to receive(:write) + # we expect the same link to be readded... + expect_any_instance_of(HTML::Proofer::Cache).to receive(:add).with('www.foofoofoo.biz', nil, 0, 'External link www.foofoofoo.biz failed: response code 0 means something\'s wrong') + + # ...even though we are within the 30d time frame, + # because it's a failure + run_proofer(['www.foofoofoo.biz'], { :cache => { :timeframe => '30d' } }) + + Timecop.return + end +end diff --git a/spec/html/proofer/fixtures/cache/.htmlproofer.log b/spec/html/proofer/fixtures/cache/.htmlproofer.log new file mode 100644 index 00000000..c22c14e0 --- /dev/null +++ b/spec/html/proofer/fixtures/cache/.htmlproofer.log @@ -0,0 +1 @@ +{"http://www.google.com/":{"time":"2015-10-27 16:39:14 -0700","filenames":["spec/html/proofer/fixtures/links/brokenLinkExternal.html"],"status":200,"message":""},"http://www.asdo3irj395295jsingrkrg4.com/":{"time":"2015-10-27 16:39:14 -0700","filenames":["spec/html/proofer/fixtures/links/brokenLinkExternal.html"],"status":0,"message":"External link http://www.asdo3irj395295jsingrkrg4.com/ failed: 0 "}} \ No newline at end of file diff --git a/spec/html/proofer/fixtures/cache/.new_url.log b/spec/html/proofer/fixtures/cache/.new_url.log new file mode 100644 index 00000000..d5867777 --- /dev/null +++ b/spec/html/proofer/fixtures/cache/.new_url.log @@ -0,0 +1 @@ +{"www.github.com":{"time":"2015-10-01 16:18:15 -0700","filenames":null,"status":200,"message":""}} diff --git a/spec/html/proofer/fixtures/cache/.not_within_date.log b/spec/html/proofer/fixtures/cache/.not_within_date.log new file mode 100644 index 00000000..31cb822b --- /dev/null +++ b/spec/html/proofer/fixtures/cache/.not_within_date.log @@ -0,0 +1 @@ +{"www.github.com":{"time":"2015-10-01 16:18:15 -0700","filenames":null,"status":200,"message":""}} \ No newline at end of file diff --git a/spec/html/proofer/fixtures/cache/.recheck_failure.log b/spec/html/proofer/fixtures/cache/.recheck_failure.log new file mode 100644 index 00000000..4aa0509e --- /dev/null +++ b/spec/html/proofer/fixtures/cache/.recheck_failure.log @@ -0,0 +1 @@ +{"www.foofoofoo.biz":{"time":"2015-10-20 12:00:00 -0700","filenames":null,"status":0,"message":"External link www.foofoofoo.biz failed: 0 "}} \ No newline at end of file diff --git a/spec/html/proofer/fixtures/cache/.simple_load.log b/spec/html/proofer/fixtures/cache/.simple_load.log new file mode 100644 index 00000000..ed33c482 --- /dev/null +++ b/spec/html/proofer/fixtures/cache/.simple_load.log @@ -0,0 +1 @@ +{"http://www.asdo3irj395295jsingrkrg4.com/":{"time":"2015-10-27 16:39:16 -0700","filenames":["spec/html/proofer/fixtures/links/bad_external_links.html"],"status":0,"message":"External link http://www.asdo3irj395295jsingrkrg4.com/ failed: 0 "},"http://www.google.com/":{"time":"2015-10-27 16:39:16 -0700","filenames":["spec/html/proofer/fixtures/links/bad_external_links.html"],"status":200,"message":""}} \ No newline at end of file diff --git a/spec/html/proofer/fixtures/cache/.within_date.log b/spec/html/proofer/fixtures/cache/.within_date.log new file mode 100644 index 00000000..4c840aaa --- /dev/null +++ b/spec/html/proofer/fixtures/cache/.within_date.log @@ -0,0 +1 @@ +{"www.github.com":{"time":"2015-10-27 12:00:00 -0700","filenames":null,"status":200,"message":""}} diff --git a/spec/html/proofer/fixtures/links/ip_href.html b/spec/html/proofer/fixtures/links/ip_href.html new file mode 100644 index 00000000..117fd9f9 --- /dev/null +++ b/spec/html/proofer/fixtures/links/ip_href.html @@ -0,0 +1,8 @@ +

The container on the Redis server and its volume have now both been moved to the second host, and Flocker has maintained its link to the web application on the first host:

+ + diff --git a/spec/html/proofer/fixtures/vcr_cassettes/links/bad_external_links_html_cache_timeframe_30d_.yml b/spec/html/proofer/fixtures/vcr_cassettes/links/bad_external_links_html_cache_timeframe_30d_.yml new file mode 100644 index 00000000..a87ea68a --- /dev/null +++ b/spec/html/proofer/fixtures/vcr_cassettes/links/bad_external_links_html_cache_timeframe_30d_.yml @@ -0,0 +1,94 @@ +--- +http_interactions: +- request: + method: head + uri: http://www.asdo3irj395295jsingrkrg4.com/ + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/2.5.1; +https://github.com/gjtorikian/html-proofer) + response: + status: + code: 0 + message: + headers: {} + body: + encoding: UTF-8 + string: '' + http_version: + adapter_metadata: + effective_url: http://www.asdo3irj395295jsingrkrg4.com/ + recorded_at: Tue, 27 Oct 2015 23:12:47 GMT +- request: + method: get + uri: http://www.asdo3irj395295jsingrkrg4.com/ + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/2.5.1; +https://github.com/gjtorikian/html-proofer) + response: + status: + code: 0 + message: + headers: {} + body: + encoding: UTF-8 + string: '' + http_version: + adapter_metadata: + effective_url: http://www.asdo3irj395295jsingrkrg4.com/ + recorded_at: Tue, 27 Oct 2015 23:12:48 GMT +- request: + method: head + uri: http://www.google.com/ + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/2.5.1; +https://github.com/gjtorikian/html-proofer) + response: + status: + code: 200 + message: OK + headers: + Date: + - Tue, 27 Oct 2015 23:12:47 GMT + Expires: + - "-1" + Cache-Control: + - private, max-age=0 + Content-Type: + - text/html; charset=UTF-8 + P3P: + - CP="This is not a P3P policy! See http://www.google.com/support/accounts/bin/answer.py?hl=en&answer=151657 + for more info." + Server: + - gws + X-XSS-Protection: + - 1; mode=block + X-Frame-Options: + - SAMEORIGIN + Set-Cookie: + - PREF=ID=1111111111111111:FF=0:TM=1445987567:LM=1445987567:V=1:S=M4Swa5ZPsFl1gVQD; + expires=Thu, 31-Dec-2015 16:02:17 GMT; path=/; domain=.google.com + - NID=72=Wbfc-s9n86mZ0dvAycdk0b_Rm26VC3jmgQHxQ-wyDU4puGlD1WrfQKW1vNMNGlBTjabfd-J5G4qorSuPCL3-3Ur24V7L0L-mSws_F1GjQLPbwr7OyK-10SbKnYX6V-p7pde_ha0KidmFOFqrDeMWK8cXN7FUEKdUP9dqm4UCV58idUbeisYuQRphzGA; + expires=Wed, 27-Apr-2016 23:12:47 GMT; path=/; domain=.google.com; HttpOnly + Transfer-Encoding: + - chunked + Accept-Ranges: + - none + Vary: + - Accept-Encoding + body: + encoding: UTF-8 + string: '' + http_version: '1.1' + adapter_metadata: + effective_url: http://www.google.com/ + recorded_at: Tue, 27 Oct 2015 23:12:48 GMT +recorded_with: VCR 2.9.3 diff --git a/spec/html/proofer/fixtures/vcr_cassettes/links/brokenLinkExternal_html_cache_timeframe_30d_.yml b/spec/html/proofer/fixtures/vcr_cassettes/links/brokenLinkExternal_html_cache_timeframe_30d_.yml new file mode 100644 index 00000000..3e019649 --- /dev/null +++ b/spec/html/proofer/fixtures/vcr_cassettes/links/brokenLinkExternal_html_cache_timeframe_30d_.yml @@ -0,0 +1,94 @@ +--- +http_interactions: +- request: + method: head + uri: http://www.asdo3irj395295jsingrkrg4.com/ + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/2.5.0; +https://github.com/gjtorikian/html-proofer) + response: + status: + code: 0 + message: + headers: {} + body: + encoding: UTF-8 + string: '' + http_version: + adapter_metadata: + effective_url: http://www.asdo3irj395295jsingrkrg4.com/ + recorded_at: Fri, 18 Sep 2015 23:43:03 GMT +- request: + method: get + uri: http://www.asdo3irj395295jsingrkrg4.com/ + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/2.5.0; +https://github.com/gjtorikian/html-proofer) + response: + status: + code: 0 + message: + headers: {} + body: + encoding: UTF-8 + string: '' + http_version: + adapter_metadata: + effective_url: http://www.asdo3irj395295jsingrkrg4.com/ + recorded_at: Fri, 18 Sep 2015 23:43:03 GMT +- request: + method: head + uri: http://www.google.com/ + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/2.5.0; +https://github.com/gjtorikian/html-proofer) + response: + status: + code: 200 + message: OK + headers: + Date: + - Fri, 18 Sep 2015 23:43:03 GMT + Expires: + - "-1" + Cache-Control: + - private, max-age=0 + Content-Type: + - text/html; charset=UTF-8 + P3P: + - CP="This is not a P3P policy! See http://www.google.com/support/accounts/bin/answer.py?hl=en&answer=151657 + for more info." + Server: + - gws + X-XSS-Protection: + - 1; mode=block + X-Frame-Options: + - SAMEORIGIN + Set-Cookie: + - PREF=ID=1111111111111111:FF=0:TM=1442619783:LM=1442619783:V=1:S=vU-QSoS8IfMQqAlh; + expires=Thu, 31-Dec-2015 16:02:17 GMT; path=/; domain=.google.com + - NID=71=2T-FfzrYMytvJPlafkm8vhNnNOIW3IputAEKO6e2e1wPf6P9qODRzUB8UTIAbrMwEu8P2js-UvPjHTItdP9lKOqbvTW12eMECQJW-X8e7ZO3DgNaHw9gZwu7WhfQuiBjDHhqrpPhQqppzBP6Xt1zooAqzQzGsTzr; + expires=Sat, 19-Mar-2016 23:43:03 GMT; path=/; domain=.google.com; HttpOnly + Transfer-Encoding: + - chunked + Accept-Ranges: + - none + Vary: + - Accept-Encoding + body: + encoding: UTF-8 + string: '' + http_version: '1.1' + adapter_metadata: + effective_url: http://www.google.com/ + recorded_at: Fri, 18 Sep 2015 23:43:04 GMT +recorded_with: VCR 2.9.3 diff --git a/spec/html/proofer/fixtures/vcr_cassettes/links/brokenLinkExternal_html_cache_timeframe_30x_.yml b/spec/html/proofer/fixtures/vcr_cassettes/links/brokenLinkExternal_html_cache_timeframe_30x_.yml new file mode 100644 index 00000000..68bde8a7 --- /dev/null +++ b/spec/html/proofer/fixtures/vcr_cassettes/links/brokenLinkExternal_html_cache_timeframe_30x_.yml @@ -0,0 +1,94 @@ +--- +http_interactions: +- request: + method: head + uri: http://www.asdo3irj395295jsingrkrg4.com/ + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/2.5.0; +https://github.com/gjtorikian/html-proofer) + response: + status: + code: 0 + message: + headers: {} + body: + encoding: UTF-8 + string: '' + http_version: + adapter_metadata: + effective_url: http://www.asdo3irj395295jsingrkrg4.com/ + recorded_at: Sun, 04 Oct 2015 20:40:43 GMT +- request: + method: get + uri: http://www.asdo3irj395295jsingrkrg4.com/ + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/2.5.0; +https://github.com/gjtorikian/html-proofer) + response: + status: + code: 0 + message: + headers: {} + body: + encoding: UTF-8 + string: '' + http_version: + adapter_metadata: + effective_url: http://www.asdo3irj395295jsingrkrg4.com/ + recorded_at: Sun, 04 Oct 2015 20:40:43 GMT +- request: + method: head + uri: http://www.google.com/ + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/2.5.0; +https://github.com/gjtorikian/html-proofer) + response: + status: + code: 200 + message: OK + headers: + Date: + - Sun, 04 Oct 2015 20:40:43 GMT + Expires: + - "-1" + Cache-Control: + - private, max-age=0 + Content-Type: + - text/html; charset=UTF-8 + P3P: + - CP="This is not a P3P policy! See http://www.google.com/support/accounts/bin/answer.py?hl=en&answer=151657 + for more info." + Server: + - gws + X-XSS-Protection: + - 1; mode=block + X-Frame-Options: + - SAMEORIGIN + Set-Cookie: + - PREF=ID=1111111111111111:FF=0:TM=1443991243:LM=1443991243:V=1:S=GCTrNO7FO2t_owmK; + expires=Thu, 31-Dec-2015 16:02:17 GMT; path=/; domain=.google.com + - NID=72=W7pMu_5D6ZBAFjvAX-sNT_l9XPapZtfD3F7pIkXK2zZSGCjn1kRQZFF7V9vZZdbqrxSnbEuwh57-hS3RFyHc8An-1xfeAoyq1KMEDoAiFGjMJwlYHSJNwticBdarzCqsjXTB7Po_8SIkPqINRYilIIDFZ1NNyvO6; + expires=Mon, 04-Apr-2016 20:40:43 GMT; path=/; domain=.google.com; HttpOnly + Transfer-Encoding: + - chunked + Accept-Ranges: + - none + Vary: + - Accept-Encoding + body: + encoding: UTF-8 + string: '' + http_version: '1.1' + adapter_metadata: + effective_url: http://www.google.com/ + recorded_at: Sun, 04 Oct 2015 20:40:43 GMT +recorded_with: VCR 2.9.3 diff --git a/spec/html/proofer/fixtures/vcr_cassettes/links/ip_href_html.yml b/spec/html/proofer/fixtures/vcr_cassettes/links/ip_href_html.yml new file mode 100644 index 00000000..f8b96131 --- /dev/null +++ b/spec/html/proofer/fixtures/vcr_cassettes/links/ip_href_html.yml @@ -0,0 +1,87 @@ +--- +http_interactions: +- request: + method: head + uri: http://1.2.3.4/ + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/2.5.2; +https://github.com/gjtorikian/html-proofer) + response: + status: + code: 0 + message: + headers: {} + body: + encoding: UTF-8 + string: '' + http_version: + adapter_metadata: + effective_url: http://1.2.3.4/ + recorded_at: Sun, 22 Nov 2015 17:03:02 GMT +- request: + method: head + uri: http://5.6.7.8/ + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/2.5.2; +https://github.com/gjtorikian/html-proofer) + response: + status: + code: 0 + message: + headers: {} + body: + encoding: UTF-8 + string: '' + http_version: + adapter_metadata: + effective_url: http://5.6.7.8/ + recorded_at: Sun, 22 Nov 2015 17:03:02 GMT +- request: + method: get + uri: http://1.2.3.4/ + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/2.5.2; +https://github.com/gjtorikian/html-proofer) + response: + status: + code: 0 + message: + headers: {} + body: + encoding: UTF-8 + string: '' + http_version: + adapter_metadata: + effective_url: http://1.2.3.4/ + recorded_at: Sun, 22 Nov 2015 17:04:42 GMT +- request: + method: get + uri: http://5.6.7.8/ + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/2.5.2; +https://github.com/gjtorikian/html-proofer) + response: + status: + code: 0 + message: + headers: {} + body: + encoding: UTF-8 + string: '' + http_version: + adapter_metadata: + effective_url: http://5.6.7.8/ + recorded_at: Sun, 22 Nov 2015 17:04:42 GMT +recorded_with: VCR 2.9.3 diff --git a/spec/html/proofer/fixtures/vcr_cassettes/www_foofoofoo_biz.yml b/spec/html/proofer/fixtures/vcr_cassettes/www_foofoofoo_biz.yml new file mode 100644 index 00000000..301af2b8 --- /dev/null +++ b/spec/html/proofer/fixtures/vcr_cassettes/www_foofoofoo_biz.yml @@ -0,0 +1,45 @@ +--- +http_interactions: +- request: + method: head + uri: www.foofoofoo.biz + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/2.5.1; +https://github.com/gjtorikian/html-proofer) + response: + status: + code: 0 + message: + headers: {} + body: + encoding: UTF-8 + string: '' + http_version: + adapter_metadata: + effective_url: HTTP://www.foofoofoo.biz/ + recorded_at: Tue, 20 Oct 2015 19:00:00 GMT +- request: + method: get + uri: www.foofoofoo.biz + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/2.5.1; +https://github.com/gjtorikian/html-proofer) + response: + status: + code: 0 + message: + headers: {} + body: + encoding: UTF-8 + string: '' + http_version: + adapter_metadata: + effective_url: HTTP://www.foofoofoo.biz/ + recorded_at: Tue, 20 Oct 2015 19:00:00 GMT +recorded_with: VCR 2.9.3 diff --git a/spec/html/proofer/fixtures/vcr_cassettes/www_foofoofoo_biz_cache_timeframe_30d_.yml b/spec/html/proofer/fixtures/vcr_cassettes/www_foofoofoo_biz_cache_timeframe_30d_.yml new file mode 100644 index 00000000..301af2b8 --- /dev/null +++ b/spec/html/proofer/fixtures/vcr_cassettes/www_foofoofoo_biz_cache_timeframe_30d_.yml @@ -0,0 +1,45 @@ +--- +http_interactions: +- request: + method: head + uri: www.foofoofoo.biz + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/2.5.1; +https://github.com/gjtorikian/html-proofer) + response: + status: + code: 0 + message: + headers: {} + body: + encoding: UTF-8 + string: '' + http_version: + adapter_metadata: + effective_url: HTTP://www.foofoofoo.biz/ + recorded_at: Tue, 20 Oct 2015 19:00:00 GMT +- request: + method: get + uri: www.foofoofoo.biz + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/2.5.1; +https://github.com/gjtorikian/html-proofer) + response: + status: + code: 0 + message: + headers: {} + body: + encoding: UTF-8 + string: '' + http_version: + adapter_metadata: + effective_url: HTTP://www.foofoofoo.biz/ + recorded_at: Tue, 20 Oct 2015 19:00:00 GMT +recorded_with: VCR 2.9.3 diff --git a/spec/html/proofer/fixtures/vcr_cassettes/www_github_com.yml b/spec/html/proofer/fixtures/vcr_cassettes/www_github_com.yml new file mode 100644 index 00000000..e9b98303 --- /dev/null +++ b/spec/html/proofer/fixtures/vcr_cassettes/www_github_com.yml @@ -0,0 +1,74 @@ +--- +http_interactions: +- request: + method: head + uri: www.github.com + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/2.5.1; +https://github.com/gjtorikian/html-proofer) + response: + status: + code: 200 + message: OK + headers: + Server: + - GitHub.com + Date: + - Tue, 27 Oct 2015 23:16:46 GMT + Content-Type: + - text/html; charset=utf-8 + Status: + - 200 OK + Content-Security-Policy: + - 'default-src *; script-src assets-cdn.github.com; object-src assets-cdn.github.com; + style-src ''self'' ''unsafe-inline'' ''unsafe-eval'' assets-cdn.github.com; + img-src ''self'' data: assets-cdn.github.com identicons.github.com www.google-analytics.com + checkout.paypal.com collector.githubapp.com *.githubusercontent.com *.gravatar.com + *.wp.com; media-src ''none''; frame-src ''self'' render.githubusercontent.com + gist.github.com www.youtube.com player.vimeo.com checkout.paypal.com; font-src + assets-cdn.github.com; connect-src ''self'' live.github.com wss://live.github.com + uploads.github.com status.github.com api.github.com www.google-analytics.com + api.braintreegateway.com client-analytics.braintreegateway.com github-cloud.s3.amazonaws.com; + base-uri ''self''; form-action ''self'' github.com gist.github.com' + Public-Key-Pins: + - max-age=300; pin-sha256="WoiWRyIOVNa9ihaBciRSC7XHjliYS9VwUGOIud4PB18="; pin-sha256="JbQbUG5JMJUoI6brnx0x3vZF6jilxsapbXGVfjhN8Fg="; + includeSubDomains + Cache-Control: + - no-cache + Vary: + - X-PJAX + - Accept-Encoding + X-UA-Compatible: + - IE=Edge,chrome=1 + Set-Cookie: + - logged_in=no; domain=.github.com; path=/; expires=Sat, 27 Oct 2035 23:16:46 + -0000; secure; HttpOnly + - _gh_sess=eyJzZXNzaW9uX2lkIjoiMDI5YTg0YzZlOWRmZWU4NjM4MTA1YTNmYmIwMGE4NzAiLCJfY3NyZl90b2tlbiI6ImNIUDlFRE1pUEw1TEJnY1pVMXlyWTJBVjZINFhIWVpnTXZZenFsMzN1MUk9In0%3D--0fd8bdcbb7d04300f5cd681832d3bed6637bbf3c; + path=/; secure; HttpOnly + X-Request-Id: + - f2b6f9fcf7c16ec3bce30f28a0256d8d + X-Runtime: + - '0.011330' + Strict-Transport-Security: + - max-age=31536000; includeSubdomains; preload + X-Content-Type-Options: + - nosniff + X-XSS-Protection: + - 1; mode=block + X-Frame-Options: + - deny + X-Served-By: + - 53e13e5d66f560f3e2b04e74a099de0d + X-GitHub-Request-Id: + - 490F8465:3709:A8ED690:563005DD + body: + encoding: UTF-8 + string: '' + http_version: '1.1' + adapter_metadata: + effective_url: https://github.com/ + recorded_at: Tue, 27 Oct 2015 23:16:46 GMT +recorded_with: VCR 2.9.3 diff --git a/spec/html/proofer/fixtures/vcr_cassettes/www_github_com_cache_timeframe_30d_.yml b/spec/html/proofer/fixtures/vcr_cassettes/www_github_com_cache_timeframe_30d_.yml new file mode 100644 index 00000000..381504c7 --- /dev/null +++ b/spec/html/proofer/fixtures/vcr_cassettes/www_github_com_cache_timeframe_30d_.yml @@ -0,0 +1,74 @@ +--- +http_interactions: +- request: + method: head + uri: www.github.com + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/2.5.1; +https://github.com/gjtorikian/html-proofer) + response: + status: + code: 200 + message: OK + headers: + Server: + - GitHub.com + Date: + - Tue, 27 Oct 2015 23:39:16 GMT + Content-Type: + - text/html; charset=utf-8 + Status: + - 200 OK + Content-Security-Policy: + - 'default-src *; script-src assets-cdn.github.com; object-src assets-cdn.github.com; + style-src ''self'' ''unsafe-inline'' ''unsafe-eval'' assets-cdn.github.com; + img-src ''self'' data: assets-cdn.github.com identicons.github.com www.google-analytics.com + checkout.paypal.com collector.githubapp.com *.githubusercontent.com *.gravatar.com + *.wp.com; media-src ''none''; frame-src ''self'' render.githubusercontent.com + gist.github.com www.youtube.com player.vimeo.com checkout.paypal.com; font-src + assets-cdn.github.com; connect-src ''self'' live.github.com wss://live.github.com + uploads.github.com status.github.com api.github.com www.google-analytics.com + api.braintreegateway.com client-analytics.braintreegateway.com github-cloud.s3.amazonaws.com; + base-uri ''self''; form-action ''self'' github.com gist.github.com' + Public-Key-Pins: + - max-age=300; pin-sha256="WoiWRyIOVNa9ihaBciRSC7XHjliYS9VwUGOIud4PB18="; pin-sha256="JbQbUG5JMJUoI6brnx0x3vZF6jilxsapbXGVfjhN8Fg="; + includeSubDomains + Cache-Control: + - no-cache + Vary: + - X-PJAX + - Accept-Encoding + X-UA-Compatible: + - IE=Edge,chrome=1 + Set-Cookie: + - logged_in=no; domain=.github.com; path=/; expires=Sat, 27 Oct 2035 23:39:16 + -0000; secure; HttpOnly + - _gh_sess=eyJzZXNzaW9uX2lkIjoiNTBmYTFmOTMyY2RiM2Y3NGFmYzczNzJhMjY3ZTViMjEiLCJfY3NyZl90b2tlbiI6IlFhaUs2M0tCTG9FcnJ0eGhDcHJFaFNIS1hMYUpWekN2ZFp4d0p3L3htd2s9In0%3D--b861f6d060912f545a23e7c62bcc7eb25ff6b8ae; + path=/; secure; HttpOnly + X-Request-Id: + - ad7807a878142492cf8861ec40f434f3 + X-Runtime: + - '0.011707' + Strict-Transport-Security: + - max-age=31536000; includeSubdomains; preload + X-Content-Type-Options: + - nosniff + X-XSS-Protection: + - 1; mode=block + X-Frame-Options: + - deny + X-Served-By: + - 353c3e3783cd5fa717715b8bdf83c23b + X-GitHub-Request-Id: + - 490F8465:798A:88B37B4:56300B23 + body: + encoding: UTF-8 + string: '' + http_version: '1.1' + adapter_metadata: + effective_url: https://github.com/ + recorded_at: Tue, 20 Oct 2015 19:00:00 GMT +recorded_with: VCR 2.9.3 diff --git a/spec/html/proofer/fixtures/vcr_cassettes/www_github_com_cache_timeframe_4d_.yml b/spec/html/proofer/fixtures/vcr_cassettes/www_github_com_cache_timeframe_4d_.yml new file mode 100644 index 00000000..35e99dbe --- /dev/null +++ b/spec/html/proofer/fixtures/vcr_cassettes/www_github_com_cache_timeframe_4d_.yml @@ -0,0 +1,74 @@ +--- +http_interactions: +- request: + method: head + uri: www.github.com + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/2.5.1; +https://github.com/gjtorikian/html-proofer) + response: + status: + code: 200 + message: OK + headers: + Server: + - GitHub.com + Date: + - Tue, 27 Oct 2015 23:33:04 GMT + Content-Type: + - text/html; charset=utf-8 + Status: + - 200 OK + Content-Security-Policy: + - 'default-src *; script-src assets-cdn.github.com; object-src assets-cdn.github.com; + style-src ''self'' ''unsafe-inline'' ''unsafe-eval'' assets-cdn.github.com; + img-src ''self'' data: assets-cdn.github.com identicons.github.com www.google-analytics.com + checkout.paypal.com collector.githubapp.com *.githubusercontent.com *.gravatar.com + *.wp.com; media-src ''none''; frame-src ''self'' render.githubusercontent.com + gist.github.com www.youtube.com player.vimeo.com checkout.paypal.com; font-src + assets-cdn.github.com; connect-src ''self'' live.github.com wss://live.github.com + uploads.github.com status.github.com api.github.com www.google-analytics.com + api.braintreegateway.com client-analytics.braintreegateway.com github-cloud.s3.amazonaws.com; + base-uri ''self''; form-action ''self'' github.com gist.github.com' + Public-Key-Pins: + - max-age=300; pin-sha256="WoiWRyIOVNa9ihaBciRSC7XHjliYS9VwUGOIud4PB18="; pin-sha256="JbQbUG5JMJUoI6brnx0x3vZF6jilxsapbXGVfjhN8Fg="; + includeSubDomains + Cache-Control: + - no-cache + Vary: + - X-PJAX + - Accept-Encoding + X-UA-Compatible: + - IE=Edge,chrome=1 + Set-Cookie: + - logged_in=no; domain=.github.com; path=/; expires=Sat, 27 Oct 2035 23:33:04 + -0000; secure; HttpOnly + - _gh_sess=eyJzZXNzaW9uX2lkIjoiMTkxMzQ4MjZhNmQ2NmJhZDNiOGNiZGQ3NTMxYzBhZDQiLCJfY3NyZl90b2tlbiI6InlGRGYxYXRwOCtuTTJBNHBkcHhnV3dlUmJVaTZvVGx4UkVMR0dzSHFoYUE9In0%3D--f4c33cd20d082d0a6efbb02dff998062aca6a8db; + path=/; secure; HttpOnly + X-Request-Id: + - ebc8f05f55d8fe9553937be4cee36ba2 + X-Runtime: + - '0.010439' + Strict-Transport-Security: + - max-age=31536000; includeSubdomains; preload + X-Content-Type-Options: + - nosniff + X-XSS-Protection: + - 1; mode=block + X-Frame-Options: + - deny + X-Served-By: + - 63914e33d55e1647962cf498030a7c16 + X-GitHub-Request-Id: + - 490F8465:7D72:6A5BF47:563009AF + body: + encoding: UTF-8 + string: '' + http_version: '1.1' + adapter_metadata: + effective_url: https://github.com/ + recorded_at: Fri, 10 Oct 2014 19:00:01 GMT +recorded_with: VCR 2.9.3 diff --git a/spec/html/proofer/fixtures/vcr_cassettes/www_github_com_www_google_com.yml b/spec/html/proofer/fixtures/vcr_cassettes/www_github_com_www_google_com.yml new file mode 100644 index 00000000..97dcc35b --- /dev/null +++ b/spec/html/proofer/fixtures/vcr_cassettes/www_github_com_www_google_com.yml @@ -0,0 +1,123 @@ +--- +http_interactions: +- request: + method: head + uri: www.google.com + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/2.5.1; +https://github.com/gjtorikian/html-proofer) + response: + status: + code: 200 + message: OK + headers: + Date: + - Tue, 27 Oct 2015 23:45:45 GMT + Expires: + - "-1" + Cache-Control: + - private, max-age=0 + Content-Type: + - text/html; charset=UTF-8 + P3P: + - CP="This is not a P3P policy! See http://www.google.com/support/accounts/bin/answer.py?hl=en&answer=151657 + for more info." + Server: + - gws + X-XSS-Protection: + - 1; mode=block + X-Frame-Options: + - SAMEORIGIN + Set-Cookie: + - PREF=ID=1111111111111111:FF=0:TM=1445989545:LM=1445989545:V=1:S=ezJJ4DJwnlbnVVYE; + expires=Thu, 31-Dec-2015 16:02:17 GMT; path=/; domain=.google.com + - NID=72=PQzL7mmUIdg1rjMewNJvMIhyNDYynWTBbrfJt9f8yrk4kJD8IBT5QVzI8wyUyMh31rrC50xD8K-I0Y_t5Oxur32IVUnVNBJ9B_iq_us63RN17aT5vIOmqf9AHGtILumCDgcqoTQZ5vfv2CdC1h4ZyeSExUG5i_1FInlZLJAqWSVKUNox7Nt3q2IHRg0; + expires=Wed, 27-Apr-2016 23:45:45 GMT; path=/; domain=.google.com; HttpOnly + Transfer-Encoding: + - chunked + Accept-Ranges: + - none + Vary: + - Accept-Encoding + body: + encoding: UTF-8 + string: '' + http_version: '1.1' + adapter_metadata: + effective_url: HTTP://www.google.com/ + recorded_at: Tue, 27 Oct 2015 23:45:45 GMT +- request: + method: head + uri: www.github.com + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/2.5.1; +https://github.com/gjtorikian/html-proofer) + response: + status: + code: 200 + message: OK + headers: + Server: + - GitHub.com + Date: + - Tue, 27 Oct 2015 23:45:58 GMT + Content-Type: + - text/html; charset=utf-8 + Status: + - 200 OK + Content-Security-Policy: + - 'default-src *; script-src assets-cdn.github.com; object-src assets-cdn.github.com; + style-src ''self'' ''unsafe-inline'' ''unsafe-eval'' assets-cdn.github.com; + img-src ''self'' data: assets-cdn.github.com identicons.github.com www.google-analytics.com + checkout.paypal.com collector.githubapp.com *.githubusercontent.com *.gravatar.com + *.wp.com; media-src ''none''; frame-src ''self'' render.githubusercontent.com + gist.github.com www.youtube.com player.vimeo.com checkout.paypal.com; font-src + assets-cdn.github.com; connect-src ''self'' live.github.com wss://live.github.com + uploads.github.com status.github.com api.github.com www.google-analytics.com + api.braintreegateway.com client-analytics.braintreegateway.com github-cloud.s3.amazonaws.com; + base-uri ''self''; form-action ''self'' github.com gist.github.com' + Public-Key-Pins: + - max-age=300; pin-sha256="WoiWRyIOVNa9ihaBciRSC7XHjliYS9VwUGOIud4PB18="; pin-sha256="JbQbUG5JMJUoI6brnx0x3vZF6jilxsapbXGVfjhN8Fg="; + includeSubDomains + Cache-Control: + - no-cache + Vary: + - X-PJAX + - Accept-Encoding + X-UA-Compatible: + - IE=Edge,chrome=1 + Set-Cookie: + - logged_in=no; domain=.github.com; path=/; expires=Sat, 27 Oct 2035 23:45:58 + -0000; secure; HttpOnly + - _gh_sess=eyJzZXNzaW9uX2lkIjoiZjg2NTc3MDdmZGFmZmVlMmY0MzBjM2Q4YjZlNWM4NmQiLCJfY3NyZl90b2tlbiI6IlVUVzVNU2c1eFRBRDZNTjFmYWtaV1BEcVNJbUhtVStBdVptZVBtejFrK3c9In0%3D--c4a32d61f67922d7e138e366139eaa0f943a6005; + path=/; secure; HttpOnly + X-Request-Id: + - 5fcecc1ed71d320ba10c3127658b714d + X-Runtime: + - '0.008166' + Strict-Transport-Security: + - max-age=31536000; includeSubdomains; preload + X-Content-Type-Options: + - nosniff + X-XSS-Protection: + - 1; mode=block + X-Frame-Options: + - deny + X-Served-By: + - ef97014f01ea59c1ef337fe51a4d0331 + X-GitHub-Request-Id: + - 490F8465:3709:A9BE601:56300CB5 + body: + encoding: UTF-8 + string: '' + http_version: '1.1' + adapter_metadata: + effective_url: https://github.com/ + recorded_at: Tue, 27 Oct 2015 23:45:58 GMT +recorded_with: VCR 2.9.3 diff --git a/spec/html/proofer/fixtures/vcr_cassettes/www_github_com_www_google_com_cache_timeframe_30d_.yml b/spec/html/proofer/fixtures/vcr_cassettes/www_github_com_www_google_com_cache_timeframe_30d_.yml new file mode 100644 index 00000000..ccdf9275 --- /dev/null +++ b/spec/html/proofer/fixtures/vcr_cassettes/www_github_com_www_google_com_cache_timeframe_30d_.yml @@ -0,0 +1,52 @@ +--- +http_interactions: +- request: + method: head + uri: www.google.com + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Mozilla/5.0 (compatible; HTML Proofer/2.5.1; +https://github.com/gjtorikian/html-proofer) + response: + status: + code: 200 + message: OK + headers: + Date: + - Tue, 27 Oct 2015 23:44:27 GMT + Expires: + - "-1" + Cache-Control: + - private, max-age=0 + Content-Type: + - text/html; charset=UTF-8 + P3P: + - CP="This is not a P3P policy! See http://www.google.com/support/accounts/bin/answer.py?hl=en&answer=151657 + for more info." + Server: + - gws + X-XSS-Protection: + - 1; mode=block + X-Frame-Options: + - SAMEORIGIN + Set-Cookie: + - PREF=ID=1111111111111111:FF=0:TM=1445989467:LM=1445989467:V=1:S=8LF-R4atOWdok09c; + expires=Thu, 31-Dec-2015 16:02:17 GMT; path=/; domain=.google.com + - NID=72=nHOu26P2ycup_mYHNAA8BCWwS6lorq6UlXns0hDKfkm0UlKbFjcQ5ACsgVQJQ8PLAInKhiCE_1AhRa2W4fsOntGp1sP3mYdx1qPsuxQDfRjKjSh88sENIDHSrYDrsjt24SJXR9_Slm9XlMDnixTgm27Yp51zWaIMaJKnUNVyr9hRd8HQSuv7jpC8GA; + expires=Wed, 27-Apr-2016 23:44:27 GMT; path=/; domain=.google.com; HttpOnly + Transfer-Encoding: + - chunked + Accept-Ranges: + - none + Vary: + - Accept-Encoding + body: + encoding: UTF-8 + string: '' + http_version: '1.1' + adapter_metadata: + effective_url: HTTP://www.google.com/ + recorded_at: Tue, 20 Oct 2015 19:00:00 GMT +recorded_with: VCR 2.9.3 diff --git a/spec/html/proofer/images_spec.rb b/spec/html/proofer/images_spec.rb index c1e7ca4c..004ed02f 100644 --- a/spec/html/proofer/images_spec.rb +++ b/spec/html/proofer/images_spec.rb @@ -35,7 +35,7 @@ it 'fails for missing external images' do externalImageFilepath = "#{FIXTURES_DIR}/images/missingImageExternal.html" proofer = run_proofer(externalImageFilepath) - expect(proofer.failed_tests.first).to match(/failed: 0/) + expect(proofer.failed_tests.first).to match(/failed: response code 0/) end it 'fails for missing internal images' do @@ -105,7 +105,7 @@ it 'properly ignores missing alt tags, but not all URLs, when asked' do ignorableLinks = "#{FIXTURES_DIR}/images/ignoreAltButNotLink.html" proofer = run_proofer(ignorableLinks, {:alt_ignore => [/.*/]}) - expect(proofer.failed_tests.first).to match(/failed: 0/) + expect(proofer.failed_tests.first).to match(/failed: response code 0/) expect(proofer.failed_tests.first).to_not match /does not have an alt attribute/ end diff --git a/spec/html/proofer/links_spec.rb b/spec/html/proofer/links_spec.rb index 5b2c4592..65dc226c 100644 --- a/spec/html/proofer/links_spec.rb +++ b/spec/html/proofer/links_spec.rb @@ -42,7 +42,7 @@ it 'fails for broken external links' do brokenLinkExternalFilepath = "#{FIXTURES_DIR}/links/brokenLinkExternal.html" proofer = run_proofer(brokenLinkExternalFilepath) - expect(proofer.failed_tests.first).to match(/failed: 0/) + expect(proofer.failed_tests.first).to match(/failed: response code 0/) end it 'passes for different filename without option' do @@ -55,7 +55,7 @@ options = { :ext => '.foo' } brokenLinkExternalFilepath = "#{FIXTURES_DIR}/links/file.foo" proofer = run_proofer(brokenLinkExternalFilepath, options) - expect(proofer.failed_tests.first).to match(/failed: 0/) + expect(proofer.failed_tests.first).to match(/failed: response code 0/) end it 'fails for broken internal links' do @@ -199,7 +199,7 @@ it 'fails for invalid links missing the protocol' do missingProtocolLink = "#{FIXTURES_DIR}/links/link_missing_protocol_invalid.html" proofer = run_proofer(missingProtocolLink) - expect(proofer.failed_tests.first).to match(/failed: 0/) + expect(proofer.failed_tests.first).to match(/failed: response code 0/) end it 'works for valid href within link elements' do @@ -236,7 +236,7 @@ it 'works for array of links' do proofer = run_proofer(['www.github.com', 'foofoofoo.biz']) - expect(proofer.failed_tests.first).to match(/failed: 0/) + expect(proofer.failed_tests.first).to match(/failed: response code 0/) end it 'works for broken anchors within pre' do @@ -410,4 +410,9 @@ expect(proofer.failed_tests.first).to match(/linking to internal hash # that does not exist/) end + it 'fails for broken IP address links' do + hash_href = "#{FIXTURES_DIR}/links/ip_href.html" + proofer = run_proofer(hash_href) + expect(proofer.failed_tests.first).to match(/response code 0/) + end end diff --git a/spec/html/proofer/scripts_spec.rb b/spec/html/proofer/scripts_spec.rb index 5379bfcd..8c70994d 100644 --- a/spec/html/proofer/scripts_spec.rb +++ b/spec/html/proofer/scripts_spec.rb @@ -5,7 +5,7 @@ it 'fails for broken external src' do file = "#{FIXTURES_DIR}/scripts/script_broken_external.html" proofer = run_proofer(file) - expect(proofer.failed_tests.first).to match(/failed: 0/) + expect(proofer.failed_tests.first).to match(/failed: response code 0/) end it 'works for valid internal src' do diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index b7a59a28..ca7bbe75 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,5 +1,6 @@ require 'bundler/setup' require 'vcr' +require 'timecop' require_relative '../lib/html/proofer' FIXTURES_DIR = 'spec/html/proofer/fixtures' @@ -22,12 +23,12 @@ def capture_stderr(*) original_stderr = $stderr original_stdout = $stdout $stderr = fake_err = StringIO.new - $stdout = fake_out = StringIO.new + $stdout = fake_out = StringIO.new unless ENV['NOISE'] begin yield rescue RuntimeError ensure - $stderr = original_stderr + $stderr = original_stderr unless ENV['NOISE'] $stdout = original_stdout end fake_err.string @@ -59,6 +60,14 @@ def make_bin(cmd, path=nil) `bin/htmlproof #{cmd} #{path}` end +def delete_cache + File.delete(HTML::Proofer::Cache::FILENAME) if File.exist?(HTML::Proofer::Cache::FILENAME) +end + +def read_cache + JSON.parse File.read(HTML::Proofer::Cache::FILENAME) +end + def make_cassette_name(file, opts) filename = if file.is_a? Array file.join('_')