Support for non-UTF-8 encoded URLs

lib/html/proofer/url_validator.rb

@@ -48,6 +48,7 @@ def external_link_checker(external_urls)
 
       def url_processor(external_urls)
         external_urls.each_pair do |href, filenames|
+          href = clean_url(href)
           if hash?(href) && @options[:check_external_hash]
             queue_request(:get, href, filenames)
           else
@@ -56,6 +57,10 @@ def url_processor(external_urls)
         end
       end
 
+      def clean_url(href)
+        Addressable::URI.parse(href).normalize
+      end
+
       def queue_request(method, href, filenames)
         request = Typhoeus::Request.new(href, @typhoeus_opts.merge({ :method => method }))
         request.on_complete { |response| response_handler(response, filenames) }
@@ -67,7 +72,6 @@ def response_handler(response, filenames)
         href = response.request.base_url
         method = response.request.options[:method]
         response_code = response.code
-
         debug_msg = "Received a #{response_code} for #{href}"
         debug_msg << " in #{filenames.join(' ')}" unless filenames.nil?
         logger.log :debug, :yellow, debug_msg

spec/html/proofer/fixtures/links/utf8Link.html

@@ -0,0 +1,2 @@
+<a href="http://ben.balter.com/2014/09/29/source-disclosed-≠-open-source/">utf8</a>
+<a href="http://ben.balter.com/2014/09/29/source-disclosed-%E2%89%A0-open-source/">another utf8</a>

spec/html/proofer/links_spec.rb

@@ -326,4 +326,10 @@
     proofer = make_proofer(fixture)
     expect(proofer.failed_tests).to eq []
   end
+
+  it 'passes for external UTF-8 links' do
+    fixture = "#{FIXTURES_DIR}/links/utf8Link.html"
+    proofer = make_proofer(fixture)
+    expect(proofer.failed_tests).to eq []
+  end
 end

spec/html/proofer/scripts_spec.rb

@@ -5,7 +5,7 @@
   it 'fails for broken external src' do
     file = "#{FIXTURES_DIR}/scripts/script_broken_external.html"
     proofer = make_proofer(file)
-    expect(proofer.failed_tests.first).to match(%r{External link http://www.asdo3IRJ395295jsingrkrg4.com/asdo3IRJ.js failed: 0 Couldn't resolve host name})
+    expect(proofer.failed_tests.first).to match(/failed: 0 Couldn't resolve host name/)
   end
 
   it 'works for valid internal src' do