Add support for ABP filter redirects as mock responses

DEPS

@@ -1,7 +1,7 @@
 use_relative_paths = True
 
 deps = {
-  "vendor/adblock_rust_ffi": "https://github.com/brave/adblock-rust-ffi.git@2360813922ceed63e3798737a268858c11e24d80",
+  "vendor/adblock_rust_ffi": "https://github.com/brave/adblock-rust-ffi.git@89127a30655eaf54cf73794309846084ea8b91b9",
   "vendor/autoplay-whitelist": "https://github.com/brave/autoplay-whitelist.git@ea527a4d36051daedb34421e129c98eda06cb5d3",
   "vendor/extension-whitelist": "https://github.com/brave/extension-whitelist.git@7843f62e26a23c51336330e220e9d7992680aae9",
   "vendor/hashset-cpp": "https://github.com/brave/hashset-cpp.git@6eab0271d014ff09bd9f38abe1e0c117e13e9aa9",

app/brave_generated_resources.grd

@@ -649,10 +649,5 @@ By installing this extension, you are agreeing to the Google Widevine Terms of U
         You can read <ph name="BEGIN_LINK_BUILD_INSTRUCTIONS">&lt;a target="_blank" href="$3"&gt;</ph>instructions on how to download and build for yourself<ph name="END_LINK_BUILD_INSTRUCTIONS">&lt;/a&gt;</ph> the specific <ph name="BEGIN_LINK_SOURCE_CODE">&lt;a target="_blank" href="$4"&gt;</ph>source code used to create this copy<ph name="END_LINK_SOURCE_CODE">&lt;/a&gt;</ph>.
       </message>
     </messages>
-    <includes>
-      <include name="IDR_BRAVE_GOOGLE_ANALYTICS_POLYFILL" file="resources/js/google_analytics_polyfill.js" type="BINDATA" />
-      <include name="IDR_BRAVE_TAG_SERVICES_POLYFILL" file="resources/js/tag_services_polyfill.js" type="BINDATA" />
-      <include name="IDR_BRAVE_TAG_MANAGER_POLYFILL" file="resources/js/tag_manager_polyfill.js" type="BINDATA" />
-    </includes>
   </release>
 </grit>

browser/net/brave_ad_block_tp_network_delegate_helper.cc

@@ -29,98 +29,27 @@ using content::ResourceType;
 
 namespace brave {
 
-std::string GetGoogleAnalyticsPolyfillJS() {
-  static std::string base64_output;
-  if (base64_output.length() != 0)  {
-    return base64_output;
-  }
-  std::string str = ui::ResourceBundle::GetSharedInstance().GetRawDataResource(
-      IDR_BRAVE_GOOGLE_ANALYTICS_POLYFILL).as_string();
-  Base64UrlEncode(str, base::Base64UrlEncodePolicy::OMIT_PADDING,
-      &base64_output);
-  base64_output = std::string(kJSDataURLPrefix) + base64_output;
-  return base64_output;
-}
-
-std::string GetGoogleTagManagerPolyfillJS() {
-  static std::string base64_output;
-  if (base64_output.length() != 0)  {
-    return base64_output;
-  }
-  std::string str = ui::ResourceBundle::GetSharedInstance().GetRawDataResource(
-      IDR_BRAVE_TAG_MANAGER_POLYFILL).as_string();
-  Base64UrlEncode(str, base::Base64UrlEncodePolicy::OMIT_PADDING,
-      &base64_output);
-  base64_output = std::string(kJSDataURLPrefix) + base64_output;
-  return base64_output;
-}
-
-std::string GetGoogleTagServicesPolyfillJS() {
-  static std::string base64_output;
-  if (base64_output.length() != 0)  {
-    return base64_output;
-  }
-  std::string str = ui::ResourceBundle::GetSharedInstance().GetRawDataResource(
-    IDR_BRAVE_TAG_SERVICES_POLYFILL).as_string();
-  Base64UrlEncode(str, base::Base64UrlEncodePolicy::OMIT_PADDING,
-      &base64_output);
-  base64_output = std::string(kJSDataURLPrefix) + base64_output;
-  return base64_output;
-}
-
-bool GetPolyfillForAdBlock(bool allow_brave_shields, bool allow_ads,
-    const GURL& tab_origin, const GURL& gurl, std::string* new_url_spec) {
-  // Polyfills which are related to adblock should only apply when shields
-  // are up.
-  if (!allow_brave_shields || allow_ads) {
-    return false;
-  }
-
-  static URLPattern analytics(URLPattern::SCHEME_ALL,
-      kGoogleAnalyticsPattern);
-  static URLPattern tag_manager(URLPattern::SCHEME_ALL,
-      kGoogleTagManagerPattern);
-  static URLPattern tag_services(URLPattern::SCHEME_ALL,
-      kGoogleTagServicesPattern);
-  if (analytics.MatchesURL(gurl)) {
-    std::string&& data_url = GetGoogleAnalyticsPolyfillJS();
-    *new_url_spec = data_url;
-    return true;
-  }
-
-  if (tag_manager.MatchesURL(gurl)) {
-    std::string&& data_url = GetGoogleTagManagerPolyfillJS();
-    *new_url_spec = data_url;
-    return true;
-  }
-
-  if (tag_services.MatchesURL(gurl)) {
-    std::string&& data_url = GetGoogleTagServicesPolyfillJS();
-    *new_url_spec = data_url;
-    return true;
-  }
-
-  return false;
-}
-
 void ShouldBlockAdOnTaskRunner(std::shared_ptr<BraveRequestInfo> ctx) {
   bool did_match_exception = false;
   std::string tab_host = ctx->tab_origin.host();
   if (!g_brave_browser_process->ad_block_service()->ShouldStartRequest(
           ctx->request_url, ctx->resource_type, tab_host,
-          &did_match_exception, &ctx->cancel_request_explicitly)) {
+          &did_match_exception, &ctx->cancel_request_explicitly,
+          &ctx->mock_data_url)) {
     ctx->blocked_by = kAdBlocked;
   } else if (!did_match_exception &&
              !g_brave_browser_process->ad_block_regional_service_manager()
                   ->ShouldStartRequest(ctx->request_url, ctx->resource_type,
                                        tab_host, &did_match_exception,
-                                       &ctx->cancel_request_explicitly)) {
+                                       &ctx->cancel_request_explicitly,
+                                       &ctx->mock_data_url)) {
     ctx->blocked_by = kAdBlocked;
   } else if (!did_match_exception &&
              !g_brave_browser_process->ad_block_custom_filters_service()
                   ->ShouldStartRequest(ctx->request_url, ctx->resource_type,
                                        tab_host, &did_match_exception,
-                                       &ctx->cancel_request_explicitly)) {
+                                       &ctx->cancel_request_explicitly,
+                                       &ctx->mock_data_url)) {
     ctx->blocked_by = kAdBlocked;
   }
 }
@@ -164,11 +93,6 @@ int OnBeforeURLRequest_AdBlockTPPreWork(
     return net::OK;
   }
 
-  if (GetPolyfillForAdBlock(ctx->allow_brave_shields, ctx->allow_ads,
-        ctx->tab_origin, ctx->request_url, &ctx->new_url_spec)) {
-    return net::OK;
-  }
-
   // Most blocked resources have been moved to our ad block lists.
   // This is only for special cases like the PDFjs ping which can
   // occur before the ad block lists are fully loaded.

browser/net/brave_ad_block_tp_network_delegate_helper.h

@@ -1,9 +1,12 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
+/* Copyright (c) 2019 The Brave Authors. All rights reserved.
+ * This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
  * You can obtain one at http://mozilla.org/MPL/2.0/. */
 
-#ifndef BRAVE_BROWSER_NET_BRAVE_AD_BLOCK_TP_NETWORK_DELEGATE_H_
-#define BRAVE_BROWSER_NET_BRAVE_AD_BLOCK_TP_NETWORK_DELEGATE_H_
+#ifndef BRAVE_BROWSER_NET_BRAVE_AD_BLOCK_TP_NETWORK_DELEGATE_HELPER_H_
+#define BRAVE_BROWSER_NET_BRAVE_AD_BLOCK_TP_NETWORK_DELEGATE_HELPER_H_
+
+#include <memory>
 
 #include "brave/browser/net/url_context.h"
 
@@ -13,9 +16,6 @@ int OnBeforeURLRequest_AdBlockTPPreWork(
     const ResponseCallback& next_callback,
     std::shared_ptr<BraveRequestInfo> ctx);
 
-bool GetPolyfillForAdBlock(bool allow_brave_shields, bool allow_ads,
-    const GURL& tab_origin, const GURL& gurl, std::string* new_url_spec);
-
 }  // namespace brave
 
-#endif  // BRAVE_BROWSER_NET_BRAVE_AD_BLOCK_TP_NETWORK_DELEGATE_H_
+#endif  // BRAVE_BROWSER_NET_BRAVE_AD_BLOCK_TP_NETWORK_DELEGATE_HELPER_H_

browser/net/brave_ad_block_tp_network_delegate_helper_unittest.cc

@@ -32,18 +32,6 @@ TEST(BraveAdBlockTPNetworkDelegateHelperTest, EmptyRequestURL) {
   EXPECT_EQ(rc, net::OK);
 }
 
-TEST(BraveAdBlockTPNetworkDelegateHelperTest, RedirectsToStubs) {
-  const std::vector<const GURL> urls(
-      {GURL(kGoogleTagManagerPattern), GURL(kGoogleTagServicesPattern)});
-  for (const auto& url : urls) {
-    auto request_info = std::make_shared<brave::BraveRequestInfo>(url);
-    int rc =
-        OnBeforeURLRequest_AdBlockTPPreWork(ResponseCallback(), request_info);
-    EXPECT_EQ(rc, net::OK);
-    EXPECT_TRUE(GURL(request_info->new_url_spec).SchemeIs("data"));
-  }
-}
-
 TEST(BraveAdBlockTPNetworkDelegateHelperTest, Blocking) {
   const std::vector<const GURL> urls({
       GURL("https://pdfjs.robwu.nl/ping"),
@@ -56,66 +44,3 @@ TEST(BraveAdBlockTPNetworkDelegateHelperTest, Blocking) {
     EXPECT_EQ(rc, net::OK);
   }
 }
-
-TEST(BraveAdBlockTPNetworkDelegateHelperTest, GetPolyfill) {
-  using brave::GetPolyfillForAdBlock;
-
-  const GURL tab_origin("https://test.com");
-  const GURL google_analytics_url(kGoogleAnalyticsPattern);
-  const GURL tag_manager_url(kGoogleTagManagerPattern);
-  const GURL tag_services_url(kGoogleTagServicesPattern);
-  const GURL normal_url("https://a.com");
-
-  std::string out_url_spec;
-  // Shields up, block ads, google analytics should get polyfill
-  ASSERT_TRUE(GetPolyfillForAdBlock(true, false, tab_origin,
-                                    google_analytics_url, &out_url_spec));
-  // Shields up, block ads, tag manager should get polyfill
-  ASSERT_TRUE(GetPolyfillForAdBlock(true, false, tab_origin, tag_manager_url,
-                                    &out_url_spec));
-  // Shields up, block ads, tag services should get polyfill
-  ASSERT_TRUE(GetPolyfillForAdBlock(true, false, tab_origin, tag_services_url,
-                                    &out_url_spec));
-  // Shields up, block ads, normal URL should NOT get polyfill
-  ASSERT_FALSE(GetPolyfillForAdBlock(true, false, tab_origin, normal_url,
-                                     &out_url_spec));
-
-  // Shields up, allow ads, google analytics should NOT get polyfill
-  ASSERT_FALSE(GetPolyfillForAdBlock(true, true, tab_origin,
-                                     google_analytics_url, &out_url_spec));
-  // Shields up, allow ads, tag manager should NOT get polyfill
-  ASSERT_FALSE(GetPolyfillForAdBlock(true, true, tab_origin, tag_manager_url,
-                                     &out_url_spec));
-  // Shields up, allow ads, tag services should NOT get polyfill
-  ASSERT_FALSE(GetPolyfillForAdBlock(true, true, tab_origin, tag_services_url,
-                                     &out_url_spec));
-  // Shields up, allow ads, normal URL should NOT get polyfill
-  ASSERT_FALSE(
-      GetPolyfillForAdBlock(true, true, tab_origin, normal_url, &out_url_spec));
-
-  // Shields down, allow ads, google analytics should NOT get polyfill
-  ASSERT_FALSE(GetPolyfillForAdBlock(false, true, tab_origin,
-                                     google_analytics_url, &out_url_spec));
-  // Shields down, allow ads, tag manager should NOT get polyfill
-  ASSERT_FALSE(GetPolyfillForAdBlock(false, true, tab_origin, tag_manager_url,
-                                     &out_url_spec));
-  // Shields down, allow ads, tag services should NOT get polyfill
-  ASSERT_FALSE(GetPolyfillForAdBlock(false, true, tab_origin, tag_services_url,
-                                     &out_url_spec));
-  // Shields down, allow ads, normal URL should NOT get polyfill
-  ASSERT_FALSE(GetPolyfillForAdBlock(false, true, tab_origin, normal_url,
-                                     &out_url_spec));
-
-  // Shields down, block ads, google analytics should NOT get polyfill
-  ASSERT_FALSE(GetPolyfillForAdBlock(false, false, tab_origin,
-                                     google_analytics_url, &out_url_spec));
-  // Shields down, block ads, tag manager should NOT get polyfill
-  ASSERT_FALSE(GetPolyfillForAdBlock(false, false, tab_origin, tag_manager_url,
-                                     &out_url_spec));
-  // Shields down, block ads, tag services should NOT get polyfill
-  ASSERT_FALSE(GetPolyfillForAdBlock(false, false, tab_origin, tag_services_url,
-                                     &out_url_spec));
-  // Shields down, block ads, normal URL should NOT get polyfill
-  ASSERT_FALSE(GetPolyfillForAdBlock(false, false, tab_origin, normal_url,
-                                     &out_url_spec));
-}

browser/net/brave_proxying_url_loader_factory.cc

@@ -110,7 +110,7 @@ void BraveProxyingURLLoaderFactory::InProgressRequest::RestartInternal() {
 
   base::RepeatingCallback<void(int)> continuation =
       base::BindRepeating(&InProgressRequest::ContinueToBeforeSendHeaders,
-                            weak_factory_.GetWeakPtr());
+                          weak_factory_.GetWeakPtr());
   redirect_url_ = GURL();
   ctx_ = std::make_shared<brave::BraveRequestInfo>();
   brave::BraveRequestInfo::FillCTX(request_, render_process_id_,
@@ -211,8 +211,8 @@ void BraveProxyingURLLoaderFactory::InProgressRequest::OnUploadProgress(
                                    std::move(callback));
 }
 
-void BraveProxyingURLLoaderFactory::InProgressRequest::
-    OnReceiveCachedMetadata(mojo_base::BigBuffer data) {
+void BraveProxyingURLLoaderFactory::InProgressRequest::OnReceiveCachedMetadata(
+    mojo_base::BigBuffer data) {
   target_client_->OnReceiveCachedMetadata(std::move(data));
 }
 
@@ -337,7 +337,8 @@ void BraveProxyingURLLoaderFactory::InProgressRequest::
     }
     network::ResourceResponseHead response;
     std::string response_data;
-    brave_shields::MakeStubResponse(request_, &response, &response_data);
+    brave_shields::MakeStubResponse(ctx_->mock_data_url, request_, &response,
+                                    &response_data);
 
     target_client_->OnReceiveResponse(response);
 
@@ -626,10 +627,8 @@ bool BraveProxyingURLLoaderFactory::MaybeProxyRequest(
   network::mojom::URLLoaderFactoryPtrInfo target_factory_info;
   *factory_receiver = mojo::MakeRequest(&target_factory_info);
 
-
   ResourceContextData::StartProxying(
-      browser_context,
-      render_process_id,
+      browser_context, render_process_id,
       render_frame_host ? render_frame_host->GetFrameTreeNodeId() : 0,
       std::move(proxied_receiver), std::move(target_factory_info));
   return true;
@@ -651,11 +650,10 @@ void BraveProxyingURLLoaderFactory::CreateLoaderAndStart(
   // unique, so we don't use it for identity here.
   const uint64_t brave_request_id = request_id_generator_->Generate();
 
-  auto result = requests_.emplace(
-      std::make_unique<InProgressRequest>(
-          this, brave_request_id, request_id, routing_id, render_process_id_,
-          frame_tree_node_id_, options, request, browser_context_,
-          traffic_annotation, std::move(loader_request), std::move(client)));
+  auto result = requests_.emplace(std::make_unique<InProgressRequest>(
+      this, brave_request_id, request_id, routing_id, render_process_id_,
+      frame_tree_node_id_, options, request, browser_context_,
+      traffic_annotation, std::move(loader_request), std::move(client)));
   (*result.first)->Restart();
 }
 

browser/net/url_context.h

@@ -90,6 +90,8 @@ struct BraveRequestInfo {
   const base::ListValue* referral_headers_list = nullptr;
   BlockedBy blocked_by = kNotBlocked;
   bool cancel_request_explicitly = false;
+  std::string mock_data_url;
+
   // Default to invalid type for resource_type, so delegate helpers
   // can properly detect that the info couldn't be obtained.
   // TODO(iefremov): Replace with something like |WebRequestResourceType| to

common/network_constants.cc

@@ -44,14 +44,8 @@ const char kCRLSetPrefix3[] =
 const char kCRLSetPrefix4[] =
     "*://storage.googleapis.com/update-delta/hfnkpimlhhgieaddgfemjhofmfblmnib"
     "/*crxd";
-const char kGoogleAnalyticsPattern[] =
-    "https://www.google-analytics.com/analytics.js";
 const char kChromeCastPrefix[] =
     "*://*.gvt1.com/edgedl/chromewebstore/*pkedcjkdefgpdelpbcmbmeomcjbeemfm*";
-const char kGoogleTagManagerPattern[] =
-    "https://www.googletagmanager.com/gtm.js";
-const char kGoogleTagServicesPattern[] =
-    "https://www.googletagservices.com/tag/js/gpt.js";
 const char kForbesPattern[] = "https://www.forbes.com/*";
 const char kForbesExtraCookies[] =
     "forbes_ab=true; welcomeAd=true; adblock_session=Off; "

common/network_constants.h

@@ -25,9 +25,6 @@ extern const char kEmptyDataURI[];
 extern const char kEmptyImageDataURI[];
 extern const char kJSDataURLPrefix[];
 extern const char kGeoLocationsPattern[];
-extern const char kGoogleAnalyticsPattern[];
-extern const char kGoogleTagManagerPattern[];
-extern const char kGoogleTagServicesPattern[];
 extern const char kForbesPattern[];
 extern const char kForbesExtraCookies[];
 extern const char kSafeBrowsingPrefix[];