From 47b456aa2691f546d860d0edd7cc58dfc6a30441 Mon Sep 17 00:00:00 2001 From: Jason Sadler Date: Mon, 3 Jun 2019 16:30:44 -0400 Subject: [PATCH] Fixes brave/brave-browser#4745 --- package.json | 2 +- test/BUILD.gn | 1 + vendor/bat-native-ledger/BUILD.gn | 2 + .../src/bat/ledger/internal/bat_get_media.cc | 5 +- .../src/bat/ledger/internal/bat_get_media.h | 2 + .../src/bat/ledger/internal/bat_publishers.cc | 5 +- .../src/bat/ledger/internal/media/reddit.cc | 284 ++++++++++++++++++ .../src/bat/ledger/internal/media/reddit.h | 99 ++++++ .../ledger/internal/media/reddit_unittest.cc | 127 ++++++++ .../src/bat/ledger/internal/static_values.h | 3 + 10 files changed, 527 insertions(+), 3 deletions(-) create mode 100644 vendor/bat-native-ledger/src/bat/ledger/internal/media/reddit.cc create mode 100644 vendor/bat-native-ledger/src/bat/ledger/internal/media/reddit.h create mode 100644 vendor/bat-native-ledger/src/bat/ledger/internal/media/reddit_unittest.cc diff --git a/package.json b/package.json index faa5b3f04f22..2633326083dd 100644 --- a/package.json +++ b/package.json @@ -277,7 +277,7 @@ "@types/react-redux": "6.0.4", "@types/redux-logger": "^3.0.7", "awesome-typescript-loader": "^5.2.1", - "brave-ui": "github:brave/brave-ui#82b3e981040b2ba89743108e6701f5fa24ce3b52", + "brave-ui": "github:brave/brave-ui#939d492a3e31e92021105684e0dbfc1399ee9df2", "css-loader": "^2.1.1", "csstype": "^2.5.5", "deep-freeze-node": "^1.1.3", diff --git a/test/BUILD.gn b/test/BUILD.gn index eb1fe2b5f07e..ba4219ee7734 100644 --- a/test/BUILD.gn +++ b/test/BUILD.gn @@ -118,6 +118,7 @@ test("brave_unit_tests") { if (brave_rewards_enabled) { sources += [ "//brave/vendor/bat-native-ledger/src/bat/ledger/internal/media/helper_unittest.cc", + "//brave/vendor/bat-native-ledger/src/bat/ledger/internal/media/reddit_unittest.cc", "//brave/vendor/bat-native-ledger/src/bat/ledger/internal/media/twitch_unittest.cc", "//brave/vendor/bat-native-ledger/src/bat/ledger/internal/media/twitter_unittest.cc", "//brave/vendor/bat-native-ledger/src/bat/ledger/internal/media/youtube_unittest.cc", diff --git a/vendor/bat-native-ledger/BUILD.gn b/vendor/bat-native-ledger/BUILD.gn index 65d444d50e64..1e8bf7cc6cd9 100644 --- a/vendor/bat-native-ledger/BUILD.gn +++ b/vendor/bat-native-ledger/BUILD.gn @@ -110,6 +110,8 @@ source_set("ledger") { "src/bat/ledger/internal/ledger_impl.h", "src/bat/ledger/internal/media/helper.h", "src/bat/ledger/internal/media/helper.cc", + "src/bat/ledger/internal/media/reddit.h", + "src/bat/ledger/internal/media/reddit.cc", "src/bat/ledger/internal/media/twitch.h", "src/bat/ledger/internal/media/twitch.cc", "src/bat/ledger/internal/media/twitter.h", diff --git a/vendor/bat-native-ledger/src/bat/ledger/internal/bat_get_media.cc b/vendor/bat-native-ledger/src/bat/ledger/internal/bat_get_media.cc index 4cf7e8a75db0..5a7c46527ff8 100644 --- a/vendor/bat-native-ledger/src/bat/ledger/internal/bat_get_media.cc +++ b/vendor/bat-native-ledger/src/bat/ledger/internal/bat_get_media.cc @@ -18,7 +18,8 @@ BatGetMedia::BatGetMedia(bat_ledger::LedgerImpl* ledger): ledger_(ledger), media_youtube_(new braveledger_media::MediaYouTube(ledger)), media_twitch_(new braveledger_media::MediaTwitch(ledger)), - media_twitter_(new braveledger_media::MediaTwitter(ledger)) { + media_twitter_(new braveledger_media::MediaTwitter(ledger)), + media_reddit_(new braveledger_media::MediaReddit(ledger)) { } BatGetMedia::~BatGetMedia() {} @@ -70,6 +71,8 @@ void BatGetMedia::GetMediaActivityFromUrl( } else if (type == TWITTER_MEDIA_TYPE) { media_twitter_->ProcessActivityFromUrl(window_id, visit_data); + } else if (type == REDDIT_MEDIA_TYPE) { + media_reddit_->ProcessActivityFromUrl(window_id, visit_data); } else { OnMediaActivityError(visit_data, type, window_id); } diff --git a/vendor/bat-native-ledger/src/bat/ledger/internal/bat_get_media.h b/vendor/bat-native-ledger/src/bat/ledger/internal/bat_get_media.h index bfdf632f99fa..98490df9b7c1 100644 --- a/vendor/bat-native-ledger/src/bat/ledger/internal/bat_get_media.h +++ b/vendor/bat-native-ledger/src/bat/ledger/internal/bat_get_media.h @@ -11,6 +11,7 @@ #include #include "bat/ledger/internal/bat_helper.h" +#include "bat/ledger/internal/media/reddit.h" #include "bat/ledger/internal/media/twitch.h" #include "bat/ledger/internal/media/twitter.h" #include "bat/ledger/internal/media/youtube.h" @@ -58,6 +59,7 @@ class BatGetMedia { std::unique_ptr media_youtube_; std::unique_ptr media_twitch_; std::unique_ptr media_twitter_; + std::unique_ptr media_reddit_; }; } // namespace braveledger_bat_get_media diff --git a/vendor/bat-native-ledger/src/bat/ledger/internal/bat_publishers.cc b/vendor/bat-native-ledger/src/bat/ledger/internal/bat_publishers.cc index a27c3891d811..85b3fff0732c 100644 --- a/vendor/bat-native-ledger/src/bat/ledger/internal/bat_publishers.cc +++ b/vendor/bat-native-ledger/src/bat/ledger/internal/bat_publishers.cc @@ -707,7 +707,8 @@ void BatPublishers::getPublisherActivityFromUrl( const bool is_media = visit_data.domain == YOUTUBE_TLD || visit_data.domain == TWITCH_TLD || - visit_data.domain == TWITTER_TLD; + visit_data.domain == TWITTER_TLD || + visit_data.domain == REDDIT_TLD; if (is_media && visit_data.path != "" && visit_data.path != "/") { @@ -716,6 +717,8 @@ void BatPublishers::getPublisherActivityFromUrl( type = TWITCH_MEDIA_TYPE; } else if (visit_data.domain == TWITTER_TLD) { type = TWITTER_MEDIA_TYPE; + } else if (visit_data.domain == REDDIT_TLD) { + type = REDDIT_MEDIA_TYPE; } ledger::VisitData new_visit_data(visit_data); diff --git a/vendor/bat-native-ledger/src/bat/ledger/internal/media/reddit.cc b/vendor/bat-native-ledger/src/bat/ledger/internal/media/reddit.cc new file mode 100644 index 000000000000..1293f60952ca --- /dev/null +++ b/vendor/bat-native-ledger/src/bat/ledger/internal/media/reddit.cc @@ -0,0 +1,284 @@ +/* Copyright (c) 2019 The Brave Authors. All rights reserved. + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include +#include +#include + +#include "base/strings/string_split.h" +#include "base/strings/stringprintf.h" +#include "bat/ledger/internal/ledger_impl.h" +#include "bat/ledger/internal/media/helper.h" +#include "bat/ledger/internal/media/reddit.h" +#include "net/http/http_status_code.h" +#include "url/url_canon.h" +#include "url/gurl.h" + +using std::placeholders::_1; +using std::placeholders::_2; +using std::placeholders::_3; + +namespace braveledger_media { + +MediaReddit::MediaReddit(bat_ledger::LedgerImpl* ledger): ledger_(ledger) { +} + +MediaReddit::~MediaReddit() { +} + +void MediaReddit::ProcessActivityFromUrl( + uint64_t window_id, + const ledger::VisitData& visit_data) { + if (visit_data.path.find("/user/") != std::string::npos) { + UserPath(window_id, visit_data); + return; + } + OnMediaActivityError(visit_data, window_id); +} + +void MediaReddit::OnMediaActivityError( + const ledger::VisitData& visit_data, + uint64_t window_id) { + + ledger::VisitData new_visit_data; + new_visit_data.domain = REDDIT_TLD; + new_visit_data.url = "https://" + (std::string)REDDIT_TLD; + new_visit_data.path = "/"; + new_visit_data.name = REDDIT_MEDIA_TYPE; + + ledger_->GetPublisherActivityFromUrl( + window_id, new_visit_data, std::string()); +} + +void MediaReddit::UserPath( + uint64_t window_id, + const ledger::VisitData& visit_data) { + const std::string user = GetUserNameFromUrl(visit_data.path); + + if (user.empty()) { + OnMediaActivityError(visit_data, window_id); + return; + } + + const std::string media_key = (std::string)REDDIT_MEDIA_TYPE + "_" + user; + ledger_->GetMediaPublisherInfo(media_key, + std::bind(&MediaReddit::OnUserActivity, + this, + window_id, + visit_data, + media_key, + _1, + _2)); +} + +void MediaReddit::OnUserActivity( + uint64_t window_id, + const ledger::VisitData& visit_data, + const std::string& media_key, + ledger::Result result, + ledger::PublisherInfoPtr publisher_info) { + if (!publisher_info || result == ledger::Result::NOT_FOUND) { + const std::string user_name = GetUserNameFromUrl(visit_data.path); + const std::string url = GetProfileUrl(user_name); + FetchDataFromUrl(visit_data.url, + std::bind(&MediaReddit::OnUserPage, + this, + window_id, + visit_data, + _1, + _2, + _3)); + } else { + GetPublisherPanelInfo( + window_id, + visit_data, + publisher_info->id); + } +} + +void MediaReddit::FetchDataFromUrl( + const std::string& url, + braveledger_media::FetchDataFromUrlCallback callback) { + /* if user is on old reddit, sub the url to get the icon + since old reddit didn't have user icons */ + GURL reddit_url(url); + if (reddit_url.DomainIs(OLD_REDDIT_DOMAIN)) { + // Canonicalize away 'old.reddit.com' and replace with 'www.reddit.com'. + std::string new_host = reddit_url.host(); + new_host.replace(0, 3, "www"); + url::Replacements replacements; + replacements.SetHost(new_host.c_str(), + url::Component(0, new_host.length())); + reddit_url = reddit_url.ReplaceComponents(replacements); + } + + ledger_->LoadURL(reddit_url.spec(), + std::vector(), + std::string(), + std::string(), + ledger::URL_METHOD::GET, + callback); +} + +// static +std::string MediaReddit::GetUserNameFromUrl(const std::string& path) { + if (path.empty()) { + return std::string(); + } + + const std::vector parts = base::SplitString( + path, "/", base::TRIM_WHITESPACE, base::SPLIT_WANT_NONEMPTY); + + if (parts.size() > 1) { + return parts.at(1); + } + + return std::string(); +} + +// static +std::string MediaReddit::GetProfileUrl(const std::string& screen_name) { + if (screen_name.empty()) { + return std::string(); + } + const std::string url_part = "https://" + (std::string)REDDIT_TLD + + "/user/%s/"; + return base::StringPrintf(url_part.c_str(), screen_name.c_str()); +} + +void MediaReddit::GetPublisherPanelInfo( + uint64_t window_id, + const ledger::VisitData& visit_data, + const std::string& publisher_key) { + const auto filter = ledger_->CreateActivityFilter( + publisher_key, + ledger::EXCLUDE_FILTER::FILTER_ALL, + false, + ledger_->GetReconcileStamp(), + true, + false); + ledger_->GetPanelPublisherInfo(filter, + std::bind(&MediaReddit::OnPublisherPanelInfo, + this, + window_id, + visit_data, + publisher_key, + _1, + _2)); +} + +void MediaReddit::OnPublisherPanelInfo( + uint64_t window_id, + const ledger::VisitData& visit_data, + const std::string& publisher_key, + ledger::Result result, + ledger::PublisherInfoPtr info) { + if (!info || result == ledger::Result::NOT_FOUND) { + FetchDataFromUrl(visit_data.url, + std::bind(&MediaReddit::OnUserPage, + this, + window_id, + visit_data, + _1, + _2, + _3)); + } else { + ledger_->OnPanelPublisherInfo(result, std::move(info), window_id); + } +} + +// static +std::string MediaReddit::GetUserId(const std::string& response) { + if (response.empty()) { + return std::string(); + } + + std::string id(braveledger_media::ExtractData( + response, "hideFromRobots\":false,\"id\":\"t2_", "\"")); + + if (id.empty()) { + id = braveledger_media::ExtractData( + response, "target_fullname\": \"t2_", "\""); // old reddit + } + return id; +} + +// static +std::string MediaReddit::GetPublisherName(const std::string& response) { + if (response.empty()) { + return std::string(); + } + + std::string user_name(braveledger_media::ExtractData( + response, "username\":\"", "\"")); + + if (user_name.empty()) { + user_name = braveledger_media::ExtractData( + response, "target_name\": \"", "\""); // old reddit + } + return user_name; +} + +void MediaReddit::OnUserPage( + uint64_t window_id, + const ledger::VisitData& visit_data, + int response_status_code, + const std::string& response, + const std::map& headers) { + if (response_status_code != net::HTTP_OK) { + OnMediaActivityError(visit_data, window_id); + return; + } + + const std::string user_name = GetUserNameFromUrl(visit_data.path); + const std::string publisher_name = GetPublisherName(response); + const std::string user_id = GetUserId(response); + const std::string publisher_key = GetPublisherKey(user_id); + const std::string media_key = GetMediaKey(user_name, REDDIT_MEDIA_TYPE); + + ledger::VisitData new_visit_data(visit_data); + new_visit_data.provider = REDDIT_MEDIA_TYPE; + new_visit_data.url = GetProfileUrl(user_name); + new_visit_data.favicon_url = GetProfileImageUrl(response); + new_visit_data.name = publisher_name.empty() ? user_name : publisher_name; + + ledger_->SaveMediaVisit( + publisher_key, + new_visit_data, + 0, + window_id, + std::bind(&MediaReddit::OnUserActivity, + this, + window_id, + visit_data, + user_id, + _1, + _2)); + + if (!media_key.empty()) { + ledger_->SetMediaPublisherInfo(media_key, publisher_key); + } +} + +// static +std::string MediaReddit::GetPublisherKey(const std::string& key) { + if (key.empty()) { + return std::string(); + } + return (std::string)REDDIT_MEDIA_TYPE + "#channel:" + key; +} + +// static +std::string MediaReddit::GetProfileImageUrl(const std::string& response) { + if (response.empty()) { + return std::string(); + } + + const std::string image_url(braveledger_media::ExtractData( + response, "accountIcon\":\"", "?")); + return image_url; // old reddit does not use account icons +} + +} // namespace braveledger_media diff --git a/vendor/bat-native-ledger/src/bat/ledger/internal/media/reddit.h b/vendor/bat-native-ledger/src/bat/ledger/internal/media/reddit.h new file mode 100644 index 000000000000..22a87bb51294 --- /dev/null +++ b/vendor/bat-native-ledger/src/bat/ledger/internal/media/reddit.h @@ -0,0 +1,99 @@ +/* Copyright (c) 2019 The Brave Authors. All rights reserved. + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef BRAVELEDGER_MEDIA_REDDIT_H_ +#define BRAVELEDGER_MEDIA_REDDIT_H_ + +#include +#include +#include + +#include "base/gtest_prod_util.h" +#include "bat/ledger/ledger.h" +#include "bat/ledger/internal/media/helper.h" + +namespace bat_ledger { +class LedgerImpl; +} + +namespace braveledger_media { + +class MediaReddit : public ledger::LedgerCallbackHandler { + public: + explicit MediaReddit(bat_ledger::LedgerImpl* ledger); + + ~MediaReddit() override; + + void ProcessActivityFromUrl( + uint64_t window_id, + const ledger::VisitData& visit_data); + + private: + void OnMediaActivityError( + const ledger::VisitData& visit_data, + uint64_t window_id); + + void UserPath( + uint64_t window_id, + const ledger::VisitData& visit_data); + + void OnUserActivity( + uint64_t window_id, + const ledger::VisitData& visit_data, + const std::string& media_key, + ledger::Result result, + ledger::PublisherInfoPtr publisher_info); + + void FetchDataFromUrl( + const std::string& url, + braveledger_media::FetchDataFromUrlCallback callback); + + void GetPublisherPanelInfo( + uint64_t window_id, + const ledger::VisitData& visit_data, + const std::string& publisher_key); + + void OnUserPage( + uint64_t window_id, + const ledger::VisitData& visit_data, + int response_status_code, + const std::string& response, + const std::map& headers); + + void OnPublisherPanelInfo( + uint64_t window_id, + const ledger::VisitData& visit_data, + const std::string& publisher_key, + ledger::Result result, + ledger::PublisherInfoPtr info); + + static std::string GetUserNameFromUrl(const std::string& path); + + static std::string GetProfileUrl(const std::string& screen_name); + + static std::string GetUserId(const std::string& response); + + static std::string GetPublisherName(const std::string& response); + + static std::string GetPublisherKey(const std::string& key); + + static std::string GetProfileImageUrl(const std::string& response); + + bat_ledger::LedgerImpl* ledger_; // NOT OWNED + + // For testing purposes + friend class MediaRedditTest; + FRIEND_TEST_ALL_PREFIXES(MediaRedditTest, GetProfileUrl); + FRIEND_TEST_ALL_PREFIXES(MediaRedditTest, GetProfileImageUrl); + FRIEND_TEST_ALL_PREFIXES(MediaRedditTest, GetPublisherKey); + FRIEND_TEST_ALL_PREFIXES(MediaRedditTest, GetMediaKey); + FRIEND_TEST_ALL_PREFIXES(MediaRedditTest, GetUserNameFromUrl); + FRIEND_TEST_ALL_PREFIXES(MediaRedditTest, GetUserId); + FRIEND_TEST_ALL_PREFIXES(MediaRedditTest, GetPublisherName); +}; + +} // namespace braveledger_media + +#endif // BRAVELEDGER_MEDIA_REDDIT_H_ diff --git a/vendor/bat-native-ledger/src/bat/ledger/internal/media/reddit_unittest.cc b/vendor/bat-native-ledger/src/bat/ledger/internal/media/reddit_unittest.cc new file mode 100644 index 000000000000..c820201c3353 --- /dev/null +++ b/vendor/bat-native-ledger/src/bat/ledger/internal/media/reddit_unittest.cc @@ -0,0 +1,127 @@ +/* Copyright (c) 2019 The Brave Authors. All rights reserved. + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include +#include + +#include "bat/ledger/internal/media/reddit.h" +#include "bat/ledger/internal/static_values.h" +#include "bat/ledger/ledger.h" +#include "testing/gtest/include/gtest/gtest.h" + +// npm run test -- brave_unit_tests --filter=MediaRedditTest.* + +namespace braveledger_media { + +class MediaRedditTest : public testing::Test { +}; + +TEST(MediaRedditTest, GetProfileUrl) { + // empty + std::string result = + braveledger_media::MediaReddit::GetProfileUrl(std::string()); + ASSERT_TRUE(result.empty()); + + result = braveledger_media::MediaReddit::GetProfileUrl("jsadler-brave"); + ASSERT_EQ(result, "https://reddit.com/user/jsadler-brave/"); +} + +TEST(MediaRedditTest, GetProfileImageUrl) { + // empty + std::string result = + braveledger_media::MediaReddit::GetProfileImageUrl(std::string()); + ASSERT_TRUE(result.empty()); + + result = braveledger_media::MediaReddit::GetProfileImageUrl( + "\"accountIcon\":\"https://www.someredditmediacdn.com/somephoto.png?" + "somequerystringparams"); + ASSERT_EQ(result, "https://www.someredditmediacdn.com/somephoto.png"); +} + +TEST(MediaRedditTest, GetPublisherKey) { + // empty + std::string result = + braveledger_media::MediaReddit::GetPublisherKey(std::string()); + ASSERT_TRUE(result.empty()); + + result = + braveledger_media::MediaReddit::GetPublisherKey("test_publisher_key"); + ASSERT_EQ(result, "reddit#channel:test_publisher_key"); +} + +TEST(MediaRedditTest, GetUserNameFromUrl) { + // empty + std::string result = + braveledger_media::MediaReddit::GetUserNameFromUrl(std::string()); + ASSERT_TRUE(result.empty()); + + // empty path + result = braveledger_media::MediaReddit:: + GetUserNameFromUrl("/"); + ASSERT_TRUE(result.empty()); + + // simple path + result = braveledger_media::MediaReddit:: + GetUserNameFromUrl("/jsadler-brave"); + ASSERT_TRUE(result.empty()); + + // long path + result = braveledger_media::MediaReddit:: + GetUserNameFromUrl("/user/jsadler-brave"); + ASSERT_EQ(result, "jsadler-brave"); +} + +TEST(MediaRedditTest, GetUserId) { + const char reddit_new[] = + "\"hideFromRobots\":false,\"id\":\"t2_78910\""; + const char reddit_old[] = + "\"target_fullname\": \"t2_123456\""; + + // empty + std::string result = + braveledger_media::MediaReddit::GetUserId(std::string()); + ASSERT_TRUE(result.empty()); + + // incorrect scrape + result = + braveledger_media::MediaReddit::GetUserId("Some random text"); + ASSERT_TRUE(result.empty()); + + // support for current Reddit + result = + braveledger_media::MediaReddit::GetUserId(reddit_old); + ASSERT_EQ(result, "123456"); + + // support for new Reddit + result = + braveledger_media::MediaReddit::GetUserId(reddit_new); + ASSERT_EQ(result, "78910"); +} + +TEST(MediaRedditTest, GetPublisherName) { + const char reddit_new[] = "\"username\":\"jsadler-brave\""; + const char reddit_old[] = "\"target_name\": \"jsadler-brave\""; + // empty + std::string result = + braveledger_media::MediaReddit::GetPublisherName(std::string()); + ASSERT_TRUE(result.empty()); + + // incorrect scrape + result = braveledger_media::MediaReddit:: + GetPublisherName("some random text"); + ASSERT_TRUE(result.empty()); + + // current reddit + result = braveledger_media::MediaReddit:: + GetPublisherName(reddit_new); + ASSERT_EQ(result, "jsadler-brave"); + + // old reddit + result = braveledger_media::MediaReddit:: + GetPublisherName(reddit_old); + ASSERT_EQ(result, "jsadler-brave"); +} + +} // namespace braveledger_media diff --git a/vendor/bat-native-ledger/src/bat/ledger/internal/static_values.h b/vendor/bat-native-ledger/src/bat/ledger/internal/static_values.h index db196b9766f6..96183d7e91ca 100644 --- a/vendor/bat-native-ledger/src/bat/ledger/internal/static_values.h +++ b/vendor/bat-native-ledger/src/bat/ledger/internal/static_values.h @@ -64,12 +64,15 @@ #define YOUTUBE_MEDIA_TYPE "youtube" #define TWITCH_MEDIA_TYPE "twitch" #define TWITTER_MEDIA_TYPE "twitter" +#define REDDIT_MEDIA_TYPE "reddit" #define YOUTUBE_PROVIDER_URL "https://www.youtube.com/oembed" #define TWITCH_PROVIDER_URL "https://api.twitch.tv/v5/oembed" #define YOUTUBE_TLD "youtube.com" #define TWITCH_TLD "twitch.tv" #define TWITTER_TLD "twitter.com" +#define REDDIT_TLD "reddit.com" #define TWITCH_VOD_URL "https://www.twitch.tv/videos/" +#define OLD_REDDIT_DOMAIN "old.reddit.com" #define MEDIA_DELIMITER '_' #define WALLET_PASSPHRASE_DELIM ' ' #define DICTIONARY_DELIMITER ','