Skip to content

Commit

Permalink
Use real Core for GUI tests
Browse files Browse the repository at this point in the history
  • Loading branch information
ichorid authored and devos50 committed Oct 1, 2020
1 parent 6d3f12a commit fecebd6
Show file tree
Hide file tree
Showing 50 changed files with 211 additions and 2,138 deletions.
7 changes: 4 additions & 3 deletions src/run_tribler.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
import tribler_gui


def start_tribler_core(base_path, api_port, api_key, root_state_dir):
def start_tribler_core(base_path, api_port, api_key, root_state_dir, core_test_mode=False):
"""
This method will start a new Tribler session.
Note that there is no direct communication between the GUI process and the core: all communication is performed
Expand Down Expand Up @@ -75,7 +75,7 @@ async def start_tribler():
# Enable tracer if --trace-debug or --trace-exceptions flag is present in sys.argv
trace_logger = check_and_enable_code_tracing('core', config.get_log_dir())

session = Session(config)
session = Session(config, core_test_mode=core_test_mode)

signal.signal(signal.SIGTERM, lambda signum, stack: shutdown(session, signum, stack))
await session.start()
Expand All @@ -97,8 +97,9 @@ async def start_tribler():
base_path = os.environ['CORE_BASE_PATH']
api_port = os.environ['CORE_API_PORT']
api_key = os.environ['CORE_API_KEY']
core_test_mode = bool(os.environ.get("TRIBLER_CORE_TEST_MODE", False))

start_tribler_core(base_path, api_port, api_key, root_state_dir)
start_tribler_core(base_path, api_port, api_key, root_state_dir, core_test_mode=core_test_mode)
else:
# Set up logging
tribler_gui.load_logger_config(root_state_dir)
Expand Down
6 changes: 3 additions & 3 deletions src/tribler-core/tribler_core/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -232,9 +232,9 @@ async def redirect_handler(_):
@pytest.fixture
def metadata_store(tmpdir):
metadata_store_path = Path(tmpdir) / 'test.db'
metadata_store = MetadataStore(metadata_store_path, tmpdir, TEST_PERSONAL_KEY)
yield metadata_store
metadata_store.shutdown()
mds = MetadataStore(metadata_store_path, tmpdir, TEST_PERSONAL_KEY, disable_sync=True)
yield mds
mds.shutdown()


@pytest.fixture
Expand Down
7 changes: 4 additions & 3 deletions src/tribler-core/tribler_core/modules/libtorrent/download.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,11 +33,12 @@
class Download(TaskManager):
""" Download subclass that represents a libtorrent download."""

def __init__(self, session, tdef):
def __init__(self, session, tdef, dummy=False):
super(Download, self).__init__()

self._logger = logging.getLogger(self.__class__.__name__)

self.dummy = dummy
self.session = session
self.config = None
self.tdef = tdef
Expand All @@ -55,7 +56,7 @@ def __init__(self, session, tdef):
self.pause_after_next_hashcheck = False
self.checkpoint_after_next_hashcheck = False
self.tracker_status = {} # {url: [num_peers, status_str]}
self.checkpoint_disabled = False
self.checkpoint_disabled = self.dummy

self.futures = defaultdict(list)
self.alert_handlers = defaultdict(list)
Expand Down Expand Up @@ -136,7 +137,7 @@ def setup(self, config=None, hidden=False, checkpoint_disabled=False):
:returns a Deferred to which a callback can be added which returns the result of network_create_engine_wrapper.
"""
self.hidden = hidden
self.checkpoint_disabled = checkpoint_disabled
self.checkpoint_disabled = checkpoint_disabled or self.dummy
self.config = config or DownloadConfig(state_dir=self.session.config.get_state_dir())

self._logger.debug("Setup: %s", hexlify(self.tdef.get_infohash()))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,9 @@ def encode_atp(atp):

class DownloadManager(TaskManager):

def __init__(self, tribler_session):
def __init__(self, tribler_session, dummy_mode=False):
super(DownloadManager, self).__init__()
self.dummy_mode = dummy_mode
self._logger = logging.getLogger(self.__class__.__name__)

self.tribler_session = tribler_session
Expand Down Expand Up @@ -87,7 +88,8 @@ def __init__(self, tribler_session):
# Status of libtorrent session to indicate if it can safely close and no pending writes to disk exists.
self.lt_session_shutdown_ready = {}
self._dht_ready_task = None
self.dht_readiness_timeout = tribler_session.config.get_libtorrent_dht_readiness_timeout()
self.dht_readiness_timeout = tribler_session.config.get_libtorrent_dht_readiness_timeout() \
if not self.dummy_mode else 0

async def _check_dht_ready(self, min_dht_peers=60):
# Checks whether we got enough DHT peers. If the number of DHT peers is low,
Expand Down Expand Up @@ -175,7 +177,14 @@ def create_session(self, hops=0, store_listen_port=True):

# Elric: Strip out the -rcX, -beta, -whatever tail on the version string.
fingerprint = ['TL'] + [int(x) for x in version_id.split('-')[0].split('.')] + [0]
ltsession = lt.session(lt.fingerprint(*fingerprint), flags=0) if hops == 0 else lt.session(flags=0)
if self.dummy_mode:
from unittest.mock import Mock
ltsession = Mock()
ltsession.pop_alerts = lambda: {}
ltsession.listen_port = lambda: 123
ltsession.get_settings = lambda: {"peer_fingerprint": "000"}
else:
ltsession = lt.session(lt.fingerprint(*fingerprint), flags=0) if hops == 0 else lt.session(flags=0)

if hops == 0:
settings['user_agent'] = 'Tribler/' + version_id
Expand Down Expand Up @@ -246,7 +255,7 @@ def create_session(self, hops=0, store_listen_port=True):
'download_rate_limit': self.tribler_session.config.get_libtorrent_max_download_rate()}
self.set_session_settings(ltsession, settings)

if self.tribler_session.config.get_libtorrent_dht_enabled():
if self.tribler_session.config.get_libtorrent_dht_enabled() and not self.dummy_mode:
ltsession.start_dht()
for router in DEFAULT_DHT_ROUTERS:
ltsession.add_dht_router(*router)
Expand Down Expand Up @@ -523,14 +532,15 @@ def start_download(self, torrent_file=None, tdef=None, config=None, checkpoint_d
config.set_time_added(int(timemod.time()))

# Create the download
download = Download(self.tribler_session, tdef)
download = Download(self.tribler_session, tdef, dummy=self.dummy_mode)
atp = download.setup(config, checkpoint_disabled=checkpoint_disabled,
hidden=hidden or config.get_bootstrap_download())
# Keep metainfo downloads in self.downloads for now because we will need to remove it later,
# and removing the download at this point will stop us from receiving any further alerts.
if infohash not in self.metainfo_requests or self.metainfo_requests[infohash][0] == download:
self.downloads[infohash] = download
self.start_handle(download, atp)
if not self.dummy_mode:
self.start_handle(download, atp)
return download

@task
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,13 @@
my_key = default_eccrypto.generate_key(u"curve25519")


ALL_PRINTABLE_CHARS = ''.join(tuple(chr(i) for i in range(32, 0x110000) if chr(i).isprintable()))


def get_random_text_string(size=200):
return "".join(random.sample(ALL_PRINTABLE_CHARS, size))


def gen_random_entry():
return {
"title": "test entry " + str(random.randint(0, 1000000)),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,16 @@

CHANNEL_DIR = TESTS_DATA_DIR / 'sample_channel'
CHANNEL_TORRENT = CHANNEL_DIR / 'channel.torrent'
CHANNEL_TORRENT_UPDATED = CHANNEL_DIR / 'channel_upd.torrent'
CHANNEL_METADATA = CHANNEL_DIR / 'channel.mdblob'
CHANNEL_METADATA_UPDATED = CHANNEL_DIR / 'channel_upd.mdblob'


@pytest.mark.asyncio
@pytest.mark.timeout(20)
async def test_channel_update_and_download(enable_chant, enable_libtorrent, channel_tdef, channel_seeder_session,
session):
async def test_channel_update_and_download(
enable_chant, enable_libtorrent, channel_tdef, channel_seeder_session, session
):
"""
Test whether we can successfully update a channel and download the new version
"""
Expand Down
60 changes: 60 additions & 0 deletions src/tribler-core/tribler_core/modules/metadata_store/utils.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
from ipv8.keyvault.crypto import default_eccrypto

from pony.orm import db_session

from tribler_core.utilities.random_utils import random_infohash, random_utf8_string


@db_session
def generate_torrent(metadata_store, parent):
metadata_store.TorrentMetadata(title=random_utf8_string(50), infohash=random_infohash(), origin_id=parent.id_)


@db_session
def generate_collection(metadata_store, parent):
coll = metadata_store.CollectionNode(title=random_utf8_string(50), origin_id=parent.id_)
for _ in range(0, 3):
generate_torrent(metadata_store, coll)


@db_session
def generate_channel(metadata_store, title=None, subscribed=False):
# Remember and restore the original key
orig_key = metadata_store.ChannelNode._my_key

metadata_store.ChannelNode._my_key = default_eccrypto.generate_key('low')
chan = metadata_store.ChannelMetadata(
title=title or random_utf8_string(100), subscribed=subscribed, infohash=random_infohash()
)

# add some collections to the channel
for _ in range(0, 3):
generate_collection(metadata_store, chan)

metadata_store.ChannelNode._my_key = orig_key


@db_session
def generate_test_channels(metadata_store):
# First, generate some foreign channels
for ind in range(0, 10):
generate_channel(metadata_store, subscribed=ind % 2 == 0)

# This one is necessary to test filters, etc
generate_channel(metadata_store, title="non-random channel name")

# The same, but subscribed
generate_channel(metadata_store, title="non-random subscribed channel name", subscribed=True)

# Now generate a couple of personal channels
chan1 = metadata_store.ChannelMetadata.create_channel(title="personal channel with non-random name")
for _ in range(0, 3):
generate_collection(metadata_store, chan1)
chan1.commit_channel_torrent()

chan2 = metadata_store.ChannelMetadata.create_channel(title="personal channel " + random_utf8_string(50))
for _ in range(0, 3):
generate_collection(metadata_store, chan2)

# add 'Tribler' entry to facilitate keyword search tests
generate_channel(metadata_store, title="Tribler tribler chan", subscribed=True)
47 changes: 33 additions & 14 deletions src/tribler-core/tribler_core/session.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
from ipv8.dht.churn import PingChurn
from ipv8.dht.provider import DHTCommunityProvider
from ipv8.messaging.anonymization.community import TunnelSettings
from ipv8.messaging.interfaces.dispatcher.endpoint import DispatcherEndpoint
from ipv8.peer import Peer
from ipv8.peerdiscovery.churn import RandomChurn
from ipv8.peerdiscovery.community import DiscoveryCommunity, PeriodicSimilarity
Expand Down Expand Up @@ -48,6 +49,7 @@
from tribler_core.modules.bootstrap import Bootstrap
from tribler_core.modules.metadata_store.gigachannel_manager import GigaChannelManager
from tribler_core.modules.metadata_store.store import MetadataStore
from tribler_core.modules.metadata_store.utils import generate_test_channels
from tribler_core.modules.payout_manager import PayoutManager
from tribler_core.modules.resource_monitor import ResourceMonitor
from tribler_core.modules.torrent_checker.torrent_checker import TorrentChecker
Expand Down Expand Up @@ -89,11 +91,11 @@ class Session(TaskManager):
"""
__single = None

def __init__(self, config):
def __init__(self, config, core_test_mode = False):
"""
A Session object is created
Only a single session instance can exist at a time in a process.
:config TriblerConfig object parametrising the Session
:config TriblerConfig object parametrizing the Session
"""
super(Session, self).__init__()

Expand Down Expand Up @@ -143,6 +145,9 @@ def __init__(self, config):
self.payout_manager = None
self.mds = None # Metadata Store

# In test mode, the Core does not communicate with the external world and the state dir is read-only
self.core_test_mode = core_test_mode

def load_ipv8_overlays(self):
if self.config.get_trustchain_testnet():
peer = Peer(self.trustchain_testnet_keypair)
Expand Down Expand Up @@ -299,6 +304,9 @@ def create_in_state_dir(path):

def get_ports_in_config(self):
"""Claim all required random ports."""
if self.core_test_mode:
self.config.selected_ports = {}
return
self.config.get_libtorrent_port()
self.config.get_anon_listen_port()
self.config.get_tunnel_community_socks5_listen_ports()
Expand Down Expand Up @@ -396,7 +404,7 @@ async def start(self):
self.readable_status = STATE_START_API
await self.api_manager.start()

if self.upgrader_enabled:
if self.upgrader_enabled and not self.core_test_mode:
self.upgrader = TriblerUpgrader(self)
self.readable_status = STATE_UPGRADING_READABLE
try:
Expand All @@ -415,7 +423,10 @@ async def start(self):
channels_dir = self.config.get_chant_channels_dir()
metadata_db_name = 'metadata.db' if not self.config.get_chant_testnet() else 'metadata_testnet.db'
database_path = self.config.get_state_dir() / 'sqlite' / metadata_db_name
self.mds = MetadataStore(database_path, channels_dir, self.trustchain_keypair)
self.mds = MetadataStore(database_path, channels_dir, self.trustchain_keypair,
disable_sync=self.core_test_mode)
if self.core_test_mode:
generate_test_channels(self.mds)

# IPv8
if self.config.get_ipv8_enabled():
Expand All @@ -433,15 +444,19 @@ async def start(self):
community_file._DEFAULT_ADDRESSES = [self.config.get_ipv8_bootstrap_override()]
community_file._DNS_ADDRESSES = []

self.ipv8 = IPv8(ipv8_config_builder.finalize(), enable_statistics=self.config.get_ipv8_statistics())
self.ipv8 = IPv8(ipv8_config_builder.finalize(),
enable_statistics=self.config.get_ipv8_statistics()) \
if not self.core_test_mode else IPv8(ipv8_config_builder.finalize(),
endpoint_override=DispatcherEndpoint([]))
await self.ipv8.start()

self.config.set_anon_proxy_settings(2, ("127.0.0.1",
self.
config.get_tunnel_community_socks5_listen_ports()))
self.ipv8_start_time = timemod.time()
self.load_ipv8_overlays()
self.enable_ipv8_statistics()
if not self.core_test_mode:
self.enable_ipv8_statistics()
if self.api_manager:
self.api_manager.set_ipv8_session(self.ipv8)
if self.config.get_tunnel_community_enabled():
Expand All @@ -456,13 +471,15 @@ async def start(self):
if self.config.get_libtorrent_enabled():
self.readable_status = STATE_START_LIBTORRENT
from tribler_core.modules.libtorrent.download_manager import DownloadManager
self.dlmgr = DownloadManager(self)
self.dlmgr = DownloadManager(self, dummy_mode=self.core_test_mode)
self.dlmgr.initialize()
self.readable_status = STATE_LOAD_CHECKPOINTS
await self.dlmgr.load_checkpoints()
if self.core_test_mode:
await self.dlmgr.start_download_from_uri("magnet:?xt=urn:btih:0000000000000000000000000000000000000000")
self.readable_status = STATE_READABLE_STARTED

if self.config.get_torrent_checking_enabled():
if self.config.get_torrent_checking_enabled() and not self.core_test_mode:
self.readable_status = STATE_START_TORRENT_CHECKER
self.torrent_checker = TorrentChecker(self)
await self.torrent_checker.initialize()
Expand All @@ -480,11 +497,11 @@ async def start(self):
self.watch_folder = WatchFolder(self)
self.watch_folder.start()

if self.config.get_resource_monitor_enabled():
if self.config.get_resource_monitor_enabled() and not self.core_test_mode:
self.resource_monitor = ResourceMonitor(self)
self.resource_monitor.start()

if self.config.get_version_checker_enabled():
if self.config.get_version_checker_enabled() and not self.core_test_mode:
self.version_check_manager = VersionCheckManager(self)
self.version_check_manager.start()

Expand All @@ -496,9 +513,10 @@ async def start(self):
if self.config.get_chant_enabled() and self.config.get_chant_manager_enabled()\
and self.config.get_libtorrent_enabled:
self.gigachannel_manager = GigaChannelManager(self)
self.gigachannel_manager.start()
if not self.core_test_mode:
self.gigachannel_manager.start()

if self.config.get_bootstrap_enabled():
if self.config.get_bootstrap_enabled() and not self.core_test_mode:
self.register_task('bootstrap_download', self.start_bootstrap_download)

self.notifier.notify(NTFY.TRIBLER_STARTED)
Expand Down Expand Up @@ -571,8 +589,9 @@ async def shutdown(self):
await self.watch_folder.stop()
self.watch_folder = None

self.notify_shutdown_state("Saving configuration...")
self.config.write()
if not self.core_test_mode:
self.notify_shutdown_state("Saving configuration...")
self.config.write()

if self.dlmgr:
await self.dlmgr.shutdown()
Expand Down

This file was deleted.

Loading

0 comments on commit fecebd6

Please sign in to comment.