From 1d7592aba80358258e4ef9c378170bf409e9c01d Mon Sep 17 00:00:00 2001 From: drupman Date: Mon, 4 Sep 2023 17:37:07 -0300 Subject: [PATCH 01/40] (fix) download and display all data instead of page data --- pages/strategy_performance/app.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pages/strategy_performance/app.py b/pages/strategy_performance/app.py index 6491ec6b..28874dc7 100644 --- a/pages/strategy_performance/app.py +++ b/pages/strategy_performance/app.py @@ -156,11 +156,11 @@ def download_csv(df: pd.DataFrame, filename: str, key: str): st.markdown("
", unsafe_allow_html=True) st.subheader("Tables") with st.expander("💵 Trades"): - st.write(strategy_data_filtered.trade_fill) - download_csv(strategy_data_filtered.trade_fill, "trade_fill", "download-trades") + st.write(strategy_data.trade_fill) + download_csv(strategy_data.trade_fill, "trade_fill", "download-trades") with st.expander("📩 Orders"): - st.write(strategy_data_filtered.orders) - download_csv(strategy_data_filtered.orders, "orders", "download-orders") + st.write(strategy_data.orders) + download_csv(strategy_data.orders, "orders", "download-orders") with st.expander("⌕ Order Status"): - st.write(strategy_data_filtered.order_status) - download_csv(strategy_data_filtered.order_status, "order_status", "download-order-status") + st.write(strategy_data.order_status) + download_csv(strategy_data.order_status, "order_status", "download-order-status") From 179f15932246bacad35669ae56fa342608909237 Mon Sep 17 00:00:00 2001 From: drupman Date: Mon, 4 Sep 2023 17:38:06 -0300 Subject: [PATCH 02/40] (refactor) replace unsafe_html with adhoc st element --- pages/strategy_performance/app.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pages/strategy_performance/app.py b/pages/strategy_performance/app.py index 28874dc7..271aea09 100644 --- a/pages/strategy_performance/app.py +++ b/pages/strategy_performance/app.py @@ -75,7 +75,7 @@ def download_csv(df: pd.DataFrame, filename: str, key: str): value=(date_array[0], date_array[-1])) strategy_data_filtered = single_market_strategy_data.get_filtered_strategy_data(start_time, end_time) - st.markdown("
", unsafe_allow_html=True) + st.divider() with st.container(): col1, col2 = st.columns(2) with col1: @@ -94,7 +94,7 @@ def download_csv(df: pd.DataFrame, filename: str, key: str): st.metric(label='Duration (Hours)', value=round(strategy_data_filtered.duration_seconds / 3600, 2)) st.metric(label='Price change', value=f"{round(strategy_data_filtered.price_change * 100, 2)} %") - st.markdown("
", unsafe_allow_html=True) + st.divider() st.subheader("📈 Performance") col131, col132, col133, col134 = st.columns(4) with col131: @@ -120,7 +120,7 @@ def download_csv(df: pd.DataFrame, filename: str, key: str): st.metric(label='Average Buy Price', value=round(strategy_data_filtered.average_buy_price, 4)) st.metric(label='Average Sell Price', value=round(strategy_data_filtered.average_sell_price, 4)) - st.markdown("
", unsafe_allow_html=True) + st.divider() st.subheader("🕯️ Candlestick") if strategy_data_filtered.market_data is not None: with st.expander("Market activity", expanded=True): @@ -153,7 +153,7 @@ def download_csv(df: pd.DataFrame, filename: str, key: str): else: st.warning("Market data is not available so the candles graph is not going to be rendered. " "Make sure that you are using the latest version of Hummingbot and market data recorder activated.") - st.markdown("
", unsafe_allow_html=True) + st.divider() st.subheader("Tables") with st.expander("💵 Trades"): st.write(strategy_data.trade_fill) From 45c6417218fdca860b006200e82b336697639862 Mon Sep 17 00:00:00 2001 From: drupman Date: Mon, 4 Sep 2023 17:49:20 -0300 Subject: [PATCH 03/40] (fix) remove session state --- pages/strategy_performance/app.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/pages/strategy_performance/app.py b/pages/strategy_performance/app.py index 271aea09..813bbc0c 100644 --- a/pages/strategy_performance/app.py +++ b/pages/strategy_performance/app.py @@ -41,8 +41,8 @@ def download_csv(df: pd.DataFrame, filename: str, key: str): ) -st.session_state["dbs"] = get_databases() -db_names = [x.db_name for x in st.session_state["dbs"].values() if x.status == 'OK'] +dbs = get_databases() +db_names = [x.db_name for x in dbs.values() if x.status == 'OK'] if not db_names: st.warning("No trades have been recorded in the selected database") selected_db_name = None @@ -52,23 +52,23 @@ def download_csv(df: pd.DataFrame, filename: str, key: str): col1, col2, col3, col4 = st.columns(4) with col1: selected_db_name = st.selectbox("Select a database to use:", db_names) - st.session_state["selected_db"] = st.session_state["dbs"][selected_db_name] + selected_db = dbs[selected_db_name] with col2: - if st.session_state.selected_db: - st.session_state.selected_config_file = st.selectbox("Select a config file to analyze:", st.session_state.selected_db.config_files) + if selected_db: + selected_config_file = st.selectbox("Select a config file to analyze:", selected_db.config_files) else: - st.session_state.selected_config_file = None + selected_config_file = None with col3: - if st.session_state.selected_config_file: - st.session_state.selected_exchange = st.selectbox("Exchange:", st.session_state.selected_db.configs[st.session_state.selected_config_file].keys()) + if selected_config_file: + selected_exchange = st.selectbox("Exchange:", selected_db.configs[selected_config_file].keys()) with col4: - if st.session_state.selected_exchange: - st.session_state.selected_trading_pair = st.selectbox("Trading Pair:", options=st.session_state.selected_db.configs[st.session_state.selected_config_file][st.session_state.selected_exchange]) + if selected_exchange: + selected_trading_pair = st.selectbox("Trading Pair:", options=selected_db.configs[selected_config_file][selected_exchange]) single_market = True if single_market: - strategy_data = st.session_state["dbs"][selected_db_name].get_strategy_data(st.session_state.selected_config_file) - single_market_strategy_data = strategy_data.get_single_market_strategy_data(st.session_state.selected_exchange, st.session_state.selected_trading_pair) + strategy_data = selected_db.get_strategy_data(selected_config_file) + single_market_strategy_data = strategy_data.get_single_market_strategy_data(selected_exchange, selected_trading_pair) date_array = pd.date_range(start=strategy_data.start_time, end=strategy_data.end_time, periods=60) start_time, end_time = st.select_slider("Select a time range to analyze", options=date_array.tolist(), From 6d645d925af9f108c9be0d36f5c12fa5a2807f5a Mon Sep 17 00:00:00 2001 From: drupman Date: Mon, 4 Sep 2023 18:29:55 -0300 Subject: [PATCH 04/40] (feat) add position executor data to strategy data --- utils/data_manipulation.py | 14 ++++++++++ utils/database_manager.py | 55 +++++++++++++++++++++++++++++++------- 2 files changed, 60 insertions(+), 9 deletions(-) diff --git a/utils/data_manipulation.py b/utils/data_manipulation.py index e7c9dd18..0e1f5774 100644 --- a/utils/data_manipulation.py +++ b/utils/data_manipulation.py @@ -9,6 +9,7 @@ class StrategyData: order_status: pd.DataFrame trade_fill: pd.DataFrame market_data: pd.DataFrame = None + position_executor: pd.DataFrame = None def get_single_market_strategy_data(self, exchange: str, trading_pair: str): orders = self.orders[(self.orders["market"] == exchange) & (self.orders["symbol"] == trading_pair)].copy() @@ -19,6 +20,11 @@ def get_single_market_strategy_data(self, exchange: str, trading_pair: str): (self.market_data["trading_pair"] == trading_pair)].copy() else: market_data = None + if self.position_executor is not None: + position_executor = self.position_executor[(self.position_executor["exchange"] == exchange) & + (self.position_executor["trading_pair"] == trading_pair)].copy() + else: + position_executor = None return SingleMarketStrategyData( exchange=exchange, trading_pair=trading_pair, @@ -26,6 +32,7 @@ def get_single_market_strategy_data(self, exchange: str, trading_pair: str): order_status=order_status, trade_fill=trade_fill, market_data=market_data, + position_executor=position_executor ) @property @@ -77,6 +84,7 @@ class SingleMarketStrategyData: order_status: pd.DataFrame trade_fill: pd.DataFrame market_data: pd.DataFrame = None + position_executor: pd.DataFrame = None def get_filtered_strategy_data(self, start_date: datetime.datetime, end_date: datetime.datetime): orders = self.orders[ @@ -88,6 +96,11 @@ def get_filtered_strategy_data(self, start_date: datetime.datetime, end_date: da (self.market_data.index >= start_date) & (self.market_data.index <= end_date)].copy() else: market_data = None + if self.position_executor is not None: + position_executor = self.position_executor[(self.position_executor.datetime >= start_date) & + (self.position_executor.datetime <= end_date)].copy() + else: + position_executor = None return SingleMarketStrategyData( exchange=self.exchange, trading_pair=self.trading_pair, @@ -95,6 +108,7 @@ def get_filtered_strategy_data(self, start_date: datetime.datetime, end_date: da order_status=order_status, trade_fill=trade_fill, market_data=market_data, + position_executor=position_executor ) def get_market_data_resampled(self, interval): diff --git a/utils/database_manager.py b/utils/database_manager.py index ce9f6e91..5db62b6e 100644 --- a/utils/database_manager.py +++ b/utils/database_manager.py @@ -9,10 +9,11 @@ class DatabaseManager: - def __init__(self, db_name): + def __init__(self, db_name: str, executors_path: str = "data"): self.db_name = db_name # TODO: Create db path for all types of db self.db_path = f'sqlite:///{os.path.join("data", db_name)}' + self.executors_path = executors_path self.engine = create_engine(self.db_path, connect_args={'check_same_thread': False}) self.session_maker = sessionmaker(bind=self.engine) @@ -157,13 +158,49 @@ def get_market_data(self, start_date=None, end_date=None): market_data["best_ask"] = market_data["best_ask"] / 1e6 return market_data - def get_strategy_data(self, config_file_path=None, start_date=None, end_date=None): - orders = self.get_orders(config_file_path, start_date, end_date) - trade_fills = self.get_trade_fills(config_file_path, start_date, end_date) - order_status = self.get_order_status(orders['id'].tolist(), start_date, end_date) + def get_position_executor_data(self, start_date=None, end_date=None) -> pd.DataFrame: + df = pd.DataFrame() + files = [file for file in os.listdir(self.executors_path) if ".csv" in file and file is not "trades_market_making_.csv"] + for file in files: + df0 = pd.read_csv(f"{self.executors_path}/{file}") + df = pd.concat([df, df0]) + df["datetime"] = pd.to_datetime(df["timestamp"], unit="s") + if start_date: + df = df[df["datetime"] >= start_date] + if end_date: + df = df[df["datetime"] <= end_date] + return df + + @staticmethod + def _safe_table_loading(func, *args, **kwargs): try: - market_data = self.get_market_data(start_date, end_date) - except Exception as e: - market_data = None - strategy_data = StrategyData(orders, order_status, trade_fills, market_data) + table = func(*args, **kwargs) + except Exception: + table = None + return table + + def get_strategy_data(self, config_file_path=None, start_date=None, end_date=None): + def load_orders(): + return self.get_orders(config_file_path, start_date, end_date) + + def load_trade_fills(): + return self.get_trade_fills(config_file_path, start_date, end_date) + + def load_order_status(): + return self.get_order_status(orders['id'].tolist(), start_date, end_date) + + def load_market_data(): + return self.get_market_data(start_date, end_date) + + def load_position_executor(): + return self.get_position_executor_data(start_date, end_date) + + # Use _safe_table_loading to load tables + orders = self._safe_table_loading(load_orders) + trade_fills = self._safe_table_loading(load_trade_fills) + order_status = self._safe_table_loading(load_order_status) + market_data = self._safe_table_loading(load_market_data) + position_executor = self._safe_table_loading(load_position_executor) + + strategy_data = StrategyData(orders, order_status, trade_fills, market_data, position_executor) return strategy_data From 851041ae80aa3a9726b4ba260e5b033d65f435b0 Mon Sep 17 00:00:00 2001 From: drupman Date: Tue, 5 Sep 2023 17:50:31 -0300 Subject: [PATCH 05/40] (feat) allow get trade fills to process multiple exchange, trading pairs and config_file_path combinations --- utils/database_manager.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/utils/database_manager.py b/utils/database_manager.py index 5db62b6e..977c7196 100644 --- a/utils/database_manager.py +++ b/utils/database_manager.py @@ -121,21 +121,22 @@ def get_orders(self, config_file_path=None, start_date=None, end_date=None): return orders def get_trade_fills(self, config_file_path=None, start_date=None, end_date=None): + groupers = ["config_file_path", "market", "symbol"] with self.session_maker() as session: query = self._get_trade_fills_query(config_file_path, start_date, end_date) trade_fills = pd.read_sql_query(query, session.connection()) + trade_fills.sort_values(by="timestamp", ascending=True, inplace=True) trade_fills["amount"] = trade_fills["amount"] / 1e6 trade_fills["price"] = trade_fills["price"] / 1e6 trade_fills["trade_fee_in_quote"] = trade_fills["trade_fee_in_quote"] / 1e6 - trade_fills["cum_fees_in_quote"] = trade_fills["trade_fee_in_quote"].cumsum() - trade_fills.loc[:, "net_amount"] = trade_fills['amount'] * trade_fills['trade_type'].apply( - lambda x: 1 if x == 'BUY' else -1) - trade_fills.loc[:, "net_amount_quote"] = trade_fills['net_amount'] * trade_fills['price'] - trade_fills.loc[:, "cum_net_amount"] = trade_fills["net_amount"].cumsum() - trade_fills.loc[:, "unrealized_trade_pnl"] = -1 * trade_fills["net_amount_quote"].cumsum() - trade_fills.loc[:, "inventory_cost"] = trade_fills["cum_net_amount"] * trade_fills["price"] - trade_fills.loc[:, "realized_trade_pnl"] = trade_fills["unrealized_trade_pnl"] + trade_fills["inventory_cost"] - trade_fills.loc[:, "net_realized_pnl"] = trade_fills["realized_trade_pnl"] - trade_fills["cum_fees_in_quote"] + trade_fills["cum_fees_in_quote"] = trade_fills.groupby(groupers)["trade_fee_in_quote"].cumsum() + trade_fills["net_amount"] = trade_fills['amount'] * trade_fills['trade_type'].apply(lambda x: 1 if x == 'BUY' else -1) + trade_fills["net_amount_quote"] = trade_fills['net_amount'] * trade_fills['price'] + trade_fills["cum_net_amount"] = trade_fills.groupby(groupers)["net_amount"].cumsum() + trade_fills["unrealized_trade_pnl"] = -1 * trade_fills.groupby(groupers)["net_amount_quote"].cumsum() + trade_fills["inventory_cost"] = trade_fills["cum_net_amount"] * trade_fills["price"] + trade_fills["realized_trade_pnl"] = trade_fills["unrealized_trade_pnl"] + trade_fills["inventory_cost"] + trade_fills["net_realized_pnl"] = trade_fills["realized_trade_pnl"] - trade_fills["cum_fees_in_quote"] trade_fills["timestamp"] = pd.to_datetime(trade_fills["timestamp"], unit="ms") trade_fills["market"] = trade_fills["market"].apply(lambda x: x.lower().replace("_papertrade", "")) From a08d7c46f7067d09f2a60a46122cd74e8ebce5fc Mon Sep 17 00:00:00 2001 From: drupman Date: Tue, 5 Sep 2023 23:32:49 -0300 Subject: [PATCH 06/40] (feat) add strategy summary table to StrategyData class --- utils/data_manipulation.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/utils/data_manipulation.py b/utils/data_manipulation.py index 0e1f5774..3372fdac 100644 --- a/utils/data_manipulation.py +++ b/utils/data_manipulation.py @@ -11,6 +11,39 @@ class StrategyData: market_data: pd.DataFrame = None position_executor: pd.DataFrame = None + @property + def strategy_summary(self): + if self.trade_fill is not None: + return self.get_strategy_summary() + else: + return None + + def get_strategy_summary(self): + def full_series(series): + return list(series) + + strategy_data = self.trade_fill.copy() + strategy_data["volume"] = strategy_data["amount"] * strategy_data["price"] + strategy_data["margin_volume"] = strategy_data["amount"] * strategy_data["price"] / strategy_data["leverage"] + strategy_summary = strategy_data.groupby(["strategy", "market", "symbol"]).agg({"order_id": "count", + "volume": "sum", + "margin_volume": "sum", + "net_realized_pnl": [full_series, + "last"]}).reset_index() + strategy_summary.columns = [f"{col[0]}_{col[1]}" if isinstance(col, tuple) and col[1] is not None else col for col in strategy_summary.columns] + strategy_summary.rename(columns={"strategy_": "Strategy", + "market_": "Exchange", + "symbol_": "Trading Pair", + "order_id_count": "# Trades", + "volume_sum": "Volume", + "margin_volume_sum": "Margin volume", + "net_realized_pnl_full_series": "PnL Over Time", + "net_realized_pnl_last": "Realized PnL"}, inplace=True) + strategy_summary.sort_values(["Realized PnL"], ascending=True, inplace=True) + strategy_summary["Examine"] = False + strategy_summary["Examine"][0] = True + return strategy_summary + def get_single_market_strategy_data(self, exchange: str, trading_pair: str): orders = self.orders[(self.orders["market"] == exchange) & (self.orders["symbol"] == trading_pair)].copy() trade_fill = self.trade_fill[self.trade_fill["order_id"].isin(orders["id"])].copy() From 3c4b9cfd0dc4337131188e01e9f497bb46b7ae97 Mon Sep 17 00:00:00 2001 From: drupman Date: Tue, 5 Sep 2023 23:39:11 -0300 Subject: [PATCH 07/40] (feat) improvements in status property and loading data in database_manager.py + including in strategy performance page --- pages/strategy_performance/app.py | 270 +++++++++++++++++------------- utils/database_manager.py | 78 ++++----- 2 files changed, 189 insertions(+), 159 deletions(-) diff --git a/pages/strategy_performance/app.py b/pages/strategy_performance/app.py index 813bbc0c..c5fda1eb 100644 --- a/pages/strategy_performance/app.py +++ b/pages/strategy_performance/app.py @@ -2,7 +2,7 @@ import pandas as pd import streamlit as st import math - +import plotly.express as px from utils.database_manager import DatabaseManager from utils.graphs import CandlesGraph from utils.st_utils import initialize_st_page @@ -10,6 +10,9 @@ initialize_st_page(title="Strategy Performance", icon="🚀") +BULLISH_COLOR = "#61C766" +BEARISH_COLOR = "#FF665A" + # Start content here intervals = { "1m": 60, @@ -41,126 +44,163 @@ def download_csv(df: pd.DataFrame, filename: str, key: str): ) +def show_strategy_summary(summary_df: pd.DataFrame): + summary = st.data_editor(summary_df, + column_config={"PnL Over Time": st.column_config.LineChartColumn("PnL Over Time", + y_min=0, + y_max=5000), + "Examine": st.column_config.CheckboxColumn(required=True) + }, + use_container_width=True + ) + selected_rows = summary[summary.Examine] + return selected_rows.drop('Examine', axis=1) + + +def summary_chart(df: pd.DataFrame): + fig = px.bar(df, x="Trading Pair", y="Realized PnL", color="Exchange") + fig.update_traces(width=min(1.0, 0.1 * len(strategy_data.strategy_summary))) + return fig + + dbs = get_databases() -db_names = [x.db_name for x in dbs.values() if x.status == 'OK'] +db_names = [x.db_name for x in dbs.values()] if not db_names: st.warning("No trades have been recorded in the selected database") selected_db_name = None selected_db = None else: - st.subheader("⚙️ Filters") - col1, col2, col3, col4 = st.columns(4) - with col1: + st.subheader("🔫 Data source") + select_tab, upload_tab = st.tabs(["Select", "Upload"]) + with select_tab: selected_db_name = st.selectbox("Select a database to use:", db_names) selected_db = dbs[selected_db_name] - with col2: - if selected_db: - selected_config_file = st.selectbox("Select a config file to analyze:", selected_db.config_files) - else: - selected_config_file = None - with col3: - if selected_config_file: - selected_exchange = st.selectbox("Exchange:", selected_db.configs[selected_config_file].keys()) - with col4: - if selected_exchange: - selected_trading_pair = st.selectbox("Trading Pair:", options=selected_db.configs[selected_config_file][selected_exchange]) - - single_market = True - if single_market: - strategy_data = selected_db.get_strategy_data(selected_config_file) - single_market_strategy_data = strategy_data.get_single_market_strategy_data(selected_exchange, selected_trading_pair) - date_array = pd.date_range(start=strategy_data.start_time, end=strategy_data.end_time, periods=60) - start_time, end_time = st.select_slider("Select a time range to analyze", - options=date_array.tolist(), - value=(date_array[0], date_array[-1])) - strategy_data_filtered = single_market_strategy_data.get_filtered_strategy_data(start_time, end_time) - - st.divider() - with st.container(): - col1, col2 = st.columns(2) - with col1: - st.subheader(f"🏦 Market") - with col2: - st.subheader("📋 General stats") - col1, col2, col3, col4 = st.columns(4) - with col1: - st.metric(label="Exchange", value=strategy_data_filtered.exchange.capitalize()) - with col2: - st.metric(label="Trading pair", value=strategy_data_filtered.trading_pair.upper()) - with col3: - st.metric(label='Start date', value=strategy_data_filtered.start_time.strftime("%Y-%m-%d %H:%M")) - st.metric(label='End date', value=strategy_data_filtered.end_time.strftime("%Y-%m-%d %H:%M")) - with col4: - st.metric(label='Duration (Hours)', value=round(strategy_data_filtered.duration_seconds / 3600, 2)) - st.metric(label='Price change', value=f"{round(strategy_data_filtered.price_change * 100, 2)} %") - - st.divider() - st.subheader("📈 Performance") - col131, col132, col133, col134 = st.columns(4) - with col131: - st.metric(label=f'Net PNL {strategy_data_filtered.quote_asset}', - value=round(strategy_data_filtered.net_pnl_quote, 2)) - st.metric(label=f'Trade PNL {strategy_data_filtered.quote_asset}', - value=round(strategy_data_filtered.trade_pnl_quote, 2)) - st.metric(label=f'Fees {strategy_data_filtered.quote_asset}', - value=round(strategy_data_filtered.cum_fees_in_quote, 2)) - with col132: - st.metric(label='Total Trades', value=strategy_data_filtered.total_orders) - st.metric(label='Total Buy Trades', value=strategy_data_filtered.total_buy_trades) - st.metric(label='Total Sell Trades', value=strategy_data_filtered.total_sell_trades) - with col133: - st.metric(label='Inventory change in Base asset', - value=round(strategy_data_filtered.inventory_change_base_asset, 4)) - st.metric(label='Total Buy Trades Amount', - value=round(strategy_data_filtered.total_buy_amount, 2)) - st.metric(label='Total Sell Trades Amount', - value=round(strategy_data_filtered.total_sell_amount, 2)) - with col134: - st.metric(label='End Price', value=round(strategy_data_filtered.end_price, 4)) - st.metric(label='Average Buy Price', value=round(strategy_data_filtered.average_buy_price, 4)) - st.metric(label='Average Sell Price', value=round(strategy_data_filtered.average_sell_price, 4)) - - st.divider() - st.subheader("🕯️ Candlestick") - if strategy_data_filtered.market_data is not None: - with st.expander("Market activity", expanded=True): - col1, col2, col3 = st.columns([1, 1, 2]) - with col1: - interval = st.selectbox("Candles Interval:", intervals.keys(), index=2) - with col2: - rows_per_page = st.number_input("Candles per Page", value=100, min_value=1, max_value=5000) - with col3: - total_rows = len(strategy_data_filtered.get_market_data_resampled(interval=f"{intervals[interval]}S")) - total_pages = math.ceil(total_rows / rows_per_page) - if total_pages > 1: - selected_page = st.select_slider("Select page", list(range(total_pages)), key="page_slider") - else: - selected_page = 0 - start_idx = selected_page * rows_per_page - end_idx = start_idx + rows_per_page - candles_df = strategy_data_filtered.get_market_data_resampled(interval=f"{intervals[interval]}S").iloc[ - start_idx:end_idx] - start_time_page = candles_df.index.min() - end_time_page = candles_df.index.max() - page_data_filtered = single_market_strategy_data.get_filtered_strategy_data(start_time_page, end_time_page) - cg = CandlesGraph(candles_df, show_volume=False, extra_rows=2) - cg.add_buy_trades(strategy_data_filtered.buys) - cg.add_sell_trades(strategy_data_filtered.sells) - cg.add_pnl(strategy_data_filtered, row=2) - cg.add_base_inventory_change(strategy_data_filtered, row=3) - fig = cg.figure() - st.plotly_chart(fig, use_container_width=True) + with upload_tab: + uploaded_db = st.file_uploader("Upload your sqlite database", type=["sqlite", "db"]) + if uploaded_db is not None: + selected_db = DatabaseManager(uploaded_db) + strategy_data = selected_db.get_strategy_data() + if strategy_data.strategy_summary is not None: + st.subheader("📝 Strategy summary") + table_tab, chart_tab = st.tabs(["Table", "Chart"]) + with table_tab: + selection = show_strategy_summary(strategy_data.strategy_summary) + selected_exchange = selection["Exchange"].values[0] + selected_trading_pair = selection["Trading Pair"].values[0] + with chart_tab: + summary_chart = summary_chart(strategy_data.strategy_summary) + st.plotly_chart(summary_chart, use_container_width=True) + st.subheader("🔍 Examine Trading Pair") + if not any("🇽" in value for value in selected_db.status.values()): + date_array = pd.date_range(start=strategy_data.start_time, end=strategy_data.end_time, periods=60) + start_time, end_time = st.select_slider("Select a time range to analyze", + options=date_array.tolist(), + value=(date_array[0], date_array[-1])) + + single_market = True + if single_market: + single_market_strategy_data = strategy_data.get_single_market_strategy_data(selected_exchange, selected_trading_pair) + strategy_data_filtered = single_market_strategy_data.get_filtered_strategy_data(start_time, end_time) + + st.divider() + with st.container(): + col1, col2 = st.columns(2) + with col1: + st.subheader(f"🏦 Market") + with col2: + st.subheader("📋 General stats") + col1, col2, col3, col4 = st.columns(4) + with col1: + st.metric(label="Exchange", value=strategy_data_filtered.exchange.capitalize()) + with col2: + st.metric(label="Trading pair", value=strategy_data_filtered.trading_pair.upper()) + with col3: + st.metric(label='Start date', value=strategy_data_filtered.start_time.strftime("%Y-%m-%d %H:%M")) + st.metric(label='End date', value=strategy_data_filtered.end_time.strftime("%Y-%m-%d %H:%M")) + with col4: + st.metric(label='Duration (Hours)', value=round(strategy_data_filtered.duration_seconds / 3600, 2)) + st.metric(label='Price change', value=f"{round(strategy_data_filtered.price_change * 100, 2)} %") + + st.divider() + st.subheader("📈 Performance") + col131, col132, col133, col134 = st.columns(4) + with col131: + st.metric(label=f'Net PNL {strategy_data_filtered.quote_asset}', + value=round(strategy_data_filtered.net_pnl_quote, 2)) + st.metric(label=f'Trade PNL {strategy_data_filtered.quote_asset}', + value=round(strategy_data_filtered.trade_pnl_quote, 2)) + st.metric(label=f'Fees {strategy_data_filtered.quote_asset}', + value=round(strategy_data_filtered.cum_fees_in_quote, 2)) + with col132: + st.metric(label='Total Trades', value=strategy_data_filtered.total_orders) + st.metric(label='Total Buy Trades', value=strategy_data_filtered.total_buy_trades) + st.metric(label='Total Sell Trades', value=strategy_data_filtered.total_sell_trades) + with col133: + st.metric(label='Inventory change in Base asset', + value=round(strategy_data_filtered.inventory_change_base_asset, 4)) + st.metric(label='Total Buy Trades Amount', + value=round(strategy_data_filtered.total_buy_amount, 2)) + st.metric(label='Total Sell Trades Amount', + value=round(strategy_data_filtered.total_sell_amount, 2)) + with col134: + st.metric(label='End Price', value=round(strategy_data_filtered.end_price, 4)) + st.metric(label='Average Buy Price', value=round(strategy_data_filtered.average_buy_price, 4)) + st.metric(label='Average Sell Price', value=round(strategy_data_filtered.average_sell_price, 4)) + + st.divider() + st.subheader("🕯️ Candlestick") + if strategy_data_filtered.market_data is not None: + with st.expander("Market activity", expanded=True): + col1, col2, col3 = st.columns([1, 1, 2]) + with col1: + interval = st.selectbox("Candles Interval:", intervals.keys(), index=2) + with col2: + rows_per_page = st.number_input("Candles per Page", value=100, min_value=1, max_value=5000) + with col3: + total_rows = len(strategy_data_filtered.get_market_data_resampled(interval=f"{intervals[interval]}S")) + total_pages = math.ceil(total_rows / rows_per_page) + if total_pages > 1: + selected_page = st.select_slider("Select page", list(range(total_pages)), key="page_slider") + else: + selected_page = 0 + start_idx = selected_page * rows_per_page + end_idx = start_idx + rows_per_page + candles_df = strategy_data_filtered.get_market_data_resampled(interval=f"{intervals[interval]}S").iloc[ + start_idx:end_idx] + start_time_page = candles_df.index.min() + end_time_page = candles_df.index.max() + page_data_filtered = single_market_strategy_data.get_filtered_strategy_data(start_time_page, end_time_page) + cg = CandlesGraph(candles_df, show_volume=False, extra_rows=2) + cg.add_buy_trades(strategy_data_filtered.buys) + cg.add_sell_trades(strategy_data_filtered.sells) + cg.add_pnl(strategy_data_filtered, row=2) + cg.add_base_inventory_change(strategy_data_filtered, row=3) + fig = cg.figure() + st.plotly_chart(fig, use_container_width=True) + else: + st.warning("Market data is not available so the candles graph is not going to be rendered. " + "Make sure that you are using the latest version of Hummingbot and market data recorder activated.") + st.divider() + st.subheader("Tables") + with st.expander("💵 Trades"): + st.write(strategy_data.trade_fill) + download_csv(strategy_data.trade_fill, "trade_fill", "download-trades") + with st.expander("📩 Orders"): + st.write(strategy_data.orders) + download_csv(strategy_data.orders, "orders", "download-orders") + with st.expander("⌕ Order Status"): + st.write(strategy_data.order_status) + download_csv(strategy_data.order_status, "order_status", "download-order-status") else: - st.warning("Market data is not available so the candles graph is not going to be rendered. " - "Make sure that you are using the latest version of Hummingbot and market data recorder activated.") - st.divider() - st.subheader("Tables") - with st.expander("💵 Trades"): - st.write(strategy_data.trade_fill) - download_csv(strategy_data.trade_fill, "trade_fill", "download-trades") - with st.expander("📩 Orders"): - st.write(strategy_data.orders) - download_csv(strategy_data.orders, "orders", "download-orders") - with st.expander("⌕ Order Status"): - st.write(strategy_data.order_status) - download_csv(strategy_data.order_status, "order_status", "download-order-status") + st.warning("We have problems to keep analyzing this database") + with st.expander("DB Status"): + status_df = pd.DataFrame([selected_db.status]).transpose().reset_index() + status_df.columns = ["Attribute", "Value"] + st.table(status_df) + + else: + st.warning("We couldn't process this sqlite database.") + with st.expander("DB Status"): + status_df = pd.DataFrame([selected_db.status]).transpose().reset_index() + status_df.columns = ["Attribute", "Value"] + st.table(status_df) diff --git a/utils/database_manager.py b/utils/database_manager.py index 977c7196..54db9727 100644 --- a/utils/database_manager.py +++ b/utils/database_manager.py @@ -17,19 +17,41 @@ def __init__(self, db_name: str, executors_path: str = "data"): self.engine = create_engine(self.db_path, connect_args={'check_same_thread': False}) self.session_maker = sessionmaker(bind=self.engine) - @property - def status(self): + def get_strategy_data(self, config_file_path=None, start_date=None, end_date=None): + def load_data(table_loader): + try: + return table_loader() + except Exception as e: + return None # Return None to indicate failure + + # Use load_data to load tables + orders = load_data(self.get_orders) + trade_fills = load_data(self.get_trade_fills) + order_status = load_data(self.get_order_status) + market_data = load_data(self.get_market_data) + position_executor = load_data(self.get_position_executor_data) + + strategy_data = StrategyData(orders, order_status, trade_fills, market_data, position_executor) + return strategy_data + + @staticmethod + def _get_table_status(table_loader): try: - with self.session_maker() as session: - query = 'SELECT DISTINCT config_file_path FROM TradeFill' - config_files = pd.read_sql_query(query, session.connection()) - if len(config_files) > 0: - # TODO: improve error handling, think what to do with other cases - return "OK" - else: - return "No records found in the TradeFill table with non-null config_file_path" + data = table_loader() + return "Correct" if len(data) > 0 else f"Error - No records matched" except Exception as e: - return f"Error: {str(e)}" + return f"Error - {str(e)}" + + @property + def status(self): + status = {"db_name": self.db_name, + "trade_fill": self._get_table_status(self.get_trade_fills), + "orders": self._get_table_status(self.get_orders), + "order_status": self._get_table_status(self.get_order_status), + "market_data": self._get_table_status(self.get_market_data), + "position_executor": self._get_table_status(self.get_position_executor_data), + } + return status @property def config_files(self): @@ -161,7 +183,7 @@ def get_market_data(self, start_date=None, end_date=None): def get_position_executor_data(self, start_date=None, end_date=None) -> pd.DataFrame: df = pd.DataFrame() - files = [file for file in os.listdir(self.executors_path) if ".csv" in file and file is not "trades_market_making_.csv"] + files = [file for file in os.listdir(self.executors_path) if ".csv" in file and file != "trades_market_making_.csv"] for file in files: df0 = pd.read_csv(f"{self.executors_path}/{file}") df = pd.concat([df, df0]) @@ -172,36 +194,4 @@ def get_position_executor_data(self, start_date=None, end_date=None) -> pd.DataF df = df[df["datetime"] <= end_date] return df - @staticmethod - def _safe_table_loading(func, *args, **kwargs): - try: - table = func(*args, **kwargs) - except Exception: - table = None - return table - - def get_strategy_data(self, config_file_path=None, start_date=None, end_date=None): - def load_orders(): - return self.get_orders(config_file_path, start_date, end_date) - - def load_trade_fills(): - return self.get_trade_fills(config_file_path, start_date, end_date) - - def load_order_status(): - return self.get_order_status(orders['id'].tolist(), start_date, end_date) - def load_market_data(): - return self.get_market_data(start_date, end_date) - - def load_position_executor(): - return self.get_position_executor_data(start_date, end_date) - - # Use _safe_table_loading to load tables - orders = self._safe_table_loading(load_orders) - trade_fills = self._safe_table_loading(load_trade_fills) - order_status = self._safe_table_loading(load_order_status) - market_data = self._safe_table_loading(load_market_data) - position_executor = self._safe_table_loading(load_position_executor) - - strategy_data = StrategyData(orders, order_status, trade_fills, market_data, position_executor) - return strategy_data From fba64f905d682237e4038a5ddb7bd15289123b20 Mon Sep 17 00:00:00 2001 From: drupman Date: Tue, 5 Sep 2023 23:40:17 -0300 Subject: [PATCH 08/40] (feat) add properties table in SingleMarketStrategyData --- utils/data_manipulation.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/utils/data_manipulation.py b/utils/data_manipulation.py index 3372fdac..018a851b 100644 --- a/utils/data_manipulation.py +++ b/utils/data_manipulation.py @@ -241,3 +241,30 @@ def net_pnl_quote(self): @property def inventory_change_base_asset(self): return self.total_buy_amount - self.total_sell_amount + + @property + def properties_table(self): + properties_dict = {"Base Asset": self.base_asset, + "Quote Asset": self.quote_asset, + # "Start Time": self.start_time, + # "End Time": self.end_time, + "Exchange": self.exchange, + "Trading pair": self.trading_pair, + "Duration (seconds)": self.duration_seconds, + "Start Price": self.start_price, + "End Price": self.end_price, + "Total Buy Amount": self.total_buy_amount, + "Total Sell Amount": self.total_sell_amount, + "Total Buy Trades": self.total_buy_trades, + "Total Sell Trades": self.total_sell_trades, + "Total Orders": self.total_orders, + "Average Buy Price": self.average_buy_price, + "Average Sell Price": self.average_sell_price, + "Price Change": self.price_change, + "Trade PnL Quote": self.trade_pnl_quote, + "Cum Fees in Quote": self.cum_fees_in_quote, + "Net PnL Quote": self.net_pnl_quote, + "Inventory Change (base asset)": self.inventory_change_base_asset} + properties_table = pd.DataFrame([properties_dict]).transpose().reset_index() + properties_table.columns = ["Metric", "Value"] + return properties_table From ff64eb266474739c73d92f058d76954db0e3469c Mon Sep 17 00:00:00 2001 From: drupman Date: Wed, 6 Sep 2023 00:05:34 -0300 Subject: [PATCH 09/40] (refactor) changes in conditional rendering --- pages/strategy_performance/app.py | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/pages/strategy_performance/app.py b/pages/strategy_performance/app.py index c5fda1eb..6f57802f 100644 --- a/pages/strategy_performance/app.py +++ b/pages/strategy_performance/app.py @@ -63,22 +63,22 @@ def summary_chart(df: pd.DataFrame): return fig +st.subheader("🔫 Data source") dbs = get_databases() db_names = [x.db_name for x in dbs.values()] -if not db_names: - st.warning("No trades have been recorded in the selected database") - selected_db_name = None - selected_db = None -else: - st.subheader("🔫 Data source") - select_tab, upload_tab = st.tabs(["Select", "Upload"]) - with select_tab: +select_tab, upload_tab = st.tabs(["Select", "Upload"]) +with select_tab: + if db_names is not None: selected_db_name = st.selectbox("Select a database to use:", db_names) selected_db = dbs[selected_db_name] - with upload_tab: - uploaded_db = st.file_uploader("Upload your sqlite database", type=["sqlite", "db"]) - if uploaded_db is not None: - selected_db = DatabaseManager(uploaded_db) + else: + st.warning("Ups! No databases were founded. Try uploading one in Upload tab.") + selected_db = None +with upload_tab: + uploaded_db = st.file_uploader("Upload your sqlite database", type=["sqlite", "db"]) + if uploaded_db is not None: + selected_db = DatabaseManager(uploaded_db) +if selected_db is not None: strategy_data = selected_db.get_strategy_data() if strategy_data.strategy_summary is not None: st.subheader("📝 Strategy summary") @@ -91,7 +91,7 @@ def summary_chart(df: pd.DataFrame): summary_chart = summary_chart(strategy_data.strategy_summary) st.plotly_chart(summary_chart, use_container_width=True) st.subheader("🔍 Examine Trading Pair") - if not any("🇽" in value for value in selected_db.status.values()): + if not any("Error" in value for key, value in selected_db.status.items() if key != "position_executor"): date_array = pd.date_range(start=strategy_data.start_time, end=strategy_data.end_time, periods=60) start_time, end_time = st.select_slider("Select a time range to analyze", options=date_array.tolist(), @@ -197,7 +197,6 @@ def summary_chart(df: pd.DataFrame): status_df = pd.DataFrame([selected_db.status]).transpose().reset_index() status_df.columns = ["Attribute", "Value"] st.table(status_df) - else: st.warning("We couldn't process this sqlite database.") with st.expander("DB Status"): From 2a9c707de1764e3b852ced761316da9e005865af Mon Sep 17 00:00:00 2001 From: drupman Date: Wed, 6 Sep 2023 00:20:48 -0300 Subject: [PATCH 10/40] (refactor) change warning msgs and add divider + remove pd warning --- pages/strategy_performance/app.py | 5 +++-- utils/data_manipulation.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pages/strategy_performance/app.py b/pages/strategy_performance/app.py index 6f57802f..657a18ac 100644 --- a/pages/strategy_performance/app.py +++ b/pages/strategy_performance/app.py @@ -81,6 +81,7 @@ def summary_chart(df: pd.DataFrame): if selected_db is not None: strategy_data = selected_db.get_strategy_data() if strategy_data.strategy_summary is not None: + st.divider() st.subheader("📝 Strategy summary") table_tab, chart_tab = st.tabs(["Table", "Chart"]) with table_tab: @@ -192,13 +193,13 @@ def summary_chart(df: pd.DataFrame): st.write(strategy_data.order_status) download_csv(strategy_data.order_status, "order_status", "download-order-status") else: - st.warning("We have problems to keep analyzing this database") + st.warning("We are encountering challenges in maintaining continuous analysis of this database.") with st.expander("DB Status"): status_df = pd.DataFrame([selected_db.status]).transpose().reset_index() status_df.columns = ["Attribute", "Value"] st.table(status_df) else: - st.warning("We couldn't process this sqlite database.") + st.warning("We were unable to process this SQLite database.") with st.expander("DB Status"): status_df = pd.DataFrame([selected_db.status]).transpose().reset_index() status_df.columns = ["Attribute", "Value"] diff --git a/utils/data_manipulation.py b/utils/data_manipulation.py index 018a851b..cf120ae9 100644 --- a/utils/data_manipulation.py +++ b/utils/data_manipulation.py @@ -41,7 +41,7 @@ def full_series(series): "net_realized_pnl_last": "Realized PnL"}, inplace=True) strategy_summary.sort_values(["Realized PnL"], ascending=True, inplace=True) strategy_summary["Examine"] = False - strategy_summary["Examine"][0] = True + strategy_summary.loc[0, "Examine"] = True return strategy_summary def get_single_market_strategy_data(self, exchange: str, trading_pair: str): From 49d1461ecc9090ecf4a4f9494fdf9d52930e2705 Mon Sep 17 00:00:00 2001 From: drupman Date: Wed, 6 Sep 2023 00:22:53 -0300 Subject: [PATCH 11/40] (fix) replace page_data_filtered in CandlesGraph methods --- pages/strategy_performance/app.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pages/strategy_performance/app.py b/pages/strategy_performance/app.py index 657a18ac..a8ab5724 100644 --- a/pages/strategy_performance/app.py +++ b/pages/strategy_performance/app.py @@ -172,10 +172,10 @@ def summary_chart(df: pd.DataFrame): end_time_page = candles_df.index.max() page_data_filtered = single_market_strategy_data.get_filtered_strategy_data(start_time_page, end_time_page) cg = CandlesGraph(candles_df, show_volume=False, extra_rows=2) - cg.add_buy_trades(strategy_data_filtered.buys) - cg.add_sell_trades(strategy_data_filtered.sells) - cg.add_pnl(strategy_data_filtered, row=2) - cg.add_base_inventory_change(strategy_data_filtered, row=3) + cg.add_buy_trades(page_data_filtered.buys) + cg.add_sell_trades(page_data_filtered.sells) + cg.add_pnl(page_data_filtered, row=2) + cg.add_base_inventory_change(page_data_filtered, row=3) fig = cg.figure() st.plotly_chart(fig, use_container_width=True) else: From 4795557b2b07f71ba79d566fd2b3348dff7567d4 Mon Sep 17 00:00:00 2001 From: drupman Date: Fri, 15 Sep 2023 23:31:20 -0300 Subject: [PATCH 12/40] (refactor) get_trade_fills refactor --- utils/database_manager.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/utils/database_manager.py b/utils/database_manager.py index 54db9727..dd3eb6e1 100644 --- a/utils/database_manager.py +++ b/utils/database_manager.py @@ -144,13 +144,11 @@ def get_orders(self, config_file_path=None, start_date=None, end_date=None): def get_trade_fills(self, config_file_path=None, start_date=None, end_date=None): groupers = ["config_file_path", "market", "symbol"] + float_cols = ["amount", "price", "trade_fee_in_quote"] with self.session_maker() as session: query = self._get_trade_fills_query(config_file_path, start_date, end_date) trade_fills = pd.read_sql_query(query, session.connection()) - trade_fills.sort_values(by="timestamp", ascending=True, inplace=True) - trade_fills["amount"] = trade_fills["amount"] / 1e6 - trade_fills["price"] = trade_fills["price"] / 1e6 - trade_fills["trade_fee_in_quote"] = trade_fills["trade_fee_in_quote"] / 1e6 + trade_fills[float_cols] = trade_fills[float_cols] / 1e6 trade_fills["cum_fees_in_quote"] = trade_fills.groupby(groupers)["trade_fee_in_quote"].cumsum() trade_fills["net_amount"] = trade_fills['amount'] * trade_fills['trade_type'].apply(lambda x: 1 if x == 'BUY' else -1) trade_fills["net_amount_quote"] = trade_fills['net_amount'] * trade_fills['price'] @@ -159,9 +157,9 @@ def get_trade_fills(self, config_file_path=None, start_date=None, end_date=None) trade_fills["inventory_cost"] = trade_fills["cum_net_amount"] * trade_fills["price"] trade_fills["realized_trade_pnl"] = trade_fills["unrealized_trade_pnl"] + trade_fills["inventory_cost"] trade_fills["net_realized_pnl"] = trade_fills["realized_trade_pnl"] - trade_fills["cum_fees_in_quote"] + trade_fills["realized_pnl"] = trade_fills["net_realized_pnl"].diff() trade_fills["timestamp"] = pd.to_datetime(trade_fills["timestamp"], unit="ms") trade_fills["market"] = trade_fills["market"].apply(lambda x: x.lower().replace("_papertrade", "")) - return trade_fills def get_order_status(self, order_ids=None, start_date=None, end_date=None): From 464fdc65814770ebf009329a3082b87049a378c3 Mon Sep 17 00:00:00 2001 From: drupman Date: Fri, 15 Sep 2023 23:31:49 -0300 Subject: [PATCH 13/40] (feat) add accuracy and profit factor metrics to single market strategy data --- utils/data_manipulation.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/utils/data_manipulation.py b/utils/data_manipulation.py index cf120ae9..cf78bb31 100644 --- a/utils/data_manipulation.py +++ b/utils/data_manipulation.py @@ -242,6 +242,18 @@ def net_pnl_quote(self): def inventory_change_base_asset(self): return self.total_buy_amount - self.total_sell_amount + @property + def accuracy(self): + total_wins = len(self.trade_fill["net_realized_pnl"] >= 0) + total_losses = len(self.trade_fill["net_realized_pnl"] < 0) + return total_wins / (total_wins + total_losses) + + @property + def profit_factor(self): + total_profit = (self.trade_fill["net_realized_pnl"] >= 0).sum() + total_loss = (self.trade_fill["net_realized_pnl"] < 0).sum() + return total_profit / total_loss + @property def properties_table(self): properties_dict = {"Base Asset": self.base_asset, From 9c7b0197a8c5760cec5a13bc0ddfd0739aaf5c8b Mon Sep 17 00:00:00 2001 From: drupman Date: Fri, 15 Sep 2023 23:32:51 -0300 Subject: [PATCH 14/40] (feat) set global colors --- utils/graphs.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/utils/graphs.py b/utils/graphs.py index 25a6f485..b462a23c 100644 --- a/utils/graphs.py +++ b/utils/graphs.py @@ -7,6 +7,9 @@ from quants_lab.strategy.strategy_analysis import StrategyAnalysis import plotly.graph_objs as go +BULLISH_COLOR = "rgba(97, 199, 102, 0.9)" +BEARISH_COLOR = "rgba(255, 102, 90, 0.9)" +FEE_COLOR = "rgba(51, 0, 51, 0.9)" class CandlesGraph: def __init__(self, candles_df: pd.DataFrame, show_volume=True, extra_rows=1): From 3fdca1aa96c37b0dd3ad4d0c5dbd502c3055908a Mon Sep 17 00:00:00 2001 From: drupman Date: Fri, 15 Sep 2023 23:46:27 -0300 Subject: [PATCH 15/40] (feat) add style metric cards function --- pages/strategy_performance/app.py | 161 +++++++++++++++++++++++++++++- 1 file changed, 160 insertions(+), 1 deletion(-) diff --git a/pages/strategy_performance/app.py b/pages/strategy_performance/app.py index a8ab5724..83a12e0a 100644 --- a/pages/strategy_performance/app.py +++ b/pages/strategy_performance/app.py @@ -7,7 +7,6 @@ from utils.graphs import CandlesGraph from utils.st_utils import initialize_st_page - initialize_st_page(title="Strategy Performance", icon="🚀") BULLISH_COLOR = "#61C766" @@ -44,6 +43,37 @@ def download_csv(df: pd.DataFrame, filename: str, key: str): ) +def style_metric_cards( + background_color: str = "rgba(255, 255, 255, 0)", + border_size_px: int = 1, + border_color: str = "rgba(255, 255, 255, 0.3)", + border_radius_px: int = 5, + border_left_color: str = "rgba(255, 255, 255, 0.5)", + box_shadow: bool = True, +): + + box_shadow_str = ( + "box-shadow: 0 0.15rem 1.75rem 0 rgba(58, 59, 69, 0.15) !important;" + if box_shadow + else "box-shadow: none !important;" + ) + st.markdown( + f""" + + """, + unsafe_allow_html=True, + ) + + def show_strategy_summary(summary_df: pd.DataFrame): summary = st.data_editor(summary_df, column_config={"PnL Over Time": st.column_config.LineChartColumn("PnL Over Time", @@ -63,6 +93,135 @@ def summary_chart(df: pd.DataFrame): return fig +def pnl_over_time(df: pd.DataFrame): + df.reset_index(drop=True, inplace=True) + df_above = df[df['net_realized_pnl'] >= 0] + df_below = df[df['net_realized_pnl'] < 0] + + fig = go.Figure() + fig.add_trace(go.Bar(x=df_above.index, y=df_above["net_realized_pnl"], marker_color=BULLISH_COLOR, showlegend=False)) + fig.add_trace(go.Bar(x=df_below.index, y=df_below["net_realized_pnl"], marker_color=BEARISH_COLOR, showlegend=False)) + fig.update_layout(title=dict( + text='Cummulative PnL', # Your title text + x=0.43, + y=0.95, + ), + plot_bgcolor='rgba(0,0,0,0)', + paper_bgcolor='rgba(0,0,0,0)') + return fig + + +def top_n_trades(series, n: int = 8): + podium = list(range(0, n)) + top_three_profits = series[series >= 0].sort_values(ascending=True)[-n:] + top_three_losses = series[series < 0].sort_values(ascending=False)[-n:] + fig = go.Figure() + fig.add_trace(go.Bar(name="Top Profits", + y=podium, + x=top_three_profits, + base=[0, 0, 0, 0], + marker_color=BULLISH_COLOR, + orientation='h', + text=top_three_profits.apply(lambda x: f"{x:.2f}"), + textposition="inside", + insidetextfont=dict(color='white'))) + fig.add_trace(go.Bar(name="Top Losses", + y=podium, + x=top_three_losses, + marker_color=BEARISH_COLOR, + orientation='h', + text=top_three_losses.apply(lambda x: f"{x:.2f}"), + textposition="inside", + insidetextfont=dict(color='white'))) + fig.update_layout(barmode='stack', + title=dict( + text='Top/Worst Realized PnLs', # Your title text + x=0.5, + y=0.95, + xanchor="center", + yanchor="top" + ), + xaxis=dict(showgrid=True, gridwidth=0.01, gridcolor="rgba(211, 211, 211, 0.5)"), # Show vertical grid lines + yaxis=dict(showgrid=False), + legend=dict(orientation="h", + x=0.5, + y=1.08, + xanchor="center", + yanchor="bottom")) + fig.update_yaxes(showticklabels=False, + showline=False, + range=[- n + 6, n + 1]) + return fig + + +def intraday_performance(df: pd.DataFrame): + def hr2angle(hr): + return (hr * 15) % 360 + + def hr_str(hr): + # Normalize hr to be between 1 and 12 + hr_str = str(((hr - 1) % 12) + 1) + suffix = ' AM' if (hr % 24) < 12 else ' PM' + return hr_str + suffix + + df["hour"] = df["timestamp"].dt.hour + profits = df[df["net_realized_pnl"] >= 0] + losses = df[df["net_realized_pnl"] < 0] + polar_profits = profits.groupby("hour")["net_realized_pnl"].sum().reset_index() + polar_losses = losses.groupby("hour")["net_realized_pnl"].sum().reset_index() + polar_losses["net_realized_pnl"] = abs(polar_losses["net_realized_pnl"]) + fig = go.Figure() + fig.add_trace(go.Barpolar( + name="Profits", + # mode="none", + r=polar_profits["net_realized_pnl"], + theta=polar_profits["hour"] * 15, + # fill="toself", + # fillcolor=BULLISH_COLOR, + marker_color=BULLISH_COLOR)) + fig.add_trace(go.Barpolar( + name="Losses", + # mode="none", + r=polar_losses["net_realized_pnl"], + theta=polar_losses["hour"] * 15, + # fill="toself", + # fillcolor=BEARISH_COLOR, + marker_color=BEARISH_COLOR)) + fig.update_layout( + polar=dict( + radialaxis=dict( + visible=True, + showline=False, + ), + angularaxis=dict( + rotation=90, + direction="clockwise", + tickvals=[hr2angle(hr) for hr in range(24)], + ticktext=[hr_str(hr) for hr in range(24)], + ), + bgcolor='rgba(255, 255, 255, 0)', + + ), + legend=dict( + orientation="h", + x=0.5, + y=1.08, + xanchor="center", + yanchor="bottom" + ), + title=dict( + text='Intraday Performance', # Your title text + x=0.5, + y=0.93, + xanchor="center", + yanchor="bottom" + ), + ) + + return fig + + +style_metric_cards() st.subheader("🔫 Data source") dbs = get_databases() db_names = [x.db_name for x in dbs.values()] From 6293cc92adb4270b48373af86f6c9d0aa2ccc287 Mon Sep 17 00:00:00 2001 From: drupman Date: Fri, 15 Sep 2023 23:47:46 -0300 Subject: [PATCH 16/40] (feat) add main metrics panel --- pages/strategy_performance/app.py | 66 ++++++++++++------------------- 1 file changed, 26 insertions(+), 40 deletions(-) diff --git a/pages/strategy_performance/app.py b/pages/strategy_performance/app.py index 83a12e0a..ca5b2257 100644 --- a/pages/strategy_performance/app.py +++ b/pages/strategy_performance/app.py @@ -1,6 +1,7 @@ import os import pandas as pd import streamlit as st +import plotly.graph_objects as go import math import plotly.express as px from utils.database_manager import DatabaseManager @@ -261,51 +262,36 @@ def hr_str(hr): if single_market: single_market_strategy_data = strategy_data.get_single_market_strategy_data(selected_exchange, selected_trading_pair) strategy_data_filtered = single_market_strategy_data.get_filtered_strategy_data(start_time, end_time) - - st.divider() with st.container(): - col1, col2 = st.columns(2) - with col1: - st.subheader(f"🏦 Market") - with col2: - st.subheader("📋 General stats") - col1, col2, col3, col4 = st.columns(4) + col1, col2, col3, col4, col5, col6, col7, col8 = st.columns(8) with col1: - st.metric(label="Exchange", value=strategy_data_filtered.exchange.capitalize()) - with col2: - st.metric(label="Trading pair", value=strategy_data_filtered.trading_pair.upper()) - with col3: - st.metric(label='Start date', value=strategy_data_filtered.start_time.strftime("%Y-%m-%d %H:%M")) - st.metric(label='End date', value=strategy_data_filtered.end_time.strftime("%Y-%m-%d %H:%M")) - with col4: - st.metric(label='Duration (Hours)', value=round(strategy_data_filtered.duration_seconds / 3600, 2)) - st.metric(label='Price change', value=f"{round(strategy_data_filtered.price_change * 100, 2)} %") - - st.divider() - st.subheader("📈 Performance") - col131, col132, col133, col134 = st.columns(4) - with col131: st.metric(label=f'Net PNL {strategy_data_filtered.quote_asset}', value=round(strategy_data_filtered.net_pnl_quote, 2)) - st.metric(label=f'Trade PNL {strategy_data_filtered.quote_asset}', - value=round(strategy_data_filtered.trade_pnl_quote, 2)) - st.metric(label=f'Fees {strategy_data_filtered.quote_asset}', - value=round(strategy_data_filtered.cum_fees_in_quote, 2)) - with col132: + with col2: st.metric(label='Total Trades', value=strategy_data_filtered.total_orders) - st.metric(label='Total Buy Trades', value=strategy_data_filtered.total_buy_trades) - st.metric(label='Total Sell Trades', value=strategy_data_filtered.total_sell_trades) - with col133: - st.metric(label='Inventory change in Base asset', - value=round(strategy_data_filtered.inventory_change_base_asset, 4)) - st.metric(label='Total Buy Trades Amount', - value=round(strategy_data_filtered.total_buy_amount, 2)) - st.metric(label='Total Sell Trades Amount', - value=round(strategy_data_filtered.total_sell_amount, 2)) - with col134: - st.metric(label='End Price', value=round(strategy_data_filtered.end_price, 4)) - st.metric(label='Average Buy Price', value=round(strategy_data_filtered.average_buy_price, 4)) - st.metric(label='Average Sell Price', value=round(strategy_data_filtered.average_sell_price, 4)) + with col3: + st.metric(label='Accuracy', + value=round(strategy_data_filtered.accuracy, 2)) + with col4: + st.metric(label="Profit Factor", + value=round(strategy_data_filtered.profit_factor, 2)) + with col5: + st.metric(label='Duration (Hours)', + value=round(strategy_data_filtered.duration_seconds / 3600, 2)) + with col6: + st.metric(label='Price change', + value=f"{round(strategy_data_filtered.price_change * 100, 2)} %") + with col7: + buy_trades_amount = round(strategy_data_filtered.total_buy_amount, 2) + avg_buy_price = round(strategy_data_filtered.average_buy_price, 4) + st.metric(label="Total Buy Volume", + value=round(buy_trades_amount * avg_buy_price, 2)) + with col8: + sell_trades_amount = round(strategy_data_filtered.total_sell_amount, 2) + avg_sell_price = round(strategy_data_filtered.average_sell_price, 4) + st.metric(label="Total Sell Volume", + value=round(sell_trades_amount * avg_sell_price, 2)) + st.plotly_chart(pnl_over_time(strategy_data_filtered.trade_fill), use_container_width=True) st.divider() st.subheader("🕯️ Candlestick") From 08d9e53a71cfdf802590bd914a6473b2e72009d3 Mon Sep 17 00:00:00 2001 From: drupman Date: Fri, 15 Sep 2023 23:49:10 -0300 Subject: [PATCH 17/40] (feat) market activity improvements --- pages/strategy_performance/app.py | 46 ++++++++++++++++++------------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/pages/strategy_performance/app.py b/pages/strategy_performance/app.py index ca5b2257..906b37a0 100644 --- a/pages/strategy_performance/app.py +++ b/pages/strategy_performance/app.py @@ -293,9 +293,8 @@ def hr_str(hr): value=round(sell_trades_amount * avg_sell_price, 2)) st.plotly_chart(pnl_over_time(strategy_data_filtered.trade_fill), use_container_width=True) - st.divider() - st.subheader("🕯️ Candlestick") - if strategy_data_filtered.market_data is not None: + st.subheader("💱 Market activity") + if "Error" not in selected_db.status["market_data"] and strategy_data_filtered.market_data is not None: with st.expander("Market activity", expanded=True): col1, col2, col3 = st.columns([1, 1, 2]) with col1: @@ -303,26 +302,35 @@ def hr_str(hr): with col2: rows_per_page = st.number_input("Candles per Page", value=100, min_value=1, max_value=5000) with col3: - total_rows = len(strategy_data_filtered.get_market_data_resampled(interval=f"{intervals[interval]}S")) + total_rows = len( + strategy_data_filtered.get_market_data_resampled(interval=f"{intervals[interval]}S")) total_pages = math.ceil(total_rows / rows_per_page) if total_pages > 1: - selected_page = st.select_slider("Select page", list(range(total_pages)), key="page_slider") + selected_page = st.select_slider("Select page", list(range(total_pages)), total_pages - 1, + key="page_slider") else: selected_page = 0 - start_idx = selected_page * rows_per_page - end_idx = start_idx + rows_per_page - candles_df = strategy_data_filtered.get_market_data_resampled(interval=f"{intervals[interval]}S").iloc[ - start_idx:end_idx] - start_time_page = candles_df.index.min() - end_time_page = candles_df.index.max() - page_data_filtered = single_market_strategy_data.get_filtered_strategy_data(start_time_page, end_time_page) - cg = CandlesGraph(candles_df, show_volume=False, extra_rows=2) - cg.add_buy_trades(page_data_filtered.buys) - cg.add_sell_trades(page_data_filtered.sells) - cg.add_pnl(page_data_filtered, row=2) - cg.add_base_inventory_change(page_data_filtered, row=3) - fig = cg.figure() - st.plotly_chart(fig, use_container_width=True) + start_idx = selected_page * rows_per_page + end_idx = start_idx + rows_per_page + candles_df = strategy_data_filtered.get_market_data_resampled( + interval=f"{intervals[interval]}S").iloc[ + start_idx:end_idx] + start_time_page = candles_df.index.min() + end_time_page = candles_df.index.max() + page_data_filtered = single_market_strategy_data.get_filtered_strategy_data(start_time_page, + end_time_page) + col1, col2 = st.columns([5.5, 1.5]) + with col1: + cg = CandlesGraph(candles_df, show_volume=False, extra_rows=2) + cg.add_buy_trades(page_data_filtered.buys) + cg.add_sell_trades(page_data_filtered.sells) + cg.add_pnl(page_data_filtered, row=2) + cg.add_base_inventory_change(page_data_filtered, row=3) + fig = cg.figure() + st.plotly_chart(fig, use_container_width=True) + with col2: + st.plotly_chart(intraday_performance(page_data_filtered.trade_fill), use_container_width=True) + st.plotly_chart(top_n_trades(page_data_filtered.trade_fill.net_realized_pnl), use_container_width=True) else: st.warning("Market data is not available so the candles graph is not going to be rendered. " "Make sure that you are using the latest version of Hummingbot and market data recorder activated.") From cb7b4de88cef06529179d7b98b98dd61f0aefeab Mon Sep 17 00:00:00 2001 From: drupman Date: Fri, 15 Sep 2023 23:49:47 -0300 Subject: [PATCH 18/40] (feat) change bullish/bearish colors --- pages/strategy_performance/app.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pages/strategy_performance/app.py b/pages/strategy_performance/app.py index 906b37a0..580c5e9b 100644 --- a/pages/strategy_performance/app.py +++ b/pages/strategy_performance/app.py @@ -10,8 +10,8 @@ initialize_st_page(title="Strategy Performance", icon="🚀") -BULLISH_COLOR = "#61C766" -BEARISH_COLOR = "#FF665A" +BULLISH_COLOR = "rgba(97, 199, 102, 0.9)" +BEARISH_COLOR = "rgba(255, 102, 90, 0.9)" # Start content here intervals = { @@ -251,6 +251,7 @@ def hr_str(hr): with chart_tab: summary_chart = summary_chart(strategy_data.strategy_summary) st.plotly_chart(summary_chart, use_container_width=True) + st.divider() st.subheader("🔍 Examine Trading Pair") if not any("Error" in value for key, value in selected_db.status.items() if key != "position_executor"): date_array = pd.date_range(start=strategy_data.start_time, end=strategy_data.end_time, periods=60) From 300038128e4729a3b676d55bc857b72a065190ba Mon Sep 17 00:00:00 2001 From: drupman Date: Fri, 15 Sep 2023 23:50:07 -0300 Subject: [PATCH 19/40] (wip) add secondary metrics panel --- pages/strategy_performance/app.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/pages/strategy_performance/app.py b/pages/strategy_performance/app.py index 580c5e9b..2fbcb973 100644 --- a/pages/strategy_performance/app.py +++ b/pages/strategy_performance/app.py @@ -336,6 +336,29 @@ def hr_str(hr): st.warning("Market data is not available so the candles graph is not going to be rendered. " "Make sure that you are using the latest version of Hummingbot and market data recorder activated.") st.divider() + st.subheader("📈 Metrics") + with st.container(): + col1, col2, col3, col4, col5 = st.columns(5) + with col1: + st.metric(label=f'Trade PNL {strategy_data_filtered.quote_asset}', + value=round(strategy_data_filtered.trade_pnl_quote, 2)) + st.metric(label=f'Fees {strategy_data_filtered.quote_asset}', + value=round(strategy_data_filtered.cum_fees_in_quote, 2)) + with col2: + st.metric(label='Total Buy Trades', value=strategy_data_filtered.total_buy_trades) + st.metric(label='Total Sell Trades', value=strategy_data_filtered.total_sell_trades) + with col3: + st.metric(label='Total Buy Trades Amount', + value=round(strategy_data_filtered.total_buy_amount, 2)) + st.metric(label='Total Sell Trades Amount', + value=round(strategy_data_filtered.total_sell_amount, 2)) + with col4: + st.metric(label='Average Buy Price', value=round(strategy_data_filtered.average_buy_price, 4)) + st.metric(label='Average Sell Price', value=round(strategy_data_filtered.average_sell_price, 4)) + with col5: + st.metric(label='Inventory change in Base asset', + value=round(strategy_data_filtered.inventory_change_base_asset, 4)) + st.divider() st.subheader("Tables") with st.expander("💵 Trades"): st.write(strategy_data.trade_fill) From 98f30b95d2e25a94874a4da9bb4d01eda48f606d Mon Sep 17 00:00:00 2001 From: drupman Date: Fri, 15 Sep 2023 23:51:04 -0300 Subject: [PATCH 20/40] (feat) update_layout adjustments --- utils/graphs.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/utils/graphs.py b/utils/graphs.py index b462a23c..d70f6f5a 100644 --- a/utils/graphs.py +++ b/utils/graphs.py @@ -224,19 +224,19 @@ def update_layout(self): self.base_figure.update_layout( title={ 'text': "Market activity", - 'y': 0.95, + 'y': 0.99, 'x': 0.5, 'xanchor': 'center', 'yanchor': 'top' }, legend=dict( orientation="h", - yanchor="bottom", - y=-0.2, - xanchor="right", - x=1 + x=0.5, + y=1.04, + xanchor="center", + yanchor="bottom" ), - height=1500, + height=1000, xaxis=dict(rangeslider_visible=False, range=[self.min_time, self.max_time]), yaxis=dict(range=[self.candles_df.low.min(), self.candles_df.high.max()]), From e728cbbe87cadec257622bda0c868cbe93487a32 Mon Sep 17 00:00:00 2001 From: drupman Date: Fri, 15 Sep 2023 23:51:49 -0300 Subject: [PATCH 21/40] (feat) pnl subchart improvements --- utils/graphs.py | 49 +++++++++++++++++++++++++++---------------------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/utils/graphs.py b/utils/graphs.py index d70f6f5a..d23ee95b 100644 --- a/utils/graphs.py +++ b/utils/graphs.py @@ -178,46 +178,51 @@ def add_base_inventory_change(self, strategy_data: StrategyData, row=3): self.base_figure.update_yaxes(title_text='Base Inventory Change', row=row, col=1) def add_pnl(self, strategy_data: SingleMarketStrategyData, row=4): - merged_df = self.get_merged_df(strategy_data) self.base_figure.add_trace( go.Scatter( - x=merged_df.index, - y=merged_df["cum_fees_in_quote"].apply(lambda x: round(-x, 2)), - name="Cum Fees", + x=strategy_data.trade_fill.timestamp, + y=[max(0, realized_pnl) for realized_pnl in strategy_data.trade_fill["realized_trade_pnl"].apply(lambda x: round(x, 4))], + name="Cum Profit", mode='lines', - line_color='teal', + line=dict(shape="hv", color="rgba(1, 1, 1, 0.5)", dash="dash", width=0.1), + # marker=dict(symbol="arrow"), fill="tozeroy", # Fill to the line below (trade pnl) - stackgroup='one' + fillcolor="rgba(0, 255, 0, 0.5)" ), row=row, col=1 ) - self.base_figure.add_trace( go.Scatter( - x=merged_df.index, - y=merged_df["trade_pnl_continuos"].apply(lambda x: round(x, 2)), - name="Cum Trade PnL", + x=strategy_data.trade_fill.timestamp, + y=[min(0, realized_pnl) for realized_pnl in strategy_data.trade_fill["realized_trade_pnl"].apply(lambda x: round(x, 4))], + name="Cum Loss", mode='lines', - line_color='pink', - fill="tonexty", # Fill to the line below (net pnl) - stackgroup='one' + line=dict(shape="hv", color="rgba(1, 1, 1, 0.5)", dash="dash", width=0.3), + # marker=dict(symbol="arrow"), + fill="tozeroy", # Fill to the line below (trade pnl) + fillcolor="rgba(255, 0, 0, 0.5)", ), row=row, col=1 ) self.base_figure.add_trace( go.Scatter( - x=merged_df.index, - y=merged_df["net_pnl_continuos"].apply(lambda x: round(x, 2)), - name="Cum Net PnL", - mode="lines+markers", - marker=dict(color="black", size=6), - line=dict(color="black", width=2), - # textposition="top center", - # text=merged_df["net_pnl_continuos"], - # texttemplate="%{text:.1f}" + x=strategy_data.trade_fill.timestamp, + y=strategy_data.trade_fill["cum_fees_in_quote"].apply(lambda x: round(x, 4)), + name="Cum Fees", + mode='lines', + line=dict(shape="hv", color="rgba(1, 1, 1, 0.1)", dash="dash", width=0.1), + fill="tozeroy", # Fill to the line below (trade pnl) + fillcolor="rgba(51, 0, 51, 0.5)" ), row=row, col=1 ) + self.base_figure.add_trace(go.Scatter(name="Net Realized Profit", + x=strategy_data.trade_fill.timestamp, + y=strategy_data.trade_fill["net_realized_pnl"], + mode="lines", + line=dict(shape="hv")), + row=row, col=1 + ) self.base_figure.update_yaxes(title_text='PNL', row=row, col=1) def update_layout(self): From daf09bde658bc4cf9d99f1f4d567b93a7e61acab Mon Sep 17 00:00:00 2001 From: drupman Date: Fri, 15 Sep 2023 23:52:42 -0300 Subject: [PATCH 22/40] (feat) replace base inventory change for quote inventory change --- pages/strategy_performance/app.py | 2 +- utils/graphs.py | 35 ++++++++++--------------------- 2 files changed, 12 insertions(+), 25 deletions(-) diff --git a/pages/strategy_performance/app.py b/pages/strategy_performance/app.py index 2fbcb973..a66453eb 100644 --- a/pages/strategy_performance/app.py +++ b/pages/strategy_performance/app.py @@ -326,7 +326,7 @@ def hr_str(hr): cg.add_buy_trades(page_data_filtered.buys) cg.add_sell_trades(page_data_filtered.sells) cg.add_pnl(page_data_filtered, row=2) - cg.add_base_inventory_change(page_data_filtered, row=3) + cg.add_quote_inventory_change(page_data_filtered, row=3) fig = cg.figure() st.plotly_chart(fig, use_container_width=True) with col2: diff --git a/utils/graphs.py b/utils/graphs.py index d23ee95b..abf13de4 100644 --- a/utils/graphs.py +++ b/utils/graphs.py @@ -146,36 +146,23 @@ def add_ema(self, length=20, row=1): row=row, col=1, ) - def add_base_inventory_change(self, strategy_data: StrategyData, row=3): - # Create a list of colors based on the sign of the amount_new column - self.base_figure.add_trace( - go.Bar( - x=strategy_data.trade_fill["timestamp"], - y=strategy_data.trade_fill["net_amount"], - name="Base Inventory Change", - opacity=0.5, - marker=dict(color=["lightgreen" if amount > 0 else "indianred" for amount in - strategy_data.trade_fill["net_amount"]]) - ), - row=row, col=1, - ) - # TODO: Review impact in different subgraphs - merged_df = self.get_merged_df(strategy_data) + def add_quote_inventory_change(self, strategy_data: StrategyData, row=3): self.base_figure.add_trace( go.Scatter( - x=merged_df.index, - y=merged_df["cum_net_amount"], - name="Cumulative Base Inventory Change", + x=strategy_data.trade_fill.timestamp, + y=strategy_data.trade_fill.inventory_cost, + name="Quote Inventory", mode="lines+markers", - marker=dict(color="black", size=6), - line=dict(color="royalblue", width=2), - # text=merged_df["cum_net_amount"], - # textposition="top center", - # texttemplate="%{text:.2f}" + marker=dict( + size=10, + symbol="arrow", + angleref="previous", + ), + line=dict(shape="hv"), ), row=row, col=1 ) - self.base_figure.update_yaxes(title_text='Base Inventory Change', row=row, col=1) + self.base_figure.update_yaxes(title_text='Quote Inventory Change', row=row, col=1) def add_pnl(self, strategy_data: SingleMarketStrategyData, row=4): self.base_figure.add_trace( From b46a9bad2089a1da85572f248566f899f909604b Mon Sep 17 00:00:00 2001 From: drupman Date: Fri, 15 Sep 2023 23:53:03 -0300 Subject: [PATCH 23/40] (feat) remove merged_df --- utils/graphs.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/utils/graphs.py b/utils/graphs.py index abf13de4..d57e1343 100644 --- a/utils/graphs.py +++ b/utils/graphs.py @@ -239,12 +239,6 @@ def update_layout(self): self.base_figure.update_yaxes(title_text="Volume", row=2, col=1) self.base_figure.update_xaxes(title_text="Time", row=self.rows, col=1) - def get_merged_df(self, strategy_data: StrategyData): - merged_df = pd.merge_asof(self.candles_df, strategy_data.trade_fill, left_index=True, right_on="timestamp", direction="backward") - merged_df["trade_pnl_continuos"] = merged_df["unrealized_trade_pnl"] + merged_df["cum_net_amount"] * merged_df["close"] - merged_df["net_pnl_continuos"] = merged_df["trade_pnl_continuos"] - merged_df["cum_fees_in_quote"] - return merged_df - class BacktestingGraphs: def __init__(self, study_df: pd.DataFrame): From e96168b8e297412e71967541c725156a54d9f63f Mon Sep 17 00:00:00 2001 From: drupman Date: Sat, 16 Sep 2023 00:29:35 -0300 Subject: [PATCH 24/40] (refactor) remove comments and error handling --- pages/strategy_performance/app.py | 258 +++++++++++++++--------------- 1 file changed, 129 insertions(+), 129 deletions(-) diff --git a/pages/strategy_performance/app.py b/pages/strategy_performance/app.py index a66453eb..5ea73eae 100644 --- a/pages/strategy_performance/app.py +++ b/pages/strategy_performance/app.py @@ -85,8 +85,10 @@ def show_strategy_summary(summary_df: pd.DataFrame): use_container_width=True ) selected_rows = summary[summary.Examine] - return selected_rows.drop('Examine', axis=1) - + if len(selected_rows) > 0: + return selected_rows.drop('Examine', axis=1) + else: + return None def summary_chart(df: pd.DataFrame): fig = px.bar(df, x="Trading Pair", y="Realized PnL", color="Exchange") @@ -174,19 +176,13 @@ def hr_str(hr): fig = go.Figure() fig.add_trace(go.Barpolar( name="Profits", - # mode="none", r=polar_profits["net_realized_pnl"], theta=polar_profits["hour"] * 15, - # fill="toself", - # fillcolor=BULLISH_COLOR, marker_color=BULLISH_COLOR)) fig.add_trace(go.Barpolar( name="Losses", - # mode="none", r=polar_losses["net_realized_pnl"], theta=polar_losses["hour"] * 15, - # fill="toself", - # fillcolor=BEARISH_COLOR, marker_color=BEARISH_COLOR)) fig.update_layout( polar=dict( @@ -211,7 +207,7 @@ def hr_str(hr): yanchor="bottom" ), title=dict( - text='Intraday Performance', # Your title text + text='Intraday Performance', x=0.5, y=0.93, xanchor="center", @@ -246,135 +242,139 @@ def hr_str(hr): table_tab, chart_tab = st.tabs(["Table", "Chart"]) with table_tab: selection = show_strategy_summary(strategy_data.strategy_summary) - selected_exchange = selection["Exchange"].values[0] - selected_trading_pair = selection["Trading Pair"].values[0] + if selection is not None: + selected_exchange = selection["Exchange"].values[0] + selected_trading_pair = selection["Trading Pair"].values[0] with chart_tab: summary_chart = summary_chart(strategy_data.strategy_summary) st.plotly_chart(summary_chart, use_container_width=True) - st.divider() - st.subheader("🔍 Examine Trading Pair") - if not any("Error" in value for key, value in selected_db.status.items() if key != "position_executor"): - date_array = pd.date_range(start=strategy_data.start_time, end=strategy_data.end_time, periods=60) - start_time, end_time = st.select_slider("Select a time range to analyze", - options=date_array.tolist(), - value=(date_array[0], date_array[-1])) - - single_market = True - if single_market: - single_market_strategy_data = strategy_data.get_single_market_strategy_data(selected_exchange, selected_trading_pair) - strategy_data_filtered = single_market_strategy_data.get_filtered_strategy_data(start_time, end_time) - with st.container(): - col1, col2, col3, col4, col5, col6, col7, col8 = st.columns(8) - with col1: - st.metric(label=f'Net PNL {strategy_data_filtered.quote_asset}', - value=round(strategy_data_filtered.net_pnl_quote, 2)) - with col2: - st.metric(label='Total Trades', value=strategy_data_filtered.total_orders) - with col3: - st.metric(label='Accuracy', - value=round(strategy_data_filtered.accuracy, 2)) - with col4: - st.metric(label="Profit Factor", - value=round(strategy_data_filtered.profit_factor, 2)) - with col5: - st.metric(label='Duration (Hours)', - value=round(strategy_data_filtered.duration_seconds / 3600, 2)) - with col6: - st.metric(label='Price change', - value=f"{round(strategy_data_filtered.price_change * 100, 2)} %") - with col7: - buy_trades_amount = round(strategy_data_filtered.total_buy_amount, 2) - avg_buy_price = round(strategy_data_filtered.average_buy_price, 4) - st.metric(label="Total Buy Volume", - value=round(buy_trades_amount * avg_buy_price, 2)) - with col8: - sell_trades_amount = round(strategy_data_filtered.total_sell_amount, 2) - avg_sell_price = round(strategy_data_filtered.average_sell_price, 4) - st.metric(label="Total Sell Volume", - value=round(sell_trades_amount * avg_sell_price, 2)) - st.plotly_chart(pnl_over_time(strategy_data_filtered.trade_fill), use_container_width=True) + if selection is None: + st.info("Choose a trading pair and start analyzing!") + else: + st.divider() + st.subheader("🔍 Examine Trading Pair") + if not any("Error" in value for key, value in selected_db.status.items() if key != "position_executor"): + date_array = pd.date_range(start=strategy_data.start_time, end=strategy_data.end_time, periods=60) + start_time, end_time = st.select_slider("Select a time range to analyze", + options=date_array.tolist(), + value=(date_array[0], date_array[-1])) - st.subheader("💱 Market activity") - if "Error" not in selected_db.status["market_data"] and strategy_data_filtered.market_data is not None: - with st.expander("Market activity", expanded=True): - col1, col2, col3 = st.columns([1, 1, 2]) + single_market = True + if single_market: + single_market_strategy_data = strategy_data.get_single_market_strategy_data(selected_exchange, selected_trading_pair) + strategy_data_filtered = single_market_strategy_data.get_filtered_strategy_data(start_time, end_time) + with st.container(): + col1, col2, col3, col4, col5, col6, col7, col8 = st.columns(8) with col1: - interval = st.selectbox("Candles Interval:", intervals.keys(), index=2) + st.metric(label=f'Net PNL {strategy_data_filtered.quote_asset}', + value=round(strategy_data_filtered.net_pnl_quote, 2)) with col2: - rows_per_page = st.number_input("Candles per Page", value=100, min_value=1, max_value=5000) + st.metric(label='Total Trades', value=strategy_data_filtered.total_orders) with col3: - total_rows = len( - strategy_data_filtered.get_market_data_resampled(interval=f"{intervals[interval]}S")) - total_pages = math.ceil(total_rows / rows_per_page) - if total_pages > 1: - selected_page = st.select_slider("Select page", list(range(total_pages)), total_pages - 1, - key="page_slider") - else: - selected_page = 0 - start_idx = selected_page * rows_per_page - end_idx = start_idx + rows_per_page - candles_df = strategy_data_filtered.get_market_data_resampled( - interval=f"{intervals[interval]}S").iloc[ - start_idx:end_idx] - start_time_page = candles_df.index.min() - end_time_page = candles_df.index.max() - page_data_filtered = single_market_strategy_data.get_filtered_strategy_data(start_time_page, - end_time_page) - col1, col2 = st.columns([5.5, 1.5]) + st.metric(label='Accuracy', + value=round(strategy_data_filtered.accuracy, 2)) + with col4: + st.metric(label="Profit Factor", + value=round(strategy_data_filtered.profit_factor, 2)) + with col5: + st.metric(label='Duration (Hours)', + value=round(strategy_data_filtered.duration_seconds / 3600, 2)) + with col6: + st.metric(label='Price change', + value=f"{round(strategy_data_filtered.price_change * 100, 2)} %") + with col7: + buy_trades_amount = round(strategy_data_filtered.total_buy_amount, 2) + avg_buy_price = round(strategy_data_filtered.average_buy_price, 4) + st.metric(label="Total Buy Volume", + value=round(buy_trades_amount * avg_buy_price, 2)) + with col8: + sell_trades_amount = round(strategy_data_filtered.total_sell_amount, 2) + avg_sell_price = round(strategy_data_filtered.average_sell_price, 4) + st.metric(label="Total Sell Volume", + value=round(sell_trades_amount * avg_sell_price, 2)) + st.plotly_chart(pnl_over_time(strategy_data_filtered.trade_fill), use_container_width=True) + + st.subheader("💱 Market activity") + if "Error" not in selected_db.status["market_data"] and strategy_data_filtered.market_data is not None: + with st.expander("Market activity", expanded=True): + col1, col2, col3 = st.columns([1, 1, 2]) + with col1: + interval = st.selectbox("Candles Interval:", intervals.keys(), index=2) + with col2: + rows_per_page = st.number_input("Candles per Page", value=100, min_value=1, max_value=5000) + with col3: + total_rows = len( + strategy_data_filtered.get_market_data_resampled(interval=f"{intervals[interval]}S")) + total_pages = math.ceil(total_rows / rows_per_page) + if total_pages > 1: + selected_page = st.select_slider("Select page", list(range(total_pages)), total_pages - 1, + key="page_slider") + else: + selected_page = 0 + start_idx = selected_page * rows_per_page + end_idx = start_idx + rows_per_page + candles_df = strategy_data_filtered.get_market_data_resampled( + interval=f"{intervals[interval]}S").iloc[ + start_idx:end_idx] + start_time_page = candles_df.index.min() + end_time_page = candles_df.index.max() + page_data_filtered = single_market_strategy_data.get_filtered_strategy_data(start_time_page, + end_time_page) + col1, col2 = st.columns([5.5, 1.5]) + with col1: + cg = CandlesGraph(candles_df, show_volume=False, extra_rows=2) + cg.add_buy_trades(page_data_filtered.buys) + cg.add_sell_trades(page_data_filtered.sells) + cg.add_pnl(page_data_filtered, row=2) + cg.add_quote_inventory_change(page_data_filtered, row=3) + fig = cg.figure() + st.plotly_chart(fig, use_container_width=True) + with col2: + st.plotly_chart(intraday_performance(page_data_filtered.trade_fill), use_container_width=True) + st.plotly_chart(top_n_trades(page_data_filtered.trade_fill.net_realized_pnl), use_container_width=True) + else: + st.warning("Market data is not available so the candles graph is not going to be rendered. " + "Make sure that you are using the latest version of Hummingbot and market data recorder activated.") + st.divider() + st.subheader("📈 Metrics") + with st.container(): + col1, col2, col3, col4, col5 = st.columns(5) with col1: - cg = CandlesGraph(candles_df, show_volume=False, extra_rows=2) - cg.add_buy_trades(page_data_filtered.buys) - cg.add_sell_trades(page_data_filtered.sells) - cg.add_pnl(page_data_filtered, row=2) - cg.add_quote_inventory_change(page_data_filtered, row=3) - fig = cg.figure() - st.plotly_chart(fig, use_container_width=True) + st.metric(label=f'Trade PNL {strategy_data_filtered.quote_asset}', + value=round(strategy_data_filtered.trade_pnl_quote, 2)) + st.metric(label=f'Fees {strategy_data_filtered.quote_asset}', + value=round(strategy_data_filtered.cum_fees_in_quote, 2)) with col2: - st.plotly_chart(intraday_performance(page_data_filtered.trade_fill), use_container_width=True) - st.plotly_chart(top_n_trades(page_data_filtered.trade_fill.net_realized_pnl), use_container_width=True) - else: - st.warning("Market data is not available so the candles graph is not going to be rendered. " - "Make sure that you are using the latest version of Hummingbot and market data recorder activated.") - st.divider() - st.subheader("📈 Metrics") - with st.container(): - col1, col2, col3, col4, col5 = st.columns(5) - with col1: - st.metric(label=f'Trade PNL {strategy_data_filtered.quote_asset}', - value=round(strategy_data_filtered.trade_pnl_quote, 2)) - st.metric(label=f'Fees {strategy_data_filtered.quote_asset}', - value=round(strategy_data_filtered.cum_fees_in_quote, 2)) - with col2: - st.metric(label='Total Buy Trades', value=strategy_data_filtered.total_buy_trades) - st.metric(label='Total Sell Trades', value=strategy_data_filtered.total_sell_trades) - with col3: - st.metric(label='Total Buy Trades Amount', - value=round(strategy_data_filtered.total_buy_amount, 2)) - st.metric(label='Total Sell Trades Amount', - value=round(strategy_data_filtered.total_sell_amount, 2)) - with col4: - st.metric(label='Average Buy Price', value=round(strategy_data_filtered.average_buy_price, 4)) - st.metric(label='Average Sell Price', value=round(strategy_data_filtered.average_sell_price, 4)) - with col5: - st.metric(label='Inventory change in Base asset', - value=round(strategy_data_filtered.inventory_change_base_asset, 4)) - st.divider() - st.subheader("Tables") - with st.expander("💵 Trades"): - st.write(strategy_data.trade_fill) - download_csv(strategy_data.trade_fill, "trade_fill", "download-trades") - with st.expander("📩 Orders"): - st.write(strategy_data.orders) - download_csv(strategy_data.orders, "orders", "download-orders") - with st.expander("⌕ Order Status"): - st.write(strategy_data.order_status) - download_csv(strategy_data.order_status, "order_status", "download-order-status") - else: - st.warning("We are encountering challenges in maintaining continuous analysis of this database.") - with st.expander("DB Status"): - status_df = pd.DataFrame([selected_db.status]).transpose().reset_index() - status_df.columns = ["Attribute", "Value"] - st.table(status_df) + st.metric(label='Total Buy Trades', value=strategy_data_filtered.total_buy_trades) + st.metric(label='Total Sell Trades', value=strategy_data_filtered.total_sell_trades) + with col3: + st.metric(label='Total Buy Trades Amount', + value=round(strategy_data_filtered.total_buy_amount, 2)) + st.metric(label='Total Sell Trades Amount', + value=round(strategy_data_filtered.total_sell_amount, 2)) + with col4: + st.metric(label='Average Buy Price', value=round(strategy_data_filtered.average_buy_price, 4)) + st.metric(label='Average Sell Price', value=round(strategy_data_filtered.average_sell_price, 4)) + with col5: + st.metric(label='Inventory change in Base asset', + value=round(strategy_data_filtered.inventory_change_base_asset, 4)) + st.divider() + st.subheader("Tables") + with st.expander("💵 Trades"): + st.write(strategy_data.trade_fill) + download_csv(strategy_data.trade_fill, "trade_fill", "download-trades") + with st.expander("📩 Orders"): + st.write(strategy_data.orders) + download_csv(strategy_data.orders, "orders", "download-orders") + with st.expander("⌕ Order Status"): + st.write(strategy_data.order_status) + download_csv(strategy_data.order_status, "order_status", "download-order-status") + else: + st.warning("We are encountering challenges in maintaining continuous analysis of this database.") + with st.expander("DB Status"): + status_df = pd.DataFrame([selected_db.status]).transpose().reset_index() + status_df.columns = ["Attribute", "Value"] + st.table(status_df) else: st.warning("We were unable to process this SQLite database.") with st.expander("DB Status"): From 98af66159448c1351decf65e23cdb517f5ecc18a Mon Sep 17 00:00:00 2001 From: drupman Date: Sat, 16 Sep 2023 00:29:47 -0300 Subject: [PATCH 25/40] (refactor) remove default selection --- utils/data_manipulation.py | 1 - 1 file changed, 1 deletion(-) diff --git a/utils/data_manipulation.py b/utils/data_manipulation.py index cf78bb31..16968a19 100644 --- a/utils/data_manipulation.py +++ b/utils/data_manipulation.py @@ -41,7 +41,6 @@ def full_series(series): "net_realized_pnl_last": "Realized PnL"}, inplace=True) strategy_summary.sort_values(["Realized PnL"], ascending=True, inplace=True) strategy_summary["Examine"] = False - strategy_summary.loc[0, "Examine"] = True return strategy_summary def get_single_market_strategy_data(self, exchange: str, trading_pair: str): From 846b527f3c30b05faf419bceaab634544df0f266 Mon Sep 17 00:00:00 2001 From: drupman Date: Sat, 16 Sep 2023 01:15:33 -0300 Subject: [PATCH 26/40] (feat) add uploading db feature --- pages/strategy_performance/app.py | 37 ++++++++++++++++++------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/pages/strategy_performance/app.py b/pages/strategy_performance/app.py index 5ea73eae..d6214673 100644 --- a/pages/strategy_performance/app.py +++ b/pages/strategy_performance/app.py @@ -12,6 +12,7 @@ BULLISH_COLOR = "rgba(97, 199, 102, 0.9)" BEARISH_COLOR = "rgba(255, 102, 90, 0.9)" +UPLOAD_FOLDER = "data" # Start content here intervals = { @@ -26,11 +27,13 @@ } -@st.cache_resource def get_databases(): sqlite_files = [db_name for db_name in os.listdir("data") if db_name.endswith(".sqlite")] databases_list = [DatabaseManager(db) for db in sqlite_files] - return {database.db_name: database for database in databases_list} + if len(databases_list) > 0: + return {database.db_name: database for database in databases_list} + else: + return None def download_csv(df: pd.DataFrame, filename: str, key: str): @@ -220,20 +223,22 @@ def hr_str(hr): style_metric_cards() st.subheader("🔫 Data source") -dbs = get_databases() -db_names = [x.db_name for x in dbs.values()] -select_tab, upload_tab = st.tabs(["Select", "Upload"]) -with select_tab: - if db_names is not None: - selected_db_name = st.selectbox("Select a database to use:", db_names) - selected_db = dbs[selected_db_name] - else: - st.warning("Ups! No databases were founded. Try uploading one in Upload tab.") - selected_db = None -with upload_tab: - uploaded_db = st.file_uploader("Upload your sqlite database", type=["sqlite", "db"]) +with st.expander("⬆️ Upload"): + uploaded_db = st.file_uploader("Select a Hummingbot SQLite Database", type=["sqlite", "db"]) if uploaded_db is not None: - selected_db = DatabaseManager(uploaded_db) + file_contents = uploaded_db.read() + with open(os.path.join(UPLOAD_FOLDER, uploaded_db.name), "wb") as f: + f.write(file_contents) + st.success("File uploaded and saved successfully!") + selected_db = DatabaseManager(uploaded_db.name) +dbs = get_databases() +if dbs is not None: + db_names = [x.db_name for x in dbs.values()] + selected_db_name = st.selectbox("Select a database to start:", db_names) + selected_db = dbs[selected_db_name] +else: + st.warning("Ups! No databases were founded. Start uploading one") + selected_db = None if selected_db is not None: strategy_data = selected_db.get_strategy_data() if strategy_data.strategy_summary is not None: @@ -249,7 +254,7 @@ def hr_str(hr): summary_chart = summary_chart(strategy_data.strategy_summary) st.plotly_chart(summary_chart, use_container_width=True) if selection is None: - st.info("Choose a trading pair and start analyzing!") + st.info("💡 Choose a trading pair and start analyzing!") else: st.divider() st.subheader("🔍 Examine Trading Pair") From 4687ba9b6a927fbbfa33c44efe97ac5ebc10429f Mon Sep 17 00:00:00 2001 From: drupman Date: Tue, 19 Sep 2023 17:16:29 -0300 Subject: [PATCH 27/40] (fix) correct profit factor --- utils/data_manipulation.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utils/data_manipulation.py b/utils/data_manipulation.py index 16968a19..7d9021be 100644 --- a/utils/data_manipulation.py +++ b/utils/data_manipulation.py @@ -249,9 +249,9 @@ def accuracy(self): @property def profit_factor(self): - total_profit = (self.trade_fill["net_realized_pnl"] >= 0).sum() - total_loss = (self.trade_fill["net_realized_pnl"] < 0).sum() - return total_profit / total_loss + total_profit = self.trade_fill.loc[self.trade_fill["realized_pnl"] >= 0, "realized_pnl"].sum() + total_loss = self.trade_fill.loc[self.trade_fill["realized_pnl"] < 0, "realized_pnl"].sum() + return total_profit / -total_loss @property def properties_table(self): From 70961a8d8d66c943c5f4382911b5e6ae9921bee7 Mon Sep 17 00:00:00 2001 From: drupman Date: Tue, 19 Sep 2023 17:17:22 -0300 Subject: [PATCH 28/40] (feat) rename examine summary column with explore and set as first column --- utils/data_manipulation.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/utils/data_manipulation.py b/utils/data_manipulation.py index 7d9021be..80f62d18 100644 --- a/utils/data_manipulation.py +++ b/utils/data_manipulation.py @@ -40,7 +40,10 @@ def full_series(series): "net_realized_pnl_full_series": "PnL Over Time", "net_realized_pnl_last": "Realized PnL"}, inplace=True) strategy_summary.sort_values(["Realized PnL"], ascending=True, inplace=True) - strategy_summary["Examine"] = False + strategy_summary["Explore"] = False + column_names = list(strategy_summary.columns) + column_names.insert(0, column_names.pop()) + strategy_summary = strategy_summary[column_names] return strategy_summary def get_single_market_strategy_data(self, exchange: str, trading_pair: str): From ec1dc6e2509f9c278892679b52ea75cb1c4730bf Mon Sep 17 00:00:00 2001 From: drupman Date: Tue, 19 Sep 2023 17:18:32 -0300 Subject: [PATCH 29/40] (feat) rename examine summary column with explore and set as first column --- pages/strategy_performance/app.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pages/strategy_performance/app.py b/pages/strategy_performance/app.py index d6214673..142dd7c0 100644 --- a/pages/strategy_performance/app.py +++ b/pages/strategy_performance/app.py @@ -83,13 +83,14 @@ def show_strategy_summary(summary_df: pd.DataFrame): column_config={"PnL Over Time": st.column_config.LineChartColumn("PnL Over Time", y_min=0, y_max=5000), - "Examine": st.column_config.CheckboxColumn(required=True) + "Explore": st.column_config.CheckboxColumn(required=True) }, - use_container_width=True + use_container_width=True, + hide_index=True ) - selected_rows = summary[summary.Examine] + selected_rows = summary[summary.Explore] if len(selected_rows) > 0: - return selected_rows.drop('Examine', axis=1) + return selected_rows.drop('Explore', axis=1) else: return None From f97d4bb97715ba177b1b886f4e563d15fbc50aee Mon Sep 17 00:00:00 2001 From: drupman Date: Tue, 19 Sep 2023 17:19:12 -0300 Subject: [PATCH 30/40] (feat) replace cummulative pnl for individual pnl --- pages/strategy_performance/app.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/pages/strategy_performance/app.py b/pages/strategy_performance/app.py index 142dd7c0..9f8ac73a 100644 --- a/pages/strategy_performance/app.py +++ b/pages/strategy_performance/app.py @@ -94,6 +94,7 @@ def show_strategy_summary(summary_df: pd.DataFrame): else: return None + def summary_chart(df: pd.DataFrame): fig = px.bar(df, x="Trading Pair", y="Realized PnL", color="Exchange") fig.update_traces(width=min(1.0, 0.1 * len(strategy_data.strategy_summary))) @@ -172,20 +173,20 @@ def hr_str(hr): return hr_str + suffix df["hour"] = df["timestamp"].dt.hour - profits = df[df["net_realized_pnl"] >= 0] - losses = df[df["net_realized_pnl"] < 0] - polar_profits = profits.groupby("hour")["net_realized_pnl"].sum().reset_index() - polar_losses = losses.groupby("hour")["net_realized_pnl"].sum().reset_index() - polar_losses["net_realized_pnl"] = abs(polar_losses["net_realized_pnl"]) + profits = df[df["realized_pnl"] >= 0] + losses = df[df["realized_pnl"] < 0] + polar_profits = profits.groupby("hour")["realized_pnl"].sum().reset_index() + polar_losses = losses.groupby("hour")["realized_pnl"].sum().reset_index() + polar_losses["realized_pnl"] = abs(polar_losses["realized_pnl"]) fig = go.Figure() fig.add_trace(go.Barpolar( name="Profits", - r=polar_profits["net_realized_pnl"], + r=polar_profits["realized_pnl"], theta=polar_profits["hour"] * 15, marker_color=BULLISH_COLOR)) fig.add_trace(go.Barpolar( name="Losses", - r=polar_losses["net_realized_pnl"], + r=polar_losses["realized_pnl"], theta=polar_losses["hour"] * 15, marker_color=BEARISH_COLOR)) fig.update_layout( @@ -258,7 +259,7 @@ def hr_str(hr): st.info("💡 Choose a trading pair and start analyzing!") else: st.divider() - st.subheader("🔍 Examine Trading Pair") + st.subheader("🔍 Explore Trading Pair") if not any("Error" in value for key, value in selected_db.status.items() if key != "position_executor"): date_array = pd.date_range(start=strategy_data.start_time, end=strategy_data.end_time, periods=60) start_time, end_time = st.select_slider("Select a time range to analyze", @@ -278,7 +279,7 @@ def hr_str(hr): st.metric(label='Total Trades', value=strategy_data_filtered.total_orders) with col3: st.metric(label='Accuracy', - value=round(strategy_data_filtered.accuracy, 2)) + value=f"{100 * strategy_data_filtered.accuracy:.2f} %") with col4: st.metric(label="Profit Factor", value=round(strategy_data_filtered.profit_factor, 2)) @@ -337,7 +338,7 @@ def hr_str(hr): st.plotly_chart(fig, use_container_width=True) with col2: st.plotly_chart(intraday_performance(page_data_filtered.trade_fill), use_container_width=True) - st.plotly_chart(top_n_trades(page_data_filtered.trade_fill.net_realized_pnl), use_container_width=True) + st.plotly_chart(top_n_trades(page_data_filtered.trade_fill.realized_pnl), use_container_width=True) else: st.warning("Market data is not available so the candles graph is not going to be rendered. " "Make sure that you are using the latest version of Hummingbot and market data recorder activated.") From 7e661e38dd6a49b865aa3ef9645e4fe0e225b2d9 Mon Sep 17 00:00:00 2001 From: drupman Date: Tue, 19 Sep 2023 19:00:50 -0300 Subject: [PATCH 31/40] (feat) handle multiple strategies selection --- pages/strategy_performance/app.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pages/strategy_performance/app.py b/pages/strategy_performance/app.py index 9f8ac73a..b23da35f 100644 --- a/pages/strategy_performance/app.py +++ b/pages/strategy_performance/app.py @@ -90,7 +90,7 @@ def show_strategy_summary(summary_df: pd.DataFrame): ) selected_rows = summary[summary.Explore] if len(selected_rows) > 0: - return selected_rows.drop('Explore', axis=1) + return selected_rows else: return None @@ -250,6 +250,9 @@ def hr_str(hr): with table_tab: selection = show_strategy_summary(strategy_data.strategy_summary) if selection is not None: + if len(selection) > 1: + st.warning("This version doesn't support multiple selections. Please try selecting only one.") + st.stop() selected_exchange = selection["Exchange"].values[0] selected_trading_pair = selection["Trading Pair"].values[0] with chart_tab: From 01a400123651ffe1f675e4589271de896b39d1fa Mon Sep 17 00:00:00 2001 From: drupman Date: Tue, 19 Sep 2023 20:48:37 -0300 Subject: [PATCH 32/40] (feat) remove quote inventory change markers --- utils/graphs.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/utils/graphs.py b/utils/graphs.py index d57e1343..c4b9b2ba 100644 --- a/utils/graphs.py +++ b/utils/graphs.py @@ -152,12 +152,7 @@ def add_quote_inventory_change(self, strategy_data: StrategyData, row=3): x=strategy_data.trade_fill.timestamp, y=strategy_data.trade_fill.inventory_cost, name="Quote Inventory", - mode="lines+markers", - marker=dict( - size=10, - symbol="arrow", - angleref="previous", - ), + mode="lines", line=dict(shape="hv"), ), row=row, col=1 @@ -172,7 +167,6 @@ def add_pnl(self, strategy_data: SingleMarketStrategyData, row=4): name="Cum Profit", mode='lines', line=dict(shape="hv", color="rgba(1, 1, 1, 0.5)", dash="dash", width=0.1), - # marker=dict(symbol="arrow"), fill="tozeroy", # Fill to the line below (trade pnl) fillcolor="rgba(0, 255, 0, 0.5)" ), From e8a80fc429bb5cec998a903ea2cfdab65c381e7b Mon Sep 17 00:00:00 2001 From: drupman Date: Tue, 19 Sep 2023 20:48:50 -0300 Subject: [PATCH 33/40] (feat) add quote volume to trade_fill --- utils/database_manager.py | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/database_manager.py b/utils/database_manager.py index dd3eb6e1..4570bf99 100644 --- a/utils/database_manager.py +++ b/utils/database_manager.py @@ -160,6 +160,7 @@ def get_trade_fills(self, config_file_path=None, start_date=None, end_date=None) trade_fills["realized_pnl"] = trade_fills["net_realized_pnl"].diff() trade_fills["timestamp"] = pd.to_datetime(trade_fills["timestamp"], unit="ms") trade_fills["market"] = trade_fills["market"].apply(lambda x: x.lower().replace("_papertrade", "")) + trade_fills["quote_volume"] = trade_fills["price"] * trade_fills["amount"] return trade_fills def get_order_status(self, order_ids=None, start_date=None, end_date=None): From b973f33eb31dd3d397ed6a062897cc37434d2a5c Mon Sep 17 00:00:00 2001 From: drupman Date: Tue, 19 Sep 2023 20:49:56 -0300 Subject: [PATCH 34/40] (feat) change radar chart: volume vs realized_pnl --- pages/strategy_performance/app.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/pages/strategy_performance/app.py b/pages/strategy_performance/app.py index b23da35f..7ddccf8d 100644 --- a/pages/strategy_performance/app.py +++ b/pages/strategy_performance/app.py @@ -173,22 +173,27 @@ def hr_str(hr): return hr_str + suffix df["hour"] = df["timestamp"].dt.hour - profits = df[df["realized_pnl"] >= 0] - losses = df[df["realized_pnl"] < 0] - polar_profits = profits.groupby("hour")["realized_pnl"].sum().reset_index() - polar_losses = losses.groupby("hour")["realized_pnl"].sum().reset_index() - polar_losses["realized_pnl"] = abs(polar_losses["realized_pnl"]) + realized_pnl_per_hour = df.groupby("hour")[["realized_pnl", "quote_volume"]].sum().reset_index() fig = go.Figure() fig.add_trace(go.Barpolar( name="Profits", - r=polar_profits["realized_pnl"], - theta=polar_profits["hour"] * 15, - marker_color=BULLISH_COLOR)) - fig.add_trace(go.Barpolar( - name="Losses", - r=polar_losses["realized_pnl"], - theta=polar_losses["hour"] * 15, - marker_color=BEARISH_COLOR)) + r=realized_pnl_per_hour["quote_volume"], + theta=realized_pnl_per_hour["hour"] * 15, + marker=dict( + color=realized_pnl_per_hour["realized_pnl"], + colorscale="RdYlGn", + cmin=-(abs(realized_pnl_per_hour["realized_pnl"]).max()), + cmid=0.0, + cmax=(abs(realized_pnl_per_hour["realized_pnl"]).max()), + colorbar=dict( + title='Realized PnL', + x=0, + y=-0.5, + xanchor='left', + yanchor='bottom', + orientation='h' + ) + ))) fig.update_layout( polar=dict( radialaxis=dict( From b91073a0d508e8379eecee90cc39475aa2d9c708 Mon Sep 17 00:00:00 2001 From: drupman Date: Tue, 19 Sep 2023 20:50:15 -0300 Subject: [PATCH 35/40] (feat) add returns histogram chart --- pages/strategy_performance/app.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/pages/strategy_performance/app.py b/pages/strategy_performance/app.py index 7ddccf8d..846ab9a3 100644 --- a/pages/strategy_performance/app.py +++ b/pages/strategy_performance/app.py @@ -228,6 +228,33 @@ def hr_str(hr): return fig +def returns_histogram(df: pd.DataFrame): + colors = [BULLISH_COLOR if val > 0 else BEARISH_COLOR for val in df["realized_pnl"]] + fig = go.Figure() + fig.add_trace(go.Histogram(name="Losses", + x=df.loc[df["realized_pnl"] < 0, "realized_pnl"], + marker_color=BEARISH_COLOR)) + fig.add_trace(go.Histogram(name="Profits", + x=df.loc[df["realized_pnl"] > 0, "realized_pnl"], + marker_color=BULLISH_COLOR)) + fig.update_layout(title=dict( + text='Returns Histogram', + x=0.5, + # y=0.93, + xanchor="center", + # yanchor="bottom" + ), + legend=dict( + orientation="h", + # entrywidth=70, + yanchor="bottom", + y=1.02, + xanchor="center", + x=.48 + )) + return fig + + style_metric_cards() st.subheader("🔫 Data source") with st.expander("⬆️ Upload"): From 38b5855ece6b1acd6170f344afc273b479325356 Mon Sep 17 00:00:00 2001 From: drupman Date: Tue, 19 Sep 2023 20:50:47 -0300 Subject: [PATCH 36/40] (feat) remove market activity expander and replace top and worst trades with histogram returns --- pages/strategy_performance/app.py | 71 +++++++++++++++---------------- 1 file changed, 35 insertions(+), 36 deletions(-) diff --git a/pages/strategy_performance/app.py b/pages/strategy_performance/app.py index 846ab9a3..8a658294 100644 --- a/pages/strategy_performance/app.py +++ b/pages/strategy_performance/app.py @@ -338,42 +338,41 @@ def returns_histogram(df: pd.DataFrame): st.subheader("💱 Market activity") if "Error" not in selected_db.status["market_data"] and strategy_data_filtered.market_data is not None: - with st.expander("Market activity", expanded=True): - col1, col2, col3 = st.columns([1, 1, 2]) - with col1: - interval = st.selectbox("Candles Interval:", intervals.keys(), index=2) - with col2: - rows_per_page = st.number_input("Candles per Page", value=100, min_value=1, max_value=5000) - with col3: - total_rows = len( - strategy_data_filtered.get_market_data_resampled(interval=f"{intervals[interval]}S")) - total_pages = math.ceil(total_rows / rows_per_page) - if total_pages > 1: - selected_page = st.select_slider("Select page", list(range(total_pages)), total_pages - 1, - key="page_slider") - else: - selected_page = 0 - start_idx = selected_page * rows_per_page - end_idx = start_idx + rows_per_page - candles_df = strategy_data_filtered.get_market_data_resampled( - interval=f"{intervals[interval]}S").iloc[ - start_idx:end_idx] - start_time_page = candles_df.index.min() - end_time_page = candles_df.index.max() - page_data_filtered = single_market_strategy_data.get_filtered_strategy_data(start_time_page, - end_time_page) - col1, col2 = st.columns([5.5, 1.5]) - with col1: - cg = CandlesGraph(candles_df, show_volume=False, extra_rows=2) - cg.add_buy_trades(page_data_filtered.buys) - cg.add_sell_trades(page_data_filtered.sells) - cg.add_pnl(page_data_filtered, row=2) - cg.add_quote_inventory_change(page_data_filtered, row=3) - fig = cg.figure() - st.plotly_chart(fig, use_container_width=True) - with col2: - st.plotly_chart(intraday_performance(page_data_filtered.trade_fill), use_container_width=True) - st.plotly_chart(top_n_trades(page_data_filtered.trade_fill.realized_pnl), use_container_width=True) + col1, col2, col3 = st.columns([1, 1, 2]) + with col1: + interval = st.selectbox("Candles Interval:", intervals.keys(), index=2) + with col2: + rows_per_page = st.number_input("Candles per Page", value=1500, min_value=1, max_value=5000) + with col3: + total_rows = len( + strategy_data_filtered.get_market_data_resampled(interval=f"{intervals[interval]}S")) + total_pages = math.ceil(total_rows / rows_per_page) + if total_pages > 1: + selected_page = st.select_slider("Select page", list(range(total_pages)), total_pages - 1, + key="page_slider") + else: + selected_page = 0 + start_idx = selected_page * rows_per_page + end_idx = start_idx + rows_per_page + candles_df = strategy_data_filtered.get_market_data_resampled( + interval=f"{intervals[interval]}S").iloc[ + start_idx:end_idx] + start_time_page = candles_df.index.min() + end_time_page = candles_df.index.max() + page_data_filtered = single_market_strategy_data.get_filtered_strategy_data(start_time_page, + end_time_page) + col1, col2 = st.columns([5.5, 1.5]) + with col1: + cg = CandlesGraph(candles_df, show_volume=False, extra_rows=2) + cg.add_buy_trades(page_data_filtered.buys) + cg.add_sell_trades(page_data_filtered.sells) + cg.add_pnl(page_data_filtered, row=2) + cg.add_quote_inventory_change(page_data_filtered, row=3) + fig = cg.figure() + st.plotly_chart(fig, use_container_width=True) + with col2: + st.plotly_chart(intraday_performance(page_data_filtered.trade_fill), use_container_width=True) + st.plotly_chart(returns_histogram(page_data_filtered.trade_fill), use_container_width=True) else: st.warning("Market data is not available so the candles graph is not going to be rendered. " "Make sure that you are using the latest version of Hummingbot and market data recorder activated.") From 3af139104b55275815cc42d7911f7e8d513d6cb8 Mon Sep 17 00:00:00 2001 From: drupman Date: Tue, 19 Sep 2023 22:54:14 -0300 Subject: [PATCH 37/40] (feat) add cummulative pnl series names --- pages/strategy_performance/app.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/pages/strategy_performance/app.py b/pages/strategy_performance/app.py index 8a658294..bb02179c 100644 --- a/pages/strategy_performance/app.py +++ b/pages/strategy_performance/app.py @@ -107,8 +107,17 @@ def pnl_over_time(df: pd.DataFrame): df_below = df[df['net_realized_pnl'] < 0] fig = go.Figure() - fig.add_trace(go.Bar(x=df_above.index, y=df_above["net_realized_pnl"], marker_color=BULLISH_COLOR, showlegend=False)) - fig.add_trace(go.Bar(x=df_below.index, y=df_below["net_realized_pnl"], marker_color=BEARISH_COLOR, showlegend=False)) + fig.add_trace(go.Bar(name="Cum Realized PnL", + x=df_above.index, + y=df_above["net_realized_pnl"], + marker_color=BULLISH_COLOR, + # hoverdq + showlegend=False)) + fig.add_trace(go.Bar(name="Cum Realized PnL", + x=df_below.index, + y=df_below["net_realized_pnl"], + marker_color=BEARISH_COLOR, + showlegend=False)) fig.update_layout(title=dict( text='Cummulative PnL', # Your title text x=0.43, From 75b9dc7ffbfcd6fd6ad45e2035d7389af63c8b1a Mon Sep 17 00:00:00 2001 From: drupman Date: Tue, 19 Sep 2023 22:54:56 -0300 Subject: [PATCH 38/40] (feat) improve returns histogram chart --- pages/strategy_performance/app.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/pages/strategy_performance/app.py b/pages/strategy_performance/app.py index bb02179c..912a917c 100644 --- a/pages/strategy_performance/app.py +++ b/pages/strategy_performance/app.py @@ -238,7 +238,6 @@ def hr_str(hr): def returns_histogram(df: pd.DataFrame): - colors = [BULLISH_COLOR if val > 0 else BEARISH_COLOR for val in df["realized_pnl"]] fig = go.Figure() fig.add_trace(go.Histogram(name="Losses", x=df.loc[df["realized_pnl"] < 0, "realized_pnl"], @@ -246,16 +245,14 @@ def returns_histogram(df: pd.DataFrame): fig.add_trace(go.Histogram(name="Profits", x=df.loc[df["realized_pnl"] > 0, "realized_pnl"], marker_color=BULLISH_COLOR)) - fig.update_layout(title=dict( - text='Returns Histogram', + fig.update_layout( + title=dict( + text='Returns Distribution', x=0.5, - # y=0.93, xanchor="center", - # yanchor="bottom" ), legend=dict( orientation="h", - # entrywidth=70, yanchor="bottom", y=1.02, xanchor="center", From 1a12e7203c0fb406539820ba169238d4783437e3 Mon Sep 17 00:00:00 2001 From: drupman Date: Tue, 19 Sep 2023 22:55:52 -0300 Subject: [PATCH 39/40] (feat) isolate candles_graph and panel metrics --- pages/strategy_performance/app.py | 43 +++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/pages/strategy_performance/app.py b/pages/strategy_performance/app.py index 912a917c..a74f73e5 100644 --- a/pages/strategy_performance/app.py +++ b/pages/strategy_performance/app.py @@ -261,6 +261,15 @@ def returns_histogram(df: pd.DataFrame): return fig +def candles_graph(candles: pd.DataFrame, strat_data, show_volume=False, extra_rows=2): + cg = CandlesGraph(candles, show_volume=show_volume, extra_rows=extra_rows) + cg.add_buy_trades(strat_data.buys) + cg.add_sell_trades(strat_data.sells) + cg.add_pnl(strategy_data, row=2) + cg.add_quote_inventory_change(strat_data, row=3) + return cg.figure() + + style_metric_cards() st.subheader("🔫 Data source") with st.expander("⬆️ Upload"): @@ -344,12 +353,15 @@ def returns_histogram(df: pd.DataFrame): st.subheader("💱 Market activity") if "Error" not in selected_db.status["market_data"] and strategy_data_filtered.market_data is not None: - col1, col2, col3 = st.columns([1, 1, 2]) + col1, col2, col3, col4 = st.columns(4) with col1: interval = st.selectbox("Candles Interval:", intervals.keys(), index=2) with col2: rows_per_page = st.number_input("Candles per Page", value=1500, min_value=1, max_value=5000) with col3: + st.markdown("##") + show_panel_metrics = st.checkbox("Show panel metrics", value=True) + with col4: total_rows = len( strategy_data_filtered.get_market_data_resampled(interval=f"{intervals[interval]}S")) total_pages = math.ceil(total_rows / rows_per_page) @@ -367,18 +379,23 @@ def returns_histogram(df: pd.DataFrame): end_time_page = candles_df.index.max() page_data_filtered = single_market_strategy_data.get_filtered_strategy_data(start_time_page, end_time_page) - col1, col2 = st.columns([5.5, 1.5]) - with col1: - cg = CandlesGraph(candles_df, show_volume=False, extra_rows=2) - cg.add_buy_trades(page_data_filtered.buys) - cg.add_sell_trades(page_data_filtered.sells) - cg.add_pnl(page_data_filtered, row=2) - cg.add_quote_inventory_change(page_data_filtered, row=3) - fig = cg.figure() - st.plotly_chart(fig, use_container_width=True) - with col2: - st.plotly_chart(intraday_performance(page_data_filtered.trade_fill), use_container_width=True) - st.plotly_chart(returns_histogram(page_data_filtered.trade_fill), use_container_width=True) + if show_panel_metrics: + col1, col2 = st.columns([2, 1]) + with col1: + candles_chart = candles_graph(candles_df, page_data_filtered) + st.plotly_chart(candles_chart, use_container_width=True) + with col2: + chart_tab, table_tab = st.tabs(["Chart", "Table"]) + with chart_tab: + st.plotly_chart(intraday_performance(page_data_filtered.trade_fill), use_container_width=True) + st.plotly_chart(returns_histogram(page_data_filtered.trade_fill), use_container_width=True) + with table_tab: + st.dataframe(page_data_filtered.trade_fill[["timestamp", "realized_pnl"]].dropna(subset="realized_pnl"), + use_container_width=True, + hide_index=True, + height=candles_chart.layout.height - 180) + else: + st.plotly_chart(candles_graph(candles_df, page_data_filtered), use_container_width=True) else: st.warning("Market data is not available so the candles graph is not going to be rendered. " "Make sure that you are using the latest version of Hummingbot and market data recorder activated.") From 5bb3ad6f550b28cd0f7667f0f5c5e24c83ebd991 Mon Sep 17 00:00:00 2001 From: drupman Date: Tue, 19 Sep 2023 23:21:31 -0300 Subject: [PATCH 40/40] (feat) add trades table in panel metrics --- pages/strategy_performance/app.py | 4 ++-- utils/database_manager.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pages/strategy_performance/app.py b/pages/strategy_performance/app.py index a74f73e5..eff2af63 100644 --- a/pages/strategy_performance/app.py +++ b/pages/strategy_performance/app.py @@ -390,10 +390,10 @@ def candles_graph(candles: pd.DataFrame, strat_data, show_volume=False, extra_ro st.plotly_chart(intraday_performance(page_data_filtered.trade_fill), use_container_width=True) st.plotly_chart(returns_histogram(page_data_filtered.trade_fill), use_container_width=True) with table_tab: - st.dataframe(page_data_filtered.trade_fill[["timestamp", "realized_pnl"]].dropna(subset="realized_pnl"), + st.dataframe(page_data_filtered.trade_fill[["timestamp", "gross_pnl", "trade_fee", "realized_pnl"]].dropna(subset="realized_pnl"), use_container_width=True, hide_index=True, - height=candles_chart.layout.height - 180) + height=(min(len(page_data_filtered.trade_fill) * 39, candles_chart.layout.height - 180))) else: st.plotly_chart(candles_graph(candles_df, page_data_filtered), use_container_width=True) else: diff --git a/utils/database_manager.py b/utils/database_manager.py index 4570bf99..18f003fd 100644 --- a/utils/database_manager.py +++ b/utils/database_manager.py @@ -158,6 +158,8 @@ def get_trade_fills(self, config_file_path=None, start_date=None, end_date=None) trade_fills["realized_trade_pnl"] = trade_fills["unrealized_trade_pnl"] + trade_fills["inventory_cost"] trade_fills["net_realized_pnl"] = trade_fills["realized_trade_pnl"] - trade_fills["cum_fees_in_quote"] trade_fills["realized_pnl"] = trade_fills["net_realized_pnl"].diff() + trade_fills["gross_pnl"] = trade_fills["realized_trade_pnl"].diff() + trade_fills["trade_fee"] = trade_fills["cum_fees_in_quote"].diff() trade_fills["timestamp"] = pd.to_datetime(trade_fills["timestamp"], unit="ms") trade_fills["market"] = trade_fills["market"].apply(lambda x: x.lower().replace("_papertrade", "")) trade_fills["quote_volume"] = trade_fills["price"] * trade_fills["amount"]