diff --git a/pages/strategy_performance/app.py b/pages/strategy_performance/app.py
index 6491ec6b..eff2af63 100644
--- a/pages/strategy_performance/app.py
+++ b/pages/strategy_performance/app.py
@@ -1,15 +1,19 @@
import os
import pandas as pd
import streamlit as st
+import plotly.graph_objects as go
import math
-
+import plotly.express as px
from utils.database_manager import DatabaseManager
from utils.graphs import CandlesGraph
from utils.st_utils import initialize_st_page
-
initialize_st_page(title="Strategy Performance", icon="🚀")
+BULLISH_COLOR = "rgba(97, 199, 102, 0.9)"
+BEARISH_COLOR = "rgba(255, 102, 90, 0.9)"
+UPLOAD_FOLDER = "data"
+
# Start content here
intervals = {
"1m": 60,
@@ -23,11 +27,13 @@
}
-@st.cache_resource
def get_databases():
sqlite_files = [db_name for db_name in os.listdir("data") if db_name.endswith(".sqlite")]
databases_list = [DatabaseManager(db) for db in sqlite_files]
- return {database.db_name: database for database in databases_list}
+ if len(databases_list) > 0:
+ return {database.db_name: database for database in databases_list}
+ else:
+ return None
def download_csv(df: pd.DataFrame, filename: str, key: str):
@@ -41,126 +47,401 @@ def download_csv(df: pd.DataFrame, filename: str, key: str):
)
-st.session_state["dbs"] = get_databases()
-db_names = [x.db_name for x in st.session_state["dbs"].values() if x.status == 'OK']
-if not db_names:
- st.warning("No trades have been recorded in the selected database")
- selected_db_name = None
- selected_db = None
+def style_metric_cards(
+ background_color: str = "rgba(255, 255, 255, 0)",
+ border_size_px: int = 1,
+ border_color: str = "rgba(255, 255, 255, 0.3)",
+ border_radius_px: int = 5,
+ border_left_color: str = "rgba(255, 255, 255, 0.5)",
+ box_shadow: bool = True,
+):
+
+ box_shadow_str = (
+ "box-shadow: 0 0.15rem 1.75rem 0 rgba(58, 59, 69, 0.15) !important;"
+ if box_shadow
+ else "box-shadow: none !important;"
+ )
+ st.markdown(
+ f"""
+
+ """,
+ unsafe_allow_html=True,
+ )
+
+
+def show_strategy_summary(summary_df: pd.DataFrame):
+ summary = st.data_editor(summary_df,
+ column_config={"PnL Over Time": st.column_config.LineChartColumn("PnL Over Time",
+ y_min=0,
+ y_max=5000),
+ "Explore": st.column_config.CheckboxColumn(required=True)
+ },
+ use_container_width=True,
+ hide_index=True
+ )
+ selected_rows = summary[summary.Explore]
+ if len(selected_rows) > 0:
+ return selected_rows
+ else:
+ return None
+
+
+def summary_chart(df: pd.DataFrame):
+ fig = px.bar(df, x="Trading Pair", y="Realized PnL", color="Exchange")
+ fig.update_traces(width=min(1.0, 0.1 * len(strategy_data.strategy_summary)))
+ return fig
+
+
+def pnl_over_time(df: pd.DataFrame):
+ df.reset_index(drop=True, inplace=True)
+ df_above = df[df['net_realized_pnl'] >= 0]
+ df_below = df[df['net_realized_pnl'] < 0]
+
+ fig = go.Figure()
+ fig.add_trace(go.Bar(name="Cum Realized PnL",
+ x=df_above.index,
+ y=df_above["net_realized_pnl"],
+ marker_color=BULLISH_COLOR,
+ # hoverdq
+ showlegend=False))
+ fig.add_trace(go.Bar(name="Cum Realized PnL",
+ x=df_below.index,
+ y=df_below["net_realized_pnl"],
+ marker_color=BEARISH_COLOR,
+ showlegend=False))
+ fig.update_layout(title=dict(
+ text='Cummulative PnL', # Your title text
+ x=0.43,
+ y=0.95,
+ ),
+ plot_bgcolor='rgba(0,0,0,0)',
+ paper_bgcolor='rgba(0,0,0,0)')
+ return fig
+
+
+def top_n_trades(series, n: int = 8):
+ podium = list(range(0, n))
+ top_three_profits = series[series >= 0].sort_values(ascending=True)[-n:]
+ top_three_losses = series[series < 0].sort_values(ascending=False)[-n:]
+ fig = go.Figure()
+ fig.add_trace(go.Bar(name="Top Profits",
+ y=podium,
+ x=top_three_profits,
+ base=[0, 0, 0, 0],
+ marker_color=BULLISH_COLOR,
+ orientation='h',
+ text=top_three_profits.apply(lambda x: f"{x:.2f}"),
+ textposition="inside",
+ insidetextfont=dict(color='white')))
+ fig.add_trace(go.Bar(name="Top Losses",
+ y=podium,
+ x=top_three_losses,
+ marker_color=BEARISH_COLOR,
+ orientation='h',
+ text=top_three_losses.apply(lambda x: f"{x:.2f}"),
+ textposition="inside",
+ insidetextfont=dict(color='white')))
+ fig.update_layout(barmode='stack',
+ title=dict(
+ text='Top/Worst Realized PnLs', # Your title text
+ x=0.5,
+ y=0.95,
+ xanchor="center",
+ yanchor="top"
+ ),
+ xaxis=dict(showgrid=True, gridwidth=0.01, gridcolor="rgba(211, 211, 211, 0.5)"), # Show vertical grid lines
+ yaxis=dict(showgrid=False),
+ legend=dict(orientation="h",
+ x=0.5,
+ y=1.08,
+ xanchor="center",
+ yanchor="bottom"))
+ fig.update_yaxes(showticklabels=False,
+ showline=False,
+ range=[- n + 6, n + 1])
+ return fig
+
+
+def intraday_performance(df: pd.DataFrame):
+ def hr2angle(hr):
+ return (hr * 15) % 360
+
+ def hr_str(hr):
+ # Normalize hr to be between 1 and 12
+ hr_str = str(((hr - 1) % 12) + 1)
+ suffix = ' AM' if (hr % 24) < 12 else ' PM'
+ return hr_str + suffix
+
+ df["hour"] = df["timestamp"].dt.hour
+ realized_pnl_per_hour = df.groupby("hour")[["realized_pnl", "quote_volume"]].sum().reset_index()
+ fig = go.Figure()
+ fig.add_trace(go.Barpolar(
+ name="Profits",
+ r=realized_pnl_per_hour["quote_volume"],
+ theta=realized_pnl_per_hour["hour"] * 15,
+ marker=dict(
+ color=realized_pnl_per_hour["realized_pnl"],
+ colorscale="RdYlGn",
+ cmin=-(abs(realized_pnl_per_hour["realized_pnl"]).max()),
+ cmid=0.0,
+ cmax=(abs(realized_pnl_per_hour["realized_pnl"]).max()),
+ colorbar=dict(
+ title='Realized PnL',
+ x=0,
+ y=-0.5,
+ xanchor='left',
+ yanchor='bottom',
+ orientation='h'
+ )
+ )))
+ fig.update_layout(
+ polar=dict(
+ radialaxis=dict(
+ visible=True,
+ showline=False,
+ ),
+ angularaxis=dict(
+ rotation=90,
+ direction="clockwise",
+ tickvals=[hr2angle(hr) for hr in range(24)],
+ ticktext=[hr_str(hr) for hr in range(24)],
+ ),
+ bgcolor='rgba(255, 255, 255, 0)',
+
+ ),
+ legend=dict(
+ orientation="h",
+ x=0.5,
+ y=1.08,
+ xanchor="center",
+ yanchor="bottom"
+ ),
+ title=dict(
+ text='Intraday Performance',
+ x=0.5,
+ y=0.93,
+ xanchor="center",
+ yanchor="bottom"
+ ),
+ )
+
+ return fig
+
+
+def returns_histogram(df: pd.DataFrame):
+ fig = go.Figure()
+ fig.add_trace(go.Histogram(name="Losses",
+ x=df.loc[df["realized_pnl"] < 0, "realized_pnl"],
+ marker_color=BEARISH_COLOR))
+ fig.add_trace(go.Histogram(name="Profits",
+ x=df.loc[df["realized_pnl"] > 0, "realized_pnl"],
+ marker_color=BULLISH_COLOR))
+ fig.update_layout(
+ title=dict(
+ text='Returns Distribution',
+ x=0.5,
+ xanchor="center",
+ ),
+ legend=dict(
+ orientation="h",
+ yanchor="bottom",
+ y=1.02,
+ xanchor="center",
+ x=.48
+ ))
+ return fig
+
+
+def candles_graph(candles: pd.DataFrame, strat_data, show_volume=False, extra_rows=2):
+ cg = CandlesGraph(candles, show_volume=show_volume, extra_rows=extra_rows)
+ cg.add_buy_trades(strat_data.buys)
+ cg.add_sell_trades(strat_data.sells)
+ cg.add_pnl(strategy_data, row=2)
+ cg.add_quote_inventory_change(strat_data, row=3)
+ return cg.figure()
+
+
+style_metric_cards()
+st.subheader("🔫 Data source")
+with st.expander("⬆️ Upload"):
+ uploaded_db = st.file_uploader("Select a Hummingbot SQLite Database", type=["sqlite", "db"])
+ if uploaded_db is not None:
+ file_contents = uploaded_db.read()
+ with open(os.path.join(UPLOAD_FOLDER, uploaded_db.name), "wb") as f:
+ f.write(file_contents)
+ st.success("File uploaded and saved successfully!")
+ selected_db = DatabaseManager(uploaded_db.name)
+dbs = get_databases()
+if dbs is not None:
+ db_names = [x.db_name for x in dbs.values()]
+ selected_db_name = st.selectbox("Select a database to start:", db_names)
+ selected_db = dbs[selected_db_name]
else:
- st.subheader("⚙️ Filters")
- col1, col2, col3, col4 = st.columns(4)
- with col1:
- selected_db_name = st.selectbox("Select a database to use:", db_names)
- st.session_state["selected_db"] = st.session_state["dbs"][selected_db_name]
- with col2:
- if st.session_state.selected_db:
- st.session_state.selected_config_file = st.selectbox("Select a config file to analyze:", st.session_state.selected_db.config_files)
+ st.warning("Ups! No databases were founded. Start uploading one")
+ selected_db = None
+if selected_db is not None:
+ strategy_data = selected_db.get_strategy_data()
+ if strategy_data.strategy_summary is not None:
+ st.divider()
+ st.subheader("📝 Strategy summary")
+ table_tab, chart_tab = st.tabs(["Table", "Chart"])
+ with table_tab:
+ selection = show_strategy_summary(strategy_data.strategy_summary)
+ if selection is not None:
+ if len(selection) > 1:
+ st.warning("This version doesn't support multiple selections. Please try selecting only one.")
+ st.stop()
+ selected_exchange = selection["Exchange"].values[0]
+ selected_trading_pair = selection["Trading Pair"].values[0]
+ with chart_tab:
+ summary_chart = summary_chart(strategy_data.strategy_summary)
+ st.plotly_chart(summary_chart, use_container_width=True)
+ if selection is None:
+ st.info("💡 Choose a trading pair and start analyzing!")
else:
- st.session_state.selected_config_file = None
- with col3:
- if st.session_state.selected_config_file:
- st.session_state.selected_exchange = st.selectbox("Exchange:", st.session_state.selected_db.configs[st.session_state.selected_config_file].keys())
- with col4:
- if st.session_state.selected_exchange:
- st.session_state.selected_trading_pair = st.selectbox("Trading Pair:", options=st.session_state.selected_db.configs[st.session_state.selected_config_file][st.session_state.selected_exchange])
-
- single_market = True
- if single_market:
- strategy_data = st.session_state["dbs"][selected_db_name].get_strategy_data(st.session_state.selected_config_file)
- single_market_strategy_data = strategy_data.get_single_market_strategy_data(st.session_state.selected_exchange, st.session_state.selected_trading_pair)
- date_array = pd.date_range(start=strategy_data.start_time, end=strategy_data.end_time, periods=60)
- start_time, end_time = st.select_slider("Select a time range to analyze",
- options=date_array.tolist(),
- value=(date_array[0], date_array[-1]))
- strategy_data_filtered = single_market_strategy_data.get_filtered_strategy_data(start_time, end_time)
-
- st.markdown("
", unsafe_allow_html=True)
- with st.container():
- col1, col2 = st.columns(2)
- with col1:
- st.subheader(f"🏦 Market")
- with col2:
- st.subheader("📋 General stats")
- col1, col2, col3, col4 = st.columns(4)
- with col1:
- st.metric(label="Exchange", value=strategy_data_filtered.exchange.capitalize())
- with col2:
- st.metric(label="Trading pair", value=strategy_data_filtered.trading_pair.upper())
- with col3:
- st.metric(label='Start date', value=strategy_data_filtered.start_time.strftime("%Y-%m-%d %H:%M"))
- st.metric(label='End date', value=strategy_data_filtered.end_time.strftime("%Y-%m-%d %H:%M"))
- with col4:
- st.metric(label='Duration (Hours)', value=round(strategy_data_filtered.duration_seconds / 3600, 2))
- st.metric(label='Price change', value=f"{round(strategy_data_filtered.price_change * 100, 2)} %")
-
- st.markdown("
", unsafe_allow_html=True)
- st.subheader("📈 Performance")
- col131, col132, col133, col134 = st.columns(4)
- with col131:
- st.metric(label=f'Net PNL {strategy_data_filtered.quote_asset}',
- value=round(strategy_data_filtered.net_pnl_quote, 2))
- st.metric(label=f'Trade PNL {strategy_data_filtered.quote_asset}',
- value=round(strategy_data_filtered.trade_pnl_quote, 2))
- st.metric(label=f'Fees {strategy_data_filtered.quote_asset}',
- value=round(strategy_data_filtered.cum_fees_in_quote, 2))
- with col132:
- st.metric(label='Total Trades', value=strategy_data_filtered.total_orders)
- st.metric(label='Total Buy Trades', value=strategy_data_filtered.total_buy_trades)
- st.metric(label='Total Sell Trades', value=strategy_data_filtered.total_sell_trades)
- with col133:
- st.metric(label='Inventory change in Base asset',
- value=round(strategy_data_filtered.inventory_change_base_asset, 4))
- st.metric(label='Total Buy Trades Amount',
- value=round(strategy_data_filtered.total_buy_amount, 2))
- st.metric(label='Total Sell Trades Amount',
- value=round(strategy_data_filtered.total_sell_amount, 2))
- with col134:
- st.metric(label='End Price', value=round(strategy_data_filtered.end_price, 4))
- st.metric(label='Average Buy Price', value=round(strategy_data_filtered.average_buy_price, 4))
- st.metric(label='Average Sell Price', value=round(strategy_data_filtered.average_sell_price, 4))
-
- st.markdown("
", unsafe_allow_html=True)
- st.subheader("🕯️ Candlestick")
- if strategy_data_filtered.market_data is not None:
- with st.expander("Market activity", expanded=True):
- col1, col2, col3 = st.columns([1, 1, 2])
- with col1:
- interval = st.selectbox("Candles Interval:", intervals.keys(), index=2)
- with col2:
- rows_per_page = st.number_input("Candles per Page", value=100, min_value=1, max_value=5000)
- with col3:
- total_rows = len(strategy_data_filtered.get_market_data_resampled(interval=f"{intervals[interval]}S"))
- total_pages = math.ceil(total_rows / rows_per_page)
- if total_pages > 1:
- selected_page = st.select_slider("Select page", list(range(total_pages)), key="page_slider")
+ st.divider()
+ st.subheader("🔍 Explore Trading Pair")
+ if not any("Error" in value for key, value in selected_db.status.items() if key != "position_executor"):
+ date_array = pd.date_range(start=strategy_data.start_time, end=strategy_data.end_time, periods=60)
+ start_time, end_time = st.select_slider("Select a time range to analyze",
+ options=date_array.tolist(),
+ value=(date_array[0], date_array[-1]))
+
+ single_market = True
+ if single_market:
+ single_market_strategy_data = strategy_data.get_single_market_strategy_data(selected_exchange, selected_trading_pair)
+ strategy_data_filtered = single_market_strategy_data.get_filtered_strategy_data(start_time, end_time)
+ with st.container():
+ col1, col2, col3, col4, col5, col6, col7, col8 = st.columns(8)
+ with col1:
+ st.metric(label=f'Net PNL {strategy_data_filtered.quote_asset}',
+ value=round(strategy_data_filtered.net_pnl_quote, 2))
+ with col2:
+ st.metric(label='Total Trades', value=strategy_data_filtered.total_orders)
+ with col3:
+ st.metric(label='Accuracy',
+ value=f"{100 * strategy_data_filtered.accuracy:.2f} %")
+ with col4:
+ st.metric(label="Profit Factor",
+ value=round(strategy_data_filtered.profit_factor, 2))
+ with col5:
+ st.metric(label='Duration (Hours)',
+ value=round(strategy_data_filtered.duration_seconds / 3600, 2))
+ with col6:
+ st.metric(label='Price change',
+ value=f"{round(strategy_data_filtered.price_change * 100, 2)} %")
+ with col7:
+ buy_trades_amount = round(strategy_data_filtered.total_buy_amount, 2)
+ avg_buy_price = round(strategy_data_filtered.average_buy_price, 4)
+ st.metric(label="Total Buy Volume",
+ value=round(buy_trades_amount * avg_buy_price, 2))
+ with col8:
+ sell_trades_amount = round(strategy_data_filtered.total_sell_amount, 2)
+ avg_sell_price = round(strategy_data_filtered.average_sell_price, 4)
+ st.metric(label="Total Sell Volume",
+ value=round(sell_trades_amount * avg_sell_price, 2))
+ st.plotly_chart(pnl_over_time(strategy_data_filtered.trade_fill), use_container_width=True)
+
+ st.subheader("💱 Market activity")
+ if "Error" not in selected_db.status["market_data"] and strategy_data_filtered.market_data is not None:
+ col1, col2, col3, col4 = st.columns(4)
+ with col1:
+ interval = st.selectbox("Candles Interval:", intervals.keys(), index=2)
+ with col2:
+ rows_per_page = st.number_input("Candles per Page", value=1500, min_value=1, max_value=5000)
+ with col3:
+ st.markdown("##")
+ show_panel_metrics = st.checkbox("Show panel metrics", value=True)
+ with col4:
+ total_rows = len(
+ strategy_data_filtered.get_market_data_resampled(interval=f"{intervals[interval]}S"))
+ total_pages = math.ceil(total_rows / rows_per_page)
+ if total_pages > 1:
+ selected_page = st.select_slider("Select page", list(range(total_pages)), total_pages - 1,
+ key="page_slider")
+ else:
+ selected_page = 0
+ start_idx = selected_page * rows_per_page
+ end_idx = start_idx + rows_per_page
+ candles_df = strategy_data_filtered.get_market_data_resampled(
+ interval=f"{intervals[interval]}S").iloc[
+ start_idx:end_idx]
+ start_time_page = candles_df.index.min()
+ end_time_page = candles_df.index.max()
+ page_data_filtered = single_market_strategy_data.get_filtered_strategy_data(start_time_page,
+ end_time_page)
+ if show_panel_metrics:
+ col1, col2 = st.columns([2, 1])
+ with col1:
+ candles_chart = candles_graph(candles_df, page_data_filtered)
+ st.plotly_chart(candles_chart, use_container_width=True)
+ with col2:
+ chart_tab, table_tab = st.tabs(["Chart", "Table"])
+ with chart_tab:
+ st.plotly_chart(intraday_performance(page_data_filtered.trade_fill), use_container_width=True)
+ st.plotly_chart(returns_histogram(page_data_filtered.trade_fill), use_container_width=True)
+ with table_tab:
+ st.dataframe(page_data_filtered.trade_fill[["timestamp", "gross_pnl", "trade_fee", "realized_pnl"]].dropna(subset="realized_pnl"),
+ use_container_width=True,
+ hide_index=True,
+ height=(min(len(page_data_filtered.trade_fill) * 39, candles_chart.layout.height - 180)))
+ else:
+ st.plotly_chart(candles_graph(candles_df, page_data_filtered), use_container_width=True)
else:
- selected_page = 0
- start_idx = selected_page * rows_per_page
- end_idx = start_idx + rows_per_page
- candles_df = strategy_data_filtered.get_market_data_resampled(interval=f"{intervals[interval]}S").iloc[
- start_idx:end_idx]
- start_time_page = candles_df.index.min()
- end_time_page = candles_df.index.max()
- page_data_filtered = single_market_strategy_data.get_filtered_strategy_data(start_time_page, end_time_page)
- cg = CandlesGraph(candles_df, show_volume=False, extra_rows=2)
- cg.add_buy_trades(strategy_data_filtered.buys)
- cg.add_sell_trades(strategy_data_filtered.sells)
- cg.add_pnl(strategy_data_filtered, row=2)
- cg.add_base_inventory_change(strategy_data_filtered, row=3)
- fig = cg.figure()
- st.plotly_chart(fig, use_container_width=True)
- else:
- st.warning("Market data is not available so the candles graph is not going to be rendered. "
- "Make sure that you are using the latest version of Hummingbot and market data recorder activated.")
- st.markdown("
", unsafe_allow_html=True)
- st.subheader("Tables")
- with st.expander("💵 Trades"):
- st.write(strategy_data_filtered.trade_fill)
- download_csv(strategy_data_filtered.trade_fill, "trade_fill", "download-trades")
- with st.expander("📩 Orders"):
- st.write(strategy_data_filtered.orders)
- download_csv(strategy_data_filtered.orders, "orders", "download-orders")
- with st.expander("⌕ Order Status"):
- st.write(strategy_data_filtered.order_status)
- download_csv(strategy_data_filtered.order_status, "order_status", "download-order-status")
+ st.warning("Market data is not available so the candles graph is not going to be rendered. "
+ "Make sure that you are using the latest version of Hummingbot and market data recorder activated.")
+ st.divider()
+ st.subheader("📈 Metrics")
+ with st.container():
+ col1, col2, col3, col4, col5 = st.columns(5)
+ with col1:
+ st.metric(label=f'Trade PNL {strategy_data_filtered.quote_asset}',
+ value=round(strategy_data_filtered.trade_pnl_quote, 2))
+ st.metric(label=f'Fees {strategy_data_filtered.quote_asset}',
+ value=round(strategy_data_filtered.cum_fees_in_quote, 2))
+ with col2:
+ st.metric(label='Total Buy Trades', value=strategy_data_filtered.total_buy_trades)
+ st.metric(label='Total Sell Trades', value=strategy_data_filtered.total_sell_trades)
+ with col3:
+ st.metric(label='Total Buy Trades Amount',
+ value=round(strategy_data_filtered.total_buy_amount, 2))
+ st.metric(label='Total Sell Trades Amount',
+ value=round(strategy_data_filtered.total_sell_amount, 2))
+ with col4:
+ st.metric(label='Average Buy Price', value=round(strategy_data_filtered.average_buy_price, 4))
+ st.metric(label='Average Sell Price', value=round(strategy_data_filtered.average_sell_price, 4))
+ with col5:
+ st.metric(label='Inventory change in Base asset',
+ value=round(strategy_data_filtered.inventory_change_base_asset, 4))
+ st.divider()
+ st.subheader("Tables")
+ with st.expander("💵 Trades"):
+ st.write(strategy_data.trade_fill)
+ download_csv(strategy_data.trade_fill, "trade_fill", "download-trades")
+ with st.expander("📩 Orders"):
+ st.write(strategy_data.orders)
+ download_csv(strategy_data.orders, "orders", "download-orders")
+ with st.expander("⌕ Order Status"):
+ st.write(strategy_data.order_status)
+ download_csv(strategy_data.order_status, "order_status", "download-order-status")
+ else:
+ st.warning("We are encountering challenges in maintaining continuous analysis of this database.")
+ with st.expander("DB Status"):
+ status_df = pd.DataFrame([selected_db.status]).transpose().reset_index()
+ status_df.columns = ["Attribute", "Value"]
+ st.table(status_df)
+ else:
+ st.warning("We were unable to process this SQLite database.")
+ with st.expander("DB Status"):
+ status_df = pd.DataFrame([selected_db.status]).transpose().reset_index()
+ status_df.columns = ["Attribute", "Value"]
+ st.table(status_df)
diff --git a/utils/data_manipulation.py b/utils/data_manipulation.py
index e7c9dd18..80f62d18 100644
--- a/utils/data_manipulation.py
+++ b/utils/data_manipulation.py
@@ -9,6 +9,42 @@ class StrategyData:
order_status: pd.DataFrame
trade_fill: pd.DataFrame
market_data: pd.DataFrame = None
+ position_executor: pd.DataFrame = None
+
+ @property
+ def strategy_summary(self):
+ if self.trade_fill is not None:
+ return self.get_strategy_summary()
+ else:
+ return None
+
+ def get_strategy_summary(self):
+ def full_series(series):
+ return list(series)
+
+ strategy_data = self.trade_fill.copy()
+ strategy_data["volume"] = strategy_data["amount"] * strategy_data["price"]
+ strategy_data["margin_volume"] = strategy_data["amount"] * strategy_data["price"] / strategy_data["leverage"]
+ strategy_summary = strategy_data.groupby(["strategy", "market", "symbol"]).agg({"order_id": "count",
+ "volume": "sum",
+ "margin_volume": "sum",
+ "net_realized_pnl": [full_series,
+ "last"]}).reset_index()
+ strategy_summary.columns = [f"{col[0]}_{col[1]}" if isinstance(col, tuple) and col[1] is not None else col for col in strategy_summary.columns]
+ strategy_summary.rename(columns={"strategy_": "Strategy",
+ "market_": "Exchange",
+ "symbol_": "Trading Pair",
+ "order_id_count": "# Trades",
+ "volume_sum": "Volume",
+ "margin_volume_sum": "Margin volume",
+ "net_realized_pnl_full_series": "PnL Over Time",
+ "net_realized_pnl_last": "Realized PnL"}, inplace=True)
+ strategy_summary.sort_values(["Realized PnL"], ascending=True, inplace=True)
+ strategy_summary["Explore"] = False
+ column_names = list(strategy_summary.columns)
+ column_names.insert(0, column_names.pop())
+ strategy_summary = strategy_summary[column_names]
+ return strategy_summary
def get_single_market_strategy_data(self, exchange: str, trading_pair: str):
orders = self.orders[(self.orders["market"] == exchange) & (self.orders["symbol"] == trading_pair)].copy()
@@ -19,6 +55,11 @@ def get_single_market_strategy_data(self, exchange: str, trading_pair: str):
(self.market_data["trading_pair"] == trading_pair)].copy()
else:
market_data = None
+ if self.position_executor is not None:
+ position_executor = self.position_executor[(self.position_executor["exchange"] == exchange) &
+ (self.position_executor["trading_pair"] == trading_pair)].copy()
+ else:
+ position_executor = None
return SingleMarketStrategyData(
exchange=exchange,
trading_pair=trading_pair,
@@ -26,6 +67,7 @@ def get_single_market_strategy_data(self, exchange: str, trading_pair: str):
order_status=order_status,
trade_fill=trade_fill,
market_data=market_data,
+ position_executor=position_executor
)
@property
@@ -77,6 +119,7 @@ class SingleMarketStrategyData:
order_status: pd.DataFrame
trade_fill: pd.DataFrame
market_data: pd.DataFrame = None
+ position_executor: pd.DataFrame = None
def get_filtered_strategy_data(self, start_date: datetime.datetime, end_date: datetime.datetime):
orders = self.orders[
@@ -88,6 +131,11 @@ def get_filtered_strategy_data(self, start_date: datetime.datetime, end_date: da
(self.market_data.index >= start_date) & (self.market_data.index <= end_date)].copy()
else:
market_data = None
+ if self.position_executor is not None:
+ position_executor = self.position_executor[(self.position_executor.datetime >= start_date) &
+ (self.position_executor.datetime <= end_date)].copy()
+ else:
+ position_executor = None
return SingleMarketStrategyData(
exchange=self.exchange,
trading_pair=self.trading_pair,
@@ -95,6 +143,7 @@ def get_filtered_strategy_data(self, start_date: datetime.datetime, end_date: da
order_status=order_status,
trade_fill=trade_fill,
market_data=market_data,
+ position_executor=position_executor
)
def get_market_data_resampled(self, interval):
@@ -194,3 +243,42 @@ def net_pnl_quote(self):
@property
def inventory_change_base_asset(self):
return self.total_buy_amount - self.total_sell_amount
+
+ @property
+ def accuracy(self):
+ total_wins = len(self.trade_fill["net_realized_pnl"] >= 0)
+ total_losses = len(self.trade_fill["net_realized_pnl"] < 0)
+ return total_wins / (total_wins + total_losses)
+
+ @property
+ def profit_factor(self):
+ total_profit = self.trade_fill.loc[self.trade_fill["realized_pnl"] >= 0, "realized_pnl"].sum()
+ total_loss = self.trade_fill.loc[self.trade_fill["realized_pnl"] < 0, "realized_pnl"].sum()
+ return total_profit / -total_loss
+
+ @property
+ def properties_table(self):
+ properties_dict = {"Base Asset": self.base_asset,
+ "Quote Asset": self.quote_asset,
+ # "Start Time": self.start_time,
+ # "End Time": self.end_time,
+ "Exchange": self.exchange,
+ "Trading pair": self.trading_pair,
+ "Duration (seconds)": self.duration_seconds,
+ "Start Price": self.start_price,
+ "End Price": self.end_price,
+ "Total Buy Amount": self.total_buy_amount,
+ "Total Sell Amount": self.total_sell_amount,
+ "Total Buy Trades": self.total_buy_trades,
+ "Total Sell Trades": self.total_sell_trades,
+ "Total Orders": self.total_orders,
+ "Average Buy Price": self.average_buy_price,
+ "Average Sell Price": self.average_sell_price,
+ "Price Change": self.price_change,
+ "Trade PnL Quote": self.trade_pnl_quote,
+ "Cum Fees in Quote": self.cum_fees_in_quote,
+ "Net PnL Quote": self.net_pnl_quote,
+ "Inventory Change (base asset)": self.inventory_change_base_asset}
+ properties_table = pd.DataFrame([properties_dict]).transpose().reset_index()
+ properties_table.columns = ["Metric", "Value"]
+ return properties_table
diff --git a/utils/database_manager.py b/utils/database_manager.py
index ce9f6e91..18f003fd 100644
--- a/utils/database_manager.py
+++ b/utils/database_manager.py
@@ -9,26 +9,49 @@
class DatabaseManager:
- def __init__(self, db_name):
+ def __init__(self, db_name: str, executors_path: str = "data"):
self.db_name = db_name
# TODO: Create db path for all types of db
self.db_path = f'sqlite:///{os.path.join("data", db_name)}'
+ self.executors_path = executors_path
self.engine = create_engine(self.db_path, connect_args={'check_same_thread': False})
self.session_maker = sessionmaker(bind=self.engine)
- @property
- def status(self):
+ def get_strategy_data(self, config_file_path=None, start_date=None, end_date=None):
+ def load_data(table_loader):
+ try:
+ return table_loader()
+ except Exception as e:
+ return None # Return None to indicate failure
+
+ # Use load_data to load tables
+ orders = load_data(self.get_orders)
+ trade_fills = load_data(self.get_trade_fills)
+ order_status = load_data(self.get_order_status)
+ market_data = load_data(self.get_market_data)
+ position_executor = load_data(self.get_position_executor_data)
+
+ strategy_data = StrategyData(orders, order_status, trade_fills, market_data, position_executor)
+ return strategy_data
+
+ @staticmethod
+ def _get_table_status(table_loader):
try:
- with self.session_maker() as session:
- query = 'SELECT DISTINCT config_file_path FROM TradeFill'
- config_files = pd.read_sql_query(query, session.connection())
- if len(config_files) > 0:
- # TODO: improve error handling, think what to do with other cases
- return "OK"
- else:
- return "No records found in the TradeFill table with non-null config_file_path"
+ data = table_loader()
+ return "Correct" if len(data) > 0 else f"Error - No records matched"
except Exception as e:
- return f"Error: {str(e)}"
+ return f"Error - {str(e)}"
+
+ @property
+ def status(self):
+ status = {"db_name": self.db_name,
+ "trade_fill": self._get_table_status(self.get_trade_fills),
+ "orders": self._get_table_status(self.get_orders),
+ "order_status": self._get_table_status(self.get_order_status),
+ "market_data": self._get_table_status(self.get_market_data),
+ "position_executor": self._get_table_status(self.get_position_executor_data),
+ }
+ return status
@property
def config_files(self):
@@ -120,24 +143,26 @@ def get_orders(self, config_file_path=None, start_date=None, end_date=None):
return orders
def get_trade_fills(self, config_file_path=None, start_date=None, end_date=None):
+ groupers = ["config_file_path", "market", "symbol"]
+ float_cols = ["amount", "price", "trade_fee_in_quote"]
with self.session_maker() as session:
query = self._get_trade_fills_query(config_file_path, start_date, end_date)
trade_fills = pd.read_sql_query(query, session.connection())
- trade_fills["amount"] = trade_fills["amount"] / 1e6
- trade_fills["price"] = trade_fills["price"] / 1e6
- trade_fills["trade_fee_in_quote"] = trade_fills["trade_fee_in_quote"] / 1e6
- trade_fills["cum_fees_in_quote"] = trade_fills["trade_fee_in_quote"].cumsum()
- trade_fills.loc[:, "net_amount"] = trade_fills['amount'] * trade_fills['trade_type'].apply(
- lambda x: 1 if x == 'BUY' else -1)
- trade_fills.loc[:, "net_amount_quote"] = trade_fills['net_amount'] * trade_fills['price']
- trade_fills.loc[:, "cum_net_amount"] = trade_fills["net_amount"].cumsum()
- trade_fills.loc[:, "unrealized_trade_pnl"] = -1 * trade_fills["net_amount_quote"].cumsum()
- trade_fills.loc[:, "inventory_cost"] = trade_fills["cum_net_amount"] * trade_fills["price"]
- trade_fills.loc[:, "realized_trade_pnl"] = trade_fills["unrealized_trade_pnl"] + trade_fills["inventory_cost"]
- trade_fills.loc[:, "net_realized_pnl"] = trade_fills["realized_trade_pnl"] - trade_fills["cum_fees_in_quote"]
+ trade_fills[float_cols] = trade_fills[float_cols] / 1e6
+ trade_fills["cum_fees_in_quote"] = trade_fills.groupby(groupers)["trade_fee_in_quote"].cumsum()
+ trade_fills["net_amount"] = trade_fills['amount'] * trade_fills['trade_type'].apply(lambda x: 1 if x == 'BUY' else -1)
+ trade_fills["net_amount_quote"] = trade_fills['net_amount'] * trade_fills['price']
+ trade_fills["cum_net_amount"] = trade_fills.groupby(groupers)["net_amount"].cumsum()
+ trade_fills["unrealized_trade_pnl"] = -1 * trade_fills.groupby(groupers)["net_amount_quote"].cumsum()
+ trade_fills["inventory_cost"] = trade_fills["cum_net_amount"] * trade_fills["price"]
+ trade_fills["realized_trade_pnl"] = trade_fills["unrealized_trade_pnl"] + trade_fills["inventory_cost"]
+ trade_fills["net_realized_pnl"] = trade_fills["realized_trade_pnl"] - trade_fills["cum_fees_in_quote"]
+ trade_fills["realized_pnl"] = trade_fills["net_realized_pnl"].diff()
+ trade_fills["gross_pnl"] = trade_fills["realized_trade_pnl"].diff()
+ trade_fills["trade_fee"] = trade_fills["cum_fees_in_quote"].diff()
trade_fills["timestamp"] = pd.to_datetime(trade_fills["timestamp"], unit="ms")
trade_fills["market"] = trade_fills["market"].apply(lambda x: x.lower().replace("_papertrade", ""))
-
+ trade_fills["quote_volume"] = trade_fills["price"] * trade_fills["amount"]
return trade_fills
def get_order_status(self, order_ids=None, start_date=None, end_date=None):
@@ -157,13 +182,17 @@ def get_market_data(self, start_date=None, end_date=None):
market_data["best_ask"] = market_data["best_ask"] / 1e6
return market_data
- def get_strategy_data(self, config_file_path=None, start_date=None, end_date=None):
- orders = self.get_orders(config_file_path, start_date, end_date)
- trade_fills = self.get_trade_fills(config_file_path, start_date, end_date)
- order_status = self.get_order_status(orders['id'].tolist(), start_date, end_date)
- try:
- market_data = self.get_market_data(start_date, end_date)
- except Exception as e:
- market_data = None
- strategy_data = StrategyData(orders, order_status, trade_fills, market_data)
- return strategy_data
+ def get_position_executor_data(self, start_date=None, end_date=None) -> pd.DataFrame:
+ df = pd.DataFrame()
+ files = [file for file in os.listdir(self.executors_path) if ".csv" in file and file != "trades_market_making_.csv"]
+ for file in files:
+ df0 = pd.read_csv(f"{self.executors_path}/{file}")
+ df = pd.concat([df, df0])
+ df["datetime"] = pd.to_datetime(df["timestamp"], unit="s")
+ if start_date:
+ df = df[df["datetime"] >= start_date]
+ if end_date:
+ df = df[df["datetime"] <= end_date]
+ return df
+
+
diff --git a/utils/graphs.py b/utils/graphs.py
index 25a6f485..c4b9b2ba 100644
--- a/utils/graphs.py
+++ b/utils/graphs.py
@@ -7,6 +7,9 @@
from quants_lab.strategy.strategy_analysis import StrategyAnalysis
import plotly.graph_objs as go
+BULLISH_COLOR = "rgba(97, 199, 102, 0.9)"
+BEARISH_COLOR = "rgba(255, 102, 90, 0.9)"
+FEE_COLOR = "rgba(51, 0, 51, 0.9)"
class CandlesGraph:
def __init__(self, candles_df: pd.DataFrame, show_volume=True, extra_rows=1):
@@ -143,97 +146,83 @@ def add_ema(self, length=20, row=1):
row=row, col=1,
)
- def add_base_inventory_change(self, strategy_data: StrategyData, row=3):
- # Create a list of colors based on the sign of the amount_new column
- self.base_figure.add_trace(
- go.Bar(
- x=strategy_data.trade_fill["timestamp"],
- y=strategy_data.trade_fill["net_amount"],
- name="Base Inventory Change",
- opacity=0.5,
- marker=dict(color=["lightgreen" if amount > 0 else "indianred" for amount in
- strategy_data.trade_fill["net_amount"]])
- ),
- row=row, col=1,
- )
- # TODO: Review impact in different subgraphs
- merged_df = self.get_merged_df(strategy_data)
+ def add_quote_inventory_change(self, strategy_data: StrategyData, row=3):
self.base_figure.add_trace(
go.Scatter(
- x=merged_df.index,
- y=merged_df["cum_net_amount"],
- name="Cumulative Base Inventory Change",
- mode="lines+markers",
- marker=dict(color="black", size=6),
- line=dict(color="royalblue", width=2),
- # text=merged_df["cum_net_amount"],
- # textposition="top center",
- # texttemplate="%{text:.2f}"
+ x=strategy_data.trade_fill.timestamp,
+ y=strategy_data.trade_fill.inventory_cost,
+ name="Quote Inventory",
+ mode="lines",
+ line=dict(shape="hv"),
),
row=row, col=1
)
- self.base_figure.update_yaxes(title_text='Base Inventory Change', row=row, col=1)
+ self.base_figure.update_yaxes(title_text='Quote Inventory Change', row=row, col=1)
def add_pnl(self, strategy_data: SingleMarketStrategyData, row=4):
- merged_df = self.get_merged_df(strategy_data)
self.base_figure.add_trace(
go.Scatter(
- x=merged_df.index,
- y=merged_df["cum_fees_in_quote"].apply(lambda x: round(-x, 2)),
- name="Cum Fees",
+ x=strategy_data.trade_fill.timestamp,
+ y=[max(0, realized_pnl) for realized_pnl in strategy_data.trade_fill["realized_trade_pnl"].apply(lambda x: round(x, 4))],
+ name="Cum Profit",
mode='lines',
- line_color='teal',
+ line=dict(shape="hv", color="rgba(1, 1, 1, 0.5)", dash="dash", width=0.1),
fill="tozeroy", # Fill to the line below (trade pnl)
- stackgroup='one'
+ fillcolor="rgba(0, 255, 0, 0.5)"
),
row=row, col=1
)
-
self.base_figure.add_trace(
go.Scatter(
- x=merged_df.index,
- y=merged_df["trade_pnl_continuos"].apply(lambda x: round(x, 2)),
- name="Cum Trade PnL",
+ x=strategy_data.trade_fill.timestamp,
+ y=[min(0, realized_pnl) for realized_pnl in strategy_data.trade_fill["realized_trade_pnl"].apply(lambda x: round(x, 4))],
+ name="Cum Loss",
mode='lines',
- line_color='pink',
- fill="tonexty", # Fill to the line below (net pnl)
- stackgroup='one'
+ line=dict(shape="hv", color="rgba(1, 1, 1, 0.5)", dash="dash", width=0.3),
+ # marker=dict(symbol="arrow"),
+ fill="tozeroy", # Fill to the line below (trade pnl)
+ fillcolor="rgba(255, 0, 0, 0.5)",
),
row=row, col=1
)
self.base_figure.add_trace(
go.Scatter(
- x=merged_df.index,
- y=merged_df["net_pnl_continuos"].apply(lambda x: round(x, 2)),
- name="Cum Net PnL",
- mode="lines+markers",
- marker=dict(color="black", size=6),
- line=dict(color="black", width=2),
- # textposition="top center",
- # text=merged_df["net_pnl_continuos"],
- # texttemplate="%{text:.1f}"
+ x=strategy_data.trade_fill.timestamp,
+ y=strategy_data.trade_fill["cum_fees_in_quote"].apply(lambda x: round(x, 4)),
+ name="Cum Fees",
+ mode='lines',
+ line=dict(shape="hv", color="rgba(1, 1, 1, 0.1)", dash="dash", width=0.1),
+ fill="tozeroy", # Fill to the line below (trade pnl)
+ fillcolor="rgba(51, 0, 51, 0.5)"
),
row=row, col=1
)
+ self.base_figure.add_trace(go.Scatter(name="Net Realized Profit",
+ x=strategy_data.trade_fill.timestamp,
+ y=strategy_data.trade_fill["net_realized_pnl"],
+ mode="lines",
+ line=dict(shape="hv")),
+ row=row, col=1
+ )
self.base_figure.update_yaxes(title_text='PNL', row=row, col=1)
def update_layout(self):
self.base_figure.update_layout(
title={
'text': "Market activity",
- 'y': 0.95,
+ 'y': 0.99,
'x': 0.5,
'xanchor': 'center',
'yanchor': 'top'
},
legend=dict(
orientation="h",
- yanchor="bottom",
- y=-0.2,
- xanchor="right",
- x=1
+ x=0.5,
+ y=1.04,
+ xanchor="center",
+ yanchor="bottom"
),
- height=1500,
+ height=1000,
xaxis=dict(rangeslider_visible=False,
range=[self.min_time, self.max_time]),
yaxis=dict(range=[self.candles_df.low.min(), self.candles_df.high.max()]),
@@ -244,12 +233,6 @@ def update_layout(self):
self.base_figure.update_yaxes(title_text="Volume", row=2, col=1)
self.base_figure.update_xaxes(title_text="Time", row=self.rows, col=1)
- def get_merged_df(self, strategy_data: StrategyData):
- merged_df = pd.merge_asof(self.candles_df, strategy_data.trade_fill, left_index=True, right_on="timestamp", direction="backward")
- merged_df["trade_pnl_continuos"] = merged_df["unrealized_trade_pnl"] + merged_df["cum_net_amount"] * merged_df["close"]
- merged_df["net_pnl_continuos"] = merged_df["trade_pnl_continuos"] - merged_df["cum_fees_in_quote"]
- return merged_df
-
class BacktestingGraphs:
def __init__(self, study_df: pd.DataFrame):