From 510a174408222079c19958bf46f75e8c2927d343 Mon Sep 17 00:00:00 2001 From: enarjord Date: Thu, 25 May 2023 17:28:25 +0200 Subject: [PATCH 01/20] add exposure ratios mean to score func --- pure_funcs.py | 68 +-------------------------------------------------- 1 file changed, 1 insertion(+), 67 deletions(-) diff --git a/pure_funcs.py b/pure_funcs.py index 1c9cd8e2e..58f4f9a5b 100644 --- a/pure_funcs.py +++ b/pure_funcs.py @@ -1399,6 +1399,7 @@ def calc_scores(config: dict, results: dict): # [(key_name, higher_is_better)] keys = [ ("adg_weighted_per_exposure", True), + ("exposure_ratios_mean", False), ("hrs_stuck_max", False), ("pa_distance_mean", False), ("pa_distance_std", False), @@ -1463,73 +1464,6 @@ def calc_scores(config: dict, results: dict): } -def calc_scores_old(config: dict, results: dict): - sides = ["long", "short"] - keys = [ - ("adg_realized_per_exposure", True), - ("pa_distance_std", False), - ("pa_distance_mean", False), - ("hrs_stuck_max", False), - ("loss_profit_ratio", False), - ("eqbal_ratio_min", True), - ] - means = {side: {} for side in sides} # adjusted means - scores = {side: -1.0 for side in sides} - raws = {side: {} for side in sides} # unadjusted means - individual_raws = {side: {sym: {} for sym in results} for side in sides} - individual_vals = {side: {sym: {} for sym in results} for side in sides} - individual_scores = {side: {sym: -1.0 for sym in results} for side in sides} - symbols_to_include = {side: [] for side in sides} - for side in sides: - for sym in results: - for key, mult in keys: - key_side = f"{key}_{side}" - if key_side not in results[sym]: - results[sym][key_side] = results[sym][key] - individual_raws[side][sym][key] = results[sym][key_side] - if (max_key := f"maximum_{key}_{side}") in config: - if config[max_key] >= 0.0: - val = max(config[max_key], results[sym][key_side]) - else: - val = 1.0 - elif (min_key := f"minimum_{key}_{side}") in config: - if config[min_key] >= 0.0: - val = min(config[min_key], results[sym][key_side]) - else: - val = 1.0 - else: - val = results[sym][key_side] - individual_vals[side][sym][key] = val - if mult: - individual_scores[side][sym] *= val - else: - individual_scores[side][sym] /= val - raws[side] = { - key: np.mean([individual_raws[side][sym][key] for sym in results]) for key, _ in keys - } - symbols_to_include[side] = sorted( - individual_scores[side], key=lambda x: individual_scores[side][x] - )[: max(1, int(len(individual_scores[side]) * (1 - config["clip_threshold"])))] - # print(symbols_to_include, individual_scores[side], config["clip_threshold"]) - means[side] = { - key: np.mean([individual_vals[side][sym][key] for sym in symbols_to_include[side]]) - for key, _ in keys - } - for key, mult in keys: - if mult: - scores[side] *= means[side][key] - else: - scores[side] /= means[side][key] - return { - "scores": scores, - "means": means, - "raws": raws, - "individual_scores": individual_scores, - "keys": keys, - "symbols_to_include": symbols_to_include, - } - - def configs_are_equal(cfg0, cfg1) -> bool: try: cfg0 = candidate_to_live_config(cfg0) From 41e376bac8d3716f132f0a58c27ca33ff002b98b Mon Sep 17 00:00:00 2001 From: enarjord Date: Thu, 25 May 2023 17:30:23 +0200 Subject: [PATCH 02/20] add maximum_exposure_ratios_mean --- configs/optimize/default.hjson | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/configs/optimize/default.hjson b/configs/optimize/default.hjson index aac1b0347..4b1cfeef0 100644 --- a/configs/optimize/default.hjson +++ b/configs/optimize/default.hjson @@ -48,6 +48,10 @@ maximum_eqbal_ratio_std_long: 0.025 maximum_eqbal_ratio_std_short: 0.025 + # score -= max(exposure_ratios_mean, eqbal_ratio_std) + maximum_exposure_ratios_mean_long: 0.1 + maximum_exposure_ratios_mean_short: 0.1 + # clip results: compute score on top performers only # clip_threshold=0.1 means drop 10% worst performers; clip_threshold=0.0 means include all clip_threshold: 0.5 @@ -74,17 +78,15 @@ "DEFIUSDT", "YFIUSDT", "BALUSDT", "CRVUSDT", "TRBUSDT", "RUNEUSDT", "SUSHIUSDT", "EGLDUSDT", "SOLUSDT", "ICXUSDT", "STORJUSDT", "BLZUSDT", "UNIUSDT", "AVAXUSDT", "FTMUSDT", - "HNTUSDT", "ENJUSDT", "FLMUSDT", "TOMOUSDT", "RENUSDT", - "KSMUSDT", "NEARUSDT", "AAVEUSDT", "FILUSDT", "RSRUSDT", - "LRCUSDT", "MATICUSDT", "OCEANUSDT", "BELUSDT", "CTKUSDT", - "AXSUSDT", "ALPHAUSDT", "ZENUSDT", "SKLUSDT", "GRTUSDT", - "1INCHUSDT", "CHZUSDT", "SANDUSDT", "ANKRUSDT", "LITUSDT", - "UNFIUSDT", "REEFUSDT", "RVNUSDT", "SFPUSDT", "XEMUSDT", - "COTIUSDT", "CHRUSDT", "MANAUSDT", "ALICEUSDT", "HBARUSDT", - "ONEUSDT", "LINAUSDT", "STMXUSDT", "DENTUSDT", "CELRUSDT", - "HOTUSDT", "MTLUSDT", "OGNUSDT", "NKNUSDT", "DGBUSDT", - - + "ENJUSDT", "FLMUSDT", "TOMOUSDT", "RENUSDT","KSMUSDT", + "NEARUSDT", "AAVEUSDT", "FILUSDT", "RSRUSDT","LRCUSDT", + "MATICUSDT", "OCEANUSDT", "BELUSDT", "CTKUSDT","AXSUSDT", + "ALPHAUSDT", "ZENUSDT", "SKLUSDT", "GRTUSDT","1INCHUSDT", + "CHZUSDT", "SANDUSDT", "ANKRUSDT", "LITUSDT","UNFIUSDT", + "REEFUSDT", "RVNUSDT", "SFPUSDT", "XEMUSDT","COTIUSDT", + "CHRUSDT", "MANAUSDT", "ALICEUSDT", "HBARUSDT","ONEUSDT", + "LINAUSDT", "STMXUSDT", "DENTUSDT", "CELRUSDT","HOTUSDT", + "MTLUSDT", "OGNUSDT", "NKNUSDT", "DGBUSDT", ] bounds_static_grid: From 2661c0fb50f41c58201a9ac6471a125280d5d536 Mon Sep 17 00:00:00 2001 From: enarjord Date: Thu, 25 May 2023 17:30:36 +0200 Subject: [PATCH 03/20] add maximum_exposure_ratios_mean --- inspect_opt_results.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/inspect_opt_results.py b/inspect_opt_results.py index 1b1139ae7..0b199dbfd 100755 --- a/inspect_opt_results.py +++ b/inspect_opt_results.py @@ -51,6 +51,8 @@ def main(): ("ers", "minimum_eqbal_ratio_mean_of_10_worst_short"), ("esl", "maximum_eqbal_ratio_std_long"), ("ess", "maximum_eqbal_ratio_std_short"), + ("exl", "maximum_exposure_ratios_mean_long"), + ("exs", "maximum_exposure_ratios_mean_short"), ("ct", "clip_threshold"), ] for k0, k1 in weights_keys: From 13a7fc7827178d97b6d23b720a3e5802033ba524 Mon Sep 17 00:00:00 2001 From: enarjord Date: Mon, 29 May 2023 14:13:04 +0200 Subject: [PATCH 04/20] further work on bybit ohlcv data --- downloader.py | 37 ++++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/downloader.py b/downloader.py index ad1c8e729..6ed0083c3 100644 --- a/downloader.py +++ b/downloader.py @@ -12,6 +12,7 @@ from urllib.request import urlopen from zipfile import ZipFile import traceback +import aiohttp import numpy as np import pandas as pd @@ -981,11 +982,41 @@ def get_first_ohlcv_ts(symbol: str, spot=False) -> int: return 0 -def get_csv_gz(url: str): +def findall(string, pattern): + """Yields all the positions of + the pattern in the string""" + i = string.find(pattern) + while i != -1: + yield i + i = string.find(pattern, i + 1) + + +async def get_bybit_monthly(base_url: str, symbol: str, month: str): + # month e.g. "2022-03" + content = urlopen(f"{base_url}{symbol}/").read().decode() + filenames = [ + content[i : i + content[i:].find("csv.gz") + 6] for i in findall(content, symbol + month) + ] + async with aiohttp.ClientSession() as session: + tasks = [] + for url in [f"{base_url}{symbol}/{filename}" for filename in filenames]: + task = asyncio.ensure_future(get_csv_gz(session, url)) + tasks.append(task) + responses = await asyncio.gather(*tasks) + return convert_to_ohlcv(pd.concat(responses).sort_values("timestamp")) + + +async def fetch_url(session, url): + async with session.get(url) as response: + content = await response.read() + return content + + +async def get_csv_gz(session, url: str): # from bybit try: - resp = urlopen(url) - with gzip.open(BytesIO(resp.read())) as f: + resp = await fetch_url(session, url) + with gzip.open(BytesIO(resp)) as f: tdf = pd.read_csv(f) return tdf except Exception as e: From 26c0d8d2572a0ed565b7983c83165e05ead98da7 Mon Sep 17 00:00:00 2001 From: enarjord Date: Mon, 29 May 2023 20:45:50 +0200 Subject: [PATCH 05/20] work on download_ohlcvs_bybit() --- downloader.py | 87 ++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 76 insertions(+), 11 deletions(-) diff --git a/downloader.py b/downloader.py index 6ed0083c3..ba1954016 100644 --- a/downloader.py +++ b/downloader.py @@ -991,19 +991,84 @@ def findall(string, pattern): i = string.find(pattern, i + 1) -async def get_bybit_monthly(base_url: str, symbol: str, month: str): - # month e.g. "2022-03" - content = urlopen(f"{base_url}{symbol}/").read().decode() - filenames = [ - content[i : i + content[i:].find("csv.gz") + 6] for i in findall(content, symbol + month) - ] +def get_days_in_between(start_day, end_day): + date_format = "%Y-%m-%d" + start_date = datetime.datetime.strptime(start_day, date_format) + end_date = datetime.datetime.strptime(end_day, date_format) + + days_in_between = [] + current_date = start_date + while current_date <= end_date: + days_in_between.append(current_date.strftime(date_format)) + current_date += datetime.timedelta(days=1) + + return days_in_between + + +def format_date(date_string): + try: + date_formats = ["%Y", "%Y-%m", "%Y-%m-%d", "%Y-%m-%dT%H:%M:%S"] + for format in date_formats: + try: + date_obj = datetime.datetime.strptime(date_string, format) + formatted_date = date_obj.strftime("%Y-%m-%d") + return formatted_date + except ValueError: + pass + raise ValueError("Invalid date format") + except Exception as e: + print("Error:", e) + return None + + +async def download_ohlcvs_bybit(symbol, start_date, end_date, download_only=False): + start_date, end_date = format_date(start_date), format_date(end_date) + assert date_to_ts(end_date) >= date_to_ts(start_date), "end_date is older than start_date" + dirpath = make_get_filepath(f"historical_data/ohlcvs_bybit/{symbol}/") + days_done = [filename[:-4] for filename in os.listdir(dirpath) if ".csv" in filename] + base_url = "https://public.bybit.com/trading/" + webpage_content = await get_bybit_webpage_content(base_url, symbol) + days = [day for day in get_days_in_between(start_date, end_date) if day not in days_done] + filenames = [cand for day in days if (cand := f"{symbol}{day}.csv.gz") in webpage_content] + dfs = {} + if len(filenames) > 0: + n_concurrent_fetches = 10 + for i in range(0, len(filenames), 10): + filenames_sublist = filenames[i : i + n_concurrent_fetches] + print( + f"fetching trades from {filenames_sublist[0][-17:-7]} to {filenames_sublist[-1][-17:-7]}" + ) + dfs_ = await get_bybit_trades(base_url, symbol, filenames_sublist) + dfs_ = {k[-17:-7]: convert_to_ohlcv(v) for k, v in dfs_.items()} + dfs.update(dfs_) + for day in days_done: + dfs[day] = pd.read_csv(f"{dirpath}{day}.csv").set_index("timestamp") + for day, df in dfs.items(): + if day in days_done: + continue + filepath = f"{dirpath}{day}.csv" + df.to_csv(filepath) + print("dumping", filepath) + if not download_only: + df = pd.concat(dfs.values()).sort_values("timestamp") + return df + + +async def get_bybit_webpage_content(base_url: str, symbol: str): + return urlopen(f"{base_url}{symbol}/").read().decode() + + +async def get_bybit_trades(base_url: str, symbol: str, filenames: [str]): + if len(filenames) == 0: + return None async with aiohttp.ClientSession() as session: - tasks = [] + tasks = {} for url in [f"{base_url}{symbol}/{filename}" for filename in filenames]: - task = asyncio.ensure_future(get_csv_gz(session, url)) - tasks.append(task) - responses = await asyncio.gather(*tasks) - return convert_to_ohlcv(pd.concat(responses).sort_values("timestamp")) + tasks[url] = asyncio.ensure_future(get_csv_gz(session, url)) + responses = {} + for url in tasks: + responses[url] = await tasks[url] + return {k: v.sort_values("timestamp") for k, v in responses.items()} async def fetch_url(session, url): From 9c12831076ec56ada7b3069bcc7f93e869c9cebe Mon Sep 17 00:00:00 2001 From: enarjord Date: Mon, 29 May 2023 20:55:11 +0200 Subject: [PATCH 06/20] return correct days --- downloader.py | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/downloader.py b/downloader.py index ba1954016..4fa5d45ad 100644 --- a/downloader.py +++ b/downloader.py @@ -1025,24 +1025,27 @@ async def download_ohlcvs_bybit(symbol, start_date, end_date, download_only=Fals start_date, end_date = format_date(start_date), format_date(end_date) assert date_to_ts(end_date) >= date_to_ts(start_date), "end_date is older than start_date" dirpath = make_get_filepath(f"historical_data/ohlcvs_bybit/{symbol}/") + ideal_days = get_days_in_between(start_date, end_date) days_done = [filename[:-4] for filename in os.listdir(dirpath) if ".csv" in filename] - base_url = "https://public.bybit.com/trading/" - webpage_content = await get_bybit_webpage_content(base_url, symbol) - days = [day for day in get_days_in_between(start_date, end_date) if day not in days_done] - filenames = [cand for day in days if (cand := f"{symbol}{day}.csv.gz") in webpage_content] + days_to_get = [day for day in ideal_days if day not in days_done] dfs = {} - if len(filenames) > 0: - n_concurrent_fetches = 10 - for i in range(0, len(filenames), 10): - filenames_sublist = filenames[i : i + n_concurrent_fetches] - print( - f"fetching trades from {filenames_sublist[0][-17:-7]} to {filenames_sublist[-1][-17:-7]}" - ) - dfs_ = await get_bybit_trades(base_url, symbol, filenames_sublist) - dfs_ = {k[-17:-7]: convert_to_ohlcv(v) for k, v in dfs_.items()} - dfs.update(dfs_) - for day in days_done: - dfs[day] = pd.read_csv(f"{dirpath}{day}.csv").set_index("timestamp") + if len(days_to_get) > 0: + base_url = "https://public.bybit.com/trading/" + webpage = await get_bybit_webpage(base_url, symbol) + filenames = [cand for day in days_to_get if (cand := f"{symbol}{day}.csv.gz") in webpage] + if len(filenames) > 0: + n_concurrent_fetches = 10 + for i in range(0, len(filenames), 10): + filenames_sublist = filenames[i : i + n_concurrent_fetches] + print( + f"fetching trades from {filenames_sublist[0][-17:-7]} to {filenames_sublist[-1][-17:-7]}" + ) + dfs_ = await get_bybit_trades(base_url, symbol, filenames_sublist) + dfs_ = {k[-17:-7]: convert_to_ohlcv(v) for k, v in dfs_.items()} + dfs.update(dfs_) + for day in ideal_days: + if day not in days_to_get: + dfs[day] = pd.read_csv(f"{dirpath}{day}.csv").set_index("timestamp") for day, df in dfs.items(): if day in days_done: continue @@ -1054,7 +1057,7 @@ async def download_ohlcvs_bybit(symbol, start_date, end_date, download_only=Fals return df -async def get_bybit_webpage_content(base_url: str, symbol: str): +async def get_bybit_webpage(base_url: str, symbol: str): return urlopen(f"{base_url}{symbol}/").read().decode() From 51ae1ccd8ae4eed83adb8957660635cae3905611 Mon Sep 17 00:00:00 2001 From: enarjord Date: Mon, 29 May 2023 21:04:33 +0200 Subject: [PATCH 07/20] dump csvs immediately --- downloader.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/downloader.py b/downloader.py index 4fa5d45ad..6905f8942 100644 --- a/downloader.py +++ b/downloader.py @@ -1042,17 +1042,21 @@ async def download_ohlcvs_bybit(symbol, start_date, end_date, download_only=Fals ) dfs_ = await get_bybit_trades(base_url, symbol, filenames_sublist) dfs_ = {k[-17:-7]: convert_to_ohlcv(v) for k, v in dfs_.items()} + dumped = [] + for day, df in sorted(dfs_.items()): + if day in days_done: + continue + filepath = f"{dirpath}{day}.csv" + df.to_csv(filepath) + dumped.append(day) + print("dumped", dirpath, ", ".join(dumped)) dfs.update(dfs_) for day in ideal_days: if day not in days_to_get: dfs[day] = pd.read_csv(f"{dirpath}{day}.csv").set_index("timestamp") - for day, df in dfs.items(): - if day in days_done: - continue - filepath = f"{dirpath}{day}.csv" - df.to_csv(filepath) - print("dumping", filepath) if not download_only: + if len(dfs) == 0: + return pd.DataFrame(columns=["open", "high", "low", "close", "volume"]) df = pd.concat(dfs.values()).sort_values("timestamp") return df From 7bebd6fbef4e2dbc859d1121a0c187def578811f Mon Sep 17 00:00:00 2001 From: enarjord Date: Tue, 30 May 2023 11:48:53 +0200 Subject: [PATCH 08/20] don't load cache if not returning df --- downloader.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/downloader.py b/downloader.py index 6905f8942..ac279c962 100644 --- a/downloader.py +++ b/downloader.py @@ -1050,11 +1050,12 @@ async def download_ohlcvs_bybit(symbol, start_date, end_date, download_only=Fals df.to_csv(filepath) dumped.append(day) print("dumped", dirpath, ", ".join(dumped)) - dfs.update(dfs_) - for day in ideal_days: - if day not in days_to_get: - dfs[day] = pd.read_csv(f"{dirpath}{day}.csv").set_index("timestamp") + if not download_only: + dfs.update(dfs_) if not download_only: + for day in ideal_days: + if day not in days_to_get: + dfs[day] = pd.read_csv(f"{dirpath}{day}.csv").set_index("timestamp") if len(dfs) == 0: return pd.DataFrame(columns=["open", "high", "low", "close", "volume"]) df = pd.concat(dfs.values()).sort_values("timestamp") From 6d401be3194d736dda9536e5605c46c031a1ccb4 Mon Sep 17 00:00:00 2001 From: enarjord Date: Tue, 30 May 2023 11:57:18 +0200 Subject: [PATCH 09/20] remove print --- downloader.py | 1 - 1 file changed, 1 deletion(-) diff --git a/downloader.py b/downloader.py index ac279c962..e81428fd8 100644 --- a/downloader.py +++ b/downloader.py @@ -1049,7 +1049,6 @@ async def download_ohlcvs_bybit(symbol, start_date, end_date, download_only=Fals filepath = f"{dirpath}{day}.csv" df.to_csv(filepath) dumped.append(day) - print("dumped", dirpath, ", ".join(dumped)) if not download_only: dfs.update(dfs_) if not download_only: From cc2c09d7b8d9cae254d7554253b8921899de7981 Mon Sep 17 00:00:00 2001 From: enarjord Date: Tue, 30 May 2023 12:14:05 +0200 Subject: [PATCH 10/20] proper dataframe index --- downloader.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/downloader.py b/downloader.py index e81428fd8..945f7aa28 100644 --- a/downloader.py +++ b/downloader.py @@ -1054,11 +1054,11 @@ async def download_ohlcvs_bybit(symbol, start_date, end_date, download_only=Fals if not download_only: for day in ideal_days: if day not in days_to_get: - dfs[day] = pd.read_csv(f"{dirpath}{day}.csv").set_index("timestamp") + dfs[day] = pd.read_csv(f"{dirpath}{day}.csv") if len(dfs) == 0: - return pd.DataFrame(columns=["open", "high", "low", "close", "volume"]) - df = pd.concat(dfs.values()).sort_values("timestamp") - return df + return pd.DataFrame(columns=["timestamp", "open", "high", "low", "close", "volume"]) + df = pd.concat(dfs.values()).sort_values("timestamp").reset_index() + return df[["timestamp", "open", "high", "low", "close", "volume"]] async def get_bybit_webpage(base_url: str, symbol: str): @@ -1203,7 +1203,7 @@ def count_longest_identical_data(hlc, symbol): return longest_consecutive -def load_hlc_cache( +async def load_hlc_cache( symbol, inverse, start_date, end_date, base_dir="backtests", spot=False, exchange="binance" ): cache_fname = ( @@ -1217,7 +1217,10 @@ def load_hlc_cache( if os.path.exists(filepath): data = np.load(filepath) else: - df = download_ohlcvs(symbol, inverse, start_date, end_date, spot) + if exchange == "bybit": + df = await download_ohlcvs_bybit(symbol, start_date, end_date, download_only=False) + else: + df = download_ohlcvs(symbol, inverse, start_date, end_date, spot) df = df[df.timestamp >= date_to_ts(start_date)] df = df[df.timestamp <= date_to_ts(end_date)] data = df[["timestamp", "high", "low", "close"]].values @@ -1244,7 +1247,7 @@ async def main(): args = parser.parse_args() config = await prepare_backtest_config(args) if config["ohlcv"]: - data = load_hlc_cache( + data = await load_hlc_cache( config["symbol"], config["inverse"], config["start_date"], From a3dd704c7fafe9869f2c97045267b807e23de2d2 Mon Sep 17 00:00:00 2001 From: enarjord Date: Tue, 30 May 2023 12:14:32 +0200 Subject: [PATCH 11/20] load_hlc_cache is async --- backtest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backtest.py b/backtest.py index 9e0a701a0..f91978873 100644 --- a/backtest.py +++ b/backtest.py @@ -238,7 +238,7 @@ async def main(): print(f"{k: <{max(map(len, keys)) + 2}} {config[k]}") print() if config["ohlcv"]: - data = load_hlc_cache( + data = await load_hlc_cache( symbol, config["inverse"], config["start_date"], From dacf085e0a708d51a5b069ebef26b570dca060e0 Mon Sep 17 00:00:00 2001 From: enarjord Date: Tue, 30 May 2023 12:14:46 +0200 Subject: [PATCH 12/20] load_hlc_cache is async --- optimize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optimize.py b/optimize.py index 54790e6cd..a582e92ed 100644 --- a/optimize.py +++ b/optimize.py @@ -267,7 +267,7 @@ async def run_opt(args, config): args.symbol = symbol tmp_cfg = await prepare_backtest_config(args) if config["ohlcv"]: - data = load_hlc_cache( + data = await load_hlc_cache( symbol, config["inverse"], config["start_date"], From 0cf2bc27b5daf774af6963f9b5f70fcbc54c5c92 Mon Sep 17 00:00:00 2001 From: enarjord Date: Tue, 30 May 2023 12:15:14 +0200 Subject: [PATCH 13/20] remove unused import --- harmony_search.py | 1 - particle_swarm_optimization.py | 1 - 2 files changed, 2 deletions(-) diff --git a/harmony_search.py b/harmony_search.py index f4db8f129..e5cb94639 100644 --- a/harmony_search.py +++ b/harmony_search.py @@ -2,7 +2,6 @@ os.environ["NOJIT"] = "false" -from downloader import Downloader, load_hlc_cache import argparse import asyncio import json diff --git a/particle_swarm_optimization.py b/particle_swarm_optimization.py index 2458ead9d..4a24638cc 100644 --- a/particle_swarm_optimization.py +++ b/particle_swarm_optimization.py @@ -2,7 +2,6 @@ os.environ["NOJIT"] = "false" -from downloader import Downloader, load_hlc_cache import argparse import asyncio import json From 2b29f865e6bf93bba23894cbbb8ebb8a55d4b86a Mon Sep 17 00:00:00 2001 From: enarjord Date: Tue, 30 May 2023 23:23:32 +0200 Subject: [PATCH 14/20] forager update frequency is parameterized --- forager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/forager.py b/forager.py index c8b7b001d..e07a64e4d 100644 --- a/forager.py +++ b/forager.py @@ -400,6 +400,7 @@ async def main(): ("symbols_to_ignore", []), ("live_configs_map_long", {}), ("live_configs_map_short", {}), + ("update_interval_minutes", 60), ]: if key not in config: config[key] = value @@ -417,7 +418,7 @@ async def main(): print() subprocess.run(["tmux", "kill-session", "-t", config["user"]]) subprocess.run(["tmuxp", "load", "-d", config["yaml_filepath"]]) - for i in range(3600, -1, -1): + for i in range(config["update_interval_minutes"] * 60, -1, -1): time.sleep(1) print(f"\rcountdown: {i} ", end=" ") print() From 676ba6b03e20d3ebdf874b4b46e75d7d3b8669c9 Mon Sep 17 00:00:00 2001 From: enarjord Date: Tue, 30 May 2023 23:24:12 +0200 Subject: [PATCH 15/20] add parameter update_interval_minutes --- configs/forager/example_config.hjson | 3 +++ 1 file changed, 3 insertions(+) diff --git a/configs/forager/example_config.hjson b/configs/forager/example_config.hjson index 1fdef7f34..1c7efa04f 100644 --- a/configs/forager/example_config.hjson +++ b/configs/forager/example_config.hjson @@ -18,6 +18,9 @@ max_n_panes: 8 + // forager restarts bots every x minutes + update_interval_minutes: 60 + // Don't create bots with these symbols symbols_to_ignore: [ SYM1USDT From 7c87f566a4ed1c7eec068153f8d20a37575fe9fb Mon Sep 17 00:00:00 2001 From: enarjord Date: Wed, 31 May 2023 13:13:59 +0200 Subject: [PATCH 16/20] proper date/ts conversios --- downloader.py | 35 ++++++++++------------------------- 1 file changed, 10 insertions(+), 25 deletions(-) diff --git a/downloader.py b/downloader.py index 945f7aa28..a38c1e5eb 100644 --- a/downloader.py +++ b/downloader.py @@ -30,7 +30,7 @@ add_argparse_args, utc_ms, ) -from pure_funcs import ts_to_date, ts_to_date_utc, date_to_ts, get_dummy_settings +from pure_funcs import ts_to_date, ts_to_date_utc, date_to_ts2, get_dummy_settings, get_day class Downloader: @@ -1005,25 +1005,9 @@ def get_days_in_between(start_day, end_day): return days_in_between -def format_date(date_string): - try: - date_formats = ["%Y", "%Y-%m", "%Y-%m-%d", "%Y-%m-%dT%H:%M:%S"] - for format in date_formats: - try: - date_obj = datetime.datetime.strptime(date_string, format) - formatted_date = date_obj.strftime("%Y-%m-%d") - return formatted_date - except ValueError: - pass - raise ValueError("Invalid date format") - except Exception as e: - print("Error:", e) - return None - - async def download_ohlcvs_bybit(symbol, start_date, end_date, download_only=False): - start_date, end_date = format_date(start_date), format_date(end_date) - assert date_to_ts(end_date) >= date_to_ts(start_date), "end_date is older than start_date" + start_date, end_date = get_day(start_date), get_day(end_date) + assert date_to_ts2(end_date) >= date_to_ts2(start_date), "end_date is older than start_date" dirpath = make_get_filepath(f"historical_data/ohlcvs_bybit/{symbol}/") ideal_days = get_days_in_between(start_date, end_date) days_done = [filename[:-4] for filename in os.listdir(dirpath) if ".csv" in filename] @@ -1126,8 +1110,8 @@ def download_ohlcvs( base_url = "https://data.binance.vision/data/" base_url += "spot/" if spot else f"futures/{'cm' if inverse else 'um'}/" col_names = ["timestamp", "open", "high", "low", "close", "volume"] - start_ts = max(get_first_ohlcv_ts(symbol, spot=spot), date_to_ts(start_date)) - end_ts = date_to_ts(end_date) + start_ts = max(get_first_ohlcv_ts(symbol, spot=spot), date_to_ts2(start_date)) + end_ts = date_to_ts2(end_date) days = [ts_to_date_utc(x)[:10] for x in list(range(start_ts, end_ts, 1000 * 60 * 60 * 24))] months = sorted({x[:7] for x in days}) month_now = ts_to_date(time())[:7] @@ -1207,8 +1191,8 @@ async def load_hlc_cache( symbol, inverse, start_date, end_date, base_dir="backtests", spot=False, exchange="binance" ): cache_fname = ( - f"{ts_to_date_utc(date_to_ts(start_date))[:10]}_" - + f"{ts_to_date_utc(date_to_ts(end_date))[:10]}_ohlcv_cache.npy" + f"{ts_to_date_utc(date_to_ts2(start_date))[:10]}_" + + f"{ts_to_date_utc(date_to_ts2(end_date))[:10]}_ohlcv_cache.npy" ) filepath = make_get_filepath( @@ -1221,8 +1205,8 @@ async def load_hlc_cache( df = await download_ohlcvs_bybit(symbol, start_date, end_date, download_only=False) else: df = download_ohlcvs(symbol, inverse, start_date, end_date, spot) - df = df[df.timestamp >= date_to_ts(start_date)] - df = df[df.timestamp <= date_to_ts(end_date)] + df = df[df.timestamp >= date_to_ts2(start_date)] + df = df[df.timestamp <= date_to_ts2(end_date)] data = df[["timestamp", "high", "low", "close"]].values np.save(filepath, data) try: @@ -1253,6 +1237,7 @@ async def main(): config["start_date"], config["end_date"], spot=config["spot"], + exchange=config["exchange"], ) else: downloader = Downloader(config) From 3256f2bcb186e3a8ed7a94587b8b7116eb0d8dad Mon Sep 17 00:00:00 2001 From: enarjord Date: Wed, 31 May 2023 13:14:13 +0200 Subject: [PATCH 17/20] add funcs date_to_ts2 and get_day --- pure_funcs.py | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/pure_funcs.py b/pure_funcs.py index 58f4f9a5b..7c19a1274 100644 --- a/pure_funcs.py +++ b/pure_funcs.py @@ -277,6 +277,46 @@ def date_to_ts(d): return int(parser.parse(d).replace(tzinfo=datetime.timezone.utc).timestamp() * 1000) +def date_to_ts2(datetime_string): + try: + date_formats = [ + "%Y", + "%Y-%m", + "%Y-%m-%d", + "%Y-%m-%dT%H", + "%Y-%m-%dT%H:%M", + "%Y-%m-%dT%H:%M:%S", + "%Y-%m-%dT%H:%M:%SZ", + ] + for format in date_formats: + try: + date_obj = datetime.datetime.strptime(datetime_string, format) + if format == "%Y" or format == "%Y-%m" or format == "%Y-%m-%d": + date_obj = date_obj.replace(hour=0, minute=0, second=0, microsecond=0) + timestamp = date_obj.replace(tzinfo=datetime.timezone.utc).timestamp() + timestamp_ms = int(timestamp * 1000) + return timestamp_ms + except ValueError: + pass + raise ValueError("Invalid datetime format") + except Exception as e: + print("Error:", e) + return None + + +def get_day(date): + # date can be str datetime or float/int timestamp + try: + return ts_to_date_utc(date_to_ts2(date))[:10] + except: + pass + try: + return ts_to_date_utc(date)[:10] + except: + pass + raise Exception(f"failed to get day from {date}") + + def get_utc_now_timestamp() -> int: """ Creates a millisecond based timestamp of UTC now. From e7469fad659a24bbaaf15e48911739bbf7c4aa29 Mon Sep 17 00:00:00 2001 From: enarjord Date: Wed, 31 May 2023 13:15:01 +0200 Subject: [PATCH 18/20] up version --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ed9cc3334..ab80214a8 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ :warning: **Use at own risk** :warning: -v5.9.8 +v5.9.10 ## Overview From b5a0e1efcf65fef215940f59aa11feced99a3960 Mon Sep 17 00:00:00 2001 From: enarjord Date: Wed, 31 May 2023 13:15:32 +0200 Subject: [PATCH 19/20] use date_to_ts2 --- procedures.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/procedures.py b/procedures.py index 1ab4d18f8..04065a461 100644 --- a/procedures.py +++ b/procedures.py @@ -25,7 +25,7 @@ ts_to_date_utc, get_dummy_settings, config_pretty_str, - date_to_ts, + date_to_ts2, get_template_live_config, sort_dict_keys, make_compatible, @@ -87,8 +87,8 @@ async def prepare_backtest_config(args) -> dict: config["spot"] = False else: config["spot"] = args.market_type == "spot" - config["start_date"] = ts_to_date_utc(date_to_ts(config["start_date"]))[:10] - config["end_date"] = ts_to_date_utc(date_to_ts(config["end_date"]))[:10] + config["start_date"] = ts_to_date_utc(date_to_ts2(config["start_date"]))[:10] + config["end_date"] = ts_to_date_utc(date_to_ts2(config["end_date"]))[:10] config["exchange"] = load_exchange_key_secret_passphrase(config["user"])[0] config["session_name"] = ( f"{config['start_date'].replace(' ', '').replace(':', '').replace('.', '')}_" @@ -458,8 +458,8 @@ def make_tick_samples(config: dict, sec_span: int = 1): """ for key in ["exchange", "symbol", "spot", "start_date", "end_date"]: assert key in config - start_ts = date_to_ts(config["start_date"]) - end_ts = date_to_ts(config["end_date"]) + start_ts = date_to_ts2(config["start_date"]) + end_ts = date_to_ts2(config["end_date"]) ticks_filepath = os.path.join( "historical_data", config["exchange"], From 7cb55d83a9a92698e9095a1030d19b8053198198 Mon Sep 17 00:00:00 2001 From: enarjord Date: Fri, 2 Jun 2023 23:34:30 +0200 Subject: [PATCH 20/20] small bug fix: correct short print mins until clock order --- passivbot.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/passivbot.py b/passivbot.py index efe37c501..c3a5186cf 100644 --- a/passivbot.py +++ b/passivbot.py @@ -1466,8 +1466,8 @@ def calc_minutes_until_next_orders(self): millis_delay_next_entry_short = calc_delay_between_fills_ms_ask( self.position["short"]["price"], self.price, - self.xk["delay_between_fills_ms_entry"][0], - self.xk["delay_weight_entry"][0], + self.xk["delay_between_fills_ms_entry"][1], + self.xk["delay_weight_entry"][1], ) millis_since_prev_close_short = ( self.server_time - self.last_fills_timestamps["clock_entry_short"] @@ -1478,8 +1478,8 @@ def calc_minutes_until_next_orders(self): millis_delay_next_close_short = calc_delay_between_fills_ms_bid( self.position["short"]["price"], self.price, - self.xk["delay_between_fills_ms_close"][0], - self.xk["delay_weight_close"][0], + self.xk["delay_between_fills_ms_close"][1], + self.xk["delay_weight_close"][1], ) millis_since_prev_close_short = ( self.server_time - self.last_fills_timestamps["clock_close_short"]