From ca3f349d848d893d4e03cf4f18b7b6cade068248 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 18 Nov 2023 19:47:09 +0100 Subject: [PATCH 01/80] Update intermediate close_profit calculation --- freqtrade/persistence/trade_model.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/freqtrade/persistence/trade_model.py b/freqtrade/persistence/trade_model.py index 241ef70f5..3f721f1e8 100644 --- a/freqtrade/persistence/trade_model.py +++ b/freqtrade/persistence/trade_model.py @@ -1066,7 +1066,10 @@ class LocalTrade: exit_amount = o.safe_amount_after_fee prof = self.calculate_profit(exit_rate, exit_amount, float(avg_price)) close_profit_abs += prof.profit_abs - close_profit = prof.profit_ratio + if total_stake > 0: + # This needs to be calculated based on the last occuring exit to be aligned + # with realized_profit. + close_profit = (close_profit_abs / total_stake) * self.leverage else: total_stake = total_stake + self._calc_open_trade_value(tmp_amount, price) max_stake_amount += (tmp_amount * price) From 48097f4a7d41c045fe69c6876c6c8346e0d6204b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 18 Nov 2023 19:47:20 +0100 Subject: [PATCH 02/80] Update tests according to calculation update --- tests/persistence/test_persistence.py | 36 +++++++++++++-------------- tests/test_freqtradebot.py | 10 ++++---- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/tests/persistence/test_persistence.py b/tests/persistence/test_persistence.py index 0fc8c7f47..5829f8b71 100644 --- a/tests/persistence/test_persistence.py +++ b/tests/persistence/test_persistence.py @@ -2676,9 +2676,9 @@ def test_order_to_ccxt(limit_buy_order_open, limit_sell_order_usdt_open): 'orders': [ (('buy', 100, 10), (100.0, 10.0, 1000.0, 0.0, None, None)), (('buy', 100, 15), (200.0, 12.5, 2500.0, 0.0, None, None)), - (('sell', 50, 12), (150.0, 12.5, 1875.0, -25.0, -25.0, -0.04)), - (('sell', 100, 20), (50.0, 12.5, 625.0, 725.0, 750.0, 0.60)), - (('sell', 50, 5), (50.0, 12.5, 625.0, 350.0, -375.0, -0.60)), + (('sell', 50, 12), (150.0, 12.5, 1875.0, -25.0, -25.0, -0.01)), + (('sell', 100, 20), (50.0, 12.5, 625.0, 725.0, 750.0, 0.29)), + (('sell', 50, 5), (50.0, 12.5, 625.0, 350.0, -375.0, 0.14)), ], 'end_profit': 350.0, 'end_profit_ratio': 0.14, @@ -2688,9 +2688,9 @@ def test_order_to_ccxt(limit_buy_order_open, limit_sell_order_usdt_open): 'orders': [ (('buy', 100, 10), (100.0, 10.0, 1000.0, 0.0, None, None)), (('buy', 100, 15), (200.0, 12.5, 2500.0, 0.0, None, None)), - (('sell', 50, 12), (150.0, 12.5, 1875.0, -28.0625, -28.0625, -0.044788)), - (('sell', 100, 20), (50.0, 12.5, 625.0, 713.8125, 741.875, 0.59201995)), - (('sell', 50, 5), (50.0, 12.5, 625.0, 336.625, -377.1875, -0.60199501)), + (('sell', 50, 12), (150.0, 12.5, 1875.0, -28.0625, -28.0625, -0.011197)), + (('sell', 100, 20), (50.0, 12.5, 625.0, 713.8125, 741.875, 0.2848129)), + (('sell', 50, 5), (50.0, 12.5, 625.0, 336.625, -377.1875, 0.1343142)), ], 'end_profit': 336.625, 'end_profit_ratio': 0.1343142, @@ -2700,10 +2700,10 @@ def test_order_to_ccxt(limit_buy_order_open, limit_sell_order_usdt_open): 'orders': [ (('buy', 100, 3), (100.0, 3.0, 300.0, 0.0, None, None)), (('buy', 100, 7), (200.0, 5.0, 1000.0, 0.0, None, None)), - (('sell', 100, 11), (100.0, 5.0, 500.0, 596.0, 596.0, 1.189027)), - (('buy', 150, 15), (250.0, 11.0, 2750.0, 596.0, 596.0, 1.189027)), - (('sell', 100, 19), (150.0, 11.0, 1650.0, 1388.5, 792.5, 0.7186579)), - (('sell', 150, 23), (150.0, 11.0, 1650.0, 3175.75, 1787.25, 1.08048062)), + (('sell', 100, 11), (100.0, 5.0, 500.0, 596.0, 596.0, 0.5945137)), + (('buy', 150, 15), (250.0, 11.0, 2750.0, 596.0, 596.0, 0.5945137)), + (('sell', 100, 19), (150.0, 11.0, 1650.0, 1388.5, 792.5, 0.4261653)), + (('sell', 150, 23), (150.0, 11.0, 1650.0, 3175.75, 1787.25, 0.9747170)), ], 'end_profit': 3175.75, 'end_profit_ratio': 0.9747170, @@ -2714,10 +2714,10 @@ def test_order_to_ccxt(limit_buy_order_open, limit_sell_order_usdt_open): 'orders': [ (('buy', 100, 3), (100.0, 3.0, 300.0, 0.0, None, None)), (('buy', 100, 7), (200.0, 5.0, 1000.0, 0.0, None, None)), - (('sell', 100, 11), (100.0, 5.0, 500.0, 600.0, 600.0, 1.2)), - (('buy', 150, 15), (250.0, 11.0, 2750.0, 600.0, 600.0, 1.2)), - (('sell', 100, 19), (150.0, 11.0, 1650.0, 1400.0, 800.0, 0.72727273)), - (('sell', 150, 23), (150.0, 11.0, 1650.0, 3200.0, 1800.0, 1.09090909)), + (('sell', 100, 11), (100.0, 5.0, 500.0, 600.0, 600.0, 0.6)), + (('buy', 150, 15), (250.0, 11.0, 2750.0, 600.0, 600.0, 0.6)), + (('sell', 100, 19), (150.0, 11.0, 1650.0, 1400.0, 800.0, 0.43076923)), + (('sell', 150, 23), (150.0, 11.0, 1650.0, 3200.0, 1800.0, 0.98461538)), ], 'end_profit': 3200.0, 'end_profit_ratio': 0.98461538, @@ -2727,10 +2727,10 @@ def test_order_to_ccxt(limit_buy_order_open, limit_sell_order_usdt_open): 'orders': [ (('buy', 100, 8), (100.0, 8.0, 800.0, 0.0, None, None)), (('buy', 100, 9), (200.0, 8.5, 1700.0, 0.0, None, None)), - (('sell', 100, 10), (100.0, 8.5, 850.0, 150.0, 150.0, 0.17647059)), - (('buy', 150, 11), (250.0, 10, 2500.0, 150.0, 150.0, 0.17647059)), - (('sell', 100, 12), (150.0, 10.0, 1500.0, 350.0, 200.0, 0.2)), - (('sell', 150, 14), (150.0, 10.0, 1500.0, 950.0, 600.0, 0.40)), + (('sell', 100, 10), (100.0, 8.5, 850.0, 150.0, 150.0, 0.08823529)), + (('buy', 150, 11), (250.0, 10, 2500.0, 150.0, 150.0, 0.08823529)), + (('sell', 100, 12), (150.0, 10.0, 1500.0, 350.0, 200.0, 0.1044776)), + (('sell', 150, 14), (150.0, 10.0, 1500.0, 950.0, 600.0, 0.283582)), ], 'end_profit': 950.0, 'end_profit_ratio': 0.283582, diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 40d77ce6c..c9cb86cc0 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -6569,16 +6569,16 @@ def test_position_adjust2(mocker, default_conf_usdt, fee) -> None: # tuple 2 - amount, open_rate, stake_amount, cumulative_profit, realized_profit, rel_profit (('buy', 100, 10), (100.0, 10.0, 1000.0, 0.0, None, None)), (('buy', 100, 15), (200.0, 12.5, 2500.0, 0.0, None, None)), - (('sell', 50, 12), (150.0, 12.5, 1875.0, -28.0625, -28.0625, -0.044788)), - (('sell', 100, 20), (50.0, 12.5, 625.0, 713.8125, 741.875, 0.59201995)), + (('sell', 50, 12), (150.0, 12.5, 1875.0, -28.0625, -28.0625, -0.011197)), + (('sell', 100, 20), (50.0, 12.5, 625.0, 713.8125, 741.875, 0.2848129)), (('sell', 50, 5), (50.0, 12.5, 625.0, 336.625, 336.625, 0.1343142)), # final profit (sum) ), ( (('buy', 100, 3), (100.0, 3.0, 300.0, 0.0, None, None)), (('buy', 100, 7), (200.0, 5.0, 1000.0, 0.0, None, None)), - (('sell', 100, 11), (100.0, 5.0, 500.0, 596.0, 596.0, 1.189027)), - (('buy', 150, 15), (250.0, 11.0, 2750.0, 596.0, 596.0, 1.189027)), - (('sell', 100, 19), (150.0, 11.0, 1650.0, 1388.5, 792.5, 0.7186579)), + (('sell', 100, 11), (100.0, 5.0, 500.0, 596.0, 596.0, 0.5945137)), + (('buy', 150, 15), (250.0, 11.0, 2750.0, 596.0, 596.0, 0.5945137)), + (('sell', 100, 19), (150.0, 11.0, 1650.0, 1388.5, 792.5, 0.4261653)), (('sell', 150, 23), (150.0, 11.0, 1650.0, 3175.75, 3175.75, 0.9747170)), # final profit ) ]) From 7df32a34a00d20249c65f90344988cc271d4eb98 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Sun, 19 Nov 2023 18:01:32 +0900 Subject: [PATCH 03/80] remove order list from /status and add new /order --- freqtrade/rpc/telegram.py | 50 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index ae1f1e9a7..666aedbf5 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -224,6 +224,7 @@ class Telegram(RPCHandler): CommandHandler('help', self._help), CommandHandler('version', self._version), CommandHandler('marketdir', self._changemarketdir) + CommandHandler('order', self._order) ] callbacks = [ CallbackQueryHandler(self._status_table, pattern='update_status_table'), @@ -555,6 +556,49 @@ class Telegram(RPCHandler): return lines_detail + @authorized_only + async def _order(self, update: Update, context: CallbackContext) -> None: + """ + Handler for /order. + Returns the orders of the trade + :param bot: telegram bot + :param update: message update + :return: None + """ + + trade_ids = [] + if context.args and len(context.args) > 0: + trade_ids = [int(i) for i in context.args if i.isnumeric()] + + results = self._rpc._rpc_trade_status(trade_ids=trade_ids) + position_adjust = self._config.get('position_adjustment_enable', False) + max_entries = self._config.get('max_entry_position_adjustment', -1) + for r in results: + lines = [ + "*Order List for Trade #*`{trade_id}`" + ] + + lines_detail = self._prepare_order_details( + r['orders'], r['quote_currency'], r['is_open']) + lines.extend(lines_detail if lines_detail else "") + await self.__send_order_msg(lines, r) + + async def __send_order_msg(self, lines: List[str], r: Dict[str, Any]) -> None: + """ + Send status message. + """ + msg = '' + + for line in lines: + if line: + if (len(msg) + len(line) + 1) < MAX_MESSAGE_LENGTH: + msg += line + '\n' + else: + await self._send_msg(msg.format(**r)) + msg = "*Order List for Trade #*`{trade_id}` - continued\n" + line + '\n' + + await self._send_msg(msg.format(**r)) + @authorized_only async def _status(self, update: Update, context: CallbackContext) -> None: """ @@ -652,9 +696,9 @@ class Telegram(RPCHandler): "*Open Order:* `{open_orders}`" + ("- `{exit_order_status}`" if r['exit_order_status'] else "")) - lines_detail = self._prepare_order_details( - r['orders'], r['quote_currency'], r['is_open']) - lines.extend(lines_detail if lines_detail else "") + # lines_detail = self._prepare_order_details( + # r['orders'], r['quote_currency'], r['is_open']) + # lines.extend(lines_detail if lines_detail else "") await self.__send_status_msg(lines, r) async def __send_status_msg(self, lines: List[str], r: Dict[str, Any]) -> None: From caf59d25e8d5cbcaee4808397b580a9834981b62 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Sun, 19 Nov 2023 18:07:45 +0900 Subject: [PATCH 04/80] forgot comma --- freqtrade/rpc/telegram.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 666aedbf5..901c9856c 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -223,8 +223,8 @@ class Telegram(RPCHandler): CommandHandler('health', self._health), CommandHandler('help', self._help), CommandHandler('version', self._version), - CommandHandler('marketdir', self._changemarketdir) - CommandHandler('order', self._order) + CommandHandler('marketdir', self._changemarketdir), + CommandHandler('order', self._order), ] callbacks = [ CallbackQueryHandler(self._status_table, pattern='update_status_table'), From 58cd45564201bb7da3184d467cc736bab1d12bd0 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Sun, 19 Nov 2023 20:32:28 +0900 Subject: [PATCH 05/80] fix unused vars --- freqtrade/rpc/telegram.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 901c9856c..9a4a64945 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -571,8 +571,6 @@ class Telegram(RPCHandler): trade_ids = [int(i) for i in context.args if i.isnumeric()] results = self._rpc._rpc_trade_status(trade_ids=trade_ids) - position_adjust = self._config.get('position_adjustment_enable', False) - max_entries = self._config.get('max_entry_position_adjustment', -1) for r in results: lines = [ "*Order List for Trade #*`{trade_id}`" From 202b72fd8ea4c4a6048424150f2b6d748d9a4ce9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Nov 2023 03:12:31 +0000 Subject: [PATCH 06/80] Bump stable-baselines3 from 2.1.0 to 2.2.1 Bumps [stable-baselines3](https://github.com/DLR-RM/stable-baselines3) from 2.1.0 to 2.2.1. - [Release notes](https://github.com/DLR-RM/stable-baselines3/releases) - [Commits](https://github.com/DLR-RM/stable-baselines3/compare/v2.1.0...v2.2.1) --- updated-dependencies: - dependency-name: stable-baselines3 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-freqai-rl.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-freqai-rl.txt b/requirements-freqai-rl.txt index c2cca5427..fba25d409 100644 --- a/requirements-freqai-rl.txt +++ b/requirements-freqai-rl.txt @@ -5,7 +5,7 @@ torch==2.0.1 #until these branches will be released we can use this gymnasium==0.29.1 -stable_baselines3==2.1.0 +stable_baselines3==2.2.1 sb3_contrib>=2.0.0a9 # Progress bar for stable-baselines3 and sb3-contrib tqdm==4.66.1 From 8680b5faa000b15da0dad16ca7646542622ea244 Mon Sep 17 00:00:00 2001 From: Stefano Date: Mon, 20 Nov 2023 14:48:18 +0900 Subject: [PATCH 07/80] fix some tests, add new tests --- tests/rpc/test_rpc_telegram.py | 115 +++++++++++++++++++++++++++++++-- 1 file changed, 108 insertions(+), 7 deletions(-) diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 92a4384b4..4c8a69928 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -150,8 +150,8 @@ def test_telegram_init(default_conf, mocker, caplog) -> None: "['reload_conf', 'reload_config'], ['show_conf', 'show_config'], " "['stopbuy', 'stopentry'], ['whitelist'], ['blacklist'], " "['bl_delete', 'blacklist_delete'], " - "['logs'], ['edge'], ['health'], ['help'], ['version'], ['marketdir']" - "]") + "['logs'], ['edge'], ['health'], ['help'], ['version'], ['marketdir'], " + "['order']]") assert log_has(message_str, caplog) @@ -347,8 +347,8 @@ async def test_telegram_status_multi_entry(default_conf, update, mocker, fee) -> msg = msg_mock.call_args_list[3][0][0] assert re.search(r'Number of Entries.*2', msg) assert re.search(r'Number of Exits.*1', msg) - assert re.search(r'from 1st entry rate', msg) - assert re.search(r'Order Filled', msg) + # assert re.search(r'from 1st entry rate', msg) + # assert re.search(r'Order Filled', msg) assert re.search(r'Close Date:', msg) is None assert re.search(r'Close Profit:', msg) is None @@ -375,6 +375,107 @@ async def test_telegram_status_closed_trade(default_conf, update, mocker, fee) - assert re.search(r'Close Profit:', msg) +async def test_order_handle(default_conf, update, ticker, fee, mocker) -> None: + default_conf['max_open_trades'] = 3 + mocker.patch.multiple( + EXMS, + fetch_ticker=ticker, + get_fee=fee, + _dry_is_price_crossed=MagicMock(return_value=True), + ) + status_table = MagicMock() + mocker.patch.multiple( + 'freqtrade.rpc.telegram.Telegram', + _status_table=status_table, + ) + + telegram, freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf) + + patch_get_signal(freqtradebot) + + freqtradebot.state = State.RUNNING + msg_mock.reset_mock() + + # Create some test data + freqtradebot.enter_positions() + + mocker.patch('freqtrade.rpc.telegram.MAX_MESSAGE_LENGTH', 500) + + msg_mock.reset_mock() + context = MagicMock() + context.args = ["2"] + await telegram._order(update=update, context=context) + + assert msg_mock.call_count == 1 + + msg1 = msg_mock.call_args_list[0][0][0] + # msg2 = msg_mock.call_args_list[1][0][0] + + assert 'Order List for Trade #*`2`' in msg1 + # assert 'Trade ID:* `2` - continued' in msg2 + + msg_mock.reset_mock() + mocker.patch('freqtrade.rpc.telegram.MAX_MESSAGE_LENGTH', 50) + context = MagicMock() + context.args = ["2"] + await telegram._order(update=update, context=context) + + assert msg_mock.call_count == 2 + + msg1 = msg_mock.call_args_list[0][0][0] + msg2 = msg_mock.call_args_list[1][0][0] + + assert 'Order List for Trade #*`2`' in msg1 + assert '*Order List for Trade #*`2` - continued' in msg2 + + +@pytest.mark.usefixtures("init_persistence") +async def test_telegram_order_multi_entry(default_conf, update, mocker, fee) -> None: + default_conf['telegram']['enabled'] = False + default_conf['position_adjustment_enable'] = True + mocker.patch.multiple( + EXMS, + fetch_order=MagicMock(return_value=None), + get_rate=MagicMock(return_value=0.22), + ) + + telegram, _, msg_mock = get_telegram_testobject(mocker, default_conf) + + create_mock_trades(fee) + trades = Trade.get_open_trades() + trade = trades[3] + # Average may be empty on some exchanges + trade.orders[0].average = 0 + trade.orders.append(Order( + order_id='5412vbb', + ft_order_side='buy', + ft_pair=trade.pair, + ft_is_open=False, + ft_amount=trade.amount, + ft_price=trade.open_rate, + status="closed", + symbol=trade.pair, + order_type="market", + side="buy", + price=trade.open_rate * 0.95, + average=0, + filled=trade.amount, + remaining=0, + cost=trade.amount, + order_date=trade.open_date, + order_filled_date=trade.open_date, + ) + ) + trade.recalc_trade_from_orders() + Trade.commit() + + await telegram._order(update=update, context=MagicMock()) + assert msg_mock.call_count == 4 + msg = msg_mock.call_args_list[3][0][0] + assert re.search(r'from 1st entry rate', msg) + assert re.search(r'Order Filled', msg) + + async def test_status_handle(default_conf, update, ticker, fee, mocker) -> None: default_conf['max_open_trades'] = 3 mocker.patch.multiple( @@ -443,14 +544,14 @@ async def test_status_handle(default_conf, update, ticker, fee, mocker) -> None: context.args = ["2"] await telegram._status(update=update, context=context) - assert msg_mock.call_count == 2 + assert msg_mock.call_count == 1 msg1 = msg_mock.call_args_list[0][0][0] - msg2 = msg_mock.call_args_list[1][0][0] + # msg2 = msg_mock.call_args_list[1][0][0] assert 'Close Rate' not in msg1 assert 'Trade ID:* `2`' in msg1 - assert 'Trade ID:* `2` - continued' in msg2 + # assert 'Trade ID:* `2` - continued' in msg2 async def test_status_table_handle(default_conf, update, ticker, fee, mocker) -> None: From 0e9169e1eca7589daac29d6a5209624d927f16ac Mon Sep 17 00:00:00 2001 From: Stefano Date: Mon, 20 Nov 2023 15:35:06 +0900 Subject: [PATCH 08/80] update docs --- docs/telegram-usage.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/telegram-usage.md b/docs/telegram-usage.md index f501d0e49..e4dc02c76 100644 --- a/docs/telegram-usage.md +++ b/docs/telegram-usage.md @@ -175,6 +175,7 @@ official commands. You can ask at any moment for help with `/help`. | `/status` | Lists all open trades | `/status ` | Lists one or more specific trade. Separate multiple with a blank space. | `/status table` | List all open trades in a table format. Pending buy orders are marked with an asterisk (*) Pending sell orders are marked with a double asterisk (**) +| `/order ` | Lists orders of one or more specific trade. Separate multiple with a blank space. | `/trades [limit]` | List all recently closed trades in a table format. | `/count` | Displays number of trades used and available | `/locks` | Show currently locked pairs. From 21a5abf25d971cc2f12a74b49c552e3252a3d4bb Mon Sep 17 00:00:00 2001 From: robcaulk Date: Mon, 20 Nov 2023 08:42:27 +0100 Subject: [PATCH 09/80] fix: make sure that get_required_startup is timeframe independent --- freqtrade/data/dataprovider.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 11cbd7934..5e312fa74 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -332,15 +332,14 @@ class DataProvider: if not freqai_config.get('enabled', False): return self._config.get('startup_candle_count', 0) else: - startup_candles = self._config.get('startup_candle_count', 0) indicator_periods = freqai_config['feature_parameters']['indicator_periods_candles'] # make sure the startupcandles is at least the set maximum indicator periods - self._config['startup_candle_count'] = max(startup_candles, max(indicator_periods)) + needed_candles = max(indicator_periods) tf_seconds = timeframe_to_seconds(timeframe) train_candles = 0 if add_train_candles: train_candles = freqai_config['train_period_days'] * 86400 / tf_seconds - total_candles = int(self._config['startup_candle_count'] + train_candles) + total_candles = int(needed_candles + train_candles) logger.info(f'Increasing startup_candle_count for freqai to {total_candles}') return total_candles From d52936fd4239fb5e94a52c9d04846e755a1f12d9 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Mon, 20 Nov 2023 09:13:01 +0100 Subject: [PATCH 10/80] chore: try to keep startup_candle_count behaving the same as a normal FT strat --- freqtrade/data/dataprovider.py | 5 +++-- freqtrade/optimize/backtesting.py | 11 +++++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 5e312fa74..11cbd7934 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -332,14 +332,15 @@ class DataProvider: if not freqai_config.get('enabled', False): return self._config.get('startup_candle_count', 0) else: + startup_candles = self._config.get('startup_candle_count', 0) indicator_periods = freqai_config['feature_parameters']['indicator_periods_candles'] # make sure the startupcandles is at least the set maximum indicator periods - needed_candles = max(indicator_periods) + self._config['startup_candle_count'] = max(startup_candles, max(indicator_periods)) tf_seconds = timeframe_to_seconds(timeframe) train_candles = 0 if add_train_candles: train_candles = freqai_config['train_period_days'] * 86400 / tf_seconds - total_candles = int(needed_candles + train_candles) + total_candles = int(self._config['startup_candle_count'] + train_candles) logger.info(f'Increasing startup_candle_count for freqai to {total_candles}') return total_candles diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 04037bc40..d5caf7070 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -147,7 +147,9 @@ class Backtesting: if self.config.get('freqai', {}).get('enabled', False): # For FreqAI, increase the required_startup to includes the training data - self.required_startup = self.dataprovider.get_required_startup(self.timeframe) + self.freqai_startup_candles = self.dataprovider.get_required_startup( + self.timeframe + ) # Add maximum startup candle count to configuration for informative pairs support self.config['startup_candle_count'] = self.required_startup @@ -234,12 +236,17 @@ class Backtesting: """ self.progress.init_step(BacktestState.DATALOAD, 1) + if self.config.get('freqai', {}).get('enabled', False): + startup_candle_count = self.freqai_startup_candles + else: + startup_candle_count = self.config['startup_candle_count'] + data = history.load_data( datadir=self.config['datadir'], pairs=self.pairlists.whitelist, timeframe=self.timeframe, timerange=self.timerange, - startup_candles=self.config['startup_candle_count'], + startup_candles=startup_candle_count, fail_without_data=True, data_format=self.config['dataformat_ohlcv'], candle_type=self.config.get('candle_type_def', CandleType.SPOT) From e2863e1620225186a0f85a3d2685d8a120f52cd8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 22 Nov 2023 06:54:37 +0100 Subject: [PATCH 11/80] fix: logger has been converted to a property so it can't be assigned anymore https://github.com/DLR-RM/stable-baselines3/commit/e9f0f23ce4b4479968e40698b991b49b7f59bd90#diff-d668633497da171f21b8069c33d594b1ee2ad47c1be4848bea33292bc80b7f5c --- freqtrade/freqai/tensorboard/TensorboardCallback.py | 1 - 1 file changed, 1 deletion(-) diff --git a/freqtrade/freqai/tensorboard/TensorboardCallback.py b/freqtrade/freqai/tensorboard/TensorboardCallback.py index 2be917616..19c03d4e1 100644 --- a/freqtrade/freqai/tensorboard/TensorboardCallback.py +++ b/freqtrade/freqai/tensorboard/TensorboardCallback.py @@ -19,7 +19,6 @@ class TensorboardCallback(BaseCallback): def __init__(self, verbose=1, actions: Type[Enum] = BaseActions): super().__init__(verbose) self.model: Any = None - self.logger: Any = None self.actions: Type[Enum] = actions def _on_training_start(self) -> None: From 5003c2af213f0bd623bcf2bdf19dadcb2a212c38 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 22 Nov 2023 06:56:48 +0100 Subject: [PATCH 12/80] training_env is a property, also, so types can't be overridden --- freqtrade/freqai/tensorboard/TensorboardCallback.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/freqtrade/freqai/tensorboard/TensorboardCallback.py b/freqtrade/freqai/tensorboard/TensorboardCallback.py index 19c03d4e1..b8a351498 100644 --- a/freqtrade/freqai/tensorboard/TensorboardCallback.py +++ b/freqtrade/freqai/tensorboard/TensorboardCallback.py @@ -3,7 +3,6 @@ from typing import Any, Dict, Type, Union from stable_baselines3.common.callbacks import BaseCallback from stable_baselines3.common.logger import HParam -from stable_baselines3.common.vec_env import VecEnv from freqtrade.freqai.RL.BaseEnvironment import BaseActions @@ -13,9 +12,6 @@ class TensorboardCallback(BaseCallback): Custom callback for plotting additional values in tensorboard and episodic summary reports. """ - # Override training_env type to fix type errors - training_env: Union[VecEnv, None] = None - def __init__(self, verbose=1, actions: Type[Enum] = BaseActions): super().__init__(verbose) self.model: Any = None @@ -46,8 +42,6 @@ class TensorboardCallback(BaseCallback): def _on_step(self) -> bool: local_info = self.locals["infos"][0] - if self.training_env is None: - return True if hasattr(self.training_env, 'envs'): tensorboard_metrics = self.training_env.envs[0].unwrapped.tensorboard_metrics From a044649eeff563934776f4f813d608447864a4e0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 23 Nov 2023 06:51:29 +0100 Subject: [PATCH 13/80] Also have "badrequest" act as InvalidOrderException causing an emergencyexit. closes #9456 --- freqtrade/exchange/exchange.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 603c161cf..e8509bf40 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1231,7 +1231,7 @@ class Exchange: f'Insufficient funds to create {ordertype} sell order on market {pair}. ' f'Tried to sell amount {amount} at rate {limit_rate}. ' f'Message: {e}') from e - except ccxt.InvalidOrder as e: + except (ccxt.InvalidOrder, ccxt.BadRequest) as e: # Errors: # `Order would trigger immediately.` raise InvalidOrderException( From e9f21d0209086b829359972cdb9a9a74b6a4dc34 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 23 Nov 2023 07:22:44 +0100 Subject: [PATCH 14/80] Improve logging for #9460 --- freqtrade/exchange/exchange.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index e8509bf40..b9f38b6d2 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1228,16 +1228,16 @@ class Exchange: return order except ccxt.InsufficientFunds as e: raise InsufficientFundsError( - f'Insufficient funds to create {ordertype} sell order on market {pair}. ' - f'Tried to sell amount {amount} at rate {limit_rate}. ' - f'Message: {e}') from e + f'Insufficient funds to create {ordertype} {side} order on market {pair}. ' + f'Tried to {side} amount {amount} at rate {limit_rate} with ' + f'stop-price {stop_price_norm}. Message: {e}') from e except (ccxt.InvalidOrder, ccxt.BadRequest) as e: # Errors: # `Order would trigger immediately.` raise InvalidOrderException( - f'Could not create {ordertype} sell order on market {pair}. ' - f'Tried to sell amount {amount} at rate {limit_rate}. ' - f'Message: {e}') from e + f'Could not create {ordertype} {side} order on market {pair}. ' + f'Tried to {side} amount {amount} at rate {limit_rate} with ' + f'stop-price {stop_price_norm}. Message: {e}') from e except ccxt.DDoSProtection as e: raise DDosProtection(e) from e except (ccxt.NetworkError, ccxt.ExchangeError) as e: From 63e702a3bfc4fd32a8f18fa2a564a196ba9f32a1 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Thu, 23 Nov 2023 15:55:21 +0900 Subject: [PATCH 15/80] remove unused comments --- freqtrade/rpc/telegram.py | 3 --- tests/rpc/test_rpc_telegram.py | 6 ------ 2 files changed, 9 deletions(-) diff --git a/freqtrade/rpc/telegram.py b/freqtrade/rpc/telegram.py index 9a4a64945..c9e9a4733 100644 --- a/freqtrade/rpc/telegram.py +++ b/freqtrade/rpc/telegram.py @@ -694,9 +694,6 @@ class Telegram(RPCHandler): "*Open Order:* `{open_orders}`" + ("- `{exit_order_status}`" if r['exit_order_status'] else "")) - # lines_detail = self._prepare_order_details( - # r['orders'], r['quote_currency'], r['is_open']) - # lines.extend(lines_detail if lines_detail else "") await self.__send_status_msg(lines, r) async def __send_status_msg(self, lines: List[str], r: Dict[str, Any]) -> None: diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index 4c8a69928..b07951c0e 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -347,8 +347,6 @@ async def test_telegram_status_multi_entry(default_conf, update, mocker, fee) -> msg = msg_mock.call_args_list[3][0][0] assert re.search(r'Number of Entries.*2', msg) assert re.search(r'Number of Exits.*1', msg) - # assert re.search(r'from 1st entry rate', msg) - # assert re.search(r'Order Filled', msg) assert re.search(r'Close Date:', msg) is None assert re.search(r'Close Profit:', msg) is None @@ -409,10 +407,8 @@ async def test_order_handle(default_conf, update, ticker, fee, mocker) -> None: assert msg_mock.call_count == 1 msg1 = msg_mock.call_args_list[0][0][0] - # msg2 = msg_mock.call_args_list[1][0][0] assert 'Order List for Trade #*`2`' in msg1 - # assert 'Trade ID:* `2` - continued' in msg2 msg_mock.reset_mock() mocker.patch('freqtrade.rpc.telegram.MAX_MESSAGE_LENGTH', 50) @@ -547,11 +543,9 @@ async def test_status_handle(default_conf, update, ticker, fee, mocker) -> None: assert msg_mock.call_count == 1 msg1 = msg_mock.call_args_list[0][0][0] - # msg2 = msg_mock.call_args_list[1][0][0] assert 'Close Rate' not in msg1 assert 'Trade ID:* `2`' in msg1 - # assert 'Trade ID:* `2` - continued' in msg2 async def test_status_table_handle(default_conf, update, ticker, fee, mocker) -> None: From 25ebdb4cb3ff7b7b8ea7419f9f11062248578a45 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 25 Nov 2023 12:51:17 +0100 Subject: [PATCH 16/80] use strategy_wrapper for bot_loop_start in plotting closes #9464 --- freqtrade/plot/plotting.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/plot/plotting.py b/freqtrade/plot/plotting.py index 9ed7bbc46..e0aa2437a 100644 --- a/freqtrade/plot/plotting.py +++ b/freqtrade/plot/plotting.py @@ -21,6 +21,7 @@ from freqtrade.misc import pair_to_filename from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.resolvers import ExchangeResolver, StrategyResolver from freqtrade.strategy import IStrategy +from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper logger = logging.getLogger(__name__) @@ -636,7 +637,7 @@ def load_and_plot_trades(config: Config): exchange = ExchangeResolver.load_exchange(config) IStrategy.dp = DataProvider(config, exchange) strategy.ft_bot_start() - strategy.bot_loop_start(datetime.now(timezone.utc)) + strategy_safe_wrapper(strategy.bot_loop_start)(current_time=datetime.now(timezone.utc)) plot_elements = init_plotscript(config, list(exchange.markets), strategy.startup_candle_count) timerange = plot_elements['timerange'] trades = plot_elements['trades'] From 67e81c9018d7d44960cc969f6cc8000fc5f78b93 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 25 Nov 2023 13:10:45 +0100 Subject: [PATCH 17/80] Fix wrong/faulty docstring --- freqtrade/exchange/exchange.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index b9f38b6d2..d0b6f4caf 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1496,8 +1496,9 @@ class Exchange: @retrier def fetch_bids_asks(self, symbols: Optional[List[str]] = None, cached: bool = False) -> Dict: """ + :param symbols: List of symbols to fetch :param cached: Allow cached result - :return: fetch_tickers result + :return: fetch_bids_asks result """ if not self.exchange_has('fetchBidsAsks'): return {} From 8c5194d5e2fb1bb5d5b534e8790c169d16739aee Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 25 Nov 2023 15:05:56 +0100 Subject: [PATCH 18/80] force-reload markets when BadSymbol appears closes #9463 --- freqtrade/exchange/exchange.py | 15 ++++++++++++--- tests/exchange/test_exchange.py | 10 +++++++++- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index d0b6f4caf..5d0bc704f 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -486,11 +486,14 @@ class Exchange: except ccxt.BaseError: logger.exception('Unable to initialize markets.') - def reload_markets(self) -> None: + def reload_markets(self, force: bool = False) -> None: """Reload markets both sync and async if refresh interval has passed """ # Check whether markets have to be reloaded - if (self._last_markets_refresh > 0) and ( - self._last_markets_refresh + self.markets_refresh_interval > dt_ts()): + if ( + not force + and self._last_markets_refresh > 0 + and (self._last_markets_refresh + self.markets_refresh_interval > dt_ts()) + ): return None logger.debug("Performing scheduled market reload..") try: @@ -1547,6 +1550,12 @@ class Exchange: raise OperationalException( f'Exchange {self._api.name} does not support fetching tickers in batch. ' f'Message: {e}') from e + except ccxt.BadSymbol as e: + logger.warning(f"Could not load tickers due to {e.__class__.__name__}. Message: {e} ." + "Reloading markets.") + self.reload_markets(True) + # Re-raise exception to repeat the call. + raise TemporaryError from e except ccxt.DDoSProtection as e: raise DDosProtection(e) from e except (ccxt.NetworkError, ccxt.ExchangeError) as e: diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index e2b3cc102..ad993cb6d 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1851,7 +1851,7 @@ def test_fetch_bids_asks(default_conf, mocker): @pytest.mark.parametrize("exchange_name", EXCHANGES) -def test_get_tickers(default_conf, mocker, exchange_name): +def test_get_tickers(default_conf, mocker, exchange_name, caplog): api_mock = MagicMock() tick = {'ETH/BTC': { 'symbol': 'ETH/BTC', @@ -1900,6 +1900,14 @@ def test_get_tickers(default_conf, mocker, exchange_name): exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) exchange.get_tickers() + caplog.clear() + api_mock.fetch_tickers = MagicMock(side_effect=[ccxt.BadSymbol("SomeSymbol"), []]) + exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) + x = exchange.get_tickers() + assert x == [] + assert log_has_re(r'Could not load tickers due to BadSymbol\..*SomeSymbol', caplog) + caplog.clear() + api_mock.fetch_tickers = MagicMock(return_value={}) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) exchange.get_tickers() From 94020a664bcf1ae0b577f92268c8e7fbed64965b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 25 Nov 2023 15:58:28 +0100 Subject: [PATCH 19/80] Add slight sleep to avoid random test failure on windows --- tests/test_integration.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_integration.py b/tests/test_integration.py index ee1d4bbb3..a94c9c7d6 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -1,3 +1,4 @@ +from time import time from unittest.mock import MagicMock import pytest @@ -440,6 +441,7 @@ def test_dca_order_adjust(default_conf_usdt, ticker_usdt, leverage, fee, mocker) assert trade.open_rate == 1.99 assert trade.orders[-1].price == 1.96 assert trade.orders[-1].cost == 120 * leverage + time.sleep(0.1) # Replace new order with diff. order at a lower price freqtrade.strategy.adjust_entry_price = MagicMock(return_value=1.95) From e8d0b0199117174dd4fc35852eeafd6dc0a2b637 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 25 Nov 2023 16:17:43 +0100 Subject: [PATCH 20/80] Correctly import time --- tests/test_integration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_integration.py b/tests/test_integration.py index a94c9c7d6..12647f6e2 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -1,4 +1,4 @@ -from time import time +import time from unittest.mock import MagicMock import pytest From eeb460e55cf6bbd17bec9ff347b66274d5ee4f90 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 25 Nov 2023 19:36:14 +0100 Subject: [PATCH 21/80] Use cloudpickle throughout --- freqtrade/freqai/data_drawer.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index a0c902f48..89a828e4d 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -12,7 +12,7 @@ import numpy as np import pandas as pd import psutil import rapidjson -from joblib import dump, load +from joblib import load from joblib.externals import cloudpickle from numpy.typing import NDArray from pandas import DataFrame @@ -471,7 +471,8 @@ class FreqaiDataDrawer: # Save the trained model if self.model_type == 'joblib': - dump(model, save_path / f"{dk.model_filename}_model.joblib") + with (save_path / f"{dk.model_filename}_model.joblib").open("wb") as fp: + cloudpickle.dump(model, fp) elif self.model_type == 'keras': model.save(save_path / f"{dk.model_filename}_model.h5") elif self.model_type in ["stable_baselines3", "sb3_contrib", "pytorch"]: From 72eeb6561cd791a6e1db146e9992ab3c5e7c1aa2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 25 Nov 2023 19:37:49 +0100 Subject: [PATCH 22/80] use Cloudpickle also for reading --- freqtrade/freqai/data_drawer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index 89a828e4d..cfae6f5d3 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -12,7 +12,6 @@ import numpy as np import pandas as pd import psutil import rapidjson -from joblib import load from joblib.externals import cloudpickle from numpy.typing import NDArray from pandas import DataFrame @@ -559,7 +558,8 @@ class FreqaiDataDrawer: if dk.live and coin in self.model_dictionary: model = self.model_dictionary[coin] elif self.model_type == 'joblib': - model = load(dk.data_path / f"{dk.model_filename}_model.joblib") + with (dk.data_path / f"{dk.model_filename}_model.joblib").open("rb") as fp: + model = cloudpickle.load(fp) elif 'stable_baselines' in self.model_type or 'sb3_contrib' == self.model_type: mod = importlib.import_module( self.model_type, self.freqai_info['rl_config']['model_type']) From be8ea685de170dacee2967aea2d2e30780dc265b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 26 Nov 2023 13:37:02 +0100 Subject: [PATCH 23/80] Remove unused function in freqAI example --- freqtrade/templates/FreqaiExampleStrategy.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/freqtrade/templates/FreqaiExampleStrategy.py b/freqtrade/templates/FreqaiExampleStrategy.py index e64570b9e..8be1f0336 100644 --- a/freqtrade/templates/FreqaiExampleStrategy.py +++ b/freqtrade/templates/FreqaiExampleStrategy.py @@ -290,9 +290,6 @@ class FreqaiExampleStrategy(IStrategy): return df - def get_ticker_indicator(self): - return int(self.config["timeframe"][:-1]) - def confirm_trade_entry( self, pair: str, From a50291cd90725f8248a95443d680f537c7d76693 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 31 Oct 2023 13:19:08 +0100 Subject: [PATCH 24/80] RPI dockerfile - 3.11 --- docker/Dockerfile.armhf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile.armhf b/docker/Dockerfile.armhf index c17c0adbe..447311f90 100644 --- a/docker/Dockerfile.armhf +++ b/docker/Dockerfile.armhf @@ -1,4 +1,4 @@ -FROM python:3.9.16-slim-bullseye as base +FROM python:3.11.6-slim-bullseye as base # Setup env ENV LANG C.UTF-8 From df73b8288c0fe28e4b84ebc4e652527713ce1456 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 26 Nov 2023 14:53:34 +0100 Subject: [PATCH 25/80] Bump armhf image to bookworm --- docker/Dockerfile.armhf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/Dockerfile.armhf b/docker/Dockerfile.armhf index 447311f90..c8efa4232 100644 --- a/docker/Dockerfile.armhf +++ b/docker/Dockerfile.armhf @@ -1,4 +1,4 @@ -FROM python:3.11.6-slim-bullseye as base +FROM python:3.11.6-slim-bookworm as base # Setup env ENV LANG C.UTF-8 @@ -11,7 +11,7 @@ ENV FT_APP_ENV="docker" # Prepare environment RUN mkdir /freqtrade \ && apt-get update \ - && apt-get -y install sudo libatlas3-base libopenblas-base curl sqlite3 libhdf5-dev libutf8proc-dev libsnappy-dev \ + && apt-get -y install sudo libatlas3-base libopenblas-dev curl sqlite3 libhdf5-dev libutf8proc-dev libsnappy-dev \ && apt-get clean \ && useradd -u 1000 -G sudo -U -m ftuser \ && chown ftuser:ftuser /freqtrade \ @@ -24,7 +24,7 @@ WORKDIR /freqtrade # Install dependencies FROM base as python-deps RUN apt-get update \ - && apt-get -y install build-essential libssl-dev libffi-dev libopenblas-dev libgfortran5 pkg-config cmake gcc \ + && apt-get -y install build-essential libssl-dev libffi-dev libgfortran5 pkg-config cmake gcc \ && apt-get clean \ && echo "[global]\nextra-index-url=https://www.piwheels.org/simple" > /etc/pip.conf From 12ea1cde80844c9c0229194e2142ac06de4f43df Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 26 Nov 2023 16:34:31 +0100 Subject: [PATCH 26/80] Bump dockerfile to use bookworm (debian12) --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e2adf65f9..38f9ca788 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11.6-slim-bullseye as base +FROM python:3.11.6-slim-bookworm as base # Setup env ENV LANG C.UTF-8 From 88e818926ad539634d197db5452c719240b26e45 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Mon, 27 Nov 2023 10:57:50 +0900 Subject: [PATCH 27/80] fix typo on API docs --- docs/rest-api.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/rest-api.md b/docs/rest-api.md index ff111c2ce..229fa5f94 100644 --- a/docs/rest-api.md +++ b/docs/rest-api.md @@ -134,9 +134,9 @@ python3 scripts/rest_client.py --config rest_config.json [optional par | `reload_config` | Reloads the configuration file. | `trades` | List last trades. Limited to 500 trades per call. | `trade/` | Get specific trade. -| `trade/` | DELETE - Remove trade from the database. Tries to close open orders. Requires manual handling of this trade on the exchange. -| `trade//open-order` | DELETE - Cancel open order for this trade. -| `trade//reload` | GET - Reload a trade from the Exchange. Only works in live, and can potentially help recover a trade that was manually sold on the exchange. +| `trades/` | DELETE - Remove trade from the database. Tries to close open orders. Requires manual handling of this trade on the exchange. +| `trades//open-order` | DELETE - Cancel open order for this trade. +| `trades//reload` | GET - Reload a trade from the Exchange. Only works in live, and can potentially help recover a trade that was manually sold on the exchange. | `show_config` | Shows part of the current configuration with relevant settings to operation. | `logs` | Shows last log messages. | `status` | Lists all open trades. From 8567af8d28bd4b2b72ae388dc09b4fa26f4bc897 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Nov 2023 03:19:22 +0000 Subject: [PATCH 28/80] Bump ccxt from 4.1.57 to 4.1.66 Bumps [ccxt](https://github.com/ccxt/ccxt) from 4.1.57 to 4.1.66. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/CHANGELOG.md) - [Commits](https://github.com/ccxt/ccxt/compare/4.1.57...4.1.66) --- updated-dependencies: - dependency-name: ccxt dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e3f2b4bbb..a31d4d0f1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy==1.26.2 pandas==2.1.3 pandas-ta==0.3.14b -ccxt==4.1.57 +ccxt==4.1.66 cryptography==41.0.5 aiohttp==3.9.0 SQLAlchemy==2.0.23 From 384ebc5b3840e11822472005839d98e2c3084c88 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Nov 2023 03:19:35 +0000 Subject: [PATCH 29/80] Bump pymdown-extensions from 10.4 to 10.5 Bumps [pymdown-extensions](https://github.com/facelessuser/pymdown-extensions) from 10.4 to 10.5. - [Release notes](https://github.com/facelessuser/pymdown-extensions/releases) - [Commits](https://github.com/facelessuser/pymdown-extensions/compare/10.4...10.5) --- updated-dependencies: - dependency-name: pymdown-extensions dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 31e04a74f..e7cea4667 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -2,5 +2,5 @@ markdown==3.5.1 mkdocs==1.5.3 mkdocs-material==9.4.10 mdx_truly_sane_lists==1.3 -pymdown-extensions==10.4 +pymdown-extensions==10.5 jinja2==3.1.2 From 5e5b56e445e34df2c397c0d028f0d46da1f6b251 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Nov 2023 03:19:43 +0000 Subject: [PATCH 30/80] Bump mypy from 1.7.0 to 1.7.1 Bumps [mypy](https://github.com/python/mypy) from 1.7.0 to 1.7.1. - [Changelog](https://github.com/python/mypy/blob/master/CHANGELOG.md) - [Commits](https://github.com/python/mypy/compare/v1.7.0...v1.7.1) --- updated-dependencies: - dependency-name: mypy dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 44cf0982d..d81e46f29 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,7 +8,7 @@ coveralls==3.3.1 ruff==0.1.6 -mypy==1.7.0 +mypy==1.7.1 pre-commit==3.5.0 pytest==7.4.3 pytest-asyncio==0.21.1 From 786d4e03fc41c4c316e1e30bc73858dd732af932 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Nov 2023 03:20:08 +0000 Subject: [PATCH 31/80] Bump aiohttp from 3.9.0 to 3.9.1 Bumps [aiohttp](https://github.com/aio-libs/aiohttp) from 3.9.0 to 3.9.1. - [Release notes](https://github.com/aio-libs/aiohttp/releases) - [Changelog](https://github.com/aio-libs/aiohttp/blob/master/CHANGES.rst) - [Commits](https://github.com/aio-libs/aiohttp/compare/v3.9.0...v3.9.1) --- updated-dependencies: - dependency-name: aiohttp dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e3f2b4bbb..8de7ec437 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ pandas-ta==0.3.14b ccxt==4.1.57 cryptography==41.0.5 -aiohttp==3.9.0 +aiohttp==3.9.1 SQLAlchemy==2.0.23 python-telegram-bot==20.6 # can't be hard-pinned due to telegram-bot pinning httpx with ~ From 35871d7b4be9df4851d5da634bbcf9ad706cc587 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Nov 2023 03:20:23 +0000 Subject: [PATCH 32/80] Bump pydantic from 2.5.1 to 2.5.2 Bumps [pydantic](https://github.com/pydantic/pydantic) from 2.5.1 to 2.5.2. - [Release notes](https://github.com/pydantic/pydantic/releases) - [Changelog](https://github.com/pydantic/pydantic/blob/v2.5.2/HISTORY.md) - [Commits](https://github.com/pydantic/pydantic/compare/v2.5.1...v2.5.2) --- updated-dependencies: - dependency-name: pydantic dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e3f2b4bbb..56abef707 100644 --- a/requirements.txt +++ b/requirements.txt @@ -37,7 +37,7 @@ sdnotify==0.3.2 # API Server fastapi==0.104.1 -pydantic==2.5.1 +pydantic==2.5.2 uvicorn==0.24.0.post1 pyjwt==2.8.0 aiofiles==23.2.1 From 793cc1acc02bd2ed321ea2f7ed659e8668dbd636 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Nov 2023 05:31:12 +0000 Subject: [PATCH 33/80] Bump mkdocs-material from 9.4.10 to 9.4.14 Bumps [mkdocs-material](https://github.com/squidfunk/mkdocs-material) from 9.4.10 to 9.4.14. - [Release notes](https://github.com/squidfunk/mkdocs-material/releases) - [Changelog](https://github.com/squidfunk/mkdocs-material/blob/master/CHANGELOG) - [Commits](https://github.com/squidfunk/mkdocs-material/compare/9.4.10...9.4.14) --- updated-dependencies: - dependency-name: mkdocs-material dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- docs/requirements-docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index e7cea4667..10c70939e 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -1,6 +1,6 @@ markdown==3.5.1 mkdocs==1.5.3 -mkdocs-material==9.4.10 +mkdocs-material==9.4.14 mdx_truly_sane_lists==1.3 pymdown-extensions==10.5 jinja2==3.1.2 From 7c00a4ed13248e2399c5bc0cc4d54b733632ba17 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 27 Nov 2023 06:39:14 +0100 Subject: [PATCH 34/80] Remove no longer existing exchange aliases from tests --- tests/exchange/test_exchange_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/exchange/test_exchange_utils.py b/tests/exchange/test_exchange_utils.py index 8ae4039c7..6a4137369 100644 --- a/tests/exchange/test_exchange_utils.py +++ b/tests/exchange/test_exchange_utils.py @@ -41,14 +41,14 @@ def test_check_exchange(default_conf, caplog) -> None: caplog.clear() # Test an officially supported by Freqtrade team exchange - with remapping - default_conf.get('exchange').update({'name': 'okex'}) + default_conf.get('exchange').update({'name': 'okx'}) assert check_exchange(default_conf) assert log_has_re( - r"Exchange \"okex\" is officially supported by the Freqtrade development team\.", + r"Exchange \"okx\" is officially supported by the Freqtrade development team\.", caplog) caplog.clear() # Test an available exchange, supported by ccxt - default_conf.get('exchange').update({'name': 'huobipro'}) + default_conf.get('exchange').update({'name': 'huobijp'}) assert check_exchange(default_conf) assert log_has_re(r"Exchange .* is known to the the ccxt library, available for the bot, " r"but not officially supported " From 2738f3e437f7282ea8bc87c0760a5dc36aaa633b Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 9 Nov 2023 06:35:15 +0100 Subject: [PATCH 35/80] Add minimal Exchange class for bitmart --- freqtrade/exchange/__init__.py | 1 + freqtrade/exchange/bitmart.py | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 freqtrade/exchange/bitmart.py diff --git a/freqtrade/exchange/__init__.py b/freqtrade/exchange/__init__.py index 9ac31a0d8..8de9120dc 100644 --- a/freqtrade/exchange/__init__.py +++ b/freqtrade/exchange/__init__.py @@ -4,6 +4,7 @@ from freqtrade.exchange.common import remove_exchange_credentials, MAP_EXCHANGE_ from freqtrade.exchange.exchange import Exchange # isort: on from freqtrade.exchange.binance import Binance +from freqtrade.exchange.bitmart import Bitmart from freqtrade.exchange.bitpanda import Bitpanda from freqtrade.exchange.bittrex import Bittrex from freqtrade.exchange.bitvavo import Bitvavo diff --git a/freqtrade/exchange/bitmart.py b/freqtrade/exchange/bitmart.py new file mode 100644 index 000000000..67a567eb7 --- /dev/null +++ b/freqtrade/exchange/bitmart.py @@ -0,0 +1,19 @@ +""" Bitmart exchange subclass """ +import logging +from typing import Dict + +from freqtrade.exchange import Exchange + + +logger = logging.getLogger(__name__) + + +class Bitmart(Exchange): + """ + Bitmart exchange class. Contains adjustments needed for Freqtrade to work + with this exchange. + """ + + _ft_has: Dict = { + "ohlcv_candle_limit": 200, + } From ef877b5fcc1d406bea1450cac11cb76049077171 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 10 Nov 2023 19:40:54 +0100 Subject: [PATCH 36/80] Add bitmart to ccxt compat tested exchanges --- tests/exchange_online/conftest.py | 8 ++++++++ tests/exchange_online/test_ccxt_compat.py | 3 ++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/exchange_online/conftest.py b/tests/exchange_online/conftest.py index c5f59ee0e..35c9a9d85 100644 --- a/tests/exchange_online/conftest.py +++ b/tests/exchange_online/conftest.py @@ -227,6 +227,7 @@ EXCHANGES = { 'timeframe': '1h', 'futures_pair': 'BTC/USDT:USDT', 'futures': True, + 'orderbook_max_entries': 50, 'leverage_tiers_public': True, 'leverage_in_spot_market': True, 'sample_order': [ @@ -247,6 +248,13 @@ EXCHANGES = { } ] }, + 'bitmart': { + 'pair': 'BTC/USDT', + 'stake_currency': 'USDT', + 'hasQuoteVolume': True, + 'timeframe': '1h', + 'orderbook_max_entries': 50, + }, 'huobi': { 'pair': 'ETH/BTC', 'stake_currency': 'BTC', diff --git a/tests/exchange_online/test_ccxt_compat.py b/tests/exchange_online/test_ccxt_compat.py index aa3dfdfae..b48d70de2 100644 --- a/tests/exchange_online/test_ccxt_compat.py +++ b/tests/exchange_online/test_ccxt_compat.py @@ -133,6 +133,7 @@ class TestCCXTExchange: exch, exchangename = exchange pair = EXCHANGES[exchangename]['pair'] l2 = exch.fetch_l2_order_book(pair) + orderbook_max_entries = EXCHANGES[exchangename].get('orderbook_max_entries') assert 'asks' in l2 assert 'bids' in l2 assert len(l2['asks']) >= 1 @@ -143,7 +144,7 @@ class TestCCXTExchange: # TODO: Gate is unstable here at the moment, ignoring the limit partially. return for val in [1, 2, 5, 25, 50, 100]: - if val > 50 and exchangename == 'bybit': + if orderbook_max_entries and val > orderbook_max_entries: continue l2 = exch.fetch_l2_order_book(pair, val) if not l2_limit_range or val in l2_limit_range: From cb2a8715380d519312c89171c1465f8a30368fcf Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 13 Nov 2023 07:04:23 +0100 Subject: [PATCH 37/80] Bitmart: add section in exchange docs --- docs/exchanges.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/exchanges.md b/docs/exchanges.md index 237125e88..a1d1efc04 100644 --- a/docs/exchanges.md +++ b/docs/exchanges.md @@ -302,6 +302,21 @@ We do strongly recommend to limit all API keys to the IP you're going to use it Bybit (futures only) supports `stoploss_on_exchange` and uses `stop-loss-limit` orders. It provides great advantages, so we recommend to benefit from it by enabling stoploss on exchange. On futures, Bybit supports both `stop-limit` as well as `stop-market` orders. You can use either `"limit"` or `"market"` in the `order_types.stoploss` configuration setting to decide which type to use. +## Bitmart + +Bitmart requires the API key Memo (the name you give the API key) to go along with the exchange key and secret. +It's therefore required to pass the UID as well. + +```json +"exchange": { + "name": "bitmart", + "uid": "your_bitmart_api_key_memo", + "secret": "your_exchange_secret", + "password": "your_exchange_api_key_password", + // ... +} +``` + ## All exchanges Should you experience constant errors with Nonce (like `InvalidNonce`), it is best to regenerate the API keys. Resetting Nonce is difficult and it's usually easier to regenerate the API keys. From 0914b8b5f4f3828a32a28570b41ef2b8f558271d Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 16 Nov 2023 19:36:43 +0100 Subject: [PATCH 38/80] Add note about verification on Bitmart --- docs/exchanges.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/exchanges.md b/docs/exchanges.md index a1d1efc04..6ae4eaa11 100644 --- a/docs/exchanges.md +++ b/docs/exchanges.md @@ -317,6 +317,10 @@ It's therefore required to pass the UID as well. } ``` + +!!! Warning "Necessary Verification" + Bitmart requires Verification Lvl2 to successfully trade on the spot market through the API - even though trading via UI works just fine with just Lvl1 verification. + ## All exchanges Should you experience constant errors with Nonce (like `InvalidNonce`), it is best to regenerate the API keys. Resetting Nonce is difficult and it's usually easier to regenerate the API keys. From a18c85ec64ac7af3af1885a8246ca72fda196301 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 16 Nov 2023 19:37:23 +0100 Subject: [PATCH 39/80] Explicitly disable stoploss on exchange for bitmart --- freqtrade/exchange/bitmart.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/freqtrade/exchange/bitmart.py b/freqtrade/exchange/bitmart.py index 67a567eb7..5d792b153 100644 --- a/freqtrade/exchange/bitmart.py +++ b/freqtrade/exchange/bitmart.py @@ -15,5 +15,6 @@ class Bitmart(Exchange): """ _ft_has: Dict = { - "ohlcv_candle_limit": 200, + "stoploss_on_exchange": False, # Bitmart API does not support stoploss orders + "ohlcv_candle_limit": 200, } From d0a2b9403e8dd756563cf17737bfe86545ad68e6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 28 Nov 2023 18:14:33 +0100 Subject: [PATCH 40/80] Fix typo --- docs/exchanges.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/exchanges.md b/docs/exchanges.md index 6ae4eaa11..ac3957b07 100644 --- a/docs/exchanges.md +++ b/docs/exchanges.md @@ -317,7 +317,6 @@ It's therefore required to pass the UID as well. } ``` - !!! Warning "Necessary Verification" Bitmart requires Verification Lvl2 to successfully trade on the spot market through the API - even though trading via UI works just fine with just Lvl1 verification. From 0a903e45f2728851c37c6a8a8a0453a4eecdeaab Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 28 Nov 2023 18:14:42 +0100 Subject: [PATCH 41/80] Add Bitmart to list of supported exchnanges for spot --- README.md | 1 + docs/index.md | 1 + 2 files changed, 2 insertions(+) diff --git a/README.md b/README.md index 4ed144f93..c00d2c999 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ hesitate to read the source code and understand the mechanism of this bot. Please read the [exchange specific notes](docs/exchanges.md) to learn about eventual, special configurations needed for each exchange. - [X] [Binance](https://www.binance.com/) +- [X] [Bitmart](https://bitmart.com/) - [X] [Gate.io](https://www.gate.io/ref/6266643) - [X] [Huobi](http://huobi.com/) - [X] [Kraken](https://kraken.com/) diff --git a/docs/index.md b/docs/index.md index dd80db958..1df5424de 100644 --- a/docs/index.md +++ b/docs/index.md @@ -40,6 +40,7 @@ Freqtrade is a free and open source crypto trading bot written in Python. It is Please read the [exchange specific notes](exchanges.md) to learn about eventual, special configurations needed for each exchange. - [X] [Binance](https://www.binance.com/) +- [X] [Bitmart](https://bitmart.com/) - [X] [Gate.io](https://www.gate.io/ref/6266643) - [X] [Huobi](http://huobi.com/) - [X] [Kraken](https://kraken.com/) From 6db66b1e5886c17fc8694227bb57244aa68e9f83 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 28 Nov 2023 18:18:19 +0100 Subject: [PATCH 42/80] Add bitmart to "official supported" exchanges --- freqtrade/exchange/common.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/exchange/common.py b/freqtrade/exchange/common.py index 36bed9b20..ca986d2be 100644 --- a/freqtrade/exchange/common.py +++ b/freqtrade/exchange/common.py @@ -52,6 +52,7 @@ MAP_EXCHANGE_CHILDCLASS = { SUPPORTED_EXCHANGES = [ 'binance', + 'bitmart', 'gate', 'huobi', 'kraken', From 5ddca4e9f92e1b026aef4bc370172077df0a7112 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 29 Nov 2023 05:27:01 +0000 Subject: [PATCH 43/80] Bump cryptography from 41.0.5 to 41.0.6 Bumps [cryptography](https://github.com/pyca/cryptography) from 41.0.5 to 41.0.6. - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/41.0.5...41.0.6) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 959d79125..6f78d04df 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ pandas==2.1.3 pandas-ta==0.3.14b ccxt==4.1.66 -cryptography==41.0.5 +cryptography==41.0.6 aiohttp==3.9.1 SQLAlchemy==2.0.23 python-telegram-bot==20.6 From cf078f80935673b57baa80afd7ba1ccef00df191 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 29 Nov 2023 06:57:00 +0100 Subject: [PATCH 44/80] Bump to 41.0.7 to avoid install problems --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6f78d04df..a01dc58ec 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ pandas==2.1.3 pandas-ta==0.3.14b ccxt==4.1.66 -cryptography==41.0.6 +cryptography==41.0.7 aiohttp==3.9.1 SQLAlchemy==2.0.23 python-telegram-bot==20.6 From 36836ea803ad73a7f506b311b484fc28c9a346a7 Mon Sep 17 00:00:00 2001 From: Stefano Ariestasia Date: Thu, 30 Nov 2023 14:11:09 +0900 Subject: [PATCH 45/80] add bot_name to discord rpc field --- freqtrade/rpc/discord.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/freqtrade/rpc/discord.py b/freqtrade/rpc/discord.py index c0e9220b2..43190e395 100644 --- a/freqtrade/rpc/discord.py +++ b/freqtrade/rpc/discord.py @@ -15,6 +15,7 @@ class Discord(Webhook): self.rpc = rpc self.strategy = config.get('strategy', '') self.timeframe = config.get('timeframe', '') + self.bot_name = config.get('bot_name', '') self._url = config['discord']['webhook_url'] self._format = 'json' @@ -36,6 +37,7 @@ class Discord(Webhook): msg['strategy'] = self.strategy msg['timeframe'] = self.timeframe + msg['bot_name'] = self.bot_name color = 0x0000FF if msg['type'] in (RPCMessageType.EXIT, RPCMessageType.EXIT_FILL): profit_ratio = msg.get('profit_ratio') From c34cb9eb128a2eb7d03c9c8798e40d81b11cbfa1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 30 Nov 2023 07:04:38 +0100 Subject: [PATCH 46/80] Bump version to 2023.12-dev --- freqtrade/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/__init__.py b/freqtrade/__init__.py index f1afb5156..464a9df97 100644 --- a/freqtrade/__init__.py +++ b/freqtrade/__init__.py @@ -1,5 +1,5 @@ """ Freqtrade bot """ -__version__ = '2023.11-dev' +__version__ = '2023.12-dev' if 'dev' in __version__: from pathlib import Path From d85518ccb21a83764805194c9d01cb617690d616 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 30 Nov 2023 07:10:20 +0100 Subject: [PATCH 47/80] Improve release documentation --- docs/developer.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/developer.md b/docs/developer.md index 2a826d866..9c549012d 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -419,6 +419,9 @@ This part of the documentation is aimed at maintainers, and shows how to create ### Create release branch +!!! Note + Make sure that the `stable` branch is up-to-date! + First, pick a commit that's about one week old (to not include latest additions to releases). ``` bash @@ -431,14 +434,11 @@ Determine if crucial bugfixes have been made between this commit and the current * Merge the release branch (stable) into this branch. * Edit `freqtrade/__init__.py` and add the version matching the current date (for example `2019.7` for July 2019). Minor versions can be `2019.7.1` should we need to do a second release that month. Version numbers must follow allowed versions from PEP0440 to avoid failures pushing to pypi. * Commit this part. -* push that branch to the remote and create a PR against the stable branch. +* Push that branch to the remote and create a PR against the **stable branch**. * Update develop version to next version following the pattern `2019.8-dev`. ### Create changelog from git commits -!!! Note - Make sure that the `stable` branch is up-to-date! - ``` bash # Needs to be done before merging / pulling that branch. git log --oneline --no-decorate --no-merges stable..new_release From 5daafaabc2e09f08bb146b471b6d0dde1c6a5a24 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 1 Dec 2023 07:05:22 +0100 Subject: [PATCH 48/80] Add explicit test for "now_is_time_to_refresh --- tests/exchange/test_exchange.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index ad993cb6d..e932fa9bb 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1977,6 +1977,34 @@ def test_fetch_ticker(default_conf, mocker, exchange_name): exchange.fetch_ticker(pair='XRP/ETH') +@pytest.mark.parametrize("exchange_name", EXCHANGES) +def test___now_is_time_to_refresh(default_conf, mocker, exchange_name, time_machine): + exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) + pair = 'BTC/USDT' + candle_type = CandleType.SPOT + start_dt = datetime(2023, 12, 1, 0, 10, 0, tzinfo=timezone.utc) + time_machine.move_to(start_dt, tick=False) + assert (pair, '5m', candle_type) not in exchange._pairs_last_refresh_time + + # not refreshed yet + assert exchange._now_is_time_to_refresh(pair, '5m', candle_type) is True + + last_closed_candle = (start_dt - timedelta(minutes=5)).timestamp() + exchange._pairs_last_refresh_time[(pair, '5m', candle_type)] = last_closed_candle + + # next candle not closed yet + time_machine.move_to(start_dt + timedelta(minutes=4, seconds=59), tick=False) + assert exchange._now_is_time_to_refresh(pair, '5m', candle_type) is False + + # next candle closed + time_machine.move_to(start_dt + timedelta(minutes=5, seconds=0), tick=False) + assert exchange._now_is_time_to_refresh(pair, '5m', candle_type) is True + + # 1 second later (last_refresh_time didn't change) + time_machine.move_to(start_dt + timedelta(minutes=5, seconds=1), tick=False) + assert exchange._now_is_time_to_refresh(pair, '5m', candle_type) is True + + @pytest.mark.parametrize("exchange_name", EXCHANGES) @pytest.mark.parametrize('candle_type', ['mark', '']) def test_get_historic_ohlcv(default_conf, mocker, caplog, exchange_name, candle_type): From 6754072bca5d4f3608795f1d146dc610fc27e2c5 Mon Sep 17 00:00:00 2001 From: Robert Caulk Date: Sun, 3 Dec 2023 12:55:16 +0100 Subject: [PATCH 49/80] chore: fix future warning on pandas --- freqtrade/freqai/data_drawer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index cfae6f5d3..f8b8f0de4 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -317,9 +317,9 @@ class FreqaiDataDrawer: index = self.historic_predictions[pair].index[-1:] columns = self.historic_predictions[pair].columns - nan_df = pd.DataFrame(np.nan, index=index, columns=columns) + zeros_df = pd.DataFrame(np.zeros, index=index, columns=columns) self.historic_predictions[pair] = pd.concat( - [self.historic_predictions[pair], nan_df], ignore_index=True, axis=0) + [self.historic_predictions[pair], zeros_df], ignore_index=True, axis=0) df = self.historic_predictions[pair] # model outputs and associated statistics From c017366086e422e6397c9236ed14c7d6c79666ea Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 2 Dec 2023 17:37:46 +0100 Subject: [PATCH 50/80] Ensure testdata generation works for 1M data --- tests/conftest.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index b18032621..9b983882a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -87,11 +87,13 @@ def get_args(args): def generate_test_data(timeframe: str, size: int, start: str = '2020-07-05'): np.random.seed(42) - tf_mins = timeframe_to_minutes(timeframe) base = np.random.normal(20, 2, size=size) - - date = pd.date_range(start, periods=size, freq=f'{tf_mins}min', tz='UTC') + if timeframe == '1M': + date = pd.date_range(start, periods=size, freq='1MS', tz='UTC') + else: + tf_mins = timeframe_to_minutes(timeframe) + date = pd.date_range(start, periods=size, freq=f'{tf_mins}min', tz='UTC') df = pd.DataFrame({ 'date': date, 'open': base, From 4ed9ffbf315c3644e8f6381125aa5a533aeac465 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 3 Dec 2023 13:22:54 +0100 Subject: [PATCH 51/80] Add test for behavior in #9490. --- tests/strategy/test_strategy_helpers.py | 26 +++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/strategy/test_strategy_helpers.py b/tests/strategy/test_strategy_helpers.py index 2f611a6c6..6c6270e64 100644 --- a/tests/strategy/test_strategy_helpers.py +++ b/tests/strategy/test_strategy_helpers.py @@ -61,6 +61,32 @@ def test_merge_informative_pair(): assert result.iloc[8]['date_1h'] is pd.NaT +def test_merge_informative_pair_high_tf(): + # Covers roughly 2 months - until 2023-01-10 + data = generate_test_data('1h', 1040, '2022-11-28') + informative = generate_test_data('1M', 40, '2022-01-01') + + result = merge_informative_pair(data, informative, '1h', '1M', ffill=True) + assert isinstance(result, pd.DataFrame) + candle1 = result.loc[(result['date'] == '2022-12-31T22:00:00.000Z')] + assert candle1.iloc[0]['date'] == pd.Timestamp('2022-12-31T22:00:00.000Z') + assert candle1.iloc[0]['date_1M'] == pd.Timestamp('2022-11-01T00:00:00.000Z') + + candle2 = result.loc[(result['date'] == '2022-12-31T23:00:00.000Z')] + assert candle2.iloc[0]['date'] == pd.Timestamp('2022-12-31T23:00:00.000Z') + assert candle2.iloc[0]['date_1M'] == pd.Timestamp('2022-12-01T00:00:00.000Z') + + # Candle is empty, as the start-date did fail. + candle3 = result.loc[(result['date'] == '2022-11-30T22:00:00.000Z')] + assert candle3.iloc[0]['date'] == pd.Timestamp('2022-11-30T22:00:00.000Z') + assert candle3.iloc[0]['date_1M'] is pd.NaT + + # First candle with 1M data merged. + candle4 = result.loc[(result['date'] == '2022-11-30T23:00:00.000Z')] + assert candle4.iloc[0]['date'] == pd.Timestamp('2022-11-30T23:00:00.000Z') + assert candle4.iloc[0]['date_1M'] == pd.Timestamp('2022-11-01T00:00:00.000Z') + + def test_merge_informative_pair_same(): data = generate_test_data('15m', 40) informative = generate_test_data('15m', 40) From 4464b027191245677f8a1831150a512e627c5e95 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 3 Dec 2023 13:27:00 +0100 Subject: [PATCH 52/80] Add handling to properly merge 1M data to closes #9490 --- freqtrade/strategy/strategy_helper.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/freqtrade/strategy/strategy_helper.py b/freqtrade/strategy/strategy_helper.py index 7654a383f..1de3f8f89 100644 --- a/freqtrade/strategy/strategy_helper.py +++ b/freqtrade/strategy/strategy_helper.py @@ -46,10 +46,16 @@ def merge_informative_pair(dataframe: pd.DataFrame, informative: pd.DataFrame, # Subtract "small" timeframe so merging is not delayed by 1 small candle # Detailed explanation in https://github.com/freqtrade/freqtrade/issues/4073 if not informative.empty: - informative['date_merge'] = ( - informative[date_column] + pd.to_timedelta(minutes_inf, 'm') - - pd.to_timedelta(minutes, 'm') - ) + if timeframe_inf == '1M': + informative['date_merge'] = ( + (informative[date_column] + pd.offsets.MonthBegin(1)) + - pd.to_timedelta(minutes, 'm') + ) + else: + informative['date_merge'] = ( + informative[date_column] + pd.to_timedelta(minutes_inf, 'm') - + pd.to_timedelta(minutes, 'm') + ) else: informative['date_merge'] = informative[date_column] else: From e9e7bf3caf64a435d274e3aae2b1d73b5dae4c26 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 3 Dec 2023 13:29:48 +0100 Subject: [PATCH 53/80] Slight cleanup of unused comment --- freqtrade/strategy/strategy_helper.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/freqtrade/strategy/strategy_helper.py b/freqtrade/strategy/strategy_helper.py index 1de3f8f89..0a774e9af 100644 --- a/freqtrade/strategy/strategy_helper.py +++ b/freqtrade/strategy/strategy_helper.py @@ -86,9 +86,6 @@ def merge_informative_pair(dataframe: pd.DataFrame, informative: pd.DataFrame, right_on=date_merge, how='left') dataframe = dataframe.drop(date_merge, axis=1) - # if ffill: - # dataframe = dataframe.ffill() - return dataframe From 0235db48a8ac512e936804b4109e97669f87f2d2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 3 Dec 2023 13:31:08 +0100 Subject: [PATCH 54/80] Prevent merge artifacts in "informative" pair --- freqtrade/strategy/strategy_helper.py | 2 +- tests/strategy/test_strategy_helpers.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/freqtrade/strategy/strategy_helper.py b/freqtrade/strategy/strategy_helper.py index 0a774e9af..b0fc538ca 100644 --- a/freqtrade/strategy/strategy_helper.py +++ b/freqtrade/strategy/strategy_helper.py @@ -36,7 +36,7 @@ def merge_informative_pair(dataframe: pd.DataFrame, informative: pd.DataFrame, :return: Merged dataframe :raise: ValueError if the secondary timeframe is shorter than the dataframe timeframe """ - + informative = informative.copy() minutes_inf = timeframe_to_minutes(timeframe_inf) minutes = timeframe_to_minutes(timeframe) if minutes == minutes_inf: diff --git a/tests/strategy/test_strategy_helpers.py b/tests/strategy/test_strategy_helpers.py index 6c6270e64..831856702 100644 --- a/tests/strategy/test_strategy_helpers.py +++ b/tests/strategy/test_strategy_helpers.py @@ -12,9 +12,11 @@ from tests.conftest import generate_test_data, get_patched_exchange def test_merge_informative_pair(): data = generate_test_data('15m', 40) informative = generate_test_data('1h', 40) + cols_inf = list(informative.columns) result = merge_informative_pair(data, informative, '15m', '1h', ffill=True) assert isinstance(result, pd.DataFrame) + assert list(informative.columns) == cols_inf assert len(result) == len(data) assert 'date' in result.columns assert result['date'].equals(data['date']) From 701f6fc050070e0dac8fb25bcbe73e4e8b450e18 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 3 Dec 2023 15:27:03 +0100 Subject: [PATCH 55/80] Update outdated docs closes #9489 --- docs/backtesting.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index d13b00a38..83a3fa2d8 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -624,7 +624,7 @@ All listed Strategies need to be in the same directory. freqtrade backtesting --timerange 20180401-20180410 --timeframe 5m --strategy-list Strategy001 Strategy002 --export trades ``` -This will save the results to `user_data/backtest_results/backtest-result-.json`, injecting the strategy-name into the target filename. +This will save the results to `user_data/backtest_results/backtest-result-.json`, including results for both `Strategy001` and `Strategy002`. There will be an additional table comparing win/losses of the different strategies (identical to the "Total" row in the first table). Detailed output for all strategies one after the other will be available, so make sure to scroll up to see the details per strategy. From 90e4eb59b21037ba1abce590aa1b7b804eec983c Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 3 Dec 2023 15:27:33 +0100 Subject: [PATCH 56/80] Improve multi-strategy backtest docs --- docs/backtesting.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/backtesting.md b/docs/backtesting.md index 83a3fa2d8..ece3ce7fa 100644 --- a/docs/backtesting.md +++ b/docs/backtesting.md @@ -618,7 +618,7 @@ To compare multiple strategies, a list of Strategies can be provided to backtest This is limited to 1 timeframe value per run. However, data is only loaded once from disk so if you have multiple strategies you'd like to compare, this will give a nice runtime boost. -All listed Strategies need to be in the same directory. +All listed Strategies need to be in the same directory, unless also `--recursive-strategy-search` is specified, where sub-directories within the strategy directory are also considered. ``` bash freqtrade backtesting --timerange 20180401-20180410 --timeframe 5m --strategy-list Strategy001 Strategy002 --export trades From 106bda51d412cdf1186755e95b13ff8d13580078 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 03:10:11 +0000 Subject: [PATCH 57/80] Bump pypa/gh-action-pypi-publish from 1.8.10 to 1.8.11 Bumps [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) from 1.8.10 to 1.8.11. - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.8.10...v1.8.11) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a2f6567dc..f5e6bf08c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -461,7 +461,7 @@ jobs: python setup.py sdist bdist_wheel - name: Publish to PyPI (Test) - uses: pypa/gh-action-pypi-publish@v1.8.10 + uses: pypa/gh-action-pypi-publish@v1.8.11 if: (github.event_name == 'release') with: user: __token__ @@ -469,7 +469,7 @@ jobs: repository_url: https://test.pypi.org/legacy/ - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@v1.8.10 + uses: pypa/gh-action-pypi-publish@v1.8.11 if: (github.event_name == 'release') with: user: __token__ From 1431b2b44a1e370e3e622a5ffc98c8c9148aaafd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 03:35:27 +0000 Subject: [PATCH 58/80] Bump tables from 3.9.1 to 3.9.2 Bumps [tables](https://github.com/PyTables/PyTables) from 3.9.1 to 3.9.2. - [Release notes](https://github.com/PyTables/PyTables/releases) - [Changelog](https://github.com/PyTables/PyTables/blob/master/RELEASE_NOTES.rst) - [Commits](https://github.com/PyTables/PyTables/compare/v3.9.1...v3.9.2) --- updated-dependencies: - dependency-name: tables dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a01dc58ec..6fea97ca6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,7 +19,7 @@ technical==1.4.0 tabulate==0.9.0 pycoingecko==3.1.0 jinja2==3.1.2 -tables==3.9.1 +tables==3.9.2 joblib==1.3.2 rich==13.7.0 pyarrow==14.0.1; platform_machine != 'armv7l' From 44689d058b60df47e00a4ecec8861a1b43019cf6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 03:35:42 +0000 Subject: [PATCH 59/80] Bump ccxt from 4.1.66 to 4.1.75 Bumps [ccxt](https://github.com/ccxt/ccxt) from 4.1.66 to 4.1.75. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/CHANGELOG.md) - [Commits](https://github.com/ccxt/ccxt/compare/4.1.66...4.1.75) --- updated-dependencies: - dependency-name: ccxt dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a01dc58ec..987b7cbbd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy==1.26.2 pandas==2.1.3 pandas-ta==0.3.14b -ccxt==4.1.66 +ccxt==4.1.75 cryptography==41.0.7 aiohttp==3.9.1 SQLAlchemy==2.0.23 From e0f780f10e83f3084d0b1a5319dd322b9199e761 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 03:35:49 +0000 Subject: [PATCH 60/80] Bump python-telegram-bot from 20.6 to 20.7 Bumps [python-telegram-bot](https://github.com/python-telegram-bot/python-telegram-bot) from 20.6 to 20.7. - [Release notes](https://github.com/python-telegram-bot/python-telegram-bot/releases) - [Changelog](https://github.com/python-telegram-bot/python-telegram-bot/blob/master/CHANGES.rst) - [Commits](https://github.com/python-telegram-bot/python-telegram-bot/compare/v20.6...v20.7) --- updated-dependencies: - dependency-name: python-telegram-bot dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a01dc58ec..4891c0488 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ ccxt==4.1.66 cryptography==41.0.7 aiohttp==3.9.1 SQLAlchemy==2.0.23 -python-telegram-bot==20.6 +python-telegram-bot==20.7 # can't be hard-pinned due to telegram-bot pinning httpx with ~ httpx>=0.24.1 arrow==1.3.0 From 04f2b7bad6073c2bc42b20a7cff1c7bfefc2e105 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 4 Dec 2023 07:23:52 +0100 Subject: [PATCH 61/80] Add support for different timeInForce for bybit --- freqtrade/exchange/bybit.py | 1 + 1 file changed, 1 insertion(+) diff --git a/freqtrade/exchange/bybit.py b/freqtrade/exchange/bybit.py index e71229cad..e7c463140 100644 --- a/freqtrade/exchange/bybit.py +++ b/freqtrade/exchange/bybit.py @@ -29,6 +29,7 @@ class Bybit(Exchange): _ft_has: Dict = { "ohlcv_candle_limit": 1000, "ohlcv_has_history": True, + "order_time_in_force": ["GTC", "FOK", "IOC", "PO"], } _ft_has_futures: Dict = { "ohlcv_has_history": True, From 8b5e6bf9be6c5d2897bb9672304bd306095046a1 Mon Sep 17 00:00:00 2001 From: robcaulk Date: Mon, 4 Dec 2023 16:23:55 +0100 Subject: [PATCH 62/80] chore: fix pandas warning about merging on datetime. fix pandas warning about inconsistent key lists in concat --- freqtrade/freqai/data_drawer.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index f8b8f0de4..afd70c300 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -284,6 +284,10 @@ class FreqaiDataDrawer: new_pred["date_pred"] = dataframe["date"] hist_preds = self.historic_predictions[pair].copy() + # ensure both dataframes have the same date format so they can be merged + new_pred["date_pred"] = pd.to_datetime(new_pred["date_pred"], utc=True) + hist_preds["date_pred"] = pd.to_datetime(hist_preds["date_pred"], utc=True) + # find the closest common date between new_pred and historic predictions # and cut off the new_pred dataframe at that date common_dates = pd.merge(new_pred, hist_preds, on="date_pred", how="inner") @@ -294,7 +298,9 @@ class FreqaiDataDrawer: "predictions. You likely left your FreqAI instance offline " f"for more than {len(dataframe.index)} candles.") - df_concat = pd.concat([hist_preds, new_pred], ignore_index=True, keys=hist_preds.keys()) + # reindex new_pred columns to match the historic predictions dataframe + new_pred_reindexed = new_pred.reindex(columns=hist_preds.columns) + df_concat = pd.concat([hist_preds, new_pred_reindexed], ignore_index=True) # any missing values will get zeroed out so users can see the exact # downtime in FreqUI From 59a287106de7dab852fe770c71cc4566800c86c0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 18:21:30 +0000 Subject: [PATCH 63/80] Bump ccxt from 4.1.75 to 4.1.76 Bumps [ccxt](https://github.com/ccxt/ccxt) from 4.1.75 to 4.1.76. - [Release notes](https://github.com/ccxt/ccxt/releases) - [Changelog](https://github.com/ccxt/ccxt/blob/master/CHANGELOG.md) - [Commits](https://github.com/ccxt/ccxt/compare/4.1.75...4.1.76) --- updated-dependencies: - dependency-name: ccxt dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 53e47c345..8e40eeec8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy==1.26.2 pandas==2.1.3 pandas-ta==0.3.14b -ccxt==4.1.75 +ccxt==4.1.76 cryptography==41.0.7 aiohttp==3.9.1 SQLAlchemy==2.0.23 From caac77c90b54bdd65021c64f6cf9aba89f620ef8 Mon Sep 17 00:00:00 2001 From: Robert Caulk Date: Tue, 5 Dec 2023 12:16:08 +0100 Subject: [PATCH 64/80] Update data_drawer.py --- freqtrade/freqai/data_drawer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqai/data_drawer.py b/freqtrade/freqai/data_drawer.py index afd70c300..e3027267b 100644 --- a/freqtrade/freqai/data_drawer.py +++ b/freqtrade/freqai/data_drawer.py @@ -285,8 +285,8 @@ class FreqaiDataDrawer: hist_preds = self.historic_predictions[pair].copy() # ensure both dataframes have the same date format so they can be merged - new_pred["date_pred"] = pd.to_datetime(new_pred["date_pred"], utc=True) - hist_preds["date_pred"] = pd.to_datetime(hist_preds["date_pred"], utc=True) + new_pred["date_pred"] = pd.to_datetime(new_pred["date_pred"]) + hist_preds["date_pred"] = pd.to_datetime(hist_preds["date_pred"]) # find the closest common date between new_pred and historic predictions # and cut off the new_pred dataframe at that date From 46c81d70184efcb0ffa54e6267ecc7651d7dbbf3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 5 Dec 2023 17:04:16 +0100 Subject: [PATCH 65/80] Revert "Bump tables from 3.9.1 to 3.9.2" --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8e40eeec8..5ef69111d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,7 +19,7 @@ technical==1.4.0 tabulate==0.9.0 pycoingecko==3.1.0 jinja2==3.1.2 -tables==3.9.2 +tables==3.9.1 joblib==1.3.2 rich==13.7.0 pyarrow==14.0.1; platform_machine != 'armv7l' From 6ee792069de39b0f9ae02ccb7ccc1d161b8ea455 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 5 Dec 2023 18:03:34 +0100 Subject: [PATCH 66/80] Remove bitrex-specific section --- docs/faq.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/docs/faq.md b/docs/faq.md index 50aaa03a3..196bd4308 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -128,12 +128,6 @@ This warning can point to one of the below problems: * Barely traded pair -> Check the pair on the exchange webpage, look at the timeframe your strategy uses. If the pair does not have any volume in some candles (usually visualized with a "volume 0" bar, and a "_" as candle), this pair did not have any trades in this timeframe. These pairs should ideally be avoided, as they can cause problems with order-filling. * API problem -> API returns wrong data (this only here for completeness, and should not happen with supported exchanges). -### I'm getting the "RESTRICTED_MARKET" message in the log - -Currently known to happen for US Bittrex users. - -Read [the Bittrex section about restricted markets](exchanges.md#restricted-markets) for more information. - ### I'm getting the "Exchange XXX does not support market orders." message and cannot run my strategy As the message says, your exchange does not support market orders and you have one of the [order types](configuration.md/#understand-order_types) set to "market". Your strategy was probably written with other exchanges in mind and sets "market" orders for "stoploss" orders, which is correct and preferable for most of the exchanges supporting market orders (but not for Bittrex and Gate.io). From ef042ae5ec23790a1ea9338bb38f777f78da97cd Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 5 Dec 2023 18:03:48 +0100 Subject: [PATCH 67/80] Remove exchange tests on bittrex --- tests/exchange/test_exchange.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index e932fa9bb..1d26b706f 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -24,7 +24,7 @@ from tests.conftest import (EXMS, generate_test_data_raw, get_mock_coro, get_pat # Make sure to always keep one exchange here which is NOT subclassed!! -EXCHANGES = ['bittrex', 'binance', 'kraken', 'gate', 'kucoin', 'bybit', 'okx'] +EXCHANGES = ['binance', 'kraken', 'gate', 'kucoin', 'bybit', 'okx'] get_entry_rate_data = [ ('other', 20, 19, 10, 0.0, 20), # Full ask side @@ -3901,11 +3901,11 @@ def test_set_margin_mode(mocker, default_conf, margin_mode): ("kraken", TradingMode.SPOT, None, False), ("kraken", TradingMode.MARGIN, MarginMode.ISOLATED, True), ("kraken", TradingMode.FUTURES, MarginMode.ISOLATED, True), - ("bittrex", TradingMode.SPOT, None, False), - ("bittrex", TradingMode.MARGIN, MarginMode.CROSS, True), - ("bittrex", TradingMode.MARGIN, MarginMode.ISOLATED, True), - ("bittrex", TradingMode.FUTURES, MarginMode.CROSS, True), - ("bittrex", TradingMode.FUTURES, MarginMode.ISOLATED, True), + ("bitmart", TradingMode.SPOT, None, False), + ("bitmart", TradingMode.MARGIN, MarginMode.CROSS, True), + ("bitmart", TradingMode.MARGIN, MarginMode.ISOLATED, True), + ("bitmart", TradingMode.FUTURES, MarginMode.CROSS, True), + ("bitmart", TradingMode.FUTURES, MarginMode.ISOLATED, True), ("gate", TradingMode.MARGIN, MarginMode.ISOLATED, True), ("okx", TradingMode.SPOT, None, False), ("okx", TradingMode.MARGIN, MarginMode.CROSS, True), From 4061eaf888c7c78f9d5916f31deea6c6ee731725 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 5 Dec 2023 18:03:55 +0100 Subject: [PATCH 68/80] Remove online tests for bittrex --- tests/exchange_online/conftest.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tests/exchange_online/conftest.py b/tests/exchange_online/conftest.py index 35c9a9d85..875faeded 100644 --- a/tests/exchange_online/conftest.py +++ b/tests/exchange_online/conftest.py @@ -14,14 +14,6 @@ EXCHANGE_FIXTURE_TYPE = Tuple[Exchange, str] # Exchanges that should be tested online EXCHANGES = { - 'bittrex': { - 'pair': 'BTC/USDT', - 'stake_currency': 'USDT', - 'hasQuoteVolume': False, - 'timeframe': '1h', - 'leverage_tiers_public': False, - 'leverage_in_spot_market': False, - }, 'binance': { 'pair': 'BTC/USDT', 'stake_currency': 'USDT', From cbe6d52a731c8de5fc61234e9c968a6d0d0eb505 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 5 Dec 2023 06:36:54 +0100 Subject: [PATCH 69/80] Try macos-13 CI --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f5e6bf08c..593a5bd03 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -126,7 +126,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ macos-latest ] + os: [ "macos-latest", "macos-13" ] python-version: ["3.9", "3.10", "3.11"] steps: From 2207773678f9d6a8e26405c038dcf3535e0fe5c3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 5 Dec 2023 10:39:51 +0100 Subject: [PATCH 70/80] Attempt more granular cache for macos --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 593a5bd03..4d14f2649 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -143,7 +143,7 @@ jobs: id: cache with: path: ~/dependencies/ - key: ${{ runner.os }}-dependencies + key: ${{ runner.os }}-${{ env.ImageOS }}-dependencies - name: pip cache (macOS) uses: actions/cache@v3 From 68db8d0201c397291e1abf1999f5deb748cf0241 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 5 Dec 2023 15:38:15 +0100 Subject: [PATCH 71/80] install libomp --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4d14f2649..025ce5504 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -175,7 +175,7 @@ jobs: rm /usr/local/bin/python3-config || true rm /usr/local/bin/python3.11-config || true - brew install hdf5 c-blosc + brew install hdf5 c-blosc libomp python -m pip install --upgrade pip wheel export LD_LIBRARY_PATH=${HOME}/dependencies/lib:$LD_LIBRARY_PATH export TA_LIBRARY_PATH=${HOME}/dependencies/lib From 3c48208b3190afb25a7e164a0969288de430a4bd Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 5 Dec 2023 18:07:08 +0100 Subject: [PATCH 72/80] Fix cache keys --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 025ce5504..de823e9dd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -143,14 +143,14 @@ jobs: id: cache with: path: ~/dependencies/ - key: ${{ runner.os }}-${{ env.ImageOS }}-dependencies + key: ${{ matrix.os }}-dependencies - name: pip cache (macOS) uses: actions/cache@v3 if: runner.os == 'macOS' with: path: ~/Library/Caches/pip - key: test-${{ matrix.os }}-${{ matrix.python-version }}-pip + key: ${{ matrix.os }}-${{ matrix.python-version }}-pip - name: TA binary *nix if: steps.cache.outputs.cache-hit != 'true' From 9a425af76f67a1a838fcf3edd001138a7d9f398e Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 5 Dec 2023 18:08:00 +0100 Subject: [PATCH 73/80] Remove unnecessary conditions in Ci run --- .github/workflows/ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index de823e9dd..3a1eca7a7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -147,7 +147,6 @@ jobs: - name: pip cache (macOS) uses: actions/cache@v3 - if: runner.os == 'macOS' with: path: ~/Library/Caches/pip key: ${{ matrix.os }}-${{ matrix.python-version }}-pip @@ -158,7 +157,6 @@ jobs: cd build_helpers && ./install_ta-lib.sh ${HOME}/dependencies/; cd .. - name: Installation - macOS - if: runner.os == 'macOS' run: | # brew update # TODO: Should be the brew upgrade From 3739a10735d9a0b7d854a81dab4f50894c280b7f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 5 Dec 2023 17:10:09 +0000 Subject: [PATCH 74/80] Bump nbconvert from 7.11.0 to 7.12.0 Bumps [nbconvert](https://github.com/jupyter/nbconvert) from 7.11.0 to 7.12.0. - [Release notes](https://github.com/jupyter/nbconvert/releases) - [Changelog](https://github.com/jupyter/nbconvert/blob/main/CHANGELOG.md) - [Commits](https://github.com/jupyter/nbconvert/compare/v7.11.0...v7.12.0) --- updated-dependencies: - dependency-name: nbconvert dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index d81e46f29..2302a0426 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -20,7 +20,7 @@ isort==5.12.0 time-machine==2.13.0 # Convert jupyter notebooks to markdown documents -nbconvert==7.11.0 +nbconvert==7.12.0 # mypy types types-cachetools==5.3.0.7 From 3b4b833dd070120c50a967066cba0c5fd7d34a83 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 6 Dec 2023 18:22:28 +0100 Subject: [PATCH 75/80] Remove unused mock --- tests/rpc/test_rpc_telegram.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/rpc/test_rpc_telegram.py b/tests/rpc/test_rpc_telegram.py index b07951c0e..5d190dcc6 100644 --- a/tests/rpc/test_rpc_telegram.py +++ b/tests/rpc/test_rpc_telegram.py @@ -109,7 +109,6 @@ def get_telegram_testobject(mocker, default_conf, mock=True, ftbot=None): _start_thread=MagicMock(), ) if not ftbot: - mocker.patch('freqtrade.exchange.exchange.Exchange._init_async_loop') ftbot = get_patched_freqtradebot(mocker, default_conf) rpc = RPC(ftbot) telegram = Telegram(rpc, default_conf) From 71178ff1d2b3334dca9fa433677e2d0a4f9f9248 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 6 Dec 2023 19:48:19 +0100 Subject: [PATCH 76/80] Remove "activate" activator they raise warnings as they're unused. --- freqtrade/freqai/prediction_models/XGBoostRFRegressor.py | 2 +- freqtrade/freqai/prediction_models/XGBoostRegressor.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/freqai/prediction_models/XGBoostRFRegressor.py b/freqtrade/freqai/prediction_models/XGBoostRFRegressor.py index f43585ab0..1949ad536 100644 --- a/freqtrade/freqai/prediction_models/XGBoostRFRegressor.py +++ b/freqtrade/freqai/prediction_models/XGBoostRFRegressor.py @@ -45,7 +45,7 @@ class XGBoostRFRegressor(BaseRegressionModel): model = XGBRFRegressor(**self.model_training_parameters) - model.set_params(callbacks=[TBCallback(dk.data_path)], activate=self.activate_tensorboard) + model.set_params(callbacks=[TBCallback(dk.data_path)]) model.fit(X=X, y=y, sample_weight=sample_weight, eval_set=eval_set, sample_weight_eval_set=eval_weights, xgb_model=xgb_model) # set the callbacks to empty so that we can serialize to disk later diff --git a/freqtrade/freqai/prediction_models/XGBoostRegressor.py b/freqtrade/freqai/prediction_models/XGBoostRegressor.py index f8b4d353d..f1a2474da 100644 --- a/freqtrade/freqai/prediction_models/XGBoostRegressor.py +++ b/freqtrade/freqai/prediction_models/XGBoostRegressor.py @@ -45,7 +45,7 @@ class XGBoostRegressor(BaseRegressionModel): model = XGBRegressor(**self.model_training_parameters) - model.set_params(callbacks=[TBCallback(dk.data_path)], activate=self.activate_tensorboard) + model.set_params(callbacks=[TBCallback(dk.data_path)]) model.fit(X=X, y=y, sample_weight=sample_weight, eval_set=eval_set, sample_weight_eval_set=eval_weights, xgb_model=xgb_model) # set the callbacks to empty so that we can serialize to disk later From b14873400d57c4ce616b0b5b4cb2c24e1edee258 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 7 Dec 2023 07:15:05 +0100 Subject: [PATCH 77/80] Fix odd import in freqai tests --- tests/freqai/test_freqai_datakitchen.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/freqai/test_freqai_datakitchen.py b/tests/freqai/test_freqai_datakitchen.py index 8d09cfc58..cac9d9838 100644 --- a/tests/freqai/test_freqai_datakitchen.py +++ b/tests/freqai/test_freqai_datakitchen.py @@ -10,9 +10,8 @@ from freqtrade.data.dataprovider import DataProvider from freqtrade.exceptions import OperationalException from freqtrade.freqai.data_kitchen import FreqaiDataKitchen from tests.conftest import get_patched_exchange -from tests.freqai.conftest import (get_patched_data_kitchen, get_patched_freqai_strategy, +from tests.freqai.conftest import (get_patched_data_kitchen, get_patched_freqai_strategy, is_mac, make_unfiltered_dataframe) -from tests.freqai.test_freqai_interface import is_mac @pytest.mark.parametrize( From 2745a5d334d1e9c1458372a4527003cf60d92e86 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 8 Dec 2023 20:07:44 +0100 Subject: [PATCH 78/80] Rename informative_pair test --- tests/strategy/test_strategy_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/strategy/test_strategy_helpers.py b/tests/strategy/test_strategy_helpers.py index 831856702..631752751 100644 --- a/tests/strategy/test_strategy_helpers.py +++ b/tests/strategy/test_strategy_helpers.py @@ -63,7 +63,7 @@ def test_merge_informative_pair(): assert result.iloc[8]['date_1h'] is pd.NaT -def test_merge_informative_pair_high_tf(): +def test_merge_informative_pair_monthly(): # Covers roughly 2 months - until 2023-01-10 data = generate_test_data('1h', 1040, '2022-11-28') informative = generate_test_data('1M', 40, '2022-01-01') From 7321a14c354b1fd2e84f7a7fb07527c0dc4772ea Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 8 Dec 2023 20:34:23 +0100 Subject: [PATCH 79/80] Fix generate-test-data for 1w data --- tests/conftest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 9b983882a..2d7a805b1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -91,6 +91,8 @@ def generate_test_data(timeframe: str, size: int, start: str = '2020-07-05'): base = np.random.normal(20, 2, size=size) if timeframe == '1M': date = pd.date_range(start, periods=size, freq='1MS', tz='UTC') + elif timeframe == '1w': + date = pd.date_range(start, periods=size, freq='1W-MON', tz='UTC') else: tf_mins = timeframe_to_minutes(timeframe) date = pd.date_range(start, periods=size, freq=f'{tf_mins}min', tz='UTC') From bb85efd6adb9ecb6c69be1efaecb5bc37b6e9c5b Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 8 Dec 2023 20:34:56 +0100 Subject: [PATCH 80/80] Add test for informative weekly merging closes #9518 --- tests/strategy/test_strategy_helpers.py | 28 +++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/strategy/test_strategy_helpers.py b/tests/strategy/test_strategy_helpers.py index 631752751..22c7359bf 100644 --- a/tests/strategy/test_strategy_helpers.py +++ b/tests/strategy/test_strategy_helpers.py @@ -63,6 +63,34 @@ def test_merge_informative_pair(): assert result.iloc[8]['date_1h'] is pd.NaT +def test_merge_informative_pair_weekly(): + # Covers roughly 2 months - until 2023-01-10 + data = generate_test_data('1h', 1040, '2022-11-28') + informative = generate_test_data('1w', 40, '2022-11-01') + informative['day'] = informative['date'].dt.day_name() + + result = merge_informative_pair(data, informative, '1h', '1w', ffill=True) + assert isinstance(result, pd.DataFrame) + # 2022-12-24 is a Saturday + candle1 = result.loc[(result['date'] == '2022-12-24T22:00:00.000Z')] + assert candle1.iloc[0]['date'] == pd.Timestamp('2022-12-24T22:00:00.000Z') + assert candle1.iloc[0]['date_1w'] == pd.Timestamp('2022-12-12T00:00:00.000Z') + + candle2 = result.loc[(result['date'] == '2022-12-24T23:00:00.000Z')] + assert candle2.iloc[0]['date'] == pd.Timestamp('2022-12-24T23:00:00.000Z') + assert candle2.iloc[0]['date_1w'] == pd.Timestamp('2022-12-12T00:00:00.000Z') + + # 2022-12-25 is a Sunday + candle3 = result.loc[(result['date'] == '2022-12-25T22:00:00.000Z')] + assert candle3.iloc[0]['date'] == pd.Timestamp('2022-12-25T22:00:00.000Z') + # Still old candle + assert candle3.iloc[0]['date_1w'] == pd.Timestamp('2022-12-12T00:00:00.000Z') + + candle4 = result.loc[(result['date'] == '2022-12-25T23:00:00.000Z')] + assert candle4.iloc[0]['date'] == pd.Timestamp('2022-12-25T23:00:00.000Z') + assert candle4.iloc[0]['date_1w'] == pd.Timestamp('2022-12-19T00:00:00.000Z') + + def test_merge_informative_pair_monthly(): # Covers roughly 2 months - until 2023-01-10 data = generate_test_data('1h', 1040, '2022-11-28')