Merge remote-tracking branch 'upstream/develop' into feature/fetch-public-trades

tests/freqai/conftest.py

@@ -25,10 +25,15 @@ def is_mac() -> bool:
     return "Darwin" in machine
 
 
+def is_arm() -> bool:
+    machine = platform.machine()
+    return "arm" in machine or "aarch64" in machine
+
+
 @pytest.fixture(autouse=True)
 def patch_torch_initlogs(mocker) -> None:
 
-    if is_mac():
+    if is_mac() and not is_arm():
         # Mock torch import completely
         import sys
         import types

tests/freqai/test_freqai_interface.py

@@ -1,5 +1,4 @@
 import logging
-import platform
 import shutil
 from pathlib import Path
 from unittest.mock import MagicMock
@@ -15,19 +14,14 @@ from freqtrade.optimize.backtesting import Backtesting
 from freqtrade.persistence import Trade
 from freqtrade.plugins.pairlistmanager import PairListManager
 from tests.conftest import EXMS, create_mock_trades, get_patched_exchange, log_has_re
-from tests.freqai.conftest import (get_patched_freqai_strategy, is_mac, is_py12, make_rl_config,
-                                   mock_pytorch_mlp_model_training_parameters)
-
-
-def is_arm() -> bool:
-    machine = platform.machine()
-    return "arm" in machine or "aarch64" in machine
+from tests.freqai.conftest import (get_patched_freqai_strategy, is_arm, is_mac, is_py12,
+                                   make_rl_config, mock_pytorch_mlp_model_training_parameters)
 
 
 def can_run_model(model: str) -> None:
     is_pytorch_model = 'Reinforcement' in model or 'PyTorch' in model
 
-    if is_py12() and ("Catboost" in model or is_pytorch_model):
+    if is_py12() and is_pytorch_model:
         pytest.skip("Model not supported on python 3.12 yet.")
 
     if is_arm() and "Catboost" in model:
@@ -243,7 +237,7 @@ def test_extract_data_and_train_model_Classifiers(mocker, freqai_conf, model):
 def test_start_backtesting(mocker, freqai_conf, model, num_files, strat, caplog):
     can_run_model(model)
     test_tb = True
-    if is_mac():
+    if is_mac() and not is_arm():
         test_tb = False
 
     freqai_conf.get("freqai", {}).update({"save_backtest_models": True})

tests/persistence/test_persistence.py

@@ -2099,6 +2099,7 @@ def test_Trade_object_idem():
         'get_mix_tag_performance',
         'get_trading_volume',
         'validate_string_len',
+        'custom_data'
     )
     EXCLUDES2 = ('trades', 'trades_open', 'bt_trades_open_pp', 'bt_open_open_trade_count',
                  'total_profit', 'from_json',)

tests/persistence/test_trade_custom_data.py

@@ -0,0 +1,160 @@
+from copy import deepcopy
+from unittest.mock import MagicMock
+
+import pytest
+
+from freqtrade.data.history.history_utils import get_timerange
+from freqtrade.optimize.backtesting import Backtesting
+from freqtrade.persistence import Trade, disable_database_use, enable_database_use
+from freqtrade.persistence.custom_data import CustomDataWrapper
+from tests.conftest import (EXMS, create_mock_trades_usdt, generate_test_data,
+                            get_patched_freqtradebot, patch_exchange)
+
+
+@pytest.mark.usefixtures("init_persistence")
+@pytest.mark.parametrize("use_db", [True, False])
+def test_trade_custom_data(fee, use_db):
+    if not use_db:
+        disable_database_use('5m')
+    Trade.reset_trades()
+    CustomDataWrapper.reset_custom_data()
+
+    create_mock_trades_usdt(fee, use_db=use_db)
+
+    trade1 = Trade.get_trades_proxy()[0]
+    if not use_db:
+        trade1.id = 1
+
+    assert trade1.get_all_custom_data() == []
+    trade1.set_custom_data('test_str', 'test_value')
+    trade1.set_custom_data('test_int', 1)
+    trade1.set_custom_data('test_float', 1.55)
+    trade1.set_custom_data('test_bool', True)
+    trade1.set_custom_data('test_dict', {'test': 'dict'})
+
+    assert len(trade1.get_all_custom_data()) == 5
+    assert trade1.get_custom_data('test_str') == 'test_value'
+    trade1.set_custom_data('test_str', 'test_value_updated')
+    assert trade1.get_custom_data('test_str') == 'test_value_updated'
+
+    assert trade1.get_custom_data('test_int') == 1
+    assert isinstance(trade1.get_custom_data('test_int'), int)
+
+    assert trade1.get_custom_data('test_float') == 1.55
+    assert isinstance(trade1.get_custom_data('test_float'), float)
+
+    assert trade1.get_custom_data('test_bool') is True
+    assert isinstance(trade1.get_custom_data('test_bool'), bool)
+
+    assert trade1.get_custom_data('test_dict') == {'test': 'dict'}
+    assert isinstance(trade1.get_custom_data('test_dict'), dict)
+    if not use_db:
+        enable_database_use()
+
+
+def test_trade_custom_data_strategy_compat(mocker, default_conf_usdt, fee):
+
+    mocker.patch(f'{EXMS}.get_rate', return_value=0.50)
+    mocker.patch('freqtrade.freqtradebot.FreqtradeBot.get_real_amount', return_value=None)
+    default_conf_usdt["minimal_roi"] = {"0":  100}
+
+    freqtrade = get_patched_freqtradebot(mocker, default_conf_usdt)
+    create_mock_trades_usdt(fee)
+
+    trade1 = Trade.get_trades_proxy(pair='ADA/USDT')[0]
+    trade1.set_custom_data('test_str', 'test_value')
+    trade1.set_custom_data('test_int', 1)
+
+    def custom_exit(pair, trade, **kwargs):
+
+        if pair == 'ADA/USDT':
+            custom_val = trade.get_custom_data('test_str')
+            custom_val_i = trade.get_custom_data('test_int')
+
+            return f"{custom_val}_{custom_val_i}"
+
+    freqtrade.strategy.custom_exit = custom_exit
+    ff_spy = mocker.spy(freqtrade.strategy, 'custom_exit')
+    trades = Trade.get_open_trades()
+    freqtrade.exit_positions(trades)
+    Trade.commit()
+
+    trade_after = Trade.get_trades_proxy(pair='ADA/USDT')[0]
+    assert trade_after.get_custom_data('test_str') == 'test_value'
+    assert trade_after.get_custom_data('test_int') == 1
+    # 2 open pairs eligible for exit
+    assert ff_spy.call_count == 2
+
+    assert trade_after.exit_reason == 'test_value_1'
+
+
+def test_trade_custom_data_strategy_backtest_compat(mocker, default_conf_usdt, fee):
+
+    mocker.patch(f'{EXMS}.get_fee', fee)
+    mocker.patch(f"{EXMS}.get_min_pair_stake_amount", return_value=10)
+    mocker.patch(f"{EXMS}.get_max_pair_stake_amount", return_value=float('inf'))
+    mocker.patch(f"{EXMS}.get_max_leverage", return_value=10)
+    mocker.patch(f"{EXMS}.get_maintenance_ratio_and_amt", return_value=(0.1, 0.1))
+    mocker.patch('freqtrade.optimize.backtesting.Backtesting._run_funding_fees')
+
+    patch_exchange(mocker)
+    default_conf_usdt.update({
+        "stake_amount": 100.0,
+        "max_open_trades": 2,
+        "dry_run_wallet": 1000.0,
+        "strategy": "StrategyTestV3",
+        "trading_mode": "futures",
+        "margin_mode": "isolated",
+        "stoploss": -2,
+        "minimal_roi": {"0":  100},
+    })
+    default_conf_usdt['pairlists'] = [{'method': 'StaticPairList', 'allow_inactive': True}]
+    backtesting = Backtesting(default_conf_usdt)
+
+    df = generate_test_data(default_conf_usdt['timeframe'], 100, '2022-01-01 00:00:00+00:00')
+
+    pair_exp = 'XRP/USDT:USDT'
+
+    def custom_exit(pair, trade, **kwargs):
+        custom_val = trade.get_custom_data('test_str')
+        custom_val_i = trade.get_custom_data('test_int', 0)
+
+        if pair == pair_exp:
+            trade.set_custom_data('test_str', 'test_value')
+            trade.set_custom_data('test_int', custom_val_i + 1)
+
+        if custom_val_i >= 2:
+            return f"{custom_val}_{custom_val_i}"
+
+    backtesting._set_strategy(backtesting.strategylist[0])
+    processed = backtesting.strategy.advise_all_indicators({
+        pair_exp: df,
+        'BTC/USDT:USDT': df,
+    })
+
+    def fun(dataframe, *args, **kwargs):
+        dataframe.loc[dataframe.index == 50, 'enter_long'] = 1
+        return dataframe
+
+    backtesting.strategy.advise_entry = fun
+    backtesting.strategy.leverage = MagicMock(return_value=1)
+    backtesting.strategy.custom_exit = custom_exit
+    ff_spy = mocker.spy(backtesting.strategy, 'custom_exit')
+
+    min_date, max_date = get_timerange(processed)
+
+    result = backtesting.backtest(
+        processed=deepcopy(processed),
+        start_date=min_date,
+        end_date=max_date,
+    )
+    results = result['results']
+    assert not results.empty
+    assert len(results) == 2
+    assert results['pair'][0] == pair_exp
+    assert results['pair'][1] == 'BTC/USDT:USDT'
+    assert results['exit_reason'][0] == 'test_value_2'
+    assert results['exit_reason'][1] == 'exit_signal'
+
+    assert ff_spy.call_count == 7
+    Backtesting.cleanup()

tests/rpc/test_rpc.py

@@ -10,6 +10,7 @@ from freqtrade.edge import PairInfo
 from freqtrade.enums import SignalDirection, State, TradingMode
 from freqtrade.exceptions import ExchangeError, InvalidOrderException, TemporaryError
 from freqtrade.persistence import Order, Trade
+from freqtrade.persistence.key_value_store import set_startup_time
 from freqtrade.persistence.pairlock_middleware import PairLocks
 from freqtrade.rpc import RPC, RPCException
 from freqtrade.rpc.fiat_convert import CryptoToFiatConverter
@@ -223,8 +224,8 @@ def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None:
     assert "Pair" in headers
     assert 'instantly' == result[0][2]
     assert 'ETH/BTC' in result[0][1]
-    assert '0.00' == result[0][3]
-    assert isnan(fiat_profit_sum)
+    assert '0.00 (0.00)' == result[0][3]
+    assert '0.00' == f'{fiat_profit_sum:.2f}'
 
     mocker.patch(f'{EXMS}._dry_is_price_crossed', return_value=True)
     freqtradebot.process()
@@ -234,8 +235,8 @@ def test_rpc_status_table(default_conf, ticker, fee, mocker) -> None:
     assert "Pair" in headers
     assert 'instantly' == result[0][2]
     assert 'ETH/BTC' in result[0][1]
-    assert '-0.41%' == result[0][3]
-    assert isnan(fiat_profit_sum)
+    assert '-0.41% (-0.00)' == result[0][3]
+    assert '-0.00' == f'{fiat_profit_sum:.2f}'
 
     # Test with fiat convert
     rpc._fiat_converter = CryptoToFiatConverter()
@@ -1298,6 +1299,7 @@ def test_rpc_health(mocker, default_conf) -> None:
     mocker.patch('freqtrade.rpc.telegram.Telegram', MagicMock())
 
     freqtradebot = get_patched_freqtradebot(mocker, default_conf)
+    set_startup_time()
     rpc = RPC(freqtradebot)
     result = rpc.health()
     assert result['last_process'] is None

tests/rpc/test_rpc_telegram.py

@@ -150,7 +150,7 @@ def test_telegram_init(default_conf, mocker, caplog) -> None:
                    "['stopbuy', 'stopentry'], ['whitelist'], ['blacklist'], "
                    "['bl_delete', 'blacklist_delete'], "
                    "['logs'], ['edge'], ['health'], ['help'], ['version'], ['marketdir'], "
-                   "['order']]")
+                   "['order'], ['list_custom_data']]")
 
     assert log_has(message_str, caplog)
 
@@ -2657,3 +2657,49 @@ async def test_change_market_direction(default_conf, mocker, update) -> None:
     context.args = ["invalid"]
     await telegram._changemarketdir(update, context)
     assert telegram._rpc._freqtrade.strategy.market_direction == MarketDirection.LONG
+
+
+async def test_telegram_list_custom_data(default_conf_usdt, update, ticker, fee, mocker) -> None:
+
+    mocker.patch.multiple(
+        EXMS,
+        fetch_ticker=ticker,
+        get_fee=fee,
+    )
+    telegram, _freqtradebot, msg_mock = get_telegram_testobject(mocker, default_conf_usdt)
+
+    # Create some test data
+    create_mock_trades_usdt(fee)
+    # No trade id
+    context = MagicMock()
+    await telegram._list_custom_data(update=update, context=context)
+    assert msg_mock.call_count == 1
+    assert 'Trade-id not set.' in msg_mock.call_args_list[0][0][0]
+    msg_mock.reset_mock()
+
+    #
+    context.args = ['1']
+    await telegram._list_custom_data(update=update, context=context)
+    assert msg_mock.call_count == 1
+    assert (
+        "Didn't find any custom-data entries for Trade ID: `1`" in msg_mock.call_args_list[0][0][0]
+    )
+    msg_mock.reset_mock()
+
+    # Add some custom data
+    trade1 = Trade.get_trades_proxy()[0]
+    trade1.set_custom_data('test_int', 1)
+    trade1.set_custom_data('test_dict', {'test': 'dict'})
+    Trade.commit()
+    context.args = [f"{trade1.id}"]
+    await telegram._list_custom_data(update=update, context=context)
+    assert msg_mock.call_count == 3
+    assert "Found custom-data entries: " in msg_mock.call_args_list[0][0][0]
+    assert (
+        "*Key:* `test_int`\n*ID:* `1`\n*Trade ID:* `1`\n*Type:* `int`\n"
+        "*Value:* `1`\n*Create Date:*") in msg_mock.call_args_list[1][0][0]
+    assert (
+        '*Key:* `test_dict`\n*ID:* `2`\n*Trade ID:* `1`\n*Type:* `dict`\n'
+        '*Value:* `{"test": "dict"}`\n*Create Date:* `') in msg_mock.call_args_list[2][0][0]
+
+    msg_mock.reset_mock()