Merge pull request #9152 from stash86/bt-metrics

Add recursive-analysis sub-command

tests/commands/test_commands.py

@@ -1556,7 +1556,7 @@ def test_start_strategy_updater(mocker, tmpdir):
     pargs['config'] = None
     start_strategy_update(pargs)
     # Number of strategies in the test directory
-    assert sc_mock.call_count == 11
+    assert sc_mock.call_count == 12
 
     sc_mock.reset_mock()
     args = [

tests/optimize/test_recursive_analysis.py

@@ -0,0 +1,188 @@
+# pragma pylint: disable=missing-docstring, W0212, line-too-long, C0103, unused-argument
+from copy import deepcopy
+from pathlib import Path
+from unittest.mock import MagicMock, PropertyMock
+
+import pytest
+
+from freqtrade.commands.optimize_commands import start_recursive_analysis
+from freqtrade.data.history import get_timerange
+from freqtrade.exceptions import OperationalException
+from freqtrade.optimize.recursive_analysis import RecursiveAnalysis
+from freqtrade.optimize.recursive_analysis_helpers import RecursiveAnalysisSubFunctions
+from tests.conftest import get_args, log_has_re, patch_exchange
+
+
+@pytest.fixture
+def recursive_conf(default_conf_usdt):
+    default_conf_usdt['timerange'] = '20220101-20220501'
+
+    default_conf_usdt['strategy_path'] = str(
+        Path(__file__).parent.parent / "strategy/strats")
+    default_conf_usdt['strategy'] = 'strategy_test_v3_recursive_issue'
+    default_conf_usdt['pairs'] = ['UNITTEST/USDT']
+    default_conf_usdt['startup_candle'] = [100]
+    return default_conf_usdt
+
+
+def test_start_recursive_analysis(mocker):
+    single_mock = MagicMock()
+    text_table_mock = MagicMock()
+    mocker.patch.multiple(
+        'freqtrade.optimize.recursive_analysis_helpers.RecursiveAnalysisSubFunctions',
+        initialize_single_recursive_analysis=single_mock,
+        text_table_recursive_analysis_instances=text_table_mock,
+    )
+    args = [
+        "recursive-analysis",
+        "--strategy",
+        "strategy_test_v3_recursive_issue",
+        "--strategy-path",
+        str(Path(__file__).parent.parent / "strategy/strats"),
+        "--pairs",
+        "UNITTEST/BTC",
+        "--timerange",
+        "20220101-20220201"
+    ]
+    pargs = get_args(args)
+    pargs['config'] = None
+
+    start_recursive_analysis(pargs)
+    assert single_mock.call_count == 1
+    assert text_table_mock.call_count == 1
+
+    single_mock.reset_mock()
+
+    # Missing timerange
+    args = [
+        "recursive-analysis",
+        "--strategy",
+        "strategy_test_v3_with_recursive_bias",
+        "--strategy-path",
+        str(Path(__file__).parent.parent / "strategy/strats"),
+        "--pairs",
+        "UNITTEST/BTC"
+    ]
+    pargs = get_args(args)
+    pargs['config'] = None
+    with pytest.raises(OperationalException,
+                       match=r"Please set a timerange\..*"):
+        start_recursive_analysis(pargs)
+
+
+def test_recursive_helper_no_strategy_defined(recursive_conf):
+    conf = deepcopy(recursive_conf)
+    conf['pairs'] = ['UNITTEST/USDT']
+    del conf['strategy']
+    with pytest.raises(OperationalException,
+                       match=r"No Strategy specified"):
+        RecursiveAnalysisSubFunctions.start(conf)
+
+
+def test_recursive_helper_start(recursive_conf, mocker) -> None:
+    single_mock = MagicMock()
+    text_table_mock = MagicMock()
+    mocker.patch.multiple(
+        'freqtrade.optimize.recursive_analysis_helpers.RecursiveAnalysisSubFunctions',
+        initialize_single_recursive_analysis=single_mock,
+        text_table_recursive_analysis_instances=text_table_mock,
+    )
+    RecursiveAnalysisSubFunctions.start(recursive_conf)
+    assert single_mock.call_count == 1
+    assert text_table_mock.call_count == 1
+
+    single_mock.reset_mock()
+    text_table_mock.reset_mock()
+
+
+def test_recursive_helper_text_table_recursive_analysis_instances(recursive_conf):
+    dict_diff = dict()
+    dict_diff['rsi'] = {}
+    dict_diff['rsi'][100] = "0.078%"
+
+    strategy_obj = {
+        'name': "strategy_test_v3_recursive_issue",
+        'location': Path(recursive_conf['strategy_path'], f"{recursive_conf['strategy']}.py")
+    }
+
+    instance = RecursiveAnalysis(recursive_conf, strategy_obj)
+    instance.dict_recursive = dict_diff
+    table, headers, data = (RecursiveAnalysisSubFunctions.
+                            text_table_recursive_analysis_instances([instance]))
+
+    # check row contents for a try that has too few signals
+    assert data[0][0] == 'rsi'
+    assert data[0][1] == '0.078%'
+    assert len(data[0]) == 2
+
+    # now check when there is no issue
+    dict_diff = dict()
+    instance = RecursiveAnalysis(recursive_conf, strategy_obj)
+    instance.dict_recursive = dict_diff
+    table, headers, data = (RecursiveAnalysisSubFunctions.
+                            text_table_recursive_analysis_instances([instance]))
+    assert len(data) == 0
+
+
+def test_initialize_single_recursive_analysis(recursive_conf, mocker, caplog):
+    mocker.patch('freqtrade.data.history.get_timerange', get_timerange)
+    patch_exchange(mocker)
+    mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist',
+                 PropertyMock(return_value=['UNITTEST/BTC']))
+    recursive_conf['pairs'] = ['UNITTEST/BTC']
+
+    recursive_conf['timeframe'] = '5m'
+    recursive_conf['timerange'] = '20180119-20180122'
+    start_mock = mocker.patch('freqtrade.optimize.recursive_analysis.RecursiveAnalysis.start')
+    strategy_obj = {
+        'name': "strategy_test_v3_recursive_issue",
+        'location': Path(recursive_conf['strategy_path'], f"{recursive_conf['strategy']}.py")
+    }
+
+    instance = RecursiveAnalysisSubFunctions.initialize_single_recursive_analysis(
+        recursive_conf, strategy_obj)
+    assert log_has_re(r"Recursive test of .* started\.", caplog)
+    assert start_mock.call_count == 1
+
+    assert instance.strategy_obj['name'] == "strategy_test_v3_recursive_issue"
+
+
+@pytest.mark.parametrize('scenario', [
+    'no_bias', 'bias1', 'bias2'
+])
+def test_recursive_biased_strategy(recursive_conf, mocker, caplog, scenario) -> None:
+    mocker.patch('freqtrade.data.history.get_timerange', get_timerange)
+    patch_exchange(mocker)
+    mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist',
+                 PropertyMock(return_value=['UNITTEST/BTC']))
+    recursive_conf['pairs'] = ['UNITTEST/BTC']
+
+    recursive_conf['timeframe'] = '5m'
+    recursive_conf['timerange'] = '20180119-20180122'
+    recursive_conf['startup_candle'] = [100]
+
+    # Patch scenario Parameter to allow for easy selection
+    mocker.patch('freqtrade.strategy.hyper.HyperStrategyMixin.load_params_from_file',
+                 return_value={
+                     'params': {
+                         "buy": {
+                             "scenario": scenario
+                         }
+                     }
+                 })
+
+    strategy_obj = {'name': "strategy_test_v3_recursive_issue"}
+    instance = RecursiveAnalysis(recursive_conf, strategy_obj)
+    instance.start()
+    # Assert init correct
+    assert log_has_re(f"Strategy Parameter: scenario = {scenario}", caplog)
+
+    if scenario == "bias2":
+        assert log_has_re("=> found lookahead in indicator rsi", caplog)
+    diff_pct = abs(float(instance.dict_recursive['rsi'][100].replace("%", "")))
+    # check non-biased strategy
+    if scenario == "no_bias":
+        assert diff_pct < 0.01
+    # check biased strategy
+    elif scenario in ("bias1", "bias2"):
+        assert diff_pct >= 0.01

tests/rpc/test_rpc_apiserver.py

@@ -1636,7 +1636,8 @@ def test_api_strategies(botclient, tmpdir):
         'freqai_test_classifier',
         'freqai_test_multimodel_classifier_strat',
         'freqai_test_multimodel_strat',
-        'freqai_test_strat'
+        'freqai_test_strat',
+        'strategy_test_v3_recursive_issue'
     ]}
 
 

tests/strategy/strats/strategy_test_v3_recursive_issue.py

@@ -0,0 +1,46 @@
+# pragma pylint: disable=missing-docstring, invalid-name, pointless-string-statement
+import talib.abstract as ta
+from pandas import DataFrame
+
+from freqtrade.strategy import IStrategy
+from freqtrade.strategy.parameters import CategoricalParameter
+
+
+class strategy_test_v3_recursive_issue(IStrategy):
+    INTERFACE_VERSION = 3
+
+    # Minimal ROI designed for the strategy
+    minimal_roi = {
+        "0": 0.04
+    }
+
+    # Optimal stoploss designed for the strategy
+    stoploss = -0.10
+
+    # Optimal timeframe for the strategy
+    timeframe = '5m'
+    scenario = CategoricalParameter(['no_bias', 'bias1', 'bias2'], default='bias1', space="buy")
+
+    # Number of candles the strategy requires before producing valid signals
+    startup_candle_count: int = 100
+
+    def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
+        # bias is introduced here
+        if self.scenario.value == 'no_bias':
+            dataframe['rsi'] = ta.RSI(dataframe, timeperiod=14)
+        else:
+            dataframe['rsi'] = ta.RSI(dataframe, timeperiod=50)
+
+        if self.scenario.value == 'bias2':
+            # Has both bias1 and bias2
+            dataframe['rsi_lookahead'] = ta.RSI(dataframe, timeperiod=50).shift(-1)
+
+        return dataframe
+
+    def populate_entry_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
+
+        return dataframe
+
+    def populate_exit_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
+
+        return dataframe

tests/strategy/test_strategy_loading.py

@@ -35,7 +35,7 @@ def test_search_all_strategies_no_failed():
     directory = Path(__file__).parent / "strats"
     strategies = StrategyResolver._search_all_objects(directory, enum_failed=False)
     assert isinstance(strategies, list)
-    assert len(strategies) == 12
+    assert len(strategies) == 13
     assert isinstance(strategies[0], dict)
 
 
@@ -43,10 +43,10 @@ def test_search_all_strategies_with_failed():
     directory = Path(__file__).parent / "strats"
     strategies = StrategyResolver._search_all_objects(directory, enum_failed=True)
     assert isinstance(strategies, list)
-    assert len(strategies) == 13
+    assert len(strategies) == 14
     # with enum_failed=True search_all_objects() shall find 2 good strategies
     # and 1 which fails to load
-    assert len([x for x in strategies if x['class'] is not None]) == 12
+    assert len([x for x in strategies if x['class'] is not None]) == 13
 
     assert len([x for x in strategies if x['class'] is None]) == 1