CTYPTO/freqtrade
1
0
Fork 0
mirror of https://github.com/freqtrade/freqtrade.git synced 2026-04-29 05:20:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #9552 from thojou/fix-freqai-populate-features-timerange

Browse Source 
Fix duplicated data loading and timerange for populate_features
This commit is contained in:
Robert Caulk
2023-12-30 13:42:08 +01:00
committed by GitHub
parent 7ccddb5d58 3f44f56f6f
commit ea8edbd23b
7 changed files with 146 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
tests/data/test_dataprovider.py
View File

@@ -508,16 +508,13 @@ def test_dp_get_required_startup(default_conf_usdt):
dp = DataProvider(default_conf_usdt, None)
# No FreqAI config
assert dp.get_required_startup('5m', False) == 0
assert dp.get_required_startup('1h', False) == 0
assert dp.get_required_startup('1d', False) == 0
assert dp.get_required_startup('1d', True) == 0
assert dp.get_required_startup('5m') == 0
assert dp.get_required_startup('1h') == 0
assert dp.get_required_startup('1d') == 0
dp._config['startup_candle_count'] = 20
assert dp.get_required_startup('5m', False) == 20
assert dp.get_required_startup('5m', True) == 20
assert dp.get_required_startup('1h', False) == 20
assert dp.get_required_startup('5m') == 20
assert dp.get_required_startup('1h') == 20
assert dp.get_required_startup('1h') == 20
# With freqAI config
@@ -532,37 +529,19 @@ def test_dp_get_required_startup(default_conf_usdt):
]
}
}
assert dp.get_required_startup('5m', False) == 20
assert dp.get_required_startup('5m', True) == 5780
assert dp.get_required_startup('1h', False) == 20
assert dp.get_required_startup('1h', True) == 500
assert dp.get_required_startup('1d', False) == 20
assert dp.get_required_startup('1d', True) == 40
assert dp.get_required_startup('5m') == 5780
assert dp.get_required_startup('1h') == 500
assert dp.get_required_startup('1d') == 40
# FreqAI kindof ignores startup_candle_count if it's below indicator_periods_candles
dp._config['startup_candle_count'] = 0
assert dp.get_required_startup('5m', False) == 20
assert dp.get_required_startup('5m', True) == 5780
assert dp.get_required_startup('1h', False) == 20
assert dp.get_required_startup('1h', True) == 500
assert dp.get_required_startup('1d', False) == 20
assert dp.get_required_startup('1d', True) == 40
assert dp.get_required_startup('5m') == 5780
assert dp.get_required_startup('1h') == 500
assert dp.get_required_startup('1d') == 40
dp._config['freqai']['feature_parameters']['indicator_periods_candles'][1] = 50
assert dp.get_required_startup('5m', False) == 50
assert dp.get_required_startup('5m', True) == 5810
assert dp.get_required_startup('1h', False) == 50
assert dp.get_required_startup('1h', True) == 530
assert dp.get_required_startup('1d', False) == 50
assert dp.get_required_startup('1d', True) == 70
assert dp.get_required_startup('5m') == 5810
assert dp.get_required_startup('1h') == 530
assert dp.get_required_startup('1d') == 70
# scenario from issue https://github.com/freqtrade/freqtrade/issues/9432
@@ -577,12 +556,6 @@ def test_dp_get_required_startup(default_conf_usdt):
}
}
dp._config['startup_candle_count'] = 40
assert dp.get_required_startup('5m', False) == 40
assert dp.get_required_startup('5m', True) == 51880
assert dp.get_required_startup('1h', False) == 40
assert dp.get_required_startup('1h', True) == 4360
assert dp.get_required_startup('1d', False) == 40
assert dp.get_required_startup('1d', True) == 220
assert dp.get_required_startup('5m') == 51880
assert dp.get_required_startup('1h') == 4360
assert dp.get_required_startup('1d') == 220

57
tests/freqai/test_freqai_backtesting.py
View File

@@ -6,11 +6,17 @@ from unittest.mock import PropertyMock
import pytest
from freqtrade.commands.optimize_commands import setup_optimize_configuration
from freqtrade.configuration.timerange import TimeRange
from freqtrade.data import history
from freqtrade.data.dataprovider import DataProvider
from freqtrade.enums import RunMode
from freqtrade.enums.candletype import CandleType
from freqtrade.exceptions import OperationalException
from freqtrade.freqai.data_kitchen import FreqaiDataKitchen
from freqtrade.optimize.backtesting import Backtesting
from tests.conftest import (CURRENT_TEST_STRATEGY, get_args, log_has_re, patch_exchange,
patched_configuration_load_config_file)
from tests.conftest import (CURRENT_TEST_STRATEGY, get_args, get_patched_exchange, log_has_re,
patch_exchange, patched_configuration_load_config_file)
from tests.freqai.conftest import get_patched_freqai_strategy
def test_freqai_backtest_start_backtest_list(freqai_conf, mocker, testdatadir, caplog):
@@ -40,7 +46,16 @@ def test_freqai_backtest_start_backtest_list(freqai_conf, mocker, testdatadir, c
Backtesting.cleanup()
def test_freqai_backtest_load_data(freqai_conf, mocker, caplog):
@pytest.mark.parametrize(
"timeframe, expected_startup_candle_count",
[
("5m", 876),
("15m", 492),
("1d", 302),
],
)
def test_freqai_backtest_load_data(freqai_conf, mocker, caplog,
timeframe, expected_startup_candle_count):
patch_exchange(mocker)
now = datetime.now(timezone.utc)
@@ -48,10 +63,14 @@ def test_freqai_backtest_load_data(freqai_conf, mocker, caplog):
PropertyMock(return_value=['HULUMULU/USDT', 'XRP/USDT']))
mocker.patch('freqtrade.optimize.backtesting.history.load_data')
mocker.patch('freqtrade.optimize.backtesting.history.get_timerange', return_value=(now, now))
freqai_conf['timeframe'] = timeframe
freqai_conf.get('freqai', {}).get('feature_parameters', {}).update({'include_timeframes': []})
backtesting = Backtesting(deepcopy(freqai_conf))
backtesting.load_bt_data()
assert log_has_re('Increasing startup_candle_count for freqai to.*', caplog)
assert log_has_re(f'Increasing startup_candle_count for freqai on {timeframe} '
f'to {expected_startup_candle_count}', caplog)
assert history.load_data.call_args[1]['startup_candles'] == expected_startup_candle_count
Backtesting.cleanup()
@@ -85,3 +104,33 @@ def test_freqai_backtest_live_models_model_not_found(freqai_conf, mocker, testda
Backtesting(bt_config)
Backtesting.cleanup()
def test_freqai_backtest_consistent_timerange(mocker, freqai_conf):
mocker.patch('freqtrade.plugins.pairlistmanager.PairListManager.whitelist',
PropertyMock(return_value=['XRP/USDT:USDT']))
gbs = mocker.patch('freqtrade.optimize.backtesting.generate_backtest_stats')
freqai_conf['candle_type_def'] = CandleType.FUTURES
freqai_conf.get('exchange', {}).update({'pair_whitelist': ['XRP/USDT:USDT']})
freqai_conf.get('freqai', {}).get('feature_parameters', {}).update(
{'include_timeframes': ['5m', '1h'], 'include_corr_pairlist': []})
freqai_conf['timerange'] = '20211120-20211121'
strategy = get_patched_freqai_strategy(mocker, freqai_conf)
exchange = get_patched_exchange(mocker, freqai_conf)
strategy.dp = DataProvider(freqai_conf, exchange)
strategy.freqai_info = freqai_conf.get("freqai", {})
freqai = strategy.freqai
freqai.dk = FreqaiDataKitchen(freqai_conf)
timerange = TimeRange.parse_timerange("20211115-20211122")
freqai.dd.load_all_pair_histories(timerange, freqai.dk)
backtesting = Backtesting(deepcopy(freqai_conf))
backtesting.start()
gbs.call_args[1]['min_date'] == datetime(2021, 11, 20, 0, 0, tzinfo=timezone.utc)
gbs.call_args[1]['max_date'] == datetime(2021, 11, 21, 0, 0, tzinfo=timezone.utc)

61
tests/freqai/test_freqai_datakitchen.py
View File

@@ -3,6 +3,7 @@ from datetime import datetime, timedelta, timezone
from pathlib import Path
from unittest.mock import MagicMock
import pandas as pd
import pytest
from freqtrade.configuration import TimeRange
@@ -135,3 +136,63 @@ def test_get_full_model_path(mocker, freqai_conf, model):
model_path = freqai.dk.get_full_models_path(freqai_conf)
assert model_path.is_dir() is True
def test_get_pair_data_for_features_with_prealoaded_data(mocker, freqai_conf):
strategy = get_patched_freqai_strategy(mocker, freqai_conf)
exchange = get_patched_exchange(mocker, freqai_conf)
strategy.dp = DataProvider(freqai_conf, exchange)
strategy.freqai_info = freqai_conf.get("freqai", {})
freqai = strategy.freqai
freqai.dk = FreqaiDataKitchen(freqai_conf)
timerange = TimeRange.parse_timerange("20180110-20180130")
freqai.dd.load_all_pair_histories(timerange, freqai.dk)
_, base_df = freqai.dd.get_base_and_corr_dataframes(timerange, "LTC/BTC", freqai.dk)
df = freqai.dk.get_pair_data_for_features("LTC/BTC", "5m", strategy, base_dataframes=base_df)
assert df is base_df["5m"]
assert not df.empty
def test_get_pair_data_for_features_without_preloaded_data(mocker, freqai_conf):
freqai_conf.update({"timerange": "20180115-20180130"})
strategy = get_patched_freqai_strategy(mocker, freqai_conf)
exchange = get_patched_exchange(mocker, freqai_conf)
strategy.dp = DataProvider(freqai_conf, exchange)
strategy.freqai_info = freqai_conf.get("freqai", {})
freqai = strategy.freqai
freqai.dk = FreqaiDataKitchen(freqai_conf)
timerange = TimeRange.parse_timerange("20180110-20180130")
freqai.dd.load_all_pair_histories(timerange, freqai.dk)
base_df = {'5m': pd.DataFrame()}
df = freqai.dk.get_pair_data_for_features("LTC/BTC", "5m", strategy, base_dataframes=base_df)
assert df is not base_df["5m"]
assert not df.empty
assert df.iloc[0]['date'].strftime("%Y-%m-%d %H:%M:%S") == "2018-01-11 23:00:00"
assert df.iloc[-1]['date'].strftime("%Y-%m-%d %H:%M:%S") == "2018-01-30 00:00:00"
def test_populate_features(mocker, freqai_conf):
strategy = get_patched_freqai_strategy(mocker, freqai_conf)
exchange = get_patched_exchange(mocker, freqai_conf)
strategy.dp = DataProvider(freqai_conf, exchange)
strategy.freqai_info = freqai_conf.get("freqai", {})
freqai = strategy.freqai
freqai.dk = FreqaiDataKitchen(freqai_conf)
timerange = TimeRange.parse_timerange("20180115-20180130")
freqai.dd.load_all_pair_histories(timerange, freqai.dk)
corr_df, base_df = freqai.dd.get_base_and_corr_dataframes(timerange, "LTC/BTC", freqai.dk)
mocker.patch.object(strategy, 'feature_engineering_expand_all', return_value=base_df["5m"])
df = freqai.dk.populate_features(base_df["5m"], "LTC/BTC", strategy,
base_dataframes=base_df, corr_dataframes=corr_df)
strategy.feature_engineering_expand_all.assert_called_once()
pd.testing.assert_frame_equal(base_df["5m"],
strategy.feature_engineering_expand_all.call_args[0][0])
assert df.iloc[0]['date'].strftime("%Y-%m-%d %H:%M:%S") == "2018-01-15 00:00:00"