diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index 229373400..0eaf5e563 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -12,7 +12,7 @@ from freqtrade.enums import RunMode, TradingMode from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_minutes from freqtrade.resolvers import ExchangeResolver -from freqtrade.util.binance_mig import migrate_binance_futures_data +from freqtrade.util.migrations import migrate_data logger = logging.getLogger(__name__) @@ -78,7 +78,7 @@ def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None: """ config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) if ohlcv: - migrate_binance_futures_data(config) + migrate_data(config) convert_ohlcv_format(config, convert_from=args['format_from'], convert_to=args['format_to'], diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index 1ad1060a4..ca8a173bf 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -18,8 +18,8 @@ from freqtrade.exceptions import OperationalException from freqtrade.exchange import Exchange from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist from freqtrade.util import dt_ts, format_ms_time -from freqtrade.util.binance_mig import migrate_binance_futures_data from freqtrade.util.datetime_helpers import dt_now +from freqtrade.util.migrations import migrate_data logger = logging.getLogger(__name__) @@ -311,15 +311,19 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes # Predefined candletype (and timeframe) depending on exchange # Downloads what is necessary to backtest based on futures data. tf_mark = exchange.get_option('mark_ohlcv_timeframe') + tf_funding_rate = exchange.get_option('funding_fee_timeframe') + fr_candle_type = CandleType.from_string(exchange.get_option('mark_ohlcv_price')) # All exchanges need FundingRate for futures trading. # The timeframe is aligned to the mark-price timeframe. - for funding_candle_type in (CandleType.FUNDING_RATE, fr_candle_type): + combs = ((CandleType.FUNDING_RATE, tf_funding_rate), (fr_candle_type, tf_mark)) + for candle_type_f, tf in combs: + logger.debug(f'Downloading pair {pair}, {candle_type_f}, interval {tf}.') _download_pair_history(pair=pair, process=process, datadir=datadir, exchange=exchange, timerange=timerange, data_handler=data_handler, - timeframe=str(tf_mark), new_pairs_days=new_pairs_days, - candle_type=funding_candle_type, + timeframe=str(tf), new_pairs_days=new_pairs_days, + candle_type=candle_type_f, erase=erase, prepend=prepend) return pairs_not_available @@ -527,7 +531,7 @@ def download_data_main(config: Config) -> None: "Please use `--dl-trades` instead for this exchange " "(will unfortunately take a long time)." ) - migrate_binance_futures_data(config) + migrate_data(config, exchange) pairs_not_available = refresh_backtest_ohlcv_data( exchange, pairs=expanded_pairs, timeframes=config['timeframes'], datadir=config['datadir'], timerange=timerange, diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index d8c063f2a..3bfd485b9 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -403,6 +403,34 @@ class IDataHandler(ABC): return file_old.rename(file_new) + def fix_funding_fee_timeframe(self, ff_timeframe: str): + """ + Temporary method to migrate data from old funding fee timeframe to the correct timeframe + Applies to bybit and okx, where funding-fee and mark candles have different timeframes. + """ + paircombs = self.ohlcv_get_available_data(self._datadir, TradingMode.FUTURES) + funding_rate_combs = [ + f for f in paircombs if f[2] == CandleType.FUNDING_RATE and f[1] != ff_timeframe + ] + + if funding_rate_combs: + logger.warning( + f'Migrating {len(funding_rate_combs)} funding fees to correct timeframe.') + + for pair, timeframe, candletype in funding_rate_combs: + old_name = self._pair_data_filename(self._datadir, pair, timeframe, candletype) + new_name = self._pair_data_filename(self._datadir, pair, ff_timeframe, candletype) + + if not Path(old_name).exists(): + logger.warning(f'{old_name} does not exist, skipping.') + continue + + if Path(new_name).exists(): + logger.warning(f'{new_name} already exists, Removing.') + Path(new_name).unlink() + + Path(old_name).rename(new_name) + def get_datahandlerclass(datatype: str) -> Type[IDataHandler]: """ diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index cea7f57ec..664000eb2 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -80,6 +80,7 @@ class Exchange: "l2_limit_range_required": True, # Allow Empty L2 limit (kucoin) "mark_ohlcv_price": "mark", "mark_ohlcv_timeframe": "8h", + "funding_fee_timeframe": "8h", "ccxt_futures_name": "swap", "needs_trading_fees": False, # use fetch_trading_fees to cache fees "order_props_in_contracts": ['amount', 'filled', 'remaining'], @@ -2734,17 +2735,16 @@ class Exchange: # Only really relevant for trades very close to the full hour open_date = timeframe_to_prev_date('1h', open_date) timeframe = self._ft_has['mark_ohlcv_timeframe'] - timeframe_ff = self._ft_has.get('funding_fee_timeframe', - self._ft_has['mark_ohlcv_timeframe']) + timeframe_ff = self._ft_has['funding_fee_timeframe'] + mark_price_type = CandleType.from_string(self._ft_has["mark_ohlcv_price"]) if not close_date: close_date = datetime.now(timezone.utc) since_ms = int(timeframe_to_prev_date(timeframe, open_date).timestamp()) * 1000 - mark_comb: PairWithTimeframe = ( - pair, timeframe, CandleType.from_string(self._ft_has["mark_ohlcv_price"])) - + mark_comb: PairWithTimeframe = (pair, timeframe, mark_price_type) funding_comb: PairWithTimeframe = (pair, timeframe_ff, CandleType.FUNDING_RATE) + candle_histories = self.refresh_latest_ohlcv( [mark_comb, funding_comb], since_ms=since_ms, diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 98c7677a7..d344cf652 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -38,7 +38,7 @@ from freqtrade.rpc.rpc_types import (ProfitLossStr, RPCCancelMsg, RPCEntryMsg, R from freqtrade.strategy.interface import IStrategy from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper from freqtrade.util import FtPrecise -from freqtrade.util.binance_mig import migrate_binance_futures_names +from freqtrade.util.migrations import migrate_binance_futures_names from freqtrade.wallets import Wallets diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index e9d4cdd8a..ce37a0dcc 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -40,7 +40,7 @@ from freqtrade.resolvers import ExchangeResolver, StrategyResolver from freqtrade.strategy.interface import IStrategy from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper from freqtrade.types import BacktestResultType, get_BacktestResultType_default -from freqtrade.util.binance_mig import migrate_binance_futures_data +from freqtrade.util.migrations import migrate_data from freqtrade.wallets import Wallets @@ -158,7 +158,7 @@ class Backtesting: self._can_short = self.trading_mode != TradingMode.SPOT self._position_stacking: bool = self.config.get('position_stacking', False) self.enable_protections: bool = self.config.get('enable_protections', False) - migrate_binance_futures_data(config) + migrate_data(config, self.exchange) self.init_backtest() @@ -277,8 +277,10 @@ class Backtesting: else: self.detail_data = {} if self.trading_mode == TradingMode.FUTURES: - self.funding_fee_timeframe: str = self.exchange.get_option('mark_ohlcv_timeframe') + self.funding_fee_timeframe: str = self.exchange.get_option('funding_fee_timeframe') self.funding_fee_timeframe_secs: int = timeframe_to_seconds(self.funding_fee_timeframe) + mark_timeframe: str = self.exchange.get_option('mark_ohlcv_timeframe') + # Load additional futures data. funding_rates_dict = history.load_data( datadir=self.config['datadir'], @@ -295,7 +297,7 @@ class Backtesting: mark_rates_dict = history.load_data( datadir=self.config['datadir'], pairs=self.pairlists.whitelist, - timeframe=self.funding_fee_timeframe, + timeframe=mark_timeframe, timerange=self.timerange, startup_candles=0, fail_without_data=True, diff --git a/freqtrade/util/migrations/__init__.py b/freqtrade/util/migrations/__init__.py new file mode 100644 index 000000000..9bd6f6288 --- /dev/null +++ b/freqtrade/util/migrations/__init__.py @@ -0,0 +1,12 @@ +from typing import Optional + +from freqtrade.exchange import Exchange +from freqtrade.util.migrations.binance_mig import migrate_binance_futures_names # noqa F401 +from freqtrade.util.migrations.binance_mig import migrate_binance_futures_data +from freqtrade.util.migrations.funding_rate_mig import migrate_funding_fee_timeframe + + +def migrate_data(config, exchange: Optional[Exchange] = None): + migrate_binance_futures_data(config) + + migrate_funding_fee_timeframe(config, exchange) diff --git a/freqtrade/util/binance_mig.py b/freqtrade/util/migrations/binance_mig.py similarity index 100% rename from freqtrade/util/binance_mig.py rename to freqtrade/util/migrations/binance_mig.py diff --git a/freqtrade/util/migrations/funding_rate_mig.py b/freqtrade/util/migrations/funding_rate_mig.py new file mode 100644 index 000000000..9fe433b2d --- /dev/null +++ b/freqtrade/util/migrations/funding_rate_mig.py @@ -0,0 +1,27 @@ +import logging +from typing import Optional + +from freqtrade.constants import Config +from freqtrade.data.history.idatahandler import get_datahandler +from freqtrade.enums import TradingMode +from freqtrade.exchange import Exchange + + +logger = logging.getLogger(__name__) + + +def migrate_funding_fee_timeframe(config: Config, exchange: Optional[Exchange]): + if ( + config.get('trading_mode', TradingMode.SPOT) != TradingMode.FUTURES + ): + # only act on futures + return + + if not exchange: + from freqtrade.resolvers import ExchangeResolver + exchange = ExchangeResolver.load_exchange(config, validate=False) + + ff_timeframe = exchange.get_option('funding_fee_timeframe') + + dhc = get_datahandler(config['datadir'], config['dataformat_ohlcv']) + dhc.fix_funding_fee_timeframe(ff_timeframe) diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 100916387..a48d34aee 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -508,8 +508,9 @@ def test_refresh_backtest_ohlcv_data( mocker.patch.object(Path, "exists", MagicMock(return_value=True)) mocker.patch.object(Path, "unlink", MagicMock()) + default_conf['trading_mode'] = trademode - ex = get_patched_exchange(mocker, default_conf) + ex = get_patched_exchange(mocker, default_conf, id='bybit') timerange = TimeRange.parse_timerange("20190101-20190102") refresh_backtest_ohlcv_data(exchange=ex, pairs=["ETH/BTC", "XRP/BTC"], timeframes=["1m", "5m"], datadir=testdatadir, @@ -521,6 +522,9 @@ def test_refresh_backtest_ohlcv_data( assert dl_mock.call_args[1]['timerange'].starttype == 'date' assert log_has_re(r"Downloading pair ETH/BTC, .* interval 1m\.", caplog) + if trademode == 'futures': + assert log_has_re(r"Downloading pair ETH/BTC, funding_rate, interval 8h\.", caplog) + assert log_has_re(r"Downloading pair ETH/BTC, mark, interval 4h\.", caplog) def test_download_data_no_markets(mocker, default_conf, caplog, testdatadir): diff --git a/tests/test_binance_mig.py b/tests/utils/test_binance_mig.py similarity index 75% rename from tests/test_binance_mig.py rename to tests/utils/test_binance_mig.py index b7c821a5a..39a853a27 100644 --- a/tests/test_binance_mig.py +++ b/tests/utils/test_binance_mig.py @@ -5,7 +5,8 @@ import shutil import pytest from freqtrade.persistence import Trade -from freqtrade.util.binance_mig import migrate_binance_futures_data, migrate_binance_futures_names +from freqtrade.util.migrations import (migrate_binance_futures_data, migrate_binance_futures_names, + migrate_data) from tests.conftest import create_mock_trades_usdt, log_has @@ -55,3 +56,13 @@ def test_binance_mig_db_conversion(default_conf_usdt, fee, caplog): default_conf_usdt['trading_mode'] = 'futures' migrate_binance_futures_names(default_conf_usdt) assert log_has('Migrating binance futures pairs in database.', caplog) + + +def test_migration_wrapper(default_conf_usdt, mocker): + default_conf_usdt['trading_mode'] = 'futures' + binmock = mocker.patch('freqtrade.util.migrations.migrate_binance_futures_data') + funding_mock = mocker.patch('freqtrade.util.migrations.migrate_funding_fee_timeframe') + migrate_data(default_conf_usdt) + + assert binmock.call_count == 1 + assert funding_mock.call_count == 1 diff --git a/tests/utils/test_funding_rate_migration.py b/tests/utils/test_funding_rate_migration.py new file mode 100644 index 000000000..ccb8435cf --- /dev/null +++ b/tests/utils/test_funding_rate_migration.py @@ -0,0 +1,29 @@ +from shutil import copytree + +from freqtrade.util.migrations import migrate_funding_fee_timeframe + + +def test_migrate_funding_rate_timeframe(default_conf_usdt, tmp_path, testdatadir): + + copytree(testdatadir / 'futures', tmp_path / 'futures') + file_4h = tmp_path / 'futures' / 'XRP_USDT_USDT-4h-funding_rate.feather' + file_8h = tmp_path / 'futures' / 'XRP_USDT_USDT-8h-funding_rate.feather' + file_1h = tmp_path / 'futures' / 'XRP_USDT_USDT-1h-futures.feather' + file_8h.rename(file_4h) + assert file_1h.exists() + assert file_4h.exists() + assert not file_8h.exists() + + default_conf_usdt['datadir'] = tmp_path + + # Inactive on spot trading ... + migrate_funding_fee_timeframe(default_conf_usdt, None) + + default_conf_usdt['trading_mode'] = 'futures' + + migrate_funding_fee_timeframe(default_conf_usdt, None) + + assert not file_4h.exists() + assert file_8h.exists() + # futures files is untouched. + assert file_1h.exists()