diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index f72164675..191f07910 100755 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -69,7 +69,8 @@ ARGS_CONVERT_DATA_TRADES = ["pairs", "format_from_trades", "format_to", "erase", ARGS_CONVERT_DATA = ["pairs", "format_from", "format_to", "erase", "exchange"] ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes", "trading_mode", "candle_types"] -ARGS_CONVERT_TRADES = ["pairs", "timeframes", "exchange", "dataformat_ohlcv", "dataformat_trades"] +ARGS_CONVERT_TRADES = ["pairs", "timeframes", "exchange", "dataformat_ohlcv", "dataformat_trades", + "trading_mode"] ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs", "trading_mode", "show_timerange"] diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index 33069885a..b183d403b 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -8,9 +8,10 @@ from freqtrade.constants import DATETIME_PRINT_FORMAT, DL_DATA_TIMEFRAMES, Confi from freqtrade.data.converter import (convert_ohlcv_format, convert_trades_format, convert_trades_to_ohlcv) from freqtrade.data.history import download_data_main -from freqtrade.enums import RunMode, TradingMode +from freqtrade.enums import CandleType, RunMode, TradingMode from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_minutes +from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist from freqtrade.resolvers import ExchangeResolver from freqtrade.util.migrations import migrate_data @@ -62,13 +63,21 @@ def start_convert_trades(args: Dict[str, Any]) -> None: for timeframe in config['timeframes']: exchange.validate_timeframes(timeframe) + available_pairs = [ + p for p in exchange.get_markets( + tradable_only=True, active_only=not config.get('include_inactive') + ).keys() + ] + + expanded_pairs = dynamic_expand_pairlist(config, available_pairs) # Convert downloaded trade data to different timeframes convert_trades_to_ohlcv( - pairs=config.get('pairs', []), timeframes=config['timeframes'], + pairs=expanded_pairs, timeframes=config['timeframes'], datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')), data_format_ohlcv=config['dataformat_ohlcv'], data_format_trades=config['dataformat_trades'], + candle_type=config.get('candle_type_def', CandleType.SPOT) ) diff --git a/freqtrade/data/converter/trade_converter.py b/freqtrade/data/converter/trade_converter.py index bd4efb77e..682430994 100644 --- a/freqtrade/data/converter/trade_converter.py +++ b/freqtrade/data/converter/trade_converter.py @@ -11,7 +11,7 @@ from pandas import DataFrame, to_datetime from freqtrade.configuration import TimeRange from freqtrade.constants import (DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, TRADES_DTYPES, Config, TradeList) -from freqtrade.enums import CandleType +from freqtrade.enums import CandleType, TradingMode from freqtrade.exceptions import OperationalException @@ -88,10 +88,10 @@ def convert_trades_to_ohlcv( timeframes: List[str], datadir: Path, timerange: TimeRange, - erase: bool = False, - data_format_ohlcv: str = 'feather', - data_format_trades: str = 'feather', - candle_type: CandleType = CandleType.SPOT + erase: bool, + data_format_ohlcv: str, + data_format_trades: str, + candle_type: CandleType, ) -> None: """ Convert stored trades data to ohlcv data @@ -99,14 +99,12 @@ def convert_trades_to_ohlcv( from freqtrade.data.history.idatahandler import get_datahandler data_handler_trades = get_datahandler(datadir, data_format=data_format_trades) data_handler_ohlcv = get_datahandler(datadir, data_format=data_format_ohlcv) - if not pairs: - pairs = data_handler_trades.trades_get_pairs(datadir) logger.info(f"About to convert pairs: '{', '.join(pairs)}', " f"intervals: '{', '.join(timeframes)}' to {datadir}") - + trading_mode = TradingMode.FUTURES if candle_type != CandleType.SPOT else TradingMode.SPOT for pair in pairs: - trades = data_handler_trades.trades_load(pair) + trades = data_handler_trades.trades_load(pair, trading_mode) for timeframe in timeframes: if erase: if data_handler_ohlcv.ohlcv_purge(pair, timeframe, candle_type=candle_type): @@ -116,7 +114,7 @@ def convert_trades_to_ohlcv( # Store ohlcv data_handler_ohlcv.ohlcv_store(pair, timeframe, data=ohlcv, candle_type=candle_type) except ValueError: - logger.exception(f'Could not convert {pair} to OHLCV.') + logger.warning(f'Could not convert {pair} to OHLCV.') def convert_trades_format(config: Config, convert_from: str, convert_to: str, erase: bool): @@ -144,11 +142,12 @@ def convert_trades_format(config: Config, convert_from: str, convert_to: str, er if 'pairs' not in config: config['pairs'] = src.trades_get_pairs(config['datadir']) logger.info(f"Converting trades for {config['pairs']}") - + trading_mode: TradingMode = config.get('trading_mode', TradingMode.SPOT) for pair in config['pairs']: - data = src.trades_load(pair=pair) + data = src.trades_load(pair, trading_mode) logger.info(f"Converting {len(data)} trades for {pair}") - trg.trades_store(pair, data) + trg.trades_store(pair, data, trading_mode) + if erase and convert_from != convert_to: logger.info(f"Deleting source Trade data for {pair}.") - src.trades_purge(pair=pair) + src.trades_purge(pair, trading_mode) diff --git a/freqtrade/data/converter/trade_converter_kraken.py b/freqtrade/data/converter/trade_converter_kraken.py index b0fa11c25..80bd917af 100644 --- a/freqtrade/data/converter/trade_converter_kraken.py +++ b/freqtrade/data/converter/trade_converter_kraken.py @@ -7,6 +7,7 @@ from freqtrade.constants import DATETIME_PRINT_FORMAT, DEFAULT_TRADES_COLUMNS, C from freqtrade.data.converter.trade_converter import (trades_convert_types, trades_df_remove_duplicates) from freqtrade.data.history.idatahandler import get_datahandler +from freqtrade.enums import TradingMode from freqtrade.exceptions import OperationalException from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.resolvers import ExchangeResolver @@ -79,4 +80,4 @@ def import_kraken_trades_from_csv(config: Config, convert_to: str): f"{trades_df['date'].min():{DATETIME_PRINT_FORMAT}} to " f"{trades_df['date'].max():{DATETIME_PRINT_FORMAT}}") - data_handler.trades_store(pair, trades_df) + data_handler.trades_store(pair, trades_df, TradingMode.SPOT) diff --git a/freqtrade/data/history/featherdatahandler.py b/freqtrade/data/history/featherdatahandler.py index 44d337836..6d57dbed7 100644 --- a/freqtrade/data/history/featherdatahandler.py +++ b/freqtrade/data/history/featherdatahandler.py @@ -5,7 +5,7 @@ from pandas import DataFrame, read_feather, to_datetime from freqtrade.configuration import TimeRange from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS -from freqtrade.enums import CandleType +from freqtrade.enums import CandleType, TradingMode from .idatahandler import IDataHandler @@ -82,14 +82,15 @@ class FeatherDataHandler(IDataHandler): """ raise NotImplementedError() - def _trades_store(self, pair: str, data: DataFrame) -> None: + def _trades_store(self, pair: str, data: DataFrame, trading_mode: TradingMode) -> None: """ Store trades data (list of Dicts) to file :param pair: Pair - used for filename :param data: Dataframe containing trades column sequence as in DEFAULT_TRADES_COLUMNS + :param trading_mode: Trading mode to use (used to determine the filename) """ - filename = self._pair_trades_filename(self._datadir, pair) + filename = self._pair_trades_filename(self._datadir, pair, trading_mode) self.create_dir_if_needed(filename) data.reset_index(drop=True).to_feather(filename, compression_level=9, compression='lz4') @@ -102,15 +103,18 @@ class FeatherDataHandler(IDataHandler): """ raise NotImplementedError() - def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> DataFrame: + def _trades_load( + self, pair: str, trading_mode: TradingMode, timerange: Optional[TimeRange] = None + ) -> DataFrame: """ Load a pair from file, either .json.gz or .json # TODO: respect timerange ... :param pair: Load trades for this pair + :param trading_mode: Trading mode to use (used to determine the filename) :param timerange: Timerange to load trades for - currently not implemented :return: Dataframe containing trades """ - filename = self._pair_trades_filename(self._datadir, pair) + filename = self._pair_trades_filename(self._datadir, pair, trading_mode) if not filename.exists(): return DataFrame(columns=DEFAULT_TRADES_COLUMNS) diff --git a/freqtrade/data/history/hdf5datahandler.py b/freqtrade/data/history/hdf5datahandler.py index b118bd7e0..cb2cdd884 100644 --- a/freqtrade/data/history/hdf5datahandler.py +++ b/freqtrade/data/history/hdf5datahandler.py @@ -6,7 +6,7 @@ import pandas as pd from freqtrade.configuration import TimeRange from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS -from freqtrade.enums import CandleType +from freqtrade.enums import CandleType, TradingMode from .idatahandler import IDataHandler @@ -100,17 +100,18 @@ class HDF5DataHandler(IDataHandler): """ raise NotImplementedError() - def _trades_store(self, pair: str, data: pd.DataFrame) -> None: + def _trades_store(self, pair: str, data: pd.DataFrame, trading_mode: TradingMode) -> None: """ Store trades data (list of Dicts) to file :param pair: Pair - used for filename :param data: Dataframe containing trades column sequence as in DEFAULT_TRADES_COLUMNS + :param trading_mode: Trading mode to use (used to determine the filename) """ key = self._pair_trades_key(pair) data.to_hdf( - self._pair_trades_filename(self._datadir, pair), key=key, + self._pair_trades_filename(self._datadir, pair, trading_mode), key=key, mode='a', complevel=9, complib='blosc', format='table', data_columns=['timestamp'] ) @@ -124,15 +125,18 @@ class HDF5DataHandler(IDataHandler): """ raise NotImplementedError() - def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> pd.DataFrame: + def _trades_load( + self, pair: str, trading_mode: TradingMode, timerange: Optional[TimeRange] = None + ) -> pd.DataFrame: """ Load a pair from h5 file. :param pair: Load trades for this pair + :param trading_mode: Trading mode to use (used to determine the filename) :param timerange: Timerange to load trades for - currently not implemented :return: Dataframe containing trades """ key = self._pair_trades_key(pair) - filename = self._pair_trades_filename(self._datadir, pair) + filename = self._pair_trades_filename(self._datadir, pair, trading_mode) if not filename.exists(): return pd.DataFrame(columns=DEFAULT_TRADES_COLUMNS) diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index ff6c2561d..208859cd3 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -13,7 +13,7 @@ from freqtrade.data.converter import (clean_ohlcv_dataframe, convert_trades_to_o ohlcv_to_dataframe, trades_df_remove_duplicates, trades_list_to_df) from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler -from freqtrade.enums import CandleType +from freqtrade.enums import CandleType, TradingMode from freqtrade.exceptions import OperationalException from freqtrade.exchange import Exchange from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist @@ -333,7 +333,8 @@ def _download_trades_history(exchange: Exchange, pair: str, *, new_pairs_days: int = 30, timerange: Optional[TimeRange] = None, - data_handler: IDataHandler + data_handler: IDataHandler, + trading_mode: TradingMode, ) -> bool: """ Download trade history from the exchange. @@ -349,7 +350,7 @@ def _download_trades_history(exchange: Exchange, if timerange.stoptype == 'date': until = timerange.stopts * 1000 - trades = data_handler.trades_load(pair) + trades = data_handler.trades_load(pair, trading_mode) # TradesList columns are defined in constants.DEFAULT_TRADES_COLUMNS # DEFAULT_TRADES_COLUMNS: 0 -> timestamp @@ -388,7 +389,7 @@ def _download_trades_history(exchange: Exchange, trades = concat([trades, new_trades_df], axis=0) # Remove duplicates to make sure we're not storing data we don't need trades = trades_df_remove_duplicates(trades) - data_handler.trades_store(pair, data=trades) + data_handler.trades_store(pair, trades, trading_mode) logger.debug("New Start: %s", 'None' if trades.empty else f"{trades.iloc[0]['date']:{DATETIME_PRINT_FORMAT}}") @@ -405,8 +406,10 @@ def _download_trades_history(exchange: Exchange, def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: Path, - timerange: TimeRange, new_pairs_days: int = 30, - erase: bool = False, data_format: str = 'feather') -> List[str]: + timerange: TimeRange, trading_mode: TradingMode, + new_pairs_days: int = 30, + erase: bool = False, data_format: str = 'feather', + ) -> List[str]: """ Refresh stored trades data for backtesting and hyperopt operations. Used by freqtrade download-data subcommand. @@ -421,7 +424,7 @@ def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: continue if erase: - if data_handler.trades_purge(pair): + if data_handler.trades_purge(pair, trading_mode): logger.info(f'Deleting existing data for pair {pair}.') logger.info(f'Downloading trades for pair {pair}.') @@ -429,7 +432,8 @@ def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: pair=pair, new_pairs_days=new_pairs_days, timerange=timerange, - data_handler=data_handler) + data_handler=data_handler, + trading_mode=trading_mode) return pairs_not_available @@ -516,12 +520,12 @@ def download_data_main(config: Config) -> None: # Start downloading try: if config.get('download_trades'): - if config.get('trading_mode') == 'futures': - raise OperationalException("Trade download not supported for futures.") pairs_not_available = refresh_backtest_trades_data( exchange, pairs=expanded_pairs, datadir=config['datadir'], timerange=timerange, new_pairs_days=config['new_pairs_days'], - erase=bool(config.get('erase')), data_format=config['dataformat_trades']) + erase=bool(config.get('erase')), data_format=config['dataformat_trades'], + trading_mode=config.get('trading_mode', TradingMode.SPOT), + ) # Convert downloaded trade data to different timeframes convert_trades_to_ohlcv( @@ -529,6 +533,7 @@ def download_data_main(config: Config) -> None: datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')), data_format_ohlcv=config['dataformat_ohlcv'], data_format_trades=config['dataformat_trades'], + candle_type=config.get('candle_type_def', CandleType.SPOT), ) else: if not exchange.get_option('ohlcv_has_history', True): diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index 47c2dd838..fbaded640 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -172,12 +172,13 @@ class IDataHandler(ABC): return [cls.rebuild_pair_from_filename(match[0]) for match in _tmp if match] @abstractmethod - def _trades_store(self, pair: str, data: DataFrame) -> None: + def _trades_store(self, pair: str, data: DataFrame, trading_mode: TradingMode) -> None: """ Store trades data (list of Dicts) to file :param pair: Pair - used for filename :param data: Dataframe containing trades column sequence as in DEFAULT_TRADES_COLUMNS + :param trading_mode: Trading mode to use (used to determine the filename) """ @abstractmethod @@ -190,45 +191,55 @@ class IDataHandler(ABC): """ @abstractmethod - def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> DataFrame: + def _trades_load( + self, pair: str, trading_mode: TradingMode, timerange: Optional[TimeRange] = None + ) -> DataFrame: """ Load a pair from file, either .json.gz or .json :param pair: Load trades for this pair + :param trading_mode: Trading mode to use (used to determine the filename) :param timerange: Timerange to load trades for - currently not implemented :return: Dataframe containing trades """ - def trades_store(self, pair: str, data: DataFrame) -> None: + def trades_store(self, pair: str, data: DataFrame, trading_mode: TradingMode) -> None: """ Store trades data (list of Dicts) to file :param pair: Pair - used for filename :param data: Dataframe containing trades column sequence as in DEFAULT_TRADES_COLUMNS + :param trading_mode: Trading mode to use (used to determine the filename) """ # Filter on expected columns (will remove the actual date column). - self._trades_store(pair, data[DEFAULT_TRADES_COLUMNS]) + self._trades_store(pair, data[DEFAULT_TRADES_COLUMNS], trading_mode) - def trades_purge(self, pair: str) -> bool: + def trades_purge(self, pair: str, trading_mode: TradingMode) -> bool: """ Remove data for this pair :param pair: Delete data for this pair. + :param trading_mode: Trading mode to use (used to determine the filename) :return: True when deleted, false if file did not exist. """ - filename = self._pair_trades_filename(self._datadir, pair) + filename = self._pair_trades_filename(self._datadir, pair, trading_mode) if filename.exists(): filename.unlink() return True return False - def trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> DataFrame: + def trades_load( + self, pair: str, trading_mode: TradingMode, timerange: Optional[TimeRange] = None + ) -> DataFrame: """ Load a pair from file, either .json.gz or .json Removes duplicates in the process. :param pair: Load trades for this pair + :param trading_mode: Trading mode to use (used to determine the filename) :param timerange: Timerange to load trades for - currently not implemented :return: List of trades """ - trades = trades_df_remove_duplicates(self._trades_load(pair, timerange=timerange)) + trades = trades_df_remove_duplicates( + self._trades_load(pair, trading_mode, timerange=timerange) + ) trades = trades_convert_types(trades) return trades @@ -264,8 +275,12 @@ class IDataHandler(ABC): return filename @classmethod - def _pair_trades_filename(cls, datadir: Path, pair: str) -> Path: + def _pair_trades_filename(cls, datadir: Path, pair: str, trading_mode: TradingMode) -> Path: pair_s = misc.pair_to_filename(pair) + if trading_mode == TradingMode.FUTURES: + # Futures pair ... + datadir = datadir.joinpath('futures') + filename = datadir.joinpath(f'{pair_s}-trades.{cls._get_file_extension()}') return filename diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index baa0c10a5..2d0333fed 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -8,7 +8,7 @@ from freqtrade import misc from freqtrade.configuration import TimeRange from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS from freqtrade.data.converter import trades_dict_to_list, trades_list_to_df -from freqtrade.enums import CandleType +from freqtrade.enums import CandleType, TradingMode from .idatahandler import IDataHandler @@ -94,14 +94,15 @@ class JsonDataHandler(IDataHandler): """ raise NotImplementedError() - def _trades_store(self, pair: str, data: DataFrame) -> None: + def _trades_store(self, pair: str, data: DataFrame, trading_mode: TradingMode) -> None: """ Store trades data (list of Dicts) to file :param pair: Pair - used for filename :param data: Dataframe containing trades column sequence as in DEFAULT_TRADES_COLUMNS + :param trading_mode: Trading mode to use (used to determine the filename) """ - filename = self._pair_trades_filename(self._datadir, pair) + filename = self._pair_trades_filename(self._datadir, pair, trading_mode) trades = data.values.tolist() misc.file_dump_json(filename, trades, is_zip=self._use_zip) @@ -114,15 +115,18 @@ class JsonDataHandler(IDataHandler): """ raise NotImplementedError() - def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> DataFrame: + def _trades_load( + self, pair: str, trading_mode: TradingMode, timerange: Optional[TimeRange] = None + ) -> DataFrame: """ Load a pair from file, either .json.gz or .json # TODO: respect timerange ... :param pair: Load trades for this pair + :param trading_mode: Trading mode to use (used to determine the filename) :param timerange: Timerange to load trades for - currently not implemented :return: Dataframe containing trades """ - filename = self._pair_trades_filename(self._datadir, pair) + filename = self._pair_trades_filename(self._datadir, pair, trading_mode) tradesdata = misc.file_load_json(filename) if not tradesdata: diff --git a/freqtrade/data/history/parquetdatahandler.py b/freqtrade/data/history/parquetdatahandler.py index c0b0cad63..01becdc84 100644 --- a/freqtrade/data/history/parquetdatahandler.py +++ b/freqtrade/data/history/parquetdatahandler.py @@ -4,8 +4,8 @@ from typing import Optional from pandas import DataFrame, read_parquet, to_datetime from freqtrade.configuration import TimeRange -from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, TradeList -from freqtrade.enums import CandleType +from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS +from freqtrade.enums import CandleType, TradingMode from .idatahandler import IDataHandler @@ -81,14 +81,15 @@ class ParquetDataHandler(IDataHandler): """ raise NotImplementedError() - def _trades_store(self, pair: str, data: DataFrame) -> None: + def _trades_store(self, pair: str, data: DataFrame, trading_mode: TradingMode) -> None: """ Store trades data (list of Dicts) to file :param pair: Pair - used for filename :param data: Dataframe containing trades column sequence as in DEFAULT_TRADES_COLUMNS + :param trading_mode: Trading mode to use (used to determine the filename) """ - filename = self._pair_trades_filename(self._datadir, pair) + filename = self._pair_trades_filename(self._datadir, pair, trading_mode) self.create_dir_if_needed(filename) data.reset_index(drop=True).to_parquet(filename) @@ -101,15 +102,18 @@ class ParquetDataHandler(IDataHandler): """ raise NotImplementedError() - def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList: + def _trades_load( + self, pair: str, trading_mode: TradingMode, timerange: Optional[TimeRange] = None + ) -> DataFrame: """ Load a pair from file, either .json.gz or .json # TODO: respect timerange ... :param pair: Load trades for this pair + :param trading_mode: Trading mode to use (used to determine the filename) :param timerange: Timerange to load trades for - currently not implemented :return: List of trades """ - filename = self._pair_trades_filename(self._datadir, pair) + filename = self._pair_trades_filename(self._datadir, pair, trading_mode) if not filename.exists(): return DataFrame(columns=DEFAULT_TRADES_COLUMNS) diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index cdad46407..2252ff9f4 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -820,11 +820,6 @@ def test_download_data_trades(mocker): "--trading-mode", "futures", "--dl-trades" ] - with pytest.raises(OperationalException, - match="Trade download not supported for futures."): - pargs = get_args(args) - pargs['config'] = None - start_download_data(pargs) def test_download_data_data_invalid(mocker): @@ -842,10 +837,11 @@ def test_download_data_data_invalid(mocker): start_download_data(pargs) -def test_start_convert_trades(mocker, caplog): +def test_start_convert_trades(mocker): convert_mock = mocker.patch('freqtrade.commands.data_commands.convert_trades_to_ohlcv', MagicMock(return_value=[])) patch_exchange(mocker) + mocker.patch(f'{EXMS}.get_markets') mocker.patch(f'{EXMS}.markets', PropertyMock(return_value={})) args = [ "trades-to-ohlcv", diff --git a/tests/data/test_converter.py b/tests/data/test_converter.py index 08fc785aa..2202ada44 100644 --- a/tests/data/test_converter.py +++ b/tests/data/test_converter.py @@ -542,7 +542,9 @@ def test_convert_trades_to_ohlcv(testdatadir, tmp_path, caplog): convert_trades_to_ohlcv([pair], timeframes=['1m', '5m'], data_format_trades='jsongz', - datadir=tmp_path, timerange=tr, erase=True) + datadir=tmp_path, timerange=tr, erase=True, + data_format_ohlcv='feather', + candle_type=CandleType.SPOT) assert log_has("Deleting existing data for pair XRP/ETH, interval 1m.", caplog) # Load new data @@ -556,5 +558,7 @@ def test_convert_trades_to_ohlcv(testdatadir, tmp_path, caplog): convert_trades_to_ohlcv(['NoDatapair'], timeframes=['1m', '5m'], data_format_trades='jsongz', - datadir=tmp_path, timerange=tr, erase=True) + datadir=tmp_path, timerange=tr, erase=True, + data_format_ohlcv='feather', + candle_type=CandleType.SPOT) assert log_has(msg, caplog) diff --git a/tests/data/test_datahandler.py b/tests/data/test_datahandler.py index a0a37c393..1217c35ad 100644 --- a/tests/data/test_datahandler.py +++ b/tests/data/test_datahandler.py @@ -261,11 +261,11 @@ def test_datahandler_trades_not_supported(datahandler, testdatadir, ): def test_jsondatahandler_trades_load(testdatadir, caplog): dh = JsonGzDataHandler(testdatadir) logmsg = "Old trades format detected - converting" - dh.trades_load('XRP/ETH') + dh.trades_load('XRP/ETH', TradingMode.SPOT) assert not log_has(logmsg, caplog) # Test conversation is happening - dh.trades_load('XRP/OLD') + dh.trades_load('XRP/OLD', TradingMode.SPOT) assert log_has(logmsg, caplog) @@ -300,16 +300,16 @@ def test_datahandler_trades_get_pairs(testdatadir, datahandler, expected): def test_hdf5datahandler_trades_load(testdatadir): dh = get_datahandler(testdatadir, 'hdf5') - trades = dh.trades_load('XRP/ETH') + trades = dh.trades_load('XRP/ETH', TradingMode.SPOT) assert isinstance(trades, DataFrame) - trades1 = dh.trades_load('UNITTEST/NONEXIST') + trades1 = dh.trades_load('UNITTEST/NONEXIST', TradingMode.SPOT) assert isinstance(trades1, DataFrame) assert trades1.empty # data goes from 2019-10-11 - 2019-10-13 timerange = TimeRange.parse_timerange('20191011-20191012') - trades2 = dh._trades_load('XRP/ETH', timerange) + trades2 = dh._trades_load('XRP/ETH', TradingMode.SPOT, timerange) assert len(trades) > len(trades2) # Check that ID is None (If it's nan, it's wrong) assert trades2.iloc[0]['type'] is None @@ -451,13 +451,13 @@ def test_hdf5datahandler_ohlcv_purge(mocker, testdatadir): @pytest.mark.parametrize('datahandler', ['jsongz', 'hdf5', 'feather', 'parquet']) def test_datahandler_trades_load(testdatadir, datahandler): dh = get_datahandler(testdatadir, datahandler) - trades = dh.trades_load('XRP/ETH') + trades = dh.trades_load('XRP/ETH', TradingMode.SPOT) assert isinstance(trades, DataFrame) assert trades.iloc[0]['timestamp'] == 1570752011620 assert trades.iloc[0]['date'] == Timestamp('2019-10-11 00:00:11.620000+0000') assert trades.iloc[-1]['cost'] == 0.1986231 - trades1 = dh.trades_load('UNITTEST/NONEXIST') + trades1 = dh.trades_load('UNITTEST/NONEXIST', TradingMode.SPOT) assert isinstance(trades, DataFrame) assert trades1.empty @@ -465,15 +465,15 @@ def test_datahandler_trades_load(testdatadir, datahandler): @pytest.mark.parametrize('datahandler', ['jsongz', 'hdf5', 'feather', 'parquet']) def test_datahandler_trades_store(testdatadir, tmp_path, datahandler): dh = get_datahandler(testdatadir, datahandler) - trades = dh.trades_load('XRP/ETH') + trades = dh.trades_load('XRP/ETH', TradingMode.SPOT) dh1 = get_datahandler(tmp_path, datahandler) - dh1.trades_store('XRP/NEW', trades) + dh1.trades_store('XRP/NEW', trades, TradingMode.SPOT) file = tmp_path / f'XRP_NEW-trades.{dh1._get_file_extension()}' assert file.is_file() # Load trades back - trades_new = dh1.trades_load('XRP/NEW') + trades_new = dh1.trades_load('XRP/NEW', TradingMode.SPOT) assert_frame_equal(trades, trades_new, check_exact=True) assert len(trades_new) == len(trades) @@ -483,11 +483,11 @@ def test_datahandler_trades_purge(mocker, testdatadir, datahandler): mocker.patch.object(Path, "exists", MagicMock(return_value=False)) unlinkmock = mocker.patch.object(Path, "unlink", MagicMock()) dh = get_datahandler(testdatadir, datahandler) - assert not dh.trades_purge('UNITTEST/NONEXIST') + assert not dh.trades_purge('UNITTEST/NONEXIST', TradingMode.SPOT) assert unlinkmock.call_count == 0 mocker.patch.object(Path, "exists", MagicMock(return_value=True)) - assert dh.trades_purge('UNITTEST/NONEXIST') + assert dh.trades_purge('UNITTEST/NONEXIST', TradingMode.SPOT) assert unlinkmock.call_count == 1 diff --git a/tests/data/test_download_data.py b/tests/data/test_download_data.py index 97640d01c..1518b28f3 100644 --- a/tests/data/test_download_data.py +++ b/tests/data/test_download_data.py @@ -78,10 +78,6 @@ def test_download_data_main_trades(mocker): "trading_mode": "futures", }) - with pytest.raises(OperationalException, - match="Trade download not supported for futures."): - download_data_main(config) - def test_download_data_main_data_invalid(mocker): patch_exchange(mocker, id="kraken") diff --git a/tests/data/test_history.py b/tests/data/test_history.py index a48d34aee..a3fe492b7 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -23,7 +23,7 @@ from freqtrade.data.history.history_utils import (_download_pair_history, _downl validate_backtest_data) from freqtrade.data.history.idatahandler import get_datahandler from freqtrade.data.history.jsondatahandler import JsonDataHandler, JsonGzDataHandler -from freqtrade.enums import CandleType +from freqtrade.enums import CandleType, TradingMode from freqtrade.exchange import timeframe_to_minutes from freqtrade.misc import file_dump_json from freqtrade.resolvers import StrategyResolver @@ -168,20 +168,21 @@ def test_json_pair_data_filename(pair, timeframe, expected_result, candle_type): assert fn == Path(expected_result + '.gz') -@pytest.mark.parametrize("pair,expected_result", [ - ("ETH/BTC", 'freqtrade/hello/world/ETH_BTC-trades.json'), - ("Fabric Token/ETH", 'freqtrade/hello/world/Fabric_Token_ETH-trades.json'), - ("ETHH20", 'freqtrade/hello/world/ETHH20-trades.json'), - (".XBTBON2H", 'freqtrade/hello/world/_XBTBON2H-trades.json'), - ("ETHUSD.d", 'freqtrade/hello/world/ETHUSD_d-trades.json'), - ("ACC_OLD_BTC", 'freqtrade/hello/world/ACC_OLD_BTC-trades.json'), +@pytest.mark.parametrize("pair,trading_mode,expected_result", [ + ("ETH/BTC", '', 'freqtrade/hello/world/ETH_BTC-trades.json'), + ("ETH/USDT:USDT", 'futures', 'freqtrade/hello/world/futures/ETH_USDT_USDT-trades.json'), + ("Fabric Token/ETH", '', 'freqtrade/hello/world/Fabric_Token_ETH-trades.json'), + ("ETHH20", '', 'freqtrade/hello/world/ETHH20-trades.json'), + (".XBTBON2H", '', 'freqtrade/hello/world/_XBTBON2H-trades.json'), + ("ETHUSD.d", '', 'freqtrade/hello/world/ETHUSD_d-trades.json'), + ("ACC_OLD_BTC", '', 'freqtrade/hello/world/ACC_OLD_BTC-trades.json'), ]) -def test_json_pair_trades_filename(pair, expected_result): - fn = JsonDataHandler._pair_trades_filename(Path('freqtrade/hello/world'), pair) +def test_json_pair_trades_filename(pair, trading_mode, expected_result): + fn = JsonDataHandler._pair_trades_filename(Path('freqtrade/hello/world'), pair, trading_mode) assert isinstance(fn, Path) assert fn == Path(expected_result) - fn = JsonGzDataHandler._pair_trades_filename(Path('freqtrade/hello/world'), pair) + fn = JsonGzDataHandler._pair_trades_filename(Path('freqtrade/hello/world'), pair, trading_mode) assert isinstance(fn, Path) assert fn == Path(expected_result + '.gz') @@ -559,7 +560,8 @@ def test_refresh_backtest_trades_data(mocker, default_conf, markets, caplog, tes unavailable_pairs = refresh_backtest_trades_data(exchange=ex, pairs=["ETH/BTC", "XRP/BTC", "XRP/ETH"], datadir=testdatadir, - timerange=timerange, erase=True + timerange=timerange, erase=True, + trading_mode=TradingMode.SPOT, ) assert dl_mock.call_count == 2 @@ -584,7 +586,7 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad assert not file1.is_file() assert _download_trades_history(data_handler=data_handler, exchange=exchange, - pair='ETH/BTC') + pair='ETH/BTC', trading_mode=TradingMode.SPOT) assert log_has("Current Amount of trades: 0", caplog) assert log_has("New Amount of trades: 6", caplog) assert ght_mock.call_count == 1 @@ -597,8 +599,9 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad since_time = int(trades_history[-3][0] // 1000) since_time2 = int(trades_history[-1][0] // 1000) timerange = TimeRange('date', None, since_time, 0) - assert _download_trades_history(data_handler=data_handler, exchange=exchange, - pair='ETH/BTC', timerange=timerange) + assert _download_trades_history( + data_handler=data_handler, exchange=exchange, pair='ETH/BTC', + timerange=timerange, trading_mode=TradingMode.SPOT) assert ght_mock.call_count == 1 # Check this in seconds - since we had to convert to seconds above too. @@ -611,7 +614,7 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad caplog.clear() assert not _download_trades_history(data_handler=data_handler, exchange=exchange, - pair='ETH/BTC') + pair='ETH/BTC', trading_mode=TradingMode.SPOT) assert log_has_re('Failed to download historic trades for pair: "ETH/BTC".*', caplog) file2 = tmp_path / 'XRP_ETH-trades.json.gz' @@ -623,8 +626,9 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad since_time = int(trades_history[0][0] // 1000) - 500 timerange = TimeRange('date', None, since_time, 0) - assert _download_trades_history(data_handler=data_handler, exchange=exchange, - pair='XRP/ETH', timerange=timerange) + assert _download_trades_history( + data_handler=data_handler, exchange=exchange, pair='XRP/ETH', + timerange=timerange, trading_mode=TradingMode.SPOT) assert ght_mock.call_count == 1 diff --git a/tests/data/test_trade_converter_kraken.py b/tests/data/test_trade_converter_kraken.py index 91de303fb..ba9221e0a 100644 --- a/tests/data/test_trade_converter_kraken.py +++ b/tests/data/test_trade_converter_kraken.py @@ -6,6 +6,7 @@ import pytest from freqtrade.data.converter.trade_converter_kraken import import_kraken_trades_from_csv from freqtrade.data.history.idatahandler import get_datahandler +from freqtrade.enums import TradingMode from freqtrade.exceptions import OperationalException from tests.conftest import EXMS, log_has, log_has_re, patch_exchange @@ -40,7 +41,7 @@ def test_import_kraken_trades_from_csv(testdatadir, tmp_path, caplog, default_co assert dstfile.is_file() dh = get_datahandler(tmp_path, 'feather') - trades = dh.trades_load('BCH_EUR') + trades = dh.trades_load('BCH_EUR', TradingMode.SPOT) assert len(trades) == 340 assert trades['date'].min().to_pydatetime() == datetime(2023, 1, 1, 0, 3, 56,