From 46e616f9975b99de2fe7fb203c58ff0111212e1c Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 1 Mar 2024 19:32:18 +0100 Subject: [PATCH 01/12] Remove defaults for converter - they're always provided and necessary. --- freqtrade/commands/data_commands.py | 3 ++- freqtrade/data/converter/trade_converter.py | 8 ++++---- freqtrade/data/history/history_utils.py | 1 + 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index 33069885a..d3600e3ef 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -8,7 +8,7 @@ from freqtrade.constants import DATETIME_PRINT_FORMAT, DL_DATA_TIMEFRAMES, Confi from freqtrade.data.converter import (convert_ohlcv_format, convert_trades_format, convert_trades_to_ohlcv) from freqtrade.data.history import download_data_main -from freqtrade.enums import RunMode, TradingMode +from freqtrade.enums import CandleType, RunMode, TradingMode from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_minutes from freqtrade.resolvers import ExchangeResolver @@ -69,6 +69,7 @@ def start_convert_trades(args: Dict[str, Any]) -> None: datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')), data_format_ohlcv=config['dataformat_ohlcv'], data_format_trades=config['dataformat_trades'], + candle_type=config.get('candle_type_def', CandleType.SPOT) ) diff --git a/freqtrade/data/converter/trade_converter.py b/freqtrade/data/converter/trade_converter.py index bd4efb77e..117f65bc6 100644 --- a/freqtrade/data/converter/trade_converter.py +++ b/freqtrade/data/converter/trade_converter.py @@ -88,10 +88,10 @@ def convert_trades_to_ohlcv( timeframes: List[str], datadir: Path, timerange: TimeRange, - erase: bool = False, - data_format_ohlcv: str = 'feather', - data_format_trades: str = 'feather', - candle_type: CandleType = CandleType.SPOT + erase: bool, + data_format_ohlcv: str, + data_format_trades: str, + candle_type: CandleType, ) -> None: """ Convert stored trades data to ohlcv data diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index ff6c2561d..27e229973 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -529,6 +529,7 @@ def download_data_main(config: Config) -> None: datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')), data_format_ohlcv=config['dataformat_ohlcv'], data_format_trades=config['dataformat_trades'], + candle_type=config.get('candle_type_def', CandleType.SPOT), ) else: if not exchange.get_option('ohlcv_has_history', True): From 5dee60921f41cc4efd562c6047ea8cc97a0ff8e6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 1 Mar 2024 19:42:33 +0100 Subject: [PATCH 02/12] Fix test for convert_trades_to_ohlcv --- tests/data/test_converter.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/data/test_converter.py b/tests/data/test_converter.py index 08fc785aa..2202ada44 100644 --- a/tests/data/test_converter.py +++ b/tests/data/test_converter.py @@ -542,7 +542,9 @@ def test_convert_trades_to_ohlcv(testdatadir, tmp_path, caplog): convert_trades_to_ohlcv([pair], timeframes=['1m', '5m'], data_format_trades='jsongz', - datadir=tmp_path, timerange=tr, erase=True) + datadir=tmp_path, timerange=tr, erase=True, + data_format_ohlcv='feather', + candle_type=CandleType.SPOT) assert log_has("Deleting existing data for pair XRP/ETH, interval 1m.", caplog) # Load new data @@ -556,5 +558,7 @@ def test_convert_trades_to_ohlcv(testdatadir, tmp_path, caplog): convert_trades_to_ohlcv(['NoDatapair'], timeframes=['1m', '5m'], data_format_trades='jsongz', - datadir=tmp_path, timerange=tr, erase=True) + datadir=tmp_path, timerange=tr, erase=True, + data_format_ohlcv='feather', + candle_type=CandleType.SPOT) assert log_has(msg, caplog) From bdd63aa1d661be6e54500329a0f50fc614d2f3e4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 1 Mar 2024 20:17:43 +0100 Subject: [PATCH 03/12] FIx futures trades pair download directory --- freqtrade/data/history/idatahandler.py | 5 +++++ tests/commands/test_commands.py | 5 ----- tests/data/test_download_data.py | 4 ---- tests/data/test_history.py | 1 + 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index 47c2dd838..01c244f38 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -266,6 +266,11 @@ class IDataHandler(ABC): @classmethod def _pair_trades_filename(cls, datadir: Path, pair: str) -> Path: pair_s = misc.pair_to_filename(pair) + if ':' in pair: + # Futures pair ... + # TODO: this should not rely on ";" in the pairname. + datadir = datadir.joinpath('futures') + filename = datadir.joinpath(f'{pair_s}-trades.{cls._get_file_extension()}') return filename diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index cdad46407..1ab9d2202 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -820,11 +820,6 @@ def test_download_data_trades(mocker): "--trading-mode", "futures", "--dl-trades" ] - with pytest.raises(OperationalException, - match="Trade download not supported for futures."): - pargs = get_args(args) - pargs['config'] = None - start_download_data(pargs) def test_download_data_data_invalid(mocker): diff --git a/tests/data/test_download_data.py b/tests/data/test_download_data.py index 97640d01c..1518b28f3 100644 --- a/tests/data/test_download_data.py +++ b/tests/data/test_download_data.py @@ -78,10 +78,6 @@ def test_download_data_main_trades(mocker): "trading_mode": "futures", }) - with pytest.raises(OperationalException, - match="Trade download not supported for futures."): - download_data_main(config) - def test_download_data_main_data_invalid(mocker): patch_exchange(mocker, id="kraken") diff --git a/tests/data/test_history.py b/tests/data/test_history.py index a48d34aee..106babd63 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -170,6 +170,7 @@ def test_json_pair_data_filename(pair, timeframe, expected_result, candle_type): @pytest.mark.parametrize("pair,expected_result", [ ("ETH/BTC", 'freqtrade/hello/world/ETH_BTC-trades.json'), + ("ETH/USDT:USDT", 'freqtrade/hello/world/futures/ETH_USDT_USDT-trades.json'), ("Fabric Token/ETH", 'freqtrade/hello/world/Fabric_Token_ETH-trades.json'), ("ETHH20", 'freqtrade/hello/world/ETHH20-trades.json'), (".XBTBON2H", 'freqtrade/hello/world/_XBTBON2H-trades.json'), From b6040e270fa253fc089230decf37d7cce98d61a8 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 2 Mar 2024 13:08:58 +0100 Subject: [PATCH 04/12] Update data handlers to accept trading_mode for trade data related functions --- freqtrade/data/history/featherdatahandler.py | 14 ++++++---- freqtrade/data/history/hdf5datahandler.py | 14 ++++++---- freqtrade/data/history/idatahandler.py | 29 ++++++++++++++------ freqtrade/data/history/jsondatahandler.py | 14 ++++++---- freqtrade/data/history/parquetdatahandler.py | 16 +++++++---- 5 files changed, 57 insertions(+), 30 deletions(-) diff --git a/freqtrade/data/history/featherdatahandler.py b/freqtrade/data/history/featherdatahandler.py index 44d337836..6d57dbed7 100644 --- a/freqtrade/data/history/featherdatahandler.py +++ b/freqtrade/data/history/featherdatahandler.py @@ -5,7 +5,7 @@ from pandas import DataFrame, read_feather, to_datetime from freqtrade.configuration import TimeRange from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS -from freqtrade.enums import CandleType +from freqtrade.enums import CandleType, TradingMode from .idatahandler import IDataHandler @@ -82,14 +82,15 @@ class FeatherDataHandler(IDataHandler): """ raise NotImplementedError() - def _trades_store(self, pair: str, data: DataFrame) -> None: + def _trades_store(self, pair: str, data: DataFrame, trading_mode: TradingMode) -> None: """ Store trades data (list of Dicts) to file :param pair: Pair - used for filename :param data: Dataframe containing trades column sequence as in DEFAULT_TRADES_COLUMNS + :param trading_mode: Trading mode to use (used to determine the filename) """ - filename = self._pair_trades_filename(self._datadir, pair) + filename = self._pair_trades_filename(self._datadir, pair, trading_mode) self.create_dir_if_needed(filename) data.reset_index(drop=True).to_feather(filename, compression_level=9, compression='lz4') @@ -102,15 +103,18 @@ class FeatherDataHandler(IDataHandler): """ raise NotImplementedError() - def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> DataFrame: + def _trades_load( + self, pair: str, trading_mode: TradingMode, timerange: Optional[TimeRange] = None + ) -> DataFrame: """ Load a pair from file, either .json.gz or .json # TODO: respect timerange ... :param pair: Load trades for this pair + :param trading_mode: Trading mode to use (used to determine the filename) :param timerange: Timerange to load trades for - currently not implemented :return: Dataframe containing trades """ - filename = self._pair_trades_filename(self._datadir, pair) + filename = self._pair_trades_filename(self._datadir, pair, trading_mode) if not filename.exists(): return DataFrame(columns=DEFAULT_TRADES_COLUMNS) diff --git a/freqtrade/data/history/hdf5datahandler.py b/freqtrade/data/history/hdf5datahandler.py index b118bd7e0..cb2cdd884 100644 --- a/freqtrade/data/history/hdf5datahandler.py +++ b/freqtrade/data/history/hdf5datahandler.py @@ -6,7 +6,7 @@ import pandas as pd from freqtrade.configuration import TimeRange from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS -from freqtrade.enums import CandleType +from freqtrade.enums import CandleType, TradingMode from .idatahandler import IDataHandler @@ -100,17 +100,18 @@ class HDF5DataHandler(IDataHandler): """ raise NotImplementedError() - def _trades_store(self, pair: str, data: pd.DataFrame) -> None: + def _trades_store(self, pair: str, data: pd.DataFrame, trading_mode: TradingMode) -> None: """ Store trades data (list of Dicts) to file :param pair: Pair - used for filename :param data: Dataframe containing trades column sequence as in DEFAULT_TRADES_COLUMNS + :param trading_mode: Trading mode to use (used to determine the filename) """ key = self._pair_trades_key(pair) data.to_hdf( - self._pair_trades_filename(self._datadir, pair), key=key, + self._pair_trades_filename(self._datadir, pair, trading_mode), key=key, mode='a', complevel=9, complib='blosc', format='table', data_columns=['timestamp'] ) @@ -124,15 +125,18 @@ class HDF5DataHandler(IDataHandler): """ raise NotImplementedError() - def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> pd.DataFrame: + def _trades_load( + self, pair: str, trading_mode: TradingMode, timerange: Optional[TimeRange] = None + ) -> pd.DataFrame: """ Load a pair from h5 file. :param pair: Load trades for this pair + :param trading_mode: Trading mode to use (used to determine the filename) :param timerange: Timerange to load trades for - currently not implemented :return: Dataframe containing trades """ key = self._pair_trades_key(pair) - filename = self._pair_trades_filename(self._datadir, pair) + filename = self._pair_trades_filename(self._datadir, pair, trading_mode) if not filename.exists(): return pd.DataFrame(columns=DEFAULT_TRADES_COLUMNS) diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index 01c244f38..bcb31a7c8 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -172,12 +172,13 @@ class IDataHandler(ABC): return [cls.rebuild_pair_from_filename(match[0]) for match in _tmp if match] @abstractmethod - def _trades_store(self, pair: str, data: DataFrame) -> None: + def _trades_store(self, pair: str, data: DataFrame, trading_mode: TradingMode) -> None: """ Store trades data (list of Dicts) to file :param pair: Pair - used for filename :param data: Dataframe containing trades column sequence as in DEFAULT_TRADES_COLUMNS + :param trading_mode: Trading mode to use (used to determine the filename) """ @abstractmethod @@ -190,45 +191,55 @@ class IDataHandler(ABC): """ @abstractmethod - def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> DataFrame: + def _trades_load( + self, pair: str, trading_mode: TradingMode, timerange: Optional[TimeRange] = None + ) -> DataFrame: """ Load a pair from file, either .json.gz or .json :param pair: Load trades for this pair + :param trading_mode: Trading mode to use (used to determine the filename) :param timerange: Timerange to load trades for - currently not implemented :return: Dataframe containing trades """ - def trades_store(self, pair: str, data: DataFrame) -> None: + def trades_store(self, pair: str, data: DataFrame, trading_mode: TradingMode) -> None: """ Store trades data (list of Dicts) to file :param pair: Pair - used for filename :param data: Dataframe containing trades column sequence as in DEFAULT_TRADES_COLUMNS + :param trading_mode: Trading mode to use (used to determine the filename) """ # Filter on expected columns (will remove the actual date column). - self._trades_store(pair, data[DEFAULT_TRADES_COLUMNS]) + self._trades_store(pair, data[DEFAULT_TRADES_COLUMNS], trading_mode) - def trades_purge(self, pair: str) -> bool: + def trades_purge(self, pair: str, trading_mode: TradingMode) -> bool: """ Remove data for this pair :param pair: Delete data for this pair. + :param trading_mode: Trading mode to use (used to determine the filename) :return: True when deleted, false if file did not exist. """ - filename = self._pair_trades_filename(self._datadir, pair) + filename = self._pair_trades_filename(self._datadir, pair, trading_mode) if filename.exists(): filename.unlink() return True return False - def trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> DataFrame: + def trades_load( + self, pair: str, trading_mode: TradingMode, timerange: Optional[TimeRange] = None + ) -> DataFrame: """ Load a pair from file, either .json.gz or .json Removes duplicates in the process. :param pair: Load trades for this pair + :param trading_mode: Trading mode to use (used to determine the filename) :param timerange: Timerange to load trades for - currently not implemented :return: List of trades """ - trades = trades_df_remove_duplicates(self._trades_load(pair, timerange=timerange)) + trades = trades_df_remove_duplicates( + self._trades_load(pair, trading_mode, timerange=timerange) + ) trades = trades_convert_types(trades) return trades @@ -264,7 +275,7 @@ class IDataHandler(ABC): return filename @classmethod - def _pair_trades_filename(cls, datadir: Path, pair: str) -> Path: + def _pair_trades_filename(cls, datadir: Path, pair: str, trading_mode: TradingMode) -> Path: pair_s = misc.pair_to_filename(pair) if ':' in pair: # Futures pair ... diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index baa0c10a5..2d0333fed 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -8,7 +8,7 @@ from freqtrade import misc from freqtrade.configuration import TimeRange from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS from freqtrade.data.converter import trades_dict_to_list, trades_list_to_df -from freqtrade.enums import CandleType +from freqtrade.enums import CandleType, TradingMode from .idatahandler import IDataHandler @@ -94,14 +94,15 @@ class JsonDataHandler(IDataHandler): """ raise NotImplementedError() - def _trades_store(self, pair: str, data: DataFrame) -> None: + def _trades_store(self, pair: str, data: DataFrame, trading_mode: TradingMode) -> None: """ Store trades data (list of Dicts) to file :param pair: Pair - used for filename :param data: Dataframe containing trades column sequence as in DEFAULT_TRADES_COLUMNS + :param trading_mode: Trading mode to use (used to determine the filename) """ - filename = self._pair_trades_filename(self._datadir, pair) + filename = self._pair_trades_filename(self._datadir, pair, trading_mode) trades = data.values.tolist() misc.file_dump_json(filename, trades, is_zip=self._use_zip) @@ -114,15 +115,18 @@ class JsonDataHandler(IDataHandler): """ raise NotImplementedError() - def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> DataFrame: + def _trades_load( + self, pair: str, trading_mode: TradingMode, timerange: Optional[TimeRange] = None + ) -> DataFrame: """ Load a pair from file, either .json.gz or .json # TODO: respect timerange ... :param pair: Load trades for this pair + :param trading_mode: Trading mode to use (used to determine the filename) :param timerange: Timerange to load trades for - currently not implemented :return: Dataframe containing trades """ - filename = self._pair_trades_filename(self._datadir, pair) + filename = self._pair_trades_filename(self._datadir, pair, trading_mode) tradesdata = misc.file_load_json(filename) if not tradesdata: diff --git a/freqtrade/data/history/parquetdatahandler.py b/freqtrade/data/history/parquetdatahandler.py index c0b0cad63..01becdc84 100644 --- a/freqtrade/data/history/parquetdatahandler.py +++ b/freqtrade/data/history/parquetdatahandler.py @@ -4,8 +4,8 @@ from typing import Optional from pandas import DataFrame, read_parquet, to_datetime from freqtrade.configuration import TimeRange -from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, TradeList -from freqtrade.enums import CandleType +from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS +from freqtrade.enums import CandleType, TradingMode from .idatahandler import IDataHandler @@ -81,14 +81,15 @@ class ParquetDataHandler(IDataHandler): """ raise NotImplementedError() - def _trades_store(self, pair: str, data: DataFrame) -> None: + def _trades_store(self, pair: str, data: DataFrame, trading_mode: TradingMode) -> None: """ Store trades data (list of Dicts) to file :param pair: Pair - used for filename :param data: Dataframe containing trades column sequence as in DEFAULT_TRADES_COLUMNS + :param trading_mode: Trading mode to use (used to determine the filename) """ - filename = self._pair_trades_filename(self._datadir, pair) + filename = self._pair_trades_filename(self._datadir, pair, trading_mode) self.create_dir_if_needed(filename) data.reset_index(drop=True).to_parquet(filename) @@ -101,15 +102,18 @@ class ParquetDataHandler(IDataHandler): """ raise NotImplementedError() - def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList: + def _trades_load( + self, pair: str, trading_mode: TradingMode, timerange: Optional[TimeRange] = None + ) -> DataFrame: """ Load a pair from file, either .json.gz or .json # TODO: respect timerange ... :param pair: Load trades for this pair + :param trading_mode: Trading mode to use (used to determine the filename) :param timerange: Timerange to load trades for - currently not implemented :return: List of trades """ - filename = self._pair_trades_filename(self._datadir, pair) + filename = self._pair_trades_filename(self._datadir, pair, trading_mode) if not filename.exists(): return DataFrame(columns=DEFAULT_TRADES_COLUMNS) From 43103f51e55aada44c81a8a23fd29d886a3e3244 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 2 Mar 2024 13:10:57 +0100 Subject: [PATCH 05/12] Update functions that use datahandler trade methods --- freqtrade/data/converter/trade_converter.py | 15 ++++++------ .../data/converter/trade_converter_kraken.py | 3 ++- freqtrade/data/history/history_utils.py | 24 ++++++++++++------- 3 files changed, 25 insertions(+), 17 deletions(-) diff --git a/freqtrade/data/converter/trade_converter.py b/freqtrade/data/converter/trade_converter.py index 117f65bc6..0e5050a05 100644 --- a/freqtrade/data/converter/trade_converter.py +++ b/freqtrade/data/converter/trade_converter.py @@ -11,7 +11,7 @@ from pandas import DataFrame, to_datetime from freqtrade.configuration import TimeRange from freqtrade.constants import (DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, TRADES_DTYPES, Config, TradeList) -from freqtrade.enums import CandleType +from freqtrade.enums import CandleType, TradingMode from freqtrade.exceptions import OperationalException @@ -104,9 +104,9 @@ def convert_trades_to_ohlcv( logger.info(f"About to convert pairs: '{', '.join(pairs)}', " f"intervals: '{', '.join(timeframes)}' to {datadir}") - + trading_mode = TradingMode.FUTURES if candle_type != CandleType.SPOT else TradingMode.SPOT for pair in pairs: - trades = data_handler_trades.trades_load(pair) + trades = data_handler_trades.trades_load(pair, trading_mode) for timeframe in timeframes: if erase: if data_handler_ohlcv.ohlcv_purge(pair, timeframe, candle_type=candle_type): @@ -144,11 +144,12 @@ def convert_trades_format(config: Config, convert_from: str, convert_to: str, er if 'pairs' not in config: config['pairs'] = src.trades_get_pairs(config['datadir']) logger.info(f"Converting trades for {config['pairs']}") - + trading_mode: TradingMode = config.get('trading_mode', TradingMode.SPOT) for pair in config['pairs']: - data = src.trades_load(pair=pair) + data = src.trades_load(pair, trading_mode) logger.info(f"Converting {len(data)} trades for {pair}") - trg.trades_store(pair, data) + trg.trades_store(pair, data, trading_mode) + if erase and convert_from != convert_to: logger.info(f"Deleting source Trade data for {pair}.") - src.trades_purge(pair=pair) + src.trades_purge(pair, trading_mode) diff --git a/freqtrade/data/converter/trade_converter_kraken.py b/freqtrade/data/converter/trade_converter_kraken.py index b0fa11c25..80bd917af 100644 --- a/freqtrade/data/converter/trade_converter_kraken.py +++ b/freqtrade/data/converter/trade_converter_kraken.py @@ -7,6 +7,7 @@ from freqtrade.constants import DATETIME_PRINT_FORMAT, DEFAULT_TRADES_COLUMNS, C from freqtrade.data.converter.trade_converter import (trades_convert_types, trades_df_remove_duplicates) from freqtrade.data.history.idatahandler import get_datahandler +from freqtrade.enums import TradingMode from freqtrade.exceptions import OperationalException from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist from freqtrade.resolvers import ExchangeResolver @@ -79,4 +80,4 @@ def import_kraken_trades_from_csv(config: Config, convert_to: str): f"{trades_df['date'].min():{DATETIME_PRINT_FORMAT}} to " f"{trades_df['date'].max():{DATETIME_PRINT_FORMAT}}") - data_handler.trades_store(pair, trades_df) + data_handler.trades_store(pair, trades_df, TradingMode.SPOT) diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index 27e229973..3f9468f7a 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -13,7 +13,7 @@ from freqtrade.data.converter import (clean_ohlcv_dataframe, convert_trades_to_o ohlcv_to_dataframe, trades_df_remove_duplicates, trades_list_to_df) from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler -from freqtrade.enums import CandleType +from freqtrade.enums import CandleType, TradingMode from freqtrade.exceptions import OperationalException from freqtrade.exchange import Exchange from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist @@ -333,7 +333,8 @@ def _download_trades_history(exchange: Exchange, pair: str, *, new_pairs_days: int = 30, timerange: Optional[TimeRange] = None, - data_handler: IDataHandler + data_handler: IDataHandler, + trading_mode: TradingMode, ) -> bool: """ Download trade history from the exchange. @@ -349,7 +350,7 @@ def _download_trades_history(exchange: Exchange, if timerange.stoptype == 'date': until = timerange.stopts * 1000 - trades = data_handler.trades_load(pair) + trades = data_handler.trades_load(pair, trading_mode) # TradesList columns are defined in constants.DEFAULT_TRADES_COLUMNS # DEFAULT_TRADES_COLUMNS: 0 -> timestamp @@ -388,7 +389,7 @@ def _download_trades_history(exchange: Exchange, trades = concat([trades, new_trades_df], axis=0) # Remove duplicates to make sure we're not storing data we don't need trades = trades_df_remove_duplicates(trades) - data_handler.trades_store(pair, data=trades) + data_handler.trades_store(pair, trades, trading_mode) logger.debug("New Start: %s", 'None' if trades.empty else f"{trades.iloc[0]['date']:{DATETIME_PRINT_FORMAT}}") @@ -405,8 +406,10 @@ def _download_trades_history(exchange: Exchange, def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: Path, - timerange: TimeRange, new_pairs_days: int = 30, - erase: bool = False, data_format: str = 'feather') -> List[str]: + timerange: TimeRange, trading_mode: TradingMode, + new_pairs_days: int = 30, + erase: bool = False, data_format: str = 'feather', + ) -> List[str]: """ Refresh stored trades data for backtesting and hyperopt operations. Used by freqtrade download-data subcommand. @@ -421,7 +424,7 @@ def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: continue if erase: - if data_handler.trades_purge(pair): + if data_handler.trades_purge(pair, trading_mode): logger.info(f'Deleting existing data for pair {pair}.') logger.info(f'Downloading trades for pair {pair}.') @@ -429,7 +432,8 @@ def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: pair=pair, new_pairs_days=new_pairs_days, timerange=timerange, - data_handler=data_handler) + data_handler=data_handler, + trading_mode=trading_mode) return pairs_not_available @@ -521,7 +525,9 @@ def download_data_main(config: Config) -> None: pairs_not_available = refresh_backtest_trades_data( exchange, pairs=expanded_pairs, datadir=config['datadir'], timerange=timerange, new_pairs_days=config['new_pairs_days'], - erase=bool(config.get('erase')), data_format=config['dataformat_trades']) + erase=bool(config.get('erase')), data_format=config['dataformat_trades'], + trading_mode=config.get('trading_mode', TradingMode.SPOT), + ) # Convert downloaded trade data to different timeframes convert_trades_to_ohlcv( From 66e43f2fe86e9948ce800e61b8362788e742c8f6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 2 Mar 2024 13:17:45 +0100 Subject: [PATCH 06/12] Adjust tests for new arguments --- tests/data/test_datahandler.py | 24 ++++++------- tests/data/test_history.py | 41 ++++++++++++----------- tests/data/test_trade_converter_kraken.py | 3 +- 3 files changed, 36 insertions(+), 32 deletions(-) diff --git a/tests/data/test_datahandler.py b/tests/data/test_datahandler.py index a0a37c393..1217c35ad 100644 --- a/tests/data/test_datahandler.py +++ b/tests/data/test_datahandler.py @@ -261,11 +261,11 @@ def test_datahandler_trades_not_supported(datahandler, testdatadir, ): def test_jsondatahandler_trades_load(testdatadir, caplog): dh = JsonGzDataHandler(testdatadir) logmsg = "Old trades format detected - converting" - dh.trades_load('XRP/ETH') + dh.trades_load('XRP/ETH', TradingMode.SPOT) assert not log_has(logmsg, caplog) # Test conversation is happening - dh.trades_load('XRP/OLD') + dh.trades_load('XRP/OLD', TradingMode.SPOT) assert log_has(logmsg, caplog) @@ -300,16 +300,16 @@ def test_datahandler_trades_get_pairs(testdatadir, datahandler, expected): def test_hdf5datahandler_trades_load(testdatadir): dh = get_datahandler(testdatadir, 'hdf5') - trades = dh.trades_load('XRP/ETH') + trades = dh.trades_load('XRP/ETH', TradingMode.SPOT) assert isinstance(trades, DataFrame) - trades1 = dh.trades_load('UNITTEST/NONEXIST') + trades1 = dh.trades_load('UNITTEST/NONEXIST', TradingMode.SPOT) assert isinstance(trades1, DataFrame) assert trades1.empty # data goes from 2019-10-11 - 2019-10-13 timerange = TimeRange.parse_timerange('20191011-20191012') - trades2 = dh._trades_load('XRP/ETH', timerange) + trades2 = dh._trades_load('XRP/ETH', TradingMode.SPOT, timerange) assert len(trades) > len(trades2) # Check that ID is None (If it's nan, it's wrong) assert trades2.iloc[0]['type'] is None @@ -451,13 +451,13 @@ def test_hdf5datahandler_ohlcv_purge(mocker, testdatadir): @pytest.mark.parametrize('datahandler', ['jsongz', 'hdf5', 'feather', 'parquet']) def test_datahandler_trades_load(testdatadir, datahandler): dh = get_datahandler(testdatadir, datahandler) - trades = dh.trades_load('XRP/ETH') + trades = dh.trades_load('XRP/ETH', TradingMode.SPOT) assert isinstance(trades, DataFrame) assert trades.iloc[0]['timestamp'] == 1570752011620 assert trades.iloc[0]['date'] == Timestamp('2019-10-11 00:00:11.620000+0000') assert trades.iloc[-1]['cost'] == 0.1986231 - trades1 = dh.trades_load('UNITTEST/NONEXIST') + trades1 = dh.trades_load('UNITTEST/NONEXIST', TradingMode.SPOT) assert isinstance(trades, DataFrame) assert trades1.empty @@ -465,15 +465,15 @@ def test_datahandler_trades_load(testdatadir, datahandler): @pytest.mark.parametrize('datahandler', ['jsongz', 'hdf5', 'feather', 'parquet']) def test_datahandler_trades_store(testdatadir, tmp_path, datahandler): dh = get_datahandler(testdatadir, datahandler) - trades = dh.trades_load('XRP/ETH') + trades = dh.trades_load('XRP/ETH', TradingMode.SPOT) dh1 = get_datahandler(tmp_path, datahandler) - dh1.trades_store('XRP/NEW', trades) + dh1.trades_store('XRP/NEW', trades, TradingMode.SPOT) file = tmp_path / f'XRP_NEW-trades.{dh1._get_file_extension()}' assert file.is_file() # Load trades back - trades_new = dh1.trades_load('XRP/NEW') + trades_new = dh1.trades_load('XRP/NEW', TradingMode.SPOT) assert_frame_equal(trades, trades_new, check_exact=True) assert len(trades_new) == len(trades) @@ -483,11 +483,11 @@ def test_datahandler_trades_purge(mocker, testdatadir, datahandler): mocker.patch.object(Path, "exists", MagicMock(return_value=False)) unlinkmock = mocker.patch.object(Path, "unlink", MagicMock()) dh = get_datahandler(testdatadir, datahandler) - assert not dh.trades_purge('UNITTEST/NONEXIST') + assert not dh.trades_purge('UNITTEST/NONEXIST', TradingMode.SPOT) assert unlinkmock.call_count == 0 mocker.patch.object(Path, "exists", MagicMock(return_value=True)) - assert dh.trades_purge('UNITTEST/NONEXIST') + assert dh.trades_purge('UNITTEST/NONEXIST', TradingMode.SPOT) assert unlinkmock.call_count == 1 diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 106babd63..a3fe492b7 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -23,7 +23,7 @@ from freqtrade.data.history.history_utils import (_download_pair_history, _downl validate_backtest_data) from freqtrade.data.history.idatahandler import get_datahandler from freqtrade.data.history.jsondatahandler import JsonDataHandler, JsonGzDataHandler -from freqtrade.enums import CandleType +from freqtrade.enums import CandleType, TradingMode from freqtrade.exchange import timeframe_to_minutes from freqtrade.misc import file_dump_json from freqtrade.resolvers import StrategyResolver @@ -168,21 +168,21 @@ def test_json_pair_data_filename(pair, timeframe, expected_result, candle_type): assert fn == Path(expected_result + '.gz') -@pytest.mark.parametrize("pair,expected_result", [ - ("ETH/BTC", 'freqtrade/hello/world/ETH_BTC-trades.json'), - ("ETH/USDT:USDT", 'freqtrade/hello/world/futures/ETH_USDT_USDT-trades.json'), - ("Fabric Token/ETH", 'freqtrade/hello/world/Fabric_Token_ETH-trades.json'), - ("ETHH20", 'freqtrade/hello/world/ETHH20-trades.json'), - (".XBTBON2H", 'freqtrade/hello/world/_XBTBON2H-trades.json'), - ("ETHUSD.d", 'freqtrade/hello/world/ETHUSD_d-trades.json'), - ("ACC_OLD_BTC", 'freqtrade/hello/world/ACC_OLD_BTC-trades.json'), +@pytest.mark.parametrize("pair,trading_mode,expected_result", [ + ("ETH/BTC", '', 'freqtrade/hello/world/ETH_BTC-trades.json'), + ("ETH/USDT:USDT", 'futures', 'freqtrade/hello/world/futures/ETH_USDT_USDT-trades.json'), + ("Fabric Token/ETH", '', 'freqtrade/hello/world/Fabric_Token_ETH-trades.json'), + ("ETHH20", '', 'freqtrade/hello/world/ETHH20-trades.json'), + (".XBTBON2H", '', 'freqtrade/hello/world/_XBTBON2H-trades.json'), + ("ETHUSD.d", '', 'freqtrade/hello/world/ETHUSD_d-trades.json'), + ("ACC_OLD_BTC", '', 'freqtrade/hello/world/ACC_OLD_BTC-trades.json'), ]) -def test_json_pair_trades_filename(pair, expected_result): - fn = JsonDataHandler._pair_trades_filename(Path('freqtrade/hello/world'), pair) +def test_json_pair_trades_filename(pair, trading_mode, expected_result): + fn = JsonDataHandler._pair_trades_filename(Path('freqtrade/hello/world'), pair, trading_mode) assert isinstance(fn, Path) assert fn == Path(expected_result) - fn = JsonGzDataHandler._pair_trades_filename(Path('freqtrade/hello/world'), pair) + fn = JsonGzDataHandler._pair_trades_filename(Path('freqtrade/hello/world'), pair, trading_mode) assert isinstance(fn, Path) assert fn == Path(expected_result + '.gz') @@ -560,7 +560,8 @@ def test_refresh_backtest_trades_data(mocker, default_conf, markets, caplog, tes unavailable_pairs = refresh_backtest_trades_data(exchange=ex, pairs=["ETH/BTC", "XRP/BTC", "XRP/ETH"], datadir=testdatadir, - timerange=timerange, erase=True + timerange=timerange, erase=True, + trading_mode=TradingMode.SPOT, ) assert dl_mock.call_count == 2 @@ -585,7 +586,7 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad assert not file1.is_file() assert _download_trades_history(data_handler=data_handler, exchange=exchange, - pair='ETH/BTC') + pair='ETH/BTC', trading_mode=TradingMode.SPOT) assert log_has("Current Amount of trades: 0", caplog) assert log_has("New Amount of trades: 6", caplog) assert ght_mock.call_count == 1 @@ -598,8 +599,9 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad since_time = int(trades_history[-3][0] // 1000) since_time2 = int(trades_history[-1][0] // 1000) timerange = TimeRange('date', None, since_time, 0) - assert _download_trades_history(data_handler=data_handler, exchange=exchange, - pair='ETH/BTC', timerange=timerange) + assert _download_trades_history( + data_handler=data_handler, exchange=exchange, pair='ETH/BTC', + timerange=timerange, trading_mode=TradingMode.SPOT) assert ght_mock.call_count == 1 # Check this in seconds - since we had to convert to seconds above too. @@ -612,7 +614,7 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad caplog.clear() assert not _download_trades_history(data_handler=data_handler, exchange=exchange, - pair='ETH/BTC') + pair='ETH/BTC', trading_mode=TradingMode.SPOT) assert log_has_re('Failed to download historic trades for pair: "ETH/BTC".*', caplog) file2 = tmp_path / 'XRP_ETH-trades.json.gz' @@ -624,8 +626,9 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad since_time = int(trades_history[0][0] // 1000) - 500 timerange = TimeRange('date', None, since_time, 0) - assert _download_trades_history(data_handler=data_handler, exchange=exchange, - pair='XRP/ETH', timerange=timerange) + assert _download_trades_history( + data_handler=data_handler, exchange=exchange, pair='XRP/ETH', + timerange=timerange, trading_mode=TradingMode.SPOT) assert ght_mock.call_count == 1 diff --git a/tests/data/test_trade_converter_kraken.py b/tests/data/test_trade_converter_kraken.py index 91de303fb..ba9221e0a 100644 --- a/tests/data/test_trade_converter_kraken.py +++ b/tests/data/test_trade_converter_kraken.py @@ -6,6 +6,7 @@ import pytest from freqtrade.data.converter.trade_converter_kraken import import_kraken_trades_from_csv from freqtrade.data.history.idatahandler import get_datahandler +from freqtrade.enums import TradingMode from freqtrade.exceptions import OperationalException from tests.conftest import EXMS, log_has, log_has_re, patch_exchange @@ -40,7 +41,7 @@ def test_import_kraken_trades_from_csv(testdatadir, tmp_path, caplog, default_co assert dstfile.is_file() dh = get_datahandler(tmp_path, 'feather') - trades = dh.trades_load('BCH_EUR') + trades = dh.trades_load('BCH_EUR', TradingMode.SPOT) assert len(trades) == 340 assert trades['date'].min().to_pydatetime() == datetime(2023, 1, 1, 0, 3, 56, From 5e7868a28dbadf96cefd27f280cd93016e0de6d9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 2 Mar 2024 13:22:41 +0100 Subject: [PATCH 07/12] Remove block from download-trades for futures --- freqtrade/data/history/history_utils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index 3f9468f7a..208859cd3 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -520,8 +520,6 @@ def download_data_main(config: Config) -> None: # Start downloading try: if config.get('download_trades'): - if config.get('trading_mode') == 'futures': - raise OperationalException("Trade download not supported for futures.") pairs_not_available = refresh_backtest_trades_data( exchange, pairs=expanded_pairs, datadir=config['datadir'], timerange=timerange, new_pairs_days=config['new_pairs_days'], From 09d763b604453ac702bc2d6e5d9adfbc3947fd69 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 2 Mar 2024 13:23:03 +0100 Subject: [PATCH 08/12] convert-trades should do proper pair expansion to support regex in pairlists --- freqtrade/commands/data_commands.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index d3600e3ef..b183d403b 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -11,6 +11,7 @@ from freqtrade.data.history import download_data_main from freqtrade.enums import CandleType, RunMode, TradingMode from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_minutes +from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist from freqtrade.resolvers import ExchangeResolver from freqtrade.util.migrations import migrate_data @@ -62,10 +63,17 @@ def start_convert_trades(args: Dict[str, Any]) -> None: for timeframe in config['timeframes']: exchange.validate_timeframes(timeframe) + available_pairs = [ + p for p in exchange.get_markets( + tradable_only=True, active_only=not config.get('include_inactive') + ).keys() + ] + + expanded_pairs = dynamic_expand_pairlist(config, available_pairs) # Convert downloaded trade data to different timeframes convert_trades_to_ohlcv( - pairs=config.get('pairs', []), timeframes=config['timeframes'], + pairs=expanded_pairs, timeframes=config['timeframes'], datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')), data_format_ohlcv=config['dataformat_ohlcv'], data_format_trades=config['dataformat_trades'], From 99da6f70c2959f75e66850237086f20b8de007c6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 2 Mar 2024 13:27:54 +0100 Subject: [PATCH 09/12] Fix failing test due to new approach for convert-trades --- tests/commands/test_commands.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 1ab9d2202..2252ff9f4 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -837,10 +837,11 @@ def test_download_data_data_invalid(mocker): start_download_data(pargs) -def test_start_convert_trades(mocker, caplog): +def test_start_convert_trades(mocker): convert_mock = mocker.patch('freqtrade.commands.data_commands.convert_trades_to_ohlcv', MagicMock(return_value=[])) patch_exchange(mocker) + mocker.patch(f'{EXMS}.get_markets') mocker.patch(f'{EXMS}.markets', PropertyMock(return_value={})) args = [ "trades-to-ohlcv", From 7ed7ed4081a1dedaecd025678268a71e0543f980 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 2 Mar 2024 15:38:36 +0100 Subject: [PATCH 10/12] Accept trading-mode for trades-to-ohlcv command --- freqtrade/commands/arguments.py | 3 ++- freqtrade/data/converter/trade_converter.py | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index f72164675..191f07910 100755 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -69,7 +69,8 @@ ARGS_CONVERT_DATA_TRADES = ["pairs", "format_from_trades", "format_to", "erase", ARGS_CONVERT_DATA = ["pairs", "format_from", "format_to", "erase", "exchange"] ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes", "trading_mode", "candle_types"] -ARGS_CONVERT_TRADES = ["pairs", "timeframes", "exchange", "dataformat_ohlcv", "dataformat_trades"] +ARGS_CONVERT_TRADES = ["pairs", "timeframes", "exchange", "dataformat_ohlcv", "dataformat_trades", + "trading_mode"] ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs", "trading_mode", "show_timerange"] diff --git a/freqtrade/data/converter/trade_converter.py b/freqtrade/data/converter/trade_converter.py index 0e5050a05..1c8327ec3 100644 --- a/freqtrade/data/converter/trade_converter.py +++ b/freqtrade/data/converter/trade_converter.py @@ -99,8 +99,6 @@ def convert_trades_to_ohlcv( from freqtrade.data.history.idatahandler import get_datahandler data_handler_trades = get_datahandler(datadir, data_format=data_format_trades) data_handler_ohlcv = get_datahandler(datadir, data_format=data_format_ohlcv) - if not pairs: - pairs = data_handler_trades.trades_get_pairs(datadir) logger.info(f"About to convert pairs: '{', '.join(pairs)}', " f"intervals: '{', '.join(timeframes)}' to {datadir}") From fcb16098d86779b17cdc889553d653547b5b3a51 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 2 Mar 2024 15:40:43 +0100 Subject: [PATCH 11/12] Reduce Error level when converting trades --- freqtrade/data/converter/trade_converter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/data/converter/trade_converter.py b/freqtrade/data/converter/trade_converter.py index 1c8327ec3..682430994 100644 --- a/freqtrade/data/converter/trade_converter.py +++ b/freqtrade/data/converter/trade_converter.py @@ -114,7 +114,7 @@ def convert_trades_to_ohlcv( # Store ohlcv data_handler_ohlcv.ohlcv_store(pair, timeframe, data=ohlcv, candle_type=candle_type) except ValueError: - logger.exception(f'Could not convert {pair} to OHLCV.') + logger.warning(f'Could not convert {pair} to OHLCV.') def convert_trades_format(config: Config, convert_from: str, convert_to: str, erase: bool): From ed8469f23ac32fe0ca6e6d6eb06b9cd7e4f07a70 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 3 Mar 2024 13:12:42 +0100 Subject: [PATCH 12/12] use trading_mode to determine trades file location --- freqtrade/data/history/idatahandler.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index bcb31a7c8..fbaded640 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -277,9 +277,8 @@ class IDataHandler(ABC): @classmethod def _pair_trades_filename(cls, datadir: Path, pair: str, trading_mode: TradingMode) -> Path: pair_s = misc.pair_to_filename(pair) - if ':' in pair: + if trading_mode == TradingMode.FUTURES: # Futures pair ... - # TODO: this should not rely on ";" in the pairname. datadir = datadir.joinpath('futures') filename = datadir.joinpath(f'{pair_s}-trades.{cls._get_file_extension()}')