CTYPTO/freqtrade
1
0
Fork 0
mirror of https://github.com/freqtrade/freqtrade.git synced 2026-02-03 12:51:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #9890 from freqtrade/feat/dl-trades_futures

Browse Source 
Support for --dl-trades for futures
This commit is contained in:
Matthias
2024-03-03 15:00:02 +01:00
committed by GitHub
parent f8cc2a6e74 ed8469f23a
commit 6bdb651573
16 changed files with 149 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
freqtrade/commands/arguments.py
View File

@@ -69,7 +69,8 @@ ARGS_CONVERT_DATA_TRADES = ["pairs", "format_from_trades", "format_to", "erase",
ARGS_CONVERT_DATA = ["pairs", "format_from", "format_to", "erase", "exchange"]
ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes", "trading_mode", "candle_types"]
ARGS_CONVERT_TRADES = ["pairs", "timeframes", "exchange", "dataformat_ohlcv", "dataformat_trades"]
ARGS_CONVERT_TRADES = ["pairs", "timeframes", "exchange", "dataformat_ohlcv", "dataformat_trades",
"trading_mode"]
ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs", "trading_mode", "show_timerange"]

13
freqtrade/commands/data_commands.py
View File

@@ -8,9 +8,10 @@ from freqtrade.constants import DATETIME_PRINT_FORMAT, DL_DATA_TIMEFRAMES, Confi
from freqtrade.data.converter import (convert_ohlcv_format, convert_trades_format,
convert_trades_to_ohlcv)
from freqtrade.data.history import download_data_main
from freqtrade.enums import RunMode, TradingMode
from freqtrade.enums import CandleType, RunMode, TradingMode
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import timeframe_to_minutes
from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist
from freqtrade.resolvers import ExchangeResolver
from freqtrade.util.migrations import migrate_data
@@ -62,13 +63,21 @@ def start_convert_trades(args: Dict[str, Any]) -> None:
for timeframe in config['timeframes']:
exchange.validate_timeframes(timeframe)
available_pairs = [
p for p in exchange.get_markets(
tradable_only=True, active_only=not config.get('include_inactive')
).keys()
]
expanded_pairs = dynamic_expand_pairlist(config, available_pairs)
# Convert downloaded trade data to different timeframes
convert_trades_to_ohlcv(
pairs=config.get('pairs', []), timeframes=config['timeframes'],
pairs=expanded_pairs, timeframes=config['timeframes'],
datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')),
data_format_ohlcv=config['dataformat_ohlcv'],
data_format_trades=config['dataformat_trades'],
candle_type=config.get('candle_type_def', CandleType.SPOT)
)

27
freqtrade/data/converter/trade_converter.py
View File

@@ -11,7 +11,7 @@ from pandas import DataFrame, to_datetime
from freqtrade.configuration import TimeRange
from freqtrade.constants import (DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, TRADES_DTYPES,
Config, TradeList)
from freqtrade.enums import CandleType
from freqtrade.enums import CandleType, TradingMode
from freqtrade.exceptions import OperationalException
@@ -88,10 +88,10 @@ def convert_trades_to_ohlcv(
timeframes: List[str],
datadir: Path,
timerange: TimeRange,
erase: bool = False,
data_format_ohlcv: str = 'feather',
data_format_trades: str = 'feather',
candle_type: CandleType = CandleType.SPOT
erase: bool,
data_format_ohlcv: str,
data_format_trades: str,
candle_type: CandleType,
) -> None:
"""
Convert stored trades data to ohlcv data
@@ -99,14 +99,12 @@ def convert_trades_to_ohlcv(
from freqtrade.data.history.idatahandler import get_datahandler
data_handler_trades = get_datahandler(datadir, data_format=data_format_trades)
data_handler_ohlcv = get_datahandler(datadir, data_format=data_format_ohlcv)
if not pairs:
pairs = data_handler_trades.trades_get_pairs(datadir)
logger.info(f"About to convert pairs: '{', '.join(pairs)}', "
f"intervals: '{', '.join(timeframes)}' to {datadir}")
trading_mode = TradingMode.FUTURES if candle_type != CandleType.SPOT else TradingMode.SPOT
for pair in pairs:
trades = data_handler_trades.trades_load(pair)
trades = data_handler_trades.trades_load(pair, trading_mode)
for timeframe in timeframes:
if erase:
if data_handler_ohlcv.ohlcv_purge(pair, timeframe, candle_type=candle_type):
@@ -116,7 +114,7 @@ def convert_trades_to_ohlcv(
# Store ohlcv
data_handler_ohlcv.ohlcv_store(pair, timeframe, data=ohlcv, candle_type=candle_type)
except ValueError:
logger.exception(f'Could not convert {pair} to OHLCV.')
logger.warning(f'Could not convert {pair} to OHLCV.')
def convert_trades_format(config: Config, convert_from: str, convert_to: str, erase: bool):
@@ -144,11 +142,12 @@ def convert_trades_format(config: Config, convert_from: str, convert_to: str, er
if 'pairs' not in config:
config['pairs'] = src.trades_get_pairs(config['datadir'])
logger.info(f"Converting trades for {config['pairs']}")
trading_mode: TradingMode = config.get('trading_mode', TradingMode.SPOT)
for pair in config['pairs']:
data = src.trades_load(pair=pair)
data = src.trades_load(pair, trading_mode)
logger.info(f"Converting {len(data)} trades for {pair}")
trg.trades_store(pair, data)
trg.trades_store(pair, data, trading_mode)
if erase and convert_from != convert_to:
logger.info(f"Deleting source Trade data for {pair}.")
src.trades_purge(pair=pair)
src.trades_purge(pair, trading_mode)

3
freqtrade/data/converter/trade_converter_kraken.py
View File

@@ -7,6 +7,7 @@ from freqtrade.constants import DATETIME_PRINT_FORMAT, DEFAULT_TRADES_COLUMNS, C
from freqtrade.data.converter.trade_converter import (trades_convert_types,
trades_df_remove_duplicates)
from freqtrade.data.history.idatahandler import get_datahandler
from freqtrade.enums import TradingMode
from freqtrade.exceptions import OperationalException
from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist
from freqtrade.resolvers import ExchangeResolver
@@ -79,4 +80,4 @@ def import_kraken_trades_from_csv(config: Config, convert_to: str):
f"{trades_df['date'].min():{DATETIME_PRINT_FORMAT}} to "
f"{trades_df['date'].max():{DATETIME_PRINT_FORMAT}}")
data_handler.trades_store(pair, trades_df)
data_handler.trades_store(pair, trades_df, TradingMode.SPOT)

14
freqtrade/data/history/featherdatahandler.py
View File

@@ -5,7 +5,7 @@ from pandas import DataFrame, read_feather, to_datetime
from freqtrade.configuration import TimeRange
from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS
from freqtrade.enums import CandleType
from freqtrade.enums import CandleType, TradingMode
from .idatahandler import IDataHandler
@@ -82,14 +82,15 @@ class FeatherDataHandler(IDataHandler):
"""
raise NotImplementedError()
def _trades_store(self, pair: str, data: DataFrame) -> None:
def _trades_store(self, pair: str, data: DataFrame, trading_mode: TradingMode) -> None:
"""
Store trades data (list of Dicts) to file
:param pair: Pair - used for filename
:param data: Dataframe containing trades
column sequence as in DEFAULT_TRADES_COLUMNS
:param trading_mode: Trading mode to use (used to determine the filename)
"""
filename = self._pair_trades_filename(self._datadir, pair)
filename = self._pair_trades_filename(self._datadir, pair, trading_mode)
self.create_dir_if_needed(filename)
data.reset_index(drop=True).to_feather(filename, compression_level=9, compression='lz4')
@@ -102,15 +103,18 @@ class FeatherDataHandler(IDataHandler):
"""
raise NotImplementedError()
def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> DataFrame:
def _trades_load(
self, pair: str, trading_mode: TradingMode, timerange: Optional[TimeRange] = None
) -> DataFrame:
"""
Load a pair from file, either .json.gz or .json
# TODO: respect timerange ...
:param pair: Load trades for this pair
:param trading_mode: Trading mode to use (used to determine the filename)
:param timerange: Timerange to load trades for - currently not implemented
:return: Dataframe containing trades
"""
filename = self._pair_trades_filename(self._datadir, pair)
filename = self._pair_trades_filename(self._datadir, pair, trading_mode)
if not filename.exists():
return DataFrame(columns=DEFAULT_TRADES_COLUMNS)

14
freqtrade/data/history/hdf5datahandler.py
View File

@@ -6,7 +6,7 @@ import pandas as pd
from freqtrade.configuration import TimeRange
from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS
from freqtrade.enums import CandleType
from freqtrade.enums import CandleType, TradingMode
from .idatahandler import IDataHandler
@@ -100,17 +100,18 @@ class HDF5DataHandler(IDataHandler):
"""
raise NotImplementedError()
def _trades_store(self, pair: str, data: pd.DataFrame) -> None:
def _trades_store(self, pair: str, data: pd.DataFrame, trading_mode: TradingMode) -> None:
"""
Store trades data (list of Dicts) to file
:param pair: Pair - used for filename
:param data: Dataframe containing trades
column sequence as in DEFAULT_TRADES_COLUMNS
:param trading_mode: Trading mode to use (used to determine the filename)
"""
key = self._pair_trades_key(pair)
data.to_hdf(
self._pair_trades_filename(self._datadir, pair), key=key,
self._pair_trades_filename(self._datadir, pair, trading_mode), key=key,
mode='a', complevel=9, complib='blosc',
format='table', data_columns=['timestamp']
)
@@ -124,15 +125,18 @@ class HDF5DataHandler(IDataHandler):
"""
raise NotImplementedError()
def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> pd.DataFrame:
def _trades_load(
self, pair: str, trading_mode: TradingMode, timerange: Optional[TimeRange] = None
) -> pd.DataFrame:
"""
Load a pair from h5 file.
:param pair: Load trades for this pair
:param trading_mode: Trading mode to use (used to determine the filename)
:param timerange: Timerange to load trades for - currently not implemented
:return: Dataframe containing trades
"""
key = self._pair_trades_key(pair)
filename = self._pair_trades_filename(self._datadir, pair)
filename = self._pair_trades_filename(self._datadir, pair, trading_mode)
if not filename.exists():
return pd.DataFrame(columns=DEFAULT_TRADES_COLUMNS)

27
freqtrade/data/history/history_utils.py
View File

@@ -13,7 +13,7 @@ from freqtrade.data.converter import (clean_ohlcv_dataframe, convert_trades_to_o
ohlcv_to_dataframe, trades_df_remove_duplicates,
trades_list_to_df)
from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler
from freqtrade.enums import CandleType
from freqtrade.enums import CandleType, TradingMode
from freqtrade.exceptions import OperationalException
from freqtrade.exchange import Exchange
from freqtrade.plugins.pairlist.pairlist_helpers import dynamic_expand_pairlist
@@ -333,7 +333,8 @@ def _download_trades_history(exchange: Exchange,
pair: str, *,
new_pairs_days: int = 30,
timerange: Optional[TimeRange] = None,
data_handler: IDataHandler
data_handler: IDataHandler,
trading_mode: TradingMode,
) -> bool:
"""
Download trade history from the exchange.
@@ -349,7 +350,7 @@ def _download_trades_history(exchange: Exchange,
if timerange.stoptype == 'date':
until = timerange.stopts * 1000
trades = data_handler.trades_load(pair)
trades = data_handler.trades_load(pair, trading_mode)
# TradesList columns are defined in constants.DEFAULT_TRADES_COLUMNS
# DEFAULT_TRADES_COLUMNS: 0 -> timestamp
@@ -388,7 +389,7 @@ def _download_trades_history(exchange: Exchange,
trades = concat([trades, new_trades_df], axis=0)
# Remove duplicates to make sure we're not storing data we don't need
trades = trades_df_remove_duplicates(trades)
data_handler.trades_store(pair, data=trades)
data_handler.trades_store(pair, trades, trading_mode)
logger.debug("New Start: %s", 'None' if trades.empty else
f"{trades.iloc[0]['date']:{DATETIME_PRINT_FORMAT}}")
@@ -405,8 +406,10 @@ def _download_trades_history(exchange: Exchange,
def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: Path,
timerange: TimeRange, new_pairs_days: int = 30,
erase: bool = False, data_format: str = 'feather') -> List[str]:
timerange: TimeRange, trading_mode: TradingMode,
new_pairs_days: int = 30,
erase: bool = False, data_format: str = 'feather',
) -> List[str]:
"""
Refresh stored trades data for backtesting and hyperopt operations.
Used by freqtrade download-data subcommand.
@@ -421,7 +424,7 @@ def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir:
continue
if erase:
if data_handler.trades_purge(pair):
if data_handler.trades_purge(pair, trading_mode):
logger.info(f'Deleting existing data for pair {pair}.')
logger.info(f'Downloading trades for pair {pair}.')
@@ -429,7 +432,8 @@ def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir:
pair=pair,
new_pairs_days=new_pairs_days,
timerange=timerange,
data_handler=data_handler)
data_handler=data_handler,
trading_mode=trading_mode)
return pairs_not_available
@@ -516,12 +520,12 @@ def download_data_main(config: Config) -> None:
# Start downloading
try:
if config.get('download_trades'):
if config.get('trading_mode') == 'futures':
raise OperationalException("Trade download not supported for futures.")
pairs_not_available = refresh_backtest_trades_data(
exchange, pairs=expanded_pairs, datadir=config['datadir'],
timerange=timerange, new_pairs_days=config['new_pairs_days'],
erase=bool(config.get('erase')), data_format=config['dataformat_trades'])
erase=bool(config.get('erase')), data_format=config['dataformat_trades'],
trading_mode=config.get('trading_mode', TradingMode.SPOT),
)
# Convert downloaded trade data to different timeframes
convert_trades_to_ohlcv(
@@ -529,6 +533,7 @@ def download_data_main(config: Config) -> None:
datadir=config['datadir'], timerange=timerange, erase=bool(config.get('erase')),
data_format_ohlcv=config['dataformat_ohlcv'],
data_format_trades=config['dataformat_trades'],
candle_type=config.get('candle_type_def', CandleType.SPOT),
)
else:
if not exchange.get_option('ohlcv_has_history', True):

33
freqtrade/data/history/idatahandler.py
View File

@@ -172,12 +172,13 @@ class IDataHandler(ABC):
return [cls.rebuild_pair_from_filename(match[0]) for match in _tmp if match]
@abstractmethod
def _trades_store(self, pair: str, data: DataFrame) -> None:
def _trades_store(self, pair: str, data: DataFrame, trading_mode: TradingMode) -> None:
"""
Store trades data (list of Dicts) to file
:param pair: Pair - used for filename
:param data: Dataframe containing trades
column sequence as in DEFAULT_TRADES_COLUMNS
:param trading_mode: Trading mode to use (used to determine the filename)
"""
@abstractmethod
@@ -190,45 +191,55 @@ class IDataHandler(ABC):
"""
@abstractmethod
def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> DataFrame:
def _trades_load(
self, pair: str, trading_mode: TradingMode, timerange: Optional[TimeRange] = None
) -> DataFrame:
"""
Load a pair from file, either .json.gz or .json
:param pair: Load trades for this pair
:param trading_mode: Trading mode to use (used to determine the filename)
:param timerange: Timerange to load trades for - currently not implemented
:return: Dataframe containing trades
"""
def trades_store(self, pair: str, data: DataFrame) -> None:
def trades_store(self, pair: str, data: DataFrame, trading_mode: TradingMode) -> None:
"""
Store trades data (list of Dicts) to file
:param pair: Pair - used for filename
:param data: Dataframe containing trades
column sequence as in DEFAULT_TRADES_COLUMNS
:param trading_mode: Trading mode to use (used to determine the filename)
"""
# Filter on expected columns (will remove the actual date column).
self._trades_store(pair, data[DEFAULT_TRADES_COLUMNS])
self._trades_store(pair, data[DEFAULT_TRADES_COLUMNS], trading_mode)
def trades_purge(self, pair: str) -> bool:
def trades_purge(self, pair: str, trading_mode: TradingMode) -> bool:
"""
Remove data for this pair
:param pair: Delete data for this pair.
:param trading_mode: Trading mode to use (used to determine the filename)
:return: True when deleted, false if file did not exist.
"""
filename = self._pair_trades_filename(self._datadir, pair)
filename = self._pair_trades_filename(self._datadir, pair, trading_mode)
if filename.exists():
filename.unlink()
return True
return False
def trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> DataFrame:
def trades_load(
self, pair: str, trading_mode: TradingMode, timerange: Optional[TimeRange] = None
) -> DataFrame:
"""
Load a pair from file, either .json.gz or .json
Removes duplicates in the process.
:param pair: Load trades for this pair
:param trading_mode: Trading mode to use (used to determine the filename)
:param timerange: Timerange to load trades for - currently not implemented
:return: List of trades
"""
trades = trades_df_remove_duplicates(self._trades_load(pair, timerange=timerange))
trades = trades_df_remove_duplicates(
self._trades_load(pair, trading_mode, timerange=timerange)
)
trades = trades_convert_types(trades)
return trades
@@ -264,8 +275,12 @@ class IDataHandler(ABC):
return filename
@classmethod
def _pair_trades_filename(cls, datadir: Path, pair: str) -> Path:
def _pair_trades_filename(cls, datadir: Path, pair: str, trading_mode: TradingMode) -> Path:
pair_s = misc.pair_to_filename(pair)
if trading_mode == TradingMode.FUTURES:
# Futures pair ...
datadir = datadir.joinpath('futures')
filename = datadir.joinpath(f'{pair_s}-trades.{cls._get_file_extension()}')
return filename

14
freqtrade/data/history/jsondatahandler.py
View File

@@ -8,7 +8,7 @@ from freqtrade import misc
from freqtrade.configuration import TimeRange
from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS
from freqtrade.data.converter import trades_dict_to_list, trades_list_to_df
from freqtrade.enums import CandleType
from freqtrade.enums import CandleType, TradingMode
from .idatahandler import IDataHandler
@@ -94,14 +94,15 @@ class JsonDataHandler(IDataHandler):
"""
raise NotImplementedError()
def _trades_store(self, pair: str, data: DataFrame) -> None:
def _trades_store(self, pair: str, data: DataFrame, trading_mode: TradingMode) -> None:
"""
Store trades data (list of Dicts) to file
:param pair: Pair - used for filename
:param data: Dataframe containing trades
column sequence as in DEFAULT_TRADES_COLUMNS
:param trading_mode: Trading mode to use (used to determine the filename)
"""
filename = self._pair_trades_filename(self._datadir, pair)
filename = self._pair_trades_filename(self._datadir, pair, trading_mode)
trades = data.values.tolist()
misc.file_dump_json(filename, trades, is_zip=self._use_zip)
@@ -114,15 +115,18 @@ class JsonDataHandler(IDataHandler):
"""
raise NotImplementedError()
def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> DataFrame:
def _trades_load(
self, pair: str, trading_mode: TradingMode, timerange: Optional[TimeRange] = None
) -> DataFrame:
"""
Load a pair from file, either .json.gz or .json
# TODO: respect timerange ...
:param pair: Load trades for this pair
:param trading_mode: Trading mode to use (used to determine the filename)
:param timerange: Timerange to load trades for - currently not implemented
:return: Dataframe containing trades
"""
filename = self._pair_trades_filename(self._datadir, pair)
filename = self._pair_trades_filename(self._datadir, pair, trading_mode)
tradesdata = misc.file_load_json(filename)
if not tradesdata:

16
freqtrade/data/history/parquetdatahandler.py
View File

@@ -4,8 +4,8 @@ from typing import Optional
from pandas import DataFrame, read_parquet, to_datetime
from freqtrade.configuration import TimeRange
from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, TradeList
from freqtrade.enums import CandleType
from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS
from freqtrade.enums import CandleType, TradingMode
from .idatahandler import IDataHandler
@@ -81,14 +81,15 @@ class ParquetDataHandler(IDataHandler):
"""
raise NotImplementedError()
def _trades_store(self, pair: str, data: DataFrame) -> None:
def _trades_store(self, pair: str, data: DataFrame, trading_mode: TradingMode) -> None:
"""
Store trades data (list of Dicts) to file
:param pair: Pair - used for filename
:param data: Dataframe containing trades
column sequence as in DEFAULT_TRADES_COLUMNS
:param trading_mode: Trading mode to use (used to determine the filename)
"""
filename = self._pair_trades_filename(self._datadir, pair)
filename = self._pair_trades_filename(self._datadir, pair, trading_mode)
self.create_dir_if_needed(filename)
data.reset_index(drop=True).to_parquet(filename)
@@ -101,15 +102,18 @@ class ParquetDataHandler(IDataHandler):
"""
raise NotImplementedError()
def _trades_load(self, pair: str, timerange: Optional[TimeRange] = None) -> TradeList:
def _trades_load(
self, pair: str, trading_mode: TradingMode, timerange: Optional[TimeRange] = None
) -> DataFrame:
"""
Load a pair from file, either .json.gz or .json
# TODO: respect timerange ...
:param pair: Load trades for this pair
:param trading_mode: Trading mode to use (used to determine the filename)
:param timerange: Timerange to load trades for - currently not implemented
:return: List of trades
"""
filename = self._pair_trades_filename(self._datadir, pair)
filename = self._pair_trades_filename(self._datadir, pair, trading_mode)
if not filename.exists():
return DataFrame(columns=DEFAULT_TRADES_COLUMNS)

8
tests/commands/test_commands.py
View File

@@ -820,11 +820,6 @@ def test_download_data_trades(mocker):
"--trading-mode", "futures",
"--dl-trades"
]
with pytest.raises(OperationalException,
match="Trade download not supported for futures."):
pargs = get_args(args)
pargs['config'] = None
start_download_data(pargs)
def test_download_data_data_invalid(mocker):
@@ -842,10 +837,11 @@ def test_download_data_data_invalid(mocker):
start_download_data(pargs)
def test_start_convert_trades(mocker, caplog):
def test_start_convert_trades(mocker):
convert_mock = mocker.patch('freqtrade.commands.data_commands.convert_trades_to_ohlcv',
MagicMock(return_value=[]))
patch_exchange(mocker)
mocker.patch(f'{EXMS}.get_markets')
mocker.patch(f'{EXMS}.markets', PropertyMock(return_value={}))
args = [
"trades-to-ohlcv",

8
tests/data/test_converter.py
View File

@@ -542,7 +542,9 @@ def test_convert_trades_to_ohlcv(testdatadir, tmp_path, caplog):
convert_trades_to_ohlcv([pair], timeframes=['1m', '5m'],
data_format_trades='jsongz',
datadir=tmp_path, timerange=tr, erase=True)
datadir=tmp_path, timerange=tr, erase=True,
data_format_ohlcv='feather',
candle_type=CandleType.SPOT)
assert log_has("Deleting existing data for pair XRP/ETH, interval 1m.", caplog)
# Load new data
@@ -556,5 +558,7 @@ def test_convert_trades_to_ohlcv(testdatadir, tmp_path, caplog):
convert_trades_to_ohlcv(['NoDatapair'], timeframes=['1m', '5m'],
data_format_trades='jsongz',
datadir=tmp_path, timerange=tr, erase=True)
datadir=tmp_path, timerange=tr, erase=True,
data_format_ohlcv='feather',
candle_type=CandleType.SPOT)
assert log_has(msg, caplog)

24
tests/data/test_datahandler.py
View File

@@ -261,11 +261,11 @@ def test_datahandler_trades_not_supported(datahandler, testdatadir, ):
def test_jsondatahandler_trades_load(testdatadir, caplog):
dh = JsonGzDataHandler(testdatadir)
logmsg = "Old trades format detected - converting"
dh.trades_load('XRP/ETH')
dh.trades_load('XRP/ETH', TradingMode.SPOT)
assert not log_has(logmsg, caplog)
# Test conversation is happening
dh.trades_load('XRP/OLD')
dh.trades_load('XRP/OLD', TradingMode.SPOT)
assert log_has(logmsg, caplog)
@@ -300,16 +300,16 @@ def test_datahandler_trades_get_pairs(testdatadir, datahandler, expected):
def test_hdf5datahandler_trades_load(testdatadir):
dh = get_datahandler(testdatadir, 'hdf5')
trades = dh.trades_load('XRP/ETH')
trades = dh.trades_load('XRP/ETH', TradingMode.SPOT)
assert isinstance(trades, DataFrame)
trades1 = dh.trades_load('UNITTEST/NONEXIST')
trades1 = dh.trades_load('UNITTEST/NONEXIST', TradingMode.SPOT)
assert isinstance(trades1, DataFrame)
assert trades1.empty
# data goes from 2019-10-11 - 2019-10-13
timerange = TimeRange.parse_timerange('20191011-20191012')
trades2 = dh._trades_load('XRP/ETH', timerange)
trades2 = dh._trades_load('XRP/ETH', TradingMode.SPOT, timerange)
assert len(trades) > len(trades2)
# Check that ID is None (If it's nan, it's wrong)
assert trades2.iloc[0]['type'] is None
@@ -451,13 +451,13 @@ def test_hdf5datahandler_ohlcv_purge(mocker, testdatadir):
@pytest.mark.parametrize('datahandler', ['jsongz', 'hdf5', 'feather', 'parquet'])
def test_datahandler_trades_load(testdatadir, datahandler):
dh = get_datahandler(testdatadir, datahandler)
trades = dh.trades_load('XRP/ETH')
trades = dh.trades_load('XRP/ETH', TradingMode.SPOT)
assert isinstance(trades, DataFrame)
assert trades.iloc[0]['timestamp'] == 1570752011620
assert trades.iloc[0]['date'] == Timestamp('2019-10-11 00:00:11.620000+0000')
assert trades.iloc[-1]['cost'] == 0.1986231
trades1 = dh.trades_load('UNITTEST/NONEXIST')
trades1 = dh.trades_load('UNITTEST/NONEXIST', TradingMode.SPOT)
assert isinstance(trades, DataFrame)
assert trades1.empty
@@ -465,15 +465,15 @@ def test_datahandler_trades_load(testdatadir, datahandler):
@pytest.mark.parametrize('datahandler', ['jsongz', 'hdf5', 'feather', 'parquet'])
def test_datahandler_trades_store(testdatadir, tmp_path, datahandler):
dh = get_datahandler(testdatadir, datahandler)
trades = dh.trades_load('XRP/ETH')
trades = dh.trades_load('XRP/ETH', TradingMode.SPOT)
dh1 = get_datahandler(tmp_path, datahandler)
dh1.trades_store('XRP/NEW', trades)
dh1.trades_store('XRP/NEW', trades, TradingMode.SPOT)
file = tmp_path / f'XRP_NEW-trades.{dh1._get_file_extension()}'
assert file.is_file()
# Load trades back
trades_new = dh1.trades_load('XRP/NEW')
trades_new = dh1.trades_load('XRP/NEW', TradingMode.SPOT)
assert_frame_equal(trades, trades_new, check_exact=True)
assert len(trades_new) == len(trades)
@@ -483,11 +483,11 @@ def test_datahandler_trades_purge(mocker, testdatadir, datahandler):
mocker.patch.object(Path, "exists", MagicMock(return_value=False))
unlinkmock = mocker.patch.object(Path, "unlink", MagicMock())
dh = get_datahandler(testdatadir, datahandler)
assert not dh.trades_purge('UNITTEST/NONEXIST')
assert not dh.trades_purge('UNITTEST/NONEXIST', TradingMode.SPOT)
assert unlinkmock.call_count == 0
mocker.patch.object(Path, "exists", MagicMock(return_value=True))
assert dh.trades_purge('UNITTEST/NONEXIST')
assert dh.trades_purge('UNITTEST/NONEXIST', TradingMode.SPOT)
assert unlinkmock.call_count == 1

4
tests/data/test_download_data.py
View File

@@ -78,10 +78,6 @@ def test_download_data_main_trades(mocker):
"trading_mode": "futures",
})
with pytest.raises(OperationalException,
match="Trade download not supported for futures."):
download_data_main(config)
def test_download_data_main_data_invalid(mocker):
patch_exchange(mocker, id="kraken")

40
tests/data/test_history.py
View File

@@ -23,7 +23,7 @@ from freqtrade.data.history.history_utils import (_download_pair_history, _downl
validate_backtest_data)
from freqtrade.data.history.idatahandler import get_datahandler
from freqtrade.data.history.jsondatahandler import JsonDataHandler, JsonGzDataHandler
from freqtrade.enums import CandleType
from freqtrade.enums import CandleType, TradingMode
from freqtrade.exchange import timeframe_to_minutes
from freqtrade.misc import file_dump_json
from freqtrade.resolvers import StrategyResolver
@@ -168,20 +168,21 @@ def test_json_pair_data_filename(pair, timeframe, expected_result, candle_type):
assert fn == Path(expected_result + '.gz')
@pytest.mark.parametrize("pair,expected_result", [
("ETH/BTC", 'freqtrade/hello/world/ETH_BTC-trades.json'),
("Fabric Token/ETH", 'freqtrade/hello/world/Fabric_Token_ETH-trades.json'),
("ETHH20", 'freqtrade/hello/world/ETHH20-trades.json'),
(".XBTBON2H", 'freqtrade/hello/world/_XBTBON2H-trades.json'),
("ETHUSD.d", 'freqtrade/hello/world/ETHUSD_d-trades.json'),
("ACC_OLD_BTC", 'freqtrade/hello/world/ACC_OLD_BTC-trades.json'),
@pytest.mark.parametrize("pair,trading_mode,expected_result", [
("ETH/BTC", '', 'freqtrade/hello/world/ETH_BTC-trades.json'),
("ETH/USDT:USDT", 'futures', 'freqtrade/hello/world/futures/ETH_USDT_USDT-trades.json'),
("Fabric Token/ETH", '', 'freqtrade/hello/world/Fabric_Token_ETH-trades.json'),
("ETHH20", '', 'freqtrade/hello/world/ETHH20-trades.json'),
(".XBTBON2H", '', 'freqtrade/hello/world/_XBTBON2H-trades.json'),
("ETHUSD.d", '', 'freqtrade/hello/world/ETHUSD_d-trades.json'),
("ACC_OLD_BTC", '', 'freqtrade/hello/world/ACC_OLD_BTC-trades.json'),
])
def test_json_pair_trades_filename(pair, expected_result):
fn = JsonDataHandler._pair_trades_filename(Path('freqtrade/hello/world'), pair)
def test_json_pair_trades_filename(pair, trading_mode, expected_result):
fn = JsonDataHandler._pair_trades_filename(Path('freqtrade/hello/world'), pair, trading_mode)
assert isinstance(fn, Path)
assert fn == Path(expected_result)
fn = JsonGzDataHandler._pair_trades_filename(Path('freqtrade/hello/world'), pair)
fn = JsonGzDataHandler._pair_trades_filename(Path('freqtrade/hello/world'), pair, trading_mode)
assert isinstance(fn, Path)
assert fn == Path(expected_result + '.gz')
@@ -559,7 +560,8 @@ def test_refresh_backtest_trades_data(mocker, default_conf, markets, caplog, tes
unavailable_pairs = refresh_backtest_trades_data(exchange=ex,
pairs=["ETH/BTC", "XRP/BTC", "XRP/ETH"],
datadir=testdatadir,
timerange=timerange, erase=True
timerange=timerange, erase=True,
trading_mode=TradingMode.SPOT,
)
assert dl_mock.call_count == 2
@@ -584,7 +586,7 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad
assert not file1.is_file()
assert _download_trades_history(data_handler=data_handler, exchange=exchange,
pair='ETH/BTC')
pair='ETH/BTC', trading_mode=TradingMode.SPOT)
assert log_has("Current Amount of trades: 0", caplog)
assert log_has("New Amount of trades: 6", caplog)
assert ght_mock.call_count == 1
@@ -597,8 +599,9 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad
since_time = int(trades_history[-3][0] // 1000)
since_time2 = int(trades_history[-1][0] // 1000)
timerange = TimeRange('date', None, since_time, 0)
assert _download_trades_history(data_handler=data_handler, exchange=exchange,
pair='ETH/BTC', timerange=timerange)
assert _download_trades_history(
data_handler=data_handler, exchange=exchange, pair='ETH/BTC',
timerange=timerange, trading_mode=TradingMode.SPOT)
assert ght_mock.call_count == 1
# Check this in seconds - since we had to convert to seconds above too.
@@ -611,7 +614,7 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad
caplog.clear()
assert not _download_trades_history(data_handler=data_handler, exchange=exchange,
pair='ETH/BTC')
pair='ETH/BTC', trading_mode=TradingMode.SPOT)
assert log_has_re('Failed to download historic trades for pair: "ETH/BTC".*', caplog)
file2 = tmp_path / 'XRP_ETH-trades.json.gz'
@@ -623,8 +626,9 @@ def test_download_trades_history(trades_history, mocker, default_conf, testdatad
since_time = int(trades_history[0][0] // 1000) - 500
timerange = TimeRange('date', None, since_time, 0)
assert _download_trades_history(data_handler=data_handler, exchange=exchange,
pair='XRP/ETH', timerange=timerange)
assert _download_trades_history(
data_handler=data_handler, exchange=exchange, pair='XRP/ETH',
timerange=timerange, trading_mode=TradingMode.SPOT)
assert ght_mock.call_count == 1

3
tests/data/test_trade_converter_kraken.py
View File

@@ -6,6 +6,7 @@ import pytest
from freqtrade.data.converter.trade_converter_kraken import import_kraken_trades_from_csv
from freqtrade.data.history.idatahandler import get_datahandler
from freqtrade.enums import TradingMode
from freqtrade.exceptions import OperationalException
from tests.conftest import EXMS, log_has, log_has_re, patch_exchange
@@ -40,7 +41,7 @@ def test_import_kraken_trades_from_csv(testdatadir, tmp_path, caplog, default_co
assert dstfile.is_file()
dh = get_datahandler(tmp_path, 'feather')
trades = dh.trades_load('BCH_EUR')
trades = dh.trades_load('BCH_EUR', TradingMode.SPOT)
assert len(trades) == 340
assert trades['date'].min().to_pydatetime() == datetime(2023, 1, 1, 0, 3, 56,