From c8162479d62dc0f87f0b4b0be4ef848b7102ae13 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Sat, 23 Oct 2021 21:10:36 -0600 Subject: [PATCH 01/64] Added price as param to fetch_ohlcv --- freqtrade/exchange/binance.py | 26 ++++++++--- freqtrade/exchange/exchange.py | 81 +++++++++++++++++++++++++-------- tests/exchange/test_exchange.py | 4 +- 3 files changed, 82 insertions(+), 29 deletions(-) diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 787285a02..b12655b86 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -200,24 +200,36 @@ class Binance(Exchange): except ccxt.BaseError as e: raise OperationalException(e) from e - async def _async_get_historic_ohlcv(self, pair: str, timeframe: str, - since_ms: int, is_new_pair: bool = False, - raise_: bool = False - ) -> Tuple[str, str, List]: + async def _async_get_historic_ohlcv( + self, + pair: str, + timeframe: str, + since_ms: int, + is_new_pair: bool, + raise_: bool = False, + price: Optional[str] = None + ) -> Tuple[str, str, List]: """ Overwrite to introduce "fast new pair" functionality by detecting the pair's listing date Does not work for other exchanges, which don't return the earliest data when called with "0" + :param price: "mark" if retrieving the mark price cnadles """ if is_new_pair: - x = await self._async_get_candle_history(pair, timeframe, 0) + x = await self._async_get_candle_history(pair, timeframe, 0, price) if x and x[2] and x[2][0] and x[2][0][0] > since_ms: # Set starting date to first available candle. since_ms = x[2][0][0] logger.info(f"Candle-data for {pair} available starting with " f"{arrow.get(since_ms // 1000).isoformat()}.") + return await super()._async_get_historic_ohlcv( - pair=pair, timeframe=timeframe, since_ms=since_ms, is_new_pair=is_new_pair, - raise_=raise_) + pair=pair, + timeframe=timeframe, + since_ms=since_ms, + is_new_pair=is_new_pair, + raise_=raise_, + price=price + ) def funding_fee_cutoff(self, open_date: datetime): """ diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 8f91e6a47..f9fa708f2 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1309,8 +1309,14 @@ class Exchange: # Historic data - def get_historic_ohlcv(self, pair: str, timeframe: str, - since_ms: int, is_new_pair: bool = False) -> List: + def get_historic_ohlcv( + self, + pair: str, + timeframe: str, + since_ms: int, + is_new_pair: bool = False, + price: Optional[str] = None + ) -> List: """ Get candle history using asyncio and returns the list of candles. Handles all async work for this. @@ -1318,34 +1324,52 @@ class Exchange: :param pair: Pair to download :param timeframe: Timeframe to get data for :param since_ms: Timestamp in milliseconds to get history from + :param price: "mark" if retrieving the mark price cnadles, "index" for index price candles :return: List with candle (OHLCV) data """ pair, timeframe, data = asyncio.get_event_loop().run_until_complete( - self._async_get_historic_ohlcv(pair=pair, timeframe=timeframe, - since_ms=since_ms, is_new_pair=is_new_pair)) + self._async_get_historic_ohlcv( + pair=pair, + timeframe=timeframe, + since_ms=since_ms, + is_new_pair=is_new_pair, + price=price + )) logger.info(f"Downloaded data for {pair} with length {len(data)}.") return data - def get_historic_ohlcv_as_df(self, pair: str, timeframe: str, - since_ms: int) -> DataFrame: + def get_historic_ohlcv_as_df( + self, + pair: str, + timeframe: str, + since_ms: int, + price: Optional[str] = None + ) -> DataFrame: """ Minimal wrapper around get_historic_ohlcv - converting the result into a dataframe :param pair: Pair to download :param timeframe: Timeframe to get data for :param since_ms: Timestamp in milliseconds to get history from + :param price: "mark" if retrieving the mark price candles, "index" for index price candles :return: OHLCV DataFrame """ - ticks = self.get_historic_ohlcv(pair, timeframe, since_ms=since_ms) + ticks = self.get_historic_ohlcv(pair, timeframe, since_ms=since_ms, price=price) return ohlcv_to_dataframe(ticks, timeframe, pair=pair, fill_missing=True, drop_incomplete=self._ohlcv_partial_candle) - async def _async_get_historic_ohlcv(self, pair: str, timeframe: str, - since_ms: int, is_new_pair: bool = False, - raise_: bool = False - ) -> Tuple[str, str, List]: + async def _async_get_historic_ohlcv( + self, + pair: str, + timeframe: str, + since_ms: int, + is_new_pair: bool, + raise_: bool = False, + price: Optional[str] = None + ) -> Tuple[str, str, List]: """ Download historic ohlcv :param is_new_pair: used by binance subclass to allow "fast" new pair downloading + :param price: "mark" if retrieving the mark price cnadles, "index" for index price candles """ one_call = timeframe_to_msecs(timeframe) * self.ohlcv_candle_limit(timeframe) @@ -1354,8 +1378,13 @@ class Exchange: one_call, arrow.utcnow().shift(seconds=one_call // 1000).humanize(only_distance=True) ) - input_coroutines = [self._async_get_candle_history( - pair, timeframe, since) for since in + input_coroutines = [ + self._async_get_candle_history( + pair, + timeframe, + since, + price + ) for since in range(since_ms, arrow.utcnow().int_timestamp * 1000, one_call)] data: List = [] @@ -1378,9 +1407,13 @@ class Exchange: data = sorted(data, key=lambda x: x[0]) return pair, timeframe, data - def refresh_latest_ohlcv(self, pair_list: ListPairsWithTimeframes, *, - since_ms: Optional[int] = None, cache: bool = True - ) -> Dict[Tuple[str, str], DataFrame]: + def refresh_latest_ohlcv( + self, + pair_list: ListPairsWithTimeframes, *, + since_ms: Optional[int] = None, + cache: bool = True, + price: Optional[str] = None + ) -> Dict[Tuple[str, str], DataFrame]: """ Refresh in-memory OHLCV asynchronously and set `_klines` with the result Loops asynchronously over pair_list and downloads all pairs async (semi-parallel). @@ -1388,6 +1421,7 @@ class Exchange: :param pair_list: List of 2 element tuples containing pair, interval to refresh :param since_ms: time since when to download, in milliseconds :param cache: Assign result to _klines. Usefull for one-off downloads like for pairlists + :param price: "mark" if retrieving the mark price cnadles, "index" for index price candles :return: Dict of [{(pair, timeframe): Dataframe}] """ logger.debug("Refreshing candle (OHLCV) data for %d pairs", len(pair_list)) @@ -1411,7 +1445,7 @@ class Exchange: else: # One call ... "regular" refresh input_coroutines.append(self._async_get_candle_history( - pair, timeframe, since_ms=since_ms)) + pair, timeframe, since_ms=since_ms, price=price)) else: logger.debug( "Using cached candle (OHLCV) data for pair %s, timeframe %s ...", @@ -1454,10 +1488,16 @@ class Exchange: + interval_in_sec) >= arrow.utcnow().int_timestamp) @retrier_async - async def _async_get_candle_history(self, pair: str, timeframe: str, - since_ms: Optional[int] = None) -> Tuple[str, str, List]: + async def _async_get_candle_history( + self, + pair: str, + timeframe: str, + since_ms: Optional[int] = None, + price: Optional[str] = None + ) -> Tuple[str, str, List]: """ Asynchronously get candle history data using fetch_ohlcv + :param price: "mark" if retrieving the mark price cnadles, "index" for index price candles returns tuple: (pair, timeframe, ohlcv_list) """ try: @@ -1467,7 +1507,8 @@ class Exchange: "Fetching pair %s, interval %s, since %s %s...", pair, timeframe, since_ms, s ) - params = self._ft_has.get('ohlcv_params', {}) + # TODO-lev: Does this put price into params correctly? + params = self._ft_has.get('ohlcv_params', {price: price}) data = await self._api_async.fetch_ohlcv(pair, timeframe=timeframe, since=since_ms, limit=self.ohlcv_candle_limit(timeframe), diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 9da05f8e0..59a64abab 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1568,7 +1568,7 @@ def test_get_historic_ohlcv(default_conf, mocker, caplog, exchange_name): ] pair = 'ETH/BTC' - async def mock_candle_hist(pair, timeframe, since_ms): + async def mock_candle_hist(pair, timeframe, since_ms, price=None): return pair, timeframe, ohlcv exchange._async_get_candle_history = Mock(wraps=mock_candle_hist) @@ -1625,7 +1625,7 @@ def test_get_historic_ohlcv_as_df(default_conf, mocker, exchange_name): ] pair = 'ETH/BTC' - async def mock_candle_hist(pair, timeframe, since_ms): + async def mock_candle_hist(pair, timeframe, since_ms, price=None): return pair, timeframe, ohlcv exchange._async_get_candle_history = Mock(wraps=mock_candle_hist) From ee2ad8ca97729b0c92afc4da6bdeb4cdf5d06e85 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Sun, 7 Nov 2021 00:35:27 -0600 Subject: [PATCH 02/64] updated historic data filenames to include the candle type --- freqtrade/data/converter.py | 49 +++++++++---- freqtrade/data/history/hdf5datahandler.py | 58 ++++++++++++--- freqtrade/data/history/history_utils.py | 40 +++++++---- freqtrade/data/history/idatahandler.py | 69 ++++++++++++++---- freqtrade/data/history/jsondatahandler.py | 56 ++++++++++++--- freqtrade/exchange/binance.py | 8 +-- freqtrade/exchange/exchange.py | 88 ++++++++--------------- tests/exchange/test_exchange.py | 4 +- 8 files changed, 247 insertions(+), 125 deletions(-) diff --git a/freqtrade/data/converter.py b/freqtrade/data/converter.py index d592b4990..67f8712a0 100644 --- a/freqtrade/data/converter.py +++ b/freqtrade/data/converter.py @@ -5,7 +5,7 @@ import itertools import logging from datetime import datetime, timezone from operator import itemgetter -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional import pandas as pd from pandas import DataFrame, to_datetime @@ -17,7 +17,8 @@ logger = logging.getLogger(__name__) def ohlcv_to_dataframe(ohlcv: list, timeframe: str, pair: str, *, - fill_missing: bool = True, drop_incomplete: bool = True) -> DataFrame: + fill_missing: bool = True, drop_incomplete: bool = True, + candle_type: Optional[str] = "") -> DataFrame: """ Converts a list with candle (OHLCV) data (in format returned by ccxt.fetch_ohlcv) to a Dataframe @@ -42,12 +43,14 @@ def ohlcv_to_dataframe(ohlcv: list, timeframe: str, pair: str, *, 'volume': 'float'}) return clean_ohlcv_dataframe(df, timeframe, pair, fill_missing=fill_missing, - drop_incomplete=drop_incomplete) + drop_incomplete=drop_incomplete, + candle_type=candle_type) def clean_ohlcv_dataframe(data: DataFrame, timeframe: str, pair: str, *, fill_missing: bool = True, - drop_incomplete: bool = True) -> DataFrame: + drop_incomplete: bool = True, + candle_type: Optional[str] = "") -> DataFrame: """ Cleanse a OHLCV dataframe by * Grouping it by date (removes duplicate tics) @@ -75,12 +78,17 @@ def clean_ohlcv_dataframe(data: DataFrame, timeframe: str, pair: str, *, logger.debug('Dropping last candle') if fill_missing: - return ohlcv_fill_up_missing_data(data, timeframe, pair) + return ohlcv_fill_up_missing_data(data, timeframe, pair, candle_type) else: return data -def ohlcv_fill_up_missing_data(dataframe: DataFrame, timeframe: str, pair: str) -> DataFrame: +def ohlcv_fill_up_missing_data( + dataframe: DataFrame, + timeframe: str, + pair: str, + candle_type: Optional[str] = "" +) -> DataFrame: """ Fills up missing data with 0 volume rows, using the previous close as price for "open", "high" "low" and "close", volume is set to 0 @@ -261,7 +269,13 @@ def convert_trades_format(config: Dict[str, Any], convert_from: str, convert_to: src.trades_purge(pair=pair) -def convert_ohlcv_format(config: Dict[str, Any], convert_from: str, convert_to: str, erase: bool): +def convert_ohlcv_format( + config: Dict[str, Any], + convert_from: str, + convert_to: str, + erase: bool, + candle_type: Optional[str] = "" +): """ Convert OHLCV from one format to another :param config: Config dictionary @@ -279,8 +293,11 @@ def convert_ohlcv_format(config: Dict[str, Any], convert_from: str, convert_to: config['pairs'] = [] # Check timeframes or fall back to timeframe. for timeframe in timeframes: - config['pairs'].extend(src.ohlcv_get_pairs(config['datadir'], - timeframe)) + config['pairs'].extend(src.ohlcv_get_pairs( + config['datadir'], + timeframe, + candle_type=candle_type + )) logger.info(f"Converting candle (OHLCV) data for {config['pairs']}") for timeframe in timeframes: @@ -289,10 +306,16 @@ def convert_ohlcv_format(config: Dict[str, Any], convert_from: str, convert_to: timerange=None, fill_missing=False, drop_incomplete=False, - startup_candles=0) - logger.info(f"Converting {len(data)} candles for {pair}") + startup_candles=0, + candle_type=candle_type) + logger.info(f"Converting {len(data)} {candle_type} candles for {pair}") if len(data) > 0: - trg.ohlcv_store(pair=pair, timeframe=timeframe, data=data) + trg.ohlcv_store( + pair=pair, + timeframe=timeframe, + data=data, + candle_type=candle_type + ) if erase and convert_from != convert_to: logger.info(f"Deleting source data for {pair} / {timeframe}") - src.ohlcv_purge(pair=pair, timeframe=timeframe) + src.ohlcv_purge(pair=pair, timeframe=timeframe, candle_type=candle_type) diff --git a/freqtrade/data/history/hdf5datahandler.py b/freqtrade/data/history/hdf5datahandler.py index dd60530aa..7d5b4f041 100644 --- a/freqtrade/data/history/hdf5datahandler.py +++ b/freqtrade/data/history/hdf5datahandler.py @@ -34,7 +34,12 @@ class HDF5DataHandler(IDataHandler): if match and len(match.groups()) > 1] @classmethod - def ohlcv_get_pairs(cls, datadir: Path, timeframe: str) -> List[str]: + def ohlcv_get_pairs( + cls, + datadir: Path, + timeframe: str, + candle_type: Optional[str] = "" + ) -> List[str]: """ Returns a list of all pairs with ohlcv data available in this datadir for the specified timeframe @@ -43,12 +48,23 @@ class HDF5DataHandler(IDataHandler): :return: List of Pairs """ - _tmp = [re.search(r'^(\S+)(?=\-' + timeframe + '.h5)', p.name) + if candle_type: + candle_type = f"-{candle_type}" + else: + candle_type = "" + + _tmp = [re.search(r'^(\S+)(?=\-' + timeframe + candle_type + '.h5)', p.name) for p in datadir.glob(f"*{timeframe}.h5")] # Check if regex found something and only return these results return [match[0].replace('_', '/') for match in _tmp if match] - def ohlcv_store(self, pair: str, timeframe: str, data: pd.DataFrame) -> None: + def ohlcv_store( + self, + pair: str, + timeframe: str, + data: pd.DataFrame, + candle_type: Optional[str] = "" + ) -> None: """ Store data in hdf5 file. :param pair: Pair - used to generate filename @@ -59,7 +75,7 @@ class HDF5DataHandler(IDataHandler): key = self._pair_ohlcv_key(pair, timeframe) _data = data.copy() - filename = self._pair_data_filename(self._datadir, pair, timeframe) + filename = self._pair_data_filename(self._datadir, pair, timeframe, candle_type) ds = pd.HDFStore(filename, mode='a', complevel=9, complib='blosc') ds.put(key, _data.loc[:, self._columns], format='table', data_columns=['date']) @@ -67,7 +83,8 @@ class HDF5DataHandler(IDataHandler): ds.close() def _ohlcv_load(self, pair: str, timeframe: str, - timerange: Optional[TimeRange] = None) -> pd.DataFrame: + timerange: Optional[TimeRange] = None, + candle_type: Optional[str] = "") -> pd.DataFrame: """ Internal method used to load data for one pair from disk. Implements the loading and conversion to a Pandas dataframe. @@ -80,7 +97,12 @@ class HDF5DataHandler(IDataHandler): :return: DataFrame with ohlcv data, or empty DataFrame """ key = self._pair_ohlcv_key(pair, timeframe) - filename = self._pair_data_filename(self._datadir, pair, timeframe) + filename = self._pair_data_filename( + self._datadir, + pair, + timeframe, + candle_type=candle_type + ) if not filename.exists(): return pd.DataFrame(columns=self._columns) @@ -99,20 +121,26 @@ class HDF5DataHandler(IDataHandler): 'low': 'float', 'close': 'float', 'volume': 'float'}) return pairdata - def ohlcv_purge(self, pair: str, timeframe: str) -> bool: + def ohlcv_purge(self, pair: str, timeframe: str, candle_type: Optional[str] = "") -> bool: """ Remove data for this pair :param pair: Delete data for this pair. :param timeframe: Timeframe (e.g. "5m") :return: True when deleted, false if file did not exist. """ - filename = self._pair_data_filename(self._datadir, pair, timeframe) + filename = self._pair_data_filename(self._datadir, pair, timeframe, candle_type) if filename.exists(): filename.unlink() return True return False - def ohlcv_append(self, pair: str, timeframe: str, data: pd.DataFrame) -> None: + def ohlcv_append( + self, + pair: str, + timeframe: str, + data: pd.DataFrame, + candle_type: Optional[str] = "" + ) -> None: """ Append data to existing data structures :param pair: Pair @@ -201,9 +229,17 @@ class HDF5DataHandler(IDataHandler): return f"{pair}/trades" @classmethod - def _pair_data_filename(cls, datadir: Path, pair: str, timeframe: str) -> Path: + def _pair_data_filename( + cls, + datadir: Path, + pair: str, + timeframe: str, + candle_type: Optional[str] = "" + ) -> Path: pair_s = misc.pair_to_filename(pair) - filename = datadir.joinpath(f'{pair_s}-{timeframe}.h5') + if candle_type: + candle_type = f"-{candle_type}" + filename = datadir.joinpath(f'{pair_s}-{timeframe}{candle_type}.h5') return filename @classmethod diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index e6b8db322..fc14474f9 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -161,7 +161,8 @@ def _download_pair_history(pair: str, *, process: str = '', new_pairs_days: int = 30, data_handler: IDataHandler = None, - timerange: Optional[TimeRange] = None) -> bool: + timerange: Optional[TimeRange] = None, + candle_type: Optional[str] = "") -> bool: """ Download latest candles from the exchange for the pair and timeframe passed in parameters The data is downloaded starting from the last correct data that @@ -198,25 +199,28 @@ def _download_pair_history(pair: str, *, since_ms=since_ms if since_ms else arrow.utcnow().shift( days=-new_pairs_days).int_timestamp * 1000, - is_new_pair=data.empty + is_new_pair=data.empty, + candle_type=candle_type, ) # TODO: Maybe move parsing to exchange class (?) new_dataframe = ohlcv_to_dataframe(new_data, timeframe, pair, - fill_missing=False, drop_incomplete=True) + fill_missing=False, drop_incomplete=True, + candle_type=candle_type) if data.empty: data = new_dataframe else: # Run cleaning again to ensure there were no duplicate candles # Especially between existing and new data. data = clean_ohlcv_dataframe(data.append(new_dataframe), timeframe, pair, - fill_missing=False, drop_incomplete=False) + fill_missing=False, drop_incomplete=False, + candle_type=candle_type) logger.debug("New Start: %s", f"{data.iloc[0]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None') logger.debug("New End: %s", f"{data.iloc[-1]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None') - data_handler.ohlcv_store(pair, timeframe, data=data) + data_handler.ohlcv_store(pair, timeframe, data=data, candle_type=candle_type) return True except Exception: @@ -229,7 +233,8 @@ def _download_pair_history(pair: str, *, def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes: List[str], datadir: Path, timerange: Optional[TimeRange] = None, new_pairs_days: int = 30, erase: bool = False, - data_format: str = None) -> List[str]: + data_format: str = None, + candle_type: Optional[str] = "") -> List[str]: """ Refresh stored ohlcv data for backtesting and hyperopt operations. Used by freqtrade download-data subcommand. @@ -245,7 +250,7 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes for timeframe in timeframes: if erase: - if data_handler.ohlcv_purge(pair, timeframe): + if data_handler.ohlcv_purge(pair, timeframe, candle_type=candle_type): logger.info( f'Deleting existing data for pair {pair}, interval {timeframe}.') @@ -254,7 +259,8 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes _download_pair_history(pair=pair, process=process, datadir=datadir, exchange=exchange, timerange=timerange, data_handler=data_handler, - timeframe=str(timeframe), new_pairs_days=new_pairs_days) + timeframe=str(timeframe), new_pairs_days=new_pairs_days, + candle_type=candle_type) return pairs_not_available @@ -353,10 +359,16 @@ def refresh_backtest_trades_data(exchange: Exchange, pairs: List[str], datadir: return pairs_not_available -def convert_trades_to_ohlcv(pairs: List[str], timeframes: List[str], - datadir: Path, timerange: TimeRange, erase: bool = False, - data_format_ohlcv: str = 'json', - data_format_trades: str = 'jsongz') -> None: +def convert_trades_to_ohlcv( + pairs: List[str], + timeframes: List[str], + datadir: Path, + timerange: TimeRange, + erase: bool = False, + data_format_ohlcv: str = 'json', + data_format_trades: str = 'jsongz', + candle_type: Optional[str] = "" +) -> None: """ Convert stored trades data to ohlcv data """ @@ -367,12 +379,12 @@ def convert_trades_to_ohlcv(pairs: List[str], timeframes: List[str], trades = data_handler_trades.trades_load(pair) for timeframe in timeframes: if erase: - if data_handler_ohlcv.ohlcv_purge(pair, timeframe): + if data_handler_ohlcv.ohlcv_purge(pair, timeframe, candle_type=candle_type): logger.info(f'Deleting existing data for pair {pair}, interval {timeframe}.') try: ohlcv = trades_to_ohlcv(trades, timeframe) # Store ohlcv - data_handler_ohlcv.ohlcv_store(pair, timeframe, data=ohlcv) + data_handler_ohlcv.ohlcv_store(pair, timeframe, data=ohlcv, candle_type=candle_type) except ValueError: logger.exception(f'Could not convert {pair} to OHLCV.') diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index 05052b2d7..debdb43b7 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -35,7 +35,12 @@ class IDataHandler(ABC): """ @abstractclassmethod - def ohlcv_get_pairs(cls, datadir: Path, timeframe: str) -> List[str]: + def ohlcv_get_pairs( + cls, + datadir: Path, + timeframe: str, + candle_type: Optional[str] = "" + ) -> List[str]: """ Returns a list of all pairs with ohlcv data available in this datadir for the specified timeframe @@ -45,7 +50,13 @@ class IDataHandler(ABC): """ @abstractmethod - def ohlcv_store(self, pair: str, timeframe: str, data: DataFrame) -> None: + def ohlcv_store( + self, + pair: str, + timeframe: str, + data: DataFrame, + candle_type: Optional[str] = "" + ) -> None: """ Store ohlcv data. :param pair: Pair - used to generate filename @@ -57,6 +68,7 @@ class IDataHandler(ABC): @abstractmethod def _ohlcv_load(self, pair: str, timeframe: str, timerange: Optional[TimeRange] = None, + candle_type: Optional[str] = "" ) -> DataFrame: """ Internal method used to load data for one pair from disk. @@ -71,7 +83,7 @@ class IDataHandler(ABC): """ @abstractmethod - def ohlcv_purge(self, pair: str, timeframe: str) -> bool: + def ohlcv_purge(self, pair: str, timeframe: str, candle_type: Optional[str] = "") -> bool: """ Remove data for this pair :param pair: Delete data for this pair. @@ -80,7 +92,13 @@ class IDataHandler(ABC): """ @abstractmethod - def ohlcv_append(self, pair: str, timeframe: str, data: DataFrame) -> None: + def ohlcv_append( + self, + pair: str, + timeframe: str, + data: DataFrame, + candle_type: Optional[str] = "" + ) -> None: """ Append data to existing data structures :param pair: Pair @@ -146,7 +164,8 @@ class IDataHandler(ABC): fill_missing: bool = True, drop_incomplete: bool = True, startup_candles: int = 0, - warn_no_data: bool = True + warn_no_data: bool = True, + candle_type: Optional[str] = "" ) -> DataFrame: """ Load cached candle (OHLCV) data for the given pair. @@ -165,9 +184,13 @@ class IDataHandler(ABC): if startup_candles > 0 and timerange_startup: timerange_startup.subtract_start(timeframe_to_seconds(timeframe) * startup_candles) - pairdf = self._ohlcv_load(pair, timeframe, - timerange=timerange_startup) - if self._check_empty_df(pairdf, pair, timeframe, warn_no_data): + pairdf = self._ohlcv_load( + pair, + timeframe, + timerange=timerange_startup, + candle_type=candle_type + ) + if self._check_empty_df(pairdf, pair, timeframe, warn_no_data, candle_type): return pairdf else: enddate = pairdf.iloc[-1]['date'] @@ -175,7 +198,13 @@ class IDataHandler(ABC): if timerange_startup: self._validate_pairdata(pair, pairdf, timerange_startup) pairdf = trim_dataframe(pairdf, timerange_startup) - if self._check_empty_df(pairdf, pair, timeframe, warn_no_data): + if self._check_empty_df( + pairdf, + pair, + timeframe, + warn_no_data, + candle_type + ): return pairdf # incomplete candles should only be dropped if we didn't trim the end beforehand. @@ -183,11 +212,19 @@ class IDataHandler(ABC): pair=pair, fill_missing=fill_missing, drop_incomplete=(drop_incomplete and - enddate == pairdf.iloc[-1]['date'])) - self._check_empty_df(pairdf, pair, timeframe, warn_no_data) + enddate == pairdf.iloc[-1]['date']), + candle_type=candle_type) + self._check_empty_df(pairdf, pair, timeframe, warn_no_data, candle_type=candle_type) return pairdf - def _check_empty_df(self, pairdf: DataFrame, pair: str, timeframe: str, warn_no_data: bool): + def _check_empty_df( + self, + pairdf: DataFrame, + pair: str, + timeframe: str, + warn_no_data: bool, + candle_type: Optional[str] = "" + ): """ Warn on empty dataframe """ @@ -200,7 +237,13 @@ class IDataHandler(ABC): return True return False - def _validate_pairdata(self, pair, pairdata: DataFrame, timerange: TimeRange): + def _validate_pairdata( + self, + pair, + pairdata: DataFrame, + timerange: TimeRange, + candle_type: Optional[str] = "" + ): """ Validates pairdata for missing data at start end end and logs warnings. :param pairdata: Dataframe to validate diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index 24d6e814b..fca80d35a 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -35,7 +35,12 @@ class JsonDataHandler(IDataHandler): if match and len(match.groups()) > 1] @classmethod - def ohlcv_get_pairs(cls, datadir: Path, timeframe: str) -> List[str]: + def ohlcv_get_pairs( + cls, + datadir: Path, + timeframe: str, + candle_type: Optional[str] = "" + ) -> List[str]: """ Returns a list of all pairs with ohlcv data available in this datadir for the specified timeframe @@ -43,13 +48,23 @@ class JsonDataHandler(IDataHandler): :param timeframe: Timeframe to search pairs for :return: List of Pairs """ + if candle_type: + candle_type = f"-{candle_type}" + else: + candle_type = "" - _tmp = [re.search(r'^(\S+)(?=\-' + timeframe + '.json)', p.name) + _tmp = [re.search(r'^(\S+)(?=\-' + timeframe + candle_type + '.json)', p.name) for p in datadir.glob(f"*{timeframe}.{cls._get_file_extension()}")] # Check if regex found something and only return these results return [match[0].replace('_', '/') for match in _tmp if match] - def ohlcv_store(self, pair: str, timeframe: str, data: DataFrame) -> None: + def ohlcv_store( + self, + pair: str, + timeframe: str, + data: DataFrame, + candle_type: Optional[str] = "" + ) -> None: """ Store data in json format "values". format looks as follows: @@ -59,7 +74,12 @@ class JsonDataHandler(IDataHandler): :param data: Dataframe containing OHLCV data :return: None """ - filename = self._pair_data_filename(self._datadir, pair, timeframe) + filename = self._pair_data_filename( + self._datadir, + pair, + timeframe, + candle_type + ) _data = data.copy() # Convert date to int _data['date'] = _data['date'].view(np.int64) // 1000 // 1000 @@ -71,6 +91,7 @@ class JsonDataHandler(IDataHandler): def _ohlcv_load(self, pair: str, timeframe: str, timerange: Optional[TimeRange] = None, + candle_type: Optional[str] = "" ) -> DataFrame: """ Internal method used to load data for one pair from disk. @@ -83,7 +104,7 @@ class JsonDataHandler(IDataHandler): all data where possible. :return: DataFrame with ohlcv data, or empty DataFrame """ - filename = self._pair_data_filename(self._datadir, pair, timeframe) + filename = self._pair_data_filename(self._datadir, pair, timeframe, candle_type=candle_type) if not filename.exists(): return DataFrame(columns=self._columns) try: @@ -100,20 +121,26 @@ class JsonDataHandler(IDataHandler): infer_datetime_format=True) return pairdata - def ohlcv_purge(self, pair: str, timeframe: str) -> bool: + def ohlcv_purge(self, pair: str, timeframe: str, candle_type: Optional[str] = "") -> bool: """ Remove data for this pair :param pair: Delete data for this pair. :param timeframe: Timeframe (e.g. "5m") :return: True when deleted, false if file did not exist. """ - filename = self._pair_data_filename(self._datadir, pair, timeframe) + filename = self._pair_data_filename(self._datadir, pair, timeframe, candle_type=candle_type) if filename.exists(): filename.unlink() return True return False - def ohlcv_append(self, pair: str, timeframe: str, data: DataFrame) -> None: + def ohlcv_append( + self, + pair: str, + timeframe: str, + data: DataFrame, + candle_type: Optional[str] = "" + ) -> None: """ Append data to existing data structures :param pair: Pair @@ -187,9 +214,18 @@ class JsonDataHandler(IDataHandler): return False @classmethod - def _pair_data_filename(cls, datadir: Path, pair: str, timeframe: str) -> Path: + def _pair_data_filename( + cls, + datadir: Path, + pair: str, + timeframe: str, + candle_type: Optional[str] = "" + ) -> Path: pair_s = misc.pair_to_filename(pair) - filename = datadir.joinpath(f'{pair_s}-{timeframe}.{cls._get_file_extension()}') + if candle_type: + candle_type = f"-{candle_type}" + filename = datadir.joinpath( + f'{pair_s}-{timeframe}{candle_type}.{cls._get_file_extension()}') return filename @classmethod diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index b12655b86..9d9146a56 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -207,15 +207,15 @@ class Binance(Exchange): since_ms: int, is_new_pair: bool, raise_: bool = False, - price: Optional[str] = None + candle_type: Optional[str] = "" ) -> Tuple[str, str, List]: """ Overwrite to introduce "fast new pair" functionality by detecting the pair's listing date Does not work for other exchanges, which don't return the earliest data when called with "0" - :param price: "mark" if retrieving the mark price cnadles + :param candle_type: "mark" if retrieving the mark price cnadles """ if is_new_pair: - x = await self._async_get_candle_history(pair, timeframe, 0, price) + x = await self._async_get_candle_history(pair, timeframe, 0, candle_type) if x and x[2] and x[2][0] and x[2][0][0] > since_ms: # Set starting date to first available candle. since_ms = x[2][0][0] @@ -228,7 +228,7 @@ class Binance(Exchange): since_ms=since_ms, is_new_pair=is_new_pair, raise_=raise_, - price=price + candle_type=candle_type ) def funding_fee_cutoff(self, open_date: datetime): diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index f9fa708f2..2cf267b7c 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1309,14 +1309,9 @@ class Exchange: # Historic data - def get_historic_ohlcv( - self, - pair: str, - timeframe: str, - since_ms: int, - is_new_pair: bool = False, - price: Optional[str] = None - ) -> List: + def get_historic_ohlcv(self, pair: str, timeframe: str, + since_ms: int, is_new_pair: bool = False, + candle_type: Optional[str] = "") -> List: """ Get candle history using asyncio and returns the list of candles. Handles all async work for this. @@ -1324,52 +1319,37 @@ class Exchange: :param pair: Pair to download :param timeframe: Timeframe to get data for :param since_ms: Timestamp in milliseconds to get history from - :param price: "mark" if retrieving the mark price cnadles, "index" for index price candles :return: List with candle (OHLCV) data """ pair, timeframe, data = asyncio.get_event_loop().run_until_complete( - self._async_get_historic_ohlcv( - pair=pair, - timeframe=timeframe, - since_ms=since_ms, - is_new_pair=is_new_pair, - price=price - )) + self._async_get_historic_ohlcv(pair=pair, timeframe=timeframe, + since_ms=since_ms, is_new_pair=is_new_pair, + candle_type=candle_type)) logger.info(f"Downloaded data for {pair} with length {len(data)}.") return data - def get_historic_ohlcv_as_df( - self, - pair: str, - timeframe: str, - since_ms: int, - price: Optional[str] = None - ) -> DataFrame: + def get_historic_ohlcv_as_df(self, pair: str, timeframe: str, + since_ms: int, candle_type: Optional[str] = "") -> DataFrame: """ Minimal wrapper around get_historic_ohlcv - converting the result into a dataframe :param pair: Pair to download :param timeframe: Timeframe to get data for :param since_ms: Timestamp in milliseconds to get history from - :param price: "mark" if retrieving the mark price candles, "index" for index price candles :return: OHLCV DataFrame """ - ticks = self.get_historic_ohlcv(pair, timeframe, since_ms=since_ms, price=price) + ticks = self.get_historic_ohlcv(pair, timeframe, since_ms=since_ms) return ohlcv_to_dataframe(ticks, timeframe, pair=pair, fill_missing=True, - drop_incomplete=self._ohlcv_partial_candle) + drop_incomplete=self._ohlcv_partial_candle, + candle_type=candle_type) - async def _async_get_historic_ohlcv( - self, - pair: str, - timeframe: str, - since_ms: int, - is_new_pair: bool, - raise_: bool = False, - price: Optional[str] = None - ) -> Tuple[str, str, List]: + async def _async_get_historic_ohlcv(self, pair: str, timeframe: str, + since_ms: int, is_new_pair: bool, + raise_: bool = False, + candle_type: Optional[str] = "" + ) -> Tuple[str, str, List]: """ Download historic ohlcv :param is_new_pair: used by binance subclass to allow "fast" new pair downloading - :param price: "mark" if retrieving the mark price cnadles, "index" for index price candles """ one_call = timeframe_to_msecs(timeframe) * self.ohlcv_candle_limit(timeframe) @@ -1378,13 +1358,8 @@ class Exchange: one_call, arrow.utcnow().shift(seconds=one_call // 1000).humanize(only_distance=True) ) - input_coroutines = [ - self._async_get_candle_history( - pair, - timeframe, - since, - price - ) for since in + input_coroutines = [self._async_get_candle_history( + pair, timeframe, since, candle_type=candle_type) for since in range(since_ms, arrow.utcnow().int_timestamp * 1000, one_call)] data: List = [] @@ -1407,13 +1382,10 @@ class Exchange: data = sorted(data, key=lambda x: x[0]) return pair, timeframe, data - def refresh_latest_ohlcv( - self, - pair_list: ListPairsWithTimeframes, *, - since_ms: Optional[int] = None, - cache: bool = True, - price: Optional[str] = None - ) -> Dict[Tuple[str, str], DataFrame]: + def refresh_latest_ohlcv(self, pair_list: ListPairsWithTimeframes, *, + since_ms: Optional[int] = None, cache: bool = True, + candle_type: Optional[str] = "" + ) -> Dict[Tuple[str, str], DataFrame]: """ Refresh in-memory OHLCV asynchronously and set `_klines` with the result Loops asynchronously over pair_list and downloads all pairs async (semi-parallel). @@ -1421,7 +1393,6 @@ class Exchange: :param pair_list: List of 2 element tuples containing pair, interval to refresh :param since_ms: time since when to download, in milliseconds :param cache: Assign result to _klines. Usefull for one-off downloads like for pairlists - :param price: "mark" if retrieving the mark price cnadles, "index" for index price candles :return: Dict of [{(pair, timeframe): Dataframe}] """ logger.debug("Refreshing candle (OHLCV) data for %d pairs", len(pair_list)) @@ -1445,7 +1416,7 @@ class Exchange: else: # One call ... "regular" refresh input_coroutines.append(self._async_get_candle_history( - pair, timeframe, since_ms=since_ms, price=price)) + pair, timeframe, since_ms=since_ms, candle_type=candle_type,)) else: logger.debug( "Using cached candle (OHLCV) data for pair %s, timeframe %s ...", @@ -1470,7 +1441,8 @@ class Exchange: # keeping parsed dataframe in cache ohlcv_df = ohlcv_to_dataframe( ticks, timeframe, pair=pair, fill_missing=True, - drop_incomplete=self._ohlcv_partial_candle) + drop_incomplete=self._ohlcv_partial_candle, + candle_type=candle_type) results_df[(pair, timeframe)] = ohlcv_df if cache: self._klines[(pair, timeframe)] = ohlcv_df @@ -1493,11 +1465,11 @@ class Exchange: pair: str, timeframe: str, since_ms: Optional[int] = None, - price: Optional[str] = None + candle_type: Optional[str] = "", ) -> Tuple[str, str, List]: """ Asynchronously get candle history data using fetch_ohlcv - :param price: "mark" if retrieving the mark price cnadles, "index" for index price candles + :param candle_type: "mark" if retrieving the mark price cnadles, "index" for index price candles returns tuple: (pair, timeframe, ohlcv_list) """ try: @@ -1507,12 +1479,12 @@ class Exchange: "Fetching pair %s, interval %s, since %s %s...", pair, timeframe, since_ms, s ) - # TODO-lev: Does this put price into params correctly? - params = self._ft_has.get('ohlcv_params', {price: price}) + params = self._ft_has.get('ohlcv_params', {}) data = await self._api_async.fetch_ohlcv(pair, timeframe=timeframe, since=since_ms, limit=self.ohlcv_candle_limit(timeframe), - params=params) + params=params, + candle_type=candle_type) # Some exchanges sort OHLCV in ASC order and others in DESC. # Ex: Bittrex returns the list of OHLCV in ASC order (oldest first, newest last) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 59a64abab..c57880bdc 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1568,7 +1568,7 @@ def test_get_historic_ohlcv(default_conf, mocker, caplog, exchange_name): ] pair = 'ETH/BTC' - async def mock_candle_hist(pair, timeframe, since_ms, price=None): + async def mock_candle_hist(pair, timeframe, since_ms, candle_type=None): return pair, timeframe, ohlcv exchange._async_get_candle_history = Mock(wraps=mock_candle_hist) @@ -1625,7 +1625,7 @@ def test_get_historic_ohlcv_as_df(default_conf, mocker, exchange_name): ] pair = 'ETH/BTC' - async def mock_candle_hist(pair, timeframe, since_ms, price=None): + async def mock_candle_hist(pair, timeframe, since_ms, candle_type=None): return pair, timeframe, ohlcv exchange._async_get_candle_history = Mock(wraps=mock_candle_hist) From 3d95533bf9fa08b35530591821e9aadc338b6c83 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Sun, 7 Nov 2021 21:37:57 -0600 Subject: [PATCH 03/64] Removed candletype from converter methods --- freqtrade/data/converter.py | 18 +++++------------- freqtrade/data/history/history_utils.py | 6 ++---- freqtrade/data/history/idatahandler.py | 3 +-- freqtrade/exchange/exchange.py | 12 ++++++------ 4 files changed, 14 insertions(+), 25 deletions(-) diff --git a/freqtrade/data/converter.py b/freqtrade/data/converter.py index 67f8712a0..baacbb948 100644 --- a/freqtrade/data/converter.py +++ b/freqtrade/data/converter.py @@ -17,8 +17,7 @@ logger = logging.getLogger(__name__) def ohlcv_to_dataframe(ohlcv: list, timeframe: str, pair: str, *, - fill_missing: bool = True, drop_incomplete: bool = True, - candle_type: Optional[str] = "") -> DataFrame: + fill_missing: bool = True, drop_incomplete: bool = True) -> DataFrame: """ Converts a list with candle (OHLCV) data (in format returned by ccxt.fetch_ohlcv) to a Dataframe @@ -43,14 +42,12 @@ def ohlcv_to_dataframe(ohlcv: list, timeframe: str, pair: str, *, 'volume': 'float'}) return clean_ohlcv_dataframe(df, timeframe, pair, fill_missing=fill_missing, - drop_incomplete=drop_incomplete, - candle_type=candle_type) + drop_incomplete=drop_incomplete) def clean_ohlcv_dataframe(data: DataFrame, timeframe: str, pair: str, *, fill_missing: bool = True, - drop_incomplete: bool = True, - candle_type: Optional[str] = "") -> DataFrame: + drop_incomplete: bool = True) -> DataFrame: """ Cleanse a OHLCV dataframe by * Grouping it by date (removes duplicate tics) @@ -78,17 +75,12 @@ def clean_ohlcv_dataframe(data: DataFrame, timeframe: str, pair: str, *, logger.debug('Dropping last candle') if fill_missing: - return ohlcv_fill_up_missing_data(data, timeframe, pair, candle_type) + return ohlcv_fill_up_missing_data(data, timeframe, pair) else: return data -def ohlcv_fill_up_missing_data( - dataframe: DataFrame, - timeframe: str, - pair: str, - candle_type: Optional[str] = "" -) -> DataFrame: +def ohlcv_fill_up_missing_data(dataframe: DataFrame, timeframe: str, pair: str) -> DataFrame: """ Fills up missing data with 0 volume rows, using the previous close as price for "open", "high" "low" and "close", volume is set to 0 diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index fc14474f9..de58c8d0f 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -204,16 +204,14 @@ def _download_pair_history(pair: str, *, ) # TODO: Maybe move parsing to exchange class (?) new_dataframe = ohlcv_to_dataframe(new_data, timeframe, pair, - fill_missing=False, drop_incomplete=True, - candle_type=candle_type) + fill_missing=False, drop_incomplete=True) if data.empty: data = new_dataframe else: # Run cleaning again to ensure there were no duplicate candles # Especially between existing and new data. data = clean_ohlcv_dataframe(data.append(new_dataframe), timeframe, pair, - fill_missing=False, drop_incomplete=False, - candle_type=candle_type) + fill_missing=False, drop_incomplete=False) logger.debug("New Start: %s", f"{data.iloc[0]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None') diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index debdb43b7..7631d76ac 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -212,8 +212,7 @@ class IDataHandler(ABC): pair=pair, fill_missing=fill_missing, drop_incomplete=(drop_incomplete and - enddate == pairdf.iloc[-1]['date']), - candle_type=candle_type) + enddate == pairdf.iloc[-1]['date'])) self._check_empty_df(pairdf, pair, timeframe, warn_no_data, candle_type=candle_type) return pairdf diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 2cf267b7c..985fc702f 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -7,7 +7,7 @@ import http import inspect import logging from copy import deepcopy -from datetime import datetime, timedelta, timezone +from datetime import datetime, timezone from math import ceil from typing import Any, Dict, List, Optional, Tuple, Union @@ -1339,8 +1339,7 @@ class Exchange: """ ticks = self.get_historic_ohlcv(pair, timeframe, since_ms=since_ms) return ohlcv_to_dataframe(ticks, timeframe, pair=pair, fill_missing=True, - drop_incomplete=self._ohlcv_partial_candle, - candle_type=candle_type) + drop_incomplete=self._ohlcv_partial_candle) async def _async_get_historic_ohlcv(self, pair: str, timeframe: str, since_ms: int, is_new_pair: bool, @@ -1441,8 +1440,7 @@ class Exchange: # keeping parsed dataframe in cache ohlcv_df = ohlcv_to_dataframe( ticks, timeframe, pair=pair, fill_missing=True, - drop_incomplete=self._ohlcv_partial_candle, - candle_type=candle_type) + drop_incomplete=self._ohlcv_partial_candle) results_df[(pair, timeframe)] = ohlcv_df if cache: self._klines[(pair, timeframe)] = ohlcv_df @@ -1469,7 +1467,9 @@ class Exchange: ) -> Tuple[str, str, List]: """ Asynchronously get candle history data using fetch_ohlcv - :param candle_type: "mark" if retrieving the mark price cnadles, "index" for index price candles + :param candle_type: + "mark" if retrieving the mark price cnadles + "index" for index price candles returns tuple: (pair, timeframe, ohlcv_list) """ try: From a657707ca3a91ebbff312aa0366078e7a1a50635 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Thu, 18 Nov 2021 01:56:25 -0600 Subject: [PATCH 04/64] Added timedelta to exchange --- freqtrade/exchange/exchange.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 985fc702f..39e986f3d 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -7,7 +7,7 @@ import http import inspect import logging from copy import deepcopy -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from math import ceil from typing import Any, Dict, List, Optional, Tuple, Union From 12060a2d2cee3e3aadd0dcab3a7cbd11fa6e06dc Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Fri, 19 Nov 2021 03:20:30 -0600 Subject: [PATCH 05/64] fixed error with fetch_ohlcv candndle_type --- freqtrade/exchange/exchange.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 39e986f3d..8511df7ee 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1480,11 +1480,12 @@ class Exchange: pair, timeframe, since_ms, s ) params = self._ft_has.get('ohlcv_params', {}) + if candle_type: + params = params.update({'price': candle_type}) data = await self._api_async.fetch_ohlcv(pair, timeframe=timeframe, since=since_ms, limit=self.ohlcv_candle_limit(timeframe), - params=params, - candle_type=candle_type) + params=params) # Some exchanges sort OHLCV in ASC order and others in DESC. # Ex: Bittrex returns the list of OHLCV in ASC order (oldest first, newest last) From 043ed3e330fc89f93705af97801b812b47fddb87 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Fri, 19 Nov 2021 03:32:37 -0600 Subject: [PATCH 06/64] Added candle_type tests for test_json_pair_data_filename --- tests/data/test_history.py | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 73ceabbbf..b6e84f32c 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -143,19 +143,31 @@ def test_testdata_path(testdatadir) -> None: assert str(Path('tests') / 'testdata') in str(testdatadir) -@pytest.mark.parametrize("pair,expected_result", [ - ("ETH/BTC", 'freqtrade/hello/world/ETH_BTC-5m.json'), - ("Fabric Token/ETH", 'freqtrade/hello/world/Fabric_Token_ETH-5m.json'), - ("ETHH20", 'freqtrade/hello/world/ETHH20-5m.json'), - (".XBTBON2H", 'freqtrade/hello/world/_XBTBON2H-5m.json'), - ("ETHUSD.d", 'freqtrade/hello/world/ETHUSD_d-5m.json'), - ("ACC_OLD/BTC", 'freqtrade/hello/world/ACC_OLD_BTC-5m.json'), +@pytest.mark.parametrize("pair,expected_result,candle_type", [ + ("ETH/BTC", 'freqtrade/hello/world/ETH_BTC-5m.json', ""), + ("Fabric Token/ETH", 'freqtrade/hello/world/Fabric_Token_ETH-5m.json', ""), + ("ETHH20", 'freqtrade/hello/world/ETHH20-5m.json', ""), + (".XBTBON2H", 'freqtrade/hello/world/_XBTBON2H-5m.json', ""), + ("ETHUSD.d", 'freqtrade/hello/world/ETHUSD_d-5m.json', ""), + ("ACC_OLD/BTC", 'freqtrade/hello/world/ACC_OLD_BTC-5m.json', ""), + ("ETH/BTC", 'freqtrade/hello/world/ETH_BTC-5m-mark.json', "mark"), + ("ACC_OLD/BTC", 'freqtrade/hello/world/ACC_OLD_BTC-5m-index.json', "index"), ]) -def test_json_pair_data_filename(pair, expected_result): - fn = JsonDataHandler._pair_data_filename(Path('freqtrade/hello/world'), pair, '5m') +def test_json_pair_data_filename(pair, expected_result, candle_type): + fn = JsonDataHandler._pair_data_filename( + Path('freqtrade/hello/world'), + pair, + '5m', + candle_type + ) assert isinstance(fn, Path) assert fn == Path(expected_result) - fn = JsonGzDataHandler._pair_data_filename(Path('freqtrade/hello/world'), pair, '5m') + fn = JsonGzDataHandler._pair_data_filename( + Path('freqtrade/hello/world'), + pair, + '5m', + candle_type + ) assert isinstance(fn, Path) assert fn == Path(expected_result + '.gz') From 91a11d01e9323abf919e8ff899322d7425635b80 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Fri, 19 Nov 2021 03:55:52 -0600 Subject: [PATCH 07/64] Added XRP_USDT-1h-mark json testdat file, and test for ohlcv_load with candle_type=mark --- tests/data/test_history.py | 6 ++ tests/testdata/XRP_USDT-1h-mark.json | 102 +++++++++++++++++++++++++++ 2 files changed, 108 insertions(+) create mode 100644 tests/testdata/XRP_USDT-1h-mark.json diff --git a/tests/data/test_history.py b/tests/data/test_history.py index b6e84f32c..0030797e8 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -669,6 +669,12 @@ def test_jsondatahandler_ohlcv_load(testdatadir, caplog): df = dh.ohlcv_load('XRP/ETH', '5m') assert len(df) == 711 + df_mark = dh.ohlcv_load('XRP/USDT', '1h', candle_type="mark") + assert len(df_mark) == 99 + + df_no_mark = dh.ohlcv_load('XRP/USDT', '1h') + assert len(df_no_mark) == 0 + # Failure case (empty array) df1 = dh.ohlcv_load('NOPAIR/XXX', '4m') assert len(df1) == 0 diff --git a/tests/testdata/XRP_USDT-1h-mark.json b/tests/testdata/XRP_USDT-1h-mark.json new file mode 100644 index 000000000..26703b945 --- /dev/null +++ b/tests/testdata/XRP_USDT-1h-mark.json @@ -0,0 +1,102 @@ +[ + [1636956000000, 1.20932, 1.21787, 1.20763, 1.21431, null], + [1636959600000, 1.21431, 1.2198, 1.20895, 1.20895, null], + [1636963200000, 1.20902, 1.21106, 1.19972, 1.20968, null], + [1636966800000, 1.20968, 1.21876, 1.20791, 1.20998, null], + [1636970400000, 1.20999, 1.21043, 1.20442, 1.20859, null], + [1636974000000, 1.20858, 1.20933, 1.20154, 1.20581, null], + [1636977600000, 1.20584, 1.20775, 1.20065, 1.20337, null], + [1636981200000, 1.20342, 1.2097, 1.19327, 1.19792, null], + [1636984800000, 1.19796, 1.1982, 1.18611, 1.19024, null], + [1636988400000, 1.19025, 1.19177, 1.18373, 1.18771, null], + [1636992000000, 1.18768, 1.19109, 1.18095, 1.1887, null], + [1636995600000, 1.18869, 1.18968, 1.18355, 1.18387, null], + [1636999200000, 1.18388, 1.18729, 1.17753, 1.18138, null], + [1637002800000, 1.18145, 1.18684, 1.17799, 1.18463, null], + [1637006400000, 1.18464, 1.18474, 1.17368, 1.17652, null], + [1637010000000, 1.17653, 1.18185, 1.16557, 1.17979, null], + [1637013600000, 1.17979, 1.18113, 1.16934, 1.18014, null], + [1637017200000, 1.18014, 1.18015, 1.16999, 1.17214, null], + [1637020800000, 1.17214, 1.17217, 1.12958, 1.14209, null], + [1637024400000, 1.14255, 1.14666, 1.10933, 1.14198, null], + [1637028000000, 1.14197, 1.14419, 1.12766, 1.12999, null], + [1637031600000, 1.12999, 1.13522, 1.11142, 1.12177, null], + [1637035200000, 1.12176, 1.13211, 1.10579, 1.1288, null], + [1637038800000, 1.12871, 1.13243, 1.12142, 1.12316, null], + [1637042400000, 1.12323, 1.1262, 1.11489, 1.12429, null], + [1637046000000, 1.12406, 1.12727, 1.11835, 1.1249, null], + [1637049600000, 1.12485, 1.13047, 1.1211, 1.12931, null], + [1637053200000, 1.12931, 1.13346, 1.10256, 1.10267, null], + [1637056800000, 1.10266, 1.10412, 1.04149, 1.0928, null], + [1637060400000, 1.09277, 1.09856, 1.08371, 1.09093, null], + [1637064000000, 1.09094, 1.09512, 1.079, 1.08003, null], + [1637067600000, 1.0802, 1.09914, 1.08016, 1.09515, null], + [1637071200000, 1.09518, 1.11627, 1.0937, 1.10985, null], + [1637074800000, 1.10985, 1.11353, 1.09618, 1.10071, null], + [1637078400000, 1.09989, 1.10852, 1.09763, 1.10461, null], + [1637082000000, 1.10459, 1.10837, 1.09662, 1.09847, null], + [1637085600000, 1.09858, 1.10506, 1.08687, 1.08716, null], + [1637089200000, 1.08677, 1.10096, 1.08151, 1.09271, null], + [1637092800000, 1.09245, 1.09269, 1.06592, 1.08025, null], + [1637096400000, 1.08026, 1.09732, 1.07953, 1.09527, null], + [1637100000000, 1.09527, 1.10506, 1.09524, 1.09933, null], + [1637103600000, 1.09933, 1.10205, 1.08761, 1.08785, null], + [1637107200000, 1.08763, 1.09518, 1.07646, 1.07999, null], + [1637110800000, 1.07997, 1.0978, 1.07651, 1.07936, null], + [1637114400000, 1.07932, 1.08758, 1.07352, 1.07603, null], + [1637118000000, 1.07604, 1.08542, 1.05931, 1.06764, null], + [1637121600000, 1.06788, 1.07848, 1.06045, 1.07608, null], + [1637125200000, 1.07613, 1.08797, 1.07293, 1.08377, null], + [1637128800000, 1.08379, 1.08567, 1.07428, 1.07942, null], + [1637132400000, 1.07958, 1.09472, 1.07356, 1.08713, null], + [1637136000000, 1.08714, 1.09149, 1.08018, 1.08021, null], + [1637139600000, 1.08021, 1.08021, 1.0668, 1.07032, null], + [1637143200000, 1.07042, 1.10563, 1.07034, 1.10255, null], + [1637146800000, 1.10284, 1.10954, 1.09767, 1.10685, null], + [1637150400000, 1.10669, 1.10848, 1.10157, 1.10537, null], + [1637154000000, 1.10537, 1.11263, 1.09554, 1.09585, null], + [1637157600000, 1.09569, 1.10051, 1.08402, 1.08431, null], + [1637161200000, 1.08444, 1.08942, 1.07569, 1.08489, null], + [1637164800000, 1.08498, 1.09581, 1.07939, 1.09485, null], + [1637168400000, 1.09443, 1.09793, 1.08778, 1.0944, null], + [1637172000000, 1.09445, 1.10227, 1.09376, 1.0992, null], + [1637175600000, 1.0992, 1.10189, 1.09216, 1.09474, null], + [1637179200000, 1.09476, 1.10198, 1.09045, 1.0993, null], + [1637182800000, 1.09934, 1.09959, 1.08755, 1.0948, null], + [1637186400000, 1.09483, 1.09519, 1.08532, 1.0923, null], + [1637190000000, 1.0923, 1.09876, 1.0874, 1.095, null], + [1637193600000, 1.09503, 1.10673, 1.09047, 1.10441, null], + [1637197200000, 1.10437, 1.16166, 1.09815, 1.12902, null], + [1637200800000, 1.12875, 1.15094, 1.1242, 1.13764, null], + [1637204400000, 1.13795, 1.14262, 1.12341, 1.12423, null], + [1637208000000, 1.12424, 1.14806, 1.11333, 1.1142, null], + [1637211600000, 1.11435, 1.12608, 1.11085, 1.11436, null], + [1637215200000, 1.11398, 1.11718, 1.10538, 1.11388, null], + [1637218800000, 1.1139, 1.11452, 1.09674, 1.1072, null], + [1637222400000, 1.10725, 1.10999, 1.10209, 1.10706, null], + [1637226000000, 1.10712, 1.10712, 1.07747, 1.08658, null], + [1637229600000, 1.08692, 1.09865, 1.0807, 1.09767, null], + [1637233200000, 1.09768, 1.10211, 1.08348, 1.08409, null], + [1637236800000, 1.08423, 1.09498, 1.08002, 1.08259, null], + [1637240400000, 1.0827, 1.08773, 1.06597, 1.07719, null], + [1637244000000, 1.07718, 1.08075, 1.06678, 1.07077, null], + [1637247600000, 1.07029, 1.07824, 1.04568, 1.05497, null], + [1637251200000, 1.05591, 1.06325, 1.03957, 1.04032, null], + [1637254800000, 1.04051, 1.05342, 1.01557, 1.04158, null], + [1637258400000, 1.04153, 1.05436, 1.04122, 1.05208, null], + [1637262000000, 1.05207, 1.05948, 1.04961, 1.05515, null], + [1637265600000, 1.05516, 1.05927, 1.04767, 1.04808, null], + [1637269200000, 1.04789, 1.05622, 1.04191, 1.04587, null], + [1637272800000, 1.04575, 1.05336, 1.03405, 1.03941, null], + [1637276400000, 1.03931, 1.04614, 1.02868, 1.0411, null], + [1637280000000, 1.04093, 1.05672, 1.0295, 1.05495, null], + [1637283600000, 1.05495, 1.0553, 1.03548, 1.03595, null], + [1637287200000, 1.0359, 1.04585, 1.02026, 1.02312, null], + [1637290800000, 1.0242, 1.02908, 1.01788, 1.02871, null], + [1637294400000, 1.02871, 1.04474, 1.02584, 1.04247, null], + [1637298000000, 1.04251, 1.04654, 1.03685, 1.0449, null], + [1637301600000, 1.0449, 1.04971, 1.04109, 1.04452, null], + [1637305200000, 1.04456, 1.04875, 1.03802, 1.04268, null], + [1637308800000, 1.04239, 1.06573, 1.04164, 1.05717, null], + [1637312400000, 1.05721, 1.06464, 1.05619, 1.06051, null] +] From 843ca22a566a8074fa24c136827d2c22b374fefc Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Fri, 19 Nov 2021 04:55:50 -0600 Subject: [PATCH 08/64] mark price test ohlcv_get_pairs --- freqtrade/data/history/hdf5datahandler.py | 2 +- freqtrade/data/history/jsondatahandler.py | 2 +- tests/data/test_history.py | 10 +++ tests/testdata/UNITTEST_USDT-1h-mark.json | 102 ++++++++++++++++++++++ 4 files changed, 114 insertions(+), 2 deletions(-) create mode 100644 tests/testdata/UNITTEST_USDT-1h-mark.json diff --git a/freqtrade/data/history/hdf5datahandler.py b/freqtrade/data/history/hdf5datahandler.py index 7d5b4f041..6a66118c4 100644 --- a/freqtrade/data/history/hdf5datahandler.py +++ b/freqtrade/data/history/hdf5datahandler.py @@ -54,7 +54,7 @@ class HDF5DataHandler(IDataHandler): candle_type = "" _tmp = [re.search(r'^(\S+)(?=\-' + timeframe + candle_type + '.h5)', p.name) - for p in datadir.glob(f"*{timeframe}.h5")] + for p in datadir.glob(f"*{timeframe}{candle_type}.h5")] # Check if regex found something and only return these results return [match[0].replace('_', '/') for match in _tmp if match] diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index fca80d35a..ddbc626bc 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -54,7 +54,7 @@ class JsonDataHandler(IDataHandler): candle_type = "" _tmp = [re.search(r'^(\S+)(?=\-' + timeframe + candle_type + '.json)', p.name) - for p in datadir.glob(f"*{timeframe}.{cls._get_file_extension()}")] + for p in datadir.glob(f"*{timeframe}{candle_type}.{cls._get_file_extension()}")] # Check if regex found something and only return these results return [match[0].replace('_', '/') for match in _tmp if match] diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 0030797e8..a1448c92d 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -629,6 +629,16 @@ def test_datahandler_ohlcv_get_pairs(testdatadir): pairs = HDF5DataHandler.ohlcv_get_pairs(testdatadir, '5m') assert set(pairs) == {'UNITTEST/BTC'} + pairs = JsonDataHandler.ohlcv_get_pairs(testdatadir, '1h', 'mark') + assert set(pairs) == {'UNITTEST/USDT', 'XRP/USDT'} + + # TODO-lev: The tests below + # pairs = JsonGzDataHandler.ohlcv_get_pairs(testdatadir, '8m') + # assert set(pairs) == {'UNITTEST/BTC'} + + # pairs = HDF5DataHandler.ohlcv_get_pairs(testdatadir, '5m') + # assert set(pairs) == {'UNITTEST/BTC'} + def test_datahandler_ohlcv_get_available_data(testdatadir): paircombs = JsonDataHandler.ohlcv_get_available_data(testdatadir) diff --git a/tests/testdata/UNITTEST_USDT-1h-mark.json b/tests/testdata/UNITTEST_USDT-1h-mark.json new file mode 100644 index 000000000..312f616fb --- /dev/null +++ b/tests/testdata/UNITTEST_USDT-1h-mark.json @@ -0,0 +1,102 @@ +[ + [1636959600000, 1.21431, 1.2198, 1.20895, 1.20895, null], + [1636963200000, 1.20902, 1.21106, 1.19972, 1.20968, null], + [1636966800000, 1.20968, 1.21876, 1.20791, 1.20998, null], + [1636970400000, 1.20999, 1.21043, 1.20442, 1.20859, null], + [1636974000000, 1.20858, 1.20933, 1.20154, 1.20581, null], + [1636977600000, 1.20584, 1.20775, 1.20065, 1.20337, null], + [1636981200000, 1.20342, 1.2097, 1.19327, 1.19792, null], + [1636984800000, 1.19796, 1.1982, 1.18611, 1.19024, null], + [1636988400000, 1.19025, 1.19177, 1.18373, 1.18771, null], + [1636992000000, 1.18768, 1.19109, 1.18095, 1.1887, null], + [1636995600000, 1.18869, 1.18968, 1.18355, 1.18387, null], + [1636999200000, 1.18388, 1.18729, 1.17753, 1.18138, null], + [1637002800000, 1.18145, 1.18684, 1.17799, 1.18463, null], + [1637006400000, 1.18464, 1.18474, 1.17368, 1.17652, null], + [1637010000000, 1.17653, 1.18185, 1.16557, 1.17979, null], + [1637013600000, 1.17979, 1.18113, 1.16934, 1.18014, null], + [1637017200000, 1.18014, 1.18015, 1.16999, 1.17214, null], + [1637020800000, 1.17214, 1.17217, 1.12958, 1.14209, null], + [1637024400000, 1.14255, 1.14666, 1.10933, 1.14198, null], + [1637028000000, 1.14197, 1.14419, 1.12766, 1.12999, null], + [1637031600000, 1.12999, 1.13522, 1.11142, 1.12177, null], + [1637035200000, 1.12176, 1.13211, 1.10579, 1.1288, null], + [1637038800000, 1.12871, 1.13243, 1.12142, 1.12316, null], + [1637042400000, 1.12323, 1.1262, 1.11489, 1.12429, null], + [1637046000000, 1.12406, 1.12727, 1.11835, 1.1249, null], + [1637049600000, 1.12485, 1.13047, 1.1211, 1.12931, null], + [1637053200000, 1.12931, 1.13346, 1.10256, 1.10267, null], + [1637056800000, 1.10266, 1.10412, 1.04149, 1.0928, null], + [1637060400000, 1.09277, 1.09856, 1.08371, 1.09093, null], + [1637064000000, 1.09094, 1.09512, 1.079, 1.08003, null], + [1637067600000, 1.0802, 1.09914, 1.08016, 1.09515, null], + [1637071200000, 1.09518, 1.11627, 1.0937, 1.10985, null], + [1637074800000, 1.10985, 1.11353, 1.09618, 1.10071, null], + [1637078400000, 1.09989, 1.10852, 1.09763, 1.10461, null], + [1637082000000, 1.10459, 1.10837, 1.09662, 1.09847, null], + [1637085600000, 1.09858, 1.10506, 1.08687, 1.08716, null], + [1637089200000, 1.08677, 1.10096, 1.08151, 1.09271, null], + [1637092800000, 1.09245, 1.09269, 1.06592, 1.08025, null], + [1637096400000, 1.08026, 1.09732, 1.07953, 1.09527, null], + [1637100000000, 1.09527, 1.10506, 1.09524, 1.09933, null], + [1637103600000, 1.09933, 1.10205, 1.08761, 1.08785, null], + [1637107200000, 1.08763, 1.09518, 1.07646, 1.07999, null], + [1637110800000, 1.07997, 1.0978, 1.07651, 1.07936, null], + [1637114400000, 1.07932, 1.08758, 1.07352, 1.07603, null], + [1637118000000, 1.07604, 1.08542, 1.05931, 1.06764, null], + [1637121600000, 1.06788, 1.07848, 1.06045, 1.07608, null], + [1637125200000, 1.07613, 1.08797, 1.07293, 1.08377, null], + [1637128800000, 1.08379, 1.08567, 1.07428, 1.07942, null], + [1637132400000, 1.07958, 1.09472, 1.07356, 1.08713, null], + [1637136000000, 1.08714, 1.09149, 1.08018, 1.08021, null], + [1637139600000, 1.08021, 1.08021, 1.0668, 1.07032, null], + [1637143200000, 1.07042, 1.10563, 1.07034, 1.10255, null], + [1637146800000, 1.10284, 1.10954, 1.09767, 1.10685, null], + [1637150400000, 1.10669, 1.10848, 1.10157, 1.10537, null], + [1637154000000, 1.10537, 1.11263, 1.09554, 1.09585, null], + [1637157600000, 1.09569, 1.10051, 1.08402, 1.08431, null], + [1637161200000, 1.08444, 1.08942, 1.07569, 1.08489, null], + [1637164800000, 1.08498, 1.09581, 1.07939, 1.09485, null], + [1637168400000, 1.09443, 1.09793, 1.08778, 1.0944, null], + [1637172000000, 1.09445, 1.10227, 1.09376, 1.0992, null], + [1637175600000, 1.0992, 1.10189, 1.09216, 1.09474, null], + [1637179200000, 1.09476, 1.10198, 1.09045, 1.0993, null], + [1637182800000, 1.09934, 1.09959, 1.08755, 1.0948, null], + [1637186400000, 1.09483, 1.09519, 1.08532, 1.0923, null], + [1637190000000, 1.0923, 1.09876, 1.0874, 1.095, null], + [1637193600000, 1.09503, 1.10673, 1.09047, 1.10441, null], + [1637197200000, 1.10437, 1.16166, 1.09815, 1.12902, null], + [1637200800000, 1.12875, 1.15094, 1.1242, 1.13764, null], + [1637204400000, 1.13795, 1.14262, 1.12341, 1.12423, null], + [1637208000000, 1.12424, 1.14806, 1.11333, 1.1142, null], + [1637211600000, 1.11435, 1.12608, 1.11085, 1.11436, null], + [1637215200000, 1.11398, 1.11718, 1.10538, 1.11388, null], + [1637218800000, 1.1139, 1.11452, 1.09674, 1.1072, null], + [1637222400000, 1.10725, 1.10999, 1.10209, 1.10706, null], + [1637226000000, 1.10712, 1.10712, 1.07747, 1.08658, null], + [1637229600000, 1.08692, 1.09865, 1.0807, 1.09767, null], + [1637233200000, 1.09768, 1.10211, 1.08348, 1.08409, null], + [1637236800000, 1.08423, 1.09498, 1.08002, 1.08259, null], + [1637240400000, 1.0827, 1.08773, 1.06597, 1.07719, null], + [1637244000000, 1.07718, 1.08075, 1.06678, 1.07077, null], + [1637247600000, 1.07029, 1.07824, 1.04568, 1.05497, null], + [1637251200000, 1.05591, 1.06325, 1.03957, 1.04032, null], + [1637254800000, 1.04051, 1.05342, 1.01557, 1.04158, null], + [1637258400000, 1.04153, 1.05436, 1.04122, 1.05208, null], + [1637262000000, 1.05207, 1.05948, 1.04961, 1.05515, null], + [1637265600000, 1.05516, 1.05927, 1.04767, 1.04808, null], + [1637269200000, 1.04789, 1.05622, 1.04191, 1.04587, null], + [1637272800000, 1.04575, 1.05336, 1.03405, 1.03941, null], + [1637276400000, 1.03931, 1.04614, 1.02868, 1.0411, null], + [1637280000000, 1.04093, 1.05672, 1.0295, 1.05495, null], + [1637283600000, 1.05495, 1.0553, 1.03548, 1.03595, null], + [1637287200000, 1.0359, 1.04585, 1.02026, 1.02312, null], + [1637290800000, 1.0242, 1.02908, 1.01788, 1.02871, null], + [1637294400000, 1.02871, 1.04474, 1.02584, 1.04247, null], + [1637298000000, 1.04251, 1.04654, 1.03685, 1.0449, null], + [1637301600000, 1.0449, 1.04971, 1.04109, 1.04452, null], + [1637305200000, 1.04456, 1.04875, 1.03802, 1.04268, null], + [1637308800000, 1.04239, 1.06573, 1.04164, 1.05717, null], + [1637312400000, 1.05721, 1.06464, 1.05619, 1.05896, null], + [1637316000000, 1.05893, 1.05918, 1.04976, 1.05188, null] +] From c3b2929e7596b6bbd107ae4c328643d17f479220 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Fri, 19 Nov 2021 05:11:42 -0600 Subject: [PATCH 09/64] Added template for test test_hdf5datahandler_ohlcv_load_and_resave --- tests/data/test_history.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/tests/data/test_history.py b/tests/data/test_history.py index a1448c92d..6c811a673 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -801,18 +801,30 @@ def test_hdf5datahandler_trades_purge(mocker, testdatadir): assert unlinkmock.call_count == 1 -def test_hdf5datahandler_ohlcv_load_and_resave(testdatadir, tmpdir): +@pytest.mark.parametrize('pair, timeframe, candle_type, candle_append', [ + ('UNITTEST/BTC', '5m', '', ''), + # TODO-lev: The test below + # ('UNITTEST/USDT', '1h', 'mark', '-mark'), +]) +def test_hdf5datahandler_ohlcv_load_and_resave( + testdatadir, + tmpdir, + pair, + timeframe, + candle_type, + candle_append +): tmpdir1 = Path(tmpdir) dh = HDF5DataHandler(testdatadir) - ohlcv = dh.ohlcv_load('UNITTEST/BTC', '5m') + ohlcv = dh.ohlcv_load(pair, timeframe, candle_type=candle_type) assert isinstance(ohlcv, DataFrame) assert len(ohlcv) > 0 - file = tmpdir1 / 'UNITTEST_NEW-5m.h5' + file = tmpdir1 / f"UNITTEST_NEW-{timeframe}{candle_append}.h5" assert not file.is_file() dh1 = HDF5DataHandler(tmpdir1) - dh1.ohlcv_store('UNITTEST/NEW', '5m', ohlcv) + dh1.ohlcv_store('UNITTEST/NEW', timeframe, ohlcv, candle_type=candle_type) assert file.is_file() assert not ohlcv[ohlcv['date'] < '2018-01-15'].empty @@ -821,15 +833,15 @@ def test_hdf5datahandler_ohlcv_load_and_resave(testdatadir, tmpdir): timerange = TimeRange.parse_timerange('20180115-20180119') # Call private function to ensure timerange is filtered in hdf5 - ohlcv = dh._ohlcv_load('UNITTEST/BTC', '5m', timerange) - ohlcv1 = dh1._ohlcv_load('UNITTEST/NEW', '5m', timerange) + ohlcv = dh._ohlcv_load(pair, timeframe, timerange, candle_type=candle_type) + ohlcv1 = dh1._ohlcv_load('UNITTEST/NEW', timeframe, timerange, candle_type=candle_type) assert len(ohlcv) == len(ohlcv1) assert ohlcv.equals(ohlcv1) assert ohlcv[ohlcv['date'] < '2018-01-15'].empty assert ohlcv[ohlcv['date'] > '2018-01-19'].empty # Try loading inexisting file - ohlcv = dh.ohlcv_load('UNITTEST/NONEXIST', '5m') + ohlcv = dh.ohlcv_load('UNITTEST/NONEXIST', timeframe, candle_type=candle_type) assert ohlcv.empty From b4029533ec60235b531f63c66bde5667efa056bc Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Sat, 20 Nov 2021 22:43:25 -0600 Subject: [PATCH 10/64] removed candle type from idatahandler.py --- freqtrade/data/history/idatahandler.py | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index 7631d76ac..cfbeb5d33 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -39,7 +39,7 @@ class IDataHandler(ABC): cls, datadir: Path, timeframe: str, - candle_type: Optional[str] = "" + candle_type: Optional[str] = '' ) -> List[str]: """ Returns a list of all pairs with ohlcv data available in this datadir @@ -55,7 +55,7 @@ class IDataHandler(ABC): pair: str, timeframe: str, data: DataFrame, - candle_type: Optional[str] = "" + candle_type: Optional[str] = '' ) -> None: """ Store ohlcv data. @@ -68,7 +68,7 @@ class IDataHandler(ABC): @abstractmethod def _ohlcv_load(self, pair: str, timeframe: str, timerange: Optional[TimeRange] = None, - candle_type: Optional[str] = "" + candle_type: Optional[str] = '' ) -> DataFrame: """ Internal method used to load data for one pair from disk. @@ -83,7 +83,7 @@ class IDataHandler(ABC): """ @abstractmethod - def ohlcv_purge(self, pair: str, timeframe: str, candle_type: Optional[str] = "") -> bool: + def ohlcv_purge(self, pair: str, timeframe: str, candle_type: Optional[str] = '') -> bool: """ Remove data for this pair :param pair: Delete data for this pair. @@ -97,7 +97,7 @@ class IDataHandler(ABC): pair: str, timeframe: str, data: DataFrame, - candle_type: Optional[str] = "" + candle_type: Optional[str] = '' ) -> None: """ Append data to existing data structures @@ -165,7 +165,7 @@ class IDataHandler(ABC): drop_incomplete: bool = True, startup_candles: int = 0, warn_no_data: bool = True, - candle_type: Optional[str] = "" + candle_type: Optional[str] = '' ) -> DataFrame: """ Load cached candle (OHLCV) data for the given pair. @@ -190,7 +190,7 @@ class IDataHandler(ABC): timerange=timerange_startup, candle_type=candle_type ) - if self._check_empty_df(pairdf, pair, timeframe, warn_no_data, candle_type): + if self._check_empty_df(pairdf, pair, timeframe, warn_no_data): return pairdf else: enddate = pairdf.iloc[-1]['date'] @@ -202,8 +202,7 @@ class IDataHandler(ABC): pairdf, pair, timeframe, - warn_no_data, - candle_type + warn_no_data ): return pairdf @@ -213,7 +212,7 @@ class IDataHandler(ABC): fill_missing=fill_missing, drop_incomplete=(drop_incomplete and enddate == pairdf.iloc[-1]['date'])) - self._check_empty_df(pairdf, pair, timeframe, warn_no_data, candle_type=candle_type) + self._check_empty_df(pairdf, pair, timeframe, warn_no_data) return pairdf def _check_empty_df( @@ -221,8 +220,7 @@ class IDataHandler(ABC): pairdf: DataFrame, pair: str, timeframe: str, - warn_no_data: bool, - candle_type: Optional[str] = "" + warn_no_data: bool ): """ Warn on empty dataframe @@ -240,8 +238,7 @@ class IDataHandler(ABC): self, pair, pairdata: DataFrame, - timerange: TimeRange, - candle_type: Optional[str] = "" + timerange: TimeRange ): """ Validates pairdata for missing data at start end end and logs warnings. From 64a6abc541541242afcb098f16c1148eac65e107 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Sat, 20 Nov 2021 22:46:47 -0600 Subject: [PATCH 11/64] Added candle type to ohlcv_get_available_data --- freqtrade/commands/data_commands.py | 16 +++++++++++----- freqtrade/constants.py | 2 +- freqtrade/data/history/hdf5datahandler.py | 10 +++++++--- freqtrade/data/history/jsondatahandler.py | 9 ++++++--- tests/commands/test_commands.py | 11 ++++++----- 5 files changed, 31 insertions(+), 17 deletions(-) diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index 5dc5fe7ea..f55a857c4 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -161,10 +161,16 @@ def start_list_data(args: Dict[str, Any]) -> None: print(f"Found {len(paircombs)} pair / timeframe combinations.") groupedpair = defaultdict(list) - for pair, timeframe in sorted(paircombs, key=lambda x: (x[0], timeframe_to_minutes(x[1]))): - groupedpair[pair].append(timeframe) + for pair, timeframe, candle_type in sorted( + paircombs, + key=lambda x: (x[0], timeframe_to_minutes(x[1]), x[2]) + ): + groupedpair[(pair, candle_type)].append(timeframe) if groupedpair: - print(tabulate([(pair, ', '.join(timeframes)) for pair, timeframes in groupedpair.items()], - headers=("Pair", "Timeframe"), - tablefmt='psql', stralign='right')) + print(tabulate([ + (pair, ', '.join(timeframes), candle_type) + for (pair, candle_type), timeframes in groupedpair.items() + ], + headers=("Pair", "Timeframe", "Type"), + tablefmt='psql', stralign='right')) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index e41ecd4f8..52c21ce58 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -467,7 +467,7 @@ CANCEL_REASON = { } # List of pairs with their timeframes -PairWithTimeframe = Tuple[str, str] +PairWithTimeframe = Tuple[str, str, str] ListPairsWithTimeframes = List[PairWithTimeframe] # Type for trades list diff --git a/freqtrade/data/history/hdf5datahandler.py b/freqtrade/data/history/hdf5datahandler.py index 6a66118c4..a59d7255d 100644 --- a/freqtrade/data/history/hdf5datahandler.py +++ b/freqtrade/data/history/hdf5datahandler.py @@ -28,9 +28,13 @@ class HDF5DataHandler(IDataHandler): :param datadir: Directory to search for ohlcv files :return: List of Tuples of (pair, timeframe) """ - _tmp = [re.search(r'^([a-zA-Z_]+)\-(\d+\S+)(?=.h5)', p.name) - for p in datadir.glob("*.h5")] - return [(match[1].replace('_', '/'), match[2]) for match in _tmp + _tmp = [ + re.search( + r'^([a-zA-Z_]+)\-(\d+\S)\-?([a-zA-Z_]*)?(?=.h5)', + p.name + ) for p in datadir.glob("*.h5") + ] + return [(match[1].replace('_', '/'), match[2], match[3]) for match in _tmp if match and len(match.groups()) > 1] @classmethod diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index ddbc626bc..57b21f894 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -29,9 +29,12 @@ class JsonDataHandler(IDataHandler): :param datadir: Directory to search for ohlcv files :return: List of Tuples of (pair, timeframe) """ - _tmp = [re.search(r'^([a-zA-Z_]+)\-(\d+\S+)(?=.json)', p.name) - for p in datadir.glob(f"*.{cls._get_file_extension()}")] - return [(match[1].replace('_', '/'), match[2]) for match in _tmp + _tmp = [ + re.search( + r'^([a-zA-Z_]+)\-(\d+\S)\-?([a-zA-Z_]*)?(?=.json)', + p.name + ) for p in datadir.glob(f"*.{cls._get_file_extension()}")] + return [(match[1].replace('_', '/'), match[2], match[3]) for match in _tmp if match and len(match.groups()) > 1] @classmethod diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 55fc4463d..72d2c6de4 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -1326,9 +1326,10 @@ def test_start_list_data(testdatadir, capsys): pargs['config'] = None start_list_data(pargs) captured = capsys.readouterr() - assert "Found 17 pair / timeframe combinations." in captured.out - assert "\n| Pair | Timeframe |\n" in captured.out - assert "\n| UNITTEST/BTC | 1m, 5m, 8m, 30m |\n" in captured.out + assert "Found 19 pair / timeframe combinations." in captured.out + assert "\n| Pair | Timeframe | Type |\n" in captured.out + assert "\n| UNITTEST/BTC | 1m, 5m, 8m, 30m | |\n" in captured.out + assert "\n| UNITTEST/USDT | 1h | mark |\n" in captured.out args = [ "list-data", @@ -1343,9 +1344,9 @@ def test_start_list_data(testdatadir, capsys): start_list_data(pargs) captured = capsys.readouterr() assert "Found 2 pair / timeframe combinations." in captured.out - assert "\n| Pair | Timeframe |\n" in captured.out + assert "\n| Pair | Timeframe | Type |\n" in captured.out assert "UNITTEST/BTC" not in captured.out - assert "\n| XRP/ETH | 1m, 5m |\n" in captured.out + assert "\n| XRP/ETH | 1m, 5m | |\n" in captured.out @pytest.mark.usefixtures("init_persistence") From e2f98a8dabc111d123d3fce352e24983ad088dd2 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Sun, 21 Nov 2021 00:21:10 -0600 Subject: [PATCH 12/64] replaced candle_type: Optional[str] = '' with candle_type: str = '' --- freqtrade/data/converter.py | 2 +- freqtrade/data/dataprovider.py | 15 +++++++++++++-- freqtrade/data/history/hdf5datahandler.py | 12 ++++++------ freqtrade/data/history/history_utils.py | 18 +++++++++++++----- freqtrade/data/history/idatahandler.py | 12 ++++++------ freqtrade/data/history/jsondatahandler.py | 12 ++++++------ freqtrade/exchange/binance.py | 14 +++++--------- freqtrade/exchange/exchange.py | 21 +++++++++++++-------- 8 files changed, 63 insertions(+), 43 deletions(-) diff --git a/freqtrade/data/converter.py b/freqtrade/data/converter.py index baacbb948..ff1be188a 100644 --- a/freqtrade/data/converter.py +++ b/freqtrade/data/converter.py @@ -266,7 +266,7 @@ def convert_ohlcv_format( convert_from: str, convert_to: str, erase: bool, - candle_type: Optional[str] = "" + candle_type: str = '' ): """ Convert OHLCV from one format to another diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index b197c159f..e091c5e2f 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -99,7 +99,12 @@ class DataProvider: logger.warning(f"No data found for ({pair}, {timeframe}).") return data - def get_analyzed_dataframe(self, pair: str, timeframe: str) -> Tuple[DataFrame, datetime]: + def get_analyzed_dataframe( + self, + pair: str, + timeframe: str, + candle_type: str = '' + ) -> Tuple[DataFrame, datetime]: """ Retrieve the analyzed dataframe. Returns the full dataframe in trade mode (live / dry), and the last 1000 candles (up to the time evaluated at this moment) in all other modes. @@ -177,7 +182,13 @@ class DataProvider: raise OperationalException(NO_EXCHANGE_EXCEPTION) return list(self._exchange._klines.keys()) - def ohlcv(self, pair: str, timeframe: str = None, copy: bool = True) -> DataFrame: + def ohlcv( + self, + pair: str, + timeframe: str = None, + copy: bool = True, + candle_type: str = '' + ) -> DataFrame: """ Get candle (OHLCV) data for the given pair as DataFrame Please use the `available_pairs` method to verify which pairs are currently cached. diff --git a/freqtrade/data/history/hdf5datahandler.py b/freqtrade/data/history/hdf5datahandler.py index a59d7255d..204229d2b 100644 --- a/freqtrade/data/history/hdf5datahandler.py +++ b/freqtrade/data/history/hdf5datahandler.py @@ -42,7 +42,7 @@ class HDF5DataHandler(IDataHandler): cls, datadir: Path, timeframe: str, - candle_type: Optional[str] = "" + candle_type: str = '' ) -> List[str]: """ Returns a list of all pairs with ohlcv data available in this datadir @@ -67,7 +67,7 @@ class HDF5DataHandler(IDataHandler): pair: str, timeframe: str, data: pd.DataFrame, - candle_type: Optional[str] = "" + candle_type: str = '' ) -> None: """ Store data in hdf5 file. @@ -88,7 +88,7 @@ class HDF5DataHandler(IDataHandler): def _ohlcv_load(self, pair: str, timeframe: str, timerange: Optional[TimeRange] = None, - candle_type: Optional[str] = "") -> pd.DataFrame: + candle_type: str = '') -> pd.DataFrame: """ Internal method used to load data for one pair from disk. Implements the loading and conversion to a Pandas dataframe. @@ -125,7 +125,7 @@ class HDF5DataHandler(IDataHandler): 'low': 'float', 'close': 'float', 'volume': 'float'}) return pairdata - def ohlcv_purge(self, pair: str, timeframe: str, candle_type: Optional[str] = "") -> bool: + def ohlcv_purge(self, pair: str, timeframe: str, candle_type: str = '') -> bool: """ Remove data for this pair :param pair: Delete data for this pair. @@ -143,7 +143,7 @@ class HDF5DataHandler(IDataHandler): pair: str, timeframe: str, data: pd.DataFrame, - candle_type: Optional[str] = "" + candle_type: str = '' ) -> None: """ Append data to existing data structures @@ -238,7 +238,7 @@ class HDF5DataHandler(IDataHandler): datadir: Path, pair: str, timeframe: str, - candle_type: Optional[str] = "" + candle_type: str = '' ) -> Path: pair_s = misc.pair_to_filename(pair) if candle_type: diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index de58c8d0f..c4476ad8d 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -29,6 +29,7 @@ def load_pair_history(pair: str, startup_candles: int = 0, data_format: str = None, data_handler: IDataHandler = None, + candle_type: str = '' ) -> DataFrame: """ Load cached ohlcv history for the given pair. @@ -64,6 +65,7 @@ def load_data(datadir: Path, startup_candles: int = 0, fail_without_data: bool = False, data_format: str = 'json', + candle_type: str = '' ) -> Dict[str, DataFrame]: """ Load ohlcv history data for a list of pairs. @@ -105,6 +107,7 @@ def refresh_data(datadir: Path, exchange: Exchange, data_format: str = None, timerange: Optional[TimeRange] = None, + candle_type: str = '' ) -> None: """ Refresh ohlcv history data for a list of pairs. @@ -124,8 +127,13 @@ def refresh_data(datadir: Path, timerange=timerange, exchange=exchange, data_handler=data_handler) -def _load_cached_data_for_updating(pair: str, timeframe: str, timerange: Optional[TimeRange], - data_handler: IDataHandler) -> Tuple[DataFrame, Optional[int]]: +def _load_cached_data_for_updating( + pair: str, + timeframe: str, + timerange: Optional[TimeRange], + data_handler: IDataHandler, + candle_type: str = '' +) -> Tuple[DataFrame, Optional[int]]: """ Load cached data to download more data. If timerange is passed in, checks whether data from an before the stored data will be @@ -162,7 +170,7 @@ def _download_pair_history(pair: str, *, new_pairs_days: int = 30, data_handler: IDataHandler = None, timerange: Optional[TimeRange] = None, - candle_type: Optional[str] = "") -> bool: + candle_type: str = '') -> bool: """ Download latest candles from the exchange for the pair and timeframe passed in parameters The data is downloaded starting from the last correct data that @@ -232,7 +240,7 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes datadir: Path, timerange: Optional[TimeRange] = None, new_pairs_days: int = 30, erase: bool = False, data_format: str = None, - candle_type: Optional[str] = "") -> List[str]: + candle_type: str = '') -> List[str]: """ Refresh stored ohlcv data for backtesting and hyperopt operations. Used by freqtrade download-data subcommand. @@ -365,7 +373,7 @@ def convert_trades_to_ohlcv( erase: bool = False, data_format_ohlcv: str = 'json', data_format_trades: str = 'jsongz', - candle_type: Optional[str] = "" + candle_type: str = '' ) -> None: """ Convert stored trades data to ohlcv data diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index cfbeb5d33..dba9ff4bd 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -39,7 +39,7 @@ class IDataHandler(ABC): cls, datadir: Path, timeframe: str, - candle_type: Optional[str] = '' + candle_type: str = '' ) -> List[str]: """ Returns a list of all pairs with ohlcv data available in this datadir @@ -55,7 +55,7 @@ class IDataHandler(ABC): pair: str, timeframe: str, data: DataFrame, - candle_type: Optional[str] = '' + candle_type: str = '' ) -> None: """ Store ohlcv data. @@ -68,7 +68,7 @@ class IDataHandler(ABC): @abstractmethod def _ohlcv_load(self, pair: str, timeframe: str, timerange: Optional[TimeRange] = None, - candle_type: Optional[str] = '' + candle_type: str = '' ) -> DataFrame: """ Internal method used to load data for one pair from disk. @@ -83,7 +83,7 @@ class IDataHandler(ABC): """ @abstractmethod - def ohlcv_purge(self, pair: str, timeframe: str, candle_type: Optional[str] = '') -> bool: + def ohlcv_purge(self, pair: str, timeframe: str, candle_type: str = '') -> bool: """ Remove data for this pair :param pair: Delete data for this pair. @@ -97,7 +97,7 @@ class IDataHandler(ABC): pair: str, timeframe: str, data: DataFrame, - candle_type: Optional[str] = '' + candle_type: str = '' ) -> None: """ Append data to existing data structures @@ -165,7 +165,7 @@ class IDataHandler(ABC): drop_incomplete: bool = True, startup_candles: int = 0, warn_no_data: bool = True, - candle_type: Optional[str] = '' + candle_type: str = '' ) -> DataFrame: """ Load cached candle (OHLCV) data for the given pair. diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index 57b21f894..1c430f542 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -42,7 +42,7 @@ class JsonDataHandler(IDataHandler): cls, datadir: Path, timeframe: str, - candle_type: Optional[str] = "" + candle_type: str = '' ) -> List[str]: """ Returns a list of all pairs with ohlcv data available in this datadir @@ -66,7 +66,7 @@ class JsonDataHandler(IDataHandler): pair: str, timeframe: str, data: DataFrame, - candle_type: Optional[str] = "" + candle_type: str = '' ) -> None: """ Store data in json format "values". @@ -94,7 +94,7 @@ class JsonDataHandler(IDataHandler): def _ohlcv_load(self, pair: str, timeframe: str, timerange: Optional[TimeRange] = None, - candle_type: Optional[str] = "" + candle_type: str = '' ) -> DataFrame: """ Internal method used to load data for one pair from disk. @@ -124,7 +124,7 @@ class JsonDataHandler(IDataHandler): infer_datetime_format=True) return pairdata - def ohlcv_purge(self, pair: str, timeframe: str, candle_type: Optional[str] = "") -> bool: + def ohlcv_purge(self, pair: str, timeframe: str, candle_type: str = '') -> bool: """ Remove data for this pair :param pair: Delete data for this pair. @@ -142,7 +142,7 @@ class JsonDataHandler(IDataHandler): pair: str, timeframe: str, data: DataFrame, - candle_type: Optional[str] = "" + candle_type: str = '' ) -> None: """ Append data to existing data structures @@ -222,7 +222,7 @@ class JsonDataHandler(IDataHandler): datadir: Path, pair: str, timeframe: str, - candle_type: Optional[str] = "" + candle_type: str = '' ) -> Path: pair_s = misc.pair_to_filename(pair) if candle_type: diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 9d9146a56..e94a97833 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -200,15 +200,11 @@ class Binance(Exchange): except ccxt.BaseError as e: raise OperationalException(e) from e - async def _async_get_historic_ohlcv( - self, - pair: str, - timeframe: str, - since_ms: int, - is_new_pair: bool, - raise_: bool = False, - candle_type: Optional[str] = "" - ) -> Tuple[str, str, List]: + async def _async_get_historic_ohlcv(self, pair: str, timeframe: str, + since_ms: int, is_new_pair: bool = False, + raise_: bool = False, + candle_type: str = '' + ) -> Tuple[str, str, List]: """ Overwrite to introduce "fast new pair" functionality by detecting the pair's listing date Does not work for other exchanges, which don't return the earliest data when called with "0" diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 8511df7ee..ef82f9f8b 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1311,7 +1311,7 @@ class Exchange: def get_historic_ohlcv(self, pair: str, timeframe: str, since_ms: int, is_new_pair: bool = False, - candle_type: Optional[str] = "") -> List: + candle_type: str = '') -> List: """ Get candle history using asyncio and returns the list of candles. Handles all async work for this. @@ -1329,7 +1329,7 @@ class Exchange: return data def get_historic_ohlcv_as_df(self, pair: str, timeframe: str, - since_ms: int, candle_type: Optional[str] = "") -> DataFrame: + since_ms: int, candle_type: str = '') -> DataFrame: """ Minimal wrapper around get_historic_ohlcv - converting the result into a dataframe :param pair: Pair to download @@ -1344,7 +1344,7 @@ class Exchange: async def _async_get_historic_ohlcv(self, pair: str, timeframe: str, since_ms: int, is_new_pair: bool, raise_: bool = False, - candle_type: Optional[str] = "" + candle_type: str = '' ) -> Tuple[str, str, List]: """ Download historic ohlcv @@ -1383,8 +1383,8 @@ class Exchange: def refresh_latest_ohlcv(self, pair_list: ListPairsWithTimeframes, *, since_ms: Optional[int] = None, cache: bool = True, - candle_type: Optional[str] = "" - ) -> Dict[Tuple[str, str], DataFrame]: + candle_type: str = '' + ) -> Dict[Tuple[str, str, str], DataFrame]: """ Refresh in-memory OHLCV asynchronously and set `_klines` with the result Loops asynchronously over pair_list and downloads all pairs async (semi-parallel). @@ -1450,7 +1450,12 @@ class Exchange: return results_df - def _now_is_time_to_refresh(self, pair: str, timeframe: str) -> bool: + def _now_is_time_to_refresh( + self, + pair: str, + timeframe: str, + candle_type: str = '' + ) -> bool: # Timeframe in seconds interval_in_sec = timeframe_to_seconds(timeframe) @@ -1463,8 +1468,8 @@ class Exchange: pair: str, timeframe: str, since_ms: Optional[int] = None, - candle_type: Optional[str] = "", - ) -> Tuple[str, str, List]: + candle_type: str = '', + ) -> Tuple[str, str, str, List]: """ Asynchronously get candle history data using fetch_ohlcv :param candle_type: From 920151934a734cce316e6f89f22ff4c63117122e Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Sun, 21 Nov 2021 01:43:05 -0600 Subject: [PATCH 13/64] Added candle_type to a lot of methods, wrote some tests --- freqtrade/data/converter.py | 2 +- freqtrade/data/dataprovider.py | 44 +++++-- freqtrade/data/history/history_utils.py | 13 +- freqtrade/edge/edge_positioning.py | 4 +- freqtrade/exchange/binance.py | 2 +- freqtrade/exchange/exchange.py | 60 +++++---- freqtrade/plugins/pairlist/AgeFilter.py | 4 +- .../plugins/pairlist/VolatilityFilter.py | 4 +- freqtrade/plugins/pairlist/VolumePairList.py | 11 +- .../plugins/pairlist/rangestabilityfilter.py | 4 +- freqtrade/plugins/pairlistmanager.py | 2 +- freqtrade/strategy/informative_decorator.py | 1 + freqtrade/strategy/interface.py | 8 +- tests/commands/test_commands.py | 2 +- tests/conftest.py | 4 +- tests/data/test_converter.py | 82 ++++++------ tests/data/test_dataprovider.py | 69 +++++++---- tests/data/test_history.py | 117 +++++++++++++----- tests/exchange/test_binance.py | 12 +- tests/exchange/test_exchange.py | 70 +++++++---- tests/optimize/test_backtesting.py | 7 +- tests/plugins/test_pairlist.py | 54 ++++---- tests/rpc/test_rpc_apiserver.py | 21 ++-- .../strats/informative_decorator_strategy.py | 4 +- tests/strategy/test_strategy_helpers.py | 37 +++--- tests/test_freqtradebot.py | 8 +- tests/testdata/XRP_USDT-1h.json | 102 +++++++++++++++ 27 files changed, 495 insertions(+), 253 deletions(-) create mode 100644 tests/testdata/XRP_USDT-1h.json diff --git a/freqtrade/data/converter.py b/freqtrade/data/converter.py index ff1be188a..dfd4a9f68 100644 --- a/freqtrade/data/converter.py +++ b/freqtrade/data/converter.py @@ -5,7 +5,7 @@ import itertools import logging from datetime import datetime, timezone from operator import itemgetter -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List import pandas as pd from pandas import DataFrame, to_datetime diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index e091c5e2f..3c8ee4b24 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -41,7 +41,13 @@ class DataProvider: """ self.__slice_index = limit_index - def _set_cached_df(self, pair: str, timeframe: str, dataframe: DataFrame) -> None: + def _set_cached_df( + self, + pair: str, + timeframe: str, + dataframe: DataFrame, + candle_type: str = '' + ) -> None: """ Store cached Dataframe. Using private method as this should never be used by a user @@ -50,7 +56,8 @@ class DataProvider: :param timeframe: Timeframe to get data for :param dataframe: analyzed dataframe """ - self.__cached_pairs[(pair, timeframe)] = (dataframe, datetime.now(timezone.utc)) + self.__cached_pairs[(pair, timeframe, candle_type)] = ( + dataframe, datetime.now(timezone.utc)) def add_pairlisthandler(self, pairlists) -> None: """ @@ -58,13 +65,18 @@ class DataProvider: """ self._pairlists = pairlists - def historic_ohlcv(self, pair: str, timeframe: str = None) -> DataFrame: + def historic_ohlcv( + self, + pair: str, + timeframe: str = None, + candle_type: str = '' + ) -> DataFrame: """ Get stored historical candle (OHLCV) data :param pair: pair to get the data for :param timeframe: timeframe to get data for """ - saved_pair = (pair, str(timeframe)) + saved_pair = (pair, str(timeframe), candle_type) if saved_pair not in self.__cached_pairs_backtesting: timerange = TimeRange.parse_timerange(None if self._config.get( 'timerange') is None else str(self._config.get('timerange'))) @@ -77,11 +89,17 @@ class DataProvider: timeframe=timeframe or self._config['timeframe'], datadir=self._config['datadir'], timerange=timerange, - data_format=self._config.get('dataformat_ohlcv', 'json') + data_format=self._config.get('dataformat_ohlcv', 'json'), + candle_type=candle_type ) return self.__cached_pairs_backtesting[saved_pair].copy() - def get_pair_dataframe(self, pair: str, timeframe: str = None) -> DataFrame: + def get_pair_dataframe( + self, + pair: str, + timeframe: str = None, + candle_type: str = '' + ) -> DataFrame: """ Return pair candle (OHLCV) data, either live or cached historical -- depending on the runmode. @@ -91,12 +109,12 @@ class DataProvider: """ if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE): # Get live OHLCV data. - data = self.ohlcv(pair=pair, timeframe=timeframe) + data = self.ohlcv(pair=pair, timeframe=timeframe, candle_type=candle_type) else: # Get historical OHLCV data (cached on disk). - data = self.historic_ohlcv(pair=pair, timeframe=timeframe) + data = self.historic_ohlcv(pair=pair, timeframe=timeframe, candle_type=candle_type) if len(data) == 0: - logger.warning(f"No data found for ({pair}, {timeframe}).") + logger.warning(f"No data found for ({pair}, {timeframe}, {candle_type}).") return data def get_analyzed_dataframe( @@ -114,7 +132,7 @@ class DataProvider: combination. Returns empty dataframe and Epoch 0 (1970-01-01) if no dataframe was cached. """ - pair_key = (pair, timeframe) + pair_key = (pair, timeframe, candle_type) if pair_key in self.__cached_pairs: if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE): df, date = self.__cached_pairs[pair_key] @@ -200,8 +218,10 @@ class DataProvider: if self._exchange is None: raise OperationalException(NO_EXCHANGE_EXCEPTION) if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE): - return self._exchange.klines((pair, timeframe or self._config['timeframe']), - copy=copy) + return self._exchange.klines( + (pair, timeframe or self._config['timeframe'], candle_type), + copy=copy + ) else: return DataFrame() diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index c4476ad8d..cfff74d93 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -54,6 +54,7 @@ def load_pair_history(pair: str, fill_missing=fill_up_missing, drop_incomplete=drop_incomplete, startup_candles=startup_candles, + candle_type=candle_type ) @@ -91,7 +92,8 @@ def load_data(datadir: Path, datadir=datadir, timerange=timerange, fill_up_missing=fill_up_missing, startup_candles=startup_candles, - data_handler=data_handler + data_handler=data_handler, + candle_type=candle_type ) if not hist.empty: result[pair] = hist @@ -124,7 +126,8 @@ def refresh_data(datadir: Path, process = f'{idx}/{len(pairs)}' _download_pair_history(pair=pair, process=process, timeframe=timeframe, datadir=datadir, - timerange=timerange, exchange=exchange, data_handler=data_handler) + timerange=timerange, exchange=exchange, data_handler=data_handler, + candle_type=candle_type) def _load_cached_data_for_updating( @@ -150,7 +153,8 @@ def _load_cached_data_for_updating( # Intentionally don't pass timerange in - since we need to load the full dataset. data = data_handler.ohlcv_load(pair, timeframe=timeframe, timerange=None, fill_missing=False, - drop_incomplete=True, warn_no_data=False) + drop_incomplete=True, warn_no_data=False, + candle_type=candle_type) if not data.empty: if start and start < data.iloc[0]['date']: # Earlier data than existing data requested, redownload all @@ -194,7 +198,8 @@ def _download_pair_history(pair: str, *, # data, since_ms = _load_cached_data_for_updating_old(datadir, pair, timeframe, timerange) data, since_ms = _load_cached_data_for_updating(pair, timeframe, timerange, - data_handler=data_handler) + data_handler=data_handler, + candle_type=candle_type) logger.debug("Current Start: %s", f"{data.iloc[0]['date']:%Y-%m-%d %H:%M:%S}" if not data.empty else 'None') diff --git a/freqtrade/edge/edge_positioning.py b/freqtrade/edge/edge_positioning.py index e08b3df2f..b2f4534f1 100644 --- a/freqtrade/edge/edge_positioning.py +++ b/freqtrade/edge/edge_positioning.py @@ -119,8 +119,8 @@ class Edge: ) # Download informative pairs too res = defaultdict(list) - for p, t in self.strategy.gather_informative_pairs(): - res[t].append(p) + for pair, timeframe, _ in self.strategy.gather_informative_pairs(): + res[timeframe].append(pair) for timeframe, inf_pairs in res.items(): timerange_startup = deepcopy(self._timerange) timerange_startup.subtract_start(timeframe_to_seconds( diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index e94a97833..0c891924f 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -204,7 +204,7 @@ class Binance(Exchange): since_ms: int, is_new_pair: bool = False, raise_: bool = False, candle_type: str = '' - ) -> Tuple[str, str, List]: + ) -> Tuple[str, str, str, List]: """ Overwrite to introduce "fast new pair" functionality by detecting the pair's listing date Does not work for other exchanges, which don't return the earliest data when called with "0" diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index ef82f9f8b..7201c025e 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -92,7 +92,7 @@ class Exchange: self._config.update(config) # Holds last candle refreshed time of each pair - self._pairs_last_refresh_time: Dict[Tuple[str, str], int] = {} + self._pairs_last_refresh_time: Dict[Tuple[str, str, str], int] = {} # Timestamp of last markets refresh self._last_markets_refresh: int = 0 @@ -105,7 +105,7 @@ class Exchange: self._buy_rate_cache: TTLCache = TTLCache(maxsize=100, ttl=1800) # Holds candles - self._klines: Dict[Tuple[str, str], DataFrame] = {} + self._klines: Dict[Tuple[str, str, str], DataFrame] = {} # Holds all open sell orders for dry_run self._dry_run_open_orders: Dict[str, Any] = {} @@ -359,7 +359,7 @@ class Exchange: or (self.trading_mode == TradingMode.FUTURES and self.market_is_future(market)) ) - def klines(self, pair_interval: Tuple[str, str], copy: bool = True) -> DataFrame: + def klines(self, pair_interval: Tuple[str, str, str], copy: bool = True) -> DataFrame: if pair_interval in self._klines: return self._klines[pair_interval].copy() if copy else self._klines[pair_interval] else: @@ -1321,7 +1321,8 @@ class Exchange: :param since_ms: Timestamp in milliseconds to get history from :return: List with candle (OHLCV) data """ - pair, timeframe, data = asyncio.get_event_loop().run_until_complete( + data: List + pair, timeframe, candle_type, data = asyncio.get_event_loop().run_until_complete( self._async_get_historic_ohlcv(pair=pair, timeframe=timeframe, since_ms=since_ms, is_new_pair=is_new_pair, candle_type=candle_type)) @@ -1337,15 +1338,15 @@ class Exchange: :param since_ms: Timestamp in milliseconds to get history from :return: OHLCV DataFrame """ - ticks = self.get_historic_ohlcv(pair, timeframe, since_ms=since_ms) + ticks = self.get_historic_ohlcv(pair, timeframe, since_ms=since_ms, candle_type=candle_type) return ohlcv_to_dataframe(ticks, timeframe, pair=pair, fill_missing=True, drop_incomplete=self._ohlcv_partial_candle) async def _async_get_historic_ohlcv(self, pair: str, timeframe: str, - since_ms: int, is_new_pair: bool, + since_ms: int, is_new_pair: bool = False, raise_: bool = False, candle_type: str = '' - ) -> Tuple[str, str, List]: + ) -> Tuple[str, str, str, List]: """ Download historic ohlcv :param is_new_pair: used by binance subclass to allow "fast" new pair downloading @@ -1374,12 +1375,12 @@ class Exchange: continue else: # Deconstruct tuple if it's not an exception - p, _, new_data = res - if p == pair: + p, _, c, new_data = res + if p == pair and c == candle_type: data.extend(new_data) # Sort data again after extending the result - above calls return in "async order" data = sorted(data, key=lambda x: x[0]) - return pair, timeframe, data + return pair, timeframe, candle_type, data def refresh_latest_ohlcv(self, pair_list: ListPairsWithTimeframes, *, since_ms: Optional[int] = None, cache: bool = True, @@ -1399,8 +1400,8 @@ class Exchange: input_coroutines = [] cached_pairs = [] # Gather coroutines to run - for pair, timeframe in set(pair_list): - if ((pair, timeframe) not in self._klines + for pair, timeframe, candle_type in set(pair_list): + if ((pair, timeframe, candle_type) not in self._klines or self._now_is_time_to_refresh(pair, timeframe)): if not since_ms and self.required_candle_call_count > 1: # Multiple calls for one pair - to get more history @@ -1411,17 +1412,17 @@ class Exchange: if since_ms: input_coroutines.append(self._async_get_historic_ohlcv( - pair, timeframe, since_ms=since_ms, raise_=True)) + pair, timeframe, since_ms=since_ms, raise_=True, candle_type=candle_type)) else: # One call ... "regular" refresh input_coroutines.append(self._async_get_candle_history( - pair, timeframe, since_ms=since_ms, candle_type=candle_type,)) + pair, timeframe, since_ms=since_ms, candle_type=candle_type)) else: logger.debug( "Using cached candle (OHLCV) data for pair %s, timeframe %s ...", - pair, timeframe + pair, timeframe, candle_type ) - cached_pairs.append((pair, timeframe)) + cached_pairs.append((pair, timeframe, candle_type)) results = asyncio.get_event_loop().run_until_complete( asyncio.gather(*input_coroutines, return_exceptions=True)) @@ -1433,20 +1434,23 @@ class Exchange: logger.warning("Async code raised an exception: %s", res.__class__.__name__) continue # Deconstruct tuple (has 3 elements) - pair, timeframe, ticks = res + pair, timeframe, c_type, ticks = res # keeping last candle time as last refreshed time of the pair if ticks: - self._pairs_last_refresh_time[(pair, timeframe)] = ticks[-1][0] // 1000 + self._pairs_last_refresh_time[(pair, timeframe, c_type)] = ticks[-1][0] // 1000 # keeping parsed dataframe in cache ohlcv_df = ohlcv_to_dataframe( ticks, timeframe, pair=pair, fill_missing=True, drop_incomplete=self._ohlcv_partial_candle) - results_df[(pair, timeframe)] = ohlcv_df + results_df[(pair, timeframe, c_type)] = ohlcv_df if cache: - self._klines[(pair, timeframe)] = ohlcv_df + self._klines[(pair, timeframe, c_type)] = ohlcv_df # Return cached klines - for pair, timeframe in cached_pairs: - results_df[(pair, timeframe)] = self.klines((pair, timeframe), copy=False) + for pair, timeframe, c_type in cached_pairs: + results_df[(pair, timeframe, c_type)] = self.klines( + (pair, timeframe, c_type), + copy=False + ) return results_df @@ -1459,8 +1463,12 @@ class Exchange: # Timeframe in seconds interval_in_sec = timeframe_to_seconds(timeframe) - return not ((self._pairs_last_refresh_time.get((pair, timeframe), 0) - + interval_in_sec) >= arrow.utcnow().int_timestamp) + return not ( + (self._pairs_last_refresh_time.get( + (pair, timeframe, candle_type), + 0 + ) + interval_in_sec) >= arrow.utcnow().int_timestamp + ) @retrier_async async def _async_get_candle_history( @@ -1501,9 +1509,9 @@ class Exchange: data = sorted(data, key=lambda x: x[0]) except IndexError: logger.exception("Error loading %s. Result was %s.", pair, data) - return pair, timeframe, [] + return pair, timeframe, candle_type, [] logger.debug("Done fetching pair %s, interval %s ...", pair, timeframe) - return pair, timeframe, data + return pair, timeframe, candle_type, data except ccxt.NotSupported as e: raise OperationalException( diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py index 5627d82ce..f2e9aa4d3 100644 --- a/freqtrade/plugins/pairlist/AgeFilter.py +++ b/freqtrade/plugins/pairlist/AgeFilter.py @@ -72,7 +72,7 @@ class AgeFilter(IPairList): :return: new allowlist """ needed_pairs = [ - (p, '1d') for p in pairlist + (p, '1d', '') for p in pairlist if p not in self._symbolsChecked and p not in self._symbolsCheckFailed] if not needed_pairs: # Remove pairs that have been removed before @@ -88,7 +88,7 @@ class AgeFilter(IPairList): candles = self._exchange.refresh_latest_ohlcv(needed_pairs, since_ms=since_ms, cache=False) if self._enabled: for p in deepcopy(pairlist): - daily_candles = candles[(p, '1d')] if (p, '1d') in candles else None + daily_candles = candles[(p, '1d', '')] if (p, '1d', '') in candles else None if not self._validate_pair_loc(p, daily_candles): pairlist.remove(p) self.log_once(f"Validated {len(pairlist)} pairs.", logger.info) diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py index 9383e5d06..c06cd0897 100644 --- a/freqtrade/plugins/pairlist/VolatilityFilter.py +++ b/freqtrade/plugins/pairlist/VolatilityFilter.py @@ -67,7 +67,7 @@ class VolatilityFilter(IPairList): :param tickers: Tickers (from exchange.get_tickers()). May be cached. :return: new allowlist """ - needed_pairs = [(p, '1d') for p in pairlist if p not in self._pair_cache] + needed_pairs = [(p, '1d', '') for p in pairlist if p not in self._pair_cache] since_ms = (arrow.utcnow() .floor('day') @@ -81,7 +81,7 @@ class VolatilityFilter(IPairList): if self._enabled: for p in deepcopy(pairlist): - daily_candles = candles[(p, '1d')] if (p, '1d') in candles else None + daily_candles = candles[(p, '1d', '')] if (p, '1d', '') in candles else None if not self._validate_pair_loc(p, daily_candles): pairlist.remove(p) return pairlist diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py index 0ffc8a8c8..0493b67c9 100644 --- a/freqtrade/plugins/pairlist/VolumePairList.py +++ b/freqtrade/plugins/pairlist/VolumePairList.py @@ -160,10 +160,9 @@ class VolumePairList(IPairList): f"{self._lookback_timeframe}, starting from {format_ms_time(since_ms)} " f"till {format_ms_time(to_ms)}", logger.info) needed_pairs = [ - (p, self._lookback_timeframe) for p in - [ - s['symbol'] for s in filtered_tickers - ] if p not in self._pair_cache + (p, self._lookback_timeframe, '') for p in + [s['symbol'] for s in filtered_tickers] + if p not in self._pair_cache ] # Get all candles @@ -174,8 +173,8 @@ class VolumePairList(IPairList): ) for i, p in enumerate(filtered_tickers): pair_candles = candles[ - (p['symbol'], self._lookback_timeframe) - ] if (p['symbol'], self._lookback_timeframe) in candles else None + (p['symbol'], self._lookback_timeframe, '') + ] if (p['symbol'], self._lookback_timeframe, '') in candles else None # in case of candle data calculate typical price and quoteVolume for candle if pair_candles is not None and not pair_candles.empty: pair_candles['typical_price'] = (pair_candles['high'] + pair_candles['low'] diff --git a/freqtrade/plugins/pairlist/rangestabilityfilter.py b/freqtrade/plugins/pairlist/rangestabilityfilter.py index 3e5a002ff..048736ae0 100644 --- a/freqtrade/plugins/pairlist/rangestabilityfilter.py +++ b/freqtrade/plugins/pairlist/rangestabilityfilter.py @@ -65,7 +65,7 @@ class RangeStabilityFilter(IPairList): :param tickers: Tickers (from exchange.get_tickers()). May be cached. :return: new allowlist """ - needed_pairs = [(p, '1d') for p in pairlist if p not in self._pair_cache] + needed_pairs = [(p, '1d', '') for p in pairlist if p not in self._pair_cache] since_ms = (arrow.utcnow() .floor('day') @@ -79,7 +79,7 @@ class RangeStabilityFilter(IPairList): if self._enabled: for p in deepcopy(pairlist): - daily_candles = candles[(p, '1d')] if (p, '1d') in candles else None + daily_candles = candles[(p, '1d', '')] if (p, '1d', '') in candles else None if not self._validate_pair_loc(p, daily_candles): pairlist.remove(p) return pairlist diff --git a/freqtrade/plugins/pairlistmanager.py b/freqtrade/plugins/pairlistmanager.py index 93b5e90e2..5ea4c2386 100644 --- a/freqtrade/plugins/pairlistmanager.py +++ b/freqtrade/plugins/pairlistmanager.py @@ -138,4 +138,4 @@ class PairListManager(): """ Create list of pair tuples with (pair, timeframe) """ - return [(pair, timeframe or self._config['timeframe']) for pair in pairs] + return [(pair, timeframe or self._config['timeframe'], '') for pair in pairs] diff --git a/freqtrade/strategy/informative_decorator.py b/freqtrade/strategy/informative_decorator.py index 4c5f21108..31d761519 100644 --- a/freqtrade/strategy/informative_decorator.py +++ b/freqtrade/strategy/informative_decorator.py @@ -14,6 +14,7 @@ class InformativeData(NamedTuple): timeframe: str fmt: Union[str, Callable[[Any], str], None] ffill: bool + candle_type: str = '' def informative(timeframe: str, asset: str = '', diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 36bf09f5f..8a4e50343 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -424,14 +424,18 @@ class IStrategy(ABC, HyperStrategyMixin): informative_pairs = self.informative_pairs() for inf_data, _ in self._ft_informative: if inf_data.asset: - pair_tf = (_format_pair_name(self.config, inf_data.asset), inf_data.timeframe) + pair_tf = ( + _format_pair_name(self.config, inf_data.asset), + inf_data.timeframe, + inf_data.candle_type + ) informative_pairs.append(pair_tf) else: if not self.dp: raise OperationalException('@informative decorator with unspecified asset ' 'requires DataProvider instance.') for pair in self.dp.current_whitelist(): - informative_pairs.append((pair, inf_data.timeframe)) + informative_pairs.append((pair, inf_data.timeframe, inf_data.candle_type)) return list(set(informative_pairs)) def get_strategy_name(self) -> str: diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 72d2c6de4..384254b74 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -1326,7 +1326,7 @@ def test_start_list_data(testdatadir, capsys): pargs['config'] = None start_list_data(pargs) captured = capsys.readouterr() - assert "Found 19 pair / timeframe combinations." in captured.out + assert "Found 20 pair / timeframe combinations." in captured.out assert "\n| Pair | Timeframe | Type |\n" in captured.out assert "\n| UNITTEST/BTC | 1m, 5m, 8m, 30m | |\n" in captured.out assert "\n| UNITTEST/USDT | 1h | mark |\n" in captured.out diff --git a/tests/conftest.py b/tests/conftest.py index e184903d1..a567b55e9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2388,7 +2388,7 @@ def market_buy_order_usdt_doublefee(market_buy_order_usdt): 'amount': 25.0, 'cost': 50.25, 'fee': {'cost': 0.00025125, 'currency': 'BNB'} - }, { + }, { 'timestamp': None, 'datetime': None, 'symbol': 'ETH/USDT', @@ -2401,7 +2401,7 @@ def market_buy_order_usdt_doublefee(market_buy_order_usdt): 'amount': 5, 'cost': 10, 'fee': {'cost': 0.0100306, 'currency': 'USDT'} - }] + }] return order diff --git a/tests/data/test_converter.py b/tests/data/test_converter.py index 6c95a9f18..39bb1b15e 100644 --- a/tests/data/test_converter.py +++ b/tests/data/test_converter.py @@ -75,7 +75,8 @@ def test_ohlcv_fill_up_missing_data(testdatadir, caplog): def test_ohlcv_fill_up_missing_data2(caplog): timeframe = '5m' - ticks = [[ + ticks = [ + [ 1511686200000, # 8:50:00 8.794e-05, # open 8.948e-05, # high @@ -106,7 +107,7 @@ def test_ohlcv_fill_up_missing_data2(caplog): 8.895e-05, 8.817e-05, 123551 - ] + ] ] # Generate test-data without filling missing @@ -134,13 +135,13 @@ def test_ohlcv_fill_up_missing_data2(caplog): def test_ohlcv_drop_incomplete(caplog): timeframe = '1d' ticks = [[ - 1559750400000, # 2019-06-04 - 8.794e-05, # open - 8.948e-05, # high - 8.794e-05, # low - 8.88e-05, # close - 2255, # volume (in quote currency) - ], + 1559750400000, # 2019-06-04 + 8.794e-05, # open + 8.948e-05, # high + 8.794e-05, # low + 8.88e-05, # close + 2255, # volume (in quote currency) + ], [ 1559836800000, # 2019-06-05 8.88e-05, @@ -148,7 +149,7 @@ def test_ohlcv_drop_incomplete(caplog): 8.88e-05, 8.893e-05, 9911, - ], + ], [ 1559923200000, # 2019-06-06 8.891e-05, @@ -156,7 +157,7 @@ def test_ohlcv_drop_incomplete(caplog): 8.875e-05, 8.877e-05, 2251 - ], + ], [ 1560009600000, # 2019-06-07 8.877e-05, @@ -164,7 +165,7 @@ def test_ohlcv_drop_incomplete(caplog): 8.895e-05, 8.817e-05, 123551 - ] + ] ] caplog.set_level(logging.DEBUG) data = ohlcv_to_dataframe(ticks, timeframe, pair="UNITTEST/BTC", @@ -287,42 +288,45 @@ def test_convert_trades_format(default_conf, testdatadir, tmpdir): file['new'].unlink() -def test_convert_ohlcv_format(default_conf, testdatadir, tmpdir): +@pytest.mark.parametrize('file_base', [ + ('XRP_ETH-5m'), + ('XRP_ETH-1m'), + # ('XRP_USDT-1h-mark'), #TODO-lev: Create .gz file +]) +def test_convert_ohlcv_format(default_conf, testdatadir, tmpdir, file_base): tmpdir1 = Path(tmpdir) - file1_orig = testdatadir / "XRP_ETH-5m.json" - file1 = tmpdir1 / "XRP_ETH-5m.json" - file1_new = tmpdir1 / "XRP_ETH-5m.json.gz" - file2_orig = testdatadir / "XRP_ETH-1m.json" - file2 = tmpdir1 / "XRP_ETH-1m.json" - file2_new = tmpdir1 / "XRP_ETH-1m.json.gz" + file_orig = testdatadir / f"{file_base}.json" + file_temp = tmpdir1 / f"{file_base}.json" + file_new = tmpdir1 / f"{file_base}.json.gz" - copyfile(file1_orig, file1) - copyfile(file2_orig, file2) + copyfile(file_orig, file_temp) default_conf['datadir'] = tmpdir1 - default_conf['pairs'] = ['XRP_ETH'] - default_conf['timeframes'] = ['1m', '5m'] + default_conf['pairs'] = ['XRP_ETH', 'XRP_USDT'] + default_conf['timeframes'] = ['1m', '5m', '1h'] - assert not file1_new.exists() - assert not file2_new.exists() + assert not file_new.exists() - convert_ohlcv_format(default_conf, convert_from='json', - convert_to='jsongz', erase=False) + convert_ohlcv_format( + default_conf, + convert_from='json', + convert_to='jsongz', + erase=False + ) - assert file1_new.exists() - assert file2_new.exists() - assert file1.exists() - assert file2.exists() + assert file_new.exists() + assert file_temp.exists() # Remove original files - file1.unlink() - file2.unlink() + file_temp.unlink() # Convert back - convert_ohlcv_format(default_conf, convert_from='jsongz', - convert_to='json', erase=True) + convert_ohlcv_format( + default_conf, + convert_from='jsongz', + convert_to='json', + erase=True + ) - assert file1.exists() - assert file2.exists() - assert not file1_new.exists() - assert not file2_new.exists() + assert file_temp.exists() + assert not file_new.exists() diff --git a/tests/data/test_dataprovider.py b/tests/data/test_dataprovider.py index 0f42068c1..4f01f816f 100644 --- a/tests/data/test_dataprovider.py +++ b/tests/data/test_dataprovider.py @@ -11,34 +11,42 @@ from freqtrade.plugins.pairlistmanager import PairListManager from tests.conftest import get_patched_exchange -def test_ohlcv(mocker, default_conf, ohlcv_history): +@pytest.mark.parametrize('candle_type', [ + 'mark', + '', +]) +def test_ohlcv(mocker, default_conf, ohlcv_history, candle_type): default_conf["runmode"] = RunMode.DRY_RUN timeframe = default_conf["timeframe"] exchange = get_patched_exchange(mocker, default_conf) - exchange._klines[("XRP/BTC", timeframe)] = ohlcv_history - exchange._klines[("UNITTEST/BTC", timeframe)] = ohlcv_history + exchange._klines[("XRP/BTC", timeframe, candle_type)] = ohlcv_history + exchange._klines[("UNITTEST/BTC", timeframe, candle_type)] = ohlcv_history dp = DataProvider(default_conf, exchange) assert dp.runmode == RunMode.DRY_RUN - assert ohlcv_history.equals(dp.ohlcv("UNITTEST/BTC", timeframe)) - assert isinstance(dp.ohlcv("UNITTEST/BTC", timeframe), DataFrame) - assert dp.ohlcv("UNITTEST/BTC", timeframe) is not ohlcv_history - assert dp.ohlcv("UNITTEST/BTC", timeframe, copy=False) is ohlcv_history - assert not dp.ohlcv("UNITTEST/BTC", timeframe).empty - assert dp.ohlcv("NONESENSE/AAA", timeframe).empty + assert ohlcv_history.equals(dp.ohlcv("UNITTEST/BTC", timeframe, candle_type=candle_type)) + assert isinstance(dp.ohlcv("UNITTEST/BTC", timeframe, candle_type=candle_type), DataFrame) + assert dp.ohlcv("UNITTEST/BTC", timeframe, candle_type=candle_type) is not ohlcv_history + assert dp.ohlcv("UNITTEST/BTC", timeframe, copy=False, candle_type=candle_type) is ohlcv_history + assert not dp.ohlcv("UNITTEST/BTC", timeframe, candle_type=candle_type).empty + assert dp.ohlcv("NONESENSE/AAA", timeframe, candle_type=candle_type).empty # Test with and without parameter - assert dp.ohlcv("UNITTEST/BTC", timeframe).equals(dp.ohlcv("UNITTEST/BTC")) + assert dp.ohlcv( + "UNITTEST/BTC", + timeframe, + candle_type=candle_type + ).equals(dp.ohlcv("UNITTEST/BTC", candle_type=candle_type)) default_conf["runmode"] = RunMode.LIVE dp = DataProvider(default_conf, exchange) assert dp.runmode == RunMode.LIVE - assert isinstance(dp.ohlcv("UNITTEST/BTC", timeframe), DataFrame) + assert isinstance(dp.ohlcv("UNITTEST/BTC", timeframe, candle_type=candle_type), DataFrame) default_conf["runmode"] = RunMode.BACKTEST dp = DataProvider(default_conf, exchange) assert dp.runmode == RunMode.BACKTEST - assert dp.ohlcv("UNITTEST/BTC", timeframe).empty + assert dp.ohlcv("UNITTEST/BTC", timeframe, candle_type=candle_type).empty def test_historic_ohlcv(mocker, default_conf, ohlcv_history): @@ -77,37 +85,46 @@ def test_historic_ohlcv_dataformat(mocker, default_conf, ohlcv_history): jsonloadmock.assert_not_called() -def test_get_pair_dataframe(mocker, default_conf, ohlcv_history): +@pytest.mark.parametrize('candle_type', [ + 'mark', + '', +]) +def test_get_pair_dataframe(mocker, default_conf, ohlcv_history, candle_type): default_conf["runmode"] = RunMode.DRY_RUN timeframe = default_conf["timeframe"] exchange = get_patched_exchange(mocker, default_conf) - exchange._klines[("XRP/BTC", timeframe)] = ohlcv_history - exchange._klines[("UNITTEST/BTC", timeframe)] = ohlcv_history + exchange._klines[("XRP/BTC", timeframe, candle_type)] = ohlcv_history + exchange._klines[("UNITTEST/BTC", timeframe, candle_type)] = ohlcv_history dp = DataProvider(default_conf, exchange) assert dp.runmode == RunMode.DRY_RUN - assert ohlcv_history.equals(dp.get_pair_dataframe("UNITTEST/BTC", timeframe)) - assert isinstance(dp.get_pair_dataframe("UNITTEST/BTC", timeframe), DataFrame) - assert dp.get_pair_dataframe("UNITTEST/BTC", timeframe) is not ohlcv_history - assert not dp.get_pair_dataframe("UNITTEST/BTC", timeframe).empty - assert dp.get_pair_dataframe("NONESENSE/AAA", timeframe).empty + assert ohlcv_history.equals(dp.get_pair_dataframe( + "UNITTEST/BTC", timeframe, candle_type=candle_type)) + assert isinstance(dp.get_pair_dataframe( + "UNITTEST/BTC", timeframe, candle_type=candle_type), DataFrame) + assert dp.get_pair_dataframe("UNITTEST/BTC", timeframe, + candle_type=candle_type) is not ohlcv_history + assert not dp.get_pair_dataframe("UNITTEST/BTC", timeframe, candle_type=candle_type).empty + assert dp.get_pair_dataframe("NONESENSE/AAA", timeframe, candle_type=candle_type).empty # Test with and without parameter - assert dp.get_pair_dataframe("UNITTEST/BTC", timeframe)\ - .equals(dp.get_pair_dataframe("UNITTEST/BTC")) + assert dp.get_pair_dataframe("UNITTEST/BTC", timeframe, candle_type=candle_type)\ + .equals(dp.get_pair_dataframe("UNITTEST/BTC", candle_type=candle_type)) default_conf["runmode"] = RunMode.LIVE dp = DataProvider(default_conf, exchange) assert dp.runmode == RunMode.LIVE - assert isinstance(dp.get_pair_dataframe("UNITTEST/BTC", timeframe), DataFrame) - assert dp.get_pair_dataframe("NONESENSE/AAA", timeframe).empty + assert isinstance(dp.get_pair_dataframe( + "UNITTEST/BTC", timeframe, candle_type=candle_type), DataFrame) + assert dp.get_pair_dataframe("NONESENSE/AAA", timeframe, candle_type=candle_type).empty historymock = MagicMock(return_value=ohlcv_history) mocker.patch("freqtrade.data.dataprovider.load_pair_history", historymock) default_conf["runmode"] = RunMode.BACKTEST dp = DataProvider(default_conf, exchange) assert dp.runmode == RunMode.BACKTEST - assert isinstance(dp.get_pair_dataframe("UNITTEST/BTC", timeframe), DataFrame) + assert isinstance(dp.get_pair_dataframe( + "UNITTEST/BTC", timeframe, candle_type=candle_type), DataFrame) # assert dp.get_pair_dataframe("NONESENSE/AAA", timeframe).empty @@ -276,7 +293,7 @@ def test_no_exchange_mode(default_conf): dp.refresh([()]) with pytest.raises(OperationalException, match=message): - dp.ohlcv('XRP/USDT', '5m') + dp.ohlcv('XRP/USDT', '5m', '') with pytest.raises(OperationalException, match=message): dp.market('XRP/USDT') diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 6c811a673..ea37ea268 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -95,6 +95,17 @@ def test_load_data_1min_timeframe(ohlcv_history, mocker, caplog, testdatadir) -> ) +def test_load_data_mark(ohlcv_history, mocker, caplog, testdatadir) -> None: + mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=ohlcv_history) + file = testdatadir / 'UNITTEST_USDT-1h-mark.json' + load_data(datadir=testdatadir, timeframe='1h', pairs=['UNITTEST/BTC'], candle_type='mark') + assert file.is_file() + assert not log_has( + 'Download history data for pair: "UNITTEST/USDT", interval: 1m ' + 'and store in None.', caplog + ) + + def test_load_data_startup_candles(mocker, caplog, default_conf, testdatadir) -> None: ltfmock = mocker.patch( 'freqtrade.data.history.jsondatahandler.JsonDataHandler._ohlcv_load', @@ -110,8 +121,9 @@ def test_load_data_startup_candles(mocker, caplog, default_conf, testdatadir) -> assert ltfmock.call_args_list[0][1]['timerange'].startts == timerange.startts - 20 * 60 +@pytest.mark.parametrize('candle_type', ['mark', '']) def test_load_data_with_new_pair_1min(ohlcv_history_list, mocker, caplog, - default_conf, tmpdir) -> None: + default_conf, tmpdir, candle_type) -> None: """ Test load_pair_history() with 1 min timeframe """ @@ -121,7 +133,7 @@ def test_load_data_with_new_pair_1min(ohlcv_history_list, mocker, caplog, file = tmpdir1 / 'MEME_BTC-1m.json' # do not download a new pair if refresh_pairs isn't set - load_pair_history(datadir=tmpdir1, timeframe='1m', pair='MEME/BTC') + load_pair_history(datadir=tmpdir1, timeframe='1m', pair='MEME/BTC', candle_type=candle_type) assert not file.is_file() assert log_has( 'No history data for pair: "MEME/BTC", timeframe: 1m. ' @@ -131,7 +143,7 @@ def test_load_data_with_new_pair_1min(ohlcv_history_list, mocker, caplog, # download a new pair if refresh_pairs is set refresh_data(datadir=tmpdir1, timeframe='1m', pairs=['MEME/BTC'], exchange=exchange) - load_pair_history(datadir=tmpdir1, timeframe='1m', pair='MEME/BTC') + load_pair_history(datadir=tmpdir1, timeframe='1m', pair='MEME/BTC', candle_type=candle_type) assert file.is_file() assert log_has_re( r'Download history data for pair: "MEME/BTC" \(0/1\), timeframe: 1m ' @@ -166,7 +178,7 @@ def test_json_pair_data_filename(pair, expected_result, candle_type): Path('freqtrade/hello/world'), pair, '5m', - candle_type + candle_type=candle_type ) assert isinstance(fn, Path) assert fn == Path(expected_result + '.gz') @@ -241,24 +253,37 @@ def test_load_cached_data_for_updating(mocker, testdatadir) -> None: assert start_ts is None -def test_download_pair_history(ohlcv_history_list, mocker, default_conf, tmpdir) -> None: +@pytest.mark.parametrize('candle_type, file_tail', [ + ('mark', '-mark'), + ('', ''), +]) +def test_download_pair_history( + ohlcv_history_list, + mocker, + default_conf, + tmpdir, + candle_type, + file_tail +) -> None: mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=ohlcv_history_list) exchange = get_patched_exchange(mocker, default_conf) tmpdir1 = Path(tmpdir) - file1_1 = tmpdir1 / 'MEME_BTC-1m.json' - file1_5 = tmpdir1 / 'MEME_BTC-5m.json' - file2_1 = tmpdir1 / 'CFI_BTC-1m.json' - file2_5 = tmpdir1 / 'CFI_BTC-5m.json' + file1_1 = tmpdir1 / f'MEME_BTC-1m{file_tail}.json' + file1_5 = tmpdir1 / f'MEME_BTC-5m{file_tail}.json' + file2_1 = tmpdir1 / f'CFI_BTC-1m{file_tail}.json' + file2_5 = tmpdir1 / f'CFI_BTC-5m{file_tail}.json' assert not file1_1.is_file() assert not file2_1.is_file() assert _download_pair_history(datadir=tmpdir1, exchange=exchange, pair='MEME/BTC', - timeframe='1m') + timeframe='1m', + candle_type=candle_type) assert _download_pair_history(datadir=tmpdir1, exchange=exchange, pair='CFI/BTC', - timeframe='1m') + timeframe='1m', + candle_type=candle_type) assert not exchange._pairs_last_refresh_time assert file1_1.is_file() assert file2_1.is_file() @@ -272,10 +297,12 @@ def test_download_pair_history(ohlcv_history_list, mocker, default_conf, tmpdir) assert _download_pair_history(datadir=tmpdir1, exchange=exchange, pair='MEME/BTC', - timeframe='5m') + timeframe='5m', + candle_type=candle_type) assert _download_pair_history(datadir=tmpdir1, exchange=exchange, pair='CFI/BTC', - timeframe='5m') + timeframe='5m', + candle_type=candle_type) assert not exchange._pairs_last_refresh_time assert file1_5.is_file() assert file2_5.is_file() @@ -295,7 +322,9 @@ def test_download_pair_history2(mocker, default_conf, testdatadir) -> None: timeframe='1m') _download_pair_history(datadir=testdatadir, exchange=exchange, pair="UNITTEST/BTC", timeframe='3m') - assert json_dump_mock.call_count == 2 + _download_pair_history(datadir=testdatadir, exchange=exchange, pair="UNITTEST/USDT", + timeframe='1h', candle_type='mark') + assert json_dump_mock.call_count == 3 def test_download_backtesting_data_exception(mocker, caplog, default_conf, tmpdir) -> None: @@ -629,7 +658,7 @@ def test_datahandler_ohlcv_get_pairs(testdatadir): pairs = HDF5DataHandler.ohlcv_get_pairs(testdatadir, '5m') assert set(pairs) == {'UNITTEST/BTC'} - pairs = JsonDataHandler.ohlcv_get_pairs(testdatadir, '1h', 'mark') + pairs = JsonDataHandler.ohlcv_get_pairs(testdatadir, '1h', candle_type='mark') assert set(pairs) == {'UNITTEST/USDT', 'XRP/USDT'} # TODO-lev: The tests below @@ -643,17 +672,33 @@ def test_datahandler_ohlcv_get_pairs(testdatadir): def test_datahandler_ohlcv_get_available_data(testdatadir): paircombs = JsonDataHandler.ohlcv_get_available_data(testdatadir) # Convert to set to avoid failures due to sorting - assert set(paircombs) == {('UNITTEST/BTC', '5m'), ('ETH/BTC', '5m'), ('XLM/BTC', '5m'), - ('TRX/BTC', '5m'), ('LTC/BTC', '5m'), ('XMR/BTC', '5m'), - ('ZEC/BTC', '5m'), ('UNITTEST/BTC', '1m'), ('ADA/BTC', '5m'), - ('ETC/BTC', '5m'), ('NXT/BTC', '5m'), ('DASH/BTC', '5m'), - ('XRP/ETH', '1m'), ('XRP/ETH', '5m'), ('UNITTEST/BTC', '30m'), - ('UNITTEST/BTC', '8m'), ('NOPAIR/XXX', '4m')} + assert set(paircombs) == { + ('UNITTEST/BTC', '5m', ''), + ('ETH/BTC', '5m', ''), + ('XLM/BTC', '5m', ''), + ('TRX/BTC', '5m', ''), + ('LTC/BTC', '5m', ''), + ('XMR/BTC', '5m', ''), + ('ZEC/BTC', '5m', ''), + ('UNITTEST/BTC', '1m', ''), + ('ADA/BTC', '5m', ''), + ('ETC/BTC', '5m', ''), + ('NXT/BTC', '5m', ''), + ('DASH/BTC', '5m', ''), + ('XRP/ETH', '1m', ''), + ('XRP/ETH', '5m', ''), + ('UNITTEST/BTC', '30m', ''), + ('UNITTEST/BTC', '8m', ''), + ('NOPAIR/XXX', '4m', ''), + ('UNITTEST/USDT', '1h', 'mark'), + ('XRP/USDT', '1h', ''), + ('XRP/USDT', '1h', 'mark'), + } paircombs = JsonGzDataHandler.ohlcv_get_available_data(testdatadir) - assert set(paircombs) == {('UNITTEST/BTC', '8m')} + assert set(paircombs) == {('UNITTEST/BTC', '8m', '')} paircombs = HDF5DataHandler.ohlcv_get_available_data(testdatadir) - assert set(paircombs) == {('UNITTEST/BTC', '5m')} + assert set(paircombs) == {('UNITTEST/BTC', '5m', '')} def test_jsondatahandler_trades_get_pairs(testdatadir): @@ -666,27 +711,29 @@ def test_jsondatahandler_ohlcv_purge(mocker, testdatadir): mocker.patch.object(Path, "exists", MagicMock(return_value=False)) unlinkmock = mocker.patch.object(Path, "unlink", MagicMock()) dh = JsonGzDataHandler(testdatadir) - assert not dh.ohlcv_purge('UNITTEST/NONEXIST', '5m') + assert not dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', '') + assert not dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', candle_type='mark') assert unlinkmock.call_count == 0 mocker.patch.object(Path, "exists", MagicMock(return_value=True)) - assert dh.ohlcv_purge('UNITTEST/NONEXIST', '5m') - assert unlinkmock.call_count == 1 + assert dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', '') + assert dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', candle_type='mark') + assert unlinkmock.call_count == 2 def test_jsondatahandler_ohlcv_load(testdatadir, caplog): dh = JsonDataHandler(testdatadir) - df = dh.ohlcv_load('XRP/ETH', '5m') + df = dh.ohlcv_load('XRP/ETH', '5m', '') assert len(df) == 711 - df_mark = dh.ohlcv_load('XRP/USDT', '1h', candle_type="mark") + df_mark = dh.ohlcv_load('UNITTEST/USDT', '1h', candle_type="mark") assert len(df_mark) == 99 - df_no_mark = dh.ohlcv_load('XRP/USDT', '1h') + df_no_mark = dh.ohlcv_load('UNITTEST/USDT', '1h', '') assert len(df_no_mark) == 0 # Failure case (empty array) - df1 = dh.ohlcv_load('NOPAIR/XXX', '4m') + df1 = dh.ohlcv_load('NOPAIR/XXX', '4m', '') assert len(df1) == 0 assert log_has("Could not load data for NOPAIR/XXX.", caplog) assert df.columns.equals(df1.columns) @@ -720,6 +767,8 @@ def test_datahandler_ohlcv_append(datahandler, testdatadir, ): dh = get_datahandler(testdatadir, datahandler) with pytest.raises(NotImplementedError): dh.ohlcv_append('UNITTEST/ETH', '5m', DataFrame()) + with pytest.raises(NotImplementedError): + dh.ohlcv_append('UNITTEST/ETH', '5m', DataFrame(), candle_type='mark') @pytest.mark.parametrize('datahandler', AVAILABLE_DATAHANDLERS) @@ -849,12 +898,14 @@ def test_hdf5datahandler_ohlcv_purge(mocker, testdatadir): mocker.patch.object(Path, "exists", MagicMock(return_value=False)) unlinkmock = mocker.patch.object(Path, "unlink", MagicMock()) dh = HDF5DataHandler(testdatadir) - assert not dh.ohlcv_purge('UNITTEST/NONEXIST', '5m') + assert not dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', '') + assert not dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', candle_type='mark') assert unlinkmock.call_count == 0 mocker.patch.object(Path, "exists", MagicMock(return_value=True)) - assert dh.ohlcv_purge('UNITTEST/NONEXIST', '5m') - assert unlinkmock.call_count == 1 + assert dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', '') + assert dh.ohlcv_purge('UNITTEST/NONEXIST', '5m', candle_type='mark') + assert unlinkmock.call_count == 2 def test_gethandlerclass(): diff --git a/tests/exchange/test_binance.py b/tests/exchange/test_binance.py index c4277daad..ac7647e73 100644 --- a/tests/exchange/test_binance.py +++ b/tests/exchange/test_binance.py @@ -343,7 +343,8 @@ def test__set_leverage_binance(mocker, default_conf): @pytest.mark.asyncio -async def test__async_get_historic_ohlcv_binance(default_conf, mocker, caplog): +@pytest.mark.parametrize('candle_type', ['mark', '']) +async def test__async_get_historic_ohlcv_binance(default_conf, mocker, caplog, candle_type): ohlcv = [ [ int((datetime.now(timezone.utc).timestamp() - 1000) * 1000), @@ -360,16 +361,17 @@ async def test__async_get_historic_ohlcv_binance(default_conf, mocker, caplog): exchange._api_async.fetch_ohlcv = get_mock_coro(ohlcv) pair = 'ETH/BTC' - respair, restf, res = await exchange._async_get_historic_ohlcv( - pair, "5m", 1500000000000, is_new_pair=False) + respair, restf, restype, res = await exchange._async_get_historic_ohlcv( + pair, "5m", 1500000000000, is_new_pair=False, candle_type=candle_type) assert respair == pair assert restf == '5m' + assert restype == candle_type # Call with very old timestamp - causes tons of requests assert exchange._api_async.fetch_ohlcv.call_count > 400 # assert res == ohlcv exchange._api_async.fetch_ohlcv.reset_mock() - _, _, res = await exchange._async_get_historic_ohlcv( - pair, "5m", 1500000000000, is_new_pair=True) + _, _, _, res = await exchange._async_get_historic_ohlcv( + pair, "5m", 1500000000000, is_new_pair=True, candle_type=candle_type) # Called twice - one "init" call - and one to get the actual data. assert exchange._api_async.fetch_ohlcv.call_count == 2 diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index c57880bdc..d8f4be660 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1554,7 +1554,8 @@ def test_fetch_ticker(default_conf, mocker, exchange_name): @pytest.mark.parametrize("exchange_name", EXCHANGES) -def test_get_historic_ohlcv(default_conf, mocker, caplog, exchange_name): +@pytest.mark.parametrize('candle_type', ['mark', '']) +def test_get_historic_ohlcv(default_conf, mocker, caplog, exchange_name, candle_type): exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) ohlcv = [ [ @@ -1569,14 +1570,18 @@ def test_get_historic_ohlcv(default_conf, mocker, caplog, exchange_name): pair = 'ETH/BTC' async def mock_candle_hist(pair, timeframe, since_ms, candle_type=None): - return pair, timeframe, ohlcv + return pair, timeframe, candle_type, ohlcv exchange._async_get_candle_history = Mock(wraps=mock_candle_hist) # one_call calculation * 1.8 should do 2 calls since = 5 * 60 * exchange.ohlcv_candle_limit('5m') * 1.8 - ret = exchange.get_historic_ohlcv(pair, "5m", int(( - arrow.utcnow().int_timestamp - since) * 1000)) + ret = exchange.get_historic_ohlcv( + pair, + "5m", + int((arrow.utcnow().int_timestamp - since) * 1000), + candle_type=candle_type + ) assert exchange._async_get_candle_history.call_count == 2 # Returns twice the above OHLCV data @@ -1589,13 +1594,18 @@ def test_get_historic_ohlcv(default_conf, mocker, caplog, exchange_name): raise TimeoutError() exchange._async_get_candle_history = MagicMock(side_effect=mock_get_candle_hist_error) - ret = exchange.get_historic_ohlcv(pair, "5m", int( - (arrow.utcnow().int_timestamp - since) * 1000)) + ret = exchange.get_historic_ohlcv( + pair, + "5m", + int((arrow.utcnow().int_timestamp - since) * 1000), + candle_type=candle_type + ) assert log_has_re(r"Async code raised an exception: .*", caplog) @pytest.mark.parametrize("exchange_name", EXCHANGES) -def test_get_historic_ohlcv_as_df(default_conf, mocker, exchange_name): +@pytest.mark.parametrize('candle_type', ['mark', '']) +def test_get_historic_ohlcv_as_df(default_conf, mocker, exchange_name, candle_type): exchange = get_patched_exchange(mocker, default_conf, id=exchange_name) ohlcv = [ [ @@ -1625,15 +1635,19 @@ def test_get_historic_ohlcv_as_df(default_conf, mocker, exchange_name): ] pair = 'ETH/BTC' - async def mock_candle_hist(pair, timeframe, since_ms, candle_type=None): - return pair, timeframe, ohlcv + async def mock_candle_hist(pair, timeframe, since_ms, candle_type): + return pair, timeframe, candle_type, ohlcv exchange._async_get_candle_history = Mock(wraps=mock_candle_hist) # one_call calculation * 1.8 should do 2 calls since = 5 * 60 * exchange.ohlcv_candle_limit('5m') * 1.8 - ret = exchange.get_historic_ohlcv_as_df(pair, "5m", int(( - arrow.utcnow().int_timestamp - since) * 1000)) + ret = exchange.get_historic_ohlcv_as_df( + pair, + "5m", + int((arrow.utcnow().int_timestamp - since) * 1000), + candle_type=candle_type + ) assert exchange._async_get_candle_history.call_count == 2 # Returns twice the above OHLCV data @@ -1647,6 +1661,7 @@ def test_get_historic_ohlcv_as_df(default_conf, mocker, exchange_name): @pytest.mark.asyncio @pytest.mark.parametrize("exchange_name", EXCHANGES) +# TODO-lev @pytest.mark.parametrize('candle_type', ['mark', '']) async def test__async_get_historic_ohlcv(default_conf, mocker, caplog, exchange_name): ohlcv = [ [ @@ -1663,7 +1678,7 @@ async def test__async_get_historic_ohlcv(default_conf, mocker, caplog, exchange_ exchange._api_async.fetch_ohlcv = get_mock_coro(ohlcv) pair = 'ETH/USDT' - respair, restf, res = await exchange._async_get_historic_ohlcv( + respair, restf, _, res = await exchange._async_get_historic_ohlcv( pair, "5m", 1500000000000, is_new_pair=False) assert respair == pair assert restf == '5m' @@ -1672,6 +1687,7 @@ async def test__async_get_historic_ohlcv(default_conf, mocker, caplog, exchange_ assert res[0] == ohlcv[0] +# TODO-lev: @pytest.mark.parametrize('candle_type', ['mark', '']) def test_refresh_latest_ohlcv(mocker, default_conf, caplog) -> None: ohlcv = [ [ @@ -1696,7 +1712,7 @@ def test_refresh_latest_ohlcv(mocker, default_conf, caplog) -> None: exchange = get_patched_exchange(mocker, default_conf) exchange._api_async.fetch_ohlcv = get_mock_coro(ohlcv) - pairs = [('IOTA/ETH', '5m'), ('XRP/ETH', '5m')] + pairs = [('IOTA/ETH', '5m', ''), ('XRP/ETH', '5m', '')] # empty dicts assert not exchange._klines res = exchange.refresh_latest_ohlcv(pairs, cache=False) @@ -1727,16 +1743,18 @@ def test_refresh_latest_ohlcv(mocker, default_conf, caplog) -> None: assert exchange.klines(pair, copy=False) is exchange.klines(pair, copy=False) # test caching - res = exchange.refresh_latest_ohlcv([('IOTA/ETH', '5m'), ('XRP/ETH', '5m')]) + res = exchange.refresh_latest_ohlcv([('IOTA/ETH', '5m', ''), ('XRP/ETH', '5m', '')]) assert len(res) == len(pairs) assert exchange._api_async.fetch_ohlcv.call_count == 0 assert log_has(f"Using cached candle (OHLCV) data for pair {pairs[0][0]}, " f"timeframe {pairs[0][1]} ...", caplog) - res = exchange.refresh_latest_ohlcv([('IOTA/ETH', '5m'), ('XRP/ETH', '5m'), ('XRP/ETH', '1d')], - cache=False) - assert len(res) == 3 + res = exchange.refresh_latest_ohlcv( + [('IOTA/ETH', '5m', ''), ('XRP/ETH', '5m', ''), ('XRP/ETH', '1d', '')], + cache=False + ) + assert len(res) == 4 @pytest.mark.asyncio @@ -1761,10 +1779,11 @@ async def test__async_get_candle_history(default_conf, mocker, caplog, exchange_ pair = 'ETH/BTC' res = await exchange._async_get_candle_history(pair, "5m") assert type(res) is tuple - assert len(res) == 3 + assert len(res) == 4 assert res[0] == pair assert res[1] == "5m" - assert res[2] == ohlcv + assert res[2] == '' + assert res[3] == ohlcv assert exchange._api_async.fetch_ohlcv.call_count == 1 assert not log_has(f"Using cached candle (OHLCV) data for {pair} ...", caplog) @@ -1803,10 +1822,11 @@ async def test__async_get_candle_history_empty(default_conf, mocker, caplog): pair = 'ETH/BTC' res = await exchange._async_get_candle_history(pair, "5m") assert type(res) is tuple - assert len(res) == 3 + assert len(res) == 4 assert res[0] == pair assert res[1] == "5m" - assert res[2] == ohlcv + assert res[2] == '' + assert res[3] == ohlcv assert exchange._api_async.fetch_ohlcv.call_count == 1 @@ -1823,7 +1843,7 @@ def test_refresh_latest_ohlcv_inv_result(default_conf, mocker, caplog): # Monkey-patch async function with empty result exchange._api_async.fetch_ohlcv = MagicMock(side_effect=mock_get_candle_hist) - pairs = [("ETH/BTC", "5m"), ("XRP/BTC", "5m")] + pairs = [("ETH/BTC", "5m", ''), ("XRP/BTC", "5m", '')] res = exchange.refresh_latest_ohlcv(pairs) assert exchange._klines assert exchange._api_async.fetch_ohlcv.call_count == 2 @@ -2107,7 +2127,7 @@ async def test___async_get_candle_history_sort(default_conf, mocker, exchange_na # Test the OHLCV data sort res = await exchange._async_get_candle_history('ETH/BTC', default_conf['timeframe']) assert res[0] == 'ETH/BTC' - res_ohlcv = res[2] + res_ohlcv = res[3] assert sort_mock.call_count == 1 assert res_ohlcv[0][0] == 1527830400000 @@ -2145,7 +2165,7 @@ async def test___async_get_candle_history_sort(default_conf, mocker, exchange_na res = await exchange._async_get_candle_history('ETH/BTC', default_conf['timeframe']) assert res[0] == 'ETH/BTC' assert res[1] == default_conf['timeframe'] - res_ohlcv = res[2] + res_ohlcv = res[3] # Sorted not called again - data is already in order assert sort_mock.call_count == 0 assert res_ohlcv[0][0] == 1527827700000 @@ -2999,7 +3019,7 @@ def test_timeframe_to_next_date(): def test_market_is_tradable( mocker, default_conf, market_symbol, base, quote, spot, margin, futures, trademode, add_dict, exchange, expected_result - ) -> None: +) -> None: default_conf['trading_mode'] = trademode mocker.patch('freqtrade.exchange.exchange.Exchange.validate_trading_mode_and_collateral') ex = get_patched_exchange(mocker, default_conf, id=exchange) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 21d11d7f7..2aa2af04d 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -855,7 +855,7 @@ def test_backtest_alternate_buy_sell(default_conf, fee, mocker, testdatadir): results = result['results'] assert len(results) == 100 # Cached data should be 200 - analyzed_df = backtesting.dataprovider.get_analyzed_dataframe('UNITTEST/BTC', '1m')[0] + analyzed_df = backtesting.dataprovider.get_analyzed_dataframe('UNITTEST/BTC', '1m', '')[0] assert len(analyzed_df) == 200 # Expect last candle to be 1 below end date (as the last candle is assumed as "incomplete" # during backtesting) @@ -924,8 +924,9 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir) offset = 1 if tres == 0 else 0 removed_candles = len(data[pair]) - offset - backtesting.strategy.startup_candle_count assert len(backtesting.dataprovider.get_analyzed_dataframe(pair, '5m')[0]) == removed_candles - assert len(backtesting.dataprovider.get_analyzed_dataframe( - 'NXT/BTC', '5m')[0]) == len(data['NXT/BTC']) - 1 - backtesting.strategy.startup_candle_count + assert len( + backtesting.dataprovider.get_analyzed_dataframe('NXT/BTC', '5m', '')[0] + ) == len(data['NXT/BTC']) - 1 - backtesting.strategy.startup_candle_count backtest_conf = { 'processed': processed, diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 692f1fb51..c1eab2fe3 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -460,11 +460,11 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t ohlcv_history_high_vola.loc[ohlcv_history_high_vola.index == 1, 'close'] = 0.00090 ohlcv_data = { - ('ETH/BTC', '1d'): ohlcv_history, - ('TKN/BTC', '1d'): ohlcv_history, - ('LTC/BTC', '1d'): ohlcv_history.append(ohlcv_history), - ('XRP/BTC', '1d'): ohlcv_history, - ('HOT/BTC', '1d'): ohlcv_history_high_vola, + ('ETH/BTC', '1d', ''): ohlcv_history, + ('TKN/BTC', '1d', ''): ohlcv_history, + ('LTC/BTC', '1d', ''): ohlcv_history.append(ohlcv_history), + ('XRP/BTC', '1d', ''): ohlcv_history, + ('HOT/BTC', '1d', ''): ohlcv_history_high_vola, } mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) @@ -578,11 +578,11 @@ def test_VolumePairList_range(mocker, whitelist_conf, shitcoinmarkets, tickers, ohlcv_history_high_volume.loc[:, 'volume'] = 10 ohlcv_data = { - ('ETH/BTC', '1d'): ohlcv_history, - ('TKN/BTC', '1d'): ohlcv_history, - ('LTC/BTC', '1d'): ohlcv_history_medium_volume, - ('XRP/BTC', '1d'): ohlcv_history_high_vola, - ('HOT/BTC', '1d'): ohlcv_history_high_volume, + ('ETH/BTC', '1d', ''): ohlcv_history, + ('TKN/BTC', '1d', ''): ohlcv_history, + ('LTC/BTC', '1d', ''): ohlcv_history_medium_volume, + ('XRP/BTC', '1d', ''): ohlcv_history_high_vola, + ('HOT/BTC', '1d', ''): ohlcv_history_high_volume, } mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) @@ -838,9 +838,9 @@ def test_agefilter_min_days_listed_too_large(mocker, default_conf, markets, tick def test_agefilter_caching(mocker, markets, whitelist_conf_agefilter, tickers, ohlcv_history): with time_machine.travel("2021-09-01 05:00:00 +00:00") as t: ohlcv_data = { - ('ETH/BTC', '1d'): ohlcv_history, - ('TKN/BTC', '1d'): ohlcv_history, - ('LTC/BTC', '1d'): ohlcv_history, + ('ETH/BTC', '1d', ''): ohlcv_history, + ('TKN/BTC', '1d', ''): ohlcv_history, + ('LTC/BTC', '1d', ''): ohlcv_history, } mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -862,10 +862,10 @@ def test_agefilter_caching(mocker, markets, whitelist_conf_agefilter, tickers, o assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 2 ohlcv_data = { - ('ETH/BTC', '1d'): ohlcv_history, - ('TKN/BTC', '1d'): ohlcv_history, - ('LTC/BTC', '1d'): ohlcv_history, - ('XRP/BTC', '1d'): ohlcv_history.iloc[[0]], + ('ETH/BTC', '1d', ''): ohlcv_history, + ('TKN/BTC', '1d', ''): ohlcv_history, + ('LTC/BTC', '1d', ''): ohlcv_history, + ('XRP/BTC', '1d', ''): ohlcv_history.iloc[[0]], } mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', return_value=ohlcv_data) freqtrade.pairlists.refresh_pairlist() @@ -883,10 +883,10 @@ def test_agefilter_caching(mocker, markets, whitelist_conf_agefilter, tickers, o t.move_to("2021-09-03 01:00:00 +00:00") # Called once for XRP/BTC ohlcv_data = { - ('ETH/BTC', '1d'): ohlcv_history, - ('TKN/BTC', '1d'): ohlcv_history, - ('LTC/BTC', '1d'): ohlcv_history, - ('XRP/BTC', '1d'): ohlcv_history, + ('ETH/BTC', '1d', ''): ohlcv_history, + ('TKN/BTC', '1d', ''): ohlcv_history, + ('LTC/BTC', '1d', ''): ohlcv_history, + ('XRP/BTC', '1d', ''): ohlcv_history, } mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', return_value=ohlcv_data) freqtrade.pairlists.refresh_pairlist() @@ -947,12 +947,12 @@ def test_rangestabilityfilter_caching(mocker, markets, default_conf, tickers, oh get_tickers=tickers ) ohlcv_data = { - ('ETH/BTC', '1d'): ohlcv_history, - ('TKN/BTC', '1d'): ohlcv_history, - ('LTC/BTC', '1d'): ohlcv_history, - ('XRP/BTC', '1d'): ohlcv_history, - ('HOT/BTC', '1d'): ohlcv_history, - ('BLK/BTC', '1d'): ohlcv_history, + ('ETH/BTC', '1d', ''): ohlcv_history, + ('TKN/BTC', '1d', ''): ohlcv_history, + ('LTC/BTC', '1d', ''): ohlcv_history, + ('XRP/BTC', '1d', ''): ohlcv_history, + ('HOT/BTC', '1d', ''): ohlcv_history, + ('BLK/BTC', '1d', ''): ohlcv_history, } mocker.patch.multiple( 'freqtrade.exchange.Exchange', diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index f2096c0c0..bdbb2e833 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -718,8 +718,8 @@ def test_api_edge_disabled(botclient, mocker, ticker, fee, markets): 'profit_closed_percent_mean': -0.75, 'profit_closed_ratio_sum': -0.015, 'profit_closed_percent_sum': -1.5, 'profit_closed_ratio': -6.739057628404269e-06, 'profit_closed_percent': -0.0, 'winning_trades': 0, 'losing_trades': 2} - ), - ( + ), + ( False, {'best_pair': 'XRP/BTC', 'best_rate': 1.0, 'best_pair_profit_ratio': 0.01, 'profit_all_coin': -44.0631579, @@ -731,8 +731,8 @@ def test_api_edge_disabled(botclient, mocker, ticker, fee, markets): 'profit_closed_percent_mean': 0.75, 'profit_closed_ratio_sum': 0.015, 'profit_closed_percent_sum': 1.5, 'profit_closed_ratio': 7.391275897987988e-07, 'profit_closed_percent': 0.0, 'winning_trades': 2, 'losing_trades': 0} - ), - ( + ), + ( None, {'best_pair': 'XRP/BTC', 'best_rate': 1.0, 'best_pair_profit_ratio': 0.01, 'profit_all_coin': -14.43790415, @@ -744,8 +744,8 @@ def test_api_edge_disabled(botclient, mocker, ticker, fee, markets): 'profit_closed_percent_mean': 0.25, 'profit_closed_ratio_sum': 0.005, 'profit_closed_percent_sum': 0.5, 'profit_closed_ratio': -5.429078808526421e-06, 'profit_closed_percent': -0.0, 'winning_trades': 1, 'losing_trades': 1} - ) - ]) + ) + ]) def test_api_profit(botclient, mocker, ticker, fee, markets, is_short, expected): ftbot, client = botclient patch_get_signal(ftbot) @@ -1331,7 +1331,7 @@ def test_list_available_pairs(botclient): rc = client_get(client, f"{BASE_URI}/available_pairs") assert_response(rc) - assert rc.json()['length'] == 13 + assert rc.json()['length'] == 15 assert isinstance(rc.json()['pairs'], list) rc = client_get(client, f"{BASE_URI}/available_pairs?timeframe=5m") @@ -1350,6 +1350,13 @@ def test_list_available_pairs(botclient): assert rc.json()['pairs'] == ['XRP/ETH'] assert len(rc.json()['pair_interval']) == 1 + rc = client_get( + client, f"{BASE_URI}/available_pairs?stake_currency=USDT&timeframe=1h&type=mark") + assert_response(rc) + assert rc.json()['length'] == 2 + assert rc.json()['pairs'] == ['UNITTEST/USDT', 'XRP/USDT'] + assert len(rc.json()['pair_interval']) == 3 # TODO-lev: What is pair_interval? Should it be 3? + def test_sysinfo(botclient): ftbot, client = botclient diff --git a/tests/strategy/strats/informative_decorator_strategy.py b/tests/strategy/strats/informative_decorator_strategy.py index 68f8651c2..331d3de08 100644 --- a/tests/strategy/strats/informative_decorator_strategy.py +++ b/tests/strategy/strats/informative_decorator_strategy.py @@ -19,7 +19,7 @@ class InformativeDecoratorTest(IStrategy): startup_candle_count: int = 20 def informative_pairs(self): - return [('BTC/USDT', '5m')] + return [('BTC/USDT', '5m', '')] def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: dataframe['buy'] = 0 @@ -67,7 +67,7 @@ class InformativeDecoratorTest(IStrategy): dataframe['rsi_less'] = dataframe['rsi'] < dataframe['rsi_1h'] # Mixing manual informative pairs with decorators. - informative = self.dp.get_pair_dataframe('BTC/USDT', '5m') + informative = self.dp.get_pair_dataframe('BTC/USDT', '5m', '') informative['rsi'] = 14 dataframe = merge_informative_pair(dataframe, informative, self.timeframe, '5m', ffill=True) diff --git a/tests/strategy/test_strategy_helpers.py b/tests/strategy/test_strategy_helpers.py index cb7cf97a1..e81b544d5 100644 --- a/tests/strategy/test_strategy_helpers.py +++ b/tests/strategy/test_strategy_helpers.py @@ -144,23 +144,24 @@ def test_stoploss_from_absolute(): assert stoploss_from_absolute(0, 100) == 1 +# TODO-lev: @pytest.mark.parametrize('candle_type', ['mark', '']) def test_informative_decorator(mocker, default_conf): test_data_5m = generate_test_data('5m', 40) test_data_30m = generate_test_data('30m', 40) test_data_1h = generate_test_data('1h', 40) data = { - ('XRP/USDT', '5m'): test_data_5m, - ('XRP/USDT', '30m'): test_data_30m, - ('XRP/USDT', '1h'): test_data_1h, - ('LTC/USDT', '5m'): test_data_5m, - ('LTC/USDT', '30m'): test_data_30m, - ('LTC/USDT', '1h'): test_data_1h, - ('BTC/USDT', '30m'): test_data_30m, - ('BTC/USDT', '5m'): test_data_5m, - ('BTC/USDT', '1h'): test_data_1h, - ('ETH/USDT', '1h'): test_data_1h, - ('ETH/USDT', '30m'): test_data_30m, - ('ETH/BTC', '1h'): test_data_1h, + ('XRP/USDT', '5m', ''): test_data_5m, + ('XRP/USDT', '30m', ''): test_data_30m, + ('XRP/USDT', '1h', ''): test_data_1h, + ('LTC/USDT', '5m', ''): test_data_5m, + ('LTC/USDT', '30m', ''): test_data_30m, + ('LTC/USDT', '1h', ''): test_data_1h, + ('BTC/USDT', '30m', ''): test_data_30m, + ('BTC/USDT', '5m', ''): test_data_5m, + ('BTC/USDT', '1h', ''): test_data_1h, + ('ETH/USDT', '1h', ''): test_data_1h, + ('ETH/USDT', '30m', ''): test_data_30m, + ('ETH/BTC', '1h', ''): test_data_1h, } from .strats.informative_decorator_strategy import InformativeDecoratorTest default_conf['stake_currency'] = 'USDT' @@ -171,19 +172,19 @@ def test_informative_decorator(mocker, default_conf): ]) assert len(strategy._ft_informative) == 6 # Equal to number of decorators used - informative_pairs = [('XRP/USDT', '1h'), ('LTC/USDT', '1h'), ('XRP/USDT', '30m'), - ('LTC/USDT', '30m'), ('BTC/USDT', '1h'), ('BTC/USDT', '30m'), - ('BTC/USDT', '5m'), ('ETH/BTC', '1h'), ('ETH/USDT', '30m')] + informative_pairs = [('XRP/USDT', '1h', ''), ('LTC/USDT', '1h', ''), ('XRP/USDT', '30m', ''), + ('LTC/USDT', '30m', ''), ('BTC/USDT', '1h', ''), ('BTC/USDT', '30m', ''), + ('BTC/USDT', '5m', ''), ('ETH/BTC', '1h', ''), ('ETH/USDT', '30m', '')] for inf_pair in informative_pairs: assert inf_pair in strategy.gather_informative_pairs() - def test_historic_ohlcv(pair, timeframe): - return data[(pair, timeframe or strategy.timeframe)].copy() + def test_historic_ohlcv(pair, timeframe, candle_type): + return data[(pair, timeframe or strategy.timeframe, candle_type)].copy() mocker.patch('freqtrade.data.dataprovider.DataProvider.historic_ohlcv', side_effect=test_historic_ohlcv) analyzed = strategy.advise_all_indicators( - {p: data[(p, strategy.timeframe)] for p in ('XRP/USDT', 'LTC/USDT')}) + {p: data[(p, strategy.timeframe, '')] for p in ('XRP/USDT', 'LTC/USDT')}) expected_columns = [ 'rsi_1h', 'rsi_30m', # Stacked informative decorators 'btc_usdt_rsi_1h', # BTC 1h informative diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index bd5fcf313..7183d17ff 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -681,7 +681,7 @@ def test_process_informative_pairs_added(default_conf_usdt, ticker_usdt, mocker) create_order=MagicMock(side_effect=TemporaryError), refresh_latest_ohlcv=refresh_mock, ) - inf_pairs = MagicMock(return_value=[("BTC/ETH", '1m'), ("ETH/USDT", "1h")]) + inf_pairs = MagicMock(return_value=[("BTC/ETH", '1m', ''), ("ETH/USDT", "1h", '')]) mocker.patch.multiple( 'freqtrade.strategy.interface.IStrategy', get_exit_signal=MagicMock(return_value=(False, False)), @@ -696,9 +696,9 @@ def test_process_informative_pairs_added(default_conf_usdt, ticker_usdt, mocker) freqtrade.process() assert inf_pairs.call_count == 1 assert refresh_mock.call_count == 1 - assert ("BTC/ETH", "1m") in refresh_mock.call_args[0][0] - assert ("ETH/USDT", "1h") in refresh_mock.call_args[0][0] - assert ("ETH/USDT", default_conf_usdt["timeframe"]) in refresh_mock.call_args[0][0] + assert ("BTC/ETH", "1m", '') in refresh_mock.call_args[0][0] + assert ("ETH/USDT", "1h", '') in refresh_mock.call_args[0][0] + assert ("ETH/USDT", default_conf_usdt["timeframe"], '') in refresh_mock.call_args[0][0] @pytest.mark.parametrize("trading_mode", [ diff --git a/tests/testdata/XRP_USDT-1h.json b/tests/testdata/XRP_USDT-1h.json new file mode 100644 index 000000000..58944e717 --- /dev/null +++ b/tests/testdata/XRP_USDT-1h.json @@ -0,0 +1,102 @@ +[ + [ 1637110800000, 1.0801, 1.09758, 1.07654, 1.07925, 3153694.607359 ], + [ 1637114400000, 1.07896, 1.0875, 1.07351, 1.07616, 2697616.070908 ], + [ 1637118000000, 1.07607, 1.08521, 1.05896, 1.06804, 4014666.826073 ], + [ 1637121600000, 1.06848, 1.07846, 1.06067, 1.07629, 3764015.567745 ], + [ 1637125200000, 1.07647, 1.08791, 1.07309, 1.0839, 1669038.113726 ], + [ 1637128800000, 1.08414, 1.0856, 1.07431, 1.0794, 1921068.874499 ], + [ 1637132400000, 1.0798, 1.09499, 1.07363, 1.08721, 2491096.863582 ], + [ 1637136000000, 1.08688, 1.09133, 1.08004, 1.08011, 1983486.794272 ], + [ 1637139600000, 1.08017, 1.08027, 1.06667, 1.07039, 3429247.985309 ], + [ 1637143200000, 1.07054, 1.10699, 1.07038, 1.10284, 4554151.954177 ], + [ 1637146800000, 1.10315, 1.10989, 1.09781, 1.1071, 2012983.10465 ], + [ 1637150400000, 1.10627, 1.10849, 1.10155, 1.10539, 1117804.08918 ], + [ 1637154000000, 1.10545, 1.11299, 1.09574, 1.09604, 2252781.33926 ], + [ 1637157600000, 1.09583, 1.10037, 1.08402, 1.08404, 1882359.279342 ], + [ 1637161200000, 1.08433, 1.08924, 1.07583, 1.08543, 1826745.82579 ], + [ 1637164800000, 1.08571, 1.09622, 1.07946, 1.09496, 1651730.678891 ], + [ 1637168400000, 1.09509, 1.0979, 1.0878, 1.0945, 1081210.614598 ], + [ 1637172000000, 1.09483, 1.10223, 1.09362, 1.09922, 1065998.492028 ], + [ 1637175600000, 1.09916, 1.10201, 1.09226, 1.09459, 924935.492048 ], + [ 1637179200000, 1.09458, 1.10196, 1.09051, 1.09916, 1253539.625345 ], + [ 1637182800000, 1.09939, 1.09948, 1.08751, 1.09485, 1066269.190094 ], + [ 1637186400000, 1.0949, 1.095, 1.08537, 1.09229, 924726.680514 ], + [ 1637190000000, 1.0923, 1.09877, 1.08753, 1.09522, 1150213.905599 ], + [ 1637193600000, 1.09538, 1.10675, 1.09058, 1.10453, 1489867.578178 ], + [ 1637197200000, 1.10446, 1.16313, 1.0978, 1.12907, 10016166.026355 ], + [ 1637200800000, 1.1287, 1.15367, 1.12403, 1.1381, 7167920.053752 ], + [ 1637204400000, 1.13818, 1.14242, 1.12358, 1.1244, 2665326.190545 ], + [ 1637208000000, 1.12432, 1.14864, 1.11061, 1.11447, 9340547.947608 ], + [ 1637211600000, 1.114, 1.12618, 1.10911, 1.11412, 11759138.472952 ], + [ 1637215200000, 1.11381, 1.11701, 1.10507, 1.1136, 3104670.727264 ], + [ 1637218800000, 1.11433, 1.1145, 1.09682, 1.10715, 2522287.830673 ], + [ 1637222400000, 1.1073, 1.11, 1.10224, 1.10697, 2021691.204473 ], + [ 1637226000000, 1.10622, 1.10707, 1.07727, 1.08674, 3679010.223352 ], + [ 1637229600000, 1.08651, 1.09861, 1.08065, 1.09771, 2041421.476307 ], + [ 1637233200000, 1.09784, 1.102, 1.08339, 1.08399, 1920597.122813 ], + [ 1637236800000, 1.08458, 1.09523, 1.07961, 1.08263, 2403158.337373 ], + [ 1637240400000, 1.08309, 1.08959, 1.06094, 1.07703, 4425686.808376 ], + [ 1637244000000, 1.07702, 1.08064, 1.063, 1.07049, 3361334.048801 ], + [ 1637247600000, 1.07126, 1.07851, 1.04538, 1.0562, 5865602.611111 ], + [ 1637251200000, 1.05616, 1.06326, 1.0395, 1.04074, 4206860.947352 ], + [ 1637254800000, 1.04023, 1.0533, 1.01478, 1.0417, 5641193.647291 ], + [ 1637258400000, 1.04177, 1.05444, 1.04132, 1.05204, 1819341.083656 ], + [ 1637262000000, 1.05201, 1.05962, 1.04964, 1.05518, 1567923.362515 ], + [ 1637265600000, 1.05579, 1.05924, 1.04772, 1.04773, 1794108.065606 ], + [ 1637269200000, 1.0484, 1.05622, 1.04183, 1.04544, 1936537.403899 ], + [ 1637272800000, 1.04543, 1.05331, 1.03396, 1.03892, 2839486.418143 ], + [ 1637276400000, 1.03969, 1.04592, 1.02886, 1.04086, 3116275.899177 ], + [ 1637280000000, 1.0409, 1.05681, 1.02922, 1.05481, 4671209.916896 ], + [ 1637283600000, 1.05489, 1.05538, 1.03539, 1.03599, 2566357.247547 ], + [ 1637287200000, 1.03596, 1.04606, 1.02038, 1.02428, 3441834.238546 ], + [ 1637290800000, 1.02483, 1.0291, 1.01785, 1.0285, 2678602.729339 ], + [ 1637294400000, 1.0287, 1.0446, 1.0259, 1.04264, 2303621.340808 ], + [ 1637298000000, 1.04313, 1.04676, 1.03662, 1.04499, 2426475.439485 ], + [ 1637301600000, 1.0451, 1.04971, 1.041, 1.04448, 2088365.810515 ], + [ 1637305200000, 1.04473, 1.04845, 1.03801, 1.04227, 2222396.213472 ], + [ 1637308800000, 1.04211, 1.06965, 1.04168, 1.05711, 3267643.936025 ], + [ 1637312400000, 1.0569, 1.06578, 1.05626, 1.05844, 1512848.016057 ], + [ 1637316000000, 1.05814, 1.05916, 1.04923, 1.05464, 1710694.805693 ], + [ 1637319600000, 1.05484, 1.05731, 1.0458, 1.05359, 1587100.45253 ], + [ 1637323200000, 1.05382, 1.06063, 1.05156, 1.05227, 1409095.236152 ], + [ 1637326800000, 1.05256, 1.06489, 1.04996, 1.06471, 1879315.174541 ], + [ 1637330400000, 1.06491, 1.1036, 1.06489, 1.09439, 6212842.71216 ], + [ 1637334000000, 1.09441, 1.10252, 1.082, 1.08879, 4833417.181969 ], + [ 1637337600000, 1.08866, 1.09485, 1.07538, 1.09045, 2554438.746366 ], + [ 1637341200000, 1.09058, 1.09906, 1.08881, 1.09039, 1961024.28963 ], + [ 1637344800000, 1.09063, 1.09447, 1.08555, 1.09041, 1427538.639232 ], + [ 1637348400000, 1.09066, 1.09521, 1.088, 1.09332, 847724.821691 ], + [ 1637352000000, 1.09335, 1.09489, 1.08402, 1.08501, 1035043.133874 ], + [ 1637355600000, 1.08474, 1.08694, 1.08, 1.08606, 969952.892274 ], + [ 1637359200000, 1.08601, 1.09, 1.08201, 1.08476, 1105782.581808 ], + [ 1637362800000, 1.08463, 1.09245, 1.08201, 1.08971, 1334467.438673 ], + [ 1637366400000, 1.0897, 1.09925, 1.08634, 1.09049, 2460070.020396 ], + [ 1637370000000, 1.0908, 1.10002, 1.09002, 1.09845, 1210028.489394 ], + [ 1637373600000, 1.09785, 1.09791, 1.08944, 1.08962, 1261987.295847 ], + [ 1637377200000, 1.08951, 1.0919, 1.08429, 1.08548, 1124938.783404 ], + [ 1637380800000, 1.08536, 1.09, 1.08424, 1.08783, 1330935.680168 ], + [ 1637384400000, 1.0877, 1.08969, 1.08266, 1.08617, 874900.746037 ], + [ 1637388000000, 1.08622, 1.09224, 1.0843, 1.0889, 1240184.759178 ], + [ 1637391600000, 1.08917, 1.0909, 1.08408, 1.08535, 706148.380072 ], + [ 1637395200000, 1.08521, 1.08857, 1.07829, 1.08349, 1713832.050838 ], + [ 1637398800000, 1.08343, 1.08841, 1.08272, 1.0855, 696597.06327 ], + [ 1637402400000, 1.08553, 1.0898, 1.08353, 1.08695, 1104159.802108 ], + [ 1637406000000, 1.08703, 1.09838, 1.08635, 1.09695, 1404001.384389 ], + [ 1637409600000, 1.09695, 1.10175, 1.09024, 1.09278, 1219090.620484 ], + [ 1637413200000, 1.093, 1.09577, 1.08615, 1.08792, 994797.546591 ], + [ 1637416800000, 1.08793, 1.09239, 1.08572, 1.08725, 1251685.429497 ], + [ 1637420400000, 1.08721, 1.08767, 1.06029, 1.06556, 3955719.53631 ], + [ 1637424000000, 1.06553, 1.07385, 1.06169, 1.07257, 1868359.179534 ], + [ 1637427600000, 1.07266, 1.0745, 1.06759, 1.07261, 1015134.469304 ], + [ 1637431200000, 1.07255, 1.0974, 1.06819, 1.09369, 4377675.964829 ], + [ 1637434800000, 1.09368, 1.09562, 1.08899, 1.09036, 914791.699929 ], + [ 1637438400000, 1.09085, 1.09262, 1.08855, 1.09214, 661436.936672 ], + [ 1637442000000, 1.0924, 1.09475, 1.08874, 1.09282, 593143.283519 ], + [ 1637445600000, 1.09301, 1.09638, 1.09154, 1.09611, 603952.916221 ], + [ 1637449200000, 1.09569, 1.09828, 1.09301, 1.09747, 676053.591571 ], + [ 1637452800000, 1.09742, 1.09822, 1.09011, 1.0902, 1375704.506469 ], + [ 1637456400000, 1.0901, 1.09311, 1.08619, 1.08856, 928706.03929 ], + [ 1637460000000, 1.08855, 1.08941, 1.07401, 1.08035, 2669150.388642 ], + [ 1637463600000, 1.08016, 1.08341, 1.07448, 1.07672, 1604049.131307 ], + [ 1637467200000, 1.07685, 1.08229, 1.07552, 1.0765, 1153357.274076 ] +] From db160989814e9a1dda15e8e3722bdd8add186cf0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 23 Nov 2021 17:43:37 +0100 Subject: [PATCH 14/64] Fix Tests --- freqtrade/exchange/binance.py | 4 ++-- freqtrade/exchange/exchange.py | 2 +- tests/exchange/test_exchange.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index 0c891924f..7540fc98a 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -212,9 +212,9 @@ class Binance(Exchange): """ if is_new_pair: x = await self._async_get_candle_history(pair, timeframe, 0, candle_type) - if x and x[2] and x[2][0] and x[2][0][0] > since_ms: + if x and x[3] and x[3][0] and x[3][0][0] > since_ms: # Set starting date to first available candle. - since_ms = x[2][0][0] + since_ms = x[3][0][0] logger.info(f"Candle-data for {pair} available starting with " f"{arrow.get(since_ms // 1000).isoformat()}.") diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 7201c025e..af1e429e3 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1419,7 +1419,7 @@ class Exchange: pair, timeframe, since_ms=since_ms, candle_type=candle_type)) else: logger.debug( - "Using cached candle (OHLCV) data for pair %s, timeframe %s ...", + "Using cached candle (OHLCV) data for pair %s, timeframe %s, candleType %s ...", pair, timeframe, candle_type ) cached_pairs.append((pair, timeframe, candle_type)) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index d8f4be660..39c367b7e 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1748,13 +1748,13 @@ def test_refresh_latest_ohlcv(mocker, default_conf, caplog) -> None: assert exchange._api_async.fetch_ohlcv.call_count == 0 assert log_has(f"Using cached candle (OHLCV) data for pair {pairs[0][0]}, " - f"timeframe {pairs[0][1]} ...", + f"timeframe {pairs[0][1]}, candleType ...", caplog) res = exchange.refresh_latest_ohlcv( [('IOTA/ETH', '5m', ''), ('XRP/ETH', '5m', ''), ('XRP/ETH', '1d', '')], cache=False ) - assert len(res) == 4 + assert len(res) == 3 @pytest.mark.asyncio From 0d30b32fdd0886dfb2b409bebd8952b1bf79e1dd Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Sat, 27 Nov 2021 02:44:06 -0600 Subject: [PATCH 15/64] Formatting changes --- tests/rpc/test_rpc_apiserver.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index bdbb2e833..a7c2357e5 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -704,9 +704,8 @@ def test_api_edge_disabled(botclient, mocker, ticker, fee, markets): assert rc.json() == {"error": "Error querying /api/v1/edge: Edge is not enabled."} -@pytest.mark.parametrize( - 'is_short,expected', - [( +@pytest.mark.parametrize('is_short,expected', [ + ( True, {'best_pair': 'ETC/BTC', 'best_rate': -0.5, 'best_pair_profit_ratio': -0.005, 'profit_all_coin': 43.61269123, @@ -719,7 +718,7 @@ def test_api_edge_disabled(botclient, mocker, ticker, fee, markets): 'profit_closed_percent_sum': -1.5, 'profit_closed_ratio': -6.739057628404269e-06, 'profit_closed_percent': -0.0, 'winning_trades': 0, 'losing_trades': 2} ), - ( + ( False, {'best_pair': 'XRP/BTC', 'best_rate': 1.0, 'best_pair_profit_ratio': 0.01, 'profit_all_coin': -44.0631579, @@ -732,7 +731,7 @@ def test_api_edge_disabled(botclient, mocker, ticker, fee, markets): 'profit_closed_percent_sum': 1.5, 'profit_closed_ratio': 7.391275897987988e-07, 'profit_closed_percent': 0.0, 'winning_trades': 2, 'losing_trades': 0} ), - ( + ( None, {'best_pair': 'XRP/BTC', 'best_rate': 1.0, 'best_pair_profit_ratio': 0.01, 'profit_all_coin': -14.43790415, @@ -745,7 +744,7 @@ def test_api_edge_disabled(botclient, mocker, ticker, fee, markets): 'profit_closed_percent_sum': 0.5, 'profit_closed_ratio': -5.429078808526421e-06, 'profit_closed_percent': -0.0, 'winning_trades': 1, 'losing_trades': 1} ) - ]) +]) def test_api_profit(botclient, mocker, ticker, fee, markets, is_short, expected): ftbot, client = botclient patch_get_signal(ftbot) From 8761649fd779fdba4b1ea269d5f3a672ac9fb679 Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Sat, 27 Nov 2021 02:55:42 -0600 Subject: [PATCH 16/64] Added candle_type in doc strings --- freqtrade/data/dataprovider.py | 5 +++++ freqtrade/data/history/hdf5datahandler.py | 5 +++++ freqtrade/data/history/history_utils.py | 4 ++++ freqtrade/data/history/idatahandler.py | 6 ++++++ freqtrade/data/history/jsondatahandler.py | 5 +++++ freqtrade/exchange/binance.py | 2 +- freqtrade/exchange/exchange.py | 8 +++++--- freqtrade/strategy/informative_decorator.py | 1 + 8 files changed, 32 insertions(+), 4 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 3c8ee4b24..d25c1b553 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -55,6 +55,7 @@ class DataProvider: :param pair: pair to get the data for :param timeframe: Timeframe to get data for :param dataframe: analyzed dataframe + :param candle_type: '', mark, index, premiumIndex, or funding_rate """ self.__cached_pairs[(pair, timeframe, candle_type)] = ( dataframe, datetime.now(timezone.utc)) @@ -75,6 +76,7 @@ class DataProvider: Get stored historical candle (OHLCV) data :param pair: pair to get the data for :param timeframe: timeframe to get data for + :param candle_type: '', mark, index, premiumIndex, or funding_rate """ saved_pair = (pair, str(timeframe), candle_type) if saved_pair not in self.__cached_pairs_backtesting: @@ -106,6 +108,7 @@ class DataProvider: :param pair: pair to get the data for :param timeframe: timeframe to get data for :return: Dataframe for this pair + :param candle_type: '', mark, index, premiumIndex, or funding_rate """ if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE): # Get live OHLCV data. @@ -128,6 +131,7 @@ class DataProvider: and the last 1000 candles (up to the time evaluated at this moment) in all other modes. :param pair: pair to get the data for :param timeframe: timeframe to get data for + :param candle_type: '', mark, index, premiumIndex, or funding_rate :return: Tuple of (Analyzed Dataframe, lastrefreshed) for the requested pair / timeframe combination. Returns empty dataframe and Epoch 0 (1970-01-01) if no dataframe was cached. @@ -212,6 +216,7 @@ class DataProvider: Please use the `available_pairs` method to verify which pairs are currently cached. :param pair: pair to get the data for :param timeframe: Timeframe to get data for + :param candle_type: '', mark, index, premiumIndex, or funding_rate :param copy: copy dataframe before returning if True. Use False only for read-only operations (where the dataframe is not modified) """ diff --git a/freqtrade/data/history/hdf5datahandler.py b/freqtrade/data/history/hdf5datahandler.py index 204229d2b..6e3f68f2e 100644 --- a/freqtrade/data/history/hdf5datahandler.py +++ b/freqtrade/data/history/hdf5datahandler.py @@ -49,6 +49,7 @@ class HDF5DataHandler(IDataHandler): for the specified timeframe :param datadir: Directory to search for ohlcv files :param timeframe: Timeframe to search pairs for + :param candle_type: '', mark, index, premiumIndex, or funding_rate :return: List of Pairs """ @@ -74,6 +75,7 @@ class HDF5DataHandler(IDataHandler): :param pair: Pair - used to generate filename :param timeframe: Timeframe - used to generate filename :param data: Dataframe containing OHLCV data + :param candle_type: '', mark, index, premiumIndex, or funding_rate :return: None """ key = self._pair_ohlcv_key(pair, timeframe) @@ -98,6 +100,7 @@ class HDF5DataHandler(IDataHandler): :param timerange: Limit data to be loaded to this timerange. Optionally implemented by subclasses to avoid loading all data where possible. + :param candle_type: '', mark, index, premiumIndex, or funding_rate :return: DataFrame with ohlcv data, or empty DataFrame """ key = self._pair_ohlcv_key(pair, timeframe) @@ -130,6 +133,7 @@ class HDF5DataHandler(IDataHandler): Remove data for this pair :param pair: Delete data for this pair. :param timeframe: Timeframe (e.g. "5m") + :param candle_type: '', mark, index, premiumIndex, or funding_rate :return: True when deleted, false if file did not exist. """ filename = self._pair_data_filename(self._datadir, pair, timeframe, candle_type) @@ -150,6 +154,7 @@ class HDF5DataHandler(IDataHandler): :param pair: Pair :param timeframe: Timeframe this ohlcv data is for :param data: Data to append. + :param candle_type: '', mark, index, premiumIndex, or funding_rate """ raise NotImplementedError() diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index cfff74d93..7b0c727c8 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -44,6 +44,7 @@ def load_pair_history(pair: str, :param startup_candles: Additional candles to load at the start of the period :param data_handler: Initialized data-handler to use. Will be initialized from data_format if not set + :param candle_type: '', mark, index, premiumIndex, or funding_rate :return: DataFrame with ohlcv data, or empty DataFrame """ data_handler = get_datahandler(datadir, data_format, data_handler) @@ -79,6 +80,7 @@ def load_data(datadir: Path, :param startup_candles: Additional candles to load at the start of the period :param fail_without_data: Raise OperationalException if no data is found. :param data_format: Data format which should be used. Defaults to json + :param candle_type: '', mark, index, premiumIndex, or funding_rate :return: dict(:) """ result: Dict[str, DataFrame] = {} @@ -120,6 +122,7 @@ def refresh_data(datadir: Path, :param exchange: Exchange object :param data_format: dataformat to use :param timerange: Limit data to be loaded to this timerange + :param candle_type: '', mark, index, premiumIndex, or funding_rate """ data_handler = get_datahandler(datadir, data_format) for idx, pair in enumerate(pairs): @@ -186,6 +189,7 @@ def _download_pair_history(pair: str, *, :param pair: pair to download :param timeframe: Timeframe (e.g "5m") :param timerange: range of time to download + :param candle_type: '', mark, index, premiumIndex, or funding_rate :return: bool with success state """ data_handler = get_datahandler(datadir, data_handler=data_handler) diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index dba9ff4bd..d3ac0232d 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -46,6 +46,7 @@ class IDataHandler(ABC): for the specified timeframe :param datadir: Directory to search for ohlcv files :param timeframe: Timeframe to search pairs for + :param candle_type: '', mark, index, premiumIndex, or funding_rate :return: List of Pairs """ @@ -62,6 +63,7 @@ class IDataHandler(ABC): :param pair: Pair - used to generate filename :param timeframe: Timeframe - used to generate filename :param data: Dataframe containing OHLCV data + :param candle_type: '', mark, index, premiumIndex, or funding_rate :return: None """ @@ -79,6 +81,7 @@ class IDataHandler(ABC): :param timerange: Limit data to be loaded to this timerange. Optionally implemented by subclasses to avoid loading all data where possible. + :param candle_type: '', mark, index, premiumIndex, or funding_rate :return: DataFrame with ohlcv data, or empty DataFrame """ @@ -88,6 +91,7 @@ class IDataHandler(ABC): Remove data for this pair :param pair: Delete data for this pair. :param timeframe: Timeframe (e.g. "5m") + :param candle_type: '', mark, index, premiumIndex, or funding_rate :return: True when deleted, false if file did not exist. """ @@ -104,6 +108,7 @@ class IDataHandler(ABC): :param pair: Pair :param timeframe: Timeframe this ohlcv data is for :param data: Data to append. + :param candle_type: '', mark, index, premiumIndex, or funding_rate """ @abstractclassmethod @@ -177,6 +182,7 @@ class IDataHandler(ABC): :param drop_incomplete: Drop last candle assuming it may be incomplete. :param startup_candles: Additional candles to load at the start of the period :param warn_no_data: Log a warning message when no data is found + :param candle_type: '', mark, index, premiumIndex, or funding_rate :return: DataFrame with ohlcv data, or empty DataFrame """ # Fix startup period diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index 1c430f542..235b5c130 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -49,6 +49,7 @@ class JsonDataHandler(IDataHandler): for the specified timeframe :param datadir: Directory to search for ohlcv files :param timeframe: Timeframe to search pairs for + :param candle_type: '', mark, index, premiumIndex, or funding_rate :return: List of Pairs """ if candle_type: @@ -75,6 +76,7 @@ class JsonDataHandler(IDataHandler): :param pair: Pair - used to generate filename :param timeframe: Timeframe - used to generate filename :param data: Dataframe containing OHLCV data + :param candle_type: '', mark, index, premiumIndex, or funding_rate :return: None """ filename = self._pair_data_filename( @@ -105,6 +107,7 @@ class JsonDataHandler(IDataHandler): :param timerange: Limit data to be loaded to this timerange. Optionally implemented by subclasses to avoid loading all data where possible. + :param candle_type: '', mark, index, premiumIndex, or funding_rate :return: DataFrame with ohlcv data, or empty DataFrame """ filename = self._pair_data_filename(self._datadir, pair, timeframe, candle_type=candle_type) @@ -129,6 +132,7 @@ class JsonDataHandler(IDataHandler): Remove data for this pair :param pair: Delete data for this pair. :param timeframe: Timeframe (e.g. "5m") + :param candle_type: '', mark, index, premiumIndex, or funding_rate :return: True when deleted, false if file did not exist. """ filename = self._pair_data_filename(self._datadir, pair, timeframe, candle_type=candle_type) @@ -149,6 +153,7 @@ class JsonDataHandler(IDataHandler): :param pair: Pair :param timeframe: Timeframe this ohlcv data is for :param data: Data to append. + :param candle_type: '', mark, index, premiumIndex, or funding_rate """ raise NotImplementedError() diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index f82c7dca9..ad18efcf5 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -204,7 +204,7 @@ class Binance(Exchange): """ Overwrite to introduce "fast new pair" functionality by detecting the pair's listing date Does not work for other exchanges, which don't return the earliest data when called with "0" - :param candle_type: "mark" if retrieving the mark price cnadles + :param candle_type: '', mark, index, premiumIndex, or funding_rate """ if is_new_pair: x = await self._async_get_candle_history(pair, timeframe, 0, candle_type) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index be503416c..91c57b788 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1319,6 +1319,7 @@ class Exchange: :param pair: Pair to download :param timeframe: Timeframe to get data for :param since_ms: Timestamp in milliseconds to get history from + :param candle_type: '', mark, index, premiumIndex, or funding_rate :return: List with candle (OHLCV) data """ data: List @@ -1336,6 +1337,7 @@ class Exchange: :param pair: Pair to download :param timeframe: Timeframe to get data for :param since_ms: Timestamp in milliseconds to get history from + :param candle_type: '', mark, index, premiumIndex, or funding_rate :return: OHLCV DataFrame """ ticks = self.get_historic_ohlcv(pair, timeframe, since_ms=since_ms, candle_type=candle_type) @@ -1350,6 +1352,7 @@ class Exchange: """ Download historic ohlcv :param is_new_pair: used by binance subclass to allow "fast" new pair downloading + :param candle_type: '', mark, index, premiumIndex, or funding_rate """ one_call = timeframe_to_msecs(timeframe) * self.ohlcv_candle_limit(timeframe) @@ -1393,6 +1396,7 @@ class Exchange: :param pair_list: List of 2 element tuples containing pair, interval to refresh :param since_ms: time since when to download, in milliseconds :param cache: Assign result to _klines. Usefull for one-off downloads like for pairlists + :param candle_type: '', mark, index, premiumIndex, or funding_rate :return: Dict of [{(pair, timeframe): Dataframe}] """ logger.debug("Refreshing candle (OHLCV) data for %d pairs", len(pair_list)) @@ -1480,9 +1484,7 @@ class Exchange: ) -> Tuple[str, str, str, List]: """ Asynchronously get candle history data using fetch_ohlcv - :param candle_type: - "mark" if retrieving the mark price cnadles - "index" for index price candles + :param candle_type: '', mark, index, premiumIndex, or funding_rate returns tuple: (pair, timeframe, ohlcv_list) """ try: diff --git a/freqtrade/strategy/informative_decorator.py b/freqtrade/strategy/informative_decorator.py index db5a70f72..1507f09ab 100644 --- a/freqtrade/strategy/informative_decorator.py +++ b/freqtrade/strategy/informative_decorator.py @@ -47,6 +47,7 @@ def informative(timeframe: str, asset: str = '', * {column} - name of dataframe column. * {timeframe} - timeframe of informative dataframe. :param ffill: ffill dataframe after merging informative pair. + :param candle_type: '', mark, index, premiumIndex, or funding_rate """ _asset = asset _timeframe = timeframe From 392128013fcf91fab6f2dbe4103261da758fa1ee Mon Sep 17 00:00:00 2001 From: Sam Germain Date: Sat, 27 Nov 2021 03:11:44 -0600 Subject: [PATCH 17/64] Updated ohlcv_get_available_data to recognize swap and futures pairs --- freqtrade/data/history/hdf5datahandler.py | 2 +- freqtrade/data/history/jsondatahandler.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/data/history/hdf5datahandler.py b/freqtrade/data/history/hdf5datahandler.py index 6e3f68f2e..16b1e5030 100644 --- a/freqtrade/data/history/hdf5datahandler.py +++ b/freqtrade/data/history/hdf5datahandler.py @@ -30,7 +30,7 @@ class HDF5DataHandler(IDataHandler): """ _tmp = [ re.search( - r'^([a-zA-Z_]+)\-(\d+\S)\-?([a-zA-Z_]*)?(?=.h5)', + r'^([a-zA-Z_]+(\:[a-zA-Z]{2,}(\-[0-9]{2,})?)?)\-(\d+\S)\-?([a-zA-Z_]*)?(?=.h5)', p.name ) for p in datadir.glob("*.h5") ] diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index 235b5c130..e1c5f5232 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -31,7 +31,7 @@ class JsonDataHandler(IDataHandler): """ _tmp = [ re.search( - r'^([a-zA-Z_]+)\-(\d+\S)\-?([a-zA-Z_]*)?(?=.json)', + r'^([a-zA-Z_]+(\:[a-zA-Z]{2,}(\-[0-9]{2,})?)?)\-(\d+\S)\-?([a-zA-Z_]*)?(?=.json)', p.name ) for p in datadir.glob(f"*.{cls._get_file_extension()}")] return [(match[1].replace('_', '/'), match[2], match[3]) for match in _tmp From 504efbd6d4c093bd8aa09eccdc43901a05f30255 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 27 Nov 2021 16:36:59 +0100 Subject: [PATCH 18/64] Add futures argument to download-data command --- freqtrade/commands/arguments.py | 2 +- freqtrade/commands/data_commands.py | 2 ++ tests/commands/test_commands.py | 12 ++++++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 025fee66c..3ebad54dc 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -69,7 +69,7 @@ ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs"] ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "new_pairs_days", "include_inactive", "timerange", "download_trades", "exchange", "timeframes", - "erase", "dataformat_ohlcv", "dataformat_trades"] + "erase", "dataformat_ohlcv", "dataformat_trades", "trading_mode"] ARGS_PLOT_DATAFRAME = ["pairs", "indicators1", "indicators2", "plot_limit", "db_url", "trade_source", "export", "exportfilename", diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index f55a857c4..2737e1013 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -64,6 +64,8 @@ def start_download_data(args: Dict[str, Any]) -> None: try: if config.get('download_trades'): + if config.get('trading_mode') == 'futures': + raise OperationalException("Trade download not supported for futures.") pairs_not_available = refresh_backtest_trades_data( exchange, pairs=expanded_pairs, datadir=config['datadir'], timerange=timerange, new_pairs_days=config['new_pairs_days'], diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 384254b74..1981e9413 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -814,6 +814,18 @@ def test_download_data_trades(mocker, caplog): assert dl_mock.call_args[1]['timerange'].starttype == "date" assert dl_mock.call_count == 1 assert convert_mock.call_count == 1 + args = [ + "download-data", + "--exchange", "kraken", + "--pairs", "ETH/BTC", "XRP/BTC", + "--days", "20", + "--trading-mode", "futures", + "--dl-trades" + ] + with pytest.raises(OperationalException, + match="Trade download not supported for futures."): + + start_download_data(get_args(args)) def test_start_convert_trades(mocker, caplog): From 107e124f608567d85e844411b6e00bbdeb3109e5 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 27 Nov 2021 17:00:06 +0100 Subject: [PATCH 19/64] Fix bug in exchange causing candles not to download --- freqtrade/exchange/exchange.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 91c57b788..635270a24 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1494,9 +1494,9 @@ class Exchange: "Fetching pair %s, interval %s, since %s %s...", pair, timeframe, since_ms, s ) - params = self._ft_has.get('ohlcv_params', {}) + params = deepcopy(self._ft_has.get('ohlcv_params', {})) if candle_type: - params = params.update({'price': candle_type}) + params.update({'price': candle_type}) data = await self._api_async.fetch_ohlcv(pair, timeframe=timeframe, since=since_ms, limit=self.ohlcv_candle_limit(timeframe), From c096c7f5cb504039fd5ee587d1160c9aa4e61a38 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 28 Nov 2021 14:33:46 +0100 Subject: [PATCH 20/64] Add explicit tests for ohlcv regex --- freqtrade/data/history/hdf5datahandler.py | 3 +-- freqtrade/data/history/idatahandler.py | 2 ++ freqtrade/data/history/jsondatahandler.py | 3 +-- tests/data/test_history.py | 15 +++++++++++++++ 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/freqtrade/data/history/hdf5datahandler.py b/freqtrade/data/history/hdf5datahandler.py index 16b1e5030..4cc2dbcc9 100644 --- a/freqtrade/data/history/hdf5datahandler.py +++ b/freqtrade/data/history/hdf5datahandler.py @@ -30,8 +30,7 @@ class HDF5DataHandler(IDataHandler): """ _tmp = [ re.search( - r'^([a-zA-Z_]+(\:[a-zA-Z]{2,}(\-[0-9]{2,})?)?)\-(\d+\S)\-?([a-zA-Z_]*)?(?=.h5)', - p.name + cls._OHLCV_REGEX, p.name ) for p in datadir.glob("*.h5") ] return [(match[1].replace('_', '/'), match[2], match[3]) for match in _tmp diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index d3ac0232d..04236e5a2 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -23,6 +23,8 @@ logger = logging.getLogger(__name__) class IDataHandler(ABC): + _OHLCV_REGEX = r'^([a-zA-Z_]+)\-(\d+\S+)(?=\.)' + def __init__(self, datadir: Path) -> None: self._datadir = datadir diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index e1c5f5232..499547523 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -31,8 +31,7 @@ class JsonDataHandler(IDataHandler): """ _tmp = [ re.search( - r'^([a-zA-Z_]+(\:[a-zA-Z]{2,}(\-[0-9]{2,})?)?)\-(\d+\S)\-?([a-zA-Z_]*)?(?=.json)', - p.name + cls._OHLCV_REGEX, p.name ) for p in datadir.glob(f"*.{cls._get_file_extension()}")] return [(match[1].replace('_', '/'), match[2], match[3]) for match in _tmp if match and len(match.groups()) > 1] diff --git a/tests/data/test_history.py b/tests/data/test_history.py index ea37ea268..246e0ea84 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -1,6 +1,7 @@ # pragma pylint: disable=missing-docstring, protected-access, C0103 import json +import re import uuid from pathlib import Path from shutil import copyfile @@ -669,6 +670,20 @@ def test_datahandler_ohlcv_get_pairs(testdatadir): # assert set(pairs) == {'UNITTEST/BTC'} +@pytest.mark.parametrize('filename,pair,timeframe', [ + ('XMR_BTC-5m.json', 'XMR_BTC', '5m'), + ('XMR_USDT-1h.h5', 'XMR_USDT', '1h'), + ('BTC_USDT-2h.jsongz', 'BTC_USDT', '2h'), +]) +def test_datahandler_ohlcv_regex(filename, pair, timeframe): + regex = JsonDataHandler._OHLCV_REGEX + + match = re.search(regex, filename) + assert len(match.groups()) > 1 + assert match[1] == pair + assert match[2] == timeframe + + def test_datahandler_ohlcv_get_available_data(testdatadir): paircombs = JsonDataHandler.ohlcv_get_available_data(testdatadir) # Convert to set to avoid failures due to sorting From 8d70672beeb512bf9d78b68240471cc185a2cf82 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 28 Nov 2021 14:37:54 +0100 Subject: [PATCH 21/64] Enhance Regex to work for mark candles --- freqtrade/data/history/idatahandler.py | 2 +- tests/data/test_history.py | 15 ++++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index 04236e5a2..46fc5fcfe 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -23,7 +23,7 @@ logger = logging.getLogger(__name__) class IDataHandler(ABC): - _OHLCV_REGEX = r'^([a-zA-Z_]+)\-(\d+\S+)(?=\.)' + _OHLCV_REGEX = r'^([a-zA-Z_]+)\-(\d+\S)\-?([a-zA-Z_]*)?(?=\.)' def __init__(self, datadir: Path) -> None: self._datadir = datadir diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 246e0ea84..af78a09af 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -670,18 +670,23 @@ def test_datahandler_ohlcv_get_pairs(testdatadir): # assert set(pairs) == {'UNITTEST/BTC'} -@pytest.mark.parametrize('filename,pair,timeframe', [ - ('XMR_BTC-5m.json', 'XMR_BTC', '5m'), - ('XMR_USDT-1h.h5', 'XMR_USDT', '1h'), - ('BTC_USDT-2h.jsongz', 'BTC_USDT', '2h'), +@pytest.mark.parametrize('filename,pair,timeframe,candletype', [ + ('XMR_BTC-5m.json', 'XMR_BTC', '5m', ''), + ('XMR_USDT-1h.h5', 'XMR_USDT', '1h', ''), + ('BTC_USDT-2h.jsongz', 'BTC_USDT', '2h', ''), + ('BTC_USDT-2h-mark.jsongz', 'BTC_USDT', '2h', 'mark'), + ('XMR_USDT-1h-mark.h5', 'XMR_USDT', '1h', 'mark'), + ('XMR_USDT-1h-random.h5', 'XMR_USDT', '1h', 'random'), + ('XMR_USDT_USDT-1h-mark.h5', 'XMR_USDT_USDT', '1h', 'mark'), ]) -def test_datahandler_ohlcv_regex(filename, pair, timeframe): +def test_datahandler_ohlcv_regex(filename, pair, timeframe, candletype): regex = JsonDataHandler._OHLCV_REGEX match = re.search(regex, filename) assert len(match.groups()) > 1 assert match[1] == pair assert match[2] == timeframe + assert match[3] == candletype def test_datahandler_ohlcv_get_available_data(testdatadir): From 7faa7539b44922ef03bfd04ce2c67a3345f29029 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 28 Nov 2021 15:03:55 +0100 Subject: [PATCH 22/64] Further enhance pair retrieval --- freqtrade/data/history/hdf5datahandler.py | 2 +- freqtrade/data/history/idatahandler.py | 13 ++++++++++++- freqtrade/data/history/jsondatahandler.py | 2 +- tests/data/test_history.py | 16 ++++++++++++++++ 4 files changed, 30 insertions(+), 3 deletions(-) diff --git a/freqtrade/data/history/hdf5datahandler.py b/freqtrade/data/history/hdf5datahandler.py index 4cc2dbcc9..34a6babb3 100644 --- a/freqtrade/data/history/hdf5datahandler.py +++ b/freqtrade/data/history/hdf5datahandler.py @@ -33,7 +33,7 @@ class HDF5DataHandler(IDataHandler): cls._OHLCV_REGEX, p.name ) for p in datadir.glob("*.h5") ] - return [(match[1].replace('_', '/'), match[2], match[3]) for match in _tmp + return [(cls.rebuild_pair_from_filename(match[1]), match[2], match[3]) for match in _tmp if match and len(match.groups()) > 1] @classmethod diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index 46fc5fcfe..787614c51 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -4,6 +4,7 @@ It's subclasses handle and storing data from disk. """ import logging +import re from abc import ABC, abstractclassmethod, abstractmethod from copy import deepcopy from datetime import datetime, timezone @@ -23,7 +24,7 @@ logger = logging.getLogger(__name__) class IDataHandler(ABC): - _OHLCV_REGEX = r'^([a-zA-Z_]+)\-(\d+\S)\-?([a-zA-Z_]*)?(?=\.)' + _OHLCV_REGEX = r'^([a-zA-Z_-]+)\-(\d+\S)\-?([a-zA-Z_]*)?(?=\.)' def __init__(self, datadir: Path) -> None: self._datadir = datadir @@ -166,6 +167,16 @@ class IDataHandler(ABC): """ return trades_remove_duplicates(self._trades_load(pair, timerange=timerange)) + @staticmethod + def rebuild_pair_from_filename(pair: str) -> str: + """ + Rebuild pair name from filename + Assumes a asset name of max. 7 length to also support BTC-PERP and BTC-PERP:USD names. + """ + res = re.sub(r'^(.{1,7})(_)', r'\g<1>/', pair, 1) + res = re.sub('_', ':', res, 1) + return res + def ohlcv_load(self, pair, timeframe: str, timerange: Optional[TimeRange] = None, fill_missing: bool = True, diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index 499547523..0cd3bb33f 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -33,7 +33,7 @@ class JsonDataHandler(IDataHandler): re.search( cls._OHLCV_REGEX, p.name ) for p in datadir.glob(f"*.{cls._get_file_extension()}")] - return [(match[1].replace('_', '/'), match[2], match[3]) for match in _tmp + return [(cls.rebuild_pair_from_filename(match[1]), match[2], match[3]) for match in _tmp if match and len(match.groups()) > 1] @classmethod diff --git a/tests/data/test_history.py b/tests/data/test_history.py index af78a09af..8557d05eb 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -673,10 +673,12 @@ def test_datahandler_ohlcv_get_pairs(testdatadir): @pytest.mark.parametrize('filename,pair,timeframe,candletype', [ ('XMR_BTC-5m.json', 'XMR_BTC', '5m', ''), ('XMR_USDT-1h.h5', 'XMR_USDT', '1h', ''), + ('BTC-PERP-1h.h5', 'BTC-PERP', '1h', ''), ('BTC_USDT-2h.jsongz', 'BTC_USDT', '2h', ''), ('BTC_USDT-2h-mark.jsongz', 'BTC_USDT', '2h', 'mark'), ('XMR_USDT-1h-mark.h5', 'XMR_USDT', '1h', 'mark'), ('XMR_USDT-1h-random.h5', 'XMR_USDT', '1h', 'random'), + ('BTC-PERP-1h-index.h5', 'BTC-PERP', '1h', 'index'), ('XMR_USDT_USDT-1h-mark.h5', 'XMR_USDT_USDT', '1h', 'mark'), ]) def test_datahandler_ohlcv_regex(filename, pair, timeframe, candletype): @@ -689,6 +691,20 @@ def test_datahandler_ohlcv_regex(filename, pair, timeframe, candletype): assert match[3] == candletype +@pytest.mark.parametrize('input,expected', [ + ('XMR_USDT', 'XMR/USDT'), + ('BTC_USDT', 'BTC/USDT'), + ('USDT_BUSD', 'USDT/BUSD'), + ('BTC_USDT_USDT', 'BTC/USDT:USDT'), # Futures + ('XRP_USDT_USDT', 'XRP/USDT:USDT'), # futures + ('BTC-PERP', 'BTC-PERP'), + ('BTC-PERP_USDT', 'BTC-PERP:USDT'), # potential FTX case +]) +def test_rebuild_pair_from_filename(input, expected): + + assert IDataHandler.rebuild_pair_from_filename(input) == expected + + def test_datahandler_ohlcv_get_available_data(testdatadir): paircombs = JsonDataHandler.ohlcv_get_available_data(testdatadir) # Convert to set to avoid failures due to sorting From 0d1324718c319f9a1414736aa9ada393f6ff339f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 28 Nov 2021 15:08:02 +0100 Subject: [PATCH 23/64] Don't replace "-" when writing pair files --- freqtrade/misc.py | 2 +- tests/test_misc.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/freqtrade/misc.py b/freqtrade/misc.py index 6f439866b..7c83c22bd 100644 --- a/freqtrade/misc.py +++ b/freqtrade/misc.py @@ -109,7 +109,7 @@ def file_load_json(file): def pair_to_filename(pair: str) -> str: - for ch in ['/', '-', ' ', '.', '@', '$', '+', ':']: + for ch in ['/', ' ', '.', '@', '$', '+', ':']: pair = pair.replace(ch, '_') return pair diff --git a/tests/test_misc.py b/tests/test_misc.py index 221c7b712..e7e2e33a0 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -71,9 +71,10 @@ def test_file_load_json(mocker, testdatadir) -> None: ("ETHH20", 'ETHH20'), (".XBTBON2H", '_XBTBON2H'), ("ETHUSD.d", 'ETHUSD_d'), - ("ADA-0327", 'ADA_0327'), - ("BTC-USD-200110", 'BTC_USD_200110'), - ("F-AKRO/USDT", 'F_AKRO_USDT'), + ("ADA-0327", 'ADA-0327'), + ("BTC-USD-200110", 'BTC-USD-200110'), + ("BTC-PERP:USDT", 'BTC-PERP_USDT'), + ("F-AKRO/USDT", 'F-AKRO_USDT'), ("LC+/ETH", 'LC__ETH'), ("CMT@18/ETH", 'CMT_18_ETH'), ("LBTC:1022/SAI", 'LBTC_1022_SAI'), From 0d6c933935f47ea555e0e36ee07254d5f7dad90a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 28 Nov 2021 15:25:57 +0100 Subject: [PATCH 24/64] Improve and fix pair detection from available data --- freqtrade/data/history/idatahandler.py | 2 +- freqtrade/rpc/api_server/api_v1.py | 7 ++++++- tests/data/test_history.py | 1 + tests/rpc/test_rpc_apiserver.py | 6 +++--- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index 787614c51..1a8e31094 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -173,7 +173,7 @@ class IDataHandler(ABC): Rebuild pair name from filename Assumes a asset name of max. 7 length to also support BTC-PERP and BTC-PERP:USD names. """ - res = re.sub(r'^(.{1,7})(_)', r'\g<1>/', pair, 1) + res = re.sub(r'^(([A-Za-z]{1,10})|^([A-Za-z\-]{1,6}))(_)', r'\g<1>/', pair, 1) res = re.sub('_', ':', res, 1) return res diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 0467e4705..975064adf 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -247,7 +247,7 @@ def get_strategy(strategy: str, config=Depends(get_config)): @router.get('/available_pairs', response_model=AvailablePairs, tags=['candle data']) def list_available_pairs(timeframe: Optional[str] = None, stake_currency: Optional[str] = None, - config=Depends(get_config)): + candletype: Optional[str] = None, config=Depends(get_config)): dh = get_datahandler(config['datadir'], config.get('dataformat_ohlcv', None)) @@ -257,6 +257,11 @@ def list_available_pairs(timeframe: Optional[str] = None, stake_currency: Option pair_interval = [pair for pair in pair_interval if pair[1] == timeframe] if stake_currency: pair_interval = [pair for pair in pair_interval if pair[0].endswith(stake_currency)] + if candletype: + pair_interval = [pair for pair in pair_interval if pair[2] == candletype] + else: + pair_interval = [pair for pair in pair_interval if pair[2] == ''] + pair_interval = sorted(pair_interval, key=lambda x: x[0]) pairs = list({x[0] for x in pair_interval}) diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 8557d05eb..bf3c1a1de 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -699,6 +699,7 @@ def test_datahandler_ohlcv_regex(filename, pair, timeframe, candletype): ('XRP_USDT_USDT', 'XRP/USDT:USDT'), # futures ('BTC-PERP', 'BTC-PERP'), ('BTC-PERP_USDT', 'BTC-PERP:USDT'), # potential FTX case + ('UNITTEST_USDT', 'UNITTEST/USDT'), ]) def test_rebuild_pair_from_filename(input, expected): diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index e2da51ba1..994d29887 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1332,7 +1332,7 @@ def test_list_available_pairs(botclient): rc = client_get(client, f"{BASE_URI}/available_pairs") assert_response(rc) - assert rc.json()['length'] == 15 + assert rc.json()['length'] == 14 assert isinstance(rc.json()['pairs'], list) rc = client_get(client, f"{BASE_URI}/available_pairs?timeframe=5m") @@ -1352,11 +1352,11 @@ def test_list_available_pairs(botclient): assert len(rc.json()['pair_interval']) == 1 rc = client_get( - client, f"{BASE_URI}/available_pairs?stake_currency=USDT&timeframe=1h&type=mark") + client, f"{BASE_URI}/available_pairs?timeframe=1h&candletype=mark") assert_response(rc) assert rc.json()['length'] == 2 assert rc.json()['pairs'] == ['UNITTEST/USDT', 'XRP/USDT'] - assert len(rc.json()['pair_interval']) == 3 # TODO-lev: What is pair_interval? Should it be 3? + assert len(rc.json()['pair_interval']) == 2 def test_sysinfo(botclient): From c20157e64f5b34669a2aad2a8e6063090601f4f3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 28 Nov 2021 15:43:04 +0100 Subject: [PATCH 25/64] Add compatibility code for existing informative_pairs implementation --- freqtrade/strategy/interface.py | 2 ++ tests/strategy/strats/informative_decorator_strategy.py | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 8a4e50343..bcb0a93b4 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -422,6 +422,8 @@ class IStrategy(ABC, HyperStrategyMixin): Internal method which gathers all informative pairs (user or automatically defined). """ informative_pairs = self.informative_pairs() + # Compatibility code for 2 tuple informative pairs + informative_pairs = [(p[0], p[1], p[2] if len(p) > 2 else '') for p in informative_pairs] for inf_data, _ in self._ft_informative: if inf_data.asset: pair_tf = ( diff --git a/tests/strategy/strats/informative_decorator_strategy.py b/tests/strategy/strats/informative_decorator_strategy.py index 448b67956..91c4642fa 100644 --- a/tests/strategy/strats/informative_decorator_strategy.py +++ b/tests/strategy/strats/informative_decorator_strategy.py @@ -19,7 +19,8 @@ class InformativeDecoratorTest(IStrategy): startup_candle_count: int = 20 def informative_pairs(self): - return [('NEO/USDT', '5m', '')] + # Intentionally return 2 tuples, must be converted to 3 in compatibility code + return [('NEO/USDT', '5m')] def populate_buy_trend(self, dataframe: DataFrame, metadata: dict) -> DataFrame: dataframe['buy'] = 0 From cb4efa6d56c2e992f906653319fdafd5d544b11b Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 28 Nov 2021 15:53:13 +0100 Subject: [PATCH 26/64] Revert unnecessary formatting changes --- freqtrade/data/dataprovider.py | 12 ++++------ freqtrade/data/history/hdf5datahandler.py | 10 ++------ freqtrade/data/history/idatahandler.py | 29 ++++------------------- freqtrade/data/history/jsondatahandler.py | 14 ++--------- 4 files changed, 12 insertions(+), 53 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index d25c1b553..0e8031554 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -105,6 +105,8 @@ class DataProvider: """ Return pair candle (OHLCV) data, either live or cached historical -- depending on the runmode. + Only combinations in the pairlist or which have been specified as informative pairs + will be available. :param pair: pair to get the data for :param timeframe: timeframe to get data for :return: Dataframe for this pair @@ -120,23 +122,17 @@ class DataProvider: logger.warning(f"No data found for ({pair}, {timeframe}, {candle_type}).") return data - def get_analyzed_dataframe( - self, - pair: str, - timeframe: str, - candle_type: str = '' - ) -> Tuple[DataFrame, datetime]: + def get_analyzed_dataframe(self, pair: str, timeframe: str) -> Tuple[DataFrame, datetime]: """ Retrieve the analyzed dataframe. Returns the full dataframe in trade mode (live / dry), and the last 1000 candles (up to the time evaluated at this moment) in all other modes. :param pair: pair to get the data for :param timeframe: timeframe to get data for - :param candle_type: '', mark, index, premiumIndex, or funding_rate :return: Tuple of (Analyzed Dataframe, lastrefreshed) for the requested pair / timeframe combination. Returns empty dataframe and Epoch 0 (1970-01-01) if no dataframe was cached. """ - pair_key = (pair, timeframe, candle_type) + pair_key = (pair, timeframe, '') if pair_key in self.__cached_pairs: if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE): df, date = self.__cached_pairs[pair_key] diff --git a/freqtrade/data/history/hdf5datahandler.py b/freqtrade/data/history/hdf5datahandler.py index 34a6babb3..ebe55a87e 100644 --- a/freqtrade/data/history/hdf5datahandler.py +++ b/freqtrade/data/history/hdf5datahandler.py @@ -37,12 +37,7 @@ class HDF5DataHandler(IDataHandler): if match and len(match.groups()) > 1] @classmethod - def ohlcv_get_pairs( - cls, - datadir: Path, - timeframe: str, - candle_type: str = '' - ) -> List[str]: + def ohlcv_get_pairs(cls, datadir: Path, timeframe: str, candle_type: str = '') -> List[str]: """ Returns a list of all pairs with ohlcv data available in this datadir for the specified timeframe @@ -88,8 +83,7 @@ class HDF5DataHandler(IDataHandler): ds.close() def _ohlcv_load(self, pair: str, timeframe: str, - timerange: Optional[TimeRange] = None, - candle_type: str = '') -> pd.DataFrame: + timerange: Optional[TimeRange] = None, candle_type: str = '') -> pd.DataFrame: """ Internal method used to load data for one pair from disk. Implements the loading and conversion to a Pandas dataframe. diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index 1a8e31094..8181d774e 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -38,12 +38,7 @@ class IDataHandler(ABC): """ @abstractclassmethod - def ohlcv_get_pairs( - cls, - datadir: Path, - timeframe: str, - candle_type: str = '' - ) -> List[str]: + def ohlcv_get_pairs(cls, datadir: Path, timeframe: str, candle_type: str = '') -> List[str]: """ Returns a list of all pairs with ohlcv data available in this datadir for the specified timeframe @@ -217,12 +212,7 @@ class IDataHandler(ABC): if timerange_startup: self._validate_pairdata(pair, pairdf, timerange_startup) pairdf = trim_dataframe(pairdf, timerange_startup) - if self._check_empty_df( - pairdf, - pair, - timeframe, - warn_no_data - ): + if self._check_empty_df(pairdf, pair, timeframe, warn_no_data): return pairdf # incomplete candles should only be dropped if we didn't trim the end beforehand. @@ -234,13 +224,7 @@ class IDataHandler(ABC): self._check_empty_df(pairdf, pair, timeframe, warn_no_data) return pairdf - def _check_empty_df( - self, - pairdf: DataFrame, - pair: str, - timeframe: str, - warn_no_data: bool - ): + def _check_empty_df(self, pairdf: DataFrame, pair: str, timeframe: str, warn_no_data: bool): """ Warn on empty dataframe """ @@ -253,12 +237,7 @@ class IDataHandler(ABC): return True return False - def _validate_pairdata( - self, - pair, - pairdata: DataFrame, - timerange: TimeRange - ): + def _validate_pairdata(self, pair, pairdata: DataFrame, timerange: TimeRange): """ Validates pairdata for missing data at start end end and logs warnings. :param pairdata: Dataframe to validate diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index 0cd3bb33f..9d3acdcaf 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -37,12 +37,7 @@ class JsonDataHandler(IDataHandler): if match and len(match.groups()) > 1] @classmethod - def ohlcv_get_pairs( - cls, - datadir: Path, - timeframe: str, - candle_type: str = '' - ) -> List[str]: + def ohlcv_get_pairs(cls, datadir: Path, timeframe: str, candle_type: str = '') -> List[str]: """ Returns a list of all pairs with ohlcv data available in this datadir for the specified timeframe @@ -78,12 +73,7 @@ class JsonDataHandler(IDataHandler): :param candle_type: '', mark, index, premiumIndex, or funding_rate :return: None """ - filename = self._pair_data_filename( - self._datadir, - pair, - timeframe, - candle_type - ) + filename = self._pair_data_filename(self._datadir, pair, timeframe, candle_type) _data = data.copy() # Convert date to int _data['date'] = _data['date'].view(np.int64) // 1000 // 1000 From 134b129d9d88268886b1bee5c5fad073df515564 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 28 Nov 2021 19:14:58 +0100 Subject: [PATCH 27/64] get_analyzed_df does not need a "candle_type" argument --- tests/optimize/test_backtesting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index 2aa2af04d..0efc2a9dc 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -925,7 +925,7 @@ def test_backtest_multi_pair(default_conf, fee, mocker, tres, pair, testdatadir) removed_candles = len(data[pair]) - offset - backtesting.strategy.startup_candle_count assert len(backtesting.dataprovider.get_analyzed_dataframe(pair, '5m')[0]) == removed_candles assert len( - backtesting.dataprovider.get_analyzed_dataframe('NXT/BTC', '5m', '')[0] + backtesting.dataprovider.get_analyzed_dataframe('NXT/BTC', '5m')[0] ) == len(data['NXT/BTC']) - 1 - backtesting.strategy.startup_candle_count backtest_conf = { From 22cda87211ac6db6642d09dff14685a03c553620 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 2 Dec 2021 19:05:06 +0100 Subject: [PATCH 28/64] Update some tests after merge --- tests/exchange/test_exchange.py | 4 ++-- tests/optimize/test_backtesting.py | 2 +- tests/test_misc.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 4a1b319d3..8a0e6e598 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1760,8 +1760,8 @@ def test_refresh_latest_ohlcv(mocker, default_conf, caplog) -> None: # Test the same again, should NOT return from cache! exchange._api_async.fetch_ohlcv.reset_mock() - res = exchange.refresh_latest_ohlcv([('IOTA/ETH', '5m'), ('XRP/ETH', '5m'), ('XRP/ETH', '1d')], - cache=False) + res = exchange.refresh_latest_ohlcv( + [('IOTA/ETH', '5m', ''), ('XRP/ETH', '5m', ''), ('XRP/ETH', '1d', '')], cache=False) assert len(res) == 3 assert exchange._api_async.fetch_ohlcv.call_count == 3 diff --git a/tests/optimize/test_backtesting.py b/tests/optimize/test_backtesting.py index c428f7e47..18996c883 100644 --- a/tests/optimize/test_backtesting.py +++ b/tests/optimize/test_backtesting.py @@ -857,7 +857,7 @@ def test_backtest_alternate_buy_sell(default_conf, fee, mocker, testdatadir): results = result['results'] assert len(results) == 100 # Cached data should be 200 - analyzed_df = backtesting.dataprovider.get_analyzed_dataframe('UNITTEST/BTC', '1m', '')[0] + analyzed_df = backtesting.dataprovider.get_analyzed_dataframe('UNITTEST/BTC', '1m')[0] assert len(analyzed_df) == 200 # Expect last candle to be 1 below end date (as the last candle is assumed as "incomplete" # during backtesting) diff --git a/tests/test_misc.py b/tests/test_misc.py index a9b256d96..0d18117b6 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -69,7 +69,7 @@ def test_file_load_json(mocker, testdatadir) -> None: ("ETH/BTC", 'ETH_BTC'), ("ETH/USDT", 'ETH_USDT'), ("ETH/USDT:USDT", 'ETH_USDT_USDT'), # swap with USDT as settlement currency - ("ETH/USDT:USDT-210625", 'ETH_USDT_USDT_210625'), # expiring futures + ("ETH/USDT:USDT-210625", 'ETH_USDT_USDT-210625'), # expiring futures ("Fabric Token/ETH", 'Fabric_Token_ETH'), ("ETHH20", 'ETHH20'), (".XBTBON2H", '_XBTBON2H'), From 7baf11a497476f2b4b56d72f4ca6ac3ff9f37a91 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 3 Dec 2021 07:04:53 +0100 Subject: [PATCH 29/64] Futures candles should go into a subdirectory --- freqtrade/commands/arguments.py | 2 +- freqtrade/commands/data_commands.py | 3 ++- freqtrade/data/history/hdf5datahandler.py | 5 ++++- freqtrade/data/history/idatahandler.py | 13 ++++++++++++- freqtrade/data/history/jsondatahandler.py | 6 +++++- freqtrade/rpc/api_server/api_v1.py | 3 ++- 6 files changed, 26 insertions(+), 6 deletions(-) diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 3ebad54dc..711a2c4a2 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -65,7 +65,7 @@ ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes"] ARGS_CONVERT_TRADES = ["pairs", "timeframes", "exchange", "dataformat_ohlcv", "dataformat_trades"] -ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs"] +ARGS_LIST_DATA = ["exchange", "dataformat_ohlcv", "pairs", "trading_mode"] ARGS_DOWNLOAD_DATA = ["pairs", "pairs_file", "days", "new_pairs_days", "include_inactive", "timerange", "download_trades", "exchange", "timeframes", diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index 2737e1013..a75b4bfd2 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -156,7 +156,8 @@ def start_list_data(args: Dict[str, Any]) -> None: from freqtrade.data.history.idatahandler import get_datahandler dhc = get_datahandler(config['datadir'], config['dataformat_ohlcv']) - paircombs = dhc.ohlcv_get_available_data(config['datadir']) + # TODO-lev: trading-mode should be parsed at config level, and available as Enum in the config. + paircombs = dhc.ohlcv_get_available_data(config['datadir'], config.get('trading_mode', 'spot')) if args['pairs']: paircombs = [comb for comb in paircombs if comb[0] in args['pairs']] diff --git a/freqtrade/data/history/hdf5datahandler.py b/freqtrade/data/history/hdf5datahandler.py index 221758020..a390e887a 100644 --- a/freqtrade/data/history/hdf5datahandler.py +++ b/freqtrade/data/history/hdf5datahandler.py @@ -21,12 +21,15 @@ class HDF5DataHandler(IDataHandler): _columns = DEFAULT_DATAFRAME_COLUMNS @classmethod - def ohlcv_get_available_data(cls, datadir: Path) -> ListPairsWithTimeframes: + def ohlcv_get_available_data(cls, datadir: Path, trading_mode: str) -> ListPairsWithTimeframes: """ Returns a list of all pairs with ohlcv data available in this datadir :param datadir: Directory to search for ohlcv files + :param trading_mode: trading-mode to be used :return: List of Tuples of (pair, timeframe) """ + if trading_mode != 'spot': + datadir = datadir.joinpath('futures') _tmp = [ re.search( cls._OHLCV_REGEX, p.name diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index f616d143d..2d7df6ca5 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -38,10 +38,11 @@ class IDataHandler(ABC): raise NotImplementedError() @abstractclassmethod - def ohlcv_get_available_data(cls, datadir: Path) -> ListPairsWithTimeframes: + def ohlcv_get_available_data(cls, datadir: Path, trading_mode: str) -> ListPairsWithTimeframes: """ Returns a list of all pairs with ohlcv data available in this datadir :param datadir: Directory to search for ohlcv files + :param trading_mode: trading-mode to be used :return: List of Tuples of (pair, timeframe) """ @@ -178,6 +179,15 @@ class IDataHandler(ABC): """ return trades_remove_duplicates(self._trades_load(pair, timerange=timerange)) + @classmethod + def create_dir_if_needed(cls, datadir: Path): + """ + Creates datadir if necessary + should only create directories for "futures" mode at the moment. + """ + if not datadir.parent.is_dir(): + datadir.parent.mkdir() + @classmethod def _pair_data_filename( cls, @@ -188,6 +198,7 @@ class IDataHandler(ABC): ) -> Path: pair_s = misc.pair_to_filename(pair) if candle_type: + datadir = datadir.joinpath('futures') candle_type = f"-{candle_type}" filename = datadir.joinpath(f'{pair_s}-{timeframe}{candle_type}.{cls._get_file_extension()}') return filename diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index 32f6a0b1d..deacda338 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -23,12 +23,15 @@ class JsonDataHandler(IDataHandler): _columns = DEFAULT_DATAFRAME_COLUMNS @classmethod - def ohlcv_get_available_data(cls, datadir: Path) -> ListPairsWithTimeframes: + def ohlcv_get_available_data(cls, datadir: Path, trading_mode: str) -> ListPairsWithTimeframes: """ Returns a list of all pairs with ohlcv data available in this datadir :param datadir: Directory to search for ohlcv files + :param trading_mode: trading-mode to be used :return: List of Tuples of (pair, timeframe) """ + if trading_mode != 'spot': + datadir = datadir.joinpath('futures') _tmp = [ re.search( cls._OHLCV_REGEX, p.name @@ -74,6 +77,7 @@ class JsonDataHandler(IDataHandler): :return: None """ filename = self._pair_data_filename(self._datadir, pair, timeframe, candle_type) + self.create_dir_if_needed(filename) _data = data.copy() # Convert date to int _data['date'] = _data['date'].view(np.int64) // 1000 // 1000 diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 235cf6de3..f97d49e96 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -254,7 +254,8 @@ def list_available_pairs(timeframe: Optional[str] = None, stake_currency: Option dh = get_datahandler(config['datadir'], config.get('dataformat_ohlcv', None)) - pair_interval = dh.ohlcv_get_available_data(config['datadir']) + # TODO-lev: xmatt to decide: use candle-type or market mode for this endpoint?? + pair_interval = dh.ohlcv_get_available_data(config['datadir'], 'spot') if timeframe: pair_interval = [pair for pair in pair_interval if pair[1] == timeframe] From b578e3125570e49a0f0d1c75d332ac2f4b1eac87 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 3 Dec 2021 07:20:00 +0100 Subject: [PATCH 30/64] Align tests to have futures data in futures/ directory --- freqtrade/data/history/hdf5datahandler.py | 1 + freqtrade/data/history/idatahandler.py | 3 +- freqtrade/data/history/jsondatahandler.py | 1 + tests/commands/test_commands.py | 25 ++++++++++++--- tests/data/test_history.py | 32 +++++++++++-------- tests/rpc/test_rpc_apiserver.py | 2 +- .../{ => futures}/UNITTEST_USDT-1h-mark.json | 0 .../{ => futures}/XRP_USDT-1h-mark.json | 0 tests/testdata/{ => futures}/XRP_USDT-1h.json | 0 9 files changed, 45 insertions(+), 19 deletions(-) rename tests/testdata/{ => futures}/UNITTEST_USDT-1h-mark.json (100%) rename tests/testdata/{ => futures}/XRP_USDT-1h-mark.json (100%) rename tests/testdata/{ => futures}/XRP_USDT-1h.json (100%) diff --git a/freqtrade/data/history/hdf5datahandler.py b/freqtrade/data/history/hdf5datahandler.py index a390e887a..e8fcad70d 100644 --- a/freqtrade/data/history/hdf5datahandler.py +++ b/freqtrade/data/history/hdf5datahandler.py @@ -50,6 +50,7 @@ class HDF5DataHandler(IDataHandler): """ if candle_type: + datadir = datadir.joinpath('futures') candle_type = f"-{candle_type}" else: candle_type = "" diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index 2d7df6ca5..bb5e83c5d 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -200,7 +200,8 @@ class IDataHandler(ABC): if candle_type: datadir = datadir.joinpath('futures') candle_type = f"-{candle_type}" - filename = datadir.joinpath(f'{pair_s}-{timeframe}{candle_type}.{cls._get_file_extension()}') + filename = datadir.joinpath( + f'{pair_s}-{timeframe}{candle_type}.{cls._get_file_extension()}') return filename @classmethod diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index deacda338..00ba5d095 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -50,6 +50,7 @@ class JsonDataHandler(IDataHandler): :return: List of Pairs """ if candle_type: + datadir = datadir.joinpath('futures') candle_type = f"-{candle_type}" else: candle_type = "" diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 1981e9413..5e5cb3c2f 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -1338,10 +1338,9 @@ def test_start_list_data(testdatadir, capsys): pargs['config'] = None start_list_data(pargs) captured = capsys.readouterr() - assert "Found 20 pair / timeframe combinations." in captured.out - assert "\n| Pair | Timeframe | Type |\n" in captured.out - assert "\n| UNITTEST/BTC | 1m, 5m, 8m, 30m | |\n" in captured.out - assert "\n| UNITTEST/USDT | 1h | mark |\n" in captured.out + assert "Found 17 pair / timeframe combinations." in captured.out + assert "\n| Pair | Timeframe | Type |\n" in captured.out + assert "\n| UNITTEST/BTC | 1m, 5m, 8m, 30m | |\n" in captured.out args = [ "list-data", @@ -1360,6 +1359,24 @@ def test_start_list_data(testdatadir, capsys): assert "UNITTEST/BTC" not in captured.out assert "\n| XRP/ETH | 1m, 5m | |\n" in captured.out + args = [ + "list-data", + "--data-format-ohlcv", + "json", + "--trading-mode", "futures", + "--datadir", + str(testdatadir), + ] + pargs = get_args(args) + pargs['config'] = None + start_list_data(pargs) + captured = capsys.readouterr() + + assert "Found 3 pair / timeframe combinations." in captured.out + assert "\n| Pair | Timeframe | Type |\n" in captured.out + assert "\n| XRP/USDT | 1h | |\n" in captured.out + assert "\n| XRP/USDT | 1h | mark |\n" in captured.out + @pytest.mark.usefixtures("init_persistence") # TODO-lev: Short trades? diff --git a/tests/data/test_history.py b/tests/data/test_history.py index bf3c1a1de..a36f933c9 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -98,7 +98,7 @@ def test_load_data_1min_timeframe(ohlcv_history, mocker, caplog, testdatadir) -> def test_load_data_mark(ohlcv_history, mocker, caplog, testdatadir) -> None: mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=ohlcv_history) - file = testdatadir / 'UNITTEST_USDT-1h-mark.json' + file = testdatadir / 'futures/UNITTEST_USDT-1h-mark.json' load_data(datadir=testdatadir, timeframe='1h', pairs=['UNITTEST/BTC'], candle_type='mark') assert file.is_file() assert not log_has( @@ -163,8 +163,8 @@ def test_testdata_path(testdatadir) -> None: (".XBTBON2H", 'freqtrade/hello/world/_XBTBON2H-5m.json', ""), ("ETHUSD.d", 'freqtrade/hello/world/ETHUSD_d-5m.json', ""), ("ACC_OLD/BTC", 'freqtrade/hello/world/ACC_OLD_BTC-5m.json', ""), - ("ETH/BTC", 'freqtrade/hello/world/ETH_BTC-5m-mark.json', "mark"), - ("ACC_OLD/BTC", 'freqtrade/hello/world/ACC_OLD_BTC-5m-index.json', "index"), + ("ETH/BTC", 'freqtrade/hello/world/futures/ETH_BTC-5m-mark.json', "mark"), + ("ACC_OLD/BTC", 'freqtrade/hello/world/futures/ACC_OLD_BTC-5m-index.json', "index"), ]) def test_json_pair_data_filename(pair, expected_result, candle_type): fn = JsonDataHandler._pair_data_filename( @@ -254,9 +254,9 @@ def test_load_cached_data_for_updating(mocker, testdatadir) -> None: assert start_ts is None -@pytest.mark.parametrize('candle_type, file_tail', [ - ('mark', '-mark'), - ('', ''), +@pytest.mark.parametrize('candle_type,subdir,file_tail', [ + ('mark', 'futures/', '-mark'), + ('', '', ''), ]) def test_download_pair_history( ohlcv_history_list, @@ -264,15 +264,16 @@ def test_download_pair_history( default_conf, tmpdir, candle_type, + subdir, file_tail ) -> None: mocker.patch('freqtrade.exchange.Exchange.get_historic_ohlcv', return_value=ohlcv_history_list) exchange = get_patched_exchange(mocker, default_conf) tmpdir1 = Path(tmpdir) - file1_1 = tmpdir1 / f'MEME_BTC-1m{file_tail}.json' - file1_5 = tmpdir1 / f'MEME_BTC-5m{file_tail}.json' - file2_1 = tmpdir1 / f'CFI_BTC-1m{file_tail}.json' - file2_5 = tmpdir1 / f'CFI_BTC-5m{file_tail}.json' + file1_1 = tmpdir1 / f'{subdir}MEME_BTC-1m{file_tail}.json' + file1_5 = tmpdir1 / f'{subdir}MEME_BTC-5m{file_tail}.json' + file2_1 = tmpdir1 / f'{subdir}CFI_BTC-1m{file_tail}.json' + file2_5 = tmpdir1 / f'{subdir}CFI_BTC-5m{file_tail}.json' assert not file1_1.is_file() assert not file2_1.is_file() @@ -707,7 +708,7 @@ def test_rebuild_pair_from_filename(input, expected): def test_datahandler_ohlcv_get_available_data(testdatadir): - paircombs = JsonDataHandler.ohlcv_get_available_data(testdatadir) + paircombs = JsonDataHandler.ohlcv_get_available_data(testdatadir, 'spot') # Convert to set to avoid failures due to sorting assert set(paircombs) == { ('UNITTEST/BTC', '5m', ''), @@ -727,14 +728,19 @@ def test_datahandler_ohlcv_get_available_data(testdatadir): ('UNITTEST/BTC', '30m', ''), ('UNITTEST/BTC', '8m', ''), ('NOPAIR/XXX', '4m', ''), + } + + paircombs = JsonDataHandler.ohlcv_get_available_data(testdatadir, 'futures') + # Convert to set to avoid failures due to sorting + assert set(paircombs) == { ('UNITTEST/USDT', '1h', 'mark'), ('XRP/USDT', '1h', ''), ('XRP/USDT', '1h', 'mark'), } - paircombs = JsonGzDataHandler.ohlcv_get_available_data(testdatadir) + paircombs = JsonGzDataHandler.ohlcv_get_available_data(testdatadir, 'spot') assert set(paircombs) == {('UNITTEST/BTC', '8m', '')} - paircombs = HDF5DataHandler.ohlcv_get_available_data(testdatadir) + paircombs = HDF5DataHandler.ohlcv_get_available_data(testdatadir, 'spot') assert set(paircombs) == {('UNITTEST/BTC', '5m', '')} diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index a62b1f2c5..75455982e 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1334,7 +1334,7 @@ def test_list_available_pairs(botclient): rc = client_get(client, f"{BASE_URI}/available_pairs") assert_response(rc) - assert rc.json()['length'] == 14 + assert rc.json()['length'] == 13 assert isinstance(rc.json()['pairs'], list) rc = client_get(client, f"{BASE_URI}/available_pairs?timeframe=5m") diff --git a/tests/testdata/UNITTEST_USDT-1h-mark.json b/tests/testdata/futures/UNITTEST_USDT-1h-mark.json similarity index 100% rename from tests/testdata/UNITTEST_USDT-1h-mark.json rename to tests/testdata/futures/UNITTEST_USDT-1h-mark.json diff --git a/tests/testdata/XRP_USDT-1h-mark.json b/tests/testdata/futures/XRP_USDT-1h-mark.json similarity index 100% rename from tests/testdata/XRP_USDT-1h-mark.json rename to tests/testdata/futures/XRP_USDT-1h-mark.json diff --git a/tests/testdata/XRP_USDT-1h.json b/tests/testdata/futures/XRP_USDT-1h.json similarity index 100% rename from tests/testdata/XRP_USDT-1h.json rename to tests/testdata/futures/XRP_USDT-1h.json From e0e4369c8eb28a2ca7973c9ef95c6c05cf8f717a Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 3 Dec 2021 08:09:32 +0100 Subject: [PATCH 31/64] list-available-pairs should be tradingmode dependent --- freqtrade/rpc/api_server/api_v1.py | 4 ++-- tests/rpc/test_rpc_apiserver.py | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index f97d49e96..f69ddef43 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -254,8 +254,8 @@ def list_available_pairs(timeframe: Optional[str] = None, stake_currency: Option dh = get_datahandler(config['datadir'], config.get('dataformat_ohlcv', None)) - # TODO-lev: xmatt to decide: use candle-type or market mode for this endpoint?? - pair_interval = dh.ohlcv_get_available_data(config['datadir'], 'spot') + pair_interval = dh.ohlcv_get_available_data(config['datadir'], + config.get('trading_mode', 'spot')) if timeframe: pair_interval = [pair for pair in pair_interval if pair[1] == timeframe] diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 75455982e..bb540dc94 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1353,6 +1353,13 @@ def test_list_available_pairs(botclient): assert rc.json()['pairs'] == ['XRP/ETH'] assert len(rc.json()['pair_interval']) == 1 + ftbot.config['trading_mode'] = 'futures' + rc = client_get( + client, f"{BASE_URI}/available_pairs?timeframe=1h") + assert_response(rc) + assert rc.json()['length'] == 1 + assert rc.json()['pairs'] == ['XRP/USDT'] + rc = client_get( client, f"{BASE_URI}/available_pairs?timeframe=1h&candletype=mark") assert_response(rc) From a87e2567372852702fda3a97744935d8490111f6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 3 Dec 2021 12:12:33 +0100 Subject: [PATCH 32/64] Add candleType enum --- freqtrade/data/history/hdf5datahandler.py | 22 +++++++++++++--------- freqtrade/data/history/idatahandler.py | 10 ++++++++-- freqtrade/data/history/jsondatahandler.py | 21 +++++++++++++-------- freqtrade/enums/__init__.py | 1 + freqtrade/enums/candletype.py | 13 +++++++++++++ 5 files changed, 48 insertions(+), 19 deletions(-) create mode 100644 freqtrade/enums/candletype.py diff --git a/freqtrade/data/history/hdf5datahandler.py b/freqtrade/data/history/hdf5datahandler.py index e8fcad70d..906a7ea6c 100644 --- a/freqtrade/data/history/hdf5datahandler.py +++ b/freqtrade/data/history/hdf5datahandler.py @@ -9,6 +9,7 @@ import pandas as pd from freqtrade.configuration import TimeRange from freqtrade.constants import (DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, ListPairsWithTimeframes, TradeList) +from freqtrade.enums.candletype import CandleType from .idatahandler import IDataHandler @@ -39,24 +40,27 @@ class HDF5DataHandler(IDataHandler): if match and len(match.groups()) > 1] @classmethod - def ohlcv_get_pairs(cls, datadir: Path, timeframe: str, candle_type: str = '') -> List[str]: + def ohlcv_get_pairs( + cls, + datadir: Path, + timeframe: str, + candle_type: CandleType = CandleType.SPOT_ + ) -> List[str]: """ Returns a list of all pairs with ohlcv data available in this datadir for the specified timeframe :param datadir: Directory to search for ohlcv files :param timeframe: Timeframe to search pairs for - :param candle_type: '', mark, index, premiumIndex, or funding_rate + :param candle_type: Any of the enum CandleType (must match your trading mode!) :return: List of Pairs """ - - if candle_type: + candle = "" + if candle_type not in (CandleType.SPOT, CandleType.SPOT_): datadir = datadir.joinpath('futures') - candle_type = f"-{candle_type}" - else: - candle_type = "" + candle = f"-{candle_type}" - _tmp = [re.search(r'^(\S+)(?=\-' + timeframe + candle_type + '.h5)', p.name) - for p in datadir.glob(f"*{timeframe}{candle_type}.h5")] + _tmp = [re.search(r'^(\S+)(?=\-' + timeframe + candle + '.h5)', p.name) + for p in datadir.glob(f"*{timeframe}{candle}.h5")] # Check if regex found something and only return these results return [match[0].replace('_', '/') for match in _tmp if match] diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index bb5e83c5d..dbf93e787 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -17,6 +17,7 @@ from freqtrade import misc from freqtrade.configuration import TimeRange from freqtrade.constants import ListPairsWithTimeframes, TradeList from freqtrade.data.converter import clean_ohlcv_dataframe, trades_remove_duplicates, trim_dataframe +from freqtrade.enums.candletype import CandleType from freqtrade.exchange import timeframe_to_seconds @@ -47,13 +48,18 @@ class IDataHandler(ABC): """ @abstractclassmethod - def ohlcv_get_pairs(cls, datadir: Path, timeframe: str, candle_type: str = '') -> List[str]: + def ohlcv_get_pairs( + cls, + datadir: Path, + timeframe: str, + candle_type: CandleType = CandleType.SPOT_ + ) -> List[str]: """ Returns a list of all pairs with ohlcv data available in this datadir for the specified timeframe :param datadir: Directory to search for ohlcv files :param timeframe: Timeframe to search pairs for - :param candle_type: '', mark, index, premiumIndex, or funding_rate + :param candle_type: Any of the enum CandleType (must match your trading mode!) :return: List of Pairs """ diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index 00ba5d095..2a8b1093c 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -10,6 +10,7 @@ from freqtrade import misc from freqtrade.configuration import TimeRange from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, ListPairsWithTimeframes, TradeList from freqtrade.data.converter import trades_dict_to_list +from freqtrade.enums.candletype import CandleType from .idatahandler import IDataHandler @@ -40,23 +41,27 @@ class JsonDataHandler(IDataHandler): if match and len(match.groups()) > 1] @classmethod - def ohlcv_get_pairs(cls, datadir: Path, timeframe: str, candle_type: str = '') -> List[str]: + def ohlcv_get_pairs( + cls, + datadir: Path, + timeframe: str, + candle_type: CandleType = CandleType.SPOT_ + ) -> List[str]: """ Returns a list of all pairs with ohlcv data available in this datadir for the specified timeframe :param datadir: Directory to search for ohlcv files :param timeframe: Timeframe to search pairs for - :param candle_type: '', mark, index, premiumIndex, or funding_rate + :param candle_type: Any of the enum CandleType (must match your trading mode!) :return: List of Pairs """ - if candle_type: + candle = "" + if candle_type not in (CandleType.SPOT, CandleType.SPOT_): datadir = datadir.joinpath('futures') - candle_type = f"-{candle_type}" - else: - candle_type = "" + candle = f"-{candle_type}" - _tmp = [re.search(r'^(\S+)(?=\-' + timeframe + candle_type + '.json)', p.name) - for p in datadir.glob(f"*{timeframe}{candle_type}.{cls._get_file_extension()}")] + _tmp = [re.search(r'^(\S+)(?=\-' + timeframe + candle + '.json)', p.name) + for p in datadir.glob(f"*{timeframe}{candle}.{cls._get_file_extension()}")] # Check if regex found something and only return these results return [match[0].replace('_', '/') for match in _tmp if match] diff --git a/freqtrade/enums/__init__.py b/freqtrade/enums/__init__.py index e9d166258..f2fee4792 100644 --- a/freqtrade/enums/__init__.py +++ b/freqtrade/enums/__init__.py @@ -1,5 +1,6 @@ # flake8: noqa: F401 from freqtrade.enums.backteststate import BacktestState +from freqtrade.enums.candletype import CandleType from freqtrade.enums.collateral import Collateral from freqtrade.enums.rpcmessagetype import RPCMessageType from freqtrade.enums.runmode import NON_UTIL_MODES, OPTIMIZE_MODES, TRADING_MODES, RunMode diff --git a/freqtrade/enums/candletype.py b/freqtrade/enums/candletype.py new file mode 100644 index 000000000..b4052a212 --- /dev/null +++ b/freqtrade/enums/candletype.py @@ -0,0 +1,13 @@ +from enum import Enum + + +class CandleType(str, Enum): + """Enum to distinguish candle types""" + SPOT = "spot" + SPOT_ = "" + FUTURES = "futures" + MARK = "mark" + INDEX = "index" + PREMIUMINDEX = "premiumIndex" + # TODO-lev: not sure this belongs here, as the datatype is really different + FUNDING_RATE = "funding_rate" From f9cf59bb4d8426c44d2b025e424d3bdf10369407 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 3 Dec 2021 12:23:35 +0100 Subject: [PATCH 33/64] Candle_type to enum --- freqtrade/data/converter.py | 3 +- freqtrade/data/history/hdf5datahandler.py | 16 ++++++----- freqtrade/data/history/history_utils.py | 25 +++++++++------- freqtrade/data/history/idatahandler.py | 35 ++++++++++++----------- freqtrade/data/history/jsondatahandler.py | 28 +++++------------- tests/data/test_history.py | 5 ++-- 6 files changed, 53 insertions(+), 59 deletions(-) diff --git a/freqtrade/data/converter.py b/freqtrade/data/converter.py index dfd4a9f68..680b95ab2 100644 --- a/freqtrade/data/converter.py +++ b/freqtrade/data/converter.py @@ -11,6 +11,7 @@ import pandas as pd from pandas import DataFrame, to_datetime from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, TradeList +from freqtrade.enums.candletype import CandleType logger = logging.getLogger(__name__) @@ -266,7 +267,7 @@ def convert_ohlcv_format( convert_from: str, convert_to: str, erase: bool, - candle_type: str = '' + candle_type: CandleType = CandleType.SPOT_ ): """ Convert OHLCV from one format to another diff --git a/freqtrade/data/history/hdf5datahandler.py b/freqtrade/data/history/hdf5datahandler.py index 906a7ea6c..35e01f279 100644 --- a/freqtrade/data/history/hdf5datahandler.py +++ b/freqtrade/data/history/hdf5datahandler.py @@ -51,7 +51,7 @@ class HDF5DataHandler(IDataHandler): for the specified timeframe :param datadir: Directory to search for ohlcv files :param timeframe: Timeframe to search pairs for - :param candle_type: Any of the enum CandleType (must match your trading mode!) + :param candle_type: Any of the enum CandleType (must match trading mode!) :return: List of Pairs """ candle = "" @@ -69,14 +69,14 @@ class HDF5DataHandler(IDataHandler): pair: str, timeframe: str, data: pd.DataFrame, - candle_type: str = '' + candle_type: CandleType = CandleType.SPOT_ ) -> None: """ Store data in hdf5 file. :param pair: Pair - used to generate filename :param timeframe: Timeframe - used to generate filename :param data: Dataframe containing OHLCV data - :param candle_type: '', mark, index, premiumIndex, or funding_rate + :param candle_type: Any of the enum CandleType (must match trading mode!) :return: None """ key = self._pair_ohlcv_key(pair, timeframe) @@ -90,7 +90,9 @@ class HDF5DataHandler(IDataHandler): ) def _ohlcv_load(self, pair: str, timeframe: str, - timerange: Optional[TimeRange] = None, candle_type: str = '') -> pd.DataFrame: + timerange: Optional[TimeRange] = None, + candle_type: CandleType = CandleType.SPOT_ + ) -> pd.DataFrame: """ Internal method used to load data for one pair from disk. Implements the loading and conversion to a Pandas dataframe. @@ -100,7 +102,7 @@ class HDF5DataHandler(IDataHandler): :param timerange: Limit data to be loaded to this timerange. Optionally implemented by subclasses to avoid loading all data where possible. - :param candle_type: '', mark, index, premiumIndex, or funding_rate + :param candle_type: Any of the enum CandleType (must match trading mode!) :return: DataFrame with ohlcv data, or empty DataFrame """ key = self._pair_ohlcv_key(pair, timeframe) @@ -133,14 +135,14 @@ class HDF5DataHandler(IDataHandler): pair: str, timeframe: str, data: pd.DataFrame, - candle_type: str = '' + candle_type: CandleType ) -> None: """ Append data to existing data structures :param pair: Pair :param timeframe: Timeframe this ohlcv data is for :param data: Data to append. - :param candle_type: '', mark, index, premiumIndex, or funding_rate + :param candle_type: Any of the enum CandleType (must match trading mode!) """ raise NotImplementedError() diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index 7b0c727c8..3fdb36e58 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -12,6 +12,7 @@ from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS from freqtrade.data.converter import (clean_ohlcv_dataframe, ohlcv_to_dataframe, trades_remove_duplicates, trades_to_ohlcv) from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler +from freqtrade.enums.candletype import CandleType from freqtrade.exceptions import OperationalException from freqtrade.exchange import Exchange from freqtrade.misc import format_ms_time @@ -29,7 +30,7 @@ def load_pair_history(pair: str, startup_candles: int = 0, data_format: str = None, data_handler: IDataHandler = None, - candle_type: str = '' + candle_type: CandleType = CandleType.SPOT ) -> DataFrame: """ Load cached ohlcv history for the given pair. @@ -44,7 +45,7 @@ def load_pair_history(pair: str, :param startup_candles: Additional candles to load at the start of the period :param data_handler: Initialized data-handler to use. Will be initialized from data_format if not set - :param candle_type: '', mark, index, premiumIndex, or funding_rate + :param candle_type: Any of the enum CandleType (must match trading mode!) :return: DataFrame with ohlcv data, or empty DataFrame """ data_handler = get_datahandler(datadir, data_format, data_handler) @@ -67,7 +68,7 @@ def load_data(datadir: Path, startup_candles: int = 0, fail_without_data: bool = False, data_format: str = 'json', - candle_type: str = '' + candle_type: CandleType = CandleType.SPOT ) -> Dict[str, DataFrame]: """ Load ohlcv history data for a list of pairs. @@ -80,7 +81,7 @@ def load_data(datadir: Path, :param startup_candles: Additional candles to load at the start of the period :param fail_without_data: Raise OperationalException if no data is found. :param data_format: Data format which should be used. Defaults to json - :param candle_type: '', mark, index, premiumIndex, or funding_rate + :param candle_type: Any of the enum CandleType (must match trading mode!) :return: dict(:) """ result: Dict[str, DataFrame] = {} @@ -111,7 +112,7 @@ def refresh_data(datadir: Path, exchange: Exchange, data_format: str = None, timerange: Optional[TimeRange] = None, - candle_type: str = '' + candle_type: CandleType = CandleType.SPOT ) -> None: """ Refresh ohlcv history data for a list of pairs. @@ -122,7 +123,7 @@ def refresh_data(datadir: Path, :param exchange: Exchange object :param data_format: dataformat to use :param timerange: Limit data to be loaded to this timerange - :param candle_type: '', mark, index, premiumIndex, or funding_rate + :param candle_type: Any of the enum CandleType (must match trading mode!) """ data_handler = get_datahandler(datadir, data_format) for idx, pair in enumerate(pairs): @@ -138,7 +139,7 @@ def _load_cached_data_for_updating( timeframe: str, timerange: Optional[TimeRange], data_handler: IDataHandler, - candle_type: str = '' + candle_type: CandleType = CandleType.SPOT ) -> Tuple[DataFrame, Optional[int]]: """ Load cached data to download more data. @@ -177,7 +178,8 @@ def _download_pair_history(pair: str, *, new_pairs_days: int = 30, data_handler: IDataHandler = None, timerange: Optional[TimeRange] = None, - candle_type: str = '') -> bool: + candle_type: CandleType = CandleType.SPOT + ) -> bool: """ Download latest candles from the exchange for the pair and timeframe passed in parameters The data is downloaded starting from the last correct data that @@ -189,7 +191,7 @@ def _download_pair_history(pair: str, *, :param pair: pair to download :param timeframe: Timeframe (e.g "5m") :param timerange: range of time to download - :param candle_type: '', mark, index, premiumIndex, or funding_rate + :param candle_type: Any of the enum CandleType (must match trading mode!) :return: bool with success state """ data_handler = get_datahandler(datadir, data_handler=data_handler) @@ -249,7 +251,8 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes datadir: Path, timerange: Optional[TimeRange] = None, new_pairs_days: int = 30, erase: bool = False, data_format: str = None, - candle_type: str = '') -> List[str]: + candle_type: CandleType = CandleType.SPOT + ) -> List[str]: """ Refresh stored ohlcv data for backtesting and hyperopt operations. Used by freqtrade download-data subcommand. @@ -382,7 +385,7 @@ def convert_trades_to_ohlcv( erase: bool = False, data_format_ohlcv: str = 'json', data_format_trades: str = 'jsongz', - candle_type: str = '' + candle_type: CandleType = CandleType.SPOT ) -> None: """ Convert stored trades data to ohlcv data diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index dbf93e787..72758a325 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -59,7 +59,7 @@ class IDataHandler(ABC): for the specified timeframe :param datadir: Directory to search for ohlcv files :param timeframe: Timeframe to search pairs for - :param candle_type: Any of the enum CandleType (must match your trading mode!) + :param candle_type: Any of the enum CandleType (must match trading mode!) :return: List of Pairs """ @@ -69,21 +69,20 @@ class IDataHandler(ABC): pair: str, timeframe: str, data: DataFrame, - candle_type: str = '' + candle_type: CandleType = CandleType.SPOT_ ) -> None: """ Store ohlcv data. :param pair: Pair - used to generate filename :param timeframe: Timeframe - used to generate filename :param data: Dataframe containing OHLCV data - :param candle_type: '', mark, index, premiumIndex, or funding_rate + :param candle_type: Any of the enum CandleType (must match trading mode!) :return: None """ @abstractmethod - def _ohlcv_load(self, pair: str, timeframe: str, - timerange: Optional[TimeRange] = None, - candle_type: str = '' + def _ohlcv_load(self, pair: str, timeframe: str, timerange: Optional[TimeRange] = None, + candle_type: CandleType = CandleType.SPOT_ ) -> DataFrame: """ Internal method used to load data for one pair from disk. @@ -94,16 +93,17 @@ class IDataHandler(ABC): :param timerange: Limit data to be loaded to this timerange. Optionally implemented by subclasses to avoid loading all data where possible. - :param candle_type: '', mark, index, premiumIndex, or funding_rate + :param candle_type: Any of the enum CandleType (must match trading mode!) :return: DataFrame with ohlcv data, or empty DataFrame """ - def ohlcv_purge(self, pair: str, timeframe: str, candle_type: str = '') -> bool: + def ohlcv_purge( + self, pair: str, timeframe: str, candle_type: CandleType = CandleType.SPOT_) -> bool: """ Remove data for this pair :param pair: Delete data for this pair. :param timeframe: Timeframe (e.g. "5m") - :param candle_type: '', mark, index, premiumIndex, or funding_rate + :param candle_type: Any of the enum CandleType (must match trading mode!) :return: True when deleted, false if file did not exist. """ filename = self._pair_data_filename(self._datadir, pair, timeframe, candle_type) @@ -118,14 +118,14 @@ class IDataHandler(ABC): pair: str, timeframe: str, data: DataFrame, - candle_type: str = '' + candle_type: CandleType ) -> None: """ Append data to existing data structures :param pair: Pair :param timeframe: Timeframe this ohlcv data is for :param data: Data to append. - :param candle_type: '', mark, index, premiumIndex, or funding_rate + :param candle_type: Any of the enum CandleType (must match trading mode!) """ @abstractclassmethod @@ -200,14 +200,15 @@ class IDataHandler(ABC): datadir: Path, pair: str, timeframe: str, - candle_type: str = '' + candle_type: CandleType ) -> Path: pair_s = misc.pair_to_filename(pair) - if candle_type: + candle = "" + if candle_type not in (CandleType.SPOT, CandleType.SPOT_): datadir = datadir.joinpath('futures') - candle_type = f"-{candle_type}" + candle = f"-{candle_type}" filename = datadir.joinpath( - f'{pair_s}-{timeframe}{candle_type}.{cls._get_file_extension()}') + f'{pair_s}-{timeframe}{candle}.{cls._get_file_extension()}') return filename @classmethod @@ -232,7 +233,7 @@ class IDataHandler(ABC): drop_incomplete: bool = True, startup_candles: int = 0, warn_no_data: bool = True, - candle_type: str = '' + candle_type: CandleType = CandleType.SPOT_ ) -> DataFrame: """ Load cached candle (OHLCV) data for the given pair. @@ -244,7 +245,7 @@ class IDataHandler(ABC): :param drop_incomplete: Drop last candle assuming it may be incomplete. :param startup_candles: Additional candles to load at the start of the period :param warn_no_data: Log a warning message when no data is found - :param candle_type: '', mark, index, premiumIndex, or funding_rate + :param candle_type: Any of the enum CandleType (must match trading mode!) :return: DataFrame with ohlcv data, or empty DataFrame """ # Fix startup period diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index 2a8b1093c..1f5439c27 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -52,7 +52,7 @@ class JsonDataHandler(IDataHandler): for the specified timeframe :param datadir: Directory to search for ohlcv files :param timeframe: Timeframe to search pairs for - :param candle_type: Any of the enum CandleType (must match your trading mode!) + :param candle_type: Any of the enum CandleType (must match trading mode!) :return: List of Pairs """ candle = "" @@ -70,7 +70,7 @@ class JsonDataHandler(IDataHandler): pair: str, timeframe: str, data: DataFrame, - candle_type: str = '' + candle_type: CandleType = CandleType.SPOT_ ) -> None: """ Store data in json format "values". @@ -79,7 +79,7 @@ class JsonDataHandler(IDataHandler): :param pair: Pair - used to generate filename :param timeframe: Timeframe - used to generate filename :param data: Dataframe containing OHLCV data - :param candle_type: '', mark, index, premiumIndex, or funding_rate + :param candle_type: Any of the enum CandleType (must match trading mode!) :return: None """ filename = self._pair_data_filename(self._datadir, pair, timeframe, candle_type) @@ -95,7 +95,7 @@ class JsonDataHandler(IDataHandler): def _ohlcv_load(self, pair: str, timeframe: str, timerange: Optional[TimeRange] = None, - candle_type: str = '' + candle_type: CandleType = CandleType.SPOT_ ) -> DataFrame: """ Internal method used to load data for one pair from disk. @@ -106,7 +106,7 @@ class JsonDataHandler(IDataHandler): :param timerange: Limit data to be loaded to this timerange. Optionally implemented by subclasses to avoid loading all data where possible. - :param candle_type: '', mark, index, premiumIndex, or funding_rate + :param candle_type: Any of the enum CandleType (must match trading mode!) :return: DataFrame with ohlcv data, or empty DataFrame """ filename = self._pair_data_filename(self._datadir, pair, timeframe, candle_type=candle_type) @@ -126,33 +126,19 @@ class JsonDataHandler(IDataHandler): infer_datetime_format=True) return pairdata - def ohlcv_purge(self, pair: str, timeframe: str, candle_type: str = '') -> bool: - """ - Remove data for this pair - :param pair: Delete data for this pair. - :param timeframe: Timeframe (e.g. "5m") - :param candle_type: '', mark, index, premiumIndex, or funding_rate - :return: True when deleted, false if file did not exist. - """ - filename = self._pair_data_filename(self._datadir, pair, timeframe, candle_type=candle_type) - if filename.exists(): - filename.unlink() - return True - return False - def ohlcv_append( self, pair: str, timeframe: str, data: DataFrame, - candle_type: str = '' + candle_type: CandleType ) -> None: """ Append data to existing data structures :param pair: Pair :param timeframe: Timeframe this ohlcv data is for :param data: Data to append. - :param candle_type: '', mark, index, premiumIndex, or funding_rate + :param candle_type: Any of the enum CandleType (must match trading mode!) """ raise NotImplementedError() diff --git a/tests/data/test_history.py b/tests/data/test_history.py index a36f933c9..7944593c1 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -24,6 +24,7 @@ from freqtrade.data.history.history_utils import (_download_pair_history, _downl validate_backtest_data) from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler, get_datahandlerclass from freqtrade.data.history.jsondatahandler import JsonDataHandler, JsonGzDataHandler +from freqtrade.enums.candletype import CandleType from freqtrade.exchange import timeframe_to_minutes from freqtrade.misc import file_dump_json from freqtrade.resolvers import StrategyResolver @@ -809,9 +810,9 @@ def test_jsondatahandler_trades_purge(mocker, testdatadir): def test_datahandler_ohlcv_append(datahandler, testdatadir, ): dh = get_datahandler(testdatadir, datahandler) with pytest.raises(NotImplementedError): - dh.ohlcv_append('UNITTEST/ETH', '5m', DataFrame()) + dh.ohlcv_append('UNITTEST/ETH', '5m', DataFrame(), CandleType.SPOT) with pytest.raises(NotImplementedError): - dh.ohlcv_append('UNITTEST/ETH', '5m', DataFrame(), candle_type='mark') + dh.ohlcv_append('UNITTEST/ETH', '5m', DataFrame(), CandleType.MARK) @pytest.mark.parametrize('datahandler', AVAILABLE_DATAHANDLERS) From f33643cacf4a4eafb3d5c925426a8a480ff0928b Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 3 Dec 2021 12:46:18 +0100 Subject: [PATCH 34/64] Add candletype from string --- freqtrade/data/converter.py | 1 + freqtrade/enums/candletype.py | 7 +++++++ tests/leverage/test_candletype.py | 18 ++++++++++++++++++ 3 files changed, 26 insertions(+) create mode 100644 tests/leverage/test_candletype.py diff --git a/freqtrade/data/converter.py b/freqtrade/data/converter.py index 680b95ab2..bdb8c3464 100644 --- a/freqtrade/data/converter.py +++ b/freqtrade/data/converter.py @@ -275,6 +275,7 @@ def convert_ohlcv_format( :param convert_from: Source format :param convert_to: Target format :param erase: Erase source data (does not apply if source and target format are identical) + :param candle_type: Any of the enum CandleType (must match trading mode!) """ from freqtrade.data.history.idatahandler import get_datahandler src = get_datahandler(config['datadir'], convert_from) diff --git a/freqtrade/enums/candletype.py b/freqtrade/enums/candletype.py index b4052a212..9818bb67b 100644 --- a/freqtrade/enums/candletype.py +++ b/freqtrade/enums/candletype.py @@ -11,3 +11,10 @@ class CandleType(str, Enum): PREMIUMINDEX = "premiumIndex" # TODO-lev: not sure this belongs here, as the datatype is really different FUNDING_RATE = "funding_rate" + + @classmethod + def from_string(cls, value: str) -> 'CandleType': + if not value: + # Default to spot + return CandleType.SPOT + return CandleType(value) diff --git a/tests/leverage/test_candletype.py b/tests/leverage/test_candletype.py new file mode 100644 index 000000000..3eb73a07c --- /dev/null +++ b/tests/leverage/test_candletype.py @@ -0,0 +1,18 @@ +import pytest + +from freqtrade.enums import CandleType + + +@pytest.mark.parametrize('input,expected', [ + ('', CandleType.SPOT), + ('spot', CandleType.SPOT), + (CandleType.SPOT, CandleType.SPOT), + (CandleType.FUTURES, CandleType.FUTURES), + (CandleType.INDEX, CandleType.INDEX), + (CandleType.MARK, CandleType.MARK), + ('futures', CandleType.FUTURES), + ('mark', CandleType.MARK), + ('premiumIndex', CandleType.PREMIUMINDEX), +]) +def test_candle_type_from_string(input, expected): + assert CandleType.from_string(input) == expected From 549321267260960a116143a38281460172cc2035 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 3 Dec 2021 13:04:31 +0100 Subject: [PATCH 35/64] More candletype changes --- freqtrade/constants.py | 2 ++ freqtrade/data/dataprovider.py | 13 ++++++++----- freqtrade/data/history/hdf5datahandler.py | 8 ++++++-- freqtrade/data/history/jsondatahandler.py | 8 ++++++-- freqtrade/optimize/backtesting.py | 4 +++- freqtrade/strategy/interface.py | 4 +++- tests/data/test_dataprovider.py | 5 +++-- tests/rpc/test_rpc_apiserver.py | 3 ++- 8 files changed, 33 insertions(+), 14 deletions(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index 59c709980..ebb817e8d 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -475,6 +475,8 @@ CANCEL_REASON = { } # List of pairs with their timeframes +# TODO-lev: This should really be +# PairWithTimeframe = Tuple[str, str, CandleType] PairWithTimeframe = Tuple[str, str, str] ListPairsWithTimeframes = List[PairWithTimeframe] diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 0e8031554..3e6eccdc8 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -14,6 +14,7 @@ from freqtrade.configuration import TimeRange from freqtrade.constants import ListPairsWithTimeframes, PairWithTimeframe from freqtrade.data.history import load_pair_history from freqtrade.enums import RunMode +from freqtrade.enums.candletype import CandleType from freqtrade.exceptions import ExchangeError, OperationalException from freqtrade.exchange import Exchange, timeframe_to_seconds @@ -46,7 +47,7 @@ class DataProvider: pair: str, timeframe: str, dataframe: DataFrame, - candle_type: str = '' + candle_type: CandleType ) -> None: """ Store cached Dataframe. @@ -55,7 +56,7 @@ class DataProvider: :param pair: pair to get the data for :param timeframe: Timeframe to get data for :param dataframe: analyzed dataframe - :param candle_type: '', mark, index, premiumIndex, or funding_rate + :param candle_type: Any of the enum CandleType (must match trading mode!) """ self.__cached_pairs[(pair, timeframe, candle_type)] = ( dataframe, datetime.now(timezone.utc)) @@ -78,7 +79,8 @@ class DataProvider: :param timeframe: timeframe to get data for :param candle_type: '', mark, index, premiumIndex, or funding_rate """ - saved_pair = (pair, str(timeframe), candle_type) + candleType = CandleType.from_string(candle_type) + saved_pair = (pair, str(timeframe), candleType) if saved_pair not in self.__cached_pairs_backtesting: timerange = TimeRange.parse_timerange(None if self._config.get( 'timerange') is None else str(self._config.get('timerange'))) @@ -92,7 +94,8 @@ class DataProvider: datadir=self._config['datadir'], timerange=timerange, data_format=self._config.get('dataformat_ohlcv', 'json'), - candle_type=candle_type + candle_type=candleType, + ) return self.__cached_pairs_backtesting[saved_pair].copy() @@ -132,7 +135,7 @@ class DataProvider: combination. Returns empty dataframe and Epoch 0 (1970-01-01) if no dataframe was cached. """ - pair_key = (pair, timeframe, '') + pair_key = (pair, timeframe, CandleType.SPOT) if pair_key in self.__cached_pairs: if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE): df, date = self.__cached_pairs[pair_key] diff --git a/freqtrade/data/history/hdf5datahandler.py b/freqtrade/data/history/hdf5datahandler.py index 35e01f279..b9585e22a 100644 --- a/freqtrade/data/history/hdf5datahandler.py +++ b/freqtrade/data/history/hdf5datahandler.py @@ -36,8 +36,12 @@ class HDF5DataHandler(IDataHandler): cls._OHLCV_REGEX, p.name ) for p in datadir.glob("*.h5") ] - return [(cls.rebuild_pair_from_filename(match[1]), match[2], match[3]) for match in _tmp - if match and len(match.groups()) > 1] + return [ + ( + cls.rebuild_pair_from_filename(match[1]), + match[2], + CandleType.from_string(match[3]) + ) for match in _tmp if match and len(match.groups()) > 1] @classmethod def ohlcv_get_pairs( diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index 1f5439c27..b4775f271 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -37,8 +37,12 @@ class JsonDataHandler(IDataHandler): re.search( cls._OHLCV_REGEX, p.name ) for p in datadir.glob(f"*.{cls._get_file_extension()}")] - return [(cls.rebuild_pair_from_filename(match[1]), match[2], match[3]) for match in _tmp - if match and len(match.groups()) > 1] + return [ + ( + cls.rebuild_pair_from_filename(match[1]), + match[2], + CandleType.from_string(match[3]) + ) for match in _tmp if match and len(match.groups()) > 1] @classmethod def ohlcv_get_pairs( diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 6c5a44da0..95c92ee0b 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -18,6 +18,7 @@ from freqtrade.data.btanalysis import trade_list_to_dataframe from freqtrade.data.converter import trim_dataframe, trim_dataframes from freqtrade.data.dataprovider import DataProvider from freqtrade.enums import BacktestState, SellType +from freqtrade.enums.candletype import CandleType from freqtrade.enums.tradingmode import TradingMode from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds @@ -290,7 +291,8 @@ class Backtesting: df_analyzed.loc[:, col] = 0 if col not in ('enter_tag', 'exit_tag') else None # Update dataprovider cache - self.dataprovider._set_cached_df(pair, self.timeframe, df_analyzed) + self.dataprovider._set_cached_df(pair, self.timeframe, df_analyzed, CandleType.SPOT) + # TODO-lev: Candle-type should be conditional, either "spot" or futures df_analyzed = df_analyzed.drop(df_analyzed.head(1).index) diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index bcb0a93b4..8abb10bc7 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -14,6 +14,7 @@ from pandas import DataFrame from freqtrade.constants import ListPairsWithTimeframes from freqtrade.data.dataprovider import DataProvider from freqtrade.enums import SellType, SignalDirection, SignalTagType, SignalType +from freqtrade.enums.candletype import CandleType from freqtrade.exceptions import OperationalException, StrategyError from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds from freqtrade.exchange.exchange import timeframe_to_next_date @@ -528,7 +529,8 @@ class IStrategy(ABC, HyperStrategyMixin): dataframe = self.analyze_ticker(dataframe, metadata) self._last_candle_seen_per_pair[pair] = dataframe.iloc[-1]['date'] if self.dp: - self.dp._set_cached_df(pair, self.timeframe, dataframe) + self.dp._set_cached_df(pair, self.timeframe, dataframe, CandleType.SPOT) + # TODO-lev: CandleType should be set conditionally else: logger.debug("Skipping TA Analysis for already analyzed candle") dataframe[SignalType.ENTER_LONG.value] = 0 diff --git a/tests/data/test_dataprovider.py b/tests/data/test_dataprovider.py index 4f01f816f..42e2bcdea 100644 --- a/tests/data/test_dataprovider.py +++ b/tests/data/test_dataprovider.py @@ -6,6 +6,7 @@ from pandas import DataFrame from freqtrade.data.dataprovider import DataProvider from freqtrade.enums import RunMode +from freqtrade.enums.candletype import CandleType from freqtrade.exceptions import ExchangeError, OperationalException from freqtrade.plugins.pairlistmanager import PairListManager from tests.conftest import get_patched_exchange @@ -247,8 +248,8 @@ def test_get_analyzed_dataframe(mocker, default_conf, ohlcv_history): exchange = get_patched_exchange(mocker, default_conf) dp = DataProvider(default_conf, exchange) - dp._set_cached_df("XRP/BTC", timeframe, ohlcv_history) - dp._set_cached_df("UNITTEST/BTC", timeframe, ohlcv_history) + dp._set_cached_df("XRP/BTC", timeframe, ohlcv_history, CandleType.SPOT) + dp._set_cached_df("UNITTEST/BTC", timeframe, ohlcv_history, CandleType.SPOT) assert dp.runmode == RunMode.DRY_RUN dataframe, time = dp.get_analyzed_dataframe("UNITTEST/BTC", timeframe) diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index bb540dc94..0823c7aee 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -17,6 +17,7 @@ from requests.auth import _basic_auth_str from freqtrade.__init__ import __version__ from freqtrade.enums import RunMode, State +from freqtrade.enums.candletype import CandleType from freqtrade.exceptions import DependencyException, ExchangeError, OperationalException from freqtrade.loggers import setup_logging, setup_logging_pre from freqtrade.persistence import PairLocks, Trade @@ -1179,7 +1180,7 @@ def test_api_pair_candles(botclient, ohlcv_history): ohlcv_history['enter_short'] = 0 ohlcv_history['exit_short'] = 0 - ftbot.dataprovider._set_cached_df("XRP/BTC", timeframe, ohlcv_history) + ftbot.dataprovider._set_cached_df("XRP/BTC", timeframe, ohlcv_history, CandleType.SPOT) rc = client_get(client, f"{BASE_URI}/pair_candles?limit={amount}&pair=XRP%2FBTC&timeframe={timeframe}") From 2f17fa2765a9fabb499b94eedd35f4e896687769 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 3 Dec 2021 14:11:24 +0100 Subject: [PATCH 36/64] Update more to use candleType --- freqtrade/data/converter.py | 2 +- freqtrade/data/dataprovider.py | 5 +-- freqtrade/data/history/hdf5datahandler.py | 2 +- freqtrade/data/history/history_utils.py | 2 +- freqtrade/data/history/idatahandler.py | 2 +- freqtrade/data/history/jsondatahandler.py | 2 +- freqtrade/exchange/binance.py | 9 ++--- freqtrade/exchange/exchange.py | 40 ++++++++----------- freqtrade/optimize/backtesting.py | 4 +- freqtrade/plugins/pairlist/AgeFilter.py | 6 ++- freqtrade/plugins/pairlist/ShuffleFilter.py | 2 +- .../plugins/pairlist/VolatilityFilter.py | 6 ++- freqtrade/plugins/pairlist/VolumePairList.py | 7 ++-- .../plugins/pairlist/rangestabilityfilter.py | 6 ++- freqtrade/plugins/pairlistmanager.py | 3 +- freqtrade/rpc/api_server/api_v1.py | 3 +- freqtrade/strategy/informative_decorator.py | 3 +- freqtrade/strategy/interface.py | 7 ++-- 18 files changed, 56 insertions(+), 55 deletions(-) diff --git a/freqtrade/data/converter.py b/freqtrade/data/converter.py index bdb8c3464..c087c87ff 100644 --- a/freqtrade/data/converter.py +++ b/freqtrade/data/converter.py @@ -11,7 +11,7 @@ import pandas as pd from pandas import DataFrame, to_datetime from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, TradeList -from freqtrade.enums.candletype import CandleType +from freqtrade.enums import CandleType logger = logging.getLogger(__name__) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 3e6eccdc8..12b02f744 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -13,8 +13,7 @@ from pandas import DataFrame from freqtrade.configuration import TimeRange from freqtrade.constants import ListPairsWithTimeframes, PairWithTimeframe from freqtrade.data.history import load_pair_history -from freqtrade.enums import RunMode -from freqtrade.enums.candletype import CandleType +from freqtrade.enums import CandleType, RunMode from freqtrade.exceptions import ExchangeError, OperationalException from freqtrade.exchange import Exchange, timeframe_to_seconds @@ -223,7 +222,7 @@ class DataProvider: raise OperationalException(NO_EXCHANGE_EXCEPTION) if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE): return self._exchange.klines( - (pair, timeframe or self._config['timeframe'], candle_type), + (pair, timeframe or self._config['timeframe'], CandleType.from_string(candle_type)), copy=copy ) else: diff --git a/freqtrade/data/history/hdf5datahandler.py b/freqtrade/data/history/hdf5datahandler.py index b9585e22a..fe840527f 100644 --- a/freqtrade/data/history/hdf5datahandler.py +++ b/freqtrade/data/history/hdf5datahandler.py @@ -9,7 +9,7 @@ import pandas as pd from freqtrade.configuration import TimeRange from freqtrade.constants import (DEFAULT_DATAFRAME_COLUMNS, DEFAULT_TRADES_COLUMNS, ListPairsWithTimeframes, TradeList) -from freqtrade.enums.candletype import CandleType +from freqtrade.enums import CandleType from .idatahandler import IDataHandler diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index 3fdb36e58..e9c31259f 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -12,7 +12,7 @@ from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS from freqtrade.data.converter import (clean_ohlcv_dataframe, ohlcv_to_dataframe, trades_remove_duplicates, trades_to_ohlcv) from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler -from freqtrade.enums.candletype import CandleType +from freqtrade.enums import CandleType from freqtrade.exceptions import OperationalException from freqtrade.exchange import Exchange from freqtrade.misc import format_ms_time diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index 72758a325..239c9ab71 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -17,7 +17,7 @@ from freqtrade import misc from freqtrade.configuration import TimeRange from freqtrade.constants import ListPairsWithTimeframes, TradeList from freqtrade.data.converter import clean_ohlcv_dataframe, trades_remove_duplicates, trim_dataframe -from freqtrade.enums.candletype import CandleType +from freqtrade.enums import CandleType from freqtrade.exchange import timeframe_to_seconds diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index b4775f271..1ed5ae023 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -10,7 +10,7 @@ from freqtrade import misc from freqtrade.configuration import TimeRange from freqtrade.constants import DEFAULT_DATAFRAME_COLUMNS, ListPairsWithTimeframes, TradeList from freqtrade.data.converter import trades_dict_to_list -from freqtrade.enums.candletype import CandleType +from freqtrade.enums import CandleType from .idatahandler import IDataHandler diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index ad18efcf5..db6cda4d7 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -8,7 +8,7 @@ from typing import Dict, List, Optional, Tuple import arrow import ccxt -from freqtrade.enums import Collateral, TradingMode +from freqtrade.enums import CandleType, Collateral, TradingMode from freqtrade.exceptions import (DDosProtection, InsufficientFundsError, InvalidOrderException, OperationalException, TemporaryError) from freqtrade.exchange import Exchange @@ -197,14 +197,13 @@ class Binance(Exchange): raise OperationalException(e) from e async def _async_get_historic_ohlcv(self, pair: str, timeframe: str, - since_ms: int, is_new_pair: bool = False, - raise_: bool = False, - candle_type: str = '' + since_ms: int, candle_type: CandleType, + is_new_pair: bool = False, raise_: bool = False, ) -> Tuple[str, str, str, List]: """ Overwrite to introduce "fast new pair" functionality by detecting the pair's listing date Does not work for other exchanges, which don't return the earliest data when called with "0" - :param candle_type: '', mark, index, premiumIndex, or funding_rate + :param candle_type: Any of the enum CandleType (must match trading mode!) """ if is_new_pair: x = await self._async_get_candle_history(pair, timeframe, 0, candle_type) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 329cecc3d..e75735c04 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -20,9 +20,9 @@ from ccxt.base.decimal_to_precision import (ROUND_DOWN, ROUND_UP, TICK_SIZE, TRU from pandas import DataFrame from freqtrade.constants import (DEFAULT_AMOUNT_RESERVE_PERCENT, NON_OPEN_EXCHANGE_STATES, - ListPairsWithTimeframes) + ListPairsWithTimeframes, PairWithTimeframe) from freqtrade.data.converter import ohlcv_to_dataframe, trades_dict_to_list -from freqtrade.enums import Collateral, TradingMode +from freqtrade.enums import CandleType, Collateral, TradingMode from freqtrade.exceptions import (DDosProtection, ExchangeError, InsufficientFundsError, InvalidOrderException, OperationalException, PricingError, RetryableOrderError, TemporaryError) @@ -92,7 +92,7 @@ class Exchange: self._config.update(config) # Holds last candle refreshed time of each pair - self._pairs_last_refresh_time: Dict[Tuple[str, str, str], int] = {} + self._pairs_last_refresh_time: Dict[PairWithTimeframe, int] = {} # Timestamp of last markets refresh self._last_markets_refresh: int = 0 @@ -105,7 +105,7 @@ class Exchange: self._buy_rate_cache: TTLCache = TTLCache(maxsize=100, ttl=1800) # Holds candles - self._klines: Dict[Tuple[str, str, str], DataFrame] = {} + self._klines: Dict[PairWithTimeframe, DataFrame] = {} # Holds all open sell orders for dry_run self._dry_run_open_orders: Dict[str, Any] = {} @@ -359,7 +359,7 @@ class Exchange: or (self.trading_mode == TradingMode.FUTURES and self.market_is_future(market)) ) - def klines(self, pair_interval: Tuple[str, str, str], copy: bool = True) -> DataFrame: + def klines(self, pair_interval: PairWithTimeframe, copy: bool = True) -> DataFrame: if pair_interval in self._klines: return self._klines[pair_interval].copy() if copy else self._klines[pair_interval] else: @@ -1314,8 +1314,8 @@ class Exchange: # Historic data def get_historic_ohlcv(self, pair: str, timeframe: str, - since_ms: int, is_new_pair: bool = False, - candle_type: str = '') -> List: + since_ms: int, candle_type: CandleType, + is_new_pair: bool = False) -> List: """ Get candle history using asyncio and returns the list of candles. Handles all async work for this. @@ -1327,7 +1327,7 @@ class Exchange: :return: List with candle (OHLCV) data """ data: List - pair, timeframe, candle_type, data = asyncio.get_event_loop().run_until_complete( + pair, _, _, data = asyncio.get_event_loop().run_until_complete( self._async_get_historic_ohlcv(pair=pair, timeframe=timeframe, since_ms=since_ms, is_new_pair=is_new_pair, candle_type=candle_type)) @@ -1335,13 +1335,13 @@ class Exchange: return data def get_historic_ohlcv_as_df(self, pair: str, timeframe: str, - since_ms: int, candle_type: str = '') -> DataFrame: + since_ms: int, candle_type: CandleType) -> DataFrame: """ Minimal wrapper around get_historic_ohlcv - converting the result into a dataframe :param pair: Pair to download :param timeframe: Timeframe to get data for :param since_ms: Timestamp in milliseconds to get history from - :param candle_type: '', mark, index, premiumIndex, or funding_rate + :param candle_type: Any of the enum CandleType (must match trading mode!) :return: OHLCV DataFrame """ ticks = self.get_historic_ohlcv(pair, timeframe, since_ms=since_ms, candle_type=candle_type) @@ -1349,14 +1349,13 @@ class Exchange: drop_incomplete=self._ohlcv_partial_candle) async def _async_get_historic_ohlcv(self, pair: str, timeframe: str, - since_ms: int, is_new_pair: bool = False, - raise_: bool = False, - candle_type: str = '' + since_ms: int, candle_type: CandleType, + is_new_pair: bool = False, raise_: bool = False, ) -> Tuple[str, str, str, List]: """ Download historic ohlcv :param is_new_pair: used by binance subclass to allow "fast" new pair downloading - :param candle_type: '', mark, index, premiumIndex, or funding_rate + :param candle_type: Any of the enum CandleType (must match trading mode!) """ one_call = timeframe_to_msecs(timeframe) * self.ohlcv_candle_limit(timeframe) @@ -1391,8 +1390,8 @@ class Exchange: def refresh_latest_ohlcv(self, pair_list: ListPairsWithTimeframes, *, since_ms: Optional[int] = None, cache: bool = True, - candle_type: str = '' - ) -> Dict[Tuple[str, str, str], DataFrame]: + candle_type: CandleType = CandleType.SPOT_ + ) -> Dict[PairWithTimeframe, DataFrame]: """ Refresh in-memory OHLCV asynchronously and set `_klines` with the result Loops asynchronously over pair_list and downloads all pairs async (semi-parallel). @@ -1410,7 +1409,7 @@ class Exchange: # Gather coroutines to run for pair, timeframe, candle_type in set(pair_list): if ((pair, timeframe, candle_type) not in self._klines or not cache - or self._now_is_time_to_refresh(pair, timeframe)): + or self._now_is_time_to_refresh(pair, timeframe, candle_type)): if not since_ms and self.required_candle_call_count > 1: # Multiple calls for one pair - to get more history one_call = timeframe_to_msecs(timeframe) * self.ohlcv_candle_limit(timeframe) @@ -1463,12 +1462,7 @@ class Exchange: return results_df - def _now_is_time_to_refresh( - self, - pair: str, - timeframe: str, - candle_type: str = '' - ) -> bool: + def _now_is_time_to_refresh(self, pair: str, timeframe: str, candle_type: CandleType) -> bool: # Timeframe in seconds interval_in_sec = timeframe_to_seconds(timeframe) diff --git a/freqtrade/optimize/backtesting.py b/freqtrade/optimize/backtesting.py index 95c92ee0b..43401be46 100644 --- a/freqtrade/optimize/backtesting.py +++ b/freqtrade/optimize/backtesting.py @@ -17,9 +17,7 @@ from freqtrade.data import history from freqtrade.data.btanalysis import trade_list_to_dataframe from freqtrade.data.converter import trim_dataframe, trim_dataframes from freqtrade.data.dataprovider import DataProvider -from freqtrade.enums import BacktestState, SellType -from freqtrade.enums.candletype import CandleType -from freqtrade.enums.tradingmode import TradingMode +from freqtrade.enums import BacktestState, CandleType, SellType, TradingMode from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds from freqtrade.mixins import LoggingMixin diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py index f2e9aa4d3..de7e26336 100644 --- a/freqtrade/plugins/pairlist/AgeFilter.py +++ b/freqtrade/plugins/pairlist/AgeFilter.py @@ -9,6 +9,7 @@ import arrow from pandas import DataFrame from freqtrade.configuration import PeriodicCache +from freqtrade.enums import CandleType from freqtrade.exceptions import OperationalException from freqtrade.misc import plural from freqtrade.plugins.pairlist.IPairList import IPairList @@ -72,7 +73,7 @@ class AgeFilter(IPairList): :return: new allowlist """ needed_pairs = [ - (p, '1d', '') for p in pairlist + (p, '1d', CandleType.SPOT_) for p in pairlist if p not in self._symbolsChecked and p not in self._symbolsCheckFailed] if not needed_pairs: # Remove pairs that have been removed before @@ -88,7 +89,8 @@ class AgeFilter(IPairList): candles = self._exchange.refresh_latest_ohlcv(needed_pairs, since_ms=since_ms, cache=False) if self._enabled: for p in deepcopy(pairlist): - daily_candles = candles[(p, '1d', '')] if (p, '1d', '') in candles else None + daily_candles = candles[(p, '1d', CandleType.SPOT_)] if ( + p, '1d', CandleType.SPOT_) in candles else None if not self._validate_pair_loc(p, daily_candles): pairlist.remove(p) self.log_once(f"Validated {len(pairlist)} pairs.", logger.info) diff --git a/freqtrade/plugins/pairlist/ShuffleFilter.py b/freqtrade/plugins/pairlist/ShuffleFilter.py index 55cf9938f..663bba49b 100644 --- a/freqtrade/plugins/pairlist/ShuffleFilter.py +++ b/freqtrade/plugins/pairlist/ShuffleFilter.py @@ -5,7 +5,7 @@ import logging import random from typing import Any, Dict, List -from freqtrade.enums.runmode import RunMode +from freqtrade.enums import RunMode from freqtrade.plugins.pairlist.IPairList import IPairList diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py index c06cd0897..299ea8c18 100644 --- a/freqtrade/plugins/pairlist/VolatilityFilter.py +++ b/freqtrade/plugins/pairlist/VolatilityFilter.py @@ -11,6 +11,7 @@ import numpy as np from cachetools.ttl import TTLCache from pandas import DataFrame +from freqtrade.enums import CandleType from freqtrade.exceptions import OperationalException from freqtrade.misc import plural from freqtrade.plugins.pairlist.IPairList import IPairList @@ -67,7 +68,7 @@ class VolatilityFilter(IPairList): :param tickers: Tickers (from exchange.get_tickers()). May be cached. :return: new allowlist """ - needed_pairs = [(p, '1d', '') for p in pairlist if p not in self._pair_cache] + needed_pairs = [(p, '1d', CandleType.SPOT_) for p in pairlist if p not in self._pair_cache] since_ms = (arrow.utcnow() .floor('day') @@ -81,7 +82,8 @@ class VolatilityFilter(IPairList): if self._enabled: for p in deepcopy(pairlist): - daily_candles = candles[(p, '1d', '')] if (p, '1d', '') in candles else None + daily_candles = candles[(p, '1d', CandleType.SPOT_)] if ( + p, '1d', CandleType.SPOT_) in candles else None if not self._validate_pair_loc(p, daily_candles): pairlist.remove(p) return pairlist diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py index 0493b67c9..7c9c9933a 100644 --- a/freqtrade/plugins/pairlist/VolumePairList.py +++ b/freqtrade/plugins/pairlist/VolumePairList.py @@ -10,6 +10,7 @@ from typing import Any, Dict, List import arrow from cachetools.ttl import TTLCache +from freqtrade.enums import CandleType from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_minutes from freqtrade.misc import format_ms_time @@ -160,7 +161,7 @@ class VolumePairList(IPairList): f"{self._lookback_timeframe}, starting from {format_ms_time(since_ms)} " f"till {format_ms_time(to_ms)}", logger.info) needed_pairs = [ - (p, self._lookback_timeframe, '') for p in + (p, self._lookback_timeframe, CandleType.SPOT_) for p in [s['symbol'] for s in filtered_tickers] if p not in self._pair_cache ] @@ -173,8 +174,8 @@ class VolumePairList(IPairList): ) for i, p in enumerate(filtered_tickers): pair_candles = candles[ - (p['symbol'], self._lookback_timeframe, '') - ] if (p['symbol'], self._lookback_timeframe, '') in candles else None + (p['symbol'], self._lookback_timeframe, CandleType.SPOT_) + ] if (p['symbol'], self._lookback_timeframe, CandleType.SPOT_) in candles else None # in case of candle data calculate typical price and quoteVolume for candle if pair_candles is not None and not pair_candles.empty: pair_candles['typical_price'] = (pair_candles['high'] + pair_candles['low'] diff --git a/freqtrade/plugins/pairlist/rangestabilityfilter.py b/freqtrade/plugins/pairlist/rangestabilityfilter.py index 048736ae0..31d92d41b 100644 --- a/freqtrade/plugins/pairlist/rangestabilityfilter.py +++ b/freqtrade/plugins/pairlist/rangestabilityfilter.py @@ -9,6 +9,7 @@ import arrow from cachetools.ttl import TTLCache from pandas import DataFrame +from freqtrade.enums import CandleType from freqtrade.exceptions import OperationalException from freqtrade.misc import plural from freqtrade.plugins.pairlist.IPairList import IPairList @@ -65,7 +66,7 @@ class RangeStabilityFilter(IPairList): :param tickers: Tickers (from exchange.get_tickers()). May be cached. :return: new allowlist """ - needed_pairs = [(p, '1d', '') for p in pairlist if p not in self._pair_cache] + needed_pairs = [(p, '1d', CandleType.SPOT_) for p in pairlist if p not in self._pair_cache] since_ms = (arrow.utcnow() .floor('day') @@ -79,7 +80,8 @@ class RangeStabilityFilter(IPairList): if self._enabled: for p in deepcopy(pairlist): - daily_candles = candles[(p, '1d', '')] if (p, '1d', '') in candles else None + daily_candles = candles[(p, '1d', CandleType.SPOT_)] if ( + p, '1d', CandleType.SPOT_) in candles else None if not self._validate_pair_loc(p, daily_candles): pairlist.remove(p) return pairlist diff --git a/freqtrade/plugins/pairlistmanager.py b/freqtrade/plugins/pairlistmanager.py index 5ea4c2386..5ae9a7e35 100644 --- a/freqtrade/plugins/pairlistmanager.py +++ b/freqtrade/plugins/pairlistmanager.py @@ -8,6 +8,7 @@ from typing import Dict, List from cachetools import TTLCache, cached from freqtrade.constants import ListPairsWithTimeframes +from freqtrade.enums import CandleType from freqtrade.exceptions import OperationalException from freqtrade.plugins.pairlist.IPairList import IPairList from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist @@ -138,4 +139,4 @@ class PairListManager(): """ Create list of pair tuples with (pair, timeframe) """ - return [(pair, timeframe or self._config['timeframe'], '') for pair in pairs] + return [(pair, timeframe or self._config['timeframe'], CandleType.SPOT) for pair in pairs] diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index f69ddef43..8bf3d41f1 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -9,6 +9,7 @@ from fastapi.exceptions import HTTPException from freqtrade import __version__ from freqtrade.constants import USERPATH_STRATEGIES from freqtrade.data.history import get_datahandler +from freqtrade.enums import CandleType from freqtrade.exceptions import OperationalException from freqtrade.rpc import RPC from freqtrade.rpc.api_server.api_schemas import (AvailablePairs, Balances, BlacklistPayload, @@ -250,7 +251,7 @@ def get_strategy(strategy: str, config=Depends(get_config)): @router.get('/available_pairs', response_model=AvailablePairs, tags=['candle data']) def list_available_pairs(timeframe: Optional[str] = None, stake_currency: Optional[str] = None, - candletype: Optional[str] = None, config=Depends(get_config)): + candletype: Optional[CandleType] = None, config=Depends(get_config)): dh = get_datahandler(config['datadir'], config.get('dataformat_ohlcv', None)) diff --git a/freqtrade/strategy/informative_decorator.py b/freqtrade/strategy/informative_decorator.py index 1507f09ab..3d939e017 100644 --- a/freqtrade/strategy/informative_decorator.py +++ b/freqtrade/strategy/informative_decorator.py @@ -2,6 +2,7 @@ from typing import Any, Callable, NamedTuple, Optional, Union from pandas import DataFrame +from freqtrade.enums import CandleType from freqtrade.exceptions import OperationalException from freqtrade.strategy.strategy_helper import merge_informative_pair @@ -14,7 +15,7 @@ class InformativeData(NamedTuple): timeframe: str fmt: Union[str, Callable[[Any], str], None] ffill: bool - candle_type: str = '' + candle_type: CandleType = CandleType.SPOT_ def informative(timeframe: str, asset: str = '', diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 8abb10bc7..2a3b4e754 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -13,8 +13,7 @@ from pandas import DataFrame from freqtrade.constants import ListPairsWithTimeframes from freqtrade.data.dataprovider import DataProvider -from freqtrade.enums import SellType, SignalDirection, SignalTagType, SignalType -from freqtrade.enums.candletype import CandleType +from freqtrade.enums import CandleType, SellType, SignalDirection, SignalTagType, SignalType from freqtrade.exceptions import OperationalException, StrategyError from freqtrade.exchange import timeframe_to_minutes, timeframe_to_seconds from freqtrade.exchange.exchange import timeframe_to_next_date @@ -424,7 +423,9 @@ class IStrategy(ABC, HyperStrategyMixin): """ informative_pairs = self.informative_pairs() # Compatibility code for 2 tuple informative pairs - informative_pairs = [(p[0], p[1], p[2] if len(p) > 2 else '') for p in informative_pairs] + informative_pairs = [ + (p[0], p[1], CandleType.from_string(p[2]) if len(p) > 2 else CandleType.SPOT_) + for p in informative_pairs] for inf_data, _ in self._ft_informative: if inf_data.asset: pair_tf = ( From d30aaaeaaa6f961fc24fda79c41e81204836a9d9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 3 Dec 2021 14:27:04 +0100 Subject: [PATCH 37/64] Tests should also use CandleType --- freqtrade/enums/candletype.py | 2 +- tests/exchange/test_exchange.py | 4 ++-- tests/test_freqtradebot.py | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/freqtrade/enums/candletype.py b/freqtrade/enums/candletype.py index 9818bb67b..ade0f68d1 100644 --- a/freqtrade/enums/candletype.py +++ b/freqtrade/enums/candletype.py @@ -16,5 +16,5 @@ class CandleType(str, Enum): def from_string(cls, value: str) -> 'CandleType': if not value: # Default to spot - return CandleType.SPOT + return CandleType.SPOT_ return CandleType(value) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index f1beedd84..0f1ba9869 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -11,7 +11,7 @@ import ccxt import pytest from pandas import DataFrame -from freqtrade.enums import Collateral, TradingMode +from freqtrade.enums import CandleType, Collateral, TradingMode from freqtrade.exceptions import (DDosProtection, DependencyException, InvalidOrderException, OperationalException, PricingError, TemporaryError) from freqtrade.exchange import Binance, Bittrex, Exchange, Kraken @@ -1685,7 +1685,7 @@ async def test__async_get_historic_ohlcv(default_conf, mocker, caplog, exchange_ pair = 'ETH/USDT' respair, restf, _, res = await exchange._async_get_historic_ohlcv( - pair, "5m", 1500000000000, is_new_pair=False) + pair, "5m", 1500000000000, candle_type=CandleType.SPOT, is_new_pair=False) assert respair == pair assert restf == '5m' # Call with very old timestamp - causes tons of requests diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 283ffc234..127fc1d1f 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -11,7 +11,7 @@ import arrow import pytest from freqtrade.constants import CANCEL_REASON, MATH_CLOSE_PREC, UNLIMITED_STAKE_AMOUNT -from freqtrade.enums import RPCMessageType, RunMode, SellType, SignalDirection, State +from freqtrade.enums import CandleType, RPCMessageType, RunMode, SellType, SignalDirection, State from freqtrade.exceptions import (DependencyException, ExchangeError, InsufficientFundsError, InvalidOrderException, OperationalException, PricingError, TemporaryError) @@ -696,9 +696,9 @@ def test_process_informative_pairs_added(default_conf_usdt, ticker_usdt, mocker) freqtrade.process() assert inf_pairs.call_count == 1 assert refresh_mock.call_count == 1 - assert ("BTC/ETH", "1m", '') in refresh_mock.call_args[0][0] - assert ("ETH/USDT", "1h", '') in refresh_mock.call_args[0][0] - assert ("ETH/USDT", default_conf_usdt["timeframe"], '') in refresh_mock.call_args[0][0] + assert ("BTC/ETH", "1m", CandleType.SPOT_) in refresh_mock.call_args[0][0] + assert ("ETH/USDT", "1h", CandleType.SPOT_) in refresh_mock.call_args[0][0] + assert ("ETH/USDT", default_conf_usdt["timeframe"], CandleType.SPOT) in refresh_mock.call_args[0][0] @pytest.mark.parametrize("trading_mode", [ From 69f371bf63beb04f96c584520650795082eac95d Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 3 Dec 2021 14:43:49 +0100 Subject: [PATCH 38/64] Update download-data to download necessary data for futures --- freqtrade/commands/data_commands.py | 4 +++- freqtrade/data/history/history_utils.py | 20 ++++++++++++++++++-- freqtrade/exchange/exchange.py | 3 +-- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index a75b4bfd2..6330de274 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -83,7 +83,9 @@ def start_download_data(args: Dict[str, Any]) -> None: exchange, pairs=expanded_pairs, timeframes=config['timeframes'], datadir=config['datadir'], timerange=timerange, new_pairs_days=config['new_pairs_days'], - erase=bool(config.get('erase')), data_format=config['dataformat_ohlcv']) + erase=bool(config.get('erase')), data_format=config['dataformat_ohlcv'], + trading_mode=config.get('trading_mode', 'spot'), + ) except KeyboardInterrupt: sys.exit("SIGINT received, aborting ...") diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index e9c31259f..eff705d06 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -248,10 +248,10 @@ def _download_pair_history(pair: str, *, def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes: List[str], - datadir: Path, timerange: Optional[TimeRange] = None, + datadir: Path, trading_mode: str, + timerange: Optional[TimeRange] = None, new_pairs_days: int = 30, erase: bool = False, data_format: str = None, - candle_type: CandleType = CandleType.SPOT ) -> List[str]: """ Refresh stored ohlcv data for backtesting and hyperopt operations. @@ -260,6 +260,7 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes """ pairs_not_available = [] data_handler = get_datahandler(datadir, data_format) + candle_type = CandleType.FUTURES if trading_mode == 'futures' else CandleType.SPOT_ for idx, pair in enumerate(pairs, start=1): if pair not in exchange.markets: pairs_not_available.append(pair) @@ -279,6 +280,21 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes timerange=timerange, data_handler=data_handler, timeframe=str(timeframe), new_pairs_days=new_pairs_days, candle_type=candle_type) + if trading_mode == 'futures': + # TODO-lev: Use correct candletype (and timeframe) depending on exchange + timeframe = '1h' + candle_type = CandleType.MARK + # TODO: this could be in most parts to the above. + if erase: + if data_handler.ohlcv_purge(pair, timeframe, candle_type=candle_type): + logger.info( + f'Deleting existing data for pair {pair}, interval {timeframe}.') + _download_pair_history(pair=pair, process=process, + datadir=datadir, exchange=exchange, + timerange=timerange, data_handler=data_handler, + timeframe=str(timeframe), new_pairs_days=new_pairs_days, + candle_type=candle_type) + return pairs_not_available diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index e75735c04..7ebf96619 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1326,7 +1326,6 @@ class Exchange: :param candle_type: '', mark, index, premiumIndex, or funding_rate :return: List with candle (OHLCV) data """ - data: List pair, _, _, data = asyncio.get_event_loop().run_until_complete( self._async_get_historic_ohlcv(pair=pair, timeframe=timeframe, since_ms=since_ms, is_new_pair=is_new_pair, @@ -1494,7 +1493,7 @@ class Exchange: pair, timeframe, since_ms, s ) params = deepcopy(self._ft_has.get('ohlcv_params', {})) - if candle_type: + if candle_type not in (CandleType.SPOT, CandleType.SPOT_): params.update({'price': candle_type}) data = await self._api_async.fetch_ohlcv(pair, timeframe=timeframe, since=since_ms, From 9421e6e61c876d271289609c36bf7c5c79530319 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 3 Dec 2021 14:57:09 +0100 Subject: [PATCH 39/64] Improve some tests --- tests/data/test_history.py | 16 ++++++++++++---- tests/leverage/test_candletype.py | 2 +- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 7944593c1..ed2822b93 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -487,7 +487,13 @@ def test_validate_backtest_data(default_conf, mocker, caplog, testdatadir) -> No assert len(caplog.record_tuples) == 0 -def test_refresh_backtest_ohlcv_data(mocker, default_conf, markets, caplog, testdatadir): +@pytest.mark.parametrize('trademode,callcount', [ + ('spot', 4), + ('margin', 4), + ('futures', 6), +]) +def test_refresh_backtest_ohlcv_data( + mocker, default_conf, markets, caplog, testdatadir, trademode, callcount): dl_mock = mocker.patch('freqtrade.data.history.history_utils._download_pair_history', MagicMock()) mocker.patch( @@ -500,10 +506,11 @@ def test_refresh_backtest_ohlcv_data(mocker, default_conf, markets, caplog, test timerange = TimeRange.parse_timerange("20190101-20190102") refresh_backtest_ohlcv_data(exchange=ex, pairs=["ETH/BTC", "XRP/BTC"], timeframes=["1m", "5m"], datadir=testdatadir, - timerange=timerange, erase=True + timerange=timerange, erase=True, + trading_mode=trademode ) - assert dl_mock.call_count == 4 + assert dl_mock.call_count == callcount assert dl_mock.call_args[1]['timerange'].starttype == 'date' assert log_has("Downloading pair ETH/BTC, interval 1m.", caplog) @@ -521,7 +528,8 @@ def test_download_data_no_markets(mocker, default_conf, caplog, testdatadir): unav_pairs = refresh_backtest_ohlcv_data(exchange=ex, pairs=["BTT/BTC", "LTC/USDT"], timeframes=["1m", "5m"], datadir=testdatadir, - timerange=timerange, erase=False + timerange=timerange, erase=False, + trading_mode='spot' ) assert dl_mock.call_count == 0 diff --git a/tests/leverage/test_candletype.py b/tests/leverage/test_candletype.py index 3eb73a07c..1424993ca 100644 --- a/tests/leverage/test_candletype.py +++ b/tests/leverage/test_candletype.py @@ -4,7 +4,7 @@ from freqtrade.enums import CandleType @pytest.mark.parametrize('input,expected', [ - ('', CandleType.SPOT), + ('', CandleType.SPOT_), ('spot', CandleType.SPOT), (CandleType.SPOT, CandleType.SPOT), (CandleType.FUTURES, CandleType.FUTURES), From bead867940f73bb4566db32bed67f5d7575b9db0 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 3 Dec 2021 15:08:00 +0100 Subject: [PATCH 40/64] Improve some typehints --- freqtrade/exchange/exchange.py | 4 +--- freqtrade/plugins/pairlist/AgeFilter.py | 3 ++- freqtrade/plugins/pairlist/VolatilityFilter.py | 4 +++- freqtrade/plugins/pairlist/VolumePairList.py | 3 ++- freqtrade/plugins/pairlist/rangestabilityfilter.py | 4 +++- 5 files changed, 11 insertions(+), 7 deletions(-) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 7ebf96619..e8e1f98c8 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1388,8 +1388,7 @@ class Exchange: return pair, timeframe, candle_type, data def refresh_latest_ohlcv(self, pair_list: ListPairsWithTimeframes, *, - since_ms: Optional[int] = None, cache: bool = True, - candle_type: CandleType = CandleType.SPOT_ + since_ms: Optional[int] = None, cache: bool = True ) -> Dict[PairWithTimeframe, DataFrame]: """ Refresh in-memory OHLCV asynchronously and set `_klines` with the result @@ -1398,7 +1397,6 @@ class Exchange: :param pair_list: List of 2 element tuples containing pair, interval to refresh :param since_ms: time since when to download, in milliseconds :param cache: Assign result to _klines. Usefull for one-off downloads like for pairlists - :param candle_type: '', mark, index, premiumIndex, or funding_rate :return: Dict of [{(pair, timeframe): Dataframe}] """ logger.debug("Refreshing candle (OHLCV) data for %d pairs", len(pair_list)) diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py index de7e26336..7c490d920 100644 --- a/freqtrade/plugins/pairlist/AgeFilter.py +++ b/freqtrade/plugins/pairlist/AgeFilter.py @@ -9,6 +9,7 @@ import arrow from pandas import DataFrame from freqtrade.configuration import PeriodicCache +from freqtrade.constants import ListPairsWithTimeframes from freqtrade.enums import CandleType from freqtrade.exceptions import OperationalException from freqtrade.misc import plural @@ -72,7 +73,7 @@ class AgeFilter(IPairList): :param tickers: Tickers (from exchange.get_tickers()). May be cached. :return: new allowlist """ - needed_pairs = [ + needed_pairs: ListPairsWithTimeframes = [ (p, '1d', CandleType.SPOT_) for p in pairlist if p not in self._symbolsChecked and p not in self._symbolsCheckFailed] if not needed_pairs: diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py index 299ea8c18..bdb7a043a 100644 --- a/freqtrade/plugins/pairlist/VolatilityFilter.py +++ b/freqtrade/plugins/pairlist/VolatilityFilter.py @@ -11,6 +11,7 @@ import numpy as np from cachetools.ttl import TTLCache from pandas import DataFrame +from freqtrade.constants import ListPairsWithTimeframes from freqtrade.enums import CandleType from freqtrade.exceptions import OperationalException from freqtrade.misc import plural @@ -68,7 +69,8 @@ class VolatilityFilter(IPairList): :param tickers: Tickers (from exchange.get_tickers()). May be cached. :return: new allowlist """ - needed_pairs = [(p, '1d', CandleType.SPOT_) for p in pairlist if p not in self._pair_cache] + needed_pairs: ListPairsWithTimeframes = [ + (p, '1d', CandleType.SPOT_) for p in pairlist if p not in self._pair_cache] since_ms = (arrow.utcnow() .floor('day') diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py index 7c9c9933a..b81a5db5c 100644 --- a/freqtrade/plugins/pairlist/VolumePairList.py +++ b/freqtrade/plugins/pairlist/VolumePairList.py @@ -10,6 +10,7 @@ from typing import Any, Dict, List import arrow from cachetools.ttl import TTLCache +from freqtrade.constants import ListPairsWithTimeframes from freqtrade.enums import CandleType from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_minutes @@ -160,7 +161,7 @@ class VolumePairList(IPairList): self.log_once(f"Using volume range of {self._lookback_period} candles, timeframe: " f"{self._lookback_timeframe}, starting from {format_ms_time(since_ms)} " f"till {format_ms_time(to_ms)}", logger.info) - needed_pairs = [ + needed_pairs: ListPairsWithTimeframes = [ (p, self._lookback_timeframe, CandleType.SPOT_) for p in [s['symbol'] for s in filtered_tickers] if p not in self._pair_cache diff --git a/freqtrade/plugins/pairlist/rangestabilityfilter.py b/freqtrade/plugins/pairlist/rangestabilityfilter.py index 31d92d41b..3e90400a6 100644 --- a/freqtrade/plugins/pairlist/rangestabilityfilter.py +++ b/freqtrade/plugins/pairlist/rangestabilityfilter.py @@ -9,6 +9,7 @@ import arrow from cachetools.ttl import TTLCache from pandas import DataFrame +from freqtrade.constants import ListPairsWithTimeframes from freqtrade.enums import CandleType from freqtrade.exceptions import OperationalException from freqtrade.misc import plural @@ -66,7 +67,8 @@ class RangeStabilityFilter(IPairList): :param tickers: Tickers (from exchange.get_tickers()). May be cached. :return: new allowlist """ - needed_pairs = [(p, '1d', CandleType.SPOT_) for p in pairlist if p not in self._pair_cache] + needed_pairs: ListPairsWithTimeframes = [ + (p, '1d', CandleType.SPOT_) for p in pairlist if p not in self._pair_cache] since_ms = (arrow.utcnow() .floor('day') From e75f31ee86805be0759e9cacc990266d76cc9181 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 3 Dec 2021 15:20:18 +0100 Subject: [PATCH 41/64] Create correct Type for PairWithTimeFrame --- freqtrade/constants.py | 6 +++--- tests/test_freqtradebot.py | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/freqtrade/constants.py b/freqtrade/constants.py index ebb817e8d..53f2c0ddf 100644 --- a/freqtrade/constants.py +++ b/freqtrade/constants.py @@ -5,6 +5,8 @@ bot constants """ from typing import List, Tuple +from freqtrade.enums import CandleType + DEFAULT_CONFIG = 'config.json' DEFAULT_EXCHANGE = 'bittrex' @@ -475,9 +477,7 @@ CANCEL_REASON = { } # List of pairs with their timeframes -# TODO-lev: This should really be -# PairWithTimeframe = Tuple[str, str, CandleType] -PairWithTimeframe = Tuple[str, str, str] +PairWithTimeframe = Tuple[str, str, CandleType] ListPairsWithTimeframes = List[PairWithTimeframe] # Type for trades list diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 127fc1d1f..6a6972b69 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -698,7 +698,8 @@ def test_process_informative_pairs_added(default_conf_usdt, ticker_usdt, mocker) assert refresh_mock.call_count == 1 assert ("BTC/ETH", "1m", CandleType.SPOT_) in refresh_mock.call_args[0][0] assert ("ETH/USDT", "1h", CandleType.SPOT_) in refresh_mock.call_args[0][0] - assert ("ETH/USDT", default_conf_usdt["timeframe"], CandleType.SPOT) in refresh_mock.call_args[0][0] + assert ("ETH/USDT", default_conf_usdt["timeframe"], + CandleType.SPOT) in refresh_mock.call_args[0][0] @pytest.mark.parametrize("trading_mode", [ From 5b779fd68ba6a95087d091049140d723f5af552f Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 3 Dec 2021 16:44:05 +0100 Subject: [PATCH 42/64] Update missing candle_type params --- freqtrade/exchange/binance.py | 2 +- freqtrade/exchange/exchange.py | 4 ++-- tests/exchange/test_exchange.py | 21 ++++++++++++--------- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/freqtrade/exchange/binance.py b/freqtrade/exchange/binance.py index db6cda4d7..10fc7ab65 100644 --- a/freqtrade/exchange/binance.py +++ b/freqtrade/exchange/binance.py @@ -206,7 +206,7 @@ class Binance(Exchange): :param candle_type: Any of the enum CandleType (must match trading mode!) """ if is_new_pair: - x = await self._async_get_candle_history(pair, timeframe, 0, candle_type) + x = await self._async_get_candle_history(pair, timeframe, candle_type, 0) if x and x[3] and x[3][0] and x[3][0][0] > since_ms: # Set starting date to first available candle. since_ms = x[3][0][0] diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index e8e1f98c8..d53f8c6e2 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1364,7 +1364,7 @@ class Exchange: arrow.utcnow().shift(seconds=one_call // 1000).humanize(only_distance=True) ) input_coroutines = [self._async_get_candle_history( - pair, timeframe, since, candle_type=candle_type) for since in + pair, timeframe, candle_type, since) for since in range(since_ms, arrow.utcnow().int_timestamp * 1000, one_call)] data: List = [] @@ -1475,8 +1475,8 @@ class Exchange: self, pair: str, timeframe: str, + candle_type: CandleType, since_ms: Optional[int] = None, - candle_type: str = '', ) -> Tuple[str, str, str, List]: """ Asynchronously get candle history data using fetch_ohlcv diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 0f1ba9869..1146acf65 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1792,12 +1792,12 @@ async def test__async_get_candle_history(default_conf, mocker, caplog, exchange_ exchange._api_async.fetch_ohlcv = get_mock_coro(ohlcv) pair = 'ETH/BTC' - res = await exchange._async_get_candle_history(pair, "5m") + res = await exchange._async_get_candle_history(pair, "5m", CandleType.SPOT) assert type(res) is tuple assert len(res) == 4 assert res[0] == pair assert res[1] == "5m" - assert res[2] == '' + assert res[2] == CandleType.SPOT assert res[3] == ohlcv assert exchange._api_async.fetch_ohlcv.call_count == 1 assert not log_has(f"Using cached candle (OHLCV) data for {pair} ...", caplog) @@ -1805,21 +1805,22 @@ async def test__async_get_candle_history(default_conf, mocker, caplog, exchange_ # exchange = Exchange(default_conf) await async_ccxt_exception(mocker, default_conf, MagicMock(), "_async_get_candle_history", "fetch_ohlcv", - pair='ABCD/BTC', timeframe=default_conf['timeframe']) + pair='ABCD/BTC', timeframe=default_conf['timeframe'], + candle_type=CandleType.SPOT) api_mock = MagicMock() with pytest.raises(OperationalException, match=r'Could not fetch historical candle \(OHLCV\) data.*'): api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.BaseError("Unknown error")) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) - await exchange._async_get_candle_history(pair, "5m", + await exchange._async_get_candle_history(pair, "5m", CandleType.SPOT, (arrow.utcnow().int_timestamp - 2000) * 1000) with pytest.raises(OperationalException, match=r'Exchange.* does not support fetching ' r'historical candle \(OHLCV\) data\..*'): api_mock.fetch_ohlcv = MagicMock(side_effect=ccxt.NotSupported("Not supported")) exchange = get_patched_exchange(mocker, default_conf, api_mock, id=exchange_name) - await exchange._async_get_candle_history(pair, "5m", + await exchange._async_get_candle_history(pair, "5m", CandleType.SPOT, (arrow.utcnow().int_timestamp - 2000) * 1000) @@ -1835,12 +1836,12 @@ async def test__async_get_candle_history_empty(default_conf, mocker, caplog): exchange = Exchange(default_conf) pair = 'ETH/BTC' - res = await exchange._async_get_candle_history(pair, "5m") + res = await exchange._async_get_candle_history(pair, "5m", CandleType.SPOT) assert type(res) is tuple assert len(res) == 4 assert res[0] == pair assert res[1] == "5m" - assert res[2] == '' + assert res[2] == zCandleType.SPOT assert res[3] == ohlcv assert exchange._api_async.fetch_ohlcv.call_count == 1 @@ -2140,7 +2141,8 @@ async def test___async_get_candle_history_sort(default_conf, mocker, exchange_na exchange._api_async.fetch_ohlcv = get_mock_coro(ohlcv) sort_mock = mocker.patch('freqtrade.exchange.exchange.sorted', MagicMock(side_effect=sort_data)) # Test the OHLCV data sort - res = await exchange._async_get_candle_history('ETH/BTC', default_conf['timeframe']) + res = await exchange._async_get_candle_history( + 'ETH/BTC', default_conf['timeframe'], CandleType.SPOT) assert res[0] == 'ETH/BTC' res_ohlcv = res[3] @@ -2177,7 +2179,8 @@ async def test___async_get_candle_history_sort(default_conf, mocker, exchange_na # Reset sort mock sort_mock = mocker.patch('freqtrade.exchange.sorted', MagicMock(side_effect=sort_data)) # Test the OHLCV data sort - res = await exchange._async_get_candle_history('ETH/BTC', default_conf['timeframe']) + res = await exchange._async_get_candle_history( + 'ETH/BTC', default_conf['timeframe'], CandleType.SPOT) assert res[0] == 'ETH/BTC' assert res[1] == default_conf['timeframe'] res_ohlcv = res[3] From 1a0861349813a07bda00be9548c3f23c9ad710c1 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sat, 4 Dec 2021 15:13:06 +0100 Subject: [PATCH 43/64] Fix parameter sequence in mock --- tests/exchange/test_exchange.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/exchange/test_exchange.py b/tests/exchange/test_exchange.py index 1146acf65..4632e6c56 100644 --- a/tests/exchange/test_exchange.py +++ b/tests/exchange/test_exchange.py @@ -1575,7 +1575,7 @@ def test_get_historic_ohlcv(default_conf, mocker, caplog, exchange_name, candle_ ] pair = 'ETH/BTC' - async def mock_candle_hist(pair, timeframe, since_ms, candle_type=None): + async def mock_candle_hist(pair, timeframe, candle_type, since_ms): return pair, timeframe, candle_type, ohlcv exchange._async_get_candle_history = Mock(wraps=mock_candle_hist) @@ -1641,7 +1641,7 @@ def test_get_historic_ohlcv_as_df(default_conf, mocker, exchange_name, candle_ty ] pair = 'ETH/BTC' - async def mock_candle_hist(pair, timeframe, since_ms, candle_type): + async def mock_candle_hist(pair, timeframe, candle_type, since_ms): return pair, timeframe, candle_type, ohlcv exchange._async_get_candle_history = Mock(wraps=mock_candle_hist) @@ -1841,7 +1841,7 @@ async def test__async_get_candle_history_empty(default_conf, mocker, caplog): assert len(res) == 4 assert res[0] == pair assert res[1] == "5m" - assert res[2] == zCandleType.SPOT + assert res[2] == CandleType.SPOT assert res[3] == ohlcv assert exchange._api_async.fetch_ohlcv.call_count == 1 From a80c3f6a1bb616bd66a1c75c23a499b58c85590a Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 5 Dec 2021 10:01:44 +0100 Subject: [PATCH 44/64] Use exchange-dependant timeframe/candletype to get mark/index candles --- freqtrade/data/history/history_utils.py | 9 ++++++--- freqtrade/exchange/exchange.py | 1 + freqtrade/exchange/ftx.py | 1 + freqtrade/exchange/kraken.py | 1 + 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index eff705d06..0970c0e95 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -281,9 +281,12 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes timeframe=str(timeframe), new_pairs_days=new_pairs_days, candle_type=candle_type) if trading_mode == 'futures': - # TODO-lev: Use correct candletype (and timeframe) depending on exchange - timeframe = '1h' - candle_type = CandleType.MARK + # Predefined candletype (and timeframe) depending on exchange + # Downloads what is necessary to backtest based on futures data. + timeframe = exchange._ft_has['mark_ohlcv_timeframe'] + candle_type = CandleType.from_string(exchange._ft_has['mark_ohlcv_price']) + # candle_type = CandleType.MARK + # TODO: this could be in most parts to the above. if erase: if data_handler.ohlcv_purge(pair, timeframe, candle_type=candle_type): diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index d53f8c6e2..6d0130b55 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -70,6 +70,7 @@ class Exchange: "l2_limit_range": None, "l2_limit_range_required": True, # Allow Empty L2 limit (kucoin) "mark_ohlcv_price": "mark", + "mark_ohlcv_timeframe": "8h", "ccxt_futures_name": "swap" } _ft_has: Dict = {} diff --git a/freqtrade/exchange/ftx.py b/freqtrade/exchange/ftx.py index fc7bc682e..36a08239d 100644 --- a/freqtrade/exchange/ftx.py +++ b/freqtrade/exchange/ftx.py @@ -21,6 +21,7 @@ class Ftx(Exchange): "stoploss_on_exchange": True, "ohlcv_candle_limit": 1500, "mark_ohlcv_price": "index", + "mark_ohlcv_timeframe": "1h", "ccxt_futures_name": "future" } diff --git a/freqtrade/exchange/kraken.py b/freqtrade/exchange/kraken.py index 42d817222..40944d15b 100644 --- a/freqtrade/exchange/kraken.py +++ b/freqtrade/exchange/kraken.py @@ -22,6 +22,7 @@ class Kraken(Exchange): "ohlcv_candle_limit": 720, "trades_pagination": "id", "trades_pagination_arg": "since", + "mark_ohlcv_timeframe": "4h", } _supported_trading_mode_collateral_pairs: List[Tuple[TradingMode, Collateral]] = [ From 9d79501c132305268c3712d20cdadbb4a7ad8603 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 5 Dec 2021 10:26:00 +0100 Subject: [PATCH 45/64] Add candletypes argument for convert-data --- freqtrade/commands/arguments.py | 4 +++- freqtrade/commands/cli_options.py | 7 +++++++ freqtrade/commands/data_commands.py | 10 ++++++---- freqtrade/configuration/configuration.py | 4 ++++ freqtrade/data/converter.py | 2 +- tests/data/test_converter.py | 7 +++++-- 6 files changed, 26 insertions(+), 8 deletions(-) diff --git a/freqtrade/commands/arguments.py b/freqtrade/commands/arguments.py index 711a2c4a2..4ddd16410 100644 --- a/freqtrade/commands/arguments.py +++ b/freqtrade/commands/arguments.py @@ -61,7 +61,9 @@ ARGS_BUILD_CONFIG = ["config"] ARGS_BUILD_STRATEGY = ["user_data_dir", "strategy", "template"] ARGS_CONVERT_DATA = ["pairs", "format_from", "format_to", "erase"] -ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes"] + +ARGS_CONVERT_DATA_OHLCV = ARGS_CONVERT_DATA + ["timeframes", "exchange", "trading_mode", + "candle_types"] ARGS_CONVERT_TRADES = ["pairs", "timeframes", "exchange", "dataformat_ohlcv", "dataformat_trades"] diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index 7d1d2edd1..d198a4b2f 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -5,6 +5,7 @@ from argparse import SUPPRESS, ArgumentTypeError from freqtrade import __version__, constants from freqtrade.constants import HYPEROPT_LOSS_BUILTIN +from freqtrade.enums import CandleType def check_int_positive(value: str) -> int: @@ -353,6 +354,12 @@ AVAILABLE_CLI_OPTIONS = { help='Select Trading mode', choices=constants.TRADING_MODES, ), + "candle_types": Arg( + '--candle-types', + help='Select Trading mode', + choices=[c.value for c in CandleType], + nargs='+', + ), # Script options "pairs": Arg( '-p', '--pairs', diff --git a/freqtrade/commands/data_commands.py b/freqtrade/commands/data_commands.py index 6330de274..0c6f48088 100644 --- a/freqtrade/commands/data_commands.py +++ b/freqtrade/commands/data_commands.py @@ -8,7 +8,7 @@ from freqtrade.configuration import TimeRange, setup_utils_configuration from freqtrade.data.converter import convert_ohlcv_format, convert_trades_format from freqtrade.data.history import (convert_trades_to_ohlcv, refresh_backtest_ohlcv_data, refresh_backtest_trades_data) -from freqtrade.enums import RunMode +from freqtrade.enums import CandleType, RunMode from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_minutes from freqtrade.exchange.exchange import market_is_active @@ -137,9 +137,11 @@ def start_convert_data(args: Dict[str, Any], ohlcv: bool = True) -> None: """ config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE) if ohlcv: - convert_ohlcv_format(config, - convert_from=args['format_from'], convert_to=args['format_to'], - erase=args['erase']) + candle_types = [CandleType.from_string(ct) for ct in config.get('candle_types', ['spot'])] + for candle_type in candle_types: + convert_ohlcv_format(config, + convert_from=args['format_from'], convert_to=args['format_to'], + erase=args['erase'], candle_type=candle_type) else: convert_trades_format(config, convert_from=args['format_from'], convert_to=args['format_to'], diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index 67617d84f..c86c43732 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -434,6 +434,10 @@ class Configuration: self._args_to_config(config, argname='trading_mode', logstring='Detected --trading-mode: {}') + self._args_to_config(config, argname='candle_types', + logstring='Detected --candle-types: {}') + + def _process_runmode(self, config: Dict[str, Any]) -> None: self._args_to_config(config, argname='dry_run', diff --git a/freqtrade/data/converter.py b/freqtrade/data/converter.py index c087c87ff..84c57be41 100644 --- a/freqtrade/data/converter.py +++ b/freqtrade/data/converter.py @@ -267,7 +267,7 @@ def convert_ohlcv_format( convert_from: str, convert_to: str, erase: bool, - candle_type: CandleType = CandleType.SPOT_ + candle_type: CandleType ): """ Convert OHLCV from one format to another diff --git a/tests/data/test_converter.py b/tests/data/test_converter.py index 39bb1b15e..fc71c9f0b 100644 --- a/tests/data/test_converter.py +++ b/tests/data/test_converter.py @@ -12,6 +12,7 @@ from freqtrade.data.converter import (convert_ohlcv_format, convert_trades_forma trades_to_ohlcv, trim_dataframe) from freqtrade.data.history import (get_timerange, load_data, load_pair_history, validate_backtest_data) +from freqtrade.enums import CandleType from tests.conftest import log_has, log_has_re from tests.data.test_history import _clean_test_file @@ -312,7 +313,8 @@ def test_convert_ohlcv_format(default_conf, testdatadir, tmpdir, file_base): default_conf, convert_from='json', convert_to='jsongz', - erase=False + erase=False, + candle_type=CandleType.SPOT ) assert file_new.exists() @@ -325,7 +327,8 @@ def test_convert_ohlcv_format(default_conf, testdatadir, tmpdir, file_base): default_conf, convert_from='jsongz', convert_to='json', - erase=True + erase=True, + candle_type=CandleType.SPOT ) assert file_temp.exists() From ce0df08ac76498644a3e66c93e7599ee3c244b21 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 6 Dec 2021 07:11:15 +0100 Subject: [PATCH 46/64] Update documentation of changed commands --- docs/data-download.md | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/docs/data-download.md b/docs/data-download.md index dbd7998c3..3bff1440c 100644 --- a/docs/data-download.md +++ b/docs/data-download.md @@ -29,6 +29,7 @@ usage: freqtrade download-data [-h] [-v] [--logfile FILE] [-V] [-c PATH] [--erase] [--data-format-ohlcv {json,jsongz,hdf5}] [--data-format-trades {json,jsongz,hdf5}] + [--trading-mode {spot,margin,futures}] optional arguments: -h, --help show this help message and exit @@ -59,6 +60,8 @@ optional arguments: --data-format-trades {json,jsongz,hdf5} Storage format for downloaded trades data. (default: `jsongz`). + --trading-mode {spot,margin,futures} + Select Trading mode Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). @@ -193,11 +196,14 @@ usage: freqtrade convert-data [-h] [-v] [--logfile FILE] [-V] [-c PATH] {json,jsongz,hdf5} --format-to {json,jsongz,hdf5} [--erase] [-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...]] + [--exchange EXCHANGE] + [--trading-mode {spot,margin,futures}] + [--candle-types {spot,,futures,mark,index,premiumIndex,funding_rate} [{spot,,futures,mark,index,premiumIndex,funding_rate} ...]] optional arguments: -h, --help show this help message and exit -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...] - Show profits for only these pairs. Pairs are space- + Limit command to these pairs. Pairs are space- separated. --format-from {json,jsongz,hdf5} Source format for data conversion. @@ -208,6 +214,12 @@ optional arguments: -t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...], --timeframes {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...] Specify which tickers to download. Space-separated list. Default: `1m 5m`. + --exchange EXCHANGE Exchange name (default: `bittrex`). Only valid if no + config is provided. + --trading-mode {spot,margin,futures} + Select Trading mode + --candle-types {spot,,futures,mark,index,premiumIndex,funding_rate} [{spot,,futures,mark,index,premiumIndex,funding_rate} ...] + Select Trading mode Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). @@ -224,6 +236,7 @@ Common arguments: Path to directory with historical backtesting data. --userdir PATH, --user-data-dir PATH Path to userdata directory. + ``` ##### Example converting data @@ -347,6 +360,7 @@ usage: freqtrade list-data [-h] [-v] [--logfile FILE] [-V] [-c PATH] [-d PATH] [--userdir PATH] [--exchange EXCHANGE] [--data-format-ohlcv {json,jsongz,hdf5}] [-p PAIRS [PAIRS ...]] + [--trading-mode {spot,margin,futures}] optional arguments: -h, --help show this help message and exit @@ -356,8 +370,10 @@ optional arguments: Storage format for downloaded candle (OHLCV) data. (default: `json`). -p PAIRS [PAIRS ...], --pairs PAIRS [PAIRS ...] - Show profits for only these pairs. Pairs are space- + Limit command to these pairs. Pairs are space- separated. + --trading-mode {spot,margin,futures} + Select Trading mode Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). From a58c2c4f6c4df021d954480175a8a2d4c7151269 Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 6 Dec 2021 20:03:32 +0100 Subject: [PATCH 47/64] Update ccxt_compat tests to also test funding_rate --- freqtrade/configuration/configuration.py | 1 - tests/exchange/test_ccxt_compat.py | 30 ++++++++++++++++++++---- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index c86c43732..beea3e507 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -437,7 +437,6 @@ class Configuration: self._args_to_config(config, argname='candle_types', logstring='Detected --candle-types: {}') - def _process_runmode(self, config: Dict[str, Any]) -> None: self._args_to_config(config, argname='dry_run', diff --git a/tests/exchange/test_ccxt_compat.py b/tests/exchange/test_ccxt_compat.py index ea0dc0fa4..8710463a6 100644 --- a/tests/exchange/test_ccxt_compat.py +++ b/tests/exchange/test_ccxt_compat.py @@ -11,6 +11,7 @@ from pathlib import Path import pytest +from freqtrade.enums import CandleType from freqtrade.exchange import timeframe_to_minutes, timeframe_to_prev_date from freqtrade.resolvers.exchange_resolver import ExchangeResolver from tests.conftest import get_default_conf_usdt @@ -51,14 +52,12 @@ EXCHANGES = { 'hasQuoteVolume': True, 'timeframe': '5m', 'futures': True, - 'futures_fundingrate_tf': '8h', 'futures_pair': 'BTC/USDT:USDT', }, 'okex': { 'pair': 'BTC/USDT', 'hasQuoteVolume': True, 'timeframe': '5m', - 'futures_fundingrate_tf': '8h', 'futures_pair': 'BTC/USDT:USDT', 'futures': True, }, @@ -182,7 +181,9 @@ class TestCCXTExchange(): exchange, exchangename = exchange pair = EXCHANGES[exchangename]['pair'] timeframe = EXCHANGES[exchangename]['timeframe'] - pair_tf = (pair, timeframe) + + pair_tf = (pair, timeframe, CandleType.SPOT) + ohlcv = exchange.refresh_latest_ohlcv([pair_tf]) assert isinstance(ohlcv, dict) assert len(ohlcv[pair_tf]) == len(exchange.klines(pair_tf)) @@ -193,7 +194,6 @@ class TestCCXTExchange(): now = datetime.now(timezone.utc) - timedelta(minutes=(timeframe_to_minutes(timeframe) * 2)) assert exchange.klines(pair_tf).iloc[-1]['date'] >= timeframe_to_prev_date(timeframe, now) - @pytest.mark.skip("No futures support yet") def test_ccxt_fetch_funding_rate_history(self, exchange_futures): # TODO-lev: enable this test once Futures mode is enabled. exchange, exchangename = exchange_futures @@ -206,12 +206,32 @@ class TestCCXTExchange(): rate = exchange.get_funding_rate_history(pair, since) assert isinstance(rate, dict) - expected_tf = EXCHANGES[exchangename].get('futures_fundingrate_tf', '1h') + + expected_tf = exchange._ft_has['mark_ohlcv_timeframe'] this_hour = timeframe_to_prev_date(expected_tf) prev_tick = timeframe_to_prev_date(expected_tf, this_hour - timedelta(minutes=1)) assert rate[int(this_hour.timestamp() * 1000)] != 0.0 assert rate[int(prev_tick.timestamp() * 1000)] != 0.0 + @pytest.mark.skip("No futures support yet") + def test_fetch_mark_price_history(self, exchange_futures): + exchange, exchangename = exchange_futures + if not exchange: + # exchange_futures only returns values for supported exchanges + return + pair = EXCHANGES[exchangename].get('futures_pair', EXCHANGES[exchangename]['pair']) + since = int((datetime.now(timezone.utc) - timedelta(days=5)).timestamp() * 1000) + + mark_candles = exchange._get_mark_price_history(pair, since) + + assert isinstance(mark_candles, dict) + expected_tf = '1h' + + this_hour = timeframe_to_prev_date(expected_tf) + prev_tick = timeframe_to_prev_date(expected_tf, this_hour - timedelta(minutes=1)) + assert mark_candles[int(this_hour.timestamp() * 1000)] != 0.0 + assert mark_candles[int(prev_tick.timestamp() * 1000)] != 0.0 + # TODO: tests fetch_trades (?) def test_ccxt_get_fee(self, exchange): From 5a3b907132ed29e23effdf13997521457f87f09a Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 7 Dec 2021 06:54:00 +0100 Subject: [PATCH 48/64] Update converter tests --- tests/commands/test_commands.py | 6 +-- tests/data/test_converter.py | 48 ++++++++++--------- tests/data/test_history.py | 2 +- tests/rpc/test_rpc_apiserver.py | 2 +- ..._USDT-1h.json => XRP_USDT-1h-futures.json} | 0 5 files changed, 31 insertions(+), 27 deletions(-) rename tests/testdata/futures/{XRP_USDT-1h.json => XRP_USDT-1h-futures.json} (100%) diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 5e5cb3c2f..90c8bb725 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -1373,9 +1373,9 @@ def test_start_list_data(testdatadir, capsys): captured = capsys.readouterr() assert "Found 3 pair / timeframe combinations." in captured.out - assert "\n| Pair | Timeframe | Type |\n" in captured.out - assert "\n| XRP/USDT | 1h | |\n" in captured.out - assert "\n| XRP/USDT | 1h | mark |\n" in captured.out + assert "\n| Pair | Timeframe | Type |\n" in captured.out + assert "\n| XRP/USDT | 1h | futures |\n" in captured.out + assert "\n| XRP/USDT | 1h | mark |\n" in captured.out @pytest.mark.usefixtures("init_persistence") diff --git a/tests/data/test_converter.py b/tests/data/test_converter.py index fc71c9f0b..fa3a7b6fe 100644 --- a/tests/data/test_converter.py +++ b/tests/data/test_converter.py @@ -12,6 +12,7 @@ from freqtrade.data.converter import (convert_ohlcv_format, convert_trades_forma trades_to_ohlcv, trim_dataframe) from freqtrade.data.history import (get_timerange, load_data, load_pair_history, validate_backtest_data) +from freqtrade.data.history.idatahandler import IDataHandler from freqtrade.enums import CandleType from tests.conftest import log_has, log_has_re from tests.data.test_history import _clean_test_file @@ -135,14 +136,15 @@ def test_ohlcv_fill_up_missing_data2(caplog): def test_ohlcv_drop_incomplete(caplog): timeframe = '1d' - ticks = [[ - 1559750400000, # 2019-06-04 - 8.794e-05, # open - 8.948e-05, # high - 8.794e-05, # low - 8.88e-05, # close - 2255, # volume (in quote currency) - ], + ticks = [ + [ + 1559750400000, # 2019-06-04 + 8.794e-05, # open + 8.948e-05, # high + 8.794e-05, # low + 8.88e-05, # close + 2255, # volume (in quote currency) + ], [ 1559836800000, # 2019-06-05 8.88e-05, @@ -150,7 +152,7 @@ def test_ohlcv_drop_incomplete(caplog): 8.88e-05, 8.893e-05, 9911, - ], + ], [ 1559923200000, # 2019-06-06 8.891e-05, @@ -158,7 +160,7 @@ def test_ohlcv_drop_incomplete(caplog): 8.875e-05, 8.877e-05, 2251 - ], + ], [ 1560009600000, # 2019-06-07 8.877e-05, @@ -166,7 +168,7 @@ def test_ohlcv_drop_incomplete(caplog): 8.895e-05, 8.817e-05, 123551 - ] + ] ] caplog.set_level(logging.DEBUG) data = ohlcv_to_dataframe(ticks, timeframe, pair="UNITTEST/BTC", @@ -289,17 +291,19 @@ def test_convert_trades_format(default_conf, testdatadir, tmpdir): file['new'].unlink() -@pytest.mark.parametrize('file_base', [ - ('XRP_ETH-5m'), - ('XRP_ETH-1m'), - # ('XRP_USDT-1h-mark'), #TODO-lev: Create .gz file +@pytest.mark.parametrize('file_base,candletype', [ + ('XRP_ETH-5m', CandleType.SPOT), + ('XRP_ETH-1m', CandleType.SPOT), + ('XRP_USDT-1h-mark', CandleType.MARK), + ('XRP_USDT-1h-futures', CandleType.FUTURES), ]) -def test_convert_ohlcv_format(default_conf, testdatadir, tmpdir, file_base): +def test_convert_ohlcv_format(default_conf, testdatadir, tmpdir, file_base, candletype): tmpdir1 = Path(tmpdir) - - file_orig = testdatadir / f"{file_base}.json" - file_temp = tmpdir1 / f"{file_base}.json" - file_new = tmpdir1 / f"{file_base}.json.gz" + prependix = '' if candletype == CandleType.SPOT else 'futures/' + file_orig = testdatadir / f"{prependix}{file_base}.json" + file_temp = tmpdir1 / f"{prependix}{file_base}.json" + file_new = tmpdir1 / f"{prependix}{file_base}.json.gz" + IDataHandler.create_dir_if_needed(file_temp) copyfile(file_orig, file_temp) @@ -314,7 +318,7 @@ def test_convert_ohlcv_format(default_conf, testdatadir, tmpdir, file_base): convert_from='json', convert_to='jsongz', erase=False, - candle_type=CandleType.SPOT + candle_type=candletype ) assert file_new.exists() @@ -328,7 +332,7 @@ def test_convert_ohlcv_format(default_conf, testdatadir, tmpdir, file_base): convert_from='jsongz', convert_to='json', erase=True, - candle_type=CandleType.SPOT + candle_type=candletype ) assert file_temp.exists() diff --git a/tests/data/test_history.py b/tests/data/test_history.py index ed2822b93..6fa62ab0c 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -743,7 +743,7 @@ def test_datahandler_ohlcv_get_available_data(testdatadir): # Convert to set to avoid failures due to sorting assert set(paircombs) == { ('UNITTEST/USDT', '1h', 'mark'), - ('XRP/USDT', '1h', ''), + ('XRP/USDT', '1h', 'futures'), ('XRP/USDT', '1h', 'mark'), } diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 0823c7aee..bf36b2e7f 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1356,7 +1356,7 @@ def test_list_available_pairs(botclient): ftbot.config['trading_mode'] = 'futures' rc = client_get( - client, f"{BASE_URI}/available_pairs?timeframe=1h") + client, f"{BASE_URI}/available_pairs?timeframe=1h&candletype=futures") assert_response(rc) assert rc.json()['length'] == 1 assert rc.json()['pairs'] == ['XRP/USDT'] diff --git a/tests/testdata/futures/XRP_USDT-1h.json b/tests/testdata/futures/XRP_USDT-1h-futures.json similarity index 100% rename from tests/testdata/futures/XRP_USDT-1h.json rename to tests/testdata/futures/XRP_USDT-1h-futures.json From cff950d7837b731abb7c1930ae1492198bcf29fb Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 7 Dec 2021 07:06:21 +0100 Subject: [PATCH 49/64] Update test_convert_ohlcv_format to test as before it did test conversion of multiple files, and that should be kept this way. --- tests/data/test_converter.py | 41 +++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/tests/data/test_converter.py b/tests/data/test_converter.py index fa3a7b6fe..c6b0059a2 100644 --- a/tests/data/test_converter.py +++ b/tests/data/test_converter.py @@ -292,23 +292,29 @@ def test_convert_trades_format(default_conf, testdatadir, tmpdir): @pytest.mark.parametrize('file_base,candletype', [ - ('XRP_ETH-5m', CandleType.SPOT), - ('XRP_ETH-1m', CandleType.SPOT), - ('XRP_USDT-1h-mark', CandleType.MARK), - ('XRP_USDT-1h-futures', CandleType.FUTURES), + (['XRP_ETH-5m', 'XRP_ETH-1m'], CandleType.SPOT), + (['UNITTEST_USDT-1h-mark', 'XRP_USDT-1h-mark'], CandleType.MARK), + (['XRP_USDT-1h-futures'], CandleType.FUTURES), ]) def test_convert_ohlcv_format(default_conf, testdatadir, tmpdir, file_base, candletype): tmpdir1 = Path(tmpdir) prependix = '' if candletype == CandleType.SPOT else 'futures/' - file_orig = testdatadir / f"{prependix}{file_base}.json" - file_temp = tmpdir1 / f"{prependix}{file_base}.json" - file_new = tmpdir1 / f"{prependix}{file_base}.json.gz" - IDataHandler.create_dir_if_needed(file_temp) + files_orig = [] + files_temp = [] + files_new = [] + for file in file_base: + file_orig = testdatadir / f"{prependix}{file}.json" + file_temp = tmpdir1 / f"{prependix}{file}.json" + file_new = tmpdir1 / f"{prependix}{file}.json.gz" + IDataHandler.create_dir_if_needed(file_temp) + copyfile(file_orig, file_temp) - copyfile(file_orig, file_temp) + files_orig.append(file_orig) + files_temp.append(file_temp) + files_new.append(file_new) default_conf['datadir'] = tmpdir1 - default_conf['pairs'] = ['XRP_ETH', 'XRP_USDT'] + default_conf['pairs'] = ['XRP_ETH', 'XRP_USDT', 'UNITTEST_USDT'] default_conf['timeframes'] = ['1m', '5m', '1h'] assert not file_new.exists() @@ -320,12 +326,12 @@ def test_convert_ohlcv_format(default_conf, testdatadir, tmpdir, file_base, cand erase=False, candle_type=candletype ) - - assert file_new.exists() - assert file_temp.exists() + for file in (files_temp + files_new): + assert file.exists() # Remove original files - file_temp.unlink() + for file in (files_temp): + file.unlink() # Convert back convert_ohlcv_format( default_conf, @@ -334,6 +340,7 @@ def test_convert_ohlcv_format(default_conf, testdatadir, tmpdir, file_base, cand erase=True, candle_type=candletype ) - - assert file_temp.exists() - assert not file_new.exists() + for file in (files_temp): + assert file.exists() + for file in (files_new): + assert not file.exists() From ba1091b9e41c39228fd244855e76f68538411699 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 7 Dec 2021 07:11:36 +0100 Subject: [PATCH 50/64] Improve dataprovider test --- tests/data/test_dataprovider.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/tests/data/test_dataprovider.py b/tests/data/test_dataprovider.py index 42e2bcdea..0e0685f96 100644 --- a/tests/data/test_dataprovider.py +++ b/tests/data/test_dataprovider.py @@ -16,27 +16,28 @@ from tests.conftest import get_patched_exchange 'mark', '', ]) -def test_ohlcv(mocker, default_conf, ohlcv_history, candle_type): +def test_dp_ohlcv(mocker, default_conf, ohlcv_history, candle_type): default_conf["runmode"] = RunMode.DRY_RUN timeframe = default_conf["timeframe"] exchange = get_patched_exchange(mocker, default_conf) exchange._klines[("XRP/BTC", timeframe, candle_type)] = ohlcv_history exchange._klines[("UNITTEST/BTC", timeframe, candle_type)] = ohlcv_history + candletype = CandleType.from_string(candle_type) dp = DataProvider(default_conf, exchange) assert dp.runmode == RunMode.DRY_RUN - assert ohlcv_history.equals(dp.ohlcv("UNITTEST/BTC", timeframe, candle_type=candle_type)) - assert isinstance(dp.ohlcv("UNITTEST/BTC", timeframe, candle_type=candle_type), DataFrame) - assert dp.ohlcv("UNITTEST/BTC", timeframe, candle_type=candle_type) is not ohlcv_history - assert dp.ohlcv("UNITTEST/BTC", timeframe, copy=False, candle_type=candle_type) is ohlcv_history - assert not dp.ohlcv("UNITTEST/BTC", timeframe, candle_type=candle_type).empty - assert dp.ohlcv("NONESENSE/AAA", timeframe, candle_type=candle_type).empty + assert ohlcv_history.equals(dp.ohlcv("UNITTEST/BTC", timeframe, candle_type=candletype)) + assert isinstance(dp.ohlcv("UNITTEST/BTC", timeframe, candle_type=candletype), DataFrame) + assert dp.ohlcv("UNITTEST/BTC", timeframe, candle_type=candletype) is not ohlcv_history + assert dp.ohlcv("UNITTEST/BTC", timeframe, copy=False, candle_type=candletype) is ohlcv_history + assert not dp.ohlcv("UNITTEST/BTC", timeframe, candle_type=candletype).empty + assert dp.ohlcv("NONESENSE/AAA", timeframe, candle_type=candletype).empty # Test with and without parameter assert dp.ohlcv( "UNITTEST/BTC", timeframe, - candle_type=candle_type + candle_type=candletype ).equals(dp.ohlcv("UNITTEST/BTC", candle_type=candle_type)) default_conf["runmode"] = RunMode.LIVE @@ -88,6 +89,7 @@ def test_historic_ohlcv_dataformat(mocker, default_conf, ohlcv_history): @pytest.mark.parametrize('candle_type', [ 'mark', + 'futures', '', ]) def test_get_pair_dataframe(mocker, default_conf, ohlcv_history, candle_type): @@ -97,10 +99,13 @@ def test_get_pair_dataframe(mocker, default_conf, ohlcv_history, candle_type): exchange._klines[("XRP/BTC", timeframe, candle_type)] = ohlcv_history exchange._klines[("UNITTEST/BTC", timeframe, candle_type)] = ohlcv_history + candletype = CandleType.from_string(candle_type) dp = DataProvider(default_conf, exchange) assert dp.runmode == RunMode.DRY_RUN assert ohlcv_history.equals(dp.get_pair_dataframe( "UNITTEST/BTC", timeframe, candle_type=candle_type)) + assert ohlcv_history.equals(dp.get_pair_dataframe( + "UNITTEST/BTC", timeframe, candle_type=candletype)) assert isinstance(dp.get_pair_dataframe( "UNITTEST/BTC", timeframe, candle_type=candle_type), DataFrame) assert dp.get_pair_dataframe("UNITTEST/BTC", timeframe, From a870e0962a49002b0a783302b44de9ebb7991630 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 7 Dec 2021 07:25:00 +0100 Subject: [PATCH 51/64] Fix some obtruse (test)bugs --- freqtrade/rpc/api_server/api_v1.py | 2 +- tests/data/test_history.py | 22 +++++----- tests/plugins/test_pairlist.py | 55 +++++++++++++------------ tests/strategy/test_strategy_helpers.py | 40 +++++++++++------- 4 files changed, 64 insertions(+), 55 deletions(-) diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 8bf3d41f1..98df84b7d 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -265,7 +265,7 @@ def list_available_pairs(timeframe: Optional[str] = None, stake_currency: Option if candletype: pair_interval = [pair for pair in pair_interval if pair[2] == candletype] else: - pair_interval = [pair for pair in pair_interval if pair[2] == ''] + pair_interval = [pair for pair in pair_interval if pair[2] == CandleType.SPOT_] pair_interval = sorted(pair_interval, key=lambda x: x[0]) diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 6fa62ab0c..b95757561 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -720,17 +720,17 @@ def test_datahandler_ohlcv_get_available_data(testdatadir): paircombs = JsonDataHandler.ohlcv_get_available_data(testdatadir, 'spot') # Convert to set to avoid failures due to sorting assert set(paircombs) == { - ('UNITTEST/BTC', '5m', ''), - ('ETH/BTC', '5m', ''), - ('XLM/BTC', '5m', ''), - ('TRX/BTC', '5m', ''), - ('LTC/BTC', '5m', ''), - ('XMR/BTC', '5m', ''), - ('ZEC/BTC', '5m', ''), - ('UNITTEST/BTC', '1m', ''), - ('ADA/BTC', '5m', ''), - ('ETC/BTC', '5m', ''), - ('NXT/BTC', '5m', ''), + ('UNITTEST/BTC', '5m', CandleType.SPOT_), + ('ETH/BTC', '5m', CandleType.SPOT_), + ('XLM/BTC', '5m', CandleType.SPOT_), + ('TRX/BTC', '5m', CandleType.SPOT_), + ('LTC/BTC', '5m', CandleType.SPOT_), + ('XMR/BTC', '5m', CandleType.SPOT_), + ('ZEC/BTC', '5m', CandleType.SPOT_), + ('UNITTEST/BTC', '1m', CandleType.SPOT_), + ('ADA/BTC', '5m', CandleType.SPOT_), + ('ETC/BTC', '5m', CandleType.SPOT_), + ('NXT/BTC', '5m', CandleType.SPOT_), ('DASH/BTC', '5m', ''), ('XRP/ETH', '1m', ''), ('XRP/ETH', '5m', ''), diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 4110ca95c..fe375a671 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -7,6 +7,7 @@ import pytest import time_machine from freqtrade.constants import AVAILABLE_PAIRLISTS +from freqtrade.enums.candletype import CandleType from freqtrade.enums.runmode import RunMode from freqtrade.exceptions import OperationalException from freqtrade.persistence import Trade @@ -461,11 +462,11 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t ohlcv_history_high_vola.loc[ohlcv_history_high_vola.index == 1, 'close'] = 0.00090 ohlcv_data = { - ('ETH/BTC', '1d', ''): ohlcv_history, - ('TKN/BTC', '1d', ''): ohlcv_history, - ('LTC/BTC', '1d', ''): ohlcv_history.append(ohlcv_history), - ('XRP/BTC', '1d', ''): ohlcv_history, - ('HOT/BTC', '1d', ''): ohlcv_history_high_vola, + ('ETH/BTC', '1d', CandleType.SPOT_): ohlcv_history, + ('TKN/BTC', '1d', CandleType.SPOT_): ohlcv_history, + ('LTC/BTC', '1d', CandleType.SPOT_): ohlcv_history.append(ohlcv_history), + ('XRP/BTC', '1d', CandleType.SPOT_): ohlcv_history, + ('HOT/BTC', '1d', CandleType.SPOT_): ohlcv_history_high_vola, } mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) @@ -579,11 +580,11 @@ def test_VolumePairList_range(mocker, whitelist_conf, shitcoinmarkets, tickers, ohlcv_history_high_volume.loc[:, 'volume'] = 10 ohlcv_data = { - ('ETH/BTC', '1d', ''): ohlcv_history, - ('TKN/BTC', '1d', ''): ohlcv_history, - ('LTC/BTC', '1d', ''): ohlcv_history_medium_volume, - ('XRP/BTC', '1d', ''): ohlcv_history_high_vola, - ('HOT/BTC', '1d', ''): ohlcv_history_high_volume, + ('ETH/BTC', '1d', CandleType.SPOT_): ohlcv_history, + ('TKN/BTC', '1d', CandleType.SPOT_): ohlcv_history, + ('LTC/BTC', '1d', CandleType.SPOT_): ohlcv_history_medium_volume, + ('XRP/BTC', '1d', CandleType.SPOT_): ohlcv_history_high_vola, + ('HOT/BTC', '1d', CandleType.SPOT_): ohlcv_history_high_volume, } mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) @@ -855,9 +856,9 @@ def test_agefilter_min_days_listed_too_large(mocker, default_conf, markets, tick def test_agefilter_caching(mocker, markets, whitelist_conf_agefilter, tickers, ohlcv_history): with time_machine.travel("2021-09-01 05:00:00 +00:00") as t: ohlcv_data = { - ('ETH/BTC', '1d', ''): ohlcv_history, - ('TKN/BTC', '1d', ''): ohlcv_history, - ('LTC/BTC', '1d', ''): ohlcv_history, + ('ETH/BTC', '1d', CandleType.SPOT_): ohlcv_history, + ('TKN/BTC', '1d', CandleType.SPOT_): ohlcv_history, + ('LTC/BTC', '1d', CandleType.SPOT_): ohlcv_history, } mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -879,10 +880,10 @@ def test_agefilter_caching(mocker, markets, whitelist_conf_agefilter, tickers, o assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 2 ohlcv_data = { - ('ETH/BTC', '1d', ''): ohlcv_history, - ('TKN/BTC', '1d', ''): ohlcv_history, - ('LTC/BTC', '1d', ''): ohlcv_history, - ('XRP/BTC', '1d', ''): ohlcv_history.iloc[[0]], + ('ETH/BTC', '1d', CandleType.SPOT_): ohlcv_history, + ('TKN/BTC', '1d', CandleType.SPOT_): ohlcv_history, + ('LTC/BTC', '1d', CandleType.SPOT_): ohlcv_history, + ('XRP/BTC', '1d', CandleType.SPOT_): ohlcv_history.iloc[[0]], } mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', return_value=ohlcv_data) freqtrade.pairlists.refresh_pairlist() @@ -900,10 +901,10 @@ def test_agefilter_caching(mocker, markets, whitelist_conf_agefilter, tickers, o t.move_to("2021-09-03 01:00:00 +00:00") # Called once for XRP/BTC ohlcv_data = { - ('ETH/BTC', '1d', ''): ohlcv_history, - ('TKN/BTC', '1d', ''): ohlcv_history, - ('LTC/BTC', '1d', ''): ohlcv_history, - ('XRP/BTC', '1d', ''): ohlcv_history, + ('ETH/BTC', '1d', CandleType.SPOT_): ohlcv_history, + ('TKN/BTC', '1d', CandleType.SPOT_): ohlcv_history, + ('LTC/BTC', '1d', CandleType.SPOT_): ohlcv_history, + ('XRP/BTC', '1d', CandleType.SPOT_): ohlcv_history, } mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', return_value=ohlcv_data) freqtrade.pairlists.refresh_pairlist() @@ -964,12 +965,12 @@ def test_rangestabilityfilter_caching(mocker, markets, default_conf, tickers, oh get_tickers=tickers ) ohlcv_data = { - ('ETH/BTC', '1d', ''): ohlcv_history, - ('TKN/BTC', '1d', ''): ohlcv_history, - ('LTC/BTC', '1d', ''): ohlcv_history, - ('XRP/BTC', '1d', ''): ohlcv_history, - ('HOT/BTC', '1d', ''): ohlcv_history, - ('BLK/BTC', '1d', ''): ohlcv_history, + ('ETH/BTC', '1d', CandleType.SPOT_): ohlcv_history, + ('TKN/BTC', '1d', CandleType.SPOT_): ohlcv_history, + ('LTC/BTC', '1d', CandleType.SPOT_): ohlcv_history, + ('XRP/BTC', '1d', CandleType.SPOT_): ohlcv_history, + ('HOT/BTC', '1d', CandleType.SPOT_): ohlcv_history, + ('BLK/BTC', '1d', CandleType.SPOT_): ohlcv_history, } mocker.patch.multiple( 'freqtrade.exchange.Exchange', diff --git a/tests/strategy/test_strategy_helpers.py b/tests/strategy/test_strategy_helpers.py index e7019b767..08fb3563e 100644 --- a/tests/strategy/test_strategy_helpers.py +++ b/tests/strategy/test_strategy_helpers.py @@ -5,6 +5,7 @@ import pandas as pd import pytest from freqtrade.data.dataprovider import DataProvider +from freqtrade.enums.candletype import CandleType from freqtrade.strategy import (merge_informative_pair, stoploss_from_absolute, stoploss_from_open, timeframe_to_minutes) from tests.conftest import get_patched_exchange @@ -151,18 +152,18 @@ def test_informative_decorator(mocker, default_conf): test_data_30m = generate_test_data('30m', 40) test_data_1h = generate_test_data('1h', 40) data = { - ('XRP/USDT', '5m', ''): test_data_5m, - ('XRP/USDT', '30m', ''): test_data_30m, - ('XRP/USDT', '1h', ''): test_data_1h, - ('LTC/USDT', '5m', ''): test_data_5m, - ('LTC/USDT', '30m', ''): test_data_30m, - ('LTC/USDT', '1h', ''): test_data_1h, - ('NEO/USDT', '30m', ''): test_data_30m, - ('NEO/USDT', '5m', ''): test_data_5m, - ('NEO/USDT', '1h', ''): test_data_1h, - ('ETH/USDT', '1h', ''): test_data_1h, - ('ETH/USDT', '30m', ''): test_data_30m, - ('ETH/BTC', '1h', ''): test_data_1h, + ('XRP/USDT', '5m', CandleType.SPOT_): test_data_5m, + ('XRP/USDT', '30m', CandleType.SPOT_): test_data_30m, + ('XRP/USDT', '1h', CandleType.SPOT_): test_data_1h, + ('LTC/USDT', '5m', CandleType.SPOT_): test_data_5m, + ('LTC/USDT', '30m', CandleType.SPOT_): test_data_30m, + ('LTC/USDT', '1h', CandleType.SPOT_): test_data_1h, + ('NEO/USDT', '30m', CandleType.SPOT_): test_data_30m, + ('NEO/USDT', '5m', CandleType.SPOT_): test_data_5m, + ('NEO/USDT', '1h', CandleType.SPOT_): test_data_1h, + ('ETH/USDT', '1h', CandleType.SPOT_): test_data_1h, + ('ETH/USDT', '30m', CandleType.SPOT_): test_data_30m, + ('ETH/BTC', '1h', CandleType.SPOT_): test_data_1h, } from .strats.informative_decorator_strategy import InformativeDecoratorTest default_conf['stake_currency'] = 'USDT' @@ -174,9 +175,16 @@ def test_informative_decorator(mocker, default_conf): ]) assert len(strategy._ft_informative) == 6 # Equal to number of decorators used - informative_pairs = [('XRP/USDT', '1h', ''), ('LTC/USDT', '1h', ''), ('XRP/USDT', '30m', ''), - ('LTC/USDT', '30m', ''), ('NEO/USDT', '1h', ''), ('NEO/USDT', '30m', ''), - ('NEO/USDT', '5m', ''), ('ETH/BTC', '1h', ''), ('ETH/USDT', '30m', '')] + informative_pairs = [ + ('XRP/USDT', '1h', CandleType.SPOT_), + ('LTC/USDT', '1h', CandleType.SPOT_), + ('XRP/USDT', '30m', CandleType.SPOT_), + ('LTC/USDT', '30m', CandleType.SPOT_), + ('NEO/USDT', '1h', CandleType.SPOT_), + ('NEO/USDT', '30m', CandleType.SPOT_), + ('NEO/USDT', '5m', CandleType.SPOT_), + ('ETH/BTC', '1h', CandleType.SPOT_), + ('ETH/USDT', '30m', CandleType.SPOT_)] for inf_pair in informative_pairs: assert inf_pair in strategy.gather_informative_pairs() @@ -186,7 +194,7 @@ def test_informative_decorator(mocker, default_conf): side_effect=test_historic_ohlcv) analyzed = strategy.advise_all_indicators( - {p: data[(p, strategy.timeframe, '')] for p in ('XRP/USDT', 'LTC/USDT')}) + {p: data[(p, strategy.timeframe, CandleType.SPOT_)] for p in ('XRP/USDT', 'LTC/USDT')}) expected_columns = [ 'rsi_1h', 'rsi_30m', # Stacked informative decorators 'neo_usdt_rsi_1h', # NEO 1h informative From 37b013c1573630c06b79bb23543c04782af3eec6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 7 Dec 2021 19:49:48 +0100 Subject: [PATCH 52/64] Update hdf5 test --- tests/data/test_history.py | 29 ++++++++++-------- .../testdata/futures/UNITTEST_USDT-1h-mark.h5 | Bin 0 -> 37751 bytes 2 files changed, 17 insertions(+), 12 deletions(-) create mode 100644 tests/testdata/futures/UNITTEST_USDT-1h-mark.h5 diff --git a/tests/data/test_history.py b/tests/data/test_history.py index b95757561..18f9dc194 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -902,10 +902,11 @@ def test_hdf5datahandler_trades_purge(mocker, testdatadir): assert unlinkmock.call_count == 1 -@pytest.mark.parametrize('pair, timeframe, candle_type, candle_append', [ - ('UNITTEST/BTC', '5m', '', ''), - # TODO-lev: The test below - # ('UNITTEST/USDT', '1h', 'mark', '-mark'), +@pytest.mark.parametrize('pair,timeframe,candle_type,candle_append,startdt,enddt', [ + # Data goes from 2018-01-10 - 2018-01-30 + ('UNITTEST/BTC', '5m', '', '', '2018-01-15', '2018-01-19'), + # Mark data goes from to 2021-11-15 2021-11-19 + ('UNITTEST/USDT', '1h', 'mark', '-mark', '2021-11-16', '2021-11-18'), ]) def test_hdf5datahandler_ohlcv_load_and_resave( testdatadir, @@ -913,33 +914,37 @@ def test_hdf5datahandler_ohlcv_load_and_resave( pair, timeframe, candle_type, - candle_append + candle_append, + startdt, enddt ): tmpdir1 = Path(tmpdir) + tmpdir2 = tmpdir1 + if candle_type not in ('', 'spot'): + tmpdir2 = tmpdir1 / 'futures' + tmpdir2.mkdir() dh = HDF5DataHandler(testdatadir) - ohlcv = dh.ohlcv_load(pair, timeframe, candle_type=candle_type) + ohlcv = dh._ohlcv_load(pair, timeframe, candle_type=candle_type) assert isinstance(ohlcv, DataFrame) assert len(ohlcv) > 0 - file = tmpdir1 / f"UNITTEST_NEW-{timeframe}{candle_append}.h5" + file = tmpdir2 / f"UNITTEST_NEW-{timeframe}{candle_append}.h5" assert not file.is_file() dh1 = HDF5DataHandler(tmpdir1) dh1.ohlcv_store('UNITTEST/NEW', timeframe, ohlcv, candle_type=candle_type) assert file.is_file() - assert not ohlcv[ohlcv['date'] < '2018-01-15'].empty + assert not ohlcv[ohlcv['date'] < startdt].empty - # Data gores from 2018-01-10 - 2018-01-30 - timerange = TimeRange.parse_timerange('20180115-20180119') + timerange = TimeRange.parse_timerange(f"{startdt.replace('-', '')}-{enddt.replace('-', '')}") # Call private function to ensure timerange is filtered in hdf5 ohlcv = dh._ohlcv_load(pair, timeframe, timerange, candle_type=candle_type) ohlcv1 = dh1._ohlcv_load('UNITTEST/NEW', timeframe, timerange, candle_type=candle_type) assert len(ohlcv) == len(ohlcv1) assert ohlcv.equals(ohlcv1) - assert ohlcv[ohlcv['date'] < '2018-01-15'].empty - assert ohlcv[ohlcv['date'] > '2018-01-19'].empty + assert ohlcv[ohlcv['date'] < startdt].empty + assert ohlcv[ohlcv['date'] > enddt].empty # Try loading inexisting file ohlcv = dh.ohlcv_load('UNITTEST/NONEXIST', timeframe, candle_type=candle_type) diff --git a/tests/testdata/futures/UNITTEST_USDT-1h-mark.h5 b/tests/testdata/futures/UNITTEST_USDT-1h-mark.h5 new file mode 100644 index 0000000000000000000000000000000000000000..ce17eb9e104af3472fbdb121cd4453a34bad6088 GIT binary patch literal 37751 zcmeG_2V4}#*Lxfyo^(XOwr5lV7McZ7z(aZ!>0*Hcjt+N$quLNfQHUZ$L}LYGz+Mpn z6)c!&RP2fhh#d<8_JDjdyE}IWh)NRw$)`8JA3Hnc&6|1e&6~F~Z;r**=9aRIJ`4;7 zX=zLnQy^Z^qm>G=nPd)1qjDeWg+|3GRGcS4zR)l!43?=;`3h9~OgfoSL*=_VTUcPA zCTbtWR54a6MZQU+3~~HTQNY^5)CmsQlNvRWr{5?!L&R`%v~hK{aB)RxB$q6d(ZyJi zlGpZfErT(MmY{sYS>tNsY7Yb$QwGxq;so6ydM1Nuiqw9tGEpXcN&B%}<~Th_Hc-uusj;pq^w!@%*F+_KijD=O>s3LPYs2qMPbpCA7orC@D9kSgB)*iM$`cD9B6XKyq^Bw<`aLJVT!08Uu1e%Tb?{yTa~@q^!eSZ{KCPLv(^DSIleFIiu#r#??EzP4KJ^%RpDpbxcvq4DzB zzF{1Y@DBDyuS5~jTjVYVs{QG>{deCRYJL4Qq_~DC1(2R%Dv`M?if{iFJtf@Vk|FlD zc~!)p`FVtHXHp+JQXY)A>LDMo=BGoQQp9pS*H`!Nq90&9jw(H!!T}>N320)<4 z5H{w{1>x-YD=E#*Jn0MuQi>Fm4n;ItIMPaKoa*!Mg#2&F7633{c2t zSz>&kmuxt?=83$h=g^d`qgjs=WLyZydt1UtezlEdq;Zwm5ZbAXCSE_x97I1zqa z|KNx~q7XdbKCbBgtedOl2&kikc)F8tk&)a;NGDKVRF0zfk->g^Q9RuV&V_MeI8X)@ zz=BDxnJfY&;v0%(uw7JsHv&&^L;x3NKz=L*f2f|OKad;94u8mw2q64Xkhy?7BN>kD zD0etfAT!);pa4Q{L`5MH8v~&X7*QR>WGpN&1O*g*fn!((x$Bh zlQbI&>S=;MhywBm21Vus0X;|c#74D(c46;f8RT?(dPAy0cnLZYfmy0cywIqA(`LhR zDGW@VC_h+_`h$Ia-il#8Kn$x&gMD`x(1Sk%ZH011Cqc3tPd^xv<#i56xEy z(6JqID#3Py{m>V*^XYydx{I#Ep@6(Z_XxUo7r`4G!RG{WA^|QQQGrJEUa;O$boCZA z0QK=W5qNT4qSiaO+>)yySjoxN5H679QVADGv>1Xl66j7Cgmi~g>wq_$!zWb;#0zy2 zqL*U_c$E*q`y;~%%A4r81H!m${+LlB_!$H|ln3Ex$@K?-3h;JZhY`I+;03)V;XmGQ zl1~I)L2t?V0k`_lcwsp|IGh8zOa$S2E=9}mmzi;9B5z-eZ9ttPb z2ZF9v+f1nJVGt;MGAVwFDIlhRm;z!7h$$eZz*j^837XgkUh zHF}J}Si^DSjf_nu;4IULX66=_R@OGQcJ>aAlbj|yySTc!dw5RqnmTRz3~xyvwyz&n zgDjMf#g-U;P$Rx%V(t08t_~+gEOJ_^enLqnckyqQvCFu6XOFDaQO5^O`^Em8@BFKc zS8g=f4=YN3y~gckWq->9@i(g#-zMd*!jlu0hMt)eqQ)7qq+i42c;~dr^E#=?)46l= z%kJw;u?mpfZg$*(?2jgfH`Ao1_4p&pw7*KE&vD*#L$}a(j|OL-UEoqYOMSI=L=>*# zr8h}WAy>z=-*KZb<>B76_sW*W+lH(e5L7efb!|p5{+O1~@QAf&kNVUzXNxwlejf3D zM{rB*?V;B<<$b66F49I)wbe^OfNxQV?D3I1qckGZ>?DkL=k(+>FrFk7AHB%0FrQ3M z&Ap13%X)b-J?~yS$Nas}WR6Ci^%HY?d~3AbO{}Mp`pI!w_~ch9T6>g_6xBXH**s^` z^|6vGCZ=%?PIoqT+;q61;%%a&ndGc!0lpU@EwSPEwbV!2Zat8dlQ6Sd>w>>}*1Dv@ z)9foH7t@sR-ixhjXBu9X4aja8zaxLlgK(=`ah_2UKdj|u9#YMwrQ?O?EwOC2#%0gc z{UK#{e>!Nm;?$MbDcbCKmZS38Zt|-o4OJPq0N;)4%D%1G4#S&u1D}~lg;uIyX}lBZ zF{&$nUEDY7Oi3f2G}DcCXrJMY%eiu)vg>h&+9ln4=4G#T+dnVgS+3YY9v_e&TYl|W zXq4)LjdBq-!>TnkCQZIuUmNDB%-Wm%@{yP1`C*BP0(>4`)7))MhxWuI8}7fjSaZ9~ z(xO+R{F|0m#Z9yLR>@`fnr)e!nbr%l0U{H>Xr_U}xYGIc>x)`F}B33&2W zg^|q{FI_lL;G6F0Y&7upFIz6(K3=>r?{?;l= z{M8J=m^wA!+u(T(H1r8M2fI8IYP_(!P}a4Uw?-Bk|(SNsH|VJ#dSia>5jSi;c2PYAUxy#@ufo^EK$y%XYPf59^ewGk;dYulV1tHP36TeOP>Q@hRijx7^J4>+_w9=nf7(FG>?Z?M?c+qHo z;%ttM-H)pVZyUmieitxJm6tZF;pO(!%)W8>%G}Vo3j+=`&T_NxNaV_ zBc3rzWz87{IXq|V`J$4ww#A`mdzJ*RS*oPGQr_=Lg8Q@Q3J3D!9qk6-axk-&lwZ(wD+1kh+#!+iIHcj(#o7Cw4zk;Q!xZJDTU%Z{ym!0I(^0Gc0LdP`zuS_ToOKgF zUsAkd?aZY+HYTSBOg9ahF+aw7S82(apCeM1=WN`#G8zAAe^!X~49V)tM+<(+Ne;AS z(JQN(8t+$@JghB_J#hFm{iLikLCaRH79h>w!?dwR3TvgeN(R<|!<0rARwKa#!rTpG zD!@(=;Yh1wI)3nYO7dzYYqKAfmks}U_{$)xa7^R!iT5vZPC8ytxaId7{>(i-bjpJu zZ5tVjSf|E`V@)hu4EwpwyL?^u3OL?apNYSm91+LfvS8WSh=<q7_cIi1})_-6FmX7gD3mP!||_3wxG#VhyGa@LQt z_oDADx4)$;UApnT%-lD+uPket-nqZf{_zgIho6Nlg|PQE{KYEn16sLtLe)il4`X}L zO>RQP823u`dvAVx5o$lLG6QGC2mcW!$9gN_Z@{M-cBR>5VLJ!sN!b*Nl-p03#vud#TPR5ShNzFCN*eM9gYva1}DZss^H z9d?m9$D zJB=}G=IS`SCB56hStA)a6SO9$)gK$Ps86r*Dd0I9m1H^LzDIubt-dVSe!E<@AR>)snK!Eoo#}%^5{Qr zxqq5MICv@6ln5mnh;>7CtK@WG@N{pLTu6KVyj8hXG8>dZHBHopk0Y&GO#Jj^jXK+v zeX@tN-Y~Fva}?MdiHCC4e)na$bC1O94PAh%ABanNH}TT^t0S6vabEhb+Yr2_{P5xa z!)C{iQdPWZ^9Em4xp$ea$(zN^>#l8Dsx_)RW^cFZaEprc`^NmmRXR*7`iRpEE>h37 zQ)#EN-E;G#&N&$zNf`x9jsv3~`Rp1|HM{u0AKv4N@hHvnIc$?lxjH_Xdudl5?cZGf z{>Y+N+p4mpv=59)d{K=TkF7siyyNJMarH0nX2dJbDxZ{jZByBuZL`m8x*(w{6G@+X zMNt;1=S+G?V6a(q)jYP|=GoaYz~W%-dV()8*}9|R5o<=qUi|7_57vT*d+uJ72$`W= zYIY+l=gB#{KU#l#AlvV}(K&rvJj-{<%kO3VEqTB8yzCJ@X3j`Y=UIBC&Hc*9I9By& zFnUXW{Eiof)U)%{koZ4qUHhy!eD?ePN;oh$lHB~)8gz3nn0Ahncsd4GGEHCq{QCaO zvIo}76uPTwZ+~P}xHKcmsH{aAJ0!m(22USRqbaYlW%t?g7{dokjnsl_E2eoxId~-- zuxgg9eJxK<{Z4uXQqL7bt`G1SF#4KVN%iotHj2RBs7>M(>6dEw{yL|5_S_qIqej4Q zLE8=+-gvh3UUZL3qh3B)!W)leH{MeIy~=sey)azfWbU+{cYJQ9*d3Enqa7OfkoL5% zRlTM^eY0{;JbS<+`htyaK1e->djfOAjx;wO5axyzVQw@7b7LmoIvFopwJp;x0hk*B zR^|r1Nps_u{W}*;nU+MD8+bzUR{7D*mo5Nv!#{nBvk`ueGBwm|0pkW58$d~GzNqOXus3>$3bJJS$4u-w81@EOB;+_Mu({w}WD;P0Dn8f-R!t(a zv~65t(Vgr|YmY_v7Qeoi(k$K=a5ZaIO=cXMIqK;zGkTv(VZGfGGU&!n4&Cs>u{sYo zOi?;D-{{45&VZ}StZ4@$hu&G^DqB9spgxbSj$cSMYu*>GQMJ-Zx-4G*aqcvgGYd^# z?+O}NnPEpSi5@|}zutpfbhKLK)NzM7JF68;f5^EgIf|2Y^0nra#YK<%+0WilCC~a{ z;|2OC+e=C)h1Cfp4bG=0C! z<1dG}FDiVfZkw?6rmS(q`U*_Xm!5I##Sm(QN?UQ2sT&_?^NTRcA1|tV|MFNr?y#io z@~i#*@}G~HbyRxN3jD^v!0j)_C6>#$tg0Roy626_f>VkfYwu`r3z{dUoK=~MZ+tKU z!lx1NdJsMx!q-E1#}zjQ(gU|QjiV5%8snSux9!|paPUy+vGP-ADlT5FymkBTgSux8 zFJAxop4LseCoZR?qShDxZiv=!9sMzeMia~|Z5*6j+`Xpz_yvZ9|G>v%6XwiYkdnG| z`HJkH*X8_@w{7R1{Rc~q94-I-kBW;|uHT~9+__&zA?$k&I{?@C!Wt8j`&z9`Hr5-- zf@bUBCb(1mVUP7b@VEek8sdq4(i61En5>9S04XqW^o>xUy@wTf-icvH4cig`ofx7M zJakgRYFq8IQ^+$Yi8`6iq43FIK13U&r?qsOn{P&-1mU%T4hNhlrXM$g&kl~@L^1=o zkxasmhN*+D>FL1xKsto8y$<2Zsw40Y6Q0C&Jh>J1v~aa?G`H~l{F7UBbB>0|5q(AU zM4k!~6G#&A;4u{lqz9ygP|XucpcU-@kRfRz8MCM~V>0<7J`th_@|8zue(s{i8AQd6 z@c0n~Bv(1EWZH?0z$qoz-2egJU2EfM0P^TWJ}kvBJ?f?=nG!zRdyqH>2+Kb7hSraF z)cW(GI^SX*;*SJHKOuj}8T=tV2c4wb=s7~S5gHiiJLt3~s*gJAFHz4jN_}-M4+?GN z0o~jco_I?WC*Hub!WCQ=?)1bPdB_Mo4CD#8sAwK1lJIMY3Qhq2QUD#0M;KVd6O9uX z&qP`X&miDMK#B6s;Yo$s9zH58*pEY^7QmC3m_%BL|HJTpLD3PRp94>w)C=+VAo|bc z@d>9q0vUL8<}m=*92kV~Z8;gJ=dk$njFk z4Onl;G?Ld-W@NoWs(MF{wy^{r8VLrVp+_4HiwcSk2mrp=NFPA`1M~ek0b$T#4)p?h zw!uJ`prbAjMSzS1XnW5#p**&echH8Y=LR9~rq0f$Q;6m%Pq=QBKPHCh+mZR5JvFI< zB%97^koiy!VMsD3&7SX}Xg$!yhe_dT?+VfHH%Lbrr zc$o#shYL57&+%uv6OPCrUkFEfx>|U;n%g*#{rT{G20$RtfONn%^QeZqIv)h%25pGq z>FRtSVafHli}Qg1{io(b*ZI}SItAzfXE5TIV|2JLFv7V3OhRKagCj^|{O|H@!zcJg zsULy;jM4(W0c}F+A{OW^^yqExnfs6HF}TLTZu~7hW@c&zOh2I>>pI_biSc%@FF-E5 z)QF@GOJddbfCIU46(D$m4zN9@UYDb9bHy1uTlF#&o+=0{RZ_gt!U;a8@ zMD28o2!9c-i1{L1M>>Ar{0r-jV8RIJSy%0DSyDH4Vh_xtzBf^fzB`?%@1%4%SwZ*` z_8ROGxKWkkMqnW9deTG$>5le5TRTeB9uUp1Kt`G&N?gV zrIU!Xp1|!7%T6j#*$7Ld>Bs9?Q^oHleY_rN?PJ&}`;XWAw@Q(mG*bJWo>WJK^XK#S zJB2rhI<6l$Xp<6K%$^;^XF79ZnNj2iDS2a%ISeT;w9k(K+TR#M`zk0#{r9EL-4I{8 zD(9c`JxR&=h3OQH01N{AVseh;f+QDl^Ac{rVaSjTr(TePVN)*&R2)vlB)q*~*97c- z!gd51h(Cya2ya(o#1C6a83^U6?YtMYFBAY0gkK;ZR0lh=m)q~_MR3A6;hYFQVU&RO z+V88N_Wh{=3EBl|l8H>B0Wk%{6cAHD zOo6Y50-f0l?Jh101}wK+@JhZ=_5u~PD-}SBSLm_>_d!JWH_$!^q6OM-L;Is>|3)d8tns;iwHaY#eHPVlpPA$f z+W(}o#nD$vfzJGDCBPLN7?51;q0&xdHaI`jB;Nk2UoC1^iP}dKwo6JL@;eahH!5J5 z3c!fw0pa&f_=7@Ee1SWtNWa>h9|3r=KPaRh=}#&42c_h<=pNV-x#>5^@2h1IP#Z30xv2^f&BGpM1G>M(htt@B`Vi?d$>M|0(tdB?T4T zll$0uCH4oUOaVZ$j_VU?Nulo>Q9r6L?cXNqrzQ3WC9O5MPQzae7q0*9z8gXLm*$J8 zT_W}e{qOKa)b6J4As~0Hzu`aq*Ze_!sqdt8QVoAef6zT>%>xED$&3%`FJVg;p6p4YahcZ;lC^V4+gZKCP6XJ zlj@^Tul@h<2Q3OBJC69$RlAqmGl$<_o!_^GIv@Bw*Dj)HguW$Vvg3OF&&WB23j6=& z59-xHp4!qwbe}>23=)0!ey#qXs_0@RVki|GbmHHE^iaF%;-r`YVhV^UAf|wr0%8h$ z1r+G2Kd577yFpWiE3#kPm5P&M3WzBnrhu3NVhV^U@D)%1`GY0|kb_xG?R$xytHJ-o z___X|akQruZWF=bC97YoRtF_)jZ%eXhcZu0i? zb?aU%49xvKYM3IZ?aVf9TrZ?>8p<=1>pq^QTm<_{ zgPZF|H@uz{F5g{IQ$|hKr!lceIy2rmQ+&fJ=Q45 zX)McbTp4@Q!e@C$tjv6Pi%PfnKN@DdqqX2(s8P21Z_$O7cb{GuySu5s+yn!TR`t5t z(?vU{wVp__-so~>jQ5qzr>57x_PIG_)5|zV>H6K{nqCi68ln?DzV&*>));OAcCySi z_iC!2`MHAYv#ZlL#)W1WG_7;K71(sFn|lwLXS!<9w}Pb8E-5_UK6S2l>2BV6PVdZ1 z!6%P|n#Vj#7;(p?_}PFXX19;p{h^`dvTU}?vWGX{+Ex|^JT*4?GbvehRa)k+{aQBn zuG)R)m!D!^t}ZT#dD~N4x6hDCCXH^Rx6MA;(`M(E%dv6cP4~ThbnRF3-=uEYS!XSK zYx3M>3rlO{Xy+d$pL=N=qgVKHMpl~lyUILYJoW1F$9sMsHX-|RzuM>P6Q=6TeyX~4 zp`TIN$oj1T+;{A&k+!-_lbL1~P?S7@@y6vS9 z-lDR+$ChS3n--StSrK}LTU8KMKkwK{$F+xeZ{KY%mS1l)_vN!YE9;N6>g|c-HT76* zYkhiHen{QJzN@BGUK?K=|4vGBk;Ms(!M_IH>z(zJ$K}xleEw^PHRJqVl~~w$Y@2c< z`0-ieKjr(6kH3HPUZ_F6NnG=j%X>Tqjpkdm826EwKVGun4sCp%WI?^!O}1`b(Rl^6 d{DDkMRb6MZ*h|$w3JSq*F>X+@J^tD9^M9^+dN%+7 literal 0 HcmV?d00001 From b4d27973b18531a7bc38fbbb6522875528d6838f Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 7 Dec 2021 19:57:18 +0100 Subject: [PATCH 53/64] Update ohlcv_get_pairs test --- freqtrade/data/history/hdf5datahandler.py | 2 +- freqtrade/data/history/idatahandler.py | 2 +- freqtrade/data/history/jsondatahandler.py | 2 +- tests/data/test_history.py | 17 ++++++++--------- .../futures/XRP_USDT-1h-futures.json.gz | Bin 0 -> 2471 bytes 5 files changed, 11 insertions(+), 12 deletions(-) create mode 100644 tests/testdata/futures/XRP_USDT-1h-futures.json.gz diff --git a/freqtrade/data/history/hdf5datahandler.py b/freqtrade/data/history/hdf5datahandler.py index fe840527f..239b9a99d 100644 --- a/freqtrade/data/history/hdf5datahandler.py +++ b/freqtrade/data/history/hdf5datahandler.py @@ -48,7 +48,7 @@ class HDF5DataHandler(IDataHandler): cls, datadir: Path, timeframe: str, - candle_type: CandleType = CandleType.SPOT_ + candle_type: CandleType ) -> List[str]: """ Returns a list of all pairs with ohlcv data available in this datadir diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index 239c9ab71..59a4bc5e4 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -52,7 +52,7 @@ class IDataHandler(ABC): cls, datadir: Path, timeframe: str, - candle_type: CandleType = CandleType.SPOT_ + candle_type: CandleType ) -> List[str]: """ Returns a list of all pairs with ohlcv data available in this datadir diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index 1ed5ae023..2180b6799 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -49,7 +49,7 @@ class JsonDataHandler(IDataHandler): cls, datadir: Path, timeframe: str, - candle_type: CandleType = CandleType.SPOT_ + candle_type: CandleType ) -> List[str]: """ Returns a list of all pairs with ohlcv data available in this datadir diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 18f9dc194..89b02f295 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -657,27 +657,26 @@ def test_convert_trades_to_ohlcv(testdatadir, tmpdir, caplog): def test_datahandler_ohlcv_get_pairs(testdatadir): - pairs = JsonDataHandler.ohlcv_get_pairs(testdatadir, '5m') + pairs = JsonDataHandler.ohlcv_get_pairs(testdatadir, '5m', candle_type=CandleType.SPOT) # Convert to set to avoid failures due to sorting assert set(pairs) == {'UNITTEST/BTC', 'XLM/BTC', 'ETH/BTC', 'TRX/BTC', 'LTC/BTC', 'XMR/BTC', 'ZEC/BTC', 'ADA/BTC', 'ETC/BTC', 'NXT/BTC', 'DASH/BTC', 'XRP/ETH'} - pairs = JsonGzDataHandler.ohlcv_get_pairs(testdatadir, '8m') + pairs = JsonGzDataHandler.ohlcv_get_pairs(testdatadir, '8m', candle_type=CandleType.SPOT) assert set(pairs) == {'UNITTEST/BTC'} - pairs = HDF5DataHandler.ohlcv_get_pairs(testdatadir, '5m') + pairs = HDF5DataHandler.ohlcv_get_pairs(testdatadir, '5m', candle_type=CandleType.SPOT) assert set(pairs) == {'UNITTEST/BTC'} - pairs = JsonDataHandler.ohlcv_get_pairs(testdatadir, '1h', candle_type='mark') + pairs = JsonDataHandler.ohlcv_get_pairs(testdatadir, '1h', candle_type=CandleType.MARK) assert set(pairs) == {'UNITTEST/USDT', 'XRP/USDT'} - # TODO-lev: The tests below - # pairs = JsonGzDataHandler.ohlcv_get_pairs(testdatadir, '8m') - # assert set(pairs) == {'UNITTEST/BTC'} + pairs = JsonGzDataHandler.ohlcv_get_pairs(testdatadir, '1h', candle_type=CandleType.FUTURES) + assert set(pairs) == {'XRP/USDT'} - # pairs = HDF5DataHandler.ohlcv_get_pairs(testdatadir, '5m') - # assert set(pairs) == {'UNITTEST/BTC'} + pairs = HDF5DataHandler.ohlcv_get_pairs(testdatadir, '1h', candle_type=CandleType.MARK) + assert set(pairs) == {'UNITTEST/USDT'} @pytest.mark.parametrize('filename,pair,timeframe,candletype', [ diff --git a/tests/testdata/futures/XRP_USDT-1h-futures.json.gz b/tests/testdata/futures/XRP_USDT-1h-futures.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..f2a223b03952d7d1d8c7d2d8f8d89dfbe667a1dd GIT binary patch literal 2471 zcmV;Y30U?YiwFpyt*>DM|5#E`UsY2?R4p-REoOCeb#i5ME^2dcZU9tSYtr312-{F5 zWAn0E%QxNsHYyA!A^p+ZbdCv@K@z@yf75u)IrfM@*SzsxGakNl$8(v=c{=IEQ`uZ@ z8;jNdUA0=|S-rN~iL&&(vd)Y$?IQDd;%)f(AUzTC^BDobPXwj2R4bzGA(xCPW_E9i z6nY4n_Jpu4nfsPpI&5WV@{5-tir#8#iDJckLMXK?B(qqa8{pyRo$)a9!TX+*7uhGm zZaB)6+{ejb31=Y3zdB7MdG96S+wtcYfT=<0XA>^&nSpU;1YAgb4hIg#x~ZqY_CmRB zXNbwUbPTB@MlQZtw%tgG)&nEE^-4$#2AwB>PZ$S4U?C@2w{HbBJpX9+%+N$5IA0xh`+g@dGSX7orz*iVS{qUpN%gc_Wv z%Ak58)ZT=Sx@kGf>{-~PR#J72K>nbaAyHz2Cwc+}jn>e$KN~S=RjDi6+lZP%l4bLw%Y}zdW&i#9` z45)6_-U0$M@Z3Go5-!hjYw$@`P##AX12%Dz2cO)ieVxxz9}T?%ZLi`c&36s(Q1UeS z#)kCEDbBqJtN$$d8NbeViBPhA+qzLdbFEK^MQ$imi{k?A^Ez}aPk>it;+N;6LP943=Qc>tjb_S#5yiL3n}iqvhxH3FUzM4M-X~e60+Ou&GY(@ZNEfBN z$j>(R6X6w^d3GGnHImZwNFV;)AhqDMHSPs+KGpMVW4kAXVCR}nlQw06oSn2aHEK%g0;K&?lL;ESE!yD$QJbdVSEqDTMDqQ#=RoCz zIL=VTfEru*Ew?5sT3Vo&aj@d66t+6zoZWJokY^o)@?X)#5i~G77!b zz;5z_pJs@vKi2F>SQ`f)8AM^-s9cb1FU0-IOZ+tRhG{=pq+=Xl`VG?=pvp}Q%kw!0 zZxalq$-oBAKH%>(5dkN{+C(#80mnhSoak+cNqs7z8H5)- z|Wwn@luL@QN>pUf^W*dbI6 z;E4Vi{1D*Y{oW>(*Qvgl&R|C|f@SBHJLZe^eA%0N#63nL{wv*Go6&1w$^rFam*cte zVQ=!WpJvgt^t({wm|7PG7ABBexY_b* z%$e)e!i{1e0^fo7Q9b)v-LJdWsQH+y+p3Gt@^jEoZh*P{S>7}we->8*zu7p^cM26F z%#%KDi;IO`G2v7jZ`9>b%|TP^4v{X0d&btGVe0SNK8mbbO*O@6;kKd&VO?#*G{C2L zFL`+LUUYSOD=+FG(WS!>Bb+l7+;0>$inpDfP?HU-6F@HkS)` z?O>~!&Pwy@&Mw5%F_&Pd4G&BY0Q`}SVjlXfu6zil#Lzx$VSeoCAB=eYh9*l!vV73(+VFvF_*>gg0xoN`TzS5009600|4j@mpVNe002*@uj&8* literal 0 HcmV?d00001 From 5b67be06c223a51f61832dd75bd81da3b9b4d013 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 7 Dec 2021 20:00:12 +0100 Subject: [PATCH 54/64] Update description of --candletypes --- docs/data-download.md | 2 +- freqtrade/commands/cli_options.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/data-download.md b/docs/data-download.md index 3bff1440c..9bfc1e685 100644 --- a/docs/data-download.md +++ b/docs/data-download.md @@ -219,7 +219,7 @@ optional arguments: --trading-mode {spot,margin,futures} Select Trading mode --candle-types {spot,,futures,mark,index,premiumIndex,funding_rate} [{spot,,futures,mark,index,premiumIndex,funding_rate} ...] - Select Trading mode + Select candle type to use Common arguments: -v, --verbose Verbose mode (-vv for more, -vvv to get all messages). diff --git a/freqtrade/commands/cli_options.py b/freqtrade/commands/cli_options.py index d198a4b2f..33d751f54 100644 --- a/freqtrade/commands/cli_options.py +++ b/freqtrade/commands/cli_options.py @@ -356,7 +356,7 @@ AVAILABLE_CLI_OPTIONS = { ), "candle_types": Arg( '--candle-types', - help='Select Trading mode', + help='Select candle type to use', choices=[c.value for c in CandleType], nargs='+', ), From f1c5a4d0651bb4383f0d7eac7c858614a5457060 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 7 Dec 2021 20:12:44 +0100 Subject: [PATCH 55/64] Use pair-reconstruction method wherever possible --- freqtrade/data/history/hdf5datahandler.py | 4 ++-- freqtrade/data/history/jsondatahandler.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/freqtrade/data/history/hdf5datahandler.py b/freqtrade/data/history/hdf5datahandler.py index 239b9a99d..73b8aedba 100644 --- a/freqtrade/data/history/hdf5datahandler.py +++ b/freqtrade/data/history/hdf5datahandler.py @@ -66,7 +66,7 @@ class HDF5DataHandler(IDataHandler): _tmp = [re.search(r'^(\S+)(?=\-' + timeframe + candle + '.h5)', p.name) for p in datadir.glob(f"*{timeframe}{candle}.h5")] # Check if regex found something and only return these results - return [match[0].replace('_', '/') for match in _tmp if match] + return [cls.rebuild_pair_from_filename(match[0]) for match in _tmp if match] def ohlcv_store( self, @@ -160,7 +160,7 @@ class HDF5DataHandler(IDataHandler): _tmp = [re.search(r'^(\S+)(?=\-trades.h5)', p.name) for p in datadir.glob("*trades.h5")] # Check if regex found something and only return these results to avoid exceptions. - return [match[0].replace('_', '/') for match in _tmp if match] + return [cls.rebuild_pair_from_filename(match[0]) for match in _tmp if match] def trades_store(self, pair: str, data: TradeList) -> None: """ diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index 2180b6799..f329d4879 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -67,7 +67,7 @@ class JsonDataHandler(IDataHandler): _tmp = [re.search(r'^(\S+)(?=\-' + timeframe + candle + '.json)', p.name) for p in datadir.glob(f"*{timeframe}{candle}.{cls._get_file_extension()}")] # Check if regex found something and only return these results - return [match[0].replace('_', '/') for match in _tmp if match] + return [cls.rebuild_pair_from_filename(match[0]) for match in _tmp if match] def ohlcv_store( self, @@ -156,7 +156,7 @@ class JsonDataHandler(IDataHandler): _tmp = [re.search(r'^(\S+)(?=\-trades.json)', p.name) for p in datadir.glob(f"*trades.{cls._get_file_extension()}")] # Check if regex found something and only return these results to avoid exceptions. - return [match[0].replace('_', '/') for match in _tmp if match] + return [cls.rebuild_pair_from_filename(match[0]) for match in _tmp if match] def trades_store(self, pair: str, data: TradeList) -> None: """ From ac2fb08aead572125bad259dd97b66977dcc2866 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 7 Dec 2021 20:21:49 +0100 Subject: [PATCH 56/64] Small updates while reviewing --- freqtrade/data/history/history_utils.py | 6 ++---- freqtrade/strategy/informative_decorator.py | 1 + freqtrade/strategy/interface.py | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index 0970c0e95..793c8b839 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -270,8 +270,7 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes if erase: if data_handler.ohlcv_purge(pair, timeframe, candle_type=candle_type): - logger.info( - f'Deleting existing data for pair {pair}, interval {timeframe}.') + logger.info(f'Deleting existing data for pair {pair}, interval {timeframe}.') logger.info(f'Downloading pair {pair}, interval {timeframe}.') process = f'{idx}/{len(pairs)}' @@ -290,8 +289,7 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes # TODO: this could be in most parts to the above. if erase: if data_handler.ohlcv_purge(pair, timeframe, candle_type=candle_type): - logger.info( - f'Deleting existing data for pair {pair}, interval {timeframe}.') + logger.info(f'Deleting existing data for pair {pair}, interval {timeframe}.') _download_pair_history(pair=pair, process=process, datadir=datadir, exchange=exchange, timerange=timerange, data_handler=data_handler, diff --git a/freqtrade/strategy/informative_decorator.py b/freqtrade/strategy/informative_decorator.py index 3d939e017..2085630cb 100644 --- a/freqtrade/strategy/informative_decorator.py +++ b/freqtrade/strategy/informative_decorator.py @@ -57,6 +57,7 @@ def informative(timeframe: str, asset: str = '', def decorator(fn: PopulateIndicators): informative_pairs = getattr(fn, '_ft_informative', []) + # TODO-lev: Add candle_type to InformativeData informative_pairs.append(InformativeData(_asset, _timeframe, _fmt, _ffill)) setattr(fn, '_ft_informative', informative_pairs) return fn diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 2a3b4e754..7c3fd60f1 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -530,7 +530,7 @@ class IStrategy(ABC, HyperStrategyMixin): dataframe = self.analyze_ticker(dataframe, metadata) self._last_candle_seen_per_pair[pair] = dataframe.iloc[-1]['date'] if self.dp: - self.dp._set_cached_df(pair, self.timeframe, dataframe, CandleType.SPOT) + self.dp._set_cached_df(pair, self.timeframe, dataframe, CandleType.SPOT_) # TODO-lev: CandleType should be set conditionally else: logger.debug("Skipping TA Analysis for already analyzed candle") From dda7283f3e88ab7fb7df22fe9ac9cdf58cb9317f Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 7 Dec 2021 20:30:58 +0100 Subject: [PATCH 57/64] Remove unnecessary default parameters --- freqtrade/data/dataprovider.py | 2 +- freqtrade/data/history/hdf5datahandler.py | 17 +++-------------- freqtrade/data/history/idatahandler.py | 18 ++++-------------- freqtrade/data/history/jsondatahandler.py | 17 +++-------------- tests/data/test_history.py | 2 +- 5 files changed, 12 insertions(+), 44 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 12b02f744..63f95ad0a 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -134,7 +134,7 @@ class DataProvider: combination. Returns empty dataframe and Epoch 0 (1970-01-01) if no dataframe was cached. """ - pair_key = (pair, timeframe, CandleType.SPOT) + pair_key = (pair, timeframe, CandleType.SPOT_) if pair_key in self.__cached_pairs: if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE): df, date = self.__cached_pairs[pair_key] diff --git a/freqtrade/data/history/hdf5datahandler.py b/freqtrade/data/history/hdf5datahandler.py index 73b8aedba..b735a19f1 100644 --- a/freqtrade/data/history/hdf5datahandler.py +++ b/freqtrade/data/history/hdf5datahandler.py @@ -44,12 +44,7 @@ class HDF5DataHandler(IDataHandler): ) for match in _tmp if match and len(match.groups()) > 1] @classmethod - def ohlcv_get_pairs( - cls, - datadir: Path, - timeframe: str, - candle_type: CandleType - ) -> List[str]: + def ohlcv_get_pairs(cls, datadir: Path, timeframe: str, candle_type: CandleType) -> List[str]: """ Returns a list of all pairs with ohlcv data available in this datadir for the specified timeframe @@ -69,12 +64,7 @@ class HDF5DataHandler(IDataHandler): return [cls.rebuild_pair_from_filename(match[0]) for match in _tmp if match] def ohlcv_store( - self, - pair: str, - timeframe: str, - data: pd.DataFrame, - candle_type: CandleType = CandleType.SPOT_ - ) -> None: + self, pair: str, timeframe: str, data: pd.DataFrame, candle_type: CandleType) -> None: """ Store data in hdf5 file. :param pair: Pair - used to generate filename @@ -94,8 +84,7 @@ class HDF5DataHandler(IDataHandler): ) def _ohlcv_load(self, pair: str, timeframe: str, - timerange: Optional[TimeRange] = None, - candle_type: CandleType = CandleType.SPOT_ + timerange: Optional[TimeRange], candle_type: CandleType ) -> pd.DataFrame: """ Internal method used to load data for one pair from disk. diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index 59a4bc5e4..a65bdd65e 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -48,12 +48,7 @@ class IDataHandler(ABC): """ @abstractclassmethod - def ohlcv_get_pairs( - cls, - datadir: Path, - timeframe: str, - candle_type: CandleType - ) -> List[str]: + def ohlcv_get_pairs(cls, datadir: Path, timeframe: str, candle_type: CandleType) -> List[str]: """ Returns a list of all pairs with ohlcv data available in this datadir for the specified timeframe @@ -65,12 +60,7 @@ class IDataHandler(ABC): @abstractmethod def ohlcv_store( - self, - pair: str, - timeframe: str, - data: DataFrame, - candle_type: CandleType = CandleType.SPOT_ - ) -> None: + self, pair: str, timeframe: str, data: DataFrame, candle_type: CandleType) -> None: """ Store ohlcv data. :param pair: Pair - used to generate filename @@ -81,8 +71,8 @@ class IDataHandler(ABC): """ @abstractmethod - def _ohlcv_load(self, pair: str, timeframe: str, timerange: Optional[TimeRange] = None, - candle_type: CandleType = CandleType.SPOT_ + def _ohlcv_load(self, pair: str, timeframe: str, timerange: Optional[TimeRange], + candle_type: CandleType ) -> DataFrame: """ Internal method used to load data for one pair from disk. diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index f329d4879..afaa89f0f 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -45,12 +45,7 @@ class JsonDataHandler(IDataHandler): ) for match in _tmp if match and len(match.groups()) > 1] @classmethod - def ohlcv_get_pairs( - cls, - datadir: Path, - timeframe: str, - candle_type: CandleType - ) -> List[str]: + def ohlcv_get_pairs(cls, datadir: Path, timeframe: str, candle_type: CandleType) -> List[str]: """ Returns a list of all pairs with ohlcv data available in this datadir for the specified timeframe @@ -70,12 +65,7 @@ class JsonDataHandler(IDataHandler): return [cls.rebuild_pair_from_filename(match[0]) for match in _tmp if match] def ohlcv_store( - self, - pair: str, - timeframe: str, - data: DataFrame, - candle_type: CandleType = CandleType.SPOT_ - ) -> None: + self, pair: str, timeframe: str, data: DataFrame, candle_type: CandleType) -> None: """ Store data in json format "values". format looks as follows: @@ -98,8 +88,7 @@ class JsonDataHandler(IDataHandler): compression='gzip' if self._use_zip else None) def _ohlcv_load(self, pair: str, timeframe: str, - timerange: Optional[TimeRange] = None, - candle_type: CandleType = CandleType.SPOT_ + timerange: Optional[TimeRange], candle_type: CandleType ) -> DataFrame: """ Internal method used to load data for one pair from disk. diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 89b02f295..138f62a6a 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -922,7 +922,7 @@ def test_hdf5datahandler_ohlcv_load_and_resave( tmpdir2 = tmpdir1 / 'futures' tmpdir2.mkdir() dh = HDF5DataHandler(testdatadir) - ohlcv = dh._ohlcv_load(pair, timeframe, candle_type=candle_type) + ohlcv = dh._ohlcv_load(pair, timeframe, None, candle_type=candle_type) assert isinstance(ohlcv, DataFrame) assert len(ohlcv) > 0 From 222c29360208bc885652c0df03d1e46b2b013b8a Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 8 Dec 2021 13:00:11 +0100 Subject: [PATCH 58/64] Add "defaultCandletype" --- freqtrade/data/dataprovider.py | 2 +- freqtrade/data/history/idatahandler.py | 5 ++--- freqtrade/enums/candletype.py | 11 +++++++++-- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 63f95ad0a..12b02f744 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -134,7 +134,7 @@ class DataProvider: combination. Returns empty dataframe and Epoch 0 (1970-01-01) if no dataframe was cached. """ - pair_key = (pair, timeframe, CandleType.SPOT_) + pair_key = (pair, timeframe, CandleType.SPOT) if pair_key in self.__cached_pairs: if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE): df, date = self.__cached_pairs[pair_key] diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index a65bdd65e..0055d378a 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -87,8 +87,7 @@ class IDataHandler(ABC): :return: DataFrame with ohlcv data, or empty DataFrame """ - def ohlcv_purge( - self, pair: str, timeframe: str, candle_type: CandleType = CandleType.SPOT_) -> bool: + def ohlcv_purge(self, pair: str, timeframe: str, candle_type: CandleType) -> bool: """ Remove data for this pair :param pair: Delete data for this pair. @@ -218,12 +217,12 @@ class IDataHandler(ABC): return res def ohlcv_load(self, pair, timeframe: str, + candle_type: CandleType, timerange: Optional[TimeRange] = None, fill_missing: bool = True, drop_incomplete: bool = True, startup_candles: int = 0, warn_no_data: bool = True, - candle_type: CandleType = CandleType.SPOT_ ) -> DataFrame: """ Load cached candle (OHLCV) data for the given pair. diff --git a/freqtrade/enums/candletype.py b/freqtrade/enums/candletype.py index ade0f68d1..9fa3ea4c3 100644 --- a/freqtrade/enums/candletype.py +++ b/freqtrade/enums/candletype.py @@ -12,9 +12,16 @@ class CandleType(str, Enum): # TODO-lev: not sure this belongs here, as the datatype is really different FUNDING_RATE = "funding_rate" - @classmethod - def from_string(cls, value: str) -> 'CandleType': + @staticmethod + def from_string(value: str) -> 'CandleType': if not value: # Default to spot return CandleType.SPOT_ return CandleType(value) + + @staticmethod + def get_default(trading_mode: str) -> 'CandleType': + if trading_mode == 'futures': + return CandleType.FUTURES + # TODO-lev: The below should be SPOT, not SPOT_ + return CandleType.SPOT_ From d89cbda7b82fbe0a0649c4bde4a929f9f6c6d90a Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 8 Dec 2021 14:10:08 +0100 Subject: [PATCH 59/64] Use `candle_type_def` where possible --- freqtrade/configuration/configuration.py | 2 ++ freqtrade/data/history/history_utils.py | 2 +- freqtrade/plugins/pairlist/AgeFilter.py | 7 +++---- freqtrade/plugins/pairlist/VolatilityFilter.py | 7 +++---- freqtrade/plugins/pairlist/VolumePairList.py | 10 ++++++---- freqtrade/plugins/pairlist/rangestabilityfilter.py | 7 +++---- freqtrade/strategy/informative_decorator.py | 5 +++-- freqtrade/strategy/interface.py | 8 +++++--- tests/conftest.py | 2 ++ 9 files changed, 28 insertions(+), 22 deletions(-) diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index beea3e507..bac20c054 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -14,6 +14,7 @@ from freqtrade.configuration.directory_operations import create_datadir, create_ from freqtrade.configuration.environment_vars import enironment_vars_to_dict from freqtrade.configuration.load_config import load_config_file, load_file from freqtrade.enums import NON_UTIL_MODES, TRADING_MODES, RunMode +from freqtrade.enums.candletype import CandleType from freqtrade.exceptions import OperationalException from freqtrade.loggers import setup_logging from freqtrade.misc import deep_merge_dicts, parse_db_uri_for_logging @@ -433,6 +434,7 @@ class Configuration: logstring='Detected --new-pairs-days: {}') self._args_to_config(config, argname='trading_mode', logstring='Detected --trading-mode: {}') + config['candle_type_def'] = CandleType.get_default(config.get('trading_mode', 'spot')) self._args_to_config(config, argname='candle_types', logstring='Detected --candle-types: {}') diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index 793c8b839..4e9ac9dcf 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -260,7 +260,7 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes """ pairs_not_available = [] data_handler = get_datahandler(datadir, data_format) - candle_type = CandleType.FUTURES if trading_mode == 'futures' else CandleType.SPOT_ + candle_type = CandleType.get_default(trading_mode) for idx, pair in enumerate(pairs, start=1): if pair not in exchange.markets: pairs_not_available.append(pair) diff --git a/freqtrade/plugins/pairlist/AgeFilter.py b/freqtrade/plugins/pairlist/AgeFilter.py index 7c490d920..f5507d0a6 100644 --- a/freqtrade/plugins/pairlist/AgeFilter.py +++ b/freqtrade/plugins/pairlist/AgeFilter.py @@ -10,7 +10,6 @@ from pandas import DataFrame from freqtrade.configuration import PeriodicCache from freqtrade.constants import ListPairsWithTimeframes -from freqtrade.enums import CandleType from freqtrade.exceptions import OperationalException from freqtrade.misc import plural from freqtrade.plugins.pairlist.IPairList import IPairList @@ -74,7 +73,7 @@ class AgeFilter(IPairList): :return: new allowlist """ needed_pairs: ListPairsWithTimeframes = [ - (p, '1d', CandleType.SPOT_) for p in pairlist + (p, '1d', self._config['candle_type_def']) for p in pairlist if p not in self._symbolsChecked and p not in self._symbolsCheckFailed] if not needed_pairs: # Remove pairs that have been removed before @@ -90,8 +89,8 @@ class AgeFilter(IPairList): candles = self._exchange.refresh_latest_ohlcv(needed_pairs, since_ms=since_ms, cache=False) if self._enabled: for p in deepcopy(pairlist): - daily_candles = candles[(p, '1d', CandleType.SPOT_)] if ( - p, '1d', CandleType.SPOT_) in candles else None + daily_candles = candles[(p, '1d', self._config['candle_type_def'])] if ( + p, '1d', self._config['candle_type_def']) in candles else None if not self._validate_pair_loc(p, daily_candles): pairlist.remove(p) self.log_once(f"Validated {len(pairlist)} pairs.", logger.info) diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py index bdb7a043a..c2dedcdeb 100644 --- a/freqtrade/plugins/pairlist/VolatilityFilter.py +++ b/freqtrade/plugins/pairlist/VolatilityFilter.py @@ -12,7 +12,6 @@ from cachetools.ttl import TTLCache from pandas import DataFrame from freqtrade.constants import ListPairsWithTimeframes -from freqtrade.enums import CandleType from freqtrade.exceptions import OperationalException from freqtrade.misc import plural from freqtrade.plugins.pairlist.IPairList import IPairList @@ -70,7 +69,7 @@ class VolatilityFilter(IPairList): :return: new allowlist """ needed_pairs: ListPairsWithTimeframes = [ - (p, '1d', CandleType.SPOT_) for p in pairlist if p not in self._pair_cache] + (p, '1d', self._config['candle_type_def']) for p in pairlist if p not in self._pair_cache] since_ms = (arrow.utcnow() .floor('day') @@ -84,8 +83,8 @@ class VolatilityFilter(IPairList): if self._enabled: for p in deepcopy(pairlist): - daily_candles = candles[(p, '1d', CandleType.SPOT_)] if ( - p, '1d', CandleType.SPOT_) in candles else None + daily_candles = candles[(p, '1d', self._config['candle_type_def'])] if ( + p, '1d', self._config['candle_type_def']) in candles else None if not self._validate_pair_loc(p, daily_candles): pairlist.remove(p) return pairlist diff --git a/freqtrade/plugins/pairlist/VolumePairList.py b/freqtrade/plugins/pairlist/VolumePairList.py index b81a5db5c..ca9771516 100644 --- a/freqtrade/plugins/pairlist/VolumePairList.py +++ b/freqtrade/plugins/pairlist/VolumePairList.py @@ -11,7 +11,6 @@ import arrow from cachetools.ttl import TTLCache from freqtrade.constants import ListPairsWithTimeframes -from freqtrade.enums import CandleType from freqtrade.exceptions import OperationalException from freqtrade.exchange import timeframe_to_minutes from freqtrade.misc import format_ms_time @@ -45,6 +44,7 @@ class VolumePairList(IPairList): self._lookback_days = self._pairlistconfig.get('lookback_days', 0) self._lookback_timeframe = self._pairlistconfig.get('lookback_timeframe', '1d') self._lookback_period = self._pairlistconfig.get('lookback_period', 0) + self._def_candletype = self._config['candle_type_def'] if (self._lookback_days > 0) & (self._lookback_period > 0): raise OperationalException( @@ -162,7 +162,7 @@ class VolumePairList(IPairList): f"{self._lookback_timeframe}, starting from {format_ms_time(since_ms)} " f"till {format_ms_time(to_ms)}", logger.info) needed_pairs: ListPairsWithTimeframes = [ - (p, self._lookback_timeframe, CandleType.SPOT_) for p in + (p, self._lookback_timeframe, self._def_candletype) for p in [s['symbol'] for s in filtered_tickers] if p not in self._pair_cache ] @@ -175,8 +175,10 @@ class VolumePairList(IPairList): ) for i, p in enumerate(filtered_tickers): pair_candles = candles[ - (p['symbol'], self._lookback_timeframe, CandleType.SPOT_) - ] if (p['symbol'], self._lookback_timeframe, CandleType.SPOT_) in candles else None + (p['symbol'], self._lookback_timeframe, self._def_candletype) + ] if ( + p['symbol'], self._lookback_timeframe, self._def_candletype + ) in candles else None # in case of candle data calculate typical price and quoteVolume for candle if pair_candles is not None and not pair_candles.empty: pair_candles['typical_price'] = (pair_candles['high'] + pair_candles['low'] diff --git a/freqtrade/plugins/pairlist/rangestabilityfilter.py b/freqtrade/plugins/pairlist/rangestabilityfilter.py index 3e90400a6..7a4aa772a 100644 --- a/freqtrade/plugins/pairlist/rangestabilityfilter.py +++ b/freqtrade/plugins/pairlist/rangestabilityfilter.py @@ -10,7 +10,6 @@ from cachetools.ttl import TTLCache from pandas import DataFrame from freqtrade.constants import ListPairsWithTimeframes -from freqtrade.enums import CandleType from freqtrade.exceptions import OperationalException from freqtrade.misc import plural from freqtrade.plugins.pairlist.IPairList import IPairList @@ -68,7 +67,7 @@ class RangeStabilityFilter(IPairList): :return: new allowlist """ needed_pairs: ListPairsWithTimeframes = [ - (p, '1d', CandleType.SPOT_) for p in pairlist if p not in self._pair_cache] + (p, '1d', self._config['candle_type_def']) for p in pairlist if p not in self._pair_cache] since_ms = (arrow.utcnow() .floor('day') @@ -82,8 +81,8 @@ class RangeStabilityFilter(IPairList): if self._enabled: for p in deepcopy(pairlist): - daily_candles = candles[(p, '1d', CandleType.SPOT_)] if ( - p, '1d', CandleType.SPOT_) in candles else None + daily_candles = candles[(p, '1d', self._config['candle_type_def'])] if ( + p, '1d', self._config['candle_type_def']) in candles else None if not self._validate_pair_loc(p, daily_candles): pairlist.remove(p) return pairlist diff --git a/freqtrade/strategy/informative_decorator.py b/freqtrade/strategy/informative_decorator.py index 2085630cb..40e9a7b47 100644 --- a/freqtrade/strategy/informative_decorator.py +++ b/freqtrade/strategy/informative_decorator.py @@ -15,7 +15,7 @@ class InformativeData(NamedTuple): timeframe: str fmt: Union[str, Callable[[Any], str], None] ffill: bool - candle_type: CandleType = CandleType.SPOT_ + candle_type: CandleType def informative(timeframe: str, asset: str = '', @@ -58,7 +58,8 @@ def informative(timeframe: str, asset: str = '', def decorator(fn: PopulateIndicators): informative_pairs = getattr(fn, '_ft_informative', []) # TODO-lev: Add candle_type to InformativeData - informative_pairs.append(InformativeData(_asset, _timeframe, _fmt, _ffill)) + informative_pairs.append(InformativeData(_asset, _timeframe, _fmt, _ffill, + CandleType.SPOT_)) setattr(fn, '_ft_informative', informative_pairs) return fn return decorator diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 7c3fd60f1..62acc3ac6 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -424,7 +424,8 @@ class IStrategy(ABC, HyperStrategyMixin): informative_pairs = self.informative_pairs() # Compatibility code for 2 tuple informative pairs informative_pairs = [ - (p[0], p[1], CandleType.from_string(p[2]) if len(p) > 2 else CandleType.SPOT_) + (p[0], p[1], CandleType.from_string(p[2]) if len( + p) > 2 else self.config['candle_type_def']) for p in informative_pairs] for inf_data, _ in self._ft_informative: if inf_data.asset: @@ -530,8 +531,9 @@ class IStrategy(ABC, HyperStrategyMixin): dataframe = self.analyze_ticker(dataframe, metadata) self._last_candle_seen_per_pair[pair] = dataframe.iloc[-1]['date'] if self.dp: - self.dp._set_cached_df(pair, self.timeframe, dataframe, CandleType.SPOT_) - # TODO-lev: CandleType should be set conditionally + self.dp._set_cached_df( + pair, self.timeframe, dataframe, + candle_type=self.config['candle_type_def']) else: logger.debug("Skipping TA Analysis for already analyzed candle") dataframe[SignalType.ENTER_LONG.value] = 0 diff --git a/tests/conftest.py b/tests/conftest.py index 38ef35abb..e60c441b7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -19,6 +19,7 @@ from freqtrade.commands import Arguments from freqtrade.data.converter import ohlcv_to_dataframe from freqtrade.edge import PairInfo from freqtrade.enums import Collateral, RunMode, TradingMode +from freqtrade.enums.candletype import CandleType from freqtrade.enums.signaltype import SignalDirection from freqtrade.exchange import Exchange from freqtrade.freqtradebot import FreqtradeBot @@ -459,6 +460,7 @@ def get_default_conf(testdatadir): "disableparamexport": True, "internals": {}, "export": "none", + "candle_type_def": CandleType.SPOT, } return configuration From 9b9d61c6d67166905fd39b1831473a4af8c0e491 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 8 Dec 2021 14:35:15 +0100 Subject: [PATCH 60/64] Remove SPOT_ candletype --- freqtrade/data/history/hdf5datahandler.py | 2 +- freqtrade/data/history/idatahandler.py | 2 +- freqtrade/data/history/jsondatahandler.py | 2 +- freqtrade/enums/candletype.py | 6 +-- freqtrade/exchange/exchange.py | 2 +- .../plugins/pairlist/VolatilityFilter.py | 7 +-- .../plugins/pairlist/rangestabilityfilter.py | 7 +-- freqtrade/rpc/api_server/api_v1.py | 8 +-- freqtrade/strategy/informative_decorator.py | 2 +- freqtrade/strategy/interface.py | 4 +- tests/commands/test_commands.py | 4 +- tests/data/test_dataprovider.py | 12 ++--- tests/data/test_history.py | 52 +++++++++--------- tests/leverage/test_candletype.py | 2 +- tests/plugins/test_pairlist.py | 54 +++++++++---------- tests/rpc/test_rpc_apiserver.py | 2 +- tests/strategy/test_interface.py | 3 +- tests/strategy/test_strategy_helpers.py | 47 ++++++++-------- tests/test_freqtradebot.py | 9 ++-- 19 files changed, 115 insertions(+), 112 deletions(-) diff --git a/freqtrade/data/history/hdf5datahandler.py b/freqtrade/data/history/hdf5datahandler.py index b735a19f1..cc01d51ca 100644 --- a/freqtrade/data/history/hdf5datahandler.py +++ b/freqtrade/data/history/hdf5datahandler.py @@ -54,7 +54,7 @@ class HDF5DataHandler(IDataHandler): :return: List of Pairs """ candle = "" - if candle_type not in (CandleType.SPOT, CandleType.SPOT_): + if candle_type != CandleType.SPOT: datadir = datadir.joinpath('futures') candle = f"-{candle_type}" diff --git a/freqtrade/data/history/idatahandler.py b/freqtrade/data/history/idatahandler.py index 0055d378a..2d0e187b8 100644 --- a/freqtrade/data/history/idatahandler.py +++ b/freqtrade/data/history/idatahandler.py @@ -193,7 +193,7 @@ class IDataHandler(ABC): ) -> Path: pair_s = misc.pair_to_filename(pair) candle = "" - if candle_type not in (CandleType.SPOT, CandleType.SPOT_): + if candle_type != CandleType.SPOT: datadir = datadir.joinpath('futures') candle = f"-{candle_type}" filename = datadir.joinpath( diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index afaa89f0f..939f18931 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -55,7 +55,7 @@ class JsonDataHandler(IDataHandler): :return: List of Pairs """ candle = "" - if candle_type not in (CandleType.SPOT, CandleType.SPOT_): + if candle_type != CandleType.SPOT: datadir = datadir.joinpath('futures') candle = f"-{candle_type}" diff --git a/freqtrade/enums/candletype.py b/freqtrade/enums/candletype.py index 9fa3ea4c3..0188650f6 100644 --- a/freqtrade/enums/candletype.py +++ b/freqtrade/enums/candletype.py @@ -4,7 +4,6 @@ from enum import Enum class CandleType(str, Enum): """Enum to distinguish candle types""" SPOT = "spot" - SPOT_ = "" FUTURES = "futures" MARK = "mark" INDEX = "index" @@ -16,12 +15,11 @@ class CandleType(str, Enum): def from_string(value: str) -> 'CandleType': if not value: # Default to spot - return CandleType.SPOT_ + return CandleType.SPOT return CandleType(value) @staticmethod def get_default(trading_mode: str) -> 'CandleType': if trading_mode == 'futures': return CandleType.FUTURES - # TODO-lev: The below should be SPOT, not SPOT_ - return CandleType.SPOT_ + return CandleType.SPOT diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 6d0130b55..6aa15f550 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -1492,7 +1492,7 @@ class Exchange: pair, timeframe, since_ms, s ) params = deepcopy(self._ft_has.get('ohlcv_params', {})) - if candle_type not in (CandleType.SPOT, CandleType.SPOT_): + if candle_type != CandleType.SPOT: params.update({'price': candle_type}) data = await self._api_async.fetch_ohlcv(pair, timeframe=timeframe, since=since_ms, diff --git a/freqtrade/plugins/pairlist/VolatilityFilter.py b/freqtrade/plugins/pairlist/VolatilityFilter.py index c2dedcdeb..55340fa14 100644 --- a/freqtrade/plugins/pairlist/VolatilityFilter.py +++ b/freqtrade/plugins/pairlist/VolatilityFilter.py @@ -34,6 +34,7 @@ class VolatilityFilter(IPairList): self._min_volatility = pairlistconfig.get('min_volatility', 0) self._max_volatility = pairlistconfig.get('max_volatility', sys.maxsize) self._refresh_period = pairlistconfig.get('refresh_period', 1440) + self._def_candletype = self._config['candle_type_def'] self._pair_cache: TTLCache = TTLCache(maxsize=1000, ttl=self._refresh_period) @@ -69,7 +70,7 @@ class VolatilityFilter(IPairList): :return: new allowlist """ needed_pairs: ListPairsWithTimeframes = [ - (p, '1d', self._config['candle_type_def']) for p in pairlist if p not in self._pair_cache] + (p, '1d', self._def_candletype) for p in pairlist if p not in self._pair_cache] since_ms = (arrow.utcnow() .floor('day') @@ -83,8 +84,8 @@ class VolatilityFilter(IPairList): if self._enabled: for p in deepcopy(pairlist): - daily_candles = candles[(p, '1d', self._config['candle_type_def'])] if ( - p, '1d', self._config['candle_type_def']) in candles else None + daily_candles = candles[(p, '1d', self._def_candletype)] if ( + p, '1d', self._def_candletype) in candles else None if not self._validate_pair_loc(p, daily_candles): pairlist.remove(p) return pairlist diff --git a/freqtrade/plugins/pairlist/rangestabilityfilter.py b/freqtrade/plugins/pairlist/rangestabilityfilter.py index 7a4aa772a..96a59808e 100644 --- a/freqtrade/plugins/pairlist/rangestabilityfilter.py +++ b/freqtrade/plugins/pairlist/rangestabilityfilter.py @@ -29,6 +29,7 @@ class RangeStabilityFilter(IPairList): self._min_rate_of_change = pairlistconfig.get('min_rate_of_change', 0.01) self._max_rate_of_change = pairlistconfig.get('max_rate_of_change', None) self._refresh_period = pairlistconfig.get('refresh_period', 1440) + self._def_candletype = self._config['candle_type_def'] self._pair_cache: TTLCache = TTLCache(maxsize=1000, ttl=self._refresh_period) @@ -67,7 +68,7 @@ class RangeStabilityFilter(IPairList): :return: new allowlist """ needed_pairs: ListPairsWithTimeframes = [ - (p, '1d', self._config['candle_type_def']) for p in pairlist if p not in self._pair_cache] + (p, '1d', self._def_candletype) for p in pairlist if p not in self._pair_cache] since_ms = (arrow.utcnow() .floor('day') @@ -81,8 +82,8 @@ class RangeStabilityFilter(IPairList): if self._enabled: for p in deepcopy(pairlist): - daily_candles = candles[(p, '1d', self._config['candle_type_def'])] if ( - p, '1d', self._config['candle_type_def']) in candles else None + daily_candles = candles[(p, '1d', self._def_candletype)] if ( + p, '1d', self._def_candletype) in candles else None if not self._validate_pair_loc(p, daily_candles): pairlist.remove(p) return pairlist diff --git a/freqtrade/rpc/api_server/api_v1.py b/freqtrade/rpc/api_server/api_v1.py index 98df84b7d..644e24655 100644 --- a/freqtrade/rpc/api_server/api_v1.py +++ b/freqtrade/rpc/api_server/api_v1.py @@ -254,9 +254,8 @@ def list_available_pairs(timeframe: Optional[str] = None, stake_currency: Option candletype: Optional[CandleType] = None, config=Depends(get_config)): dh = get_datahandler(config['datadir'], config.get('dataformat_ohlcv', None)) - - pair_interval = dh.ohlcv_get_available_data(config['datadir'], - config.get('trading_mode', 'spot')) + trading_mode = config.get('trading_mode', 'spot') + pair_interval = dh.ohlcv_get_available_data(config['datadir'], trading_mode) if timeframe: pair_interval = [pair for pair in pair_interval if pair[1] == timeframe] @@ -265,7 +264,8 @@ def list_available_pairs(timeframe: Optional[str] = None, stake_currency: Option if candletype: pair_interval = [pair for pair in pair_interval if pair[2] == candletype] else: - pair_interval = [pair for pair in pair_interval if pair[2] == CandleType.SPOT_] + candle_type = CandleType.get_default(trading_mode) + pair_interval = [pair for pair in pair_interval if pair[2] == candle_type] pair_interval = sorted(pair_interval, key=lambda x: x[0]) diff --git a/freqtrade/strategy/informative_decorator.py b/freqtrade/strategy/informative_decorator.py index 40e9a7b47..986b457a2 100644 --- a/freqtrade/strategy/informative_decorator.py +++ b/freqtrade/strategy/informative_decorator.py @@ -59,7 +59,7 @@ def informative(timeframe: str, asset: str = '', informative_pairs = getattr(fn, '_ft_informative', []) # TODO-lev: Add candle_type to InformativeData informative_pairs.append(InformativeData(_asset, _timeframe, _fmt, _ffill, - CandleType.SPOT_)) + CandleType.SPOT)) setattr(fn, '_ft_informative', informative_pairs) return fn return decorator diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 62acc3ac6..25b7404f7 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -425,7 +425,7 @@ class IStrategy(ABC, HyperStrategyMixin): # Compatibility code for 2 tuple informative pairs informative_pairs = [ (p[0], p[1], CandleType.from_string(p[2]) if len( - p) > 2 else self.config['candle_type_def']) + p) > 2 else self.config.get('candle_type_def', CandleType.SPOT)) for p in informative_pairs] for inf_data, _ in self._ft_informative: if inf_data.asset: @@ -533,7 +533,7 @@ class IStrategy(ABC, HyperStrategyMixin): if self.dp: self.dp._set_cached_df( pair, self.timeframe, dataframe, - candle_type=self.config['candle_type_def']) + candle_type=self.config.get('candle_type_def', CandleType.SPOT)) else: logger.debug("Skipping TA Analysis for already analyzed candle") dataframe[SignalType.ENTER_LONG.value] = 0 diff --git a/tests/commands/test_commands.py b/tests/commands/test_commands.py index 90c8bb725..2b5504324 100644 --- a/tests/commands/test_commands.py +++ b/tests/commands/test_commands.py @@ -1340,7 +1340,7 @@ def test_start_list_data(testdatadir, capsys): captured = capsys.readouterr() assert "Found 17 pair / timeframe combinations." in captured.out assert "\n| Pair | Timeframe | Type |\n" in captured.out - assert "\n| UNITTEST/BTC | 1m, 5m, 8m, 30m | |\n" in captured.out + assert "\n| UNITTEST/BTC | 1m, 5m, 8m, 30m | spot |\n" in captured.out args = [ "list-data", @@ -1357,7 +1357,7 @@ def test_start_list_data(testdatadir, capsys): assert "Found 2 pair / timeframe combinations." in captured.out assert "\n| Pair | Timeframe | Type |\n" in captured.out assert "UNITTEST/BTC" not in captured.out - assert "\n| XRP/ETH | 1m, 5m | |\n" in captured.out + assert "\n| XRP/ETH | 1m, 5m | spot |\n" in captured.out args = [ "list-data", diff --git a/tests/data/test_dataprovider.py b/tests/data/test_dataprovider.py index 0e0685f96..17a90aca4 100644 --- a/tests/data/test_dataprovider.py +++ b/tests/data/test_dataprovider.py @@ -20,10 +20,10 @@ def test_dp_ohlcv(mocker, default_conf, ohlcv_history, candle_type): default_conf["runmode"] = RunMode.DRY_RUN timeframe = default_conf["timeframe"] exchange = get_patched_exchange(mocker, default_conf) - exchange._klines[("XRP/BTC", timeframe, candle_type)] = ohlcv_history - exchange._klines[("UNITTEST/BTC", timeframe, candle_type)] = ohlcv_history - candletype = CandleType.from_string(candle_type) + exchange._klines[("XRP/BTC", timeframe, candletype)] = ohlcv_history + exchange._klines[("UNITTEST/BTC", timeframe, candletype)] = ohlcv_history + dp = DataProvider(default_conf, exchange) assert dp.runmode == RunMode.DRY_RUN assert ohlcv_history.equals(dp.ohlcv("UNITTEST/BTC", timeframe, candle_type=candletype)) @@ -96,10 +96,10 @@ def test_get_pair_dataframe(mocker, default_conf, ohlcv_history, candle_type): default_conf["runmode"] = RunMode.DRY_RUN timeframe = default_conf["timeframe"] exchange = get_patched_exchange(mocker, default_conf) - exchange._klines[("XRP/BTC", timeframe, candle_type)] = ohlcv_history - exchange._klines[("UNITTEST/BTC", timeframe, candle_type)] = ohlcv_history - candletype = CandleType.from_string(candle_type) + exchange._klines[("XRP/BTC", timeframe, candletype)] = ohlcv_history + exchange._klines[("UNITTEST/BTC", timeframe, candletype)] = ohlcv_history + dp = DataProvider(default_conf, exchange) assert dp.runmode == RunMode.DRY_RUN assert ohlcv_history.equals(dp.get_pair_dataframe( diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 138f62a6a..4ec31ccd6 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -172,7 +172,7 @@ def test_json_pair_data_filename(pair, expected_result, candle_type): Path('freqtrade/hello/world'), pair, '5m', - candle_type + CandleType.from_string(candle_type) ) assert isinstance(fn, Path) assert fn == Path(expected_result) @@ -180,7 +180,7 @@ def test_json_pair_data_filename(pair, expected_result, candle_type): Path('freqtrade/hello/world'), pair, '5m', - candle_type=candle_type + candle_type=CandleType.from_string(candle_type) ) assert isinstance(fn, Path) assert fn == Path(expected_result + '.gz') @@ -257,7 +257,7 @@ def test_load_cached_data_for_updating(mocker, testdatadir) -> None: @pytest.mark.parametrize('candle_type,subdir,file_tail', [ ('mark', 'futures/', '-mark'), - ('', '', ''), + ('spot', '', ''), ]) def test_download_pair_history( ohlcv_history_list, @@ -719,23 +719,23 @@ def test_datahandler_ohlcv_get_available_data(testdatadir): paircombs = JsonDataHandler.ohlcv_get_available_data(testdatadir, 'spot') # Convert to set to avoid failures due to sorting assert set(paircombs) == { - ('UNITTEST/BTC', '5m', CandleType.SPOT_), - ('ETH/BTC', '5m', CandleType.SPOT_), - ('XLM/BTC', '5m', CandleType.SPOT_), - ('TRX/BTC', '5m', CandleType.SPOT_), - ('LTC/BTC', '5m', CandleType.SPOT_), - ('XMR/BTC', '5m', CandleType.SPOT_), - ('ZEC/BTC', '5m', CandleType.SPOT_), - ('UNITTEST/BTC', '1m', CandleType.SPOT_), - ('ADA/BTC', '5m', CandleType.SPOT_), - ('ETC/BTC', '5m', CandleType.SPOT_), - ('NXT/BTC', '5m', CandleType.SPOT_), - ('DASH/BTC', '5m', ''), - ('XRP/ETH', '1m', ''), - ('XRP/ETH', '5m', ''), - ('UNITTEST/BTC', '30m', ''), - ('UNITTEST/BTC', '8m', ''), - ('NOPAIR/XXX', '4m', ''), + ('UNITTEST/BTC', '5m', CandleType.SPOT), + ('ETH/BTC', '5m', CandleType.SPOT), + ('XLM/BTC', '5m', CandleType.SPOT), + ('TRX/BTC', '5m', CandleType.SPOT), + ('LTC/BTC', '5m', CandleType.SPOT), + ('XMR/BTC', '5m', CandleType.SPOT), + ('ZEC/BTC', '5m', CandleType.SPOT), + ('UNITTEST/BTC', '1m', CandleType.SPOT), + ('ADA/BTC', '5m', CandleType.SPOT), + ('ETC/BTC', '5m', CandleType.SPOT), + ('NXT/BTC', '5m', CandleType.SPOT), + ('DASH/BTC', '5m', CandleType.SPOT), + ('XRP/ETH', '1m', CandleType.SPOT), + ('XRP/ETH', '5m', CandleType.SPOT), + ('UNITTEST/BTC', '30m', CandleType.SPOT), + ('UNITTEST/BTC', '8m', CandleType.SPOT), + ('NOPAIR/XXX', '4m', CandleType.SPOT), } paircombs = JsonDataHandler.ohlcv_get_available_data(testdatadir, 'futures') @@ -747,9 +747,9 @@ def test_datahandler_ohlcv_get_available_data(testdatadir): } paircombs = JsonGzDataHandler.ohlcv_get_available_data(testdatadir, 'spot') - assert set(paircombs) == {('UNITTEST/BTC', '8m', '')} + assert set(paircombs) == {('UNITTEST/BTC', '8m', CandleType.SPOT)} paircombs = HDF5DataHandler.ohlcv_get_available_data(testdatadir, 'spot') - assert set(paircombs) == {('UNITTEST/BTC', '5m', '')} + assert set(paircombs) == {('UNITTEST/BTC', '5m', CandleType.SPOT)} def test_jsondatahandler_trades_get_pairs(testdatadir): @@ -774,17 +774,17 @@ def test_jsondatahandler_ohlcv_purge(mocker, testdatadir): def test_jsondatahandler_ohlcv_load(testdatadir, caplog): dh = JsonDataHandler(testdatadir) - df = dh.ohlcv_load('XRP/ETH', '5m', '') + df = dh.ohlcv_load('XRP/ETH', '5m', 'spot') assert len(df) == 711 df_mark = dh.ohlcv_load('UNITTEST/USDT', '1h', candle_type="mark") assert len(df_mark) == 99 - df_no_mark = dh.ohlcv_load('UNITTEST/USDT', '1h', '') + df_no_mark = dh.ohlcv_load('UNITTEST/USDT', '1h', 'spot') assert len(df_no_mark) == 0 # Failure case (empty array) - df1 = dh.ohlcv_load('NOPAIR/XXX', '4m', '') + df1 = dh.ohlcv_load('NOPAIR/XXX', '4m', 'spot') assert len(df1) == 0 assert log_has("Could not load data for NOPAIR/XXX.", caplog) assert df.columns.equals(df1.columns) @@ -903,7 +903,7 @@ def test_hdf5datahandler_trades_purge(mocker, testdatadir): @pytest.mark.parametrize('pair,timeframe,candle_type,candle_append,startdt,enddt', [ # Data goes from 2018-01-10 - 2018-01-30 - ('UNITTEST/BTC', '5m', '', '', '2018-01-15', '2018-01-19'), + ('UNITTEST/BTC', '5m', 'spot', '', '2018-01-15', '2018-01-19'), # Mark data goes from to 2021-11-15 2021-11-19 ('UNITTEST/USDT', '1h', 'mark', '-mark', '2021-11-16', '2021-11-18'), ]) diff --git a/tests/leverage/test_candletype.py b/tests/leverage/test_candletype.py index 1424993ca..3eb73a07c 100644 --- a/tests/leverage/test_candletype.py +++ b/tests/leverage/test_candletype.py @@ -4,7 +4,7 @@ from freqtrade.enums import CandleType @pytest.mark.parametrize('input,expected', [ - ('', CandleType.SPOT_), + ('', CandleType.SPOT), ('spot', CandleType.SPOT), (CandleType.SPOT, CandleType.SPOT), (CandleType.FUTURES, CandleType.FUTURES), diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index fe375a671..3ef6dacd6 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -462,11 +462,11 @@ def test_VolumePairList_whitelist_gen(mocker, whitelist_conf, shitcoinmarkets, t ohlcv_history_high_vola.loc[ohlcv_history_high_vola.index == 1, 'close'] = 0.00090 ohlcv_data = { - ('ETH/BTC', '1d', CandleType.SPOT_): ohlcv_history, - ('TKN/BTC', '1d', CandleType.SPOT_): ohlcv_history, - ('LTC/BTC', '1d', CandleType.SPOT_): ohlcv_history.append(ohlcv_history), - ('XRP/BTC', '1d', CandleType.SPOT_): ohlcv_history, - ('HOT/BTC', '1d', CandleType.SPOT_): ohlcv_history_high_vola, + ('ETH/BTC', '1d', CandleType.SPOT): ohlcv_history, + ('TKN/BTC', '1d', CandleType.SPOT): ohlcv_history, + ('LTC/BTC', '1d', CandleType.SPOT): ohlcv_history.append(ohlcv_history), + ('XRP/BTC', '1d', CandleType.SPOT): ohlcv_history, + ('HOT/BTC', '1d', CandleType.SPOT): ohlcv_history_high_vola, } mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) @@ -580,11 +580,11 @@ def test_VolumePairList_range(mocker, whitelist_conf, shitcoinmarkets, tickers, ohlcv_history_high_volume.loc[:, 'volume'] = 10 ohlcv_data = { - ('ETH/BTC', '1d', CandleType.SPOT_): ohlcv_history, - ('TKN/BTC', '1d', CandleType.SPOT_): ohlcv_history, - ('LTC/BTC', '1d', CandleType.SPOT_): ohlcv_history_medium_volume, - ('XRP/BTC', '1d', CandleType.SPOT_): ohlcv_history_high_vola, - ('HOT/BTC', '1d', CandleType.SPOT_): ohlcv_history_high_volume, + ('ETH/BTC', '1d', CandleType.SPOT): ohlcv_history, + ('TKN/BTC', '1d', CandleType.SPOT): ohlcv_history, + ('LTC/BTC', '1d', CandleType.SPOT): ohlcv_history_medium_volume, + ('XRP/BTC', '1d', CandleType.SPOT): ohlcv_history_high_vola, + ('HOT/BTC', '1d', CandleType.SPOT): ohlcv_history_high_volume, } mocker.patch('freqtrade.exchange.Exchange.exchange_has', MagicMock(return_value=True)) @@ -856,9 +856,9 @@ def test_agefilter_min_days_listed_too_large(mocker, default_conf, markets, tick def test_agefilter_caching(mocker, markets, whitelist_conf_agefilter, tickers, ohlcv_history): with time_machine.travel("2021-09-01 05:00:00 +00:00") as t: ohlcv_data = { - ('ETH/BTC', '1d', CandleType.SPOT_): ohlcv_history, - ('TKN/BTC', '1d', CandleType.SPOT_): ohlcv_history, - ('LTC/BTC', '1d', CandleType.SPOT_): ohlcv_history, + ('ETH/BTC', '1d', CandleType.SPOT): ohlcv_history, + ('TKN/BTC', '1d', CandleType.SPOT): ohlcv_history, + ('LTC/BTC', '1d', CandleType.SPOT): ohlcv_history, } mocker.patch.multiple( 'freqtrade.exchange.Exchange', @@ -880,10 +880,10 @@ def test_agefilter_caching(mocker, markets, whitelist_conf_agefilter, tickers, o assert freqtrade.exchange.refresh_latest_ohlcv.call_count == 2 ohlcv_data = { - ('ETH/BTC', '1d', CandleType.SPOT_): ohlcv_history, - ('TKN/BTC', '1d', CandleType.SPOT_): ohlcv_history, - ('LTC/BTC', '1d', CandleType.SPOT_): ohlcv_history, - ('XRP/BTC', '1d', CandleType.SPOT_): ohlcv_history.iloc[[0]], + ('ETH/BTC', '1d', CandleType.SPOT): ohlcv_history, + ('TKN/BTC', '1d', CandleType.SPOT): ohlcv_history, + ('LTC/BTC', '1d', CandleType.SPOT): ohlcv_history, + ('XRP/BTC', '1d', CandleType.SPOT): ohlcv_history.iloc[[0]], } mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', return_value=ohlcv_data) freqtrade.pairlists.refresh_pairlist() @@ -901,10 +901,10 @@ def test_agefilter_caching(mocker, markets, whitelist_conf_agefilter, tickers, o t.move_to("2021-09-03 01:00:00 +00:00") # Called once for XRP/BTC ohlcv_data = { - ('ETH/BTC', '1d', CandleType.SPOT_): ohlcv_history, - ('TKN/BTC', '1d', CandleType.SPOT_): ohlcv_history, - ('LTC/BTC', '1d', CandleType.SPOT_): ohlcv_history, - ('XRP/BTC', '1d', CandleType.SPOT_): ohlcv_history, + ('ETH/BTC', '1d', CandleType.SPOT): ohlcv_history, + ('TKN/BTC', '1d', CandleType.SPOT): ohlcv_history, + ('LTC/BTC', '1d', CandleType.SPOT): ohlcv_history, + ('XRP/BTC', '1d', CandleType.SPOT): ohlcv_history, } mocker.patch('freqtrade.exchange.Exchange.refresh_latest_ohlcv', return_value=ohlcv_data) freqtrade.pairlists.refresh_pairlist() @@ -965,12 +965,12 @@ def test_rangestabilityfilter_caching(mocker, markets, default_conf, tickers, oh get_tickers=tickers ) ohlcv_data = { - ('ETH/BTC', '1d', CandleType.SPOT_): ohlcv_history, - ('TKN/BTC', '1d', CandleType.SPOT_): ohlcv_history, - ('LTC/BTC', '1d', CandleType.SPOT_): ohlcv_history, - ('XRP/BTC', '1d', CandleType.SPOT_): ohlcv_history, - ('HOT/BTC', '1d', CandleType.SPOT_): ohlcv_history, - ('BLK/BTC', '1d', CandleType.SPOT_): ohlcv_history, + ('ETH/BTC', '1d', CandleType.SPOT): ohlcv_history, + ('TKN/BTC', '1d', CandleType.SPOT): ohlcv_history, + ('LTC/BTC', '1d', CandleType.SPOT): ohlcv_history, + ('XRP/BTC', '1d', CandleType.SPOT): ohlcv_history, + ('HOT/BTC', '1d', CandleType.SPOT): ohlcv_history, + ('BLK/BTC', '1d', CandleType.SPOT): ohlcv_history, } mocker.patch.multiple( 'freqtrade.exchange.Exchange', diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index bf36b2e7f..0823c7aee 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -1356,7 +1356,7 @@ def test_list_available_pairs(botclient): ftbot.config['trading_mode'] = 'futures' rc = client_get( - client, f"{BASE_URI}/available_pairs?timeframe=1h&candletype=futures") + client, f"{BASE_URI}/available_pairs?timeframe=1h") assert_response(rc) assert rc.json()['length'] == 1 assert rc.json()['pairs'] == ['XRP/USDT'] diff --git a/tests/strategy/test_interface.py b/tests/strategy/test_interface.py index 7115f7aab..61a07191d 100644 --- a/tests/strategy/test_interface.py +++ b/tests/strategy/test_interface.py @@ -11,8 +11,7 @@ from pandas import DataFrame from freqtrade.configuration import TimeRange from freqtrade.data.dataprovider import DataProvider from freqtrade.data.history import load_data -from freqtrade.enums import SellType -from freqtrade.enums.signaltype import SignalDirection +from freqtrade.enums import SellType, SignalDirection from freqtrade.exceptions import OperationalException, StrategyError from freqtrade.optimize.space import SKDecimal from freqtrade.persistence import PairLocks, Trade diff --git a/tests/strategy/test_strategy_helpers.py b/tests/strategy/test_strategy_helpers.py index 08fb3563e..8c7da75f5 100644 --- a/tests/strategy/test_strategy_helpers.py +++ b/tests/strategy/test_strategy_helpers.py @@ -152,18 +152,18 @@ def test_informative_decorator(mocker, default_conf): test_data_30m = generate_test_data('30m', 40) test_data_1h = generate_test_data('1h', 40) data = { - ('XRP/USDT', '5m', CandleType.SPOT_): test_data_5m, - ('XRP/USDT', '30m', CandleType.SPOT_): test_data_30m, - ('XRP/USDT', '1h', CandleType.SPOT_): test_data_1h, - ('LTC/USDT', '5m', CandleType.SPOT_): test_data_5m, - ('LTC/USDT', '30m', CandleType.SPOT_): test_data_30m, - ('LTC/USDT', '1h', CandleType.SPOT_): test_data_1h, - ('NEO/USDT', '30m', CandleType.SPOT_): test_data_30m, - ('NEO/USDT', '5m', CandleType.SPOT_): test_data_5m, - ('NEO/USDT', '1h', CandleType.SPOT_): test_data_1h, - ('ETH/USDT', '1h', CandleType.SPOT_): test_data_1h, - ('ETH/USDT', '30m', CandleType.SPOT_): test_data_30m, - ('ETH/BTC', '1h', CandleType.SPOT_): test_data_1h, + ('XRP/USDT', '5m', CandleType.SPOT): test_data_5m, + ('XRP/USDT', '30m', CandleType.SPOT): test_data_30m, + ('XRP/USDT', '1h', CandleType.SPOT): test_data_1h, + ('LTC/USDT', '5m', CandleType.SPOT): test_data_5m, + ('LTC/USDT', '30m', CandleType.SPOT): test_data_30m, + ('LTC/USDT', '1h', CandleType.SPOT): test_data_1h, + ('NEO/USDT', '30m', CandleType.SPOT): test_data_30m, + ('NEO/USDT', '5m', CandleType.SPOT): test_data_5m, + ('NEO/USDT', '1h', CandleType.SPOT): test_data_1h, + ('ETH/USDT', '1h', CandleType.SPOT): test_data_1h, + ('ETH/USDT', '30m', CandleType.SPOT): test_data_30m, + ('ETH/BTC', '1h', CandleType.SPOT): test_data_1h, } from .strats.informative_decorator_strategy import InformativeDecoratorTest default_conf['stake_currency'] = 'USDT' @@ -176,25 +176,26 @@ def test_informative_decorator(mocker, default_conf): assert len(strategy._ft_informative) == 6 # Equal to number of decorators used informative_pairs = [ - ('XRP/USDT', '1h', CandleType.SPOT_), - ('LTC/USDT', '1h', CandleType.SPOT_), - ('XRP/USDT', '30m', CandleType.SPOT_), - ('LTC/USDT', '30m', CandleType.SPOT_), - ('NEO/USDT', '1h', CandleType.SPOT_), - ('NEO/USDT', '30m', CandleType.SPOT_), - ('NEO/USDT', '5m', CandleType.SPOT_), - ('ETH/BTC', '1h', CandleType.SPOT_), - ('ETH/USDT', '30m', CandleType.SPOT_)] + ('XRP/USDT', '1h', CandleType.SPOT), + ('LTC/USDT', '1h', CandleType.SPOT), + ('XRP/USDT', '30m', CandleType.SPOT), + ('LTC/USDT', '30m', CandleType.SPOT), + ('NEO/USDT', '1h', CandleType.SPOT), + ('NEO/USDT', '30m', CandleType.SPOT), + ('NEO/USDT', '5m', CandleType.SPOT), + ('ETH/BTC', '1h', CandleType.SPOT), + ('ETH/USDT', '30m', CandleType.SPOT)] for inf_pair in informative_pairs: assert inf_pair in strategy.gather_informative_pairs() def test_historic_ohlcv(pair, timeframe, candle_type): - return data[(pair, timeframe or strategy.timeframe, candle_type)].copy() + return data[ + (pair, timeframe or strategy.timeframe, CandleType.from_string(candle_type))].copy() mocker.patch('freqtrade.data.dataprovider.DataProvider.historic_ohlcv', side_effect=test_historic_ohlcv) analyzed = strategy.advise_all_indicators( - {p: data[(p, strategy.timeframe, CandleType.SPOT_)] for p in ('XRP/USDT', 'LTC/USDT')}) + {p: data[(p, strategy.timeframe, CandleType.SPOT)] for p in ('XRP/USDT', 'LTC/USDT')}) expected_columns = [ 'rsi_1h', 'rsi_30m', # Stacked informative decorators 'neo_usdt_rsi_1h', # NEO 1h informative diff --git a/tests/test_freqtradebot.py b/tests/test_freqtradebot.py index 6a6972b69..7c22078e2 100644 --- a/tests/test_freqtradebot.py +++ b/tests/test_freqtradebot.py @@ -681,7 +681,10 @@ def test_process_informative_pairs_added(default_conf_usdt, ticker_usdt, mocker) create_order=MagicMock(side_effect=TemporaryError), refresh_latest_ohlcv=refresh_mock, ) - inf_pairs = MagicMock(return_value=[("BTC/ETH", '1m', ''), ("ETH/USDT", "1h", '')]) + inf_pairs = MagicMock(return_value=[ + ("BTC/ETH", '1m', CandleType.SPOT), + ("ETH/USDT", "1h", CandleType.SPOT) + ]) mocker.patch.multiple( 'freqtrade.strategy.interface.IStrategy', get_exit_signal=MagicMock(return_value=(False, False)), @@ -696,8 +699,8 @@ def test_process_informative_pairs_added(default_conf_usdt, ticker_usdt, mocker) freqtrade.process() assert inf_pairs.call_count == 1 assert refresh_mock.call_count == 1 - assert ("BTC/ETH", "1m", CandleType.SPOT_) in refresh_mock.call_args[0][0] - assert ("ETH/USDT", "1h", CandleType.SPOT_) in refresh_mock.call_args[0][0] + assert ("BTC/ETH", "1m", CandleType.SPOT) in refresh_mock.call_args[0][0] + assert ("ETH/USDT", "1h", CandleType.SPOT) in refresh_mock.call_args[0][0] assert ("ETH/USDT", default_conf_usdt["timeframe"], CandleType.SPOT) in refresh_mock.call_args[0][0] From d079b444a6d0559df135dff14db328d2c46d9bb2 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 8 Dec 2021 14:48:56 +0100 Subject: [PATCH 61/64] Add optional "has" (as comment for now) --- freqtrade/exchange/common.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/freqtrade/exchange/common.py b/freqtrade/exchange/common.py index fc21c0f02..3beb253df 100644 --- a/freqtrade/exchange/common.py +++ b/freqtrade/exchange/common.py @@ -43,10 +43,14 @@ EXCHANGE_HAS_REQUIRED = [ EXCHANGE_HAS_OPTIONAL = [ # Private 'fetchMyTrades', # Trades for order - fee detection + # 'setLeverage', # Margin/Futures trading + # 'setMarginMode', # Margin/Futures trading + # 'fetchFundingHistory', # Futures trading # Public 'fetchOrderBook', 'fetchL2OrderBook', 'fetchTicker', # OR for pricing 'fetchTickers', # For volumepairlist? 'fetchTrades', # Downloading trades data + # 'fetchFundingRateHistory', # Futures trading ] From 25e1142f8924f2f91009fb16f0d49d2a0c9b6047 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 8 Dec 2021 15:59:20 +0100 Subject: [PATCH 62/64] Update Enum imports --- freqtrade/configuration/configuration.py | 3 +-- tests/conftest.py | 4 +--- tests/data/test_dataprovider.py | 3 +-- tests/data/test_history.py | 2 +- tests/plugins/test_pairlist.py | 3 +-- tests/rpc/test_rpc_apiserver.py | 3 +-- tests/strategy/test_strategy_helpers.py | 2 +- 7 files changed, 7 insertions(+), 13 deletions(-) diff --git a/freqtrade/configuration/configuration.py b/freqtrade/configuration/configuration.py index bac20c054..48bd7bdb3 100644 --- a/freqtrade/configuration/configuration.py +++ b/freqtrade/configuration/configuration.py @@ -13,8 +13,7 @@ from freqtrade.configuration.deprecated_settings import process_temporary_deprec from freqtrade.configuration.directory_operations import create_datadir, create_userdata_dir from freqtrade.configuration.environment_vars import enironment_vars_to_dict from freqtrade.configuration.load_config import load_config_file, load_file -from freqtrade.enums import NON_UTIL_MODES, TRADING_MODES, RunMode -from freqtrade.enums.candletype import CandleType +from freqtrade.enums import NON_UTIL_MODES, TRADING_MODES, CandleType, RunMode from freqtrade.exceptions import OperationalException from freqtrade.loggers import setup_logging from freqtrade.misc import deep_merge_dicts, parse_db_uri_for_logging diff --git a/tests/conftest.py b/tests/conftest.py index e60c441b7..0b625ab68 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,9 +18,7 @@ from freqtrade import constants from freqtrade.commands import Arguments from freqtrade.data.converter import ohlcv_to_dataframe from freqtrade.edge import PairInfo -from freqtrade.enums import Collateral, RunMode, TradingMode -from freqtrade.enums.candletype import CandleType -from freqtrade.enums.signaltype import SignalDirection +from freqtrade.enums import CandleType, Collateral, RunMode, SignalDirection, TradingMode from freqtrade.exchange import Exchange from freqtrade.freqtradebot import FreqtradeBot from freqtrade.persistence import LocalTrade, Trade, init_db diff --git a/tests/data/test_dataprovider.py b/tests/data/test_dataprovider.py index 17a90aca4..93f82de5d 100644 --- a/tests/data/test_dataprovider.py +++ b/tests/data/test_dataprovider.py @@ -5,8 +5,7 @@ import pytest from pandas import DataFrame from freqtrade.data.dataprovider import DataProvider -from freqtrade.enums import RunMode -from freqtrade.enums.candletype import CandleType +from freqtrade.enums import CandleType, RunMode from freqtrade.exceptions import ExchangeError, OperationalException from freqtrade.plugins.pairlistmanager import PairListManager from tests.conftest import get_patched_exchange diff --git a/tests/data/test_history.py b/tests/data/test_history.py index 4ec31ccd6..678a0b31b 100644 --- a/tests/data/test_history.py +++ b/tests/data/test_history.py @@ -24,7 +24,7 @@ from freqtrade.data.history.history_utils import (_download_pair_history, _downl validate_backtest_data) from freqtrade.data.history.idatahandler import IDataHandler, get_datahandler, get_datahandlerclass from freqtrade.data.history.jsondatahandler import JsonDataHandler, JsonGzDataHandler -from freqtrade.enums.candletype import CandleType +from freqtrade.enums import CandleType from freqtrade.exchange import timeframe_to_minutes from freqtrade.misc import file_dump_json from freqtrade.resolvers import StrategyResolver diff --git a/tests/plugins/test_pairlist.py b/tests/plugins/test_pairlist.py index 3ef6dacd6..f70f2e388 100644 --- a/tests/plugins/test_pairlist.py +++ b/tests/plugins/test_pairlist.py @@ -7,8 +7,7 @@ import pytest import time_machine from freqtrade.constants import AVAILABLE_PAIRLISTS -from freqtrade.enums.candletype import CandleType -from freqtrade.enums.runmode import RunMode +from freqtrade.enums import CandleType, RunMode from freqtrade.exceptions import OperationalException from freqtrade.persistence import Trade from freqtrade.plugins.pairlist.pairlist_helpers import expand_pairlist diff --git a/tests/rpc/test_rpc_apiserver.py b/tests/rpc/test_rpc_apiserver.py index 0823c7aee..8d25d1f5f 100644 --- a/tests/rpc/test_rpc_apiserver.py +++ b/tests/rpc/test_rpc_apiserver.py @@ -16,8 +16,7 @@ from numpy import isnan from requests.auth import _basic_auth_str from freqtrade.__init__ import __version__ -from freqtrade.enums import RunMode, State -from freqtrade.enums.candletype import CandleType +from freqtrade.enums import CandleType, RunMode, State from freqtrade.exceptions import DependencyException, ExchangeError, OperationalException from freqtrade.loggers import setup_logging, setup_logging_pre from freqtrade.persistence import PairLocks, Trade diff --git a/tests/strategy/test_strategy_helpers.py b/tests/strategy/test_strategy_helpers.py index 8c7da75f5..a1b6f57d5 100644 --- a/tests/strategy/test_strategy_helpers.py +++ b/tests/strategy/test_strategy_helpers.py @@ -5,7 +5,7 @@ import pandas as pd import pytest from freqtrade.data.dataprovider import DataProvider -from freqtrade.enums.candletype import CandleType +from freqtrade.enums import CandleType from freqtrade.strategy import (merge_informative_pair, stoploss_from_absolute, stoploss_from_open, timeframe_to_minutes) from tests.conftest import get_patched_exchange From 35afc7b478e8d2e928699a733cca62d409dd471c Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 8 Dec 2021 16:07:27 +0100 Subject: [PATCH 63/64] Fix wrong tradingMOde comparison --- freqtrade/data/history/hdf5datahandler.py | 2 +- freqtrade/data/history/jsondatahandler.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/freqtrade/data/history/hdf5datahandler.py b/freqtrade/data/history/hdf5datahandler.py index cc01d51ca..6483cfb21 100644 --- a/freqtrade/data/history/hdf5datahandler.py +++ b/freqtrade/data/history/hdf5datahandler.py @@ -29,7 +29,7 @@ class HDF5DataHandler(IDataHandler): :param trading_mode: trading-mode to be used :return: List of Tuples of (pair, timeframe) """ - if trading_mode != 'spot': + if trading_mode == 'futures': datadir = datadir.joinpath('futures') _tmp = [ re.search( diff --git a/freqtrade/data/history/jsondatahandler.py b/freqtrade/data/history/jsondatahandler.py index 939f18931..82ab1abbf 100644 --- a/freqtrade/data/history/jsondatahandler.py +++ b/freqtrade/data/history/jsondatahandler.py @@ -31,7 +31,7 @@ class JsonDataHandler(IDataHandler): :param trading_mode: trading-mode to be used :return: List of Tuples of (pair, timeframe) """ - if trading_mode != 'spot': + if trading_mode == 'futures': datadir = datadir.joinpath('futures') _tmp = [ re.search( From 26797442282c676d6561cec382f3d435ebcdbca4 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 8 Dec 2021 16:20:26 +0100 Subject: [PATCH 64/64] Explicit test for candletype get_default --- freqtrade/data/history/history_utils.py | 1 - tests/leverage/test_candletype.py | 11 ++++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/freqtrade/data/history/history_utils.py b/freqtrade/data/history/history_utils.py index 4e9ac9dcf..64297c7e5 100644 --- a/freqtrade/data/history/history_utils.py +++ b/freqtrade/data/history/history_utils.py @@ -284,7 +284,6 @@ def refresh_backtest_ohlcv_data(exchange: Exchange, pairs: List[str], timeframes # Downloads what is necessary to backtest based on futures data. timeframe = exchange._ft_has['mark_ohlcv_timeframe'] candle_type = CandleType.from_string(exchange._ft_has['mark_ohlcv_price']) - # candle_type = CandleType.MARK # TODO: this could be in most parts to the above. if erase: diff --git a/tests/leverage/test_candletype.py b/tests/leverage/test_candletype.py index 3eb73a07c..ed7991d26 100644 --- a/tests/leverage/test_candletype.py +++ b/tests/leverage/test_candletype.py @@ -14,5 +14,14 @@ from freqtrade.enums import CandleType ('mark', CandleType.MARK), ('premiumIndex', CandleType.PREMIUMINDEX), ]) -def test_candle_type_from_string(input, expected): +def test_CandleType_from_string(input, expected): assert CandleType.from_string(input) == expected + + +@pytest.mark.parametrize('input,expected', [ + ('futures', CandleType.FUTURES), + ('spot', CandleType.SPOT), + ('margin', CandleType.SPOT), +]) +def test_CandleType_get_default(input, expected): + assert CandleType.get_default(input) == expected