From bf257469658d1917e8fe55ec7b28917e20863849 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sat, 16 May 2020 11:09:50 +0300 Subject: [PATCH 1/9] Introduce datatype for informative pairs --- freqtrade/data/dataprovider.py | 8 +++++--- freqtrade/exchange/exchange.py | 3 ++- freqtrade/strategy/interface.py | 3 ++- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 7ada4f642..58cd5442b 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -9,11 +9,13 @@ from typing import Any, Dict, List, Optional, Tuple from pandas import DataFrame +from freqtrade.data.common import ListPairsWithTimeframes from freqtrade.data.history import load_pair_history from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.exchange import Exchange from freqtrade.state import RunMode + logger = logging.getLogger(__name__) @@ -25,8 +27,8 @@ class DataProvider: self._pairlists = pairlists def refresh(self, - pairlist: List[Tuple[str, str]], - helping_pairs: List[Tuple[str, str]] = None) -> None: + pairlist: ListPairsWithTimeframes, + helping_pairs: ListPairsWithTimeframes = None) -> None: """ Refresh data, called with each cycle """ @@ -36,7 +38,7 @@ class DataProvider: self._exchange.refresh_latest_ohlcv(pairlist) @property - def available_pairs(self) -> List[Tuple[str, str]]: + def available_pairs(self) -> ListPairsWithTimeframes: """ Return a list of tuples containing (pair, timeframe) for which data is currently cached. Should be whitelist + open trades. diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 6ad7ad582..43ed65787 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -18,6 +18,7 @@ from ccxt.base.decimal_to_precision import (ROUND_DOWN, ROUND_UP, TICK_SIZE, TRUNCATE, decimal_to_precision) from pandas import DataFrame +from freqtrade.data.common import ListPairsWithTimeframes from freqtrade.data.converter import ohlcv_to_dataframe, trades_dict_to_list from freqtrade.exceptions import (DependencyException, InvalidOrderException, OperationalException, TemporaryError) @@ -676,7 +677,7 @@ class Exchange: logger.info("Downloaded data for %s with length %s.", pair, len(data)) return data - def refresh_latest_ohlcv(self, pair_list: List[Tuple[str, str]]) -> List[Tuple[str, List]]: + def refresh_latest_ohlcv(self, pair_list: ListPairsWithTimeframes) -> List[Tuple[str, List]]: """ Refresh in-memory OHLCV asynchronously and set `_klines` with the result Loops asynchronously over pair_list and downloads all pairs async (semi-parallel). diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 6268b8a43..68666efab 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -12,6 +12,7 @@ from typing import Dict, List, NamedTuple, Optional, Tuple import arrow from pandas import DataFrame +from freqtrade.data.common import ListPairsWithTimeframes from freqtrade.data.dataprovider import DataProvider from freqtrade.exceptions import StrategyError from freqtrade.exchange import timeframe_to_minutes @@ -185,7 +186,7 @@ class IStrategy(ABC): """ return False - def informative_pairs(self) -> List[Tuple[str, str]]: + def informative_pairs(self) -> ListPairsWithTimeframes: """ Define additional, informative pair/interval combinations to be cached from the exchange. These pair/interval combinations are non-tradeable, unless they are part From 035a12ce6171289c26127b6f679df6e8de1f5836 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sat, 16 May 2020 11:25:32 +0300 Subject: [PATCH 2/9] Move _create_pair_whitelist to dataprovider --- freqtrade/data/dataprovider.py | 11 +++++++++-- freqtrade/freqtradebot.py | 8 +------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 58cd5442b..ad7b7ada3 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -25,6 +25,7 @@ class DataProvider: self._config = config self._exchange = exchange self._pairlists = pairlists + self._timeframe = self._config['ticker_interval'] def refresh(self, pairlist: ListPairsWithTimeframes, @@ -45,6 +46,12 @@ class DataProvider: """ return list(self._exchange._klines.keys()) + def create_pair_list(self, pairs: List[str], timeframe: str = None) -> ListPairsWithTimeframes: + """ + Create list of pair tuples with (pair, ticker_interval) + """ + return [(pair, timeframe or self._timeframe) for pair in pairs] + def ohlcv(self, pair: str, timeframe: str = None, copy: bool = True) -> DataFrame: """ Get candle (OHLCV) data for the given pair as DataFrame @@ -55,7 +62,7 @@ class DataProvider: Use False only for read-only operations (where the dataframe is not modified) """ if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE): - return self._exchange.klines((pair, timeframe or self._config['ticker_interval']), + return self._exchange.klines((pair, timeframe or self._timeframe), copy=copy) else: return DataFrame() @@ -67,7 +74,7 @@ class DataProvider: :param timeframe: timeframe to get data for """ return load_pair_history(pair=pair, - timeframe=timeframe or self._config['ticker_interval'], + timeframe=timeframe or self._timeframe, datadir=self._config['datadir'] ) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 3893a3ee0..f365a30fd 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -145,7 +145,7 @@ class FreqtradeBot: self.active_pair_whitelist = self._refresh_whitelist(trades) # Refreshing candles - self.dataprovider.refresh(self._create_pair_whitelist(self.active_pair_whitelist), + self.dataprovider.refresh(self.dataprovider.create_pair_list(self.active_pair_whitelist), self.strategy.informative_pairs()) with self._sell_lock: @@ -184,12 +184,6 @@ class FreqtradeBot: _whitelist.extend([trade.pair for trade in trades if trade.pair not in _whitelist]) return _whitelist - def _create_pair_whitelist(self, pairs: List[str]) -> List[Tuple[str, str]]: - """ - Create pair-whitelist tuple with (pair, ticker_interval) - """ - return [(pair, self.config['ticker_interval']) for pair in pairs] - def get_free_open_trades(self): """ Return the number of free open trades slots or 0 if From f8b01f5a43981e71534f2e4ce70717da9374a3ed Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sat, 16 May 2020 11:28:36 +0300 Subject: [PATCH 3/9] Make flake happy --- freqtrade/data/dataprovider.py | 2 +- freqtrade/freqtradebot.py | 2 +- freqtrade/strategy/interface.py | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index ad7b7ada3..95348e56e 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -5,7 +5,7 @@ including ticker and orderbook data, live and historical candle (OHLCV) data Common Interface for bot and strategy to access data. """ import logging -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, List, Optional from pandas import DataFrame diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index f365a30fd..151a319e2 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -7,7 +7,7 @@ import traceback from datetime import datetime from math import isclose from threading import Lock -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, List, Optional import arrow from cachetools import TTLCache diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 68666efab..5173265bc 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -7,7 +7,7 @@ import warnings from abc import ABC, abstractmethod from datetime import datetime, timezone from enum import Enum -from typing import Dict, List, NamedTuple, Optional, Tuple +from typing import Dict, NamedTuple, Optional, Tuple import arrow from pandas import DataFrame @@ -20,6 +20,7 @@ from freqtrade.persistence import Trade from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper from freqtrade.wallets import Wallets + logger = logging.getLogger(__name__) From facaaabc1eaec70639919087b8dcd5a55c882599 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sat, 16 May 2020 11:49:24 +0300 Subject: [PATCH 4/9] Rename _refresh_whitelist() --- freqtrade/freqtradebot.py | 9 +++++---- tests/conftest.py | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 151a319e2..6b99e95b6 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -84,7 +84,7 @@ class FreqtradeBot: self.edge = Edge(self.config, self.exchange, self.strategy) if \ self.config.get('edge', {}).get('enabled', False) else None - self.active_pair_whitelist = self._refresh_whitelist() + self.active_pair_whitelist = self._refresh_active_whitelist() # Set initial bot state from config initial_state = self.config.get('initial_state') @@ -142,7 +142,7 @@ class FreqtradeBot: # Query trades from persistence layer trades = Trade.get_open_trades() - self.active_pair_whitelist = self._refresh_whitelist(trades) + self.active_pair_whitelist = self._refresh_active_whitelist(trades) # Refreshing candles self.dataprovider.refresh(self.dataprovider.create_pair_list(self.active_pair_whitelist), @@ -165,9 +165,10 @@ class FreqtradeBot: Trade.session.flush() - def _refresh_whitelist(self, trades: List[Trade] = []) -> List[str]: + def _refresh_active_whitelist(self, trades: List[Trade] = []) -> List[str]: """ - Refresh whitelist from pairlist or edge and extend it with trades. + Refresh active whitelist from pairlist or edge and extend it with + pairs that have open trades. """ # Refresh whitelist self.pairlists.refresh_pairlist() diff --git a/tests/conftest.py b/tests/conftest.py index 40ab436cf..623447e9c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -92,7 +92,7 @@ def patch_wallet(mocker, free=999.9) -> None: def patch_whitelist(mocker, conf) -> None: - mocker.patch('freqtrade.freqtradebot.FreqtradeBot._refresh_whitelist', + mocker.patch('freqtrade.freqtradebot.FreqtradeBot._refresh_active_whitelist', MagicMock(return_value=conf['exchange']['pair_whitelist'])) From e7c11ed2cfe16574e455e0706da04a8982cda8bd Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sat, 16 May 2020 11:59:47 +0300 Subject: [PATCH 5/9] Fix fetching timeframe (failed in backtesting) --- freqtrade/data/dataprovider.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 95348e56e..602a92358 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -25,7 +25,6 @@ class DataProvider: self._config = config self._exchange = exchange self._pairlists = pairlists - self._timeframe = self._config['ticker_interval'] def refresh(self, pairlist: ListPairsWithTimeframes, @@ -50,7 +49,7 @@ class DataProvider: """ Create list of pair tuples with (pair, ticker_interval) """ - return [(pair, timeframe or self._timeframe) for pair in pairs] + return [(pair, timeframe or self._config['ticker_interval']) for pair in pairs] def ohlcv(self, pair: str, timeframe: str = None, copy: bool = True) -> DataFrame: """ @@ -62,7 +61,7 @@ class DataProvider: Use False only for read-only operations (where the dataframe is not modified) """ if self.runmode in (RunMode.DRY_RUN, RunMode.LIVE): - return self._exchange.klines((pair, timeframe or self._timeframe), + return self._exchange.klines((pair, timeframe or self._config['ticker_interval']), copy=copy) else: return DataFrame() @@ -74,7 +73,7 @@ class DataProvider: :param timeframe: timeframe to get data for """ return load_pair_history(pair=pair, - timeframe=timeframe or self._timeframe, + timeframe=timeframe or self._config['ticker_interval'], datadir=self._config['datadir'] ) From 5f2a871637b62a0a327750118b4d28e86169c428 Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Sat, 16 May 2020 17:15:58 +0300 Subject: [PATCH 6/9] Add missing module --- freqtrade/data/common.py | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 freqtrade/data/common.py diff --git a/freqtrade/data/common.py b/freqtrade/data/common.py new file mode 100644 index 000000000..7a61d51ea --- /dev/null +++ b/freqtrade/data/common.py @@ -0,0 +1,7 @@ +""" +Common datatypes +""" +from typing import List, Tuple + +# List of pairs with their timeframes +ListPairsWithTimeframes = List[Tuple[str, str]] From 627c5059f0066ca7bbe6e2c5e3fc0a2ed7e9c8cd Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Mon, 18 May 2020 13:54:21 +0300 Subject: [PATCH 7/9] Move create_pair_list to pairlistmanager --- freqtrade/data/common.py | 7 ------- freqtrade/data/dataprovider.py | 8 +------- freqtrade/exchange/exchange.py | 2 +- freqtrade/freqtradebot.py | 2 +- freqtrade/pairlist/pairlistmanager.py | 13 ++++++++++++- freqtrade/strategy/interface.py | 2 +- 6 files changed, 16 insertions(+), 18 deletions(-) delete mode 100644 freqtrade/data/common.py diff --git a/freqtrade/data/common.py b/freqtrade/data/common.py deleted file mode 100644 index 7a61d51ea..000000000 --- a/freqtrade/data/common.py +++ /dev/null @@ -1,7 +0,0 @@ -""" -Common datatypes -""" -from typing import List, Tuple - -# List of pairs with their timeframes -ListPairsWithTimeframes = List[Tuple[str, str]] diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 602a92358..5668e2437 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -9,10 +9,10 @@ from typing import Any, Dict, List, Optional from pandas import DataFrame -from freqtrade.data.common import ListPairsWithTimeframes from freqtrade.data.history import load_pair_history from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.exchange import Exchange +from freqtrade.pairlist.pairlistmanager import ListPairsWithTimeframes from freqtrade.state import RunMode @@ -45,12 +45,6 @@ class DataProvider: """ return list(self._exchange._klines.keys()) - def create_pair_list(self, pairs: List[str], timeframe: str = None) -> ListPairsWithTimeframes: - """ - Create list of pair tuples with (pair, ticker_interval) - """ - return [(pair, timeframe or self._config['ticker_interval']) for pair in pairs] - def ohlcv(self, pair: str, timeframe: str = None, copy: bool = True) -> DataFrame: """ Get candle (OHLCV) data for the given pair as DataFrame diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 43ed65787..858cdecab 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -18,12 +18,12 @@ from ccxt.base.decimal_to_precision import (ROUND_DOWN, ROUND_UP, TICK_SIZE, TRUNCATE, decimal_to_precision) from pandas import DataFrame -from freqtrade.data.common import ListPairsWithTimeframes from freqtrade.data.converter import ohlcv_to_dataframe, trades_dict_to_list from freqtrade.exceptions import (DependencyException, InvalidOrderException, OperationalException, TemporaryError) from freqtrade.exchange.common import BAD_EXCHANGES, retrier, retrier_async from freqtrade.misc import deep_merge_dicts, safe_value_fallback +from freqtrade.pairlist.pairlistmanager import ListPairsWithTimeframes CcxtModuleType = Any diff --git a/freqtrade/freqtradebot.py b/freqtrade/freqtradebot.py index 6b99e95b6..6ea5ec970 100644 --- a/freqtrade/freqtradebot.py +++ b/freqtrade/freqtradebot.py @@ -145,7 +145,7 @@ class FreqtradeBot: self.active_pair_whitelist = self._refresh_active_whitelist(trades) # Refreshing candles - self.dataprovider.refresh(self.dataprovider.create_pair_list(self.active_pair_whitelist), + self.dataprovider.refresh(self.pairlists.create_pair_list(self.active_pair_whitelist), self.strategy.informative_pairs()) with self._sell_lock: diff --git a/freqtrade/pairlist/pairlistmanager.py b/freqtrade/pairlist/pairlistmanager.py index 5b4c5b602..163836c4d 100644 --- a/freqtrade/pairlist/pairlistmanager.py +++ b/freqtrade/pairlist/pairlistmanager.py @@ -5,7 +5,7 @@ Provides lists as configured in config.json """ import logging -from typing import Dict, List +from typing import Dict, List, Tuple from cachetools import TTLCache, cached @@ -13,9 +13,14 @@ from freqtrade.exceptions import OperationalException from freqtrade.pairlist.IPairList import IPairList from freqtrade.resolvers import PairListResolver + logger = logging.getLogger(__name__) +# List of pairs with their timeframes +ListPairsWithTimeframes = List[Tuple[str, str]] + + class PairListManager(): def __init__(self, exchange, config: dict) -> None: @@ -94,3 +99,9 @@ class PairListManager(): pairlist = IPairList.verify_blacklist(pairlist, self.blacklist, True) self._whitelist = pairlist + + def create_pair_list(self, pairs: List[str], timeframe: str = None) -> ListPairsWithTimeframes: + """ + Create list of pair tuples with (pair, ticker_interval) + """ + return [(pair, timeframe or self._config['ticker_interval']) for pair in pairs] diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index 5173265bc..d5d171bec 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -12,10 +12,10 @@ from typing import Dict, NamedTuple, Optional, Tuple import arrow from pandas import DataFrame -from freqtrade.data.common import ListPairsWithTimeframes from freqtrade.data.dataprovider import DataProvider from freqtrade.exceptions import StrategyError from freqtrade.exchange import timeframe_to_minutes +from freqtrade.pairlist.pairlistmanager import ListPairsWithTimeframes from freqtrade.persistence import Trade from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper from freqtrade.wallets import Wallets From 0c8aff98e093eb539802885e3247d4ed5c0f32fb Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Mon, 18 May 2020 15:03:36 +0300 Subject: [PATCH 8/9] Make flake happy --- freqtrade/pairlist/pairlistmanager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/freqtrade/pairlist/pairlistmanager.py b/freqtrade/pairlist/pairlistmanager.py index 9c8f0de1f..5c3a3aaa7 100644 --- a/freqtrade/pairlist/pairlistmanager.py +++ b/freqtrade/pairlist/pairlistmanager.py @@ -116,4 +116,4 @@ class PairListManager(): """ Create list of pair tuples with (pair, ticker_interval) """ - return [(pair, timeframe or self._config['ticker_interval']) for pair in pairs] \ No newline at end of file + return [(pair, timeframe or self._config['ticker_interval']) for pair in pairs] From 115586a50f2cc96580ce0fa989b68995fd9d087d Mon Sep 17 00:00:00 2001 From: hroff-1902 Date: Mon, 18 May 2020 21:59:50 +0300 Subject: [PATCH 9/9] Introduce freqtrade.typing --- freqtrade/data/dataprovider.py | 2 +- freqtrade/exchange/exchange.py | 2 +- freqtrade/pairlist/pairlistmanager.py | 7 ++----- freqtrade/strategy/interface.py | 2 +- freqtrade/typing.py | 8 ++++++++ 5 files changed, 13 insertions(+), 8 deletions(-) create mode 100644 freqtrade/typing.py diff --git a/freqtrade/data/dataprovider.py b/freqtrade/data/dataprovider.py index 5668e2437..ef03307cc 100644 --- a/freqtrade/data/dataprovider.py +++ b/freqtrade/data/dataprovider.py @@ -12,8 +12,8 @@ from pandas import DataFrame from freqtrade.data.history import load_pair_history from freqtrade.exceptions import DependencyException, OperationalException from freqtrade.exchange import Exchange -from freqtrade.pairlist.pairlistmanager import ListPairsWithTimeframes from freqtrade.state import RunMode +from freqtrade.typing import ListPairsWithTimeframes logger = logging.getLogger(__name__) diff --git a/freqtrade/exchange/exchange.py b/freqtrade/exchange/exchange.py index 36d1ddca4..c60eb766a 100644 --- a/freqtrade/exchange/exchange.py +++ b/freqtrade/exchange/exchange.py @@ -23,7 +23,7 @@ from freqtrade.exceptions import (DependencyException, InvalidOrderException, OperationalException, TemporaryError) from freqtrade.exchange.common import BAD_EXCHANGES, retrier, retrier_async from freqtrade.misc import deep_merge_dicts, safe_value_fallback -from freqtrade.pairlist.pairlistmanager import ListPairsWithTimeframes +from freqtrade.typing import ListPairsWithTimeframes CcxtModuleType = Any diff --git a/freqtrade/pairlist/pairlistmanager.py b/freqtrade/pairlist/pairlistmanager.py index 5c3a3aaa7..78a6a9668 100644 --- a/freqtrade/pairlist/pairlistmanager.py +++ b/freqtrade/pairlist/pairlistmanager.py @@ -3,22 +3,19 @@ PairList manager class """ import logging from copy import deepcopy -from typing import Dict, List, Tuple +from typing import Dict, List from cachetools import TTLCache, cached from freqtrade.exceptions import OperationalException from freqtrade.pairlist.IPairList import IPairList from freqtrade.resolvers import PairListResolver +from freqtrade.typing import ListPairsWithTimeframes logger = logging.getLogger(__name__) -# List of pairs with their timeframes -ListPairsWithTimeframes = List[Tuple[str, str]] - - class PairListManager(): def __init__(self, exchange, config: dict) -> None: diff --git a/freqtrade/strategy/interface.py b/freqtrade/strategy/interface.py index d5d171bec..8031c7932 100644 --- a/freqtrade/strategy/interface.py +++ b/freqtrade/strategy/interface.py @@ -15,9 +15,9 @@ from pandas import DataFrame from freqtrade.data.dataprovider import DataProvider from freqtrade.exceptions import StrategyError from freqtrade.exchange import timeframe_to_minutes -from freqtrade.pairlist.pairlistmanager import ListPairsWithTimeframes from freqtrade.persistence import Trade from freqtrade.strategy.strategy_wrapper import strategy_safe_wrapper +from freqtrade.typing import ListPairsWithTimeframes from freqtrade.wallets import Wallets diff --git a/freqtrade/typing.py b/freqtrade/typing.py new file mode 100644 index 000000000..3cb7cf816 --- /dev/null +++ b/freqtrade/typing.py @@ -0,0 +1,8 @@ +""" +Common Freqtrade types +""" + +from typing import List, Tuple + +# List of pairs with their timeframes +ListPairsWithTimeframes = List[Tuple[str, str]]